You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@paimon.apache.org by ho...@apache.org on 2026/09/20 03:57:56 UTC
(paimon) branch release-2.1 updated (a5f3dd4b99 -> 1fc0f17ad2)
This is an automated email from the ASF dual-hosted git repository.
XiaoHongbo-Hope pushed a change to branch release-2.1
in repository https://gitbox.apache.org/repos/asf/paimon.git
from a5f3dd4b99 [rest] Add semantic view definition management to REST Catalog (#9848)
new f3862f6b4e [python] Cover native dynamic PK events and compacted first-row scans (#9854)
new c1f92f1a41 [python] Speed up nested VARIANT reads by vectorizing offsets (#9847)
new 8a429cfc36 [python] Expand native planning across resolved tables and scan modes (#9873)
new 72e4a32b56 [core] Execute complete filters with unprojected fields (#9872)
new 03d68bfec7 [python] Speed up LeRobot video window reads (#9852)
new 478a7201c4 [python] Support writing shared-shredding MAPs in Parquet (#9837)
new b6d91b6fd8 [core] Add standalone manifest sidecar format and utilities (#9845)
new 301713fe6c [core] Manage manifest extra file lifecycle (#9886)
new 32f0a9261e [python] Use conditional OSS puts for atomic metadata writes (#9715)
new 9686b2a1c2 [core] Generate sidecars for newly written manifests (#9889)
new 3f0d716f97 [python] Reduce memory usage for Parquet row-id updates (#9840)
new ce81e89b9d [spark][core] Support more expressions in V2 filter pushdown (#9891)
new 084cde4772 [core] Prune manifest blocks with sidecars during scans (#9897)
new a5fb087bde [spark][core] Support more scalar transforms in V2 filter pushdown (#9905)
new 4151992545 [python] Resolve value stats columns using file schema (#9906)
new d6644bbc97 [python] Make BLOB I/O merge parameters configurable (#9890)
new f951059590 [format] Support file.block-size for Avro (#9901)
new 394cbaaa11 Revert "[format] Support file.block-size for Avro (#9901)"
new 2e8803c498 [python] Honor snapshot.ignore-empty-commit for empty appends (#9878)
new 1fc0f17ad2 [python] Extend ContiguousWindowDataset for frame-level training (#9875)
The 20 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/workflows/ci-python.yml | 1 +
docs/docs/concepts/spec/manifest.md | 247 +++++
docs/docs/pypaimon/multimodal-reading.md | 119 ++-
docs/docs/pypaimon/pytorch.md | 55 +-
docs/generated/catalog_configuration.html | 6 +
docs/generated/core_configuration.html | 6 +
.../main/java/org/apache/paimon/CoreOptions.java | 11 +
.../org/apache/paimon/options/CatalogOptions.java | 10 +
.../benchmark/ManifestFileSorterBenchmark.java | 4 +-
.../apache/paimon/fileindex/FileIndexReader.java | 20 +
.../paimon/globalindex/GlobalIndexReader.java | 24 +
...FieldTransform.java => BitLengthTransform.java} | 57 +-
.../java/org/apache/paimon/predicate/Contains.java | 2 +-
.../{FieldTransform.java => DateAddTransform.java} | 58 +-
...{FieldTransform.java => DateDiffTransform.java} | 58 +-
.../paimon/predicate/DateExtractTransform.java | 147 +++
.../paimon/predicate/DateTruncTransform.java | 135 +++
...oncatTransform.java => DayOfWeekTransform.java} | 32 +-
...oncatTransform.java => DayOfYearTransform.java} | 32 +-
.../{ConcatTransform.java => DayTransform.java} | 31 +-
.../java/org/apache/paimon/predicate/EndsWith.java | 2 +-
.../apache/paimon/predicate/FunctionVisitor.java | 16 +
.../{ConcatTransform.java => HourTransform.java} | 31 +-
...atTransform.java => IsoDayOfWeekTransform.java} | 32 +-
.../org/apache/paimon/predicate/LeafFunction.java | 4 +
.../apache/paimon/predicate/LengthTransform.java | 129 +++
.../java/org/apache/paimon/predicate/Like.java | 2 +-
.../{ConcatTransform.java => MinuteTransform.java} | 31 +-
.../{ConcatTransform.java => MonthTransform.java} | 31 +-
.../predicate/{Contains.java => NotContains.java} | 20 +-
.../predicate/{Contains.java => NotEndsWith.java} | 20 +-
.../predicate/{LessThan.java => NotLike.java} | 22 +-
.../{Contains.java => NotStartsWith.java} | 20 +-
.../predicate/OnlyPartitionKeyEqualVisitor.java | 20 +
.../apache/paimon/predicate/OverlayTransform.java | 124 +++
.../org/apache/paimon/predicate/PadTransform.java | 145 +++
.../apache/paimon/predicate/PredicateBuilder.java | 8 +
.../apache/paimon/predicate/PredicateRemapper.java | 87 ++
...{ConcatTransform.java => QuarterTransform.java} | 31 +-
.../{ConcatTransform.java => SecondTransform.java} | 31 +-
.../org/apache/paimon/predicate/StartsWith.java | 2 +-
.../org/apache/paimon/predicate/Transform.java | 21 +
.../paimon/predicate/TransformInputUtils.java | 77 ++
.../paimon/predicate/TranslateTransform.java | 95 ++
.../{ConcatTransform.java => WeekTransform.java} | 32 +-
...{ConcatTransform.java => WeekdayTransform.java} | 31 +-
...ncatTransform.java => YearOfWeekTransform.java} | 33 +-
.../{ConcatTransform.java => YearTransform.java} | 31 +-
.../org/apache/paimon/utils/DeltaVarintCodec.java | 126 +++
.../java/org/apache/paimon/utils/TypeUtils.java | 19 +
.../org/apache/paimon/utils/VarLengthIntUtils.java | 29 +
.../paimon/predicate/DateExtractTransformTest.java | 150 +++
.../paimon/predicate/LengthTransformTest.java | 82 ++
.../paimon/predicate/PredicateJsonSerdeTest.java | 31 +-
.../paimon/predicate/PredicateRemapperTest.java | 144 +++
.../org/apache/paimon/predicate/PredicateTest.java | 60 ++
.../paimon/predicate/ScalarTransformTest.java | 171 +++
.../paimon/predicate/TransformJsonSerdeTest.java | 118 +-
.../apache/paimon/utils/DeltaVarintCodecTest.java | 225 ++++
.../apache/paimon/utils/VarLengthIntUtilsTest.java | 101 ++
.../java/org/apache/paimon/AbstractFileStore.java | 10 +-
.../src/main/java/org/apache/paimon/FileStore.java | 2 +
.../org/apache/paimon/catalog/CachingCatalog.java | 21 +
.../paimon/catalog/TableQueryAuthResult.java | 133 +--
.../org/apache/paimon/manifest/BucketFilter.java | 16 +
.../apache/paimon/manifest/ManifestAvroWriter.java | 52 +-
.../org/apache/paimon/manifest/ManifestFile.java | 113 +-
.../apache/paimon/manifest/ManifestSidecar.java | 942 ++++++++++++++++
.../paimon/operation/AbstractFileStoreScan.java | 17 +-
.../apache/paimon/operation/ChangelogDeletion.java | 13 +-
.../operation/DataEvolutionFileStoreScan.java | 17 +-
.../apache/paimon/operation/FileDeletionBase.java | 20 +-
.../paimon/operation/ManifestFileMerger.java | 2 +-
.../paimon/operation/commit/CommitCleaner.java | 6 +-
.../paimon/table/AbstractFileStoreTable.java | 16 +
.../paimon/table/DelegatedFileStoreTable.java | 11 +
.../paimon/table/FallbackReadFileStoreTable.java | 6 +
.../org/apache/paimon/table/FileStoreTable.java | 5 +
.../paimon/table/format/FormatReadBuilder.java | 18 +-
.../paimon/table/format/FormatTableRead.java | 40 +-
.../paimon/table/source/AbstractDataTableRead.java | 199 +---
.../paimon/table/source/AbstractDataTableScan.java | 56 +-
.../apache/paimon/table/source/ReadTransform.java | 178 ++++
.../org/apache/paimon/table/source/TableRead.java | 12 +
.../java/org/apache/paimon/utils/ObjectsCache.java | 6 +
.../java/org/apache/paimon/CoreOptionsTest.java | 18 +
.../test/java/org/apache/paimon/JavaPyE2ETest.java | 61 ++
.../apache/paimon/catalog/CachingCatalogTest.java | 142 +++
.../paimon/manifest/ManifestBlockIndexTest.java | 891 ++++++++++++++++
.../manifest/ManifestFileMetaSerializerTest.java | 3 +-
.../paimon/manifest/ManifestFileMetaTest.java | 8 +-
.../paimon/manifest/ManifestFileMetaTestBase.java | 4 +-
.../apache/paimon/manifest/ManifestFileTest.java | 688 +++++++++++-
.../paimon/manifest/ManifestIndexTestUtils.java | 100 ++
.../paimon/manifest/ManifestSidecarTest.java | 1123 ++++++++++++++++++++
.../paimon/manifest/ManifestSidecarWriteTest.java | 348 ++++++
.../paimon/operation/ExpireSnapshotsTest.java | 72 ++
.../operation/LocalOrphanFilesCleanTest.java | 35 +
.../operation/ManifestRewriteCleanupTest.java | 4 +-
.../operation/ManifestSidecarLifecycleTest.java | 106 ++
.../org/apache/paimon/rest/RESTCatalogTest.java | 87 ++
.../paimon/table/AppendOnlySimpleTableTest.java | 10 +-
.../paimon/table/DataEvolutionFileIndexTest.java | 44 +-
.../paimon/table/PrimaryKeySimpleTableTest.java | 26 +
.../paimon/table/format/FormatTableReadTest.java | 109 ++
.../table/source/AbstractDataTableReadTest.java | 369 ++++++-
.../resources/compatibility/manifest-sidecar-v1 | Bin 0 -> 2107 bytes
.../orc/filter/OrcPredicateFunctionVisitor.java | 22 +
.../format/orc/filter/OrcFilterConverterTest.java | 15 +
.../paimon/format/parquet/ParquetFiltersTest.java | 15 +
paimon-python/README.md | 128 ++-
paimon-python/dev/requirements-dev.txt | 1 +
paimon-python/dev/run_mixed_tests.sh | 15 +
paimon-python/pypaimon/benchmark/act/paimon.py | 11 +-
.../pypaimon/catalog/rest/rest_token_file_io.py | 47 +-
paimon-python/pypaimon/common/file_io.py | 30 +-
paimon-python/pypaimon/common/json_util.py | 2 +
paimon-python/pypaimon/common/options/config.py | 32 +
.../pypaimon/common/options/core_options.py | 57 +
.../pypaimon/data/map_shared_shredding.py | 51 +-
.../apply_deletion_vector_reader.py | 13 +-
paimon-python/pypaimon/filesystem/oss_file_io.py | 157 +++
.../pypaimon/filesystem/resolving_file_io.py | 3 +
.../pypaimon/manifest/manifest_file_manager.py | 3 +-
.../pypaimon/multimodal/lerobot/dataset.py | 144 ++-
paimon-python/pypaimon/multimodal/query.py | 51 +-
.../pypaimon/multimodal/window_dataset.py | 292 +++--
.../pypaimon/multimodal/window_transforms.py | 88 ++
paimon-python/pypaimon/read/native_plan.py | 84 +-
.../pypaimon/read/reader/format_pyarrow_reader.py | 19 +-
.../read/scanner/chunk_shuffle_split_generator.py | 13 +-
paimon-python/pypaimon/read/split_read.py | 18 +-
paimon-python/pypaimon/read/table_read.py | 6 +
paimon-python/pypaimon/read/table_scan.py | 150 ++-
.../rest_catalog_blob_as_descriptor_sample.py | 23 +-
paimon-python/pypaimon/table/file_store_table.py | 5 +-
paimon-python/pypaimon/tests/act_runner_test.py | 20 +-
paimon-python/pypaimon/tests/blob_test.py | 132 ++-
.../tests/contiguous_window_dataset_test.py | 236 ++++
.../pypaimon/tests/e2e/java_py_read_write_test.py | 46 +
paimon-python/pypaimon/tests/file_io_test.py | 7 +-
.../tests/format_pyarrow_variant_row_group_test.py | 32 +-
paimon-python/pypaimon/tests/lance_utils_test.py | 6 +-
.../tests/map_shared_shredding_write_test.py | 429 ++++++++
.../pypaimon/tests/multimodal_lerobot_test.py | 203 ++++
.../tests/native_plan_capabilities_test.py | 36 +
.../tests/native_plan_chunk_shuffle_test.py | 172 +++
.../tests/native_plan_dynamic_bucket_test.py | 33 +
.../pypaimon/tests/native_plan_integration_test.py | 13 +-
.../tests/native_plan_materialized_pk_test.py | 186 ++++
.../tests/native_plan_resolved_schema_test.py | 184 ++++
.../pypaimon/tests/native_plan_rest_test.py | 151 +++
paimon-python/pypaimon/tests/native_plan_test.py | 57 +-
.../pypaimon/tests/oss_atomic_write_test.py | 308 ++++++
paimon-python/pypaimon/tests/oss_file_io_test.py | 12 +-
.../pypaimon/tests/oss_legacy_mode_test.py | 19 +-
.../pypaimon/tests/py36/ao_simple_test.py | 8 +-
paimon-python/pypaimon/tests/reader_base_test.py | 23 +
.../pypaimon/tests/reader_predicate_test.py | 78 ++
.../pypaimon/tests/resolving_file_io_test.py | 21 +
.../pypaimon/tests/rest/rest_token_file_io_test.py | 3 +
.../pypaimon/tests/schema_evolution_read_test.py | 53 +-
.../pypaimon/tests/snapshot_manager_test.py | 24 +
paimon-python/pypaimon/tests/table_commit_test.py | 47 +
.../tests/table_update_by_row_id_chunked_test.py | 112 +-
.../pypaimon/tests/table_upsert_by_key_test.py | 250 +++++
.../pypaimon/tests/window_transforms_test.py | 111 ++
paimon-python/pypaimon/write/file_store_commit.py | 16 +-
.../pypaimon/write/map_shared_shredding_writer.py | 573 ++++++++++
paimon-python/pypaimon/write/table_commit.py | 3 +-
paimon-python/pypaimon/write/table_update.py | 1 +
.../pypaimon/write/table_update_by_row_id.py | 227 +++-
.../pypaimon/write/writer/data_vector_writer.py | 8 +-
paimon-python/pypaimon/write/writer/data_writer.py | 75 +-
.../write/writer/dedicated_format_writer.py | 8 +-
.../pypaimon/write/writer/single_file_writer.py | 106 ++
paimon-python/setup.py | 4 +
.../paimon/spark/SparkV2FilterConverter.scala | 9 +
.../spark/util/SparkExpressionConverter.scala | 121 ++-
.../paimon/spark/SparkFilterConverterTest.java | 17 +-
.../spark/procedure/PermissionProcedureTest.scala | 4 +-
.../spark/sql/SparkV2FilterConverterTestBase.scala | 284 ++++-
182 files changed, 14369 insertions(+), 1192 deletions(-)
copy paimon-common/src/main/java/org/apache/paimon/predicate/{FieldTransform.java => BitLengthTransform.java} (55%)
copy paimon-common/src/main/java/org/apache/paimon/predicate/{FieldTransform.java => DateAddTransform.java} (53%)
copy paimon-common/src/main/java/org/apache/paimon/predicate/{FieldTransform.java => DateDiffTransform.java} (53%)
create mode 100644 paimon-common/src/main/java/org/apache/paimon/predicate/DateExtractTransform.java
create mode 100644 paimon-common/src/main/java/org/apache/paimon/predicate/DateTruncTransform.java
copy paimon-common/src/main/java/org/apache/paimon/predicate/{ConcatTransform.java => DayOfWeekTransform.java} (61%)
copy paimon-common/src/main/java/org/apache/paimon/predicate/{ConcatTransform.java => DayOfYearTransform.java} (61%)
copy paimon-common/src/main/java/org/apache/paimon/predicate/{ConcatTransform.java => DayTransform.java} (61%)
copy paimon-common/src/main/java/org/apache/paimon/predicate/{ConcatTransform.java => HourTransform.java} (61%)
copy paimon-common/src/main/java/org/apache/paimon/predicate/{ConcatTransform.java => IsoDayOfWeekTransform.java} (61%)
create mode 100644 paimon-common/src/main/java/org/apache/paimon/predicate/LengthTransform.java
copy paimon-common/src/main/java/org/apache/paimon/predicate/{ConcatTransform.java => MinuteTransform.java} (61%)
copy paimon-common/src/main/java/org/apache/paimon/predicate/{ConcatTransform.java => MonthTransform.java} (61%)
copy paimon-common/src/main/java/org/apache/paimon/predicate/{Contains.java => NotContains.java} (75%)
copy paimon-common/src/main/java/org/apache/paimon/predicate/{Contains.java => NotEndsWith.java} (75%)
copy paimon-common/src/main/java/org/apache/paimon/predicate/{LessThan.java => NotLike.java} (70%)
copy paimon-common/src/main/java/org/apache/paimon/predicate/{Contains.java => NotStartsWith.java} (74%)
create mode 100644 paimon-common/src/main/java/org/apache/paimon/predicate/OverlayTransform.java
create mode 100644 paimon-common/src/main/java/org/apache/paimon/predicate/PadTransform.java
create mode 100644 paimon-common/src/main/java/org/apache/paimon/predicate/PredicateRemapper.java
copy paimon-common/src/main/java/org/apache/paimon/predicate/{ConcatTransform.java => QuarterTransform.java} (61%)
copy paimon-common/src/main/java/org/apache/paimon/predicate/{ConcatTransform.java => SecondTransform.java} (61%)
create mode 100644 paimon-common/src/main/java/org/apache/paimon/predicate/TransformInputUtils.java
create mode 100644 paimon-common/src/main/java/org/apache/paimon/predicate/TranslateTransform.java
copy paimon-common/src/main/java/org/apache/paimon/predicate/{ConcatTransform.java => WeekTransform.java} (61%)
copy paimon-common/src/main/java/org/apache/paimon/predicate/{ConcatTransform.java => WeekdayTransform.java} (61%)
copy paimon-common/src/main/java/org/apache/paimon/predicate/{ConcatTransform.java => YearOfWeekTransform.java} (61%)
copy paimon-common/src/main/java/org/apache/paimon/predicate/{ConcatTransform.java => YearTransform.java} (61%)
create mode 100644 paimon-common/src/main/java/org/apache/paimon/utils/DeltaVarintCodec.java
create mode 100644 paimon-common/src/test/java/org/apache/paimon/predicate/DateExtractTransformTest.java
create mode 100644 paimon-common/src/test/java/org/apache/paimon/predicate/LengthTransformTest.java
create mode 100644 paimon-common/src/test/java/org/apache/paimon/predicate/PredicateRemapperTest.java
create mode 100644 paimon-common/src/test/java/org/apache/paimon/predicate/ScalarTransformTest.java
create mode 100644 paimon-common/src/test/java/org/apache/paimon/utils/DeltaVarintCodecTest.java
create mode 100644 paimon-common/src/test/java/org/apache/paimon/utils/VarLengthIntUtilsTest.java
create mode 100644 paimon-core/src/main/java/org/apache/paimon/manifest/ManifestSidecar.java
create mode 100644 paimon-core/src/main/java/org/apache/paimon/table/source/ReadTransform.java
create mode 100644 paimon-core/src/test/java/org/apache/paimon/manifest/ManifestBlockIndexTest.java
create mode 100644 paimon-core/src/test/java/org/apache/paimon/manifest/ManifestIndexTestUtils.java
create mode 100644 paimon-core/src/test/java/org/apache/paimon/manifest/ManifestSidecarTest.java
create mode 100644 paimon-core/src/test/java/org/apache/paimon/manifest/ManifestSidecarWriteTest.java
create mode 100644 paimon-core/src/test/java/org/apache/paimon/operation/ManifestSidecarLifecycleTest.java
create mode 100644 paimon-core/src/test/java/org/apache/paimon/table/format/FormatTableReadTest.java
create mode 100644 paimon-core/src/test/resources/compatibility/manifest-sidecar-v1
create mode 100644 paimon-python/pypaimon/filesystem/oss_file_io.py
create mode 100644 paimon-python/pypaimon/multimodal/window_transforms.py
create mode 100644 paimon-python/pypaimon/tests/map_shared_shredding_write_test.py
create mode 100644 paimon-python/pypaimon/tests/native_plan_chunk_shuffle_test.py
create mode 100644 paimon-python/pypaimon/tests/native_plan_materialized_pk_test.py
create mode 100644 paimon-python/pypaimon/tests/native_plan_resolved_schema_test.py
create mode 100644 paimon-python/pypaimon/tests/native_plan_rest_test.py
create mode 100644 paimon-python/pypaimon/tests/oss_atomic_write_test.py
create mode 100644 paimon-python/pypaimon/tests/window_transforms_test.py
create mode 100644 paimon-python/pypaimon/write/map_shared_shredding_writer.py
create mode 100644 paimon-python/pypaimon/write/writer/single_file_writer.py
(paimon) 17/20: [format] Support file.block-size for Avro (#9901)
Posted by ho...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
XiaoHongbo-Hope pushed a commit to branch release-2.1
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit f95105959014176e44f1638b44eeb3833b79a29f
Author: umi <55...@users.noreply.github.com>
AuthorDate: Thu Sep 17 11:32:37 2026 +0800
[format] Support file.block-size for Avro (#9901)
---
docs/generated/core_configuration.html | 2 +-
.../main/java/org/apache/paimon/CoreOptions.java | 3 +-
.../apache/paimon/format/avro/AvroFileFormat.java | 6 ++
.../paimon/format/avro/AvroFileFormatTest.java | 71 ++++++++++++++++++++++
4 files changed, 80 insertions(+), 2 deletions(-)
diff --git a/docs/generated/core_configuration.html b/docs/generated/core_configuration.html
index cf48d473c4..19f6d52603 100644
--- a/docs/generated/core_configuration.html
+++ b/docs/generated/core_configuration.html
@@ -738,7 +738,7 @@ under the License.
<td><h5>file.block-size</h5></td>
<td style="word-wrap: break-word;">(none)</td>
<td>MemorySize</td>
- <td>File block size of format, default value of orc stripe is 64 MB, and parquet row group is 128 MB.</td>
+ <td>File block size of format, default value of orc stripe is 64 MB, parquet row group is 128 MB, and avro block is 64000 bytes. For avro, this is the uncompressed block size threshold.</td>
</tr>
<tr>
<td><h5>file.compression</h5></td>
diff --git a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java
index 9bb324b780..20c59b1f95 100644
--- a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java
+++ b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java
@@ -426,7 +426,8 @@ public class CoreOptions implements Serializable {
.memoryType()
.noDefaultValue()
.withDescription(
- "File block size of format, default value of orc stripe is 64 MB, and parquet row group is 128 MB.");
+ "File block size of format, default value of orc stripe is 64 MB, parquet row group is 128 MB, "
+ + "and avro block is 64000 bytes. For avro, this is the uncompressed block size threshold.");
public static final ConfigOption<MemorySize> FILE_INDEX_IN_MANIFEST_THRESHOLD =
key("file-index.in-manifest-threshold")
diff --git a/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroFileFormat.java b/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroFileFormat.java
index 69512d5ee7..a23c66af21 100644
--- a/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroFileFormat.java
+++ b/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroFileFormat.java
@@ -29,6 +29,7 @@ import org.apache.paimon.fs.CloseShieldOutputStream;
import org.apache.paimon.fs.PositionOutputStream;
import org.apache.paimon.options.ConfigOption;
import org.apache.paimon.options.ConfigOptions;
+import org.apache.paimon.options.MemorySize;
import org.apache.paimon.options.Options;
import org.apache.paimon.predicate.Predicate;
import org.apache.paimon.statistics.SimpleColStatsCollector;
@@ -65,12 +66,14 @@ public class AvroFileFormat extends FileFormat {
private final Options options;
private final int zstdLevel;
+ @Nullable private final MemorySize blockSize;
public AvroFileFormat(FormatContext context) {
super(IDENTIFIER);
this.options = getIdentifierPrefixOptions(context.options());
this.zstdLevel = context.zstdLevel();
+ this.blockSize = context.blockSize();
}
@Override
@@ -93,6 +96,9 @@ public class AvroFileFormat extends FileFormat {
AvroRowDatumWriter datumWriter = new AvroRowDatumWriter(rowType);
DataFileWriter<InternalRow> writer = new DataFileWriter<>(datumWriter);
writer.setCodec(createCodecFactory(compression));
+ if (blockSize != null) {
+ writer.setSyncInterval(Math.toIntExact(blockSize.getBytes()));
+ }
writer.setFlushOnEveryBlock(false);
writer.create(schema, new CloseShieldOutputStream(out));
return new AvroBlockWriter(writer, out, schema);
diff --git a/paimon-format/src/test/java/org/apache/paimon/format/avro/AvroFileFormatTest.java b/paimon-format/src/test/java/org/apache/paimon/format/avro/AvroFileFormatTest.java
index c7ca45b88f..33b5798b5b 100644
--- a/paimon-format/src/test/java/org/apache/paimon/format/avro/AvroFileFormatTest.java
+++ b/paimon-format/src/test/java/org/apache/paimon/format/avro/AvroFileFormatTest.java
@@ -48,6 +48,9 @@ import org.apache.avro.io.EncoderFactory;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
+import org.junit.jupiter.params.provider.ValueSource;
import java.io.ByteArrayOutputStream;
import java.io.File;
@@ -76,6 +79,74 @@ public class AvroFileFormatTest {
fileFormat = new AvroFileFormat(new FormatContext(new Options(), 1024, 1024));
}
+ @ParameterizedTest
+ @CsvSource({
+ ", 64000, null",
+ ", 64000, deflate",
+ "1 kb, 1024, null",
+ "1 kb, 1024, deflate",
+ "4 kb, 4096, null",
+ "4 kb, 4096, deflate",
+ "128 kb, 131072, null",
+ "128 kb, 131072, deflate"
+ })
+ void testFileBlockSize(String blockSize, int expectedBlockSize, String compression)
+ throws IOException {
+ Options options = new Options();
+ if (blockSize != null) {
+ options.setString("file.block-size", blockSize);
+ }
+ FileFormat format = FileFormat.fromIdentifier("avro", options);
+ RowType rowType = DataTypes.ROW(DataTypes.INT().notNull()).notNull();
+ LocalFileIO fileIO = LocalFileIO.create();
+ Path file = new Path(new Path(tempPath.toUri()), UUID.randomUUID().toString());
+ int numRecords = 300_000;
+
+ try (PositionOutputStream out = fileIO.newOutputStream(file, false);
+ FormatWriter writer =
+ format.createWriterFactory(rowType).create(out, compression)) {
+ // Each record is encoded as one byte, so record counts also give uncompressed sizes.
+ for (int i = 0; i < numRecords; i++) {
+ writer.addElement(GenericRow.of(0));
+ }
+ }
+
+ long records = 0;
+ try (AvroBlockReader reader = new AvroBlockReader(fileIO.newInputStream(file))) {
+ while (reader.hasNextBlock()) {
+ AvroRawBlock block = reader.nextBorrowedRawBlock();
+ long expectedRecords = Math.min(expectedBlockSize, numRecords - records);
+ assertThat(block.recordCount()).isEqualTo(expectedRecords);
+ assertThat(block.decompress(null).remaining()).isEqualTo((int) expectedRecords);
+ records += block.recordCount();
+ }
+ }
+ assertThat(records).isEqualTo(numRecords);
+ }
+
+ @ParameterizedTest
+ @ValueSource(longs = {2147483648L, 4294968320L, Long.MAX_VALUE})
+ void testFileBlockSizeOverflow(long blockSize) throws IOException {
+ Options options = new Options();
+ options.setString("file.block-size", Long.toString(blockSize));
+ FileFormat format = FileFormat.fromIdentifier("avro", options);
+ RowType rowType = DataTypes.ROW(DataTypes.INT().notNull()).notNull();
+ LocalFileIO fileIO = LocalFileIO.create();
+ Path file = new Path(new Path(tempPath.toUri()), UUID.randomUUID().toString());
+
+ try (PositionOutputStream out = fileIO.newOutputStream(file, false)) {
+ assertThatThrownBy(
+ () -> {
+ try (FormatWriter writer =
+ format.createWriterFactory(rowType).create(out, "null")) {
+ writer.addElement(GenericRow.of(0));
+ }
+ })
+ .isInstanceOf(ArithmeticException.class)
+ .hasMessage("integer overflow");
+ }
+ }
+
@Test
public void testSupportedDataTypes() {
ArrayList<DataField> dataFields = new ArrayList<>();
(paimon) 11/20: [python] Reduce memory usage for Parquet row-id updates (#9840)
Posted by ho...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
XiaoHongbo-Hope pushed a commit to branch release-2.1
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit 3f0d716f97ae43742d103e87e0aed55faca85b93
Author: XiaoHongbo <xi...@alibaba-inc.com>
AuthorDate: Wed Sep 16 19:35:30 2026 +0800
[python] Reduce memory usage for Parquet row-id updates (#9840)
---
paimon-python/pypaimon/read/table_read.py | 6 +
.../tests/map_shared_shredding_write_test.py | 44 ++++
.../tests/table_update_by_row_id_chunked_test.py | 112 ++++++++-
.../pypaimon/tests/table_upsert_by_key_test.py | 250 +++++++++++++++++++++
paimon-python/pypaimon/write/table_update.py | 1 +
.../pypaimon/write/table_update_by_row_id.py | 227 ++++++++++++++++++-
paimon-python/pypaimon/write/writer/data_writer.py | 43 +++-
.../pypaimon/write/writer/single_file_writer.py | 106 +++++++++
8 files changed, 767 insertions(+), 22 deletions(-)
diff --git a/paimon-python/pypaimon/read/table_read.py b/paimon-python/pypaimon/read/table_read.py
index 2b0d5a4ea3..5b8407874e 100644
--- a/paimon-python/pypaimon/read/table_read.py
+++ b/paimon-python/pypaimon/read/table_read.py
@@ -196,6 +196,12 @@ class TableRead:
self,
splits: List[Split],
blob_parallelism: Optional[int] = None):
+ """Return a closeable batch reader supporting context management.
+
+ Newer PyArrow versions use ``RecordBatchReader.from_stream``. Older
+ versions fall back to ``_ClosableArrowBatchReader``, which closes both
+ the batch iterator and its underlying reader.
+ """
reader, batch_iterator = self._new_arrow_batch_reader(
splits, blob_parallelism)
if (_RECORD_BATCH_READER_FROM_STREAM is not None
diff --git a/paimon-python/pypaimon/tests/map_shared_shredding_write_test.py b/paimon-python/pypaimon/tests/map_shared_shredding_write_test.py
index eba20b4270..fe82a6d2ed 100644
--- a/paimon-python/pypaimon/tests/map_shared_shredding_write_test.py
+++ b/paimon-python/pypaimon/tests/map_shared_shredding_write_test.py
@@ -92,6 +92,50 @@ class MapSharedShreddingWriteTest(unittest.TestCase):
result.column("metrics_overflow").to_pylist(),
)
+ def test_row_id_update_preserves_shared_shredding(self):
+ table = self._create_table('parquet', 2, {
+ 'data-evolution.enabled': 'true',
+ 'row-tracking.enabled': 'true',
+ })
+ self._write(table, pa.Table.from_pydict({
+ 'id': [1, 2],
+ 'metrics': [[('hot', 1)], [('warm', 2)]],
+ }, schema=self.arrow_schema))
+
+ read_builder = table.new_read_builder().with_projection(['id', '_ROW_ID'])
+ rows = read_builder.new_read().to_arrow(
+ read_builder.new_scan().plan().splits())
+ row_ids = dict(zip(rows.column('id').to_pylist(),
+ rows.column('_ROW_ID').to_pylist()))
+ row_id = row_ids[2]
+ update = pa.Table.from_pydict({
+ '_ROW_ID': [row_id],
+ 'metrics': [[('hot', 99), ('new', 7)]],
+ }, schema=pa.schema([
+ ('_ROW_ID', pa.int64()),
+ self.arrow_schema.field('metrics'),
+ ]))
+
+ builder = table.new_batch_write_builder()
+ messages = builder.new_update().with_update_type(
+ ['metrics']).update_by_arrow_with_row_id(update)
+ self.assertEqual(1, len(messages[0].new_files))
+ overlay = messages[0].new_files[0]
+ self.assertTrue(is_shared_shredding(
+ pq.read_schema(overlay.file_path).field('metrics')))
+ commit = builder.new_commit()
+ commit.commit(messages)
+ commit.close()
+
+ read_builder = table.new_read_builder().with_projection(['id', 'metrics'])
+ result = read_builder.new_read().to_arrow(
+ read_builder.new_scan().plan().splits())
+ self.assertEqual(
+ {1: [('hot', 1)], 2: [('hot', 99), ('new', 7)]},
+ dict(zip(result.column('id').to_pylist(),
+ result.column('metrics').to_pylist())),
+ )
+
def test_reject_orc(self):
with self.assertRaisesRegex(
ValueError,
diff --git a/paimon-python/pypaimon/tests/table_update_by_row_id_chunked_test.py b/paimon-python/pypaimon/tests/table_update_by_row_id_chunked_test.py
index c0902c5421..3ce93701ee 100644
--- a/paimon-python/pypaimon/tests/table_update_by_row_id_chunked_test.py
+++ b/paimon-python/pypaimon/tests/table_update_by_row_id_chunked_test.py
@@ -21,13 +21,76 @@ from unittest import mock
import pyarrow as pa
import pyarrow.compute as pc
+import pyarrow.parquet as pq
+from pypaimon.common.options.core_options import ChangelogProducer
from pypaimon.table.special_fields import SpecialFields
-from pypaimon.write.table_update_by_row_id import TableUpdateByRowId
+from pypaimon.write.table_update_by_row_id import (
+ TableUpdateByRowId,
+ _RowIdUpdateFileWriter,
+)
+from pypaimon.write.writer.single_file_writer import SingleFileWriter
class TableUpdateByRowIdChunkedTest(unittest.TestCase):
+ def test_streaming_overlay_rejects_sidecar_before_output(self):
+ table = mock.Mock()
+ table.is_primary_key_table = False
+ table.fields = []
+ options = table.options
+ options.file_format.return_value = 'parquet'
+ options.variant_shredding_enabled.return_value = False
+ options.data_evolution_row_sidecar_enabled.return_value = False
+ options.with_vector_format.return_value = False
+ options.changelog_producer.return_value = ChangelogProducer.NONE
+ self.assertTrue(_RowIdUpdateFileWriter.supports_table(table))
+
+ options.data_evolution_row_sidecar_enabled.return_value = True
+ with self.assertRaisesRegex(ValueError, 'requires plain Parquet'):
+ _RowIdUpdateFileWriter(table, (), ['id'])
+ table.file_io.new_output_stream.assert_not_called()
+
+ def test_streaming_overlay_rejects_shared_shredding_before_output(self):
+ table = mock.Mock()
+ table.is_primary_key_table = False
+ field = mock.Mock()
+ field.name = 'metrics'
+ table.fields = [field]
+ options = table.options
+ options.file_format.return_value = 'parquet'
+ options.variant_shredding_enabled.return_value = False
+ options.data_evolution_row_sidecar_enabled.return_value = False
+ options.with_vector_format.return_value = False
+ options.changelog_producer.return_value = ChangelogProducer.NONE
+ options.map_storage_layout.return_value = 'shared-shredding'
+
+ with self.assertRaisesRegex(ValueError, 'requires plain Parquet'):
+ _RowIdUpdateFileWriter(table, (), ['metrics'])
+ table.file_io.new_output_stream.assert_not_called()
+
+ def test_single_file_writer_rejects_other_formats_before_open(self):
+ file_io = mock.Mock()
+ with self.assertRaisesRegex(NotImplementedError, 'only supports Parquet'):
+ SingleFileWriter(file_io, 'unused.orc', pa.schema([]), 'orc',
+ 'zstd', 1, [], mock.Mock())
+ file_io.new_output_stream.assert_not_called()
+
+ def test_single_file_writer_failed_open_only_deletes_owned_file(self):
+ file_io = mock.Mock()
+ file_io.new_output_stream.side_effect = FileExistsError('already exists')
+ with self.assertRaises(FileExistsError):
+ SingleFileWriter(file_io, 'existing.parquet', pa.schema([]), 'parquet',
+ 'zstd', 1, [], mock.Mock())
+ file_io.delete_quietly.assert_not_called()
+
+ file_io.new_output_stream.side_effect = None
+ with mock.patch.object(pq, 'ParquetWriter', side_effect=OSError('failed to open')):
+ with self.assertRaisesRegex(OSError, 'failed to open'):
+ SingleFileWriter(file_io, 'new.parquet', pa.schema([]), 'parquet',
+ 'zstd', 1, [], mock.Mock())
+ file_io.delete_quietly.assert_called_once_with('new.parquet')
+
@staticmethod
def _updater():
updater = TableUpdateByRowId.__new__(TableUpdateByRowId)
@@ -105,6 +168,53 @@ class TableUpdateByRowIdChunkedTest(unittest.TestCase):
self._updater()._merge_update_with_original(
original, updates, ["payload"], first_row_id=0)
+ def test_streaming_update_rejects_non_contiguous_original_row_ids(self):
+ updates = pa.table({
+ SpecialFields.ROW_ID.name: pa.array([0], type=pa.int64()),
+ "payload": pa.array([3]),
+ })
+
+ for row_ids in ([0, 2], [0, 2, 1]):
+ with self.subTest(row_ids=row_ids):
+ batch = pa.RecordBatch.from_pydict({
+ SpecialFields.ROW_ID.name:
+ pa.array(row_ids, type=pa.int64()),
+ "payload": pa.array(row_ids),
+ })
+ managed_reader = mock.MagicMock()
+ managed_reader.__enter__.return_value = iter([batch])
+ table_read = mock.Mock()
+ table_read._to_managed_arrow_batch_reader.return_value = (
+ managed_reader)
+ updater = self._updater()
+ updater._original_file_read = mock.Mock(
+ return_value=(table_read, mock.sentinel.split))
+
+ with self.assertRaisesRegex(
+ ValueError, "not contiguous at row ID 0"):
+ list(updater._merged_batches(0, updates, ["payload"]))
+
+ def test_streaming_update_rejects_row_ids_before_original_group(self):
+ updates = pa.table({
+ SpecialFields.ROW_ID.name: pa.array([9], type=pa.int64()),
+ "payload": pa.array([3]),
+ })
+ batch = pa.RecordBatch.from_pydict({
+ SpecialFields.ROW_ID.name: pa.array([10, 11], type=pa.int64()),
+ "payload": pa.array([1, 2]),
+ })
+ managed_reader = mock.MagicMock()
+ managed_reader.__enter__.return_value = iter([batch])
+ table_read = mock.Mock()
+ table_read._to_managed_arrow_batch_reader.return_value = managed_reader
+ updater = self._updater()
+ updater._original_file_read = mock.Mock(
+ return_value=(table_read, mock.sentinel.split))
+
+ with self.assertRaisesRegex(
+ ValueError, "precede the original file group"):
+ list(updater._merged_batches(10, updates, ["payload"]))
+
def test_total_offsets_over_int32_remain_in_separate_chunks(self):
child_length = 1_100_000_000
large_chunk = self._list_chunk([0, child_length])
diff --git a/paimon-python/pypaimon/tests/table_upsert_by_key_test.py b/paimon-python/pypaimon/tests/table_upsert_by_key_test.py
index 7f06fcab42..564d61f3ed 100644
--- a/paimon-python/pypaimon/tests/table_upsert_by_key_test.py
+++ b/paimon-python/pypaimon/tests/table_upsert_by_key_test.py
@@ -16,12 +16,16 @@
# under the License.
import os
+from contextlib import contextmanager
import unittest
from unittest import mock
import pyarrow as pa
+import pyarrow.parquet as pq
+from pypaimon.read.table_read import TableRead
from pypaimon.table.special_fields import SpecialFields
+from pypaimon.write.table_update_by_row_id import _RowIdUpdateFileWriter
from pypaimon.tests.data_evolution_test_helpers import (
BatchModeMixin,
DataEvolutionTestBase,
@@ -61,6 +65,197 @@ class _TableUpsertByKeyTestBase(DataEvolutionTestBase):
def _apply_upsert_rows(self, table_update, rows, upsert_keys, cid):
raise NotImplementedError
+ def test_upsert_row_groups_do_not_follow_read_batches(self):
+ schema = pa.schema([('id', pa.int32()), ('score', pa.int32())])
+ for read_size in (73, 1024):
+ with self.subTest(read_size=read_size):
+ table = self._create_table(pa_schema=schema, options={
+ **self.table_options, 'read.batch-size': str(read_size)})
+ original = pa.Table.from_pydict(
+ {'id': list(range(5000)), 'score': list(range(5000))}, schema=schema)
+ self._write_arrow(table, original)
+ updates = pa.Table.from_pydict({'id': [13], 'score': [-1]}, schema=schema)
+ messages = self._upsert(table, updates, ['id'], ['score'])
+ files = [f for message in messages for f in message.new_files]
+ self.assertEqual(len(files), 1)
+ metadata = pq.read_metadata(files[0].file_path)
+ self.assertEqual(metadata.num_row_groups, 1)
+ self.assertEqual(metadata.row_group(0).num_rows, 5000)
+ scores = self._read_all(table).to_pydict()['score']
+ expected = list(range(5000))
+ expected[13] = -1
+ self.assertEqual(scores, expected)
+
+ def test_row_group_byte_budget_and_oversized_row(self):
+ schema = pa.schema([('id', pa.int32())])
+ table = self._create_table(pa_schema=schema, options={
+ **self.table_options, 'file.block-size': '400 b'})
+ writer = _RowIdUpdateFileWriter(table, (), ['id'])
+ data = pa.Table.from_pydict({'id': list(range(250))}, schema=schema)
+ try:
+ for batch_size in (1, 73, 250):
+ groups = list(writer._row_groups(
+ data.to_batches(max_chunksize=batch_size)))
+ self.assertEqual([group.num_rows for group in groups], [100, 100, 50])
+ self.assertTrue(pa.concat_tables(groups).equals(data))
+ self.assertTrue(all(group.nbytes <= 400 for group in groups))
+ large = pa.Table.from_pydict({'text': ['a', 'x' * 500, 'b']})
+ groups = list(writer._row_groups(large.to_batches()))
+ self.assertEqual([group.num_rows for group in groups], [1, 1, 1])
+ self.assertTrue(pa.concat_tables(groups).equals(large))
+ # Empty batches, nulls and sliced variable-width columns must not
+ # lose rows or make the output depend on input batch boundaries.
+ mixed = pa.Table.from_pydict({
+ 'text': [
+ 'skip', None, '', 'a' * 300,
+ 'b' * 300, None, 'last', 'skip',
+ ],
+ }).slice(1, 6)
+ layouts = []
+ for batch_size in (1, 2, 6):
+ batches = mixed.to_batches(max_chunksize=batch_size)
+ batches.insert(0, batches[0].slice(0, 0))
+ groups = list(writer._row_groups(batches))
+ self.assertTrue(pa.concat_tables(groups).equals(mixed))
+ self.assertTrue(all(group.nbytes <= 400 for group in groups))
+ layouts.append([group.num_rows for group in groups])
+ self.assertTrue(all(layout == layouts[0] for layout in layouts))
+ self.assertEqual(list(writer._row_groups([])), [])
+ with mock.patch.object(_RowIdUpdateFileWriter, '_ROW_GROUP_MAX_ROWS', 17):
+ groups = list(writer._row_groups(
+ data.to_batches(max_chunksize=73)))
+ self.assertEqual([group.num_rows for group in groups], [17] * 14 + [12])
+ self.assertTrue(pa.concat_tables(groups).equals(data))
+ finally:
+ writer.close()
+
+ def test_invalid_row_group_size_fails_before_opening_output(self):
+ schema = pa.schema([('id', pa.int32())])
+ table = self._create_table(pa_schema=schema, options={
+ **self.table_options, 'file.block-size': '0 b'})
+ with mock.patch.object(
+ table.file_io, 'new_output_stream') as output_stream:
+ with self.assertRaisesRegex(
+ ValueError, 'file.block-size must be positive'):
+ _RowIdUpdateFileWriter(table, (), ['id'])
+ output_stream.assert_not_called()
+
+ @mock.patch.object(_RowIdUpdateFileWriter, '_ROW_GROUP_MAX_ROWS', 2)
+ def test_partial_upsert_streams_original_file_group(self):
+ schema = pa.schema([
+ ('id', pa.int32()),
+ ('payload', pa.list_(pa.struct([('text', pa.string())]))),
+ ('score', pa.int32()),
+ ])
+ table = self._create_table(pa_schema=schema, options={
+ **self.table_options, 'metadata.stats-mode': 'full'})
+ expected = [{'id': i, 'payload': [{'text': str(i)}], 'score': i}
+ for i in range(12)]
+
+ def as_table(rows):
+ return pa.Table.from_pydict(
+ {name: [row[name] for row in rows] for name in schema.names},
+ schema=schema)
+
+ self._write_arrow(table, as_table(expected))
+ read_batches = TableRead._to_managed_arrow_batch_reader
+ write = pq.ParquetWriter.write_table
+ progress = {'read': 0, 'written': 0}
+ closed = []
+
+ @contextmanager
+ def bounded_reader(reader, splits, **kwargs):
+ source = read_batches(reader, splits, **kwargs)
+
+ def batches():
+ try:
+ for batch in source:
+ for start in range(0, batch.num_rows, 2):
+ self.assertEqual(progress['read'], progress['written'])
+ piece = batch.slice(start, 2)
+ progress['read'] += piece.num_rows
+ yield piece
+ finally:
+ source.close()
+ closed.append(True)
+ iterator = batches()
+ try:
+ yield iterator
+ finally:
+ iterator.close()
+
+ def record_write(writer, batch, **kwargs):
+ result = write(writer, batch, **kwargs)
+ progress['written'] += batch.num_rows
+ return result
+
+ for replacements in ([9, 1, 5], [0, 11]):
+ updates = [{'id': i, 'payload': None if i == 5 else
+ [{'text': 'updated-' + str(i)}],
+ 'score': None if i == 5 else -i} for i in replacements]
+ progress.update(read=0, written=0)
+ with mock.patch.object(TableRead, 'to_arrow', side_effect=AssertionError(
+ 'upsert must not materialize the original file group')):
+ with mock.patch.object(TableRead, '_to_managed_arrow_batch_reader', bounded_reader):
+ with mock.patch.object(pq.ParquetWriter, 'write_table', record_write):
+ messages = self._upsert(
+ table, as_table(updates),
+ ['id'], ['payload', 'score'])
+ self.assertEqual(progress, {'read': 12, 'written': 12})
+ for row in updates:
+ expected[row['id']] = row
+ self.assertEqual(self._read_all(table).to_pydict(), as_table(expected).to_pydict())
+ files = [f for msg in messages for f in msg.new_files]
+ self.assertEqual(len(files), 1)
+ self.assertEqual((files[0].first_row_id, files[0].row_count), (0, 12))
+ scores = [r['score'] for r in expected if r['score'] is not None]
+ self.assertEqual(files[0].value_stats.min_values.values[1], min(scores))
+ self.assertEqual(files[0].value_stats.max_values.values[1], max(scores))
+ self.assertEqual(files[0].value_stats.null_counts, [1, 1])
+
+ # A failed streamed write must leave the committed table intact and
+ # remove the partially written overlay.
+ self.assertEqual(len(closed), 2)
+ progress.update(read=0, written=0)
+
+ def fail_second_write(writer, batch, **kwargs):
+ if progress['written']:
+ raise OSError('injected write failure')
+ return record_write(writer, batch, **kwargs)
+
+ with mock.patch.object(table.file_io, 'delete_quietly',
+ wraps=table.file_io.delete_quietly) as delete:
+ with mock.patch.object(pq.ParquetWriter, 'write_table',
+ fail_second_write):
+ with mock.patch.object(TableRead, '_to_managed_arrow_batch_reader', bounded_reader):
+ with self.assertRaisesRegex(OSError, 'injected write failure'):
+ self._upsert(table, as_table(updates),
+ ['id'], ['payload', 'score'])
+ self.assertEqual(len(closed), 3)
+ self.assertTrue(delete.called)
+ for call in delete.call_args_list:
+ self.assertFalse(table.file_io.exists(call[0][0]))
+ self.assertEqual(self._read_all(table).to_pydict(), as_table(expected).to_pydict())
+
+ # Closing the format writer is part of the file transaction too.
+ close = pq.ParquetWriter.close
+
+ def fail_close(writer):
+ was_open = writer.is_open
+ close(writer)
+ if was_open:
+ raise OSError('injected close failure')
+
+ with mock.patch.object(table.file_io, 'delete_quietly',
+ wraps=table.file_io.delete_quietly) as delete:
+ with mock.patch.object(pq.ParquetWriter, 'close', fail_close):
+ with self.assertRaisesRegex(OSError, 'injected close failure'):
+ self._upsert(table, as_table(updates), ['id'], ['payload', 'score'])
+ self.assertTrue(delete.called)
+ for call in delete.call_args_list:
+ self.assertFalse(table.file_io.exists(call[0][0]))
+ self.assertEqual(self._read_all(table).to_pydict(), as_table(expected).to_pydict())
+
# ------------------------------------------------------------------
# Helpers built on the primitives
# ------------------------------------------------------------------
@@ -953,6 +1148,61 @@ class _TableUpsertByKeyTestBase(DataEvolutionTestBase):
self.assertEqual((1, 'Alice', 99, 'NYC'), rows[0])
self.assertEqual((2, 'Bob', 88, 'LA'), rows[1])
+ def test_duplicate_update_cols_are_deduplicated(self):
+ table = self._create_table()
+ self._write_arrow(table, pa.Table.from_pydict({
+ 'id': [1],
+ 'name': ['Alice'],
+ 'age': [25],
+ 'city': ['NYC'],
+ }, schema=self.pa_schema))
+
+ messages = self._upsert(
+ table,
+ pa.Table.from_pydict({
+ 'id': [1],
+ 'name': ['ignored'],
+ 'age': [99],
+ 'city': ['ignored'],
+ }, schema=self.pa_schema),
+ upsert_keys=['id'],
+ # Matching the schema width must not mean "update all columns".
+ update_cols=['age'] * len(table.field_names),
+ )
+
+ self.assertEqual(
+ self._read_all(table).to_pydict(),
+ {'id': [1], 'name': ['Alice'], 'age': [99], 'city': ['NYC']},
+ )
+ files = [file for message in messages for file in message.new_files]
+ self.assertEqual([file.write_cols for file in files], [['age']])
+
+ def test_not_null_update_across_read_batches(self):
+ schema = pa.schema([
+ pa.field('id', pa.int32(), nullable=False),
+ pa.field('score', pa.int32(), nullable=False),
+ ])
+ table = self._create_table(pa_schema=schema, options={
+ **self.table_options, 'read.batch-size': '2'})
+ original = pa.Table.from_pydict({
+ 'id': list(range(4)),
+ 'score': list(range(4)),
+ }, schema=schema)
+ self._write_arrow(table, original)
+
+ updates = pa.Table.from_pydict({'id': [2], 'score': [99]}, schema=schema)
+ messages = self._upsert(table, updates, ['id'], ['score'])
+
+ expected = original.set_column(
+ 1,
+ schema.field('score'),
+ pa.array([0, 1, 99, 3], type=pa.int32()),
+ )
+ self.assertTrue(self._read_all(table).equals(expected))
+ files = [file for message in messages for file in message.new_files]
+ self.assertEqual(len(files), 1)
+ self.assertFalse(pq.read_schema(files[0].file_path).field('score').nullable)
+
# ==================================================================
# Duplicate-key dedup tests — parametrised
# ==================================================================
diff --git a/paimon-python/pypaimon/write/table_update.py b/paimon-python/pypaimon/write/table_update.py
index c358644460..922b1bc8c3 100644
--- a/paimon-python/pypaimon/write/table_update.py
+++ b/paimon-python/pypaimon/write/table_update.py
@@ -126,6 +126,7 @@ class TableUpdate:
self.projection = None
def with_update_type(self, update_cols: List[str]):
+ update_cols = list(dict.fromkeys(update_cols))
for col in update_cols:
if col not in self.table.field_names:
raise ValueError(f"Column {col} is not in table schema.")
diff --git a/paimon-python/pypaimon/write/table_update_by_row_id.py b/paimon-python/pypaimon/write/table_update_by_row_id.py
index 90c08d20d2..9226f8d823 100644
--- a/paimon-python/pypaimon/write/table_update_by_row_id.py
+++ b/paimon-python/pypaimon/write/table_update_by_row_id.py
@@ -16,6 +16,7 @@
# under the License.
import bisect
+import uuid
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional, Set, Tuple
@@ -23,8 +24,10 @@ import numpy as np
import pyarrow as pa
import pyarrow.compute as pc
+from pypaimon.common.options.core_options import ChangelogProducer, CoreOptions
from pypaimon.manifest.schema.data_file_meta import DataFileMeta
from pypaimon.manifest.schema.manifest_entry import ManifestEntry
+from pypaimon.manifest.schema.simple_stats import SimpleStats
from pypaimon.read.scanner.data_evolution_split_generator import (
DataEvolutionSplitGenerator,
)
@@ -49,6 +52,13 @@ from pypaimon.write.row_utils import (
value_for_arrow,
)
from pypaimon.write.writer.blob_writer import BlobWriter
+from pypaimon.write.writer.append_only_data_writer import AppendOnlyDataWriter
+from pypaimon.write.writer.single_file_writer import SingleFileWriter
+from pypaimon.write.writer.write_buffer import WriteBuffer
+
+_ARROW_MAJOR = int(pa.__version__.split('.')[0])
+# Keep aligned with org.apache.parquet.hadoop.ParquetWriter.DEFAULT_BLOCK_SIZE.
+_DEFAULT_PARQUET_BLOCK_SIZE = 128 * 1024 * 1024
@dataclass(frozen=True)
@@ -66,6 +76,134 @@ class _FilesInfo:
valid_row_id_ranges: List[Range] = field(default_factory=list)
+class _RowIdUpdateFileWriter:
+ """Write one plain-Parquet update file for a row-id file group."""
+
+ _ROW_GROUP_MAX_ROWS = 1024 * 1024
+
+ @staticmethod
+ def supports_table(table):
+ options = table.options
+ return (not table.is_primary_key_table
+ and options.file_format(CoreOptions.FILE_FORMAT_PARQUET)
+ == CoreOptions.FILE_FORMAT_PARQUET
+ and not (options.variant_shredding_enabled()
+ and options.variant_shredding_schema())
+ and not options.data_evolution_row_sidecar_enabled(False)
+ and not options.with_vector_format()
+ and options.changelog_producer() == ChangelogProducer.NONE
+ and not any(options.map_storage_layout(f.name) == 'shared-shredding'
+ for f in table.fields)
+ and not any(is_blob_file_field(f) for f in table.fields))
+
+ def __init__(self, table, partition, column_names):
+ if not self.supports_table(table):
+ raise ValueError('Row-id update file writer requires plain Parquet without sidecars')
+ configured = table.options.file_block_size()
+ self._target_bytes = (configured.get_bytes() if configured is not None
+ else _DEFAULT_PARQUET_BLOCK_SIZE)
+ if self._target_bytes <= 0:
+ raise ValueError('file.block-size must be positive')
+ self._data_writer = AppendOnlyDataWriter(
+ table, partition, 0, 0, table.options, write_cols=column_names)
+
+ @staticmethod
+ def _row_group_slice(batch, offset, count):
+ piece = batch.slice(offset, count)
+ # Arrow 6 nbytes counts full backing buffers even for slices. Compact
+ # the slice there so both accounting and retained buffers stay bounded.
+ if _ARROW_MAJOR < 7:
+ piece = pa.RecordBatch.from_arrays(
+ [pa.concat_arrays([column]) for column in piece.columns], schema=piece.schema)
+ return piece
+
+ def _row_groups(self, batches):
+ """Bound row-group buffering using Arrow-side bytes, not encoded size."""
+ buffer = WriteBuffer(self._data_writer._merge_data)
+ try:
+ for batch in batches:
+ offset = 0
+ while offset < batch.num_rows:
+ count = min(batch.num_rows - offset,
+ self._ROW_GROUP_MAX_ROWS - buffer.num_rows)
+ piece = self._row_group_slice(batch, offset, count)
+ available = self._target_bytes - buffer.nbytes
+ if piece.nbytes > available:
+ low, high = 0, count
+ while low < high:
+ middle = (low + high + 1) // 2
+ if self._row_group_slice(piece, 0, middle).nbytes <= available:
+ low = middle
+ else:
+ high = middle - 1
+ count = low
+ if count == 0 and buffer.num_rows:
+ yield buffer.take()
+ continue
+ count = max(1, count)
+ piece = self._row_group_slice(piece, 0, count)
+ buffer.append(pa.Table.from_batches([piece]))
+ offset += count
+ del piece
+ if buffer.nbytes >= self._target_bytes or buffer.num_rows >= self._ROW_GROUP_MAX_ROWS:
+ yield buffer.take()
+ del batch
+ if buffer.num_rows:
+ yield buffer.take()
+ finally:
+ buffer.reset()
+
+ def write_batches(self, batches):
+ """Write one file while keeping input and output row groups bounded."""
+ writer = self._data_writer
+ file_name = '{}{}-0.parquet'.format(
+ writer.options.data_file_prefix(), uuid.uuid4())
+ file_path = writer._generate_file_path(file_name)
+ fields = []
+ groups = self._row_groups(batches)
+ file_writer = None
+ try:
+ for batch in groups:
+ if not batch.num_rows:
+ continue
+ if file_writer is None:
+ if writer.options.metadata_stats_enabled():
+ fields = PyarrowFieldParser.to_paimon_schema(batch.schema)
+ file_writer = SingleFileWriter(
+ writer.file_io, file_path, batch.schema, writer.file_format,
+ writer.compression, writer.zstd_level,
+ fields, writer._get_column_stats)
+ file_writer.write(batch, row_group_size=batch.num_rows)
+ del batch
+ if file_writer is None:
+ return []
+ file_writer.close()
+ meta = writer._create_data_file_meta(
+ file_name=file_name,
+ file_path=file_path,
+ row_count=file_writer.row_count,
+ min_key=GenericRow([], []), max_key=GenericRow([], []),
+ key_stats=SimpleStats.empty_stats(),
+ value_stats=writer._collect_value_stats(
+ None, fields, file_writer.column_stats),
+ min_sequence_number=0, max_sequence_number=0,
+ )
+ writer._finish_data_file(meta)
+ return [meta]
+ except Exception:
+ if file_writer is not None:
+ file_writer.abort()
+ raise
+ finally:
+ groups.close()
+
+ def abort(self):
+ self._data_writer.abort()
+
+ def close(self):
+ self._data_writer.close()
+
+
class TableUpdateByRowId:
"""
Table update for partial column updates (data evolution).
@@ -211,6 +349,7 @@ class TableUpdateByRowId:
if not column_names:
raise ValueError("column_names cannot be empty")
+ column_names = list(dict.fromkeys(column_names))
if SpecialFields.ROW_ID.name not in data.column_names:
raise ValueError(f"Input data must contain {SpecialFields.ROW_ID.name} column")
@@ -257,6 +396,7 @@ class TableUpdateByRowId:
) -> List[CommitMessage]:
if not column_names:
raise ValueError("column_names cannot be empty")
+ column_names = list(dict.fromkeys(column_names))
if len(rows) != len(row_ids_by_row):
raise ValueError(
"rows and row_ids_by_row must have the same length: "
@@ -390,7 +530,7 @@ class TableUpdateByRowId:
group_blob_object_columns,
)
- def _read_original_file_data(self, first_row_id: int, column_names: List[str]) -> Optional[pa.Table]:
+ def _read_original_file_data(self, first_row_id: int, column_names: List[str]) -> pa.Table:
"""Read original file data for the given first_row_id.
Only reads columns that exist in the original file and need to be updated.
@@ -402,16 +542,17 @@ class TableUpdateByRowId:
column_names: The column names to update
Returns:
- PyArrow Table containing the original data for columns that exist in the file,
- or None if no columns need to be read from the original file.
+ PyArrow Table containing the original values for the requested columns.
"""
+ table_read, origin_split = self._original_file_read(first_row_id, column_names)
+ original = table_read.to_arrow([origin_split])
+ return original.select(column_names)
+
+ def _original_file_read(self, first_row_id, column_names):
wanted = set(column_names)
read_fields: List[DataField] = [
table_field for table_field in self.table.fields if table_field.name in wanted
]
- if not read_fields:
- return None
-
entry = self._first_row_id_index.get(first_row_id)
if entry is None:
raise ValueError(f"No file found for first_row_id {first_row_id}")
@@ -432,8 +573,66 @@ class TableUpdateByRowId:
predicate=None,
read_type=read_fields + [SpecialFields.ROW_ID],
)
- original = table_read.to_arrow([origin_split])
- return original.select([field.name for field in read_fields])
+ return table_read, origin_split
+
+ def _merged_batches(self, first_row_id, data, column_names):
+ """Merge ordinary columns a batch at a time in physical row order."""
+ table_read, split = self._original_file_read(first_row_id, column_names)
+ updates = sorted(enumerate(data[SpecialFields.ROW_ID.name].to_pylist()),
+ key=lambda item: item[1])
+ update_index = 0
+ offset = first_row_id
+ with table_read._to_managed_arrow_batch_reader([split]) as reader:
+ for batch in reader:
+ if not batch.num_rows:
+ continue
+ row_ids = batch[SpecialFields.ROW_ID.name]
+ if row_ids.null_count:
+ raise ValueError(
+ 'Original file group contains null _ROW_ID values')
+ row_id_values = row_ids.to_numpy(zero_copy_only=True)
+ if (row_id_values[0] != offset
+ or (len(row_id_values) > 1
+ and not np.all(np.diff(row_id_values) == 1))):
+ raise ValueError(
+ f'Original file group is not contiguous at row ID {offset}')
+ end = offset + batch.num_rows
+ selected = []
+ while update_index < len(updates) and updates[update_index][1] < end:
+ if updates[update_index][1] < offset:
+ raise ValueError(
+ 'Update row IDs precede the original file group')
+ selected.append(updates[update_index][0])
+ update_index += 1
+ original = pa.Table.from_batches([batch]).select(column_names)
+ if selected:
+ merged, _ = self._merge_update_with_original(
+ original, data.take(selected), column_names, offset)
+ else:
+ merged = original
+ yield from merged.to_batches()
+ offset = end
+ del batch, original, merged
+ if update_index != len(updates):
+ raise ValueError('Update row IDs extend past the original file group')
+
+ def _write_group_streaming(self, partition, first_row_id, data, column_names):
+ writer = _RowIdUpdateFileWriter(
+ self.table, tuple(partition.values), column_names)
+ batches = self._merged_batches(first_row_id, data, column_names)
+ try:
+ files = writer.write_batches(batches)
+ self._assign_update_file_metadata(files, first_row_id, column_names, {})
+ if files:
+ self.commit_messages.append(CommitMessage(
+ partition=tuple(partition.values), bucket=0, new_files=files,
+ check_from_snapshot=self.snapshot_id))
+ except Exception:
+ writer.abort()
+ raise
+ finally:
+ batches.close()
+ writer.close()
def _merge_update_with_original(
self,
@@ -533,7 +732,13 @@ class TableUpdateByRowId:
merged_columns[col_name] = self._merge_chunked_column(
original_col, update_col, sorted_updates)
- merged_table = pa.table(merged_columns) if merged_columns else None
+ merged_table = None
+ if merged_columns:
+ merged_schema = pa.schema([
+ original_data.schema.field(name)
+ for name in merged_columns
+ ])
+ merged_table = pa.table(merged_columns, schema=merged_schema)
return merged_table, blob_columns
@@ -770,6 +975,10 @@ class TableUpdateByRowId:
Reads the original file data, merges in the update values, and
writes a single output file (rolling disabled) for the group.
"""
+ # Specialized writers still own their sidecars and physical encoding.
+ if _RowIdUpdateFileWriter.supports_table(self.table):
+ self._write_group_streaming(partition, first_row_id, data, column_names)
+ return
original_data = self._read_original_file_data(first_row_id, column_names)
_, target_files = self._first_row_id_index[first_row_id]
blob_columns_with_baseline = {
diff --git a/paimon-python/pypaimon/write/writer/data_writer.py b/paimon-python/pypaimon/write/writer/data_writer.py
index 46db0578ce..71f5004f7e 100644
--- a/paimon-python/pypaimon/write/writer/data_writer.py
+++ b/paimon-python/pypaimon/write/writer/data_writer.py
@@ -343,9 +343,9 @@ class DataWriter(ABC):
min_seq = self.sequence_generator.start
max_seq = self.sequence_generator.current
creation_time = Timestamp.now()
- data_meta = DataFileMeta.create(
+ data_meta = self._create_data_file_meta(
file_name=file_name,
- file_size=self.file_io.get_file_size(file_path),
+ file_path=file_path,
row_count=data.num_rows,
min_key=GenericRow(min_key, self.trimmed_primary_keys_fields),
max_key=GenericRow(max_key, self.trimmed_primary_keys_fields),
@@ -353,17 +353,8 @@ class DataWriter(ABC):
value_stats=value_stats,
min_sequence_number=min_seq,
max_sequence_number=max_seq,
- schema_id=self.table.table_schema.id,
- level=0,
extra_files=extra_files,
creation_time=creation_time,
- delete_row_count=0,
- file_source=0,
- value_stats_cols=None if value_stats_enabled else [],
- external_path=external_path_str,
- first_row_id=None,
- write_cols=self.write_cols,
- file_path=file_path,
)
if self.changelog_producer == ChangelogProducer.INPUT:
@@ -378,8 +369,14 @@ class DataWriter(ABC):
self.file_io.delete_quietly(row_sidecar_path)
raise
+ self._finish_data_file(data_meta, changelog_meta, shared_shredding_stats)
+
+ def _finish_data_file(self, data_meta, changelog_meta=None,
+ shared_shredding_stats=None):
+ """Record a fully written data file and its optional changelog."""
self.sequence_generator.start = self.sequence_generator.current
- self._map_shared_shredding.file_completed(shared_shredding_stats)
+ if shared_shredding_stats is not None:
+ self._map_shared_shredding.file_completed(shared_shredding_stats)
self.committed_files.append(data_meta)
if changelog_meta is not None:
self.committed_changelog_files.append(changelog_meta)
@@ -391,6 +388,28 @@ class DataWriter(ABC):
self.file_io.write_parquet(path, data, compression=self.compression, zstd_level=self.zstd_level)
return {}
+ def _create_data_file_meta(self, file_name, file_path, row_count,
+ min_key, max_key, key_stats, value_stats,
+ min_sequence_number, max_sequence_number,
+ extra_files=None, creation_time=None):
+ """Common metadata finalization for buffered and incremental files."""
+ return DataFileMeta.create(
+ file_name=file_name,
+ file_size=self.file_io.get_file_size(file_path),
+ row_count=row_count,
+ min_key=min_key, max_key=max_key,
+ key_stats=key_stats, value_stats=value_stats,
+ min_sequence_number=min_sequence_number,
+ max_sequence_number=max_sequence_number,
+ schema_id=self.table.table_schema.id, level=0,
+ extra_files=extra_files if extra_files is not None else [],
+ creation_time=creation_time if creation_time is not None else Timestamp.now(),
+ delete_row_count=0, file_source=0,
+ value_stats_cols=None if self.options.metadata_stats_enabled() else [],
+ external_path=file_path if self.external_path_provider is not None else None,
+ first_row_id=None, write_cols=self.write_cols, file_path=file_path,
+ )
+
def _apply_variant_shredding(self, data: pa.Table) -> pa.Table:
"""Transform VARIANT columns into shredded Parquet format.
diff --git a/paimon-python/pypaimon/write/writer/single_file_writer.py b/paimon-python/pypaimon/write/writer/single_file_writer.py
new file mode 100644
index 0000000000..184d013822
--- /dev/null
+++ b/paimon-python/pypaimon/write/writer/single_file_writer.py
@@ -0,0 +1,106 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from contextlib import suppress
+
+import pyarrow.parquet as pq
+
+from pypaimon.common.options.core_options import CoreOptions
+
+
+class SingleFileWriter:
+ """Write one file incrementally; currently only Parquet is supported."""
+
+ def __init__(self, file_io, path, schema, file_format, compression, zstd_level,
+ stats_fields, stats_collector):
+ if file_format != CoreOptions.FILE_FORMAT_PARQUET:
+ raise NotImplementedError(
+ 'SingleFileWriter only supports Parquet, got {}'.format(file_format))
+ self._file_io = file_io
+ self._path = path
+ self._stats_fields = stats_fields
+ self._stats_collector = stats_collector
+ self._stream = None
+ self._writer = None
+ self._owns_file = False
+ self._closed = False
+ self.row_count = 0
+ self.column_stats = {}
+
+ kwargs = {'compression': compression}
+ if compression.lower() == 'zstd':
+ kwargs['compression_level'] = zstd_level
+ try:
+ self._stream = file_io.new_output_stream(path)
+ self._owns_file = True
+ self._writer = pq.ParquetWriter(self._stream, schema, **kwargs)
+ except Exception:
+ self.abort()
+ raise
+
+ def write(self, data, row_group_size=None):
+ if self._closed:
+ raise RuntimeError('Writer is already closed')
+ try:
+ kwargs = {}
+ if row_group_size is not None:
+ kwargs['row_group_size'] = row_group_size
+ self._writer.write_table(data, **kwargs)
+ self.row_count += data.num_rows
+ for field in self._stats_fields:
+ current = self._stats_collector(data, field.name)
+ previous = self.column_stats.get(field.name)
+ if previous is not None:
+ current['null_counts'] += previous['null_counts']
+ for key, choose in (
+ ('min_values', min), ('max_values', max)):
+ values = [
+ value for value in (previous[key], current[key])
+ if value is not None
+ ]
+ current[key] = choose(values) if values else None
+ self.column_stats[field.name] = current
+ except Exception:
+ self.abort()
+ raise
+
+ def close(self):
+ if self._closed:
+ return
+ try:
+ self._writer.close()
+ self._writer = None
+ self._stream.close()
+ self._stream = None
+ self._closed = True
+ except Exception:
+ self.abort()
+ raise
+
+ def abort(self):
+ if self._writer is not None:
+ with suppress(Exception):
+ self._writer.close()
+ self._writer = None
+ if self._stream is not None:
+ with suppress(Exception):
+ self._stream.close()
+ self._stream = None
+ self._closed = True
+ if self._owns_file:
+ self._file_io.delete_quietly(self._path)
+ self._owns_file = False
(paimon) 10/20: [core] Generate sidecars for newly written manifests (#9889)
Posted by ho...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
XiaoHongbo-Hope pushed a commit to branch release-2.1
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit 9686b2a1c2be4fd10edbd5a4ef94043d8261a2a7
Author: YeJunHao <41...@users.noreply.github.com>
AuthorDate: Wed Sep 16 19:21:52 2026 +0800
[core] Generate sidecars for newly written manifests (#9889)
---
docs/docs/concepts/spec/manifest.md | 13 +-
docs/generated/core_configuration.html | 6 +
.../main/java/org/apache/paimon/CoreOptions.java | 11 +
.../benchmark/ManifestFileSorterBenchmark.java | 3 +-
.../java/org/apache/paimon/AbstractFileStore.java | 3 +-
.../apache/paimon/manifest/ManifestAvroWriter.java | 52 ++-
.../org/apache/paimon/manifest/ManifestFile.java | 20 +-
.../java/org/apache/paimon/CoreOptionsTest.java | 18 ++
.../paimon/manifest/ManifestFileMetaTest.java | 6 +-
.../paimon/manifest/ManifestFileMetaTestBase.java | 3 +-
.../apache/paimon/manifest/ManifestFileTest.java | 3 +-
.../paimon/manifest/ManifestIndexTestUtils.java | 1 +
.../paimon/manifest/ManifestSidecarTest.java | 4 +-
.../paimon/manifest/ManifestSidecarWriteTest.java | 347 +++++++++++++++++++++
.../operation/ManifestRewriteCleanupTest.java | 3 +-
15 files changed, 471 insertions(+), 22 deletions(-)
diff --git a/docs/docs/concepts/spec/manifest.md b/docs/docs/concepts/spec/manifest.md
index ada9baf1ce..29466c1ad6 100644
--- a/docs/docs/concepts/spec/manifest.md
+++ b/docs/docs/concepts/spec/manifest.md
@@ -71,11 +71,14 @@ using independent partition, row-ID and bucket coverage. A sidecar uses the
`.avro.sidecar` reference in the manifest metadata's `_EXTRA_FILES`, without probing a
derived file name. The Avro schemas and `_VERSION` identifiers remain unchanged.
-The utility includes construction, validation, block selection and optional caching. Table
-writers and scans do not yet invoke it automatically. Callers are responsible for publishing
-sidecar references, managing file ownership, applying entry filters and reconciling ADD/DELETE
-entries after block selection. `build` reads the completed physical manifest and returns
-sidecar bytes; it does not write or publish another file.
+The utility includes construction, validation, block selection and optional caching. Java table
+writers generate sidecars when `manifest.sidecar.enabled` is true; when unset, it inherits
+`manifest-sort.enabled`. Both ordinary writes and raw manifest rewrites build the sidecar from
+the completed output manifest and publish its `_EXTRA_FILES` reference only after both files
+close successfully. Failed writes and aborted writers clean up their own manifest/sidecar pairs.
+Scans do not yet invoke sidecar pruning automatically. Callers remain responsible for applying
+entry filters and reconciling ADD/DELETE entries after block selection. The low-level `build`
+method returns sidecar bytes without writing or publishing another file.
Callers decide whether to invoke `build` and `read`; these utilities have no read/write switches.
`build` and `Builder` accept `rowIdEnabled` and `bucketEnabled` arguments for independent
diff --git a/docs/generated/core_configuration.html b/docs/generated/core_configuration.html
index 16d3339f41..cf48d473c4 100644
--- a/docs/generated/core_configuration.html
+++ b/docs/generated/core_configuration.html
@@ -1113,6 +1113,12 @@ Mainly to resolve data skew on primary keys. We recommend starting with 64 mb wh
<td>Boolean</td>
<td>Whether to skip automatic manifest merging during commit when write-only is true. This also skips automatic manifest sort rewrite. Explicit manifest compaction is not affected.</td>
</tr>
+ <tr>
+ <td><h5>manifest.sidecar.enabled</h5></td>
+ <td style="word-wrap: break-word;">(none)</td>
+ <td>Boolean</td>
+ <td>Whether to enable manifest sidecars with independent partition, row-id and bucket coverage. Defaults to manifest-sort.enabled when unset.</td>
+ </tr>
<tr>
<td><h5>manifest.target-file-size</h5></td>
<td style="word-wrap: break-word;">8 mb</td>
diff --git a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java
index 68215e18e4..9bb324b780 100644
--- a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java
+++ b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java
@@ -522,6 +522,13 @@ public class CoreOptions implements Serializable {
.defaultValue(MemorySize.ofMebiBytes(8))
.withDescription("Suggested file size of a manifest file.");
+ public static final ConfigOption<Boolean> MANIFEST_SIDECAR_ENABLED =
+ key("manifest.sidecar.enabled")
+ .booleanType()
+ .noDefaultValue()
+ .withDescription(
+ "Whether to enable manifest sidecars with independent partition, row-id and bucket coverage. Defaults to manifest-sort.enabled when unset.");
+
public static final ConfigOption<MemorySize> MANIFEST_FULL_COMPACTION_FILE_SIZE =
key("manifest.full-compaction-threshold-size")
.memoryType()
@@ -3217,6 +3224,10 @@ public class CoreOptions implements Serializable {
return options.get(MANIFEST_TARGET_FILE_SIZE);
}
+ public boolean manifestSidecarEnabled() {
+ return options.getOptional(MANIFEST_SIDECAR_ENABLED).orElseGet(this::manifestSortEnabled);
+ }
+
public MemorySize manifestFullCompactionThresholdSize() {
return options.get(MANIFEST_FULL_COMPACTION_FILE_SIZE);
}
diff --git a/paimon-benchmark/paimon-micro-benchmarks/src/test/java/org/apache/paimon/benchmark/ManifestFileSorterBenchmark.java b/paimon-benchmark/paimon-micro-benchmarks/src/test/java/org/apache/paimon/benchmark/ManifestFileSorterBenchmark.java
index 5a5f31d875..e805efb694 100644
--- a/paimon-benchmark/paimon-micro-benchmarks/src/test/java/org/apache/paimon/benchmark/ManifestFileSorterBenchmark.java
+++ b/paimon-benchmark/paimon-micro-benchmarks/src/test/java/org/apache/paimon/benchmark/ManifestFileSorterBenchmark.java
@@ -300,7 +300,8 @@ public class ManifestFileSorterBenchmark {
"zstd",
pathFactory,
TARGET_MANIFEST_SIZE,
- null)
+ null,
+ new CoreOptions(new Options()))
.create();
}
diff --git a/paimon-core/src/main/java/org/apache/paimon/AbstractFileStore.java b/paimon-core/src/main/java/org/apache/paimon/AbstractFileStore.java
index 7399e05778..29c7ab8de0 100644
--- a/paimon-core/src/main/java/org/apache/paimon/AbstractFileStore.java
+++ b/paimon-core/src/main/java/org/apache/paimon/AbstractFileStore.java
@@ -211,7 +211,8 @@ abstract class AbstractFileStore<T> implements FileStore<T> {
options.manifestCompression(),
pathFactory(),
options.manifestTargetSize().getBytes(),
- readManifestCache);
+ readManifestCache,
+ options);
}
@Override
diff --git a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestAvroWriter.java b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestAvroWriter.java
index 4dabe54f3f..92c81bea7c 100644
--- a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestAvroWriter.java
+++ b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestAvroWriter.java
@@ -18,6 +18,7 @@
package org.apache.paimon.manifest;
+import org.apache.paimon.CoreOptions;
import org.apache.paimon.data.BinaryRow;
import org.apache.paimon.data.InternalRow;
import org.apache.paimon.format.SimpleColStats;
@@ -43,6 +44,7 @@ import java.io.IOException;
import java.io.UncheckedIOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.IdentityHashMap;
import java.util.List;
import java.util.Map;
@@ -68,6 +70,7 @@ public final class ManifestAvroWriter implements AutoCloseable {
private final String compression;
private final PathFactory pathFactory;
private final long targetFileSize;
+ private final CoreOptions options;
private final List<ManifestFileMeta> results = new ArrayList<>();
private final List<Path> completedPaths = new ArrayList<>();
@@ -83,7 +86,8 @@ public final class ManifestAvroWriter implements AutoCloseable {
ObjectSerializer<ManifestEntry> serializer,
String compression,
PathFactory pathFactory,
- long targetFileSize) {
+ long targetFileSize,
+ CoreOptions options) {
this.fileIO = fileIO;
this.schemaManager = schemaManager;
this.partitionType = partitionType;
@@ -92,6 +96,7 @@ public final class ManifestAvroWriter implements AutoCloseable {
this.compression = compression;
this.pathFactory = pathFactory;
this.targetFileSize = targetFileSize;
+ this.options = options;
}
public void write(ManifestEntry entry) throws IOException {
@@ -218,6 +223,9 @@ public final class ManifestAvroWriter implements AutoCloseable {
currentWriter.close();
ManifestFileMeta result = currentWriter.result();
completedPaths.add(currentWriter.path);
+ if (currentWriter.sidecarCreated) {
+ completedPaths.add(ManifestSidecar.path(currentWriter.path));
+ }
results.add(result);
currentWriter = null;
}
@@ -413,6 +421,7 @@ public final class ManifestAvroWriter implements AutoCloseable {
private @Nullable RowIdStats rowIdStats = new RowIdStats();
private boolean closed;
private boolean aborted;
+ private boolean sidecarCreated;
private FileWriter(Path path) {
this.path = path;
@@ -488,7 +497,7 @@ public final class ManifestAvroWriter implements AutoCloseable {
maxLevel = Math.max(maxLevel, entry.level());
if (rowIdStats != null) {
Long firstRowId = entry.file().firstRowId();
- if (firstRowId == null) {
+ if (!validRowIdRange(firstRowId, entry.file().rowCount())) {
rowIdStats = null;
} else {
rowIdStats.collect(firstRowId, entry.file().rowCount());
@@ -515,7 +524,7 @@ public final class ManifestAvroWriter implements AutoCloseable {
minLevel = Math.min(minLevel, entry.level);
maxLevel = Math.max(maxLevel, entry.level);
if (rowIdStats != null) {
- if (!entry.hasRowId) {
+ if (!entry.hasRowId || !validRowIdRange(entry.firstRowId, entry.rowCount)) {
rowIdStats = null;
} else {
rowIdStats.collect(entry.firstRowId, entry.rowCount);
@@ -697,6 +706,14 @@ public final class ManifestAvroWriter implements AutoCloseable {
ExceptionUtils.firstOrSuppressed(cleanupFailure, primaryFailure);
}
}
+ if (sidecarCreated) {
+ try {
+ fileIO.deleteQuietly(ManifestSidecar.path(path));
+ } catch (Throwable cleanupFailure) {
+ primaryFailure =
+ ExceptionUtils.firstOrSuppressed(cleanupFailure, primaryFailure);
+ }
+ }
return primaryFailure;
}
@@ -711,6 +728,7 @@ public final class ManifestAvroWriter implements AutoCloseable {
outputBytes = out.getPos();
out.close();
out = null;
+ writeSidecar();
} catch (IOException | RuntimeException | Error failure) {
abortCollecting(failure, true);
throw failure;
@@ -719,6 +737,26 @@ public final class ManifestAvroWriter implements AutoCloseable {
}
}
+ private void writeSidecar() throws IOException {
+ if (!options.manifestSidecarEnabled()) {
+ return;
+ }
+ byte[] bytes =
+ ManifestSidecar.build(
+ fileIO,
+ path,
+ outputBytes,
+ Math.addExact(numAddedFiles, numDeletedFiles),
+ options.dataEvolutionEnabled(),
+ options.bucket() != -1);
+ // Publish result() only after both immutable objects have closed. No rename.
+ try (PositionOutputStream sidecarOut =
+ fileIO.newOutputStream(ManifestSidecar.path(path), false)) {
+ sidecarCreated = true;
+ sidecarOut.write(bytes);
+ }
+ }
+
private ManifestFileMeta result() {
if (!closed || outputBytes == null) {
throw new IllegalStateException(
@@ -740,10 +778,16 @@ public final class ManifestAvroWriter implements AutoCloseable {
rowIdStats == null ? null : rowIdStats.minRowId,
rowIdStats == null ? null : rowIdStats.maxRowId,
totalBucketsKnown ? totalBuckets : null,
- null);
+ sidecarCreated
+ ? Collections.singletonList(ManifestSidecar.path(path).getName())
+ : null);
}
}
+ private static boolean validRowIdRange(@Nullable Long first, long count) {
+ return first != null && first >= 0 && count > 0 && count - 1 <= Long.MAX_VALUE - first;
+ }
+
private static class RowIdStats {
private long minRowId = Long.MAX_VALUE;
diff --git a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFile.java b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFile.java
index 9781be4b90..30e0adcf1b 100644
--- a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFile.java
+++ b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFile.java
@@ -18,6 +18,7 @@
package org.apache.paimon.manifest;
+import org.apache.paimon.CoreOptions;
import org.apache.paimon.annotation.VisibleForTesting;
import org.apache.paimon.data.InternalRow;
import org.apache.paimon.format.FileFormat;
@@ -59,6 +60,7 @@ public class ManifestFile extends ObjectsFile<ManifestEntry> {
private final RowType partitionType;
private final AvroFileFormat avroFileFormat;
private final long suggestedFileSize;
+ private final CoreOptions options;
private ManifestFile(
FileIO fileIO,
@@ -69,7 +71,8 @@ public class ManifestFile extends ObjectsFile<ManifestEntry> {
String compression,
PathFactory pathFactory,
long suggestedFileSize,
- @Nullable SegmentsCache<Path> cache) {
+ @Nullable SegmentsCache<Path> cache,
+ CoreOptions options) {
super(
fileIO,
serializer,
@@ -85,6 +88,7 @@ public class ManifestFile extends ObjectsFile<ManifestEntry> {
this.partitionType = partitionType;
this.avroFileFormat = avroFileFormat;
this.suggestedFileSize = suggestedFileSize;
+ this.options = options;
}
@Override
@@ -301,7 +305,8 @@ public class ManifestFile extends ObjectsFile<ManifestEntry> {
serializer,
compression,
pathFactory,
- suggestedFileSize);
+ suggestedFileSize,
+ options);
}
/** Creates an Avro manifest writer for one explicit path. */
@@ -314,7 +319,8 @@ public class ManifestFile extends ObjectsFile<ManifestEntry> {
serializer,
compression,
singlePathFactory(manifestPath),
- Long.MAX_VALUE);
+ Long.MAX_VALUE,
+ options);
}
private PathFactory singlePathFactory(Path manifestPath) {
@@ -357,6 +363,7 @@ public class ManifestFile extends ObjectsFile<ManifestEntry> {
private final String compression;
private final FileStorePathFactory pathFactory;
private final long suggestedFileSize;
+ private final CoreOptions options;
@Nullable private final SegmentsCache<Path> cache;
public Factory(
@@ -367,7 +374,8 @@ public class ManifestFile extends ObjectsFile<ManifestEntry> {
String compression,
FileStorePathFactory pathFactory,
long suggestedFileSize,
- @Nullable SegmentsCache<Path> cache) {
+ @Nullable SegmentsCache<Path> cache,
+ CoreOptions options) {
this.fileIO = fileIO;
this.schemaManager = schemaManager;
this.partitionType = partitionType;
@@ -376,6 +384,7 @@ public class ManifestFile extends ObjectsFile<ManifestEntry> {
this.pathFactory = pathFactory;
this.suggestedFileSize = suggestedFileSize;
this.cache = cache;
+ this.options = options;
}
public boolean isCacheEnabled() {
@@ -392,7 +401,8 @@ public class ManifestFile extends ObjectsFile<ManifestEntry> {
compression,
pathFactory.manifestFileFactory(),
suggestedFileSize,
- cache);
+ cache,
+ options);
}
}
}
diff --git a/paimon-core/src/test/java/org/apache/paimon/CoreOptionsTest.java b/paimon-core/src/test/java/org/apache/paimon/CoreOptionsTest.java
index 47aa94acd9..e83ca0d4ac 100644
--- a/paimon-core/src/test/java/org/apache/paimon/CoreOptionsTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/CoreOptionsTest.java
@@ -29,6 +29,24 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy;
/** Tests for {@link org.apache.paimon.CoreOptions}. */
public class CoreOptionsTest {
+ @Test
+ void testManifestSidecarDefaultsToManifestSort() {
+ assertThat(CoreOptions.MANIFEST_SIDECAR_ENABLED.defaultValue()).isNull();
+ for (Boolean sort : new Boolean[] {null, false, true}) {
+ for (Boolean configured : new Boolean[] {null, false, true}) {
+ Options options = new Options();
+ if (sort != null) {
+ options.set(CoreOptions.MANIFEST_SORT_ENABLED, sort);
+ }
+ if (configured != null) {
+ options.set(CoreOptions.MANIFEST_SIDECAR_ENABLED, configured);
+ }
+ assertThat(new CoreOptions(options).manifestSidecarEnabled())
+ .isEqualTo(configured == null ? Boolean.TRUE.equals(sort) : configured);
+ }
+ }
+ }
+
@Test
public void testDefaultStartupMode() {
Options conf = new Options();
diff --git a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaTest.java b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaTest.java
index ad3641b121..16f33f4cee 100644
--- a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaTest.java
@@ -2744,7 +2744,8 @@ public class ManifestFileMetaTest extends ManifestFileMetaTestBase {
false,
null),
Long.MAX_VALUE,
- null)
+ null,
+ new CoreOptions(new Options()))
.create();
List<ManifestFileMeta> input = new ArrayList<>();
@@ -3204,7 +3205,8 @@ public class ManifestFileMetaTest extends ManifestFileMetaTestBase {
false,
null),
Long.MAX_VALUE,
- null)
+ null,
+ new CoreOptions(new Options()))
.create();
}
diff --git a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaTestBase.java b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaTestBase.java
index 5dd1e4787b..a4516015c2 100644
--- a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaTestBase.java
+++ b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaTestBase.java
@@ -166,7 +166,8 @@ public abstract class ManifestFileMetaTestBase {
false,
null),
Long.MAX_VALUE,
- null)
+ null,
+ new CoreOptions(new Options()))
.create();
}
diff --git a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileTest.java b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileTest.java
index 50097b041a..0362871ef8 100644
--- a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileTest.java
@@ -1442,7 +1442,8 @@ public class ManifestFileTest {
"zstd",
pathFactory,
suggestedFileSize,
- cache)
+ cache,
+ new CoreOptions(new Options()))
.create();
}
diff --git a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestIndexTestUtils.java b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestIndexTestUtils.java
index 32a94c9a5b..e9be1afc3f 100644
--- a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestIndexTestUtils.java
+++ b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestIndexTestUtils.java
@@ -35,6 +35,7 @@ import java.util.List;
/** Synthetic index references for manifest serialization and lifecycle tests. */
public final class ManifestIndexTestUtils {
+
private ManifestIndexTestUtils() {}
public static ManifestFileMeta withIndexFileName(ManifestFileMeta meta, String indexFileName) {
diff --git a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestSidecarTest.java b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestSidecarTest.java
index b4a7b3e29b..1110c70158 100644
--- a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestSidecarTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestSidecarTest.java
@@ -18,6 +18,7 @@
package org.apache.paimon.manifest;
+import org.apache.paimon.CoreOptions;
import org.apache.paimon.data.BinaryRow;
import org.apache.paimon.data.BinaryRowWriter;
import org.apache.paimon.data.BinaryString;
@@ -276,7 +277,8 @@ class ManifestSidecarTest {
new ManifestEntrySerializer(),
"zstd",
paths,
- Long.MAX_VALUE);
+ Long.MAX_VALUE,
+ new CoreOptions(new Options()));
}
@Test
diff --git a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestSidecarWriteTest.java b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestSidecarWriteTest.java
new file mode 100644
index 0000000000..cea952943f
--- /dev/null
+++ b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestSidecarWriteTest.java
@@ -0,0 +1,347 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.manifest;
+
+import org.apache.paimon.CoreOptions;
+import org.apache.paimon.data.BinaryRow;
+import org.apache.paimon.format.FileFormat;
+import org.apache.paimon.fs.FileIO;
+import org.apache.paimon.fs.Path;
+import org.apache.paimon.fs.PositionOutputStream;
+import org.apache.paimon.fs.PositionOutputStreamWrapper;
+import org.apache.paimon.fs.SeekableInputStream;
+import org.apache.paimon.fs.local.LocalFileIO;
+import org.apache.paimon.options.Options;
+import org.apache.paimon.schema.FileSystemSchemaManager;
+import org.apache.paimon.types.RowType;
+import org.apache.paimon.utils.FileStorePathFactory;
+import org.apache.paimon.utils.Range;
+import org.apache.paimon.utils.RowRangeIndex;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import javax.annotation.Nullable;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.BiPredicate;
+import java.util.stream.Stream;
+
+import static org.apache.paimon.TestKeyValueGenerator.DEFAULT_PART_TYPE;
+import static org.apache.paimon.utils.VarLengthIntUtils.decodeInt;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+/** Tests automatic generation and ownership of manifest sidecars. */
+class ManifestSidecarWriteTest {
+
+ @TempDir java.nio.file.Path temp;
+ private final ManifestTestDataGenerator gen = ManifestTestDataGenerator.builder().build();
+
+ @Test
+ void sidecarOptionControlsWriteIO() {
+ for (Boolean sort : new Boolean[] {null, false, true}) {
+ for (Boolean configured : new Boolean[] {null, false, true}) {
+ Options options = new Options();
+ if (sort != null) {
+ options.set(CoreOptions.MANIFEST_SORT_ENABLED, sort);
+ }
+ if (configured != null) {
+ options.set(CoreOptions.MANIFEST_SIDECAR_ENABLED, configured);
+ }
+ AtomicInteger reads = new AtomicInteger();
+ FileIO io =
+ new LocalFileIO() {
+
+ @Override
+ public SeekableInputStream newInputStream(Path path)
+ throws IOException {
+ reads.incrementAndGet();
+ return super.newInputStream(path);
+ }
+ };
+ Path root = root(sort + "-" + configured);
+ ManifestFile manifests =
+ manifests(root, io, DEFAULT_PART_TYPE, Long.MAX_VALUE, options);
+ ManifestFileMeta meta =
+ manifests.write(Collections.singletonList(gen.next())).get(0);
+ boolean enabled = configured == null ? Boolean.TRUE.equals(sort) : configured;
+ assertThat(reads.get()).isEqualTo(enabled ? 1 : 0);
+ assertThat(ManifestSidecar.fileName(meta) != null).isEqualTo(enabled);
+ }
+ }
+ }
+
+ @Test
+ void rollingAndRawRewritesGenerateTheirOwnSidecars() throws Exception {
+ Options options = enabledOptions();
+ options.set(CoreOptions.MANIFEST_SORT_ENABLED, true);
+ FileIO io = LocalFileIO.create();
+ Path root = root("rolling");
+ ManifestFile manifests = manifests(root, io, DEFAULT_PART_TYPE, 1, options);
+ List<ManifestFileMeta> metas = manifests.write(entries(2200));
+ assertThat(metas.size()).isGreaterThan(1);
+ for (ManifestFileMeta meta : metas) {
+ assertThat(ManifestSidecar.fileName(meta))
+ .isEqualTo(meta.fileName() + ManifestSidecar.SUFFIX);
+ List<ManifestEntry> actual = manifests.read(meta.fileName());
+ for (ManifestEntry entry :
+ Arrays.asList(actual.get(0), actual.get(actual.size() - 1))) {
+ long row = entry.file().firstRowId();
+ assertThat(select(root, io, meta, DEFAULT_PART_TYPE, point(row), null).blocks())
+ .isNotEmpty();
+ }
+ long gap = actual.get(0).file().firstRowId() + actual.get(0).file().rowCount();
+ assertThat(select(root, io, meta, DEFAULT_PART_TYPE, point(gap), null).blocks())
+ .isEmpty();
+ }
+
+ ManifestFileMeta source = metas.get(0);
+ Path rewrittenPath = new Path(new Path(root, "manifest"), "explicit-rewrite");
+ ManifestAvroWriter writer = manifests.createAvroWriter(rewrittenPath);
+ try (ManifestAvroReader reader =
+ manifests.scanAvroBlocks(source.fileName(), source.fileSize())) {
+ writer.writeEncodedManifest(reader, source);
+ }
+ assertThatThrownBy(writer::result).isInstanceOf(IllegalStateException.class);
+ writer.close();
+ ManifestFileMeta rewritten = writer.result().get(0);
+ assertThat(ManifestSidecar.fileName(rewritten))
+ .isEqualTo("explicit-rewrite" + ManifestSidecar.SUFFIX);
+ assertThat(manifests.read(rewritten.fileName()))
+ .isEqualTo(manifests.read(source.fileName()));
+ long outside = metas.get(metas.size() - 1).maxRowId();
+ assertThat(select(root, io, rewritten, DEFAULT_PART_TYPE, point(outside), null).blocks())
+ .isEmpty();
+ writer.abort();
+ assertThat(io.exists(rewrittenPath)).isFalse();
+ assertThat(io.exists(ManifestSidecar.path(rewrittenPath))).isFalse();
+ assertThat(io.exists(manifestPath(root, source))).isTrue();
+ for (ManifestFileMeta meta : metas) {
+ manifests.delete(meta);
+ assertThat(io.exists(ManifestSidecar.path(manifestPath(root, meta)))).isFalse();
+ }
+ }
+
+ @Test
+ void payloadsFollowTableMetadata() throws Exception {
+ for (boolean partitioned : new boolean[] {false, true}) {
+ for (boolean evolution : new boolean[] {false, true}) {
+ for (int bucket : new int[] {-2, -1, 4}) {
+ Options options = enabledOptions();
+ options.set(CoreOptions.DATA_EVOLUTION_ENABLED, evolution);
+ options.set(CoreOptions.BUCKET, bucket);
+ RowType partitionType = partitioned ? DEFAULT_PART_TYPE : RowType.of();
+ Path root = root(partitioned + "-" + evolution + "-" + bucket);
+ FileIO io = LocalFileIO.create();
+ ManifestFile manifests =
+ manifests(root, io, partitionType, Long.MAX_VALUE, options);
+ ManifestEntry source = gen.next();
+ ManifestEntry entry =
+ ManifestEntry.create(
+ FileKind.ADD,
+ partitioned ? source.partition() : BinaryRow.EMPTY_ROW,
+ 1,
+ 4,
+ source.file().newFirstRowId(100L));
+ ManifestFileMeta meta =
+ manifests.write(Collections.singletonList(entry)).get(0);
+ byte[] bytes =
+ Files.readAllBytes(
+ java.nio.file.Paths.get(
+ ManifestSidecar.path(manifestPath(root, meta))
+ .toString()));
+ ByteBuffer in = ByteBuffer.wrap(bytes);
+ in.getInt();
+ decodeInt(in);
+ int headerLength = decodeInt(in);
+ in.position(in.position() + headerLength);
+ assertThat(decodeInt(in)).isEqualTo(1);
+ assertThat(select(root, io, meta, partitionType, point(99), null).blocks())
+ .hasSize(evolution ? 0 : 1);
+ assertThat(
+ select(root, io, meta, partitionType, null, (b, t) -> b == 99)
+ .blocks())
+ .hasSize(bucket == -1 ? 1 : 0);
+ assertThat(manifests.read(meta.fileName())).containsExactly(entry);
+ }
+ }
+ }
+ }
+
+ @Test
+ void invalidRowIdRangeKeepsCoverageUnavailable() throws Exception {
+ FileIO io = LocalFileIO.create();
+ Path root = root("unknown");
+ ManifestFile manifests =
+ manifests(root, io, DEFAULT_PART_TYPE, Long.MAX_VALUE, enabledOptions());
+ ManifestEntry source = gen.next();
+ ManifestEntry entry =
+ ManifestEntry.create(
+ FileKind.ADD, source.partition(), 1, 4, source.file().newFirstRowId(-1L));
+ ManifestFileMeta meta = manifests.write(Collections.singletonList(entry)).get(0);
+ assertThat(meta.minRowId()).isNull();
+ assertThat(meta.maxRowId()).isNull();
+ assertThat(select(root, io, meta, DEFAULT_PART_TYPE, point(123), null).blocks()).hasSize(1);
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"open", "write", "close"})
+ void sidecarFailureCleansAllRollingOutputs(String phase) throws Exception {
+ AtomicInteger sidecars = new AtomicInteger();
+ FileIO io =
+ new LocalFileIO() {
+
+ @Override
+ public PositionOutputStream newOutputStream(Path path, boolean overwrite)
+ throws IOException {
+ if (!path.getName().endsWith(ManifestSidecar.SUFFIX)
+ || sidecars.incrementAndGet() != 2) {
+ return super.newOutputStream(path, overwrite);
+ }
+ if (phase.equals("open")) {
+ throw new IOException("sidecar " + phase + " failed");
+ }
+ return new PositionOutputStreamWrapper(
+ super.newOutputStream(path, overwrite)) {
+
+ @Override
+ public void write(byte[] bytes) throws IOException {
+ if (phase.equals("write")) {
+ throw new IOException("sidecar " + phase + " failed");
+ }
+ super.write(bytes);
+ }
+
+ @Override
+ public void close() throws IOException {
+ super.close();
+ if (phase.equals("close")) {
+ throw new IOException("sidecar " + phase + " failed");
+ }
+ }
+ };
+ }
+ };
+ Path root = root("failure-" + phase);
+ ManifestFile manifests = manifests(root, io, DEFAULT_PART_TYPE, 1, enabledOptions());
+ assertThatThrownBy(() -> manifests.write(entries(2200)))
+ .hasRootCauseMessage("sidecar " + phase + " failed");
+ try (Stream<java.nio.file.Path> files =
+ Files.list(temp.resolve("failure-" + phase).resolve("manifest"))) {
+ assertThat(files).isEmpty();
+ }
+ }
+
+ private Options enabledOptions() {
+ Options options = new Options();
+ options.set(CoreOptions.MANIFEST_SIDECAR_ENABLED, true);
+ options.set(CoreOptions.DATA_EVOLUTION_ENABLED, true);
+ return options;
+ }
+
+ private List<ManifestEntry> entries(int count) {
+ List<ManifestEntry> result = new ArrayList<>();
+ for (int i = 0; i < count; i++) {
+ ManifestEntry entry = gen.next();
+ result.add(
+ ManifestEntry.create(
+ i % 2 == 0 ? FileKind.ADD : FileKind.DELETE,
+ entry.partition(),
+ entry.bucket(),
+ entry.totalBuckets(),
+ entry.file().newFirstRowId(i * 100000000L)));
+ }
+ return result;
+ }
+
+ private Path root(String name) {
+ return new Path(temp.resolve(name).toString());
+ }
+
+ private static RowRangeIndex point(long rowId) {
+ return RowRangeIndex.create(Collections.singletonList(new Range(rowId, rowId)));
+ }
+
+ private static Path manifestPath(Path root, ManifestFileMeta meta) {
+ return new Path(new Path(root, "manifest"), meta.fileName());
+ }
+
+ private ManifestSidecar.Selection select(
+ Path root,
+ FileIO io,
+ ManifestFileMeta meta,
+ RowType partitionType,
+ @Nullable RowRangeIndex rows,
+ @Nullable BiPredicate<Integer, Integer> buckets) {
+ ManifestSidecar.Selection result =
+ ManifestSidecar.read(
+ io,
+ manifestPath(root, meta),
+ meta,
+ rows,
+ null,
+ partitionType,
+ buckets,
+ null);
+ assertThat(result).isNotNull();
+ return result;
+ }
+
+ private ManifestFile manifests(
+ Path root, FileIO io, RowType partitionType, long targetSize, Options options) {
+ FileStorePathFactory paths =
+ new FileStorePathFactory(
+ root,
+ partitionType,
+ "default",
+ CoreOptions.FILE_FORMAT.defaultValue(),
+ CoreOptions.DATA_FILE_PREFIX.defaultValue(),
+ CoreOptions.CHANGELOG_FILE_PREFIX.defaultValue(),
+ CoreOptions.PARTITION_GENERATE_LEGACY_NAME.defaultValue(),
+ CoreOptions.FILE_SUFFIX_INCLUDE_COMPRESSION.defaultValue(),
+ CoreOptions.FILE_COMPRESSION.defaultValue(),
+ null,
+ null,
+ CoreOptions.ExternalPathStrategy.NONE,
+ null,
+ false,
+ null);
+ return new ManifestFile.Factory(
+ io,
+ new FileSystemSchemaManager(io, root),
+ partitionType,
+ FileFormat.fromIdentifier("avro", new Options()),
+ "zstd",
+ paths,
+ targetSize,
+ null,
+ new CoreOptions(options))
+ .create();
+ }
+}
diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/ManifestRewriteCleanupTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/ManifestRewriteCleanupTest.java
index 8247d23262..20a1b26bee 100644
--- a/paimon-core/src/test/java/org/apache/paimon/operation/ManifestRewriteCleanupTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/operation/ManifestRewriteCleanupTest.java
@@ -668,7 +668,8 @@ class ManifestRewriteCleanupTest extends ManifestFileMetaTestBase {
false,
null),
suggestedFileSize,
- null)
+ null,
+ new CoreOptions(new Options()))
.create();
}
(paimon) 19/20: [python] Honor snapshot.ignore-empty-commit for empty appends (#9878)
Posted by ho...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
XiaoHongbo-Hope pushed a commit to branch release-2.1
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit 2e8803c4981c7bbeea0bb9f9e573fab7e39e3786
Author: Yann Byron <bi...@gmail.com>
AuthorDate: Thu Sep 17 11:39:01 2026 +0800
[python] Honor snapshot.ignore-empty-commit for empty appends (#9878)
---
.../pypaimon/common/options/core_options.py | 13 ++++++
paimon-python/pypaimon/tests/table_commit_test.py | 47 ++++++++++++++++++++++
paimon-python/pypaimon/write/file_store_commit.py | 16 +++++---
paimon-python/pypaimon/write/table_commit.py | 3 +-
4 files changed, 72 insertions(+), 7 deletions(-)
diff --git a/paimon-python/pypaimon/common/options/core_options.py b/paimon-python/pypaimon/common/options/core_options.py
index 2fe0d039fd..0ac4610864 100644
--- a/paimon-python/pypaimon/common/options/core_options.py
+++ b/paimon-python/pypaimon/common/options/core_options.py
@@ -727,6 +727,16 @@ class CoreOptions:
.with_description("The prefix for commit user.")
)
+ SNAPSHOT_IGNORE_EMPTY_COMMIT: ConfigOption[bool] = (
+ ConfigOptions.key("snapshot.ignore-empty-commit")
+ .boolean_type()
+ .no_default_value()
+ .with_description(
+ "Whether to skip append commits without changes. "
+ "PyPaimon defaults to true; false allows tagging an empty table."
+ )
+ )
+
COMMIT_MAX_RETRIES: ConfigOption[int] = (
ConfigOptions.key("commit.max-retries")
.int_type()
@@ -1609,6 +1619,9 @@ class CoreOptions:
weights.append(parsed)
return weights
+ def snapshot_ignore_empty_commit(self) -> bool:
+ return self.options.get(CoreOptions.SNAPSHOT_IGNORE_EMPTY_COMMIT, True)
+
def commit_max_retries(self) -> int:
return self.options.get(CoreOptions.COMMIT_MAX_RETRIES)
diff --git a/paimon-python/pypaimon/tests/table_commit_test.py b/paimon-python/pypaimon/tests/table_commit_test.py
index d607e73921..2c7079bcc7 100644
--- a/paimon-python/pypaimon/tests/table_commit_test.py
+++ b/paimon-python/pypaimon/tests/table_commit_test.py
@@ -16,6 +16,7 @@
# under the License.
import unittest
+from tempfile import TemporaryDirectory
from unittest.mock import Mock
from parameterized import parameterized
@@ -27,6 +28,52 @@ from pypaimon.write.table_commit import BatchTableCommit, StreamTableCommit
class TestTableCommit(unittest.TestCase):
+ def test_empty_append_snapshot_is_opt_in_and_can_be_tagged(self):
+ import pyarrow as pa
+ import pypaimon.multimodal as pmm
+
+ with TemporaryDirectory(prefix="paimon-empty-commit-") as warehouse:
+ connection = pmm.connect(options={"warehouse": warehouse})
+ schema = pa.schema([pa.field("feature", pa.string(), False)])
+ table = connection.create_table("stat", schema=schema)
+ empty = pa.Table.from_pylist([], schema=schema)
+ table.add(empty)
+ snapshots = table.raw_table.snapshot_manager()
+ self.assertIsNone(snapshots.get_latest_snapshot())
+
+ def commit_empty():
+ writable = table.raw_table.copy({
+ "snapshot.ignore-empty-commit": "false",
+ })
+ commit = writable.new_batch_write_builder().new_commit()
+ try:
+ commit.commit([], snapshot_properties={"source": "empty-stat"})
+ finally:
+ commit.close()
+
+ commit_empty()
+ snapshot = snapshots.get_latest_snapshot()
+ self.assertIsNotNone(snapshot)
+ self.assertEqual((1, 0, 0), (
+ snapshot.id, snapshot.total_record_count,
+ snapshot.delta_record_count))
+ self.assertEqual({"source": "empty-stat"}, snapshot.properties)
+ table.raw_table.create_tag("empty")
+ tagged = table.scan(tag_name="empty").to_arrow()
+ self.assertEqual(0, tagged.num_rows)
+ self.assertEqual(schema, tagged.schema)
+
+ table.add([{"feature": "state_imu_body"}])
+ table.add(empty)
+ self.assertEqual(2, snapshots.get_latest_snapshot().id)
+ commit_empty()
+ snapshot = snapshots.get_latest_snapshot()
+ self.assertEqual((3, 1, 0), (
+ snapshot.id, snapshot.total_record_count,
+ snapshot.delta_record_count))
+ self.assertEqual([{"feature": "state_imu_body"}], table.scan().to_list())
+ self.assertEqual([], table.scan(tag_name="empty").to_list())
+
def _create_commit(self, cls, overwrite_partition=None):
commit = cls.__new__(cls)
commit.table = Mock()
diff --git a/paimon-python/pypaimon/write/file_store_commit.py b/paimon-python/pypaimon/write/file_store_commit.py
index 10ac3e2b53..8c7cab1175 100644
--- a/paimon-python/pypaimon/write/file_store_commit.py
+++ b/paimon-python/pypaimon/write/file_store_commit.py
@@ -262,7 +262,8 @@ class FileStoreCommit:
commit_identifier: int,
snapshot_properties: Optional[Dict[str, str]] = None):
"""Commit the given commit messages in normal append mode."""
- if not commit_messages:
+ ignore_empty_commit = self.table.options.snapshot_ignore_empty_commit()
+ if not commit_messages and ignore_empty_commit:
return
# Extract the minimum check_from_snapshot from commit messages
@@ -345,7 +346,8 @@ class FileStoreCommit:
index_deletes=index_deletes,
index_adds=index_adds,
hash_index_base_snapshot=hash_index_base_snapshot,
- snapshot_properties=snapshot_properties)
+ snapshot_properties=snapshot_properties,
+ allow_empty_commit=not ignore_empty_commit)
def overwrite(
self,
@@ -505,7 +507,8 @@ class FileStoreCommit:
detect_conflicts=False, allow_rollback=False, index_deletes=None,
index_adds=None, changelog_entries=None,
hash_index_base_snapshot=None,
- snapshot_properties: Optional[Dict[str, str]] = None):
+ snapshot_properties: Optional[Dict[str, str]] = None,
+ allow_empty_commit=False):
retry_count = 0
retry_result = None
@@ -528,9 +531,10 @@ class FileStoreCommit:
else commit_entries_plan(latest_snapshot)
)
- # No entries to commit (e.g. drop_partitions with no matching
- # data): skip an empty snapshot.
- if not commit_entries and not index_deletes and not index_adds:
+ # Append can explicitly publish an empty snapshot for tagging.
+ # No-op overwrite/drop operations retain their existing behavior.
+ if (not allow_empty_commit and not commit_entries
+ and not index_deletes and not index_adds):
break
result = self._try_commit_once(
diff --git a/paimon-python/pypaimon/write/table_commit.py b/paimon-python/pypaimon/write/table_commit.py
index f6eb6748cd..35ee83f2fc 100644
--- a/paimon-python/pypaimon/write/table_commit.py
+++ b/paimon-python/pypaimon/write/table_commit.py
@@ -88,7 +88,8 @@ class TableCommit:
overwrite_partition=self.overwrite_partition,
**commit_kwargs)
else:
- if not non_empty_messages:
+ if (not non_empty_messages
+ and self.table.options.snapshot_ignore_empty_commit()):
return
logger.info(
"Committing table %s, %d non-empty messages",
(paimon) 20/20: [python] Extend ContiguousWindowDataset for frame-level training (#9875)
Posted by ho...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
XiaoHongbo-Hope pushed a commit to branch release-2.1
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit 1fc0f17ad2dbb94c17372a348cfbc4a0a9ec133b
Author: Yann Byron <bi...@gmail.com>
AuthorDate: Thu Sep 17 11:39:54 2026 +0800
[python] Extend ContiguousWindowDataset for frame-level training (#9875)
---
docs/docs/pypaimon/multimodal-reading.md | 119 +++++++--
docs/docs/pypaimon/pytorch.md | 55 ++--
paimon-python/pypaimon/benchmark/act/paimon.py | 11 +-
.../pypaimon/multimodal/lerobot/dataset.py | 26 +-
paimon-python/pypaimon/multimodal/query.py | 51 ++--
.../pypaimon/multimodal/window_dataset.py | 292 ++++++++++++++-------
.../pypaimon/multimodal/window_transforms.py | 88 +++++++
paimon-python/pypaimon/tests/act_runner_test.py | 20 +-
.../tests/contiguous_window_dataset_test.py | 236 +++++++++++++++++
.../pypaimon/tests/window_transforms_test.py | 111 ++++++++
paimon-python/setup.py | 1 +
11 files changed, 828 insertions(+), 182 deletions(-)
diff --git a/docs/docs/pypaimon/multimodal-reading.md b/docs/docs/pypaimon/multimodal-reading.md
index 234dd4f6e2..d3df05a9bd 100644
--- a/docs/docs/pypaimon/multimodal-reading.md
+++ b/docs/docs/pypaimon/multimodal-reading.md
@@ -182,35 +182,37 @@ Notes:
### Contiguous windows for PyTorch
-Install the `torch` extra, then use `to_contiguous_window_dataset` to expose
-map-style windows without loading the selected rows or BLOB payloads into Python
-memory up front. The Dataset builds a compact index from the group column, order
-column, and Paimon row IDs. Each `__getitem__` call fetches only that window from
-the snapshot recorded in `dataset.snapshot_id`.
+Use `to_contiguous_window_dataset` to build map-style training samples from a
+single frame table. Each row represents one time step; task text, labels, and
+other context must already be materialized in that table. The Dataset does not
+read companion tables. It builds a compact index of group/order values and
+Paimon row IDs, then fetches requested columns and BLOB payloads on demand from
+the snapshot recorded in `dataset.snapshot_id`. Row tracking must be enabled.
```shell
pip install 'pypaimon[torch]'
```
```python
-import torch
-
+from functools import partial
-def float32_window(values):
- return torch.tensor(values, dtype=torch.float32)
+import torch
+from pypaimon.multimodal.window_transforms import images_to_tensor, to_tensor
+float32_window = partial(to_tensor, dtype=torch.float32)
windows = (
- frames.scan()
+ frames.scan(tag_name="train-v1")
.where("split = 'train'")
.to_contiguous_window_dataset(
- window_size=16,
- columns=["state", "action"],
+ columns=["state", "image", "action"],
group_key="episode_index",
order_key="frame_index",
- tail="pad",
+ frame_offsets={"state": [-2, -1, 0], "action": list(range(16))},
+ boundary="pad",
column_transforms={
"state": float32_window,
+ "image": images_to_tensor,
"action": float32_window,
},
)
@@ -218,25 +220,86 @@ windows = (
sample = windows[0]
assert sample["action"].shape == (16, action_size)
-assert sample["is_pad"].shape == (16,)
+assert sample["action_is_pad"].shape == (16,)
+assert sample["state"].shape == (3, state_size)
+assert sample["image"].shape == (1, channels, height, width)
```
-The group and order keys in a sample identify the window anchor. Every projected
-column contains the whole window. With `tail="drop"`, only full windows are
-exposed. With `tail="pad"`, every real row is an anchor; missing suffix values
-repeat the last real value by default and `is_pad` is `True` exactly at those
-positions. With `tail="error"`, construction fails if any scheduled anchor is
-incomplete. Use `pad_values` to override the repeated value for individual
-columns. Anchors advance by `stride`, which defaults to one row.
+The group and order keys identify the anchor. `frame_offsets` defines each
+column's relative frame positions, including history, future, and sparse
+sampling. Offset order and duplicates are preserved. A selected column omitted
+from the mapping uses `[0]`, so the image above is read only at the anchor.
+Without any window definition, every selected column uses `[0]`. Each column
+has its own Boolean `<column>_is_pad` tensor, where `True` marks an out-of-group
+position. These generated names must not collide with other output fields.
+
+Anchors start at each group's first row and advance by `stride` (default one).
+The stride controls anchor spacing, not the spacing inside a window:
+
+| `boundary` | Behavior at either end of a group |
+|------------|----------------------------------|
+| `drop` (default) | Keep only anchors for which all requested offsets are valid. |
+| `pad` | Keep scheduled anchors and repeat the nearest endpoint for missing positions. |
+| `error` | Fail construction if any scheduled anchor has an incomplete window. |
+
+`pad_values={"action": [0.0, 0.0]}` overrides endpoint repetition for that
+column. Supply values in the same raw representation as a table cell. Padding
+happens before column transforms, so normalization also affects constant padding.
+Use the mask when excluding padded actions from a loss.
+
+As an alternative, specify seconds with `delta_timestamps`, an explicit positive
+`fps`, and optional `tolerance_s` (default `1e-4`). For example,
+`delta_timestamps={"state": [-0.1, 0.0]}, fps=10` selects offsets `[-1, 0]`.
+Both `fps` and `tolerance_s` are only accepted with `delta_timestamps`.
+Offsets must align to the regular frame grid within the tolerance; this API
+does not search actual timestamps or interpolate missing frames. Do not combine
+`delta_timestamps`, `frame_offsets`, and `window_size`.
`column_transforms` receive one padded Python list per projected column. This is
-where applications define tensor dtype and shape or decode BLOB bytes. The
-optional `adapter` receives the resulting sample mapping and can rename or
-combine fields for a model-specific batch contract. The core Dataset does not
-know model field names, image formats, or normalization rules. Top-level
-functions and callable classes are recommended for transforms and adapters so
-the Dataset remains picklable by multi-worker `torch.utils.data.DataLoader`
-instances.
+where applications define tensor dtype and shape or decode BLOB bytes. Even
+singleton windows retain the time dimension. `images_to_tensor` returns TCHW
+images with EXIF orientation applied. Eight-bit pixels become float32 in
+`[0, 1]`; use `partial(images_to_tensor, return_uint8=True)` to keep uint8.
+Higher-bit-depth images become float32 in their original units. All images in
+a column window must have the same shape.
+
+The optional `adapter` receives the transformed sample and can normalize values,
+rename/combine fields, remove singleton time dimensions, or coordinate random
+augmentation across cameras and time. Statistics must be supplied explicitly;
+the Dataset never computes or discovers normalization statistics. For example:
+
+```python
+class NormalizeAction:
+ def __init__(self, mean, std):
+ self.mean = torch.as_tensor(mean, dtype=torch.float32)
+ self.std = torch.as_tensor(std, dtype=torch.float32)
+ if not torch.isfinite(self.std).all() or not (self.std > 0).all():
+ raise ValueError("Action standard deviations must be finite and positive.")
+
+ def __call__(self, sample):
+ sample["action"] = (sample["action"] - self.mean) / self.std
+ return sample
+```
+
+Pass `adapter=NormalizeAction(train_action_mean, train_action_std)` when
+constructing the Dataset. Use training-set statistics, with the same values for
+validation. Keep callbacks picklable (top-level functions, `partial`, or callable
+classes) for multi-worker DataLoader use. Physical reads are coalesced in
+`__getitems__`, while samples and transforms retain their requested order and
+duplicates. Negative indices and slices follow Python sequence semantics.
+
+Rows are sorted by group and order. Order values must be non-null integers
+increasing by exactly one within each group. Duplicate steps and internal gaps,
+including gaps introduced by a scan filter, fail validation. Windows never
+cross groups. A snapshot pin fixes the data being read; use retained tags for
+long-running training. Sampler progress and augmentation RNG state remain the
+training application's responsibility.
+
+Existing `window_size=16` calls remain supported, with `anchor_columns` selecting
+singleton fields and `tail="drop"/"pad"/"error"` controlling the old forward
+window. These calls preserve their single `is_pad` mask and previous shapes.
+`anchor_columns` and `tail` require `window_size`; do not combine `tail` and
+`boundary`. New applications should use field offsets and per-column masks.
Columns configured by `video-frame-field` are rejected: a window read would drop
the `frame_index` and other metadata carried by their `VideoFrameDescriptor`
diff --git a/docs/docs/pypaimon/pytorch.md b/docs/docs/pypaimon/pytorch.md
index 6f973ade3f..26770bdbba 100644
--- a/docs/docs/pypaimon/pytorch.md
+++ b/docs/docs/pypaimon/pytorch.md
@@ -159,42 +159,55 @@ physical video ranges and cache decoder sessions per worker. See
for the write path and a complete decoder example.
## Contiguous Windows
-Use a map-style `ContiguousWindowDataset` when training samples are fixed-size
-windows which must not cross a sequence boundary. The dataset builds an index
+Use a map-style `ContiguousWindowDataset` to construct training samples from
+one materialized frame table. Each field can have its own history or future
+window, and windows never cross a sequence boundary. The dataset builds an index
from only the group column, order column, and Paimon row IDs. Projected values,
including BLOB payloads, are read from the pinned snapshot when a sample is
requested; they are not retained in the index.
```python
+from functools import partial
+
+import torch
from torch.utils.data import DataLoader
+from pypaimon.multimodal.window_transforms import images_to_tensor, to_tensor
dataset = (
frames.scan()
.to_contiguous_window_dataset(
- window_size=16,
- columns=["state", "image"],
- anchor_columns=["image"],
+ columns=["state", "image", "action"],
+ frame_offsets={"state": [-2, -1, 0], "action": list(range(16))},
group_key="episode_index",
order_key="frame_index",
- tail="pad",
+ boundary="pad",
+ column_transforms={
+ "state": partial(to_tensor, dtype=torch.float32),
+ "action": partial(to_tensor, dtype=torch.float32),
+ "image": images_to_tensor,
+ },
)
)
loader = DataLoader(dataset, batch_size=32, num_workers=4, shuffle=True)
```
-Each item contains the group and order keys, one list for each requested
-column, and a boolean `is_pad` tensor where `True` marks padding. Padding
-repeats the final real value by default; `pad_values` can override individual
-columns. Columns named in `anchor_columns` contain only the first row's value,
-which is useful when an observation applies to a full action window. Use
-`column_transforms` to convert column lists to tensors and
-`adapter` to produce a model-specific sample mapping. Keep these callbacks
-picklable when using multiple DataLoader workers.
-
-Scheduled anchors start at row zero and advance by `stride` (default `1`).
-`tail="drop"` omits incomplete windows, `tail="pad"` includes and pads them,
-and `tail="error"` rejects a sequence with any scheduled incomplete window.
+Each raw item contains scalar group/order keys, a list for every selected column,
+and a Boolean `<column>_is_pad` tensor where `True` marks padding. Unspecified
+field offsets default to `[0]`. The example produces state/action tensors with
+time lengths 3/16 and an image tensor of shape `(1, C, H, W)`; DataLoader adds the
+batch dimension. `pad_values` can replace endpoint repetition for individual
+columns, before any transforms. `adapter` can normalize or rename fields and
+produce a model-specific mapping. Keep callbacks picklable for multiple workers.
+
+Scheduled anchors start at each group's first row and advance by `stride`
+(default `1`). `boundary="drop"` (default) omits anchors incomplete for any
+field, `boundary="pad"` pads either end, and `boundary="error"` rejects any
+scheduled incomplete window. `delta_timestamps` with an explicit `fps` is an
+alternative to integer offsets, with frame-grid alignment checked against
+`tolerance_s`. Existing `window_size`/`anchor_columns`/`tail` calls retain their
+single `is_pad` output. See [multimodal reading](multimodal-reading#contiguous-windows-for-pytorch)
+for the full sample and conversion contract.
Rows are sorted by `order_key` inside each `group_key` value. Order values must
be integers which increase by exactly one; duplicates and missing steps are
rejected, and windows never cross groups. The resolved Paimon
@@ -203,6 +216,12 @@ change its index or sample contents. A dataset pinned through `tag_name` fails
its reads if the tag is moved to another snapshot, rather than mixing rows from
the two snapshots.
+Use normal PyTorch samplers for shuffling and distributed training. For DDP,
+pass `DistributedSampler(dataset)` to DataLoader, omit `shuffle=True`, and call
+the sampler's `set_epoch(epoch)` each epoch. Its padding or drop behavior still
+applies when the sample count is not divisible by the number of ranks. Dataset
+version pinning does not save sampler progress or random augmentation state.
+
Columns configured by `video-frame-field` are rejected: a window read would drop
the `frame_index` and other metadata carried by their `VideoFrameDescriptor`
values. Read those columns with `to_torch()` instead.
diff --git a/paimon-python/pypaimon/benchmark/act/paimon.py b/paimon-python/pypaimon/benchmark/act/paimon.py
index 18b50664b6..161edadbbc 100644
--- a/paimon-python/pypaimon/benchmark/act/paimon.py
+++ b/paimon-python/pypaimon/benchmark/act/paimon.py
@@ -52,7 +52,7 @@ class PaimonACTAdapter:
The persisted ``frame_index`` is forwarded as the shared ACT sample
position.
State and image columns are singleton lists; action retains the full
- horizon and ``is_pad`` is forwarded unchanged.
+ horizon and ``action_is_pad`` becomes the ACT ``is_pad`` mask.
"""
qpos = np.concatenate([
np.asarray(sample[name][0], dtype=np.float32)
@@ -80,7 +80,7 @@ class PaimonACTAdapter:
"qpos": torch.from_numpy(np.ascontiguousarray(qpos)),
"action": torch.from_numpy(np.ascontiguousarray(action)),
"images": torch.from_numpy(np.ascontiguousarray(images)),
- "is_pad": sample["is_pad"],
+ "is_pad": sample["action_is_pad"],
}
@@ -114,13 +114,14 @@ def create_datasets(
frames.scan(snapshot_id=snapshot_id).where(
"episode_id = '%s'" % episode_id.replace("'", "''")
).to_contiguous_window_dataset(
- window_size=config.action_horizon,
+ frame_offsets={
+ name: range(config.action_horizon) for name in ACTION_COLUMNS
+ },
columns=QPOS_COLUMNS + ACTION_COLUMNS + IMAGE_COLUMNS,
- anchor_columns=QPOS_COLUMNS + IMAGE_COLUMNS,
group_key="episode_id",
order_key="frame_index",
stride=1,
- tail="drop",
+ boundary="drop",
adapter=PaimonACTAdapter(normalization),
)
for episode_id in (train_episode_id, validation_episode_id)
diff --git a/paimon-python/pypaimon/multimodal/lerobot/dataset.py b/paimon-python/pypaimon/multimodal/lerobot/dataset.py
index eca82cbfd6..006ca6357b 100644
--- a/paimon-python/pypaimon/multimodal/lerobot/dataset.py
+++ b/paimon-python/pypaimon/multimodal/lerobot/dataset.py
@@ -18,7 +18,6 @@
"""LeRobot-compatible map-style reads from a multimodal Paimon table."""
import bisect
-import io
import json
import math
import operator
@@ -1438,14 +1437,10 @@ def _torch_row(row, features, return_uint8=False):
def _image_tensor(payload, feature, return_uint8=False):
if payload is None:
raise ValueError("LeRobot image feature contains a null frame.")
- import numpy as np
- import torch
- try:
- from PIL import Image, ImageOps
- except ImportError as error:
- raise ImportError(
- "PaimonLeRobotDataset requires Pillow from "
- "'pypaimon[lerobot]'.") from error
+ from pypaimon.multimodal.window_transforms import (
+ _decode_image,
+ _image_array_to_tensor,
+ )
expected_shape = _feature_shape(feature, "image")
if len(expected_shape) != 3:
@@ -1455,21 +1450,12 @@ def _image_tensor(payload, feature, return_uint8=False):
payload_shape = expected_shape[1:] + expected_shape[:1] \
if names and names[0] in ("channel", "channels") \
else expected_shape
- with Image.open(io.BytesIO(payload)) as image:
- array = np.array(ImageOps.exif_transpose(image), copy=True)
- if array.ndim == 2:
- array = array[:, :, None]
+ array = _decode_image(payload)
if array.shape != payload_shape:
raise ValueError(
"LeRobot image payload has shape %s, expected %s."
% (array.shape, payload_shape))
- normalize = array.dtype == np.uint8
- tensor = torch.from_numpy(array).permute(2, 0, 1)
- if normalize and return_uint8:
- return tensor
- # Preserve high-bit-depth and floating-point images in native units.
- tensor = tensor.float()
- return tensor.div_(255) if normalize else tensor
+ return _image_array_to_tensor(array, return_uint8)
def _video_tensor(frame, feature, return_uint8=False):
diff --git a/paimon-python/pypaimon/multimodal/query.py b/paimon-python/pypaimon/multimodal/query.py
index 489ff64152..e21edb2f1e 100644
--- a/paimon-python/pypaimon/multimodal/query.py
+++ b/paimon-python/pypaimon/multimodal/query.py
@@ -167,42 +167,58 @@ class ScanQuery:
def to_contiguous_window_dataset(
self,
*,
- window_size,
+ window_size=None,
columns=None,
anchor_columns=None,
group_key="episode_index",
order_key="frame_index",
stride=1,
- tail="drop",
+ tail=None,
column_transforms=None,
pad_values=None,
adapter=None,
- blob_parallelism=64):
- """Build a snapshot-pinned, map-style Dataset of contiguous rows.
+ blob_parallelism=64,
+ frame_offsets=None,
+ delta_timestamps=None,
+ fps=None,
+ tolerance_s=None,
+ boundary=None):
+ """Build snapshot-pinned training windows from this single-table scan.
The Dataset indexes only ``group_key``, ``order_key``, and Paimon row
- IDs, then reads projected values on demand. Columns listed in
- ``anchor_columns`` are provided to ``column_transforms`` as one-element
- lists read from the first row of each window; ``adapter`` receives the
- transformed values. ``order_key`` must contain non-null integers that
+ IDs, then reads projected values on demand. Each column has its own
+ relative frame offsets; unspecified columns use ``[0]``. Transforms
+ receive padded lists, including singleton windows, before ``adapter``
+ receives the sample. ``order_key`` must contain non-null integers that
increase by exactly one within each group. The Dataset sorts rows within
each group and never creates a window across groups.
Args:
- window_size: Number of rows in a complete window.
+ window_size: Legacy forward window size. Mutually exclusive with
+ frame_offsets and delta_timestamps. Retains a single is_pad
+ mask instead of the new per-column masks.
columns: Value columns to return, excluding the group and order
keys. The scan projection is used when omitted.
- anchor_columns: Subset of ``columns`` read only from the window's
- first row.
+ anchor_columns: With window_size, columns using only the anchor.
group_key: Column identifying an independent row sequence.
order_key: Integer position column within each group.
- stride: Distance between scheduled window starts.
- tail: Handling for incomplete final windows: ``drop``, ``pad``, or
- ``error``.
+ stride: Distance between anchors, starting at each group's first row.
+ tail: Legacy boundary alias requiring window_size; cannot be used
+ together with boundary.
column_transforms: Per-column callables applied to value lists.
- pad_values: Optional replacement values used by ``tail='pad'``.
+ pad_values: Per-column raw padding values; otherwise repeat the
+ nearest endpoint. Padding happens before column transforms.
adapter: Callable that converts the complete sample mapping.
blob_parallelism: Maximum concurrent BLOB body reads per fetch.
+ frame_offsets: Mapping of columns to nonempty integer offset
+ sequences. Supports history, future, sparse and repeated frames.
+ delta_timestamps: Alternative per-column offsets in seconds, aligned
+ to the regular frame grid defined by fps within tolerance_s.
+ fps: Finite positive frame rate, required with delta_timestamps.
+ tolerance_s: Finite nonnegative seconds-conversion tolerance,
+ defaulting to 1e-4. Only valid with delta_timestamps.
+ boundary: Handling at both group ends: drop (default), pad, or error.
+ Drop retains only anchors valid for every requested offset.
Returns:
A snapshot-pinned ``ContiguousWindowDataset``. See that class for
@@ -222,6 +238,11 @@ class ScanQuery:
pad_values=pad_values,
adapter=adapter,
blob_parallelism=blob_parallelism,
+ frame_offsets=frame_offsets,
+ delta_timestamps=delta_timestamps,
+ fps=fps,
+ tolerance_s=tolerance_s,
+ boundary=boundary,
)
def to_ray(
diff --git a/paimon-python/pypaimon/multimodal/window_dataset.py b/paimon-python/pypaimon/multimodal/window_dataset.py
index 2a63845962..ec5f639969 100644
--- a/paimon-python/pypaimon/multimodal/window_dataset.py
+++ b/paimon-python/pypaimon/multimodal/window_dataset.py
@@ -18,6 +18,7 @@
"""Snapshot-pinned PyTorch Dataset for contiguous Paimon row windows."""
import copy
+import math
import operator
import numpy as np
@@ -42,77 +43,116 @@ from pypaimon.utils.range import Range
class ContiguousWindowDataset(Dataset):
- """Map-style Dataset which reads fixed row windows on demand.
+ """Map-style training samples from a single table of contiguous frames.
The in-memory index contains only group values, order bounds, and Paimon
row IDs, stored in Arrow and NumPy arrays. Each ``__getitem__`` reads the
projected rows from the snapshot resolved while the index was built, reusing
that snapshot's authorized scan plan instead of planning again. Within each
group, ``order_key`` must contain non-null integers that increase by exactly
- one; rows from different groups never share a window. ``tail`` controls
- scheduled anchors whose remaining rows are shorter than ``window_size``:
+ one; rows from different groups never share a window. ``frame_offsets``
+ selects relative frames independently for each projected column. Unspecified
+ columns use ``[0]``. Alternatively, ``delta_timestamps`` specifies offsets in
+ seconds on a regular grid defined by ``fps``. ``boundary`` handles scheduled
+ anchors whose offsets extend beyond either end of a group:
* ``drop`` omits them;
- * ``pad`` repeats final values and marks repeats in ``is_pad``;
+ * ``pad`` repeats the closest endpoint, or uses the column's ``pad_values``;
* ``error`` rejects the dataset.
- The raw result mapping contains scalar group and order values, a
- length-``window_size`` Boolean ``is_pad`` tensor, one-element lists for
- ``anchor_columns``, and length-``window_size`` lists for other projected
- columns. ``anchor_columns`` therefore avoids loading repeated context such
- as observation images or initial robot state. ``column_transforms`` then
- convert individual column lists before ``adapter`` adapts the complete
- mapping to a model-specific contract.
+ The result contains scalar group/order values identifying the anchor, a
+ list for each value column (including singleton windows), and a Boolean
+ ``<column>_is_pad`` tensor per column. Offsets retain their order and
+ duplicates. Padding happens before ``column_transforms`` convert each list;
+ ``adapter`` then adapts the complete mapping to a model-specific contract.
+ Both transforms operate on independent copies of the requested values.
+
+ For compatibility, ``window_size`` selects forward consecutive frames for
+ all columns except ``anchor_columns``, which use ``[0]``. These calls retain
+ the original ``tail`` argument and single ``is_pad`` mask in their output.
+ Only one of the three window definitions may be supplied. Without any,
+ every column uses ``[0]``. ``stride`` schedules anchors from each group's
+ first row; boundary handling never shifts that schedule.
``blob_parallelism`` controls concurrent BLOB reads for each item or batch.
Video frame columns are not supported yet, because a window read would drop
the frame metadata carried by their descriptors.
"""
- _TAIL_POLICIES = ("drop", "pad", "error")
+ _BOUNDARY_POLICIES = ("drop", "pad", "error")
def __init__(
self,
query,
*,
- window_size,
+ window_size=None,
columns=None,
anchor_columns=None,
group_key="episode_index",
order_key="frame_index",
stride=1,
- tail="drop",
+ tail=None,
column_transforms=None,
pad_values=None,
adapter=None,
- blob_parallelism=64):
+ blob_parallelism=64,
+ frame_offsets=None,
+ delta_timestamps=None,
+ fps=None,
+ tolerance_s=None,
+ boundary=None):
if not isinstance(query, ScanQuery) or isinstance(query, _PreFilterQuery):
raise TypeError(
"ContiguousWindowDataset is only supported on scan(), "
"not search queries.")
- self.window_size = _positive_int(window_size, "window_size")
+ self.window_size = (
+ _positive_int(window_size, "window_size")
+ if window_size is not None else None)
self.stride = _positive_int(stride, "stride")
- if tail not in self._TAIL_POLICIES:
+ if sum(value is not None for value in (
+ window_size, frame_offsets, delta_timestamps)) > 1:
raise ValueError(
- "tail must be one of %s; got %r."
- % (self._TAIL_POLICIES, tail))
- self.tail = tail
+ "Specify only one of window_size, frame_offsets, "
+ "or delta_timestamps.")
+ if window_size is None and (anchor_columns is not None or tail is not None):
+ raise ValueError("anchor_columns and tail require window_size.")
+ if tail is not None and boundary is not None:
+ raise ValueError("Specify boundary or tail, not both.")
+ self.boundary = boundary if boundary is not None else (
+ tail if tail is not None else "drop")
+ if self.boundary not in self._BOUNDARY_POLICIES:
+ raise ValueError(
+ "boundary must be one of %s; got %r."
+ % (self._BOUNDARY_POLICIES, self.boundary))
+ if self.window_size is not None:
+ self.tail = self.boundary
self.group_key = _column(query, group_key, "group_key")
self.order_key = _column(query, order_key, "order_key")
if self.group_key == self.order_key:
raise ValueError("group_key and order_key must name different columns.")
- if "is_pad" in (self.group_key, self.order_key):
+ if self.window_size is not None and "is_pad" in (self.group_key, self.order_key):
raise ValueError("group_key and order_key must not be is_pad.")
self.columns = _columns(
query, columns, self.group_key, self.order_key)
_reject_video_columns(query._table, self.columns)
self.anchor_columns = _anchor_columns(anchor_columns, self.columns)
- anchor_column_set = set(self.anchor_columns)
- self._window_columns = [
- name for name in self.columns if name not in anchor_column_set
- ]
+ self.frame_offsets = _frame_offsets(
+ self.columns, frame_offsets, delta_timestamps, fps, tolerance_s,
+ self.window_size, self.anchor_columns)
+ reserved = {"is_pad"} if self.window_size is not None else {
+ "%s_is_pad" % name for name in self.columns}
+ collisions = reserved.intersection(
+ self.columns + [self.group_key, self.order_key])
+ if collisions:
+ raise ValueError(
+ "Generated padding mask names conflict with output "
+ "columns: %s." % sorted(collisions))
self.column_transforms = _column_transforms(
column_transforms, self.columns)
self.pad_values = _pad_values(pad_values, self.columns)
+ self._read_groups = {}
+ for name, offsets in self.frame_offsets.items():
+ key = (offsets, self.boundary == "pad" and name in self.pad_values)
+ self._read_groups.setdefault(key, []).append(name)
if adapter is not None and not callable(adapter):
raise TypeError("adapter must be callable or None.")
self.adapter = adapter
@@ -139,19 +179,15 @@ class ContiguousWindowDataset(Dataset):
return int(self._anchor_groups.size)
def __getitem__(self, index):
- """Read one window by map-style Dataset index.
+ """Read one sample, or a list of samples for a Python slice.
Negative indices follow Python sequence semantics. The return value is
the pre-adapter mapping described by the class, or the adapter result
when an adapter is configured.
"""
- anchor, row_ids = self._resolve_window(index)
- rows = self._read_window_rows(row_ids)
- anchor_row = (
- self._read_rows(row_ids[:1], self.anchor_columns)[0]
- if self.anchor_columns else None
- )
- return self._sample(anchor, rows, anchor_row)
+ if isinstance(index, slice):
+ return self.__getitems__(range(*index.indices(len(self))))
+ return self.__getitems__([index])[0]
def __getitems__(self, indices):
"""Read several Dataset indices while coalescing overlapping row IDs.
@@ -162,28 +198,20 @@ class ContiguousWindowDataset(Dataset):
windows = [self._resolve_window(index) for index in indices]
if not windows:
return []
- row_ids = list(dict.fromkeys(
- row_id for _, window_row_ids in windows
- for row_id in window_row_ids
- ))
- rows_by_id = dict(zip(row_ids, self._read_window_rows(row_ids)))
- anchor_row_ids = list(dict.fromkeys(
- window_row_ids[0] for _, window_row_ids in windows
- ))
- anchor_rows_by_id = (
- dict(zip(
- anchor_row_ids,
- self._read_rows(anchor_row_ids, self.anchor_columns),
- ))
- if self.anchor_columns else {}
- )
+ reads = {}
+ for (offsets, constant_pad), columns in self._read_groups.items():
+ plans = [self._column_window(anchor, offsets, constant_pad)
+ for anchor in windows]
+ row_ids = list(dict.fromkeys(
+ row_id for ids, _ in plans for row_id in ids
+ if row_id is not None))
+ rows = self._read_rows(row_ids, columns) if row_ids else []
+ by_id = dict(zip(row_ids, rows))
+ for name in columns:
+ reads[name] = (plans, by_id)
return [
- self._sample(
- anchor,
- [rows_by_id[row_id] for row_id in window_row_ids],
- anchor_rows_by_id.get(window_row_ids[0]),
- )
- for anchor, window_row_ids in windows
+ self._sample(anchor, position, reads)
+ for position, anchor in enumerate(windows)
]
def __getstate__(self):
@@ -202,32 +230,38 @@ class ContiguousWindowDataset(Dataset):
group_index = int(self._anchor_groups[index])
start = int(self._anchor_starts[index])
- valid_count = min(
- self.window_size, int(self._group_lengths[group_index]) - start)
- offset = int(self._group_starts[group_index]) + start
- row_ids = self._row_ids[offset:offset + valid_count].tolist()
- return (group_index, start, valid_count), row_ids
-
- def _sample(self, anchor, rows, anchor_row=None):
- group_index, start, valid_count = anchor
- padding_count = self.window_size - valid_count
- padding_mask = torch.zeros(self.window_size, dtype=torch.bool)
- if padding_count:
- padding_mask[valid_count:] = True
+ return group_index, start
+
+ def _column_window(self, anchor, offsets, constant_pad):
+ group_index, start = anchor
+ length = int(self._group_lengths[group_index])
+ group_start = int(self._group_starts[group_index])
+ positions = [start + delta for delta in offsets]
+ padding = [position < 0 or position >= length for position in positions]
+ row_ids = [
+ None if padded and constant_pad else int(self._row_ids[
+ group_start + min(max(position, 0), length - 1)])
+ for position, padded in zip(positions, padding)
+ ]
+ return row_ids, padding
+
+ def _sample(self, anchor, position, reads):
+ group_index, start = anchor
sample = {
self.group_key: self._group_keys[group_index],
self.order_key: int(self._group_first_orders[group_index]) + start,
- "is_pad": padding_mask,
}
+ if self.window_size is not None:
+ available = int(self._group_lengths[group_index]) - start
+ sample["is_pad"] = torch.arange(self.window_size) >= available
for name in self.columns:
- if name in self.anchor_columns:
- values = [copy.deepcopy(anchor_row[name])]
- else:
- values = [copy.deepcopy(row[name]) for row in rows]
- if padding_count and name not in self.anchor_columns:
- pad_value = self.pad_values.get(name, values[-1])
- values.extend(
- copy.deepcopy(pad_value) for _ in range(padding_count))
+ plans, rows = reads[name]
+ row_ids, padding = plans[position]
+ values = [copy.deepcopy(
+ self.pad_values[name] if row_id is None else rows[row_id][name])
+ for row_id in row_ids]
+ if self.window_size is None:
+ sample["%s_is_pad" % name] = torch.tensor(padding, dtype=torch.bool)
transform = self.column_transforms.get(name)
sample[name] = transform(values) if transform is not None else values
if self.adapter is not None:
@@ -320,24 +354,33 @@ class ContiguousWindowDataset(Dataset):
def _build_anchors(self):
groups = []
starts = []
+ minimum = min(min(offsets) for offsets in self.frame_offsets.values())
+ maximum = max(max(offsets) for offsets in self.frame_offsets.values())
+ # Legacy windows schedule their full horizon even with anchor-only
+ # columns, so their cardinality and global mask remain unchanged.
+ if self.window_size is not None:
+ maximum = max(maximum, self.window_size - 1)
for group_index, length in enumerate(self._group_lengths):
length = int(length)
positions = np.arange(0, length, self.stride, dtype=np.int64)
- valid_counts = np.minimum(self.window_size, length - positions)
- incomplete = np.flatnonzero(valid_counts < self.window_size)
+ lower = min(length, max(0, -minimum))
+ upper = max(0, min(length, length - maximum))
+ valid = (positions >= lower) & (positions < upper)
+ incomplete = np.flatnonzero(~valid)
if incomplete.size:
- if self.tail == "error":
+ if self.boundary == "error":
first = int(incomplete[0])
+ size = "window_size=%s, " % self.window_size \
+ if self.window_size is not None else ""
raise ValueError(
"Group %s has an incomplete window at %s: "
- "window_size=%d, available=%d."
+ "%soffset bounds=[%d, %d]."
% (self._group_keys[group_index],
int(self._group_first_orders[group_index])
+ int(positions[first]),
- self.window_size,
- int(valid_counts[first])))
- if self.tail == "drop":
- positions = positions[valid_counts == self.window_size]
+ size, minimum, maximum))
+ if self.boundary == "drop":
+ positions = positions[valid]
if positions.size:
groups.append(np.full(positions.size, group_index, dtype=np.int64))
starts.append(positions)
@@ -346,11 +389,6 @@ class ContiguousWindowDataset(Dataset):
return empty, empty.copy()
return np.concatenate(groups), np.concatenate(starts)
- def _read_window_rows(self, row_ids):
- if not self._window_columns:
- return [{} for _ in row_ids]
- return self._read_rows(row_ids, self._window_columns)
-
def _read_rows(self, row_ids, columns=None):
"""Read projected rows by ID from the pinned snapshot.
@@ -608,6 +646,80 @@ def _reject_video_columns(table, columns):
"frame_index." % video_columns)
+def _frame_offsets(columns, offsets, timestamps, fps, tolerance_s,
+ window_size, anchor_columns):
+ """Normalize the public window forms to nonempty integer offset tuples."""
+ if timestamps is not None:
+ tolerance = _finite_number(
+ 1e-4 if tolerance_s is None else tolerance_s, "tolerance_s")
+ if tolerance < 0:
+ raise ValueError("tolerance_s must be non-negative.")
+ if fps is None:
+ raise ValueError("delta_timestamps requires an explicit fps.")
+ fps = _finite_number(fps, "fps")
+ if fps <= 0:
+ raise ValueError("fps must be positive.")
+ elif fps is not None or tolerance_s is not None:
+ raise ValueError("fps and tolerance_s are only used with delta_timestamps.")
+ if window_size is not None:
+ forward = tuple(range(window_size))
+ return {
+ name: (0,) if name in anchor_columns else forward
+ for name in columns
+ }
+ parameter = "delta_timestamps" if timestamps is not None else "frame_offsets"
+ supplied = _mapping(timestamps if timestamps is not None else offsets, parameter)
+ _validate_mapping_columns(supplied, columns, parameter)
+ result = {}
+ for name in columns:
+ if name not in supplied:
+ result[name] = (0,)
+ continue
+ values = supplied[name]
+ if isinstance(values, (str, bytes, dict)):
+ raise TypeError("%s[%r] must be a sequence of offsets." % (parameter, name))
+ try:
+ values = list(values)
+ except TypeError:
+ raise TypeError("%s[%r] must be a sequence of offsets." % (parameter, name))
+ if not values:
+ raise ValueError("%s[%r] must not be empty." % (parameter, name))
+ converted = []
+ for value in values:
+ if timestamps is not None:
+ seconds = _finite_number(value, "delta_timestamps[%r]" % name)
+ scaled = seconds * fps
+ if not math.isfinite(scaled):
+ raise ValueError("delta_timestamps * fps must be finite.")
+ delta = round(scaled)
+ if abs(seconds - delta / fps) > tolerance:
+ raise ValueError(
+ "delta_timestamps for %s must align to multiples of 1/fps."
+ % name)
+ else:
+ if isinstance(value, (bool, np.bool_)):
+ raise TypeError("frame_offsets must contain integers, not booleans.")
+ try:
+ delta = operator.index(value)
+ except TypeError:
+ raise TypeError("frame_offsets must contain integer offsets.")
+ converted.append(delta)
+ result[name] = tuple(converted)
+ return result
+
+
+def _finite_number(value, name):
+ if isinstance(value, (bool, np.bool_)):
+ raise ValueError("%s must be a finite number, not a boolean." % name)
+ try:
+ number = float(value)
+ except (TypeError, ValueError, OverflowError):
+ raise ValueError("%s must be a finite number." % name)
+ if not math.isfinite(number):
+ raise ValueError("%s must be a finite number." % name)
+ return number
+
+
def _columns(query, columns, group_key, order_key):
available = {field.name for field in query._table.fields}
if columns is None:
@@ -635,10 +747,10 @@ def _columns(query, columns, group_key, order_key):
if invalid:
raise ValueError("columns do not exist: %s." % invalid)
reserved = [name for name in columns
- if name in (group_key, order_key, "is_pad")]
+ if name in (group_key, order_key)]
if reserved:
raise ValueError(
- "columns must not include group_key, order_key, or is_pad: %s."
+ "columns must not include group_key or order_key: %s."
% reserved)
return columns
diff --git a/paimon-python/pypaimon/multimodal/window_transforms.py b/paimon-python/pypaimon/multimodal/window_transforms.py
new file mode 100644
index 0000000000..3385b8525d
--- /dev/null
+++ b/paimon-python/pypaimon/multimodal/window_transforms.py
@@ -0,0 +1,88 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Column transforms for materialized training windows."""
+
+import io
+
+
+def to_tensor(values, dtype=None):
+ """Convert numeric window values to a tensor with a leading time axis.
+
+ Use ``partial(to_tensor, dtype=torch.float32)`` from ``functools``
+ to choose a dtype.
+ This conversion does not normalize values.
+ """
+ import torch
+
+ result = torch.as_tensor(values, dtype=dtype)
+ if result.ndim == 0:
+ raise ValueError("Window values must have a time axis.")
+ return result
+
+
+def images_to_tensor(values, return_uint8=False):
+ """Decode a non-empty sequence of image bytes to a TCHW tensor.
+
+ Apply EXIF orientation and preserve grayscale as one channel. Eight-bit
+ pixels become float32 in [0, 1], or stay uint8 with ``return_uint8=True``.
+ Higher-bit-depth pixels always become float32 in their original units.
+ All decoded frames must have the same shape.
+ """
+ import torch
+
+ if not isinstance(return_uint8, bool):
+ raise TypeError("return_uint8 must be a boolean.")
+ frames = []
+ for payload in values:
+ if not isinstance(payload, (bytes, bytearray, memoryview)):
+ raise ValueError("Image window values must contain image bytes.")
+ frame = _image_array_to_tensor(_decode_image(payload), return_uint8)
+ if frames and frame.shape != frames[0].shape:
+ raise ValueError("Image window frames must have the same shape.")
+ frames.append(frame)
+ if not frames:
+ raise ValueError("Image window must contain at least one frame.")
+ return torch.stack(frames)
+
+
+def _decode_image(payload):
+ import numpy as np
+ try:
+ from PIL import Image, ImageOps
+ except ImportError as error:
+ raise ImportError(
+ "Image decoding requires Pillow; install 'pypaimon[torch]' "
+ "or Pillow.") from error
+
+ with Image.open(io.BytesIO(payload)) as image:
+ array = np.array(ImageOps.exif_transpose(image), copy=True)
+ if array.ndim == 2:
+ array = array[:, :, None]
+ return array
+
+
+def _image_array_to_tensor(array, return_uint8=False):
+ import numpy as np
+ import torch
+
+ normalize = array.dtype == np.uint8
+ tensor = torch.from_numpy(array).permute(2, 0, 1)
+ if normalize and return_uint8:
+ return tensor
+ tensor = tensor.float()
+ return tensor.div_(255) if normalize else tensor
diff --git a/paimon-python/pypaimon/tests/act_runner_test.py b/paimon-python/pypaimon/tests/act_runner_test.py
index 57d64bf8c1..a7fbc69e92 100644
--- a/paimon-python/pypaimon/tests/act_runner_test.py
+++ b/paimon-python/pypaimon/tests/act_runner_test.py
@@ -418,7 +418,16 @@ def test_backends_match_the_golden_act_window_contract(benchmark_input):
)
expected = hdf5[1]
- actual = paimon[1]
+ with patch.object(paimon, "adapter", wraps=paimon.adapter) as adapt:
+ actual = paimon[1]
+ generic_sample = adapt.call_args.args[0]
+ assert "is_pad" not in generic_sample
+ assert len(generic_sample["action"]) == 3
+ assert generic_sample["action_is_pad"].tolist() == [False] * 3
+ for name in QPOS_COLUMNS + IMAGE_COLUMNS:
+ assert len(generic_sample[name]) == 1
+ assert generic_sample[name + "_is_pad"].tolist() == [False]
+ assert actual["is_pad"] is generic_sample["action_is_pad"]
assert set(expected) == {
"sample_id", "episode_id", "frame_index", "qpos", "action",
@@ -605,11 +614,10 @@ def test_paimon_windows_are_lazy_snapshot_pinned_and_vortex_independent(
with patch.object(
train, "_read_rows", wraps=train._read_rows) as read_rows:
sample_before_append = train[0]
- assert [call.args[1] for call in read_rows.call_args_list] == [
- list(ACTION_COLUMNS),
- list(QPOS_COLUMNS + IMAGE_COLUMNS),
- ]
- assert [len(call.args[0]) for call in read_rows.call_args_list] == [3, 1]
+ assert {
+ tuple(call.args[1]): len(call.args[0])
+ for call in read_rows.call_args_list
+ } == {ACTION_COLUMNS: 3, QPOS_COLUMNS + IMAGE_COLUMNS: 1}
assert fetch.call_count == 1
assert {
name: len(fetch.call_args.args[1][name])
diff --git a/paimon-python/pypaimon/tests/contiguous_window_dataset_test.py b/paimon-python/pypaimon/tests/contiguous_window_dataset_test.py
index c3582ea7b5..c0e5d4d7dd 100644
--- a/paimon-python/pypaimon/tests/contiguous_window_dataset_test.py
+++ b/paimon-python/pypaimon/tests/contiguous_window_dataset_test.py
@@ -116,6 +116,242 @@ class ContiguousWindowDatasetTest(unittest.TestCase):
)
)
+ def test_per_column_offsets_combine_history_future_and_anchor(self):
+ dataset = self._table().scan().to_contiguous_window_dataset(
+ columns=["value", "payload"], group_key="episode", order_key="step",
+ frame_offsets={"value": [-1, 0, 2]}, boundary="pad")
+
+ self.assertEqual(6, len(dataset))
+ sample = dataset[0]
+ self.assertEqual(("episode-a", 0), (sample["episode"], sample["step"]))
+ self.assertEqual([0, 0, 1], sample["value"])
+ self.assertEqual([True, False, True], sample["value_is_pad"].tolist())
+ self.assertEqual([b"episode-a-0"], sample["payload"])
+ self.assertEqual([False], sample["payload_is_pad"].tolist())
+ self.assertEqual(torch.bool, sample["value_is_pad"].dtype)
+ self.assertNotIn("is_pad", sample)
+ self.assertEqual([101, 102, 103], dataset[4]["value"])
+ self.assertEqual([False, False, True], dataset[4]["value_is_pad"].tolist())
+
+ def test_frame_mode_defaults_to_single_frame_with_time_dimension(self):
+ dataset = self._table().scan().select(["value"]).to_contiguous_window_dataset(
+ group_key="episode", order_key="step")
+
+ self.assertEqual(6, len(dataset))
+ self.assertEqual({"episode", "step", "value", "value_is_pad"}, set(dataset[0]))
+ self.assertEqual([0], dataset[0]["value"])
+ self.assertEqual([False], dataset[0]["value_is_pad"].tolist())
+ self.assertEqual(103, dataset[-1]["value"][0])
+
+ def test_drop_offsets_keep_stride_anchored_at_group_start(self):
+ table = self._table()
+ dataset = table.scan().to_contiguous_window_dataset(
+ columns=["value", "payload"], group_key="episode", order_key="step",
+ frame_offsets={"value": [-1, 0], "payload": [1]}, stride=2)
+
+ self.assertEqual(1, len(dataset))
+ self.assertEqual(("episode-b", 2), (dataset[0]["episode"], dataset[0]["step"]))
+ self.assertEqual([101, 102], dataset[0]["value"])
+ self.assertEqual([b"episode-b-3"], dataset[0]["payload"])
+ self.assertFalse(dataset[0]["value_is_pad"].any())
+ with self.assertRaisesRegex(
+ ValueError, r"episode-a.*at 0: offset bounds=\[-1, 0\]"):
+ table.scan().to_contiguous_window_dataset(
+ columns=["value"], group_key="episode", order_key="step",
+ frame_offsets={"value": [-1, 0]}, boundary="error")
+
+ def test_sparse_repeated_offsets_preserve_batch_and_slice_order(self):
+ dataset = self._table().scan().to_contiguous_window_dataset(
+ columns=["value"], group_key="episode", order_key="step",
+ frame_offsets={"value": [2, -1, 2, 0]}, boundary="pad")
+
+ samples = dataset.__getitems__([4, 0, 4, -1])
+ self.assertEqual([[103, 101, 103, 102], [1, 0, 1, 0],
+ [103, 101, 103, 102], [103, 102, 103, 103]],
+ [sample["value"] for sample in samples])
+ self.assertEqual([True, False, True, False], samples[0]["value_is_pad"].tolist())
+ self.assertEqual([dataset[i]["value"] for i in (5, 3, 1)],
+ [sample["value"] for sample in dataset[::-2]])
+ self.assertEqual([], dataset.__getitems__([]))
+ with self.assertRaises(IndexError):
+ dataset.__getitems__([0, len(dataset)])
+
+ def test_custom_padding_precedes_transforms_even_for_all_padding(self):
+ dataset = self._table().scan().to_contiguous_window_dataset(
+ columns=["value", "payload"], group_key="episode", order_key="step",
+ frame_offsets={"value": [-10, 10], "payload": [0, 10]},
+ boundary="pad", pad_values={"value": -7, "payload": b"missing"},
+ column_transforms={"value": _TensorColumnTransform()})
+
+ with patch.object(dataset, "_read_rows", wraps=dataset._read_rows) as read:
+ sample = dataset[0]
+ self.assertEqual(1, read.call_count)
+ self.assertEqual(["payload"], read.call_args.args[1])
+ self.assertTrue(torch.equal(torch.tensor([-7, -7]), sample["value"]))
+ self.assertEqual([True, True], sample["value_is_pad"].tolist())
+ self.assertEqual([b"episode-a-0", b"missing"], sample["payload"])
+ self.assertEqual([False, True], sample["payload_is_pad"].tolist())
+
+ def test_seconds_offsets_match_frame_offsets_with_alignment_tolerance(self):
+ table = self._table()
+ kwargs = dict(columns=["value", "payload"], group_key="episode",
+ order_key="step", boundary="pad")
+ frames = table.scan().to_contiguous_window_dataset(
+ frame_offsets={"value": [-1, 0, 2]}, **kwargs)
+ seconds = table.scan().to_contiguous_window_dataset(
+ delta_timestamps={"value": [-0.1, 0, 0.20001]}, fps=10,
+ tolerance_s=0.0001, **kwargs)
+
+ self.assertEqual(len(frames), len(seconds))
+ for expected, actual in zip(frames[:], seconds[:]):
+ self.assertEqual(expected["value"], actual["value"])
+ self.assertEqual(expected["payload"], actual["payload"])
+ self.assertTrue(torch.equal(expected["value_is_pad"], actual["value_is_pad"]))
+
+ def test_offset_projections_do_not_fetch_anchor_images_for_action_rows(self):
+ dataset = self._table().scan().to_contiguous_window_dataset(
+ columns=["value", "payload"], group_key="episode", order_key="step",
+ frame_offsets={"value": [-1, 0, 1], "payload": [0]}, boundary="pad")
+
+ with patch.object(dataset, "_read_rows", wraps=dataset._read_rows) as read, \
+ patch("pypaimon.multimodal.window_dataset.fetch_blob_bodies",
+ side_effect=window_dataset.fetch_blob_bodies) as fetch:
+ samples = dataset.__getitems__([3, 4, 3])
+
+ projections = {tuple(call.args[1]): len(call.args[0]) for call in read.call_args_list}
+ self.assertEqual({("value",): 4, ("payload",): 2}, projections)
+ self.assertEqual(2, sum(len(call.args[1]["payload"]) for call in fetch.call_args_list))
+ self.assertEqual([[100, 101, 102], [101, 102, 103], [100, 101, 102]],
+ [sample["value"] for sample in samples])
+
+ def test_unused_pad_values_do_not_split_reads(self):
+ table = self._table()
+ for boundary in ("drop", "error"):
+ with self.subTest(boundary=boundary):
+ dataset = table.scan().to_contiguous_window_dataset(
+ columns=["value", "payload"], group_key="episode", order_key="step",
+ frame_offsets={"value": [0, 1], "payload": [0, 1]},
+ stride=2, boundary=boundary, pad_values={"value": -1})
+ with patch.object(dataset, "_read_rows", wraps=dataset._read_rows) as read:
+ samples = dataset.__getitems__([0, 1, 0])
+ self.assertEqual(1, read.call_count)
+ self.assertEqual(["value", "payload"], read.call_args.args[1])
+ self.assertEqual(4, len(read.call_args.args[0]))
+ self.assertEqual([[0, 1], [100, 101], [0, 1]],
+ [sample["value"] for sample in samples])
+ self.assertEqual([b"episode-a-0", b"episode-a-1"], samples[0]["payload"])
+
+ def test_offset_padding_and_repeated_cells_are_mutably_isolated(self):
+ table = self.conn.create_table(
+ "offset_mutable", schema=pa.schema([
+ pa.field("episode", pa.string(), nullable=False),
+ pa.field("step", pa.int32(), nullable=False),
+ pa.field("values", pa.list_(pa.int32()), nullable=False),
+ ]), options=_TABLE_OPTIONS)
+ table.add([{"episode": "a", "step": 0, "values": [0]}])
+ pad = [-1]
+ dataset = table.scan().to_contiguous_window_dataset(
+ columns=["values"], group_key="episode", order_key="step",
+ frame_offsets={"values": [-1, 0, 0, 1]}, boundary="pad",
+ pad_values={"values": pad})
+
+ first, second = dataset.__getitems__([0, 0])
+ first["values"][0].append(99)
+ first["values"][1].append(99)
+ self.assertEqual([0], first["values"][2])
+ self.assertEqual([-1], first["values"][3])
+ self.assertEqual([[-1], [0], [0], [-1]], second["values"])
+ self.assertEqual([[-1], [0], [0], [-1]], dataset[0]["values"])
+ self.assertEqual([-1], pad)
+
+ def test_offset_dataloader_workers_and_distributed_sampler(self):
+ dataset = self._table().scan().to_contiguous_window_dataset(
+ columns=["value"], group_key="episode", order_key="step",
+ frame_offsets={"value": [-1, 0, 1]}, boundary="pad",
+ column_transforms={"value": _TensorColumnTransform()})
+ restored = pickle.loads(pickle.dumps(dataset))
+ self.assertEqual(dataset.snapshot_id, restored.snapshot_id)
+ batches = list(torch.utils.data.DataLoader(
+ restored, batch_size=2, num_workers=2, shuffle=False))
+ self.assertEqual([[0, 0, 1], [0, 1, 1], [100, 100, 101],
+ [100, 101, 102], [101, 102, 103], [102, 103, 103]],
+ [row for batch in batches for row in batch["value"].tolist()])
+ self.assertEqual((2, 3), tuple(batches[0]["value_is_pad"].shape))
+ self.assertEqual(torch.bool, batches[0]["value_is_pad"].dtype)
+ partitions = [list(torch.utils.data.DistributedSampler(
+ dataset, num_replicas=2, rank=rank, shuffle=False)) for rank in range(2)]
+ self.assertEqual([[0, 2, 4], [1, 3, 5]], partitions)
+ self.assertEqual([0, 100, 102], [dataset[i]["value"][1].item() for i in partitions[0]])
+
+ def test_rejects_invalid_offset_configuration_at_construction(self):
+ table = self._table()
+ cases = [
+ {"frame_offsets": {"missing": [0]}},
+ {"frame_offsets": {"value": []}},
+ {"frame_offsets": {"value": [1.0]}},
+ {"frame_offsets": {"value": [True]}},
+ {"frame_offsets": {"value": [0]}, "window_size": 2},
+ {"delta_timestamps": {"value": [0]}, "fps": 10, "window_size": 2},
+ {"frame_offsets": {"value": [0]}, "delta_timestamps": {"value": [0]}, "fps": 10},
+ {"delta_timestamps": {"value": [0]}},
+ {"delta_timestamps": {"value": []}, "fps": 10},
+ {"delta_timestamps": {"value": [float("nan")]}, "fps": 10},
+ {"delta_timestamps": {"value": [float("inf")]}, "fps": 10},
+ {"delta_timestamps": {"value": [0]}, "fps": 0},
+ {"delta_timestamps": {"value": [0]}, "fps": -1},
+ {"delta_timestamps": {"value": [0]}, "fps": float("inf")},
+ {"delta_timestamps": {"value": [0]}, "fps": float("nan")},
+ {"delta_timestamps": {"value": [0.05]}, "fps": 10},
+ {"frame_offsets": {"value": [0]}, "anchor_columns": ["value"]},
+ {"frame_offsets": {"value": [0]}, "tail": "pad"},
+ {"frame_offsets": {"value": [0]}, "tolerance_s": 0.01},
+ {"window_size": 2, "tolerance_s": 0.01},
+ {"window_size": 2, "tail": "pad", "boundary": "pad"},
+ {"boundary": "unknown"},
+ ]
+ for kwargs in cases:
+ with self.subTest(kwargs=kwargs), self.assertRaises((TypeError, ValueError)):
+ table.scan().to_contiguous_window_dataset(
+ columns=["value"], group_key="episode", order_key="step", **kwargs)
+
+ def test_rejects_generated_padding_mask_collisions(self):
+ table = self.conn.create_table(
+ "mask_collision", schema=pa.schema([
+ pa.field("episode", pa.string(), nullable=False),
+ pa.field("step", pa.int32(), nullable=False),
+ pa.field("value", pa.int32(), nullable=False),
+ pa.field("value_is_pad", pa.int32(), nullable=False),
+ ]), options=_TABLE_OPTIONS)
+ table.add([{"episode": "a", "step": 0, "value": 1, "value_is_pad": 2}])
+ with self.assertRaises(ValueError):
+ table.scan().to_contiguous_window_dataset(
+ columns=["value", "value_is_pad"], group_key="episode", order_key="step")
+ with self.assertRaises(ValueError):
+ table.scan().to_contiguous_window_dataset(
+ columns=["value"], group_key="episode", order_key="value_is_pad")
+
+ def test_offset_mode_allows_is_pad_as_a_stored_column(self):
+ table = self.conn.create_table(
+ "stored_is_pad", schema=pa.schema([
+ pa.field("episode", pa.string(), nullable=False),
+ pa.field("step", pa.int32(), nullable=False),
+ pa.field("is_pad", pa.int32(), nullable=False),
+ pa.field("value", pa.int32(), nullable=False),
+ ]), options=_TABLE_OPTIONS)
+ table.add([{"episode": "a", "step": i, "is_pad": i, "value": 100 + i}
+ for i in range(3)])
+ for group, order, column in (("episode", "step", "is_pad"),
+ ("is_pad", "step", "value"),
+ ("episode", "is_pad", "value")):
+ with self.subTest(group=group, order=order, column=column):
+ kwargs = dict(group_key=group, order_key=order, columns=[column])
+ dataset = table.scan().to_contiguous_window_dataset(**kwargs)
+ self.assertEqual(3, len(dataset))
+ self.assertEqual([0] if column == "is_pad" else [100], dataset[0][column])
+ self.assertEqual([False], dataset[0][column + "_is_pad"].tolist())
+ with self.assertRaises(ValueError):
+ table.scan().to_contiguous_window_dataset(window_size=1, **kwargs)
+
def test_sorts_rows_and_never_crosses_episode_boundaries(self):
dataset = self._dataset(self._table())
diff --git a/paimon-python/pypaimon/tests/window_transforms_test.py b/paimon-python/pypaimon/tests/window_transforms_test.py
new file mode 100644
index 0000000000..45df0766bb
--- /dev/null
+++ b/paimon-python/pypaimon/tests/window_transforms_test.py
@@ -0,0 +1,111 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import io
+import unittest
+from functools import partial
+from unittest.mock import patch
+
+import numpy as np
+import pytest
+
+torch = pytest.importorskip("torch")
+Image = pytest.importorskip("PIL.Image")
+
+from pypaimon.multimodal.window_transforms import images_to_tensor, to_tensor
+
+
+def _png(array, **kwargs):
+ buffer = io.BytesIO()
+ Image.fromarray(array).save(buffer, format="PNG", **kwargs)
+ return buffer.getvalue()
+
+
+class WindowTransformsTest(unittest.TestCase):
+
+ def test_numeric_values_keep_time_axis_and_explicit_dtype(self):
+ convert = partial(to_tensor, dtype=torch.float64)
+ values = convert([[1, 2], [3, 4]])
+ self.assertEqual((2, 2), tuple(values.shape))
+ self.assertEqual(torch.float64, values.dtype)
+ self.assertEqual([1, 2], to_tensor([1, 2]).tolist())
+ with self.assertRaisesRegex(ValueError, "time axis"):
+ to_tensor(1)
+
+ def test_rgb_and_gray_pixels_and_time_axis(self):
+ rgb = _png(np.full((2, 3, 3), 128, dtype=np.uint8))
+ values = images_to_tensor([rgb, rgb])
+ self.assertEqual((2, 3, 2, 3), tuple(values.shape))
+ self.assertEqual(torch.float32, values.dtype)
+ torch.testing.assert_close(values, torch.full_like(values, 128 / 255))
+ raw = images_to_tensor([rgb], return_uint8=True)
+ self.assertEqual(torch.uint8, raw.dtype)
+ self.assertEqual(128, raw[0, 0, 0, 0].item())
+ gray = _png(np.array([[0, 255]], dtype=np.uint8))
+ self.assertEqual([[[[0., 1.]]]], images_to_tensor([gray]).tolist())
+
+ def test_high_bit_depth_keeps_native_units(self):
+ payload = _png(np.array([[0, 1024, 65535]], dtype=np.uint16))
+ for return_uint8 in (False, True):
+ values = images_to_tensor([payload], return_uint8=return_uint8)
+ self.assertEqual(torch.float32, values.dtype)
+ self.assertEqual([[[[0., 1024., 65535.]]]], values.tolist())
+
+ def test_exif_orientation_is_applied_before_stacking(self):
+ exif = Image.Exif()
+ exif[274] = 6
+ payload = _png(np.array([[1, 2], [3, 4], [5, 6]], dtype=np.uint8),
+ exif=exif)
+ values = images_to_tensor([payload], return_uint8=True)
+ self.assertEqual([[[[5, 3, 1], [6, 4, 2]]]], values.tolist())
+
+ def test_invalid_image_inputs_fail_clearly(self):
+ for values in ([], [None], ["not bytes"]):
+ with self.subTest(values=values), self.assertRaises(ValueError):
+ images_to_tensor(values)
+ with self.assertRaises(OSError):
+ images_to_tensor([b"not an image"])
+ with self.assertRaises(TypeError):
+ images_to_tensor([b"unused"], return_uint8=1)
+ with self.assertRaises(ValueError):
+ images_to_tensor([_png(np.zeros((2, 2), dtype=np.uint8)),
+ _png(np.zeros((3, 2), dtype=np.uint8))])
+
+ def test_image_import_errors_identify_the_failing_dependency(self):
+ from pypaimon.multimodal.lerobot.dataset import _image_tensor
+
+ payload = _png(np.zeros((2, 3, 3), dtype=np.uint8))
+ feature = {"dtype": "image", "shape": (2, 3, 3)}
+ readers = (lambda: images_to_tensor([payload]),
+ lambda: _image_tensor(payload, feature))
+ for read in readers:
+ with patch.dict("sys.modules", {"PIL": None}):
+ with self.assertRaisesRegex(ImportError, "requires Pillow"):
+ read()
+ with patch.dict("sys.modules", {"numpy": None}):
+ with self.assertRaises(ImportError) as raised:
+ read()
+ self.assertEqual("numpy", raised.exception.name)
+ failure = ImportError("decoder plugin unavailable")
+ with patch.object(Image, "open", side_effect=failure):
+ with self.assertRaises(ImportError) as raised:
+ read()
+ self.assertIs(failure, raised.exception)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/paimon-python/setup.py b/paimon-python/setup.py
index 60cfe979d3..246f77714c 100644
--- a/paimon-python/setup.py
+++ b/paimon-python/setup.py
@@ -263,6 +263,7 @@ setup(
],
'torch': [
'torch',
+ 'Pillow',
],
'act': LEROBOT_DEPENDENCIES + [
'Pillow; python_version>="3.10"',
(paimon) 18/20: Revert "[format] Support file.block-size for Avro (#9901)"
Posted by ho...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
XiaoHongbo-Hope pushed a commit to branch release-2.1
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit 394cbaaa11e141362f5a5d6c5dadd07d72b56430
Author: JingsongLi <ji...@gmail.com>
AuthorDate: Thu Sep 17 11:35:21 2026 +0800
Revert "[format] Support file.block-size for Avro (#9901)"
This reverts commit 7f8c08e16977101c9207816f89e69576f0861717.
---
docs/generated/core_configuration.html | 2 +-
.../main/java/org/apache/paimon/CoreOptions.java | 3 +-
.../apache/paimon/format/avro/AvroFileFormat.java | 6 --
.../paimon/format/avro/AvroFileFormatTest.java | 71 ----------------------
4 files changed, 2 insertions(+), 80 deletions(-)
diff --git a/docs/generated/core_configuration.html b/docs/generated/core_configuration.html
index 19f6d52603..cf48d473c4 100644
--- a/docs/generated/core_configuration.html
+++ b/docs/generated/core_configuration.html
@@ -738,7 +738,7 @@ under the License.
<td><h5>file.block-size</h5></td>
<td style="word-wrap: break-word;">(none)</td>
<td>MemorySize</td>
- <td>File block size of format, default value of orc stripe is 64 MB, parquet row group is 128 MB, and avro block is 64000 bytes. For avro, this is the uncompressed block size threshold.</td>
+ <td>File block size of format, default value of orc stripe is 64 MB, and parquet row group is 128 MB.</td>
</tr>
<tr>
<td><h5>file.compression</h5></td>
diff --git a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java
index 20c59b1f95..9bb324b780 100644
--- a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java
+++ b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java
@@ -426,8 +426,7 @@ public class CoreOptions implements Serializable {
.memoryType()
.noDefaultValue()
.withDescription(
- "File block size of format, default value of orc stripe is 64 MB, parquet row group is 128 MB, "
- + "and avro block is 64000 bytes. For avro, this is the uncompressed block size threshold.");
+ "File block size of format, default value of orc stripe is 64 MB, and parquet row group is 128 MB.");
public static final ConfigOption<MemorySize> FILE_INDEX_IN_MANIFEST_THRESHOLD =
key("file-index.in-manifest-threshold")
diff --git a/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroFileFormat.java b/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroFileFormat.java
index a23c66af21..69512d5ee7 100644
--- a/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroFileFormat.java
+++ b/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroFileFormat.java
@@ -29,7 +29,6 @@ import org.apache.paimon.fs.CloseShieldOutputStream;
import org.apache.paimon.fs.PositionOutputStream;
import org.apache.paimon.options.ConfigOption;
import org.apache.paimon.options.ConfigOptions;
-import org.apache.paimon.options.MemorySize;
import org.apache.paimon.options.Options;
import org.apache.paimon.predicate.Predicate;
import org.apache.paimon.statistics.SimpleColStatsCollector;
@@ -66,14 +65,12 @@ public class AvroFileFormat extends FileFormat {
private final Options options;
private final int zstdLevel;
- @Nullable private final MemorySize blockSize;
public AvroFileFormat(FormatContext context) {
super(IDENTIFIER);
this.options = getIdentifierPrefixOptions(context.options());
this.zstdLevel = context.zstdLevel();
- this.blockSize = context.blockSize();
}
@Override
@@ -96,9 +93,6 @@ public class AvroFileFormat extends FileFormat {
AvroRowDatumWriter datumWriter = new AvroRowDatumWriter(rowType);
DataFileWriter<InternalRow> writer = new DataFileWriter<>(datumWriter);
writer.setCodec(createCodecFactory(compression));
- if (blockSize != null) {
- writer.setSyncInterval(Math.toIntExact(blockSize.getBytes()));
- }
writer.setFlushOnEveryBlock(false);
writer.create(schema, new CloseShieldOutputStream(out));
return new AvroBlockWriter(writer, out, schema);
diff --git a/paimon-format/src/test/java/org/apache/paimon/format/avro/AvroFileFormatTest.java b/paimon-format/src/test/java/org/apache/paimon/format/avro/AvroFileFormatTest.java
index 33b5798b5b..c7ca45b88f 100644
--- a/paimon-format/src/test/java/org/apache/paimon/format/avro/AvroFileFormatTest.java
+++ b/paimon-format/src/test/java/org/apache/paimon/format/avro/AvroFileFormatTest.java
@@ -48,9 +48,6 @@ import org.apache.avro.io.EncoderFactory;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
-import org.junit.jupiter.params.ParameterizedTest;
-import org.junit.jupiter.params.provider.CsvSource;
-import org.junit.jupiter.params.provider.ValueSource;
import java.io.ByteArrayOutputStream;
import java.io.File;
@@ -79,74 +76,6 @@ public class AvroFileFormatTest {
fileFormat = new AvroFileFormat(new FormatContext(new Options(), 1024, 1024));
}
- @ParameterizedTest
- @CsvSource({
- ", 64000, null",
- ", 64000, deflate",
- "1 kb, 1024, null",
- "1 kb, 1024, deflate",
- "4 kb, 4096, null",
- "4 kb, 4096, deflate",
- "128 kb, 131072, null",
- "128 kb, 131072, deflate"
- })
- void testFileBlockSize(String blockSize, int expectedBlockSize, String compression)
- throws IOException {
- Options options = new Options();
- if (blockSize != null) {
- options.setString("file.block-size", blockSize);
- }
- FileFormat format = FileFormat.fromIdentifier("avro", options);
- RowType rowType = DataTypes.ROW(DataTypes.INT().notNull()).notNull();
- LocalFileIO fileIO = LocalFileIO.create();
- Path file = new Path(new Path(tempPath.toUri()), UUID.randomUUID().toString());
- int numRecords = 300_000;
-
- try (PositionOutputStream out = fileIO.newOutputStream(file, false);
- FormatWriter writer =
- format.createWriterFactory(rowType).create(out, compression)) {
- // Each record is encoded as one byte, so record counts also give uncompressed sizes.
- for (int i = 0; i < numRecords; i++) {
- writer.addElement(GenericRow.of(0));
- }
- }
-
- long records = 0;
- try (AvroBlockReader reader = new AvroBlockReader(fileIO.newInputStream(file))) {
- while (reader.hasNextBlock()) {
- AvroRawBlock block = reader.nextBorrowedRawBlock();
- long expectedRecords = Math.min(expectedBlockSize, numRecords - records);
- assertThat(block.recordCount()).isEqualTo(expectedRecords);
- assertThat(block.decompress(null).remaining()).isEqualTo((int) expectedRecords);
- records += block.recordCount();
- }
- }
- assertThat(records).isEqualTo(numRecords);
- }
-
- @ParameterizedTest
- @ValueSource(longs = {2147483648L, 4294968320L, Long.MAX_VALUE})
- void testFileBlockSizeOverflow(long blockSize) throws IOException {
- Options options = new Options();
- options.setString("file.block-size", Long.toString(blockSize));
- FileFormat format = FileFormat.fromIdentifier("avro", options);
- RowType rowType = DataTypes.ROW(DataTypes.INT().notNull()).notNull();
- LocalFileIO fileIO = LocalFileIO.create();
- Path file = new Path(new Path(tempPath.toUri()), UUID.randomUUID().toString());
-
- try (PositionOutputStream out = fileIO.newOutputStream(file, false)) {
- assertThatThrownBy(
- () -> {
- try (FormatWriter writer =
- format.createWriterFactory(rowType).create(out, "null")) {
- writer.addElement(GenericRow.of(0));
- }
- })
- .isInstanceOf(ArithmeticException.class)
- .hasMessage("integer overflow");
- }
- }
-
@Test
public void testSupportedDataTypes() {
ArrayList<DataField> dataFields = new ArrayList<>();
(paimon) 02/20: [python] Speed up nested VARIANT reads by vectorizing offsets (#9847)
Posted by ho...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
XiaoHongbo-Hope pushed a commit to branch release-2.1
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit c1f92f1a415850f7724256631177cbab506c4f80
Author: XiaoHongbo <xi...@alibaba-inc.com>
AuthorDate: Wed Sep 16 10:29:49 2026 +0800
[python] Speed up nested VARIANT reads by vectorizing offsets (#9847)
---
.../pypaimon/read/reader/format_pyarrow_reader.py | 16 +++++------
.../tests/format_pyarrow_variant_row_group_test.py | 32 +++++++++++++++++++++-
2 files changed, 39 insertions(+), 9 deletions(-)
diff --git a/paimon-python/pypaimon/read/reader/format_pyarrow_reader.py b/paimon-python/pypaimon/read/reader/format_pyarrow_reader.py
index f3403e260b..7ce7ec1d7a 100644
--- a/paimon-python/pypaimon/read/reader/format_pyarrow_reader.py
+++ b/paimon-python/pypaimon/read/reader/format_pyarrow_reader.py
@@ -25,6 +25,7 @@ from concurrent.futures import Future
from typing import Any, Callable, Deque, Dict, Iterator, List, Optional, Set, Tuple
import pyarrow as pa
+import pyarrow.compute as pc
import pyarrow.dataset as ds
from pyarrow import RecordBatch
@@ -1125,11 +1126,10 @@ def _normalized_offsets(column):
[None, column.buffers()[1]],
offset=column.offset,
)
- raw_offsets = offsets_array.to_pylist()
- start = raw_offsets[0]
- end = raw_offsets[-1]
- offsets = [value - start for value in raw_offsets]
- for index, is_null in enumerate(column.is_null().to_pylist()):
- if is_null:
- offsets[index] = None
- return pa.array(offsets, type=offsets_array.type), start, end
+ start = offsets_array[0].as_py()
+ end = offsets_array[-1].as_py()
+ offsets = pc.subtract(offsets_array, pa.scalar(start, type=offsets_array.type))
+ if column.null_count:
+ mask = pa.concat_arrays([column.is_null(), pa.array([False])])
+ offsets = pc.if_else(mask, pa.scalar(None, type=offsets_array.type), offsets)
+ return offsets, start, end
diff --git a/paimon-python/pypaimon/tests/format_pyarrow_variant_row_group_test.py b/paimon-python/pypaimon/tests/format_pyarrow_variant_row_group_test.py
index bb0f06bc6e..ef58bcbe51 100644
--- a/paimon-python/pypaimon/tests/format_pyarrow_variant_row_group_test.py
+++ b/paimon-python/pypaimon/tests/format_pyarrow_variant_row_group_test.py
@@ -33,7 +33,10 @@ from pypaimon.data.variant_shredding import (
shredding_schema_to_arrow_type,
shred_variant_column,
)
-from pypaimon.read.reader.format_pyarrow_reader import FormatPyArrowReader
+from pypaimon.read.reader.format_pyarrow_reader import (
+ FormatPyArrowReader,
+ _normalized_offsets,
+)
from pypaimon.schema.data_types import (
ArrayType,
AtomicType,
@@ -71,6 +74,33 @@ def _drain(reader):
return rows, columns, content_keys
+class NormalizedOffsetsTest(unittest.TestCase):
+
+ def test_nested_offsets(self):
+ for arrow_type, values in [
+ (pa.list_(pa.int64()), [[1, 2], None, [], [3], None]),
+ (pa.large_list(pa.int64()), [[1, 2], None, [], [3], None]),
+ (pa.map_(pa.string(), pa.int64()),
+ [[('a', 1), ('b', 2)], None, [], [('c', 3)], None]),
+ ]:
+ array = pa.array(values, type=arrow_type)
+ for column in [array, array.slice(1, 3), array.slice(4),
+ array.slice(2, 0), pa.array([], type=arrow_type)]:
+ with self.subTest(type=arrow_type, length=len(column), offset=column.offset):
+ offset_type = pa.int64() if pa.types.is_large_list(arrow_type) else pa.int32()
+ raw = pa.Array.from_buffers(
+ offset_type, len(column) + 1,
+ [None, column.buffers()[1]], offset=column.offset).to_pylist()
+ expected = [value - raw[0] for value in raw]
+ for i, null in enumerate(column.is_null().to_pylist()):
+ if null:
+ expected[i] = None
+ offsets, start, end = _normalized_offsets(column)
+ self.assertEqual((start, end), (raw[0], raw[-1]))
+ self.assertEqual(offsets.type, offset_type)
+ self.assertEqual(offsets.to_pylist(), expected)
+
+
class VariantRowGroupReaderTest(unittest.TestCase):
def setUp(self):
(paimon) 07/20: [core] Add standalone manifest sidecar format and utilities (#9845)
Posted by ho...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
XiaoHongbo-Hope pushed a commit to branch release-2.1
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit b6d91b6fd89b31bfe20153b74867edb2c900d3b3
Author: YeJunHao <41...@users.noreply.github.com>
AuthorDate: Wed Sep 16 16:24:47 2026 +0800
[core] Add standalone manifest sidecar format and utilities (#9845)
---
docs/docs/concepts/spec/manifest.md | 236 +++++
.../org/apache/paimon/utils/DeltaVarintCodec.java | 126 +++
.../org/apache/paimon/utils/VarLengthIntUtils.java | 29 +
.../apache/paimon/utils/DeltaVarintCodecTest.java | 225 ++++
.../apache/paimon/utils/VarLengthIntUtilsTest.java | 101 ++
.../apache/paimon/manifest/ManifestSidecar.java | 942 ++++++++++++++++
.../paimon/manifest/ManifestBlockIndexTest.java | 891 ++++++++++++++++
.../paimon/manifest/ManifestSidecarTest.java | 1121 ++++++++++++++++++++
.../resources/compatibility/manifest-sidecar-v1 | Bin 0 -> 2107 bytes
9 files changed, 3671 insertions(+)
diff --git a/docs/docs/concepts/spec/manifest.md b/docs/docs/concepts/spec/manifest.md
index 6ad271bb5a..ada9baf1ce 100644
--- a/docs/docs/concepts/spec/manifest.md
+++ b/docs/docs/concepts/spec/manifest.md
@@ -63,6 +63,242 @@ skip manifests before opening them.
Each extra file belongs exclusively to one manifest. It is retained and cleaned up together with
that manifest during snapshot, tag, or changelog deletion.
+### Manifest Sidecar
+
+`ManifestSidecar` provides a binary sidecar for selecting complete Avro manifest blocks
+using independent partition, row-ID and bucket coverage. A sidecar uses the
+`<manifest-file-name>.avro.sidecar` naming convention. Readers find it through an explicit
+`.avro.sidecar` reference in the manifest metadata's `_EXTRA_FILES`, without probing a
+derived file name. The Avro schemas and `_VERSION` identifiers remain unchanged.
+
+The utility includes construction, validation, block selection and optional caching. Table
+writers and scans do not yet invoke it automatically. Callers are responsible for publishing
+sidecar references, managing file ownership, applying entry filters and reconciling ADD/DELETE
+entries after block selection. `build` reads the completed physical manifest and returns
+sidecar bytes; it does not write or publish another file.
+
+Callers decide whether to invoke `build` and `read`; these utilities have no read/write switches.
+`build` and `Builder` accept `rowIdEnabled` and `bucketEnabled` arguments for independent
+payload generation. Partition generation is always enabled,
+including the empty partition tuple for unpartitioned tables. Missing or invalid
+metadata makes only the affected block's dimension unavailable. There is no sidecar byte budget:
+construction keeps complete coverage and `read` consumes the entire file once it is opened.
+
+`read` returns null for an absent sidecar reference or an `IOException`, allowing the caller
+to fall back to the manifest. If the thread is interrupted, the I/O failure is propagated as
+`UncheckedIOException`. Other exceptions and errors propagate unchanged. `select` validates
+supplied bytes directly and reports invalid containers with `IOException`.
+
+Version 1 uses the following layout. Counts, lengths, offsets and the version use canonical
+nonnegative unsigned LEB128 varints. Counts and payload lengths are bounded by `Integer.MAX_VALUE`;
+block offsets, lengths and record counts are bounded by `Long.MAX_VALUE`. Row-ID envelope
+endpoints remain fixed-width, eight-byte big-endian longs. Encoding IDs are unsigned bytes
+with separate namespaces. The existing serialized partition tuple bytes are unchanged.
+
+```text
+magic : 4 bytes // ASCII PMSC
+formatVersion : varint // 1
+avroHeaderLength : varint
+avroHeader : bytes // original schema, codec and sync marker
+partitionCount : varint
+partitionDictionary[]
+ partitionByteLength : varint
+ partitionBytes : bytes // existing manifest BinaryRow serialization
+blockCount : varint
+blocks[] // original physical order
+ offset : varint
+ length : varint // complete encoded block, including sync marker
+ recordCount : varint
+ partitionEncoding : byte
+ if partitionEncoding != 0:
+ partitionPayloadLength : varint
+ partitionPayload : bytes
+ rowIdEncoding : byte
+ if rowIdEncoding != 0:
+ rowIdPayloadLength : varint
+ rowIdPayload : bytes
+ bucketEncoding : byte
+ if bucketEncoding != 0:
+ bucketPayloadLength : varint
+ bucketPayload : bytes
+checksum : 4 bytes // big-endian CRC32 of all preceding bytes
+```
+
+The block ID is its position. Its first entry ordinal is the sum of preceding record counts
+and is not stored. Each complete partition tuple appears once in the dictionary, including
+all its fields and nulls. The scan's partition type interprets the existing serialized tuple.
+Partition predicates are evaluated once per dictionary entry.
+
+| Dimension | Encoding | Payload |
+| --- | --- | --- |
+| Any | `0` | Unavailable; only the encoding byte is present. |
+| Partition | `1` | `intsDeltaPayload` of sorted unique dictionary IDs. |
+| Row ID | `1` | Minimum, maximum, and `intsDeltaPayload` of sorted interior interval endpoints. |
+| Bucket | `1` | Two paired `intsDeltaPayload` sequences: sorted bucket IDs and their recorded total bucket counts. |
+| Any | Other nonzero ID | Skip the declared payload length; treat only this dimension as unavailable. |
+
+Only nonzero encodings are followed by a length and payload. Payload lengths exclude the
+encoding and length fields, but include the count and other fields within the payload.
+Partition IDs and bucket pairs have positive counts no greater than the block's record count.
+Row-ID coverage contains one or more intervals; its interior endpoint count can be zero for
+a single interval. Encoding 0 represents unavailable coverage, not an empty known set.
+
+#### Integer Delta Payload
+
+Partition, row-ID and both bucket sequences share this structure:
+
+```text
+intsDeltaPayload
+ count : varint
+ deltas[count] : varint
+```
+
+The count is in `[0, Integer.MAX_VALUE]`. Nonnegative deltas use unsigned LEB128 varints,
+occupying one to nine bytes for values from 0 through
+`Long.MAX_VALUE`. Seven value bits are stored per byte, least significant group first; the
+high bit indicates another byte follows.
+Encodings use the shortest representation without padding.
+
+A nondecreasing sequence is delta-encoded from a specified base. Each value contributes one
+unsigned varint containing its difference from the preceding value. The first difference
+is relative to the base:
+
+```text
+deltas[] : varint
+value[0] = base + deltas[0]
+value[i] = value[i - 1] + deltas[i]
+```
+
+The shared `DeltaVarintCodec` utility writes the count and then each delta immediately,
+and reads values on demand using `VarLengthIntUtils`. A reader consumes exactly the declared
+number of values, leaving any following sequence available in the buffer. Callers check
+their enclosing payload boundaries. Counts, overflow and value bounds are checked without
+materializing arrays. Reads may stop early.
+
+Only the `totalBuckets` sequence uses signed differences, because totals need not increase
+when pairs are sorted by bucket. Its differences use ZigZag before unsigned varint encoding:
+`encoded = (delta << 1) ^ (delta >> 63)` and
+`delta = (encoded >>> 1) ^ -(encoded & 1)`. Values are nonnegative ints, so encoded deltas
+are at most `2 * Integer.MAX_VALUE` and require at most five bytes. The field defines this
+signed mode; no additional mode byte is stored. Other sequences use nonnegative differences.
+
+#### Partition Payload
+
+When `partitionEncoding == 1`, the block stores IDs of all distinct partition tuples
+represented by its entries:
+
+```text
+partitionPayload
+ intsDeltaPayload // dictionary IDs, base = 0
+```
+
+An ID is the zero-based position of a complete tuple in the sidecar's shared dictionary.
+IDs satisfy `0 <= id < partitionCount` and are strictly increasing. Tuple bytes appear only
+in the dictionary and are not repeated in each block. For IDs `[0, 1, 2, 3, 4]`, the deltas
+are `[0, 1, 1, 1, 1]`. The payload contains a one-byte count of 5 followed by these five
+varint bytes: 6 bytes, or 8 bytes including the encoding and length fields.
+
+With a partition filter, a block matches if any referenced tuple matches. A tuple containing
+a null partition value still has a dictionary ID. Unpartitioned tables record the empty
+tuple. If any entry's entire partition tuple is unavailable, the block uses encoding 0,
+so a dictionary miss cannot exclude that block. Later blocks can still use existing IDs.
+
+#### Row-ID Payload
+
+The writer merges overlapping and adjacent inclusive intervals contributed by entries.
+An entry contributes `[firstRowId, firstRowId + rowCount - 1]`. The resulting intervals are
+sorted and disjoint; they are never expanded into individual row IDs or coarsened to include gaps.
+
+```text
+rowIdPayload
+ minRowId : long // first interval's start
+ maxRowId : long // last interval's inclusive end
+ intsDeltaPayload // stores longs: 2 * (N - 1) sorted interior endpoints, base = minRowId
+```
+
+The envelope satisfies `0 <= minRowId <= maxRowId <= Long.MAX_VALUE`. Flatten the intervals
+as `[start0, end0, start1, end1, ...]`. The first start is supplied by `minRowId`, and the last
+end by `maxRowId`; only the remaining `2 * (N - 1)` interior endpoints are delta/varint encoded.
+Their count must be even, and the derived interval count `N = count / 2 + 1` must not exceed
+the block's record count. There is no separate stored interval count.
+Pairing the reconstructed endpoints recovers the intervals. Each pair satisfies
+`0 <= start <= end <= Long.MAX_VALUE`; each following start must exceed the preceding end.
+
+For `[(10, 19), (30, 39)]`, the minimum is 10 and maximum is 39. The interior
+endpoints `[19, 30]` have deltas `[9, 11]` from base 10, each encoded as one varint byte.
+The payload starts with an eight-byte minimum of 10 and an eight-byte maximum of 39,
+followed by `intsDeltaPayload` bytes `[2, 9, 11]`: 19 bytes, or 21 bytes with framing.
+For a single interval, the two eight-byte endpoints and a one-byte zero count define the
+interval: 17 payload bytes and no deltas.
+
+The reader first tests the envelope without decoding any deltas. A query for row ID 25
+passes the example's envelope check but matches neither interval. Unknown or invalid row-ID
+metadata makes that block's row-ID payload unavailable; partition and bucket coverage remain usable.
+
+#### Bucket Payload
+
+When `bucketEncoding == 1`, the block stores distinct bucket/count pairs:
+
+```text
+bucketPayload
+ buckets : intsDeltaPayload // N > 0, base = 0, nonnegative deltas
+ totalBuckets : intsDeltaPayload // N values, base = 0, ZigZag signed deltas
+```
+
+Pairs are sorted first by bucket, then by total bucket count, with duplicates removed.
+Each pair satisfies `0 <= bucket < totalBuckets <= Integer.MAX_VALUE`. Both sequences have
+the same count, and values at the same position form one pair. The totals must not be sorted
+independently. The same bucket may occur with different totals after rescaling.
+
+For `[(1, 4), (1, 8), (3, 4)]`, bucket values `[1, 1, 3]` have deltas `[1, 0, 2]`.
+Paired totals `[4, 8, 4]` have signed deltas `[4, 4, -4]`, ZigZag-encoded as `[8, 8, 7]`.
+The two payloads are `[3, 1, 0, 2]` and `[3, 8, 8, 7]`: 8 bytes total, or 10 bytes with framing.
+
+Missing, invalid or negative/synthetic bucket metadata makes the block's bucket coverage
+unavailable. A caller can supply a predicate on `(bucket, totalBuckets)` which conservatively
+retains every potentially matching pair. Filters requiring an entry's partition belong at
+the entry-filtering stage; omit the bucket predicate if no safe check is available.
+
+#### Validation and Reading
+
+Readers validate the checksum, container fields, payload lengths and leading count headers,
+and the complete physical block directory regardless of the query. Byte spans must cover
+the whole original manifest after its header; record counts must sum to the manifest entry
+count. Manifest length and entry count come from the supplied `ManifestFileMeta` rather
+than being duplicated in the sidecar. Unknown nonzero encodings skip their declared bytes
+without interpreting a count.
+
+Compressed contents are decoded only for dimensions needed by the filters. A row-ID envelope
+rejection skips all its deltas; a matching interval or partition ID skips remaining values.
+Bucket filtering first walks the bucket sequence to locate the paired totals without
+allocating arrays, then decodes pairs until a match. Unused totals may be skipped.
+Invalid varints, value counts, overflows,
+out-of-range values or ordering encountered while decoding invalidate the container. Delta
+contents skipped by short-circuiting are not individually validated.
+
+For conjunctive filters a block is retained only if every dimension is unavailable or matches.
+Matching tests row ID, partition, then bucket coverage. Absent filters are skipped, and a
+rejection skips the remaining dimensions. Matches in different dimensions can come from
+different entries, so entry filtering and ADD/DELETE reconciliation remain necessary.
+
+The entire sidecar is read in chunks of at most 1 MiB, including payloads unused by a query.
+There is no size-based fallback or payload dropping. Payload lengths save decoding work,
+not sidecar storage I/O. Selected compressed Avro blocks are read by byte range with adjacent
+spans coalesced and individual read requests bounded to 4 MiB. Building a sidecar does not
+modify the original manifest.
+
+`read` and `openManifest` accept an optional caller-supplied `SegmentsCache<Object>`. Complete
+sidecar bytes are keyed by their explicit `Path`. Only successful reads and selections
+populate the cache; query-specific selections are not cached. Cache entry-size limits affect
+admission only: larger sidecars are still fully read, validated and used.
+
+Selected Avro blocks also share this cache. Each entry contains one complete compressed block,
+keyed by the manifest's full path, original offset and encoded length, separately from whole-file
+keys. Different selections reuse the same blocks. Only complete reads populate the cache;
+oversized blocks stream through the read buffer. Adjacent uncached blocks fitting the buffer
+are read together and cached individually. Fully cached selections do not open the manifest.
+The cache retains its configured memory budget, entry-size limit, expiration and eviction policy.
+
## Manifest
Data manifests record **ADD** (`0`) and **DELETE** (`1`) entries. Readers reconcile these entries
diff --git a/paimon-common/src/main/java/org/apache/paimon/utils/DeltaVarintCodec.java b/paimon-common/src/main/java/org/apache/paimon/utils/DeltaVarintCodec.java
new file mode 100644
index 0000000000..9c6575bb4f
--- /dev/null
+++ b/paimon-common/src/main/java/org/apache/paimon/utils/DeltaVarintCodec.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.utils;
+
+import java.io.DataOutput;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Objects;
+
+/** Streaming count-prefixed delta/varint encoding of nonnegative integer sequences. */
+public final class DeltaVarintCodec {
+
+ private DeltaVarintCodec() {}
+
+ /** Writes a varint count followed by deltas, without buffering the sequence. */
+ public static final class Writer {
+ private final DataOutput out;
+ private final boolean signedDeltas;
+ private long remaining;
+ private long previous;
+
+ public Writer(DataOutput out, int count, long base) throws IOException {
+ this(out, count, base, false);
+ }
+
+ /** Signed deltas support non-monotonic nonnegative int values using ZigZag. */
+ public Writer(DataOutput out, int count, long base, boolean signedDeltas)
+ throws IOException {
+ if (base < 0 || count < 0 || (signedDeltas && base > Integer.MAX_VALUE)) {
+ throw new IllegalArgumentException("Invalid delta/varint count or base");
+ }
+ this.out = Objects.requireNonNull(out);
+ this.signedDeltas = signedDeltas;
+ this.remaining = count;
+ previous = base;
+ VarLengthIntUtils.encodeInt(out, count);
+ }
+
+ public void write(long value) throws IOException {
+ require(remaining > 0 && value >= 0);
+ long delta = value - previous;
+ if (signedDeltas) {
+ require(value <= Integer.MAX_VALUE);
+ delta = (delta << 1) ^ (delta >> 63);
+ } else {
+ require(delta >= 0);
+ }
+ VarLengthIntUtils.encodeLong(out, delta);
+ previous = value;
+ remaining--;
+ }
+ }
+
+ /**
+ * Reads the count and then values on demand. A complete read leaves the buffer at the next
+ * payload, allowing count-prefixed sequences to be concatenated. Reading may stop early.
+ */
+ public static final class Reader {
+ private final ByteBuffer data;
+ private final long max;
+ private final int count;
+ private final boolean signedDeltas;
+ private long remaining;
+ private long value;
+
+ public Reader(ByteBuffer data, long base, long max) throws IOException {
+ this(data, base, max, false);
+ }
+
+ public Reader(ByteBuffer data, long base, long max, boolean signedDeltas)
+ throws IOException {
+ this.data = Objects.requireNonNull(data);
+ long encodedCount = VarLengthIntUtils.decodeLong(data);
+ require(encodedCount <= Integer.MAX_VALUE && encodedCount <= data.remaining());
+ require(base >= 0 && max >= base && (!signedDeltas || max <= Integer.MAX_VALUE));
+ count = (int) encodedCount;
+ remaining = count;
+ value = base;
+ this.max = max;
+ this.signedDeltas = signedDeltas;
+ }
+
+ public int count() {
+ return count;
+ }
+
+ public boolean hasNext() {
+ return remaining > 0;
+ }
+
+ public long next() throws IOException {
+ require(remaining > 0);
+ long delta = VarLengthIntUtils.decodeLong(data);
+ if (signedDeltas) {
+ require(delta <= 2L * Integer.MAX_VALUE);
+ delta = (delta >>> 1) ^ -(delta & 1);
+ }
+ require(delta >= -value && delta <= max - value);
+ value += delta;
+ remaining--;
+ return value;
+ }
+ }
+
+ private static void require(boolean valid) throws IOException {
+ if (!valid) {
+ throw new IOException("Invalid delta/varint sequence");
+ }
+ }
+}
diff --git a/paimon-common/src/main/java/org/apache/paimon/utils/VarLengthIntUtils.java b/paimon-common/src/main/java/org/apache/paimon/utils/VarLengthIntUtils.java
index a05eb8a994..cc7a56e5c5 100644
--- a/paimon-common/src/main/java/org/apache/paimon/utils/VarLengthIntUtils.java
+++ b/paimon-common/src/main/java/org/apache/paimon/utils/VarLengthIntUtils.java
@@ -24,6 +24,7 @@ import java.io.EOFException;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
+import java.nio.ByteBuffer;
/* This file is based on source code of LongPacker from the PalDB Project (https://github.com/linkedin/PalDB), licensed by the Apache
* Software Foundation (ASF) under the Apache License, Version 2.0. See the NOTICE file distributed with this work for
@@ -82,6 +83,25 @@ public final class VarLengthIntUtils {
throw new Error("Malformed long.");
}
+ /** Decodes a canonical nonnegative long from the buffer and advances its position. */
+ public static long decodeLong(ByteBuffer in) throws IOException {
+ long value = 0;
+ for (int shift = 0; shift < 63; shift += 7) {
+ if (!in.hasRemaining()) {
+ throw new EOFException("Truncated variable-length long");
+ }
+ int b = Byte.toUnsignedInt(in.get());
+ value |= (long) (b & 0x7f) << shift;
+ if ((b & 0x80) == 0) {
+ if (shift != 0 && (b & 0x7f) == 0) {
+ throw new IOException("Noncanonical variable-length long");
+ }
+ return value;
+ }
+ }
+ throw new IOException("Invalid variable-length long");
+ }
+
public static long decodeLong(byte[] ba, int index) {
long result = 0;
for (int offset = 0; offset < 64; offset += 7) {
@@ -158,6 +178,15 @@ public final class VarLengthIntUtils {
throw new Error("Malformed integer.");
}
+ /** Decodes a canonical nonnegative int from the buffer and advances its position. */
+ public static int decodeInt(ByteBuffer in) throws IOException {
+ long value = decodeLong(in);
+ if (value > Integer.MAX_VALUE) {
+ throw new IOException("Variable-length integer exceeds Integer.MAX_VALUE");
+ }
+ return (int) value;
+ }
+
public static int decodeInt(InputStream is) throws IOException {
for (int offset = 0, result = 0; offset < 32; offset += 7) {
int b = is.read();
diff --git a/paimon-common/src/test/java/org/apache/paimon/utils/DeltaVarintCodecTest.java b/paimon-common/src/test/java/org/apache/paimon/utils/DeltaVarintCodecTest.java
new file mode 100644
index 0000000000..dd667f475d
--- /dev/null
+++ b/paimon-common/src/test/java/org/apache/paimon/utils/DeltaVarintCodecTest.java
@@ -0,0 +1,225 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.utils;
+
+import org.apache.paimon.io.DataOutputSerializer;
+
+import org.junit.jupiter.api.Test;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.Random;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+/** Tests for count-prefixed streaming delta/varint payloads. */
+class DeltaVarintCodecTest {
+ @Test
+ void fixedBytesAndSlicedBuffer() throws Exception {
+ long[] values = {10, 12, 14, 16, 16, 1024};
+ byte[] encoded = encode(10, values);
+ assertThat(encoded).containsExactly(new byte[] {6, 0, 2, 2, 2, 0, (byte) 0xf0, 7});
+ ByteBuffer data = ByteBuffer.allocate(encoded.length + 4);
+ data.position(2);
+ data.put(encoded);
+ data.limit(data.position()).position(2);
+ DeltaVarintCodec.Reader reader = new DeltaVarintCodec.Reader(data, 10, 1024);
+ assertThat(reader.count()).isEqualTo(values.length);
+ for (long value : values) {
+ assertThat(reader.hasNext()).isTrue();
+ assertThat(reader.next()).isEqualTo(value);
+ }
+ assertThat(reader.hasNext()).isFalse();
+ assertThat(data.hasRemaining()).isFalse();
+ assertThatThrownBy(reader::next).isInstanceOf(IOException.class);
+ }
+
+ @Test
+ void longBoundariesAndEmptySequence() throws Exception {
+ byte[] encoded = encode(0, Long.MAX_VALUE);
+ assertThat(encoded).hasSize(10);
+ assertThat(new DeltaVarintCodec.Reader(ByteBuffer.wrap(encoded), 0, Long.MAX_VALUE).next())
+ .isEqualTo(Long.MAX_VALUE);
+ assertThat(encode(Long.MAX_VALUE - 1, Long.MAX_VALUE, Long.MAX_VALUE))
+ .containsExactly(new byte[] {2, 1, 0});
+ DeltaVarintCodec.Reader empty =
+ new DeltaVarintCodec.Reader(ByteBuffer.wrap(new byte[] {0}), 0, 0);
+ assertThat(empty.hasNext()).isFalse();
+ assertThatThrownBy(empty::next).isInstanceOf(IOException.class);
+ assertThat(encode(0)).containsExactly((byte) 0);
+ }
+
+ @Test
+ void stopsBeforeUnusedMalformedData() throws Exception {
+ DeltaVarintCodec.Reader reader =
+ new DeltaVarintCodec.Reader(
+ ByteBuffer.wrap(new byte[] {2, 0, (byte) 0x80}), 10, 100);
+ assertThat(reader.next()).isEqualTo(10);
+ assertThat(reader.hasNext()).isTrue();
+ assertThatThrownBy(reader::next).isInstanceOf(IOException.class);
+ }
+
+ @Test
+ void rejectsInvalidCountsBoundsAndOrdering() throws Exception {
+ assertThatThrownBy(() -> new DeltaVarintCodec.Reader(ByteBuffer.allocate(0), 0, 1))
+ .isInstanceOf(IOException.class);
+ assertThatThrownBy(
+ () -> new DeltaVarintCodec.Reader(ByteBuffer.wrap(new byte[] {2, 0}), 0, 1))
+ .isInstanceOf(IOException.class);
+ assertThatThrownBy(
+ () ->
+ new DeltaVarintCodec.Reader(
+ ByteBuffer.wrap(new byte[] {(byte) 0x81, 0, 0}), 0, 1))
+ .isInstanceOf(IOException.class);
+ assertThatThrownBy(() -> new DeltaVarintCodec.Reader(ByteBuffer.wrap(new byte[] {0}), 2, 1))
+ .isInstanceOf(IOException.class);
+ assertThatThrownBy(() -> new DeltaVarintCodec.Writer(new DataOutputSerializer(8), 0, -1))
+ .isInstanceOf(IllegalArgumentException.class);
+ assertThatThrownBy(() -> encode(10, 9)).isInstanceOf(IOException.class);
+ assertThatThrownBy(() -> encode(0, 2, 1)).isInstanceOf(IOException.class);
+ DeltaVarintCodec.Reader overflow =
+ new DeltaVarintCodec.Reader(
+ ByteBuffer.wrap(encode(0, Long.MAX_VALUE)), 1, Long.MAX_VALUE);
+ assertThatThrownBy(overflow::next).isInstanceOf(IOException.class);
+ }
+
+ @Test
+ void rejectsMalformedVarints() throws Exception {
+ byte[] overlong = new byte[10];
+ Arrays.fill(overlong, (byte) 0x80);
+ for (byte[] bytes :
+ Arrays.asList(new byte[] {(byte) 0x80}, new byte[] {(byte) 0x81, 0}, overlong)) {
+ byte[] payload = new byte[bytes.length + 1];
+ payload[0] = 1;
+ System.arraycopy(bytes, 0, payload, 1, bytes.length);
+ DeltaVarintCodec.Reader reader =
+ new DeltaVarintCodec.Reader(ByteBuffer.wrap(payload), 0, Long.MAX_VALUE);
+ assertThatThrownBy(reader::next).isInstanceOf(IOException.class);
+ }
+ }
+
+ @Test
+ void randomizedRoundTrips() throws Exception {
+ Random random = new Random(9845);
+ for (int trial = 0; trial < 1000; trial++) {
+ long base = random.nextInt(10000);
+ long[] values = new long[random.nextInt(128)];
+ long value = base;
+ for (int i = 0; i < values.length; i++) {
+ value += random.nextInt(10000);
+ values[i] = value;
+ }
+ DeltaVarintCodec.Reader reader =
+ new DeltaVarintCodec.Reader(ByteBuffer.wrap(encode(base, values)), base, value);
+ for (long expected : values) {
+ assertThat(reader.next()).isEqualTo(expected);
+ }
+ assertThat(reader.hasNext()).isFalse();
+ }
+ }
+
+ private static byte[] encode(long base, long... values) throws IOException {
+ DataOutputSerializer out = new DataOutputSerializer(32);
+ DeltaVarintCodec.Writer writer = new DeltaVarintCodec.Writer(out, values.length, base);
+ for (long value : values) {
+ writer.write(value);
+ }
+ return out.getCopyOfBuffer();
+ }
+
+ @Test
+ void concatenatedPayloadsRetainTheirOwnBoundaries() throws Exception {
+ byte[] first = encode(0, 1, 1, 3);
+ byte[] second = encode(0, 4, 8, 16);
+ ByteBuffer buffer = ByteBuffer.allocate(first.length + second.length);
+ buffer.put(first).put(second).flip();
+ DeltaVarintCodec.Reader reader = new DeltaVarintCodec.Reader(buffer, 0, 3);
+ assertThat(reader.count()).isEqualTo(3);
+ assertThat(reader.next()).isEqualTo(1);
+ assertThat(reader.next()).isEqualTo(1);
+ assertThat(reader.next()).isEqualTo(3);
+ assertThat(reader.hasNext()).isFalse();
+ assertThat(buffer.position()).isEqualTo(first.length);
+ reader = new DeltaVarintCodec.Reader(buffer, 0, 16);
+ assertThat(reader.next()).isEqualTo(4);
+ assertThat(reader.next()).isEqualTo(8);
+ assertThat(reader.next()).isEqualTo(16);
+ assertThat(buffer.hasRemaining()).isFalse();
+ }
+
+ @Test
+ void signedIntDeltasPreserveDecreasingTotals() throws Exception {
+ DataOutputSerializer out = new DataOutputSerializer(32);
+ DeltaVarintCodec.Writer writer = new DeltaVarintCodec.Writer(out, 3, 0, true);
+ for (int value : new int[] {4, 8, 4}) {
+ writer.write(value);
+ }
+ assertThat(out.getCopyOfBuffer()).containsExactly(new byte[] {3, 8, 8, 7});
+ DeltaVarintCodec.Reader reader =
+ new DeltaVarintCodec.Reader(ByteBuffer.wrap(out.getCopyOfBuffer()), 0, 8, true);
+ assertThat(reader.next()).isEqualTo(4);
+ assertThat(reader.next()).isEqualTo(8);
+ assertThat(reader.next()).isEqualTo(4);
+ assertThat(reader.hasNext()).isFalse();
+ assertThatThrownBy(() -> writer.write(4)).isInstanceOf(IOException.class);
+ }
+
+ @Test
+ void signedIntBoundariesAndRandomSequences() throws Exception {
+ Random random = new Random(20260916);
+ for (int trial = 0; trial < 100; trial++) {
+ int[] values = new int[128];
+ for (int i = 0; i < values.length; i++) {
+ values[i] = random.nextInt(Integer.MAX_VALUE);
+ }
+ values[0] = Integer.MAX_VALUE;
+ values[1] = 0;
+ values[2] = Integer.MAX_VALUE;
+ DataOutputSerializer out = new DataOutputSerializer(32);
+ DeltaVarintCodec.Writer writer =
+ new DeltaVarintCodec.Writer(out, values.length, 0, true);
+ for (int value : values) {
+ writer.write(value);
+ }
+ ByteBuffer buffer = ByteBuffer.wrap(out.getCopyOfBuffer());
+ DeltaVarintCodec.Reader reader =
+ new DeltaVarintCodec.Reader(buffer, 0, Integer.MAX_VALUE, true);
+ for (int value : values) {
+ assertThat(reader.next()).isEqualTo(value);
+ }
+ assertThat(buffer.hasRemaining()).isFalse();
+ }
+ DeltaVarintCodec.Reader negative =
+ new DeltaVarintCodec.Reader(ByteBuffer.wrap(new byte[] {1, 1}), 0, 10, true);
+ assertThatThrownBy(negative::next).isInstanceOf(IOException.class);
+ DeltaVarintCodec.Reader tooLarge =
+ new DeltaVarintCodec.Reader(ByteBuffer.wrap(new byte[] {1, 22}), 0, 10, true);
+ assertThatThrownBy(tooLarge::next).isInstanceOf(IOException.class);
+ assertThatThrownBy(
+ () ->
+ new DeltaVarintCodec.Writer(
+ new DataOutputSerializer(8),
+ 0,
+ (long) Integer.MAX_VALUE + 1,
+ true))
+ .isInstanceOf(IllegalArgumentException.class);
+ }
+}
diff --git a/paimon-common/src/test/java/org/apache/paimon/utils/VarLengthIntUtilsTest.java b/paimon-common/src/test/java/org/apache/paimon/utils/VarLengthIntUtilsTest.java
new file mode 100644
index 0000000000..76a8f4f8d5
--- /dev/null
+++ b/paimon-common/src/test/java/org/apache/paimon/utils/VarLengthIntUtilsTest.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.utils;
+
+import org.apache.paimon.io.DataOutputSerializer;
+
+import org.junit.jupiter.api.Test;
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+
+import static org.apache.paimon.utils.VarLengthIntUtils.decodeInt;
+import static org.apache.paimon.utils.VarLengthIntUtils.encodeInt;
+import static org.apache.paimon.utils.VarLengthIntUtils.encodeLong;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+/** Tests for variable-length integer buffer decoding. */
+class VarLengthIntUtilsTest {
+
+ @Test
+ void decodeIntBoundariesAndBufferPosition() throws Exception {
+ int[] values = {
+ 0,
+ 1,
+ 127,
+ 128,
+ 16383,
+ 16384,
+ (1 << 21) - 1,
+ 1 << 21,
+ (1 << 28) - 1,
+ 1 << 28,
+ Integer.MAX_VALUE
+ };
+ DataOutputSerializer out = new DataOutputSerializer(64);
+ for (int value : values) {
+ encodeInt(out, value);
+ }
+ byte[] bytes = out.getCopyOfBuffer();
+ for (ByteBuffer buffer :
+ Arrays.asList(
+ ByteBuffer.allocate(bytes.length + 4),
+ ByteBuffer.allocateDirect(bytes.length + 4))) {
+ buffer.position(2);
+ buffer.put(bytes).put((byte) 42).flip();
+ buffer.position(2);
+ ByteBuffer in = buffer.slice().asReadOnlyBuffer();
+ for (int value : values) {
+ assertThat(decodeInt(in)).isEqualTo(value);
+ }
+ assertThat(in.position()).isEqualTo(bytes.length);
+ assertThat(in.get()).isEqualTo((byte) 42);
+ }
+ }
+
+ @Test
+ void decodeIntRejectsOverflow() throws Exception {
+ for (long value : new long[] {(long) Integer.MAX_VALUE + 1, 1L << 32, Long.MAX_VALUE}) {
+ DataOutputSerializer out = new DataOutputSerializer(16);
+ encodeLong(out, value);
+ assertThatThrownBy(() -> decodeInt(ByteBuffer.wrap(out.getCopyOfBuffer())))
+ .isInstanceOf(IOException.class)
+ .hasMessageContaining("Integer.MAX_VALUE");
+ }
+ }
+
+ @Test
+ void decodeIntRejectsTruncatedAndMalformedValues() {
+ for (byte[] bytes :
+ Arrays.asList(new byte[0], new byte[] {(byte) 0x80}, new byte[] {(byte) 0xff})) {
+ assertThatThrownBy(() -> decodeInt(ByteBuffer.wrap(bytes)))
+ .isInstanceOf(EOFException.class);
+ }
+ byte[] overlong = new byte[10];
+ Arrays.fill(overlong, (byte) 0x80);
+ for (byte[] bytes :
+ Arrays.asList(new byte[] {(byte) 0x80, 0}, new byte[] {(byte) 0x81, 0}, overlong)) {
+ assertThatThrownBy(() -> decodeInt(ByteBuffer.wrap(bytes)))
+ .isInstanceOf(IOException.class);
+ }
+ }
+}
diff --git a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestSidecar.java b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestSidecar.java
new file mode 100644
index 0000000000..c6ee9e9f56
--- /dev/null
+++ b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestSidecar.java
@@ -0,0 +1,942 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.manifest;
+
+import org.apache.paimon.annotation.VisibleForTesting;
+import org.apache.paimon.data.BinaryRow;
+import org.apache.paimon.data.Segments;
+import org.apache.paimon.fs.FileIO;
+import org.apache.paimon.fs.Path;
+import org.apache.paimon.fs.SeekableInputStream;
+import org.apache.paimon.partition.PartitionPredicate;
+import org.apache.paimon.types.DataField;
+import org.apache.paimon.types.RowType;
+import org.apache.paimon.utils.DeltaVarintCodec;
+import org.apache.paimon.utils.RowRangeIndex;
+import org.apache.paimon.utils.SegmentsCache;
+import org.apache.paimon.utils.SerializationUtils;
+import org.apache.paimon.utils.VarLengthIntUtils;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.annotation.Nullable;
+
+import java.io.ByteArrayOutputStream;
+import java.io.DataOutputStream;
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.UncheckedIOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.function.BiPredicate;
+import java.util.zip.CRC32;
+
+import static org.apache.paimon.utils.VarLengthIntUtils.decodeInt;
+import static org.apache.paimon.utils.VarLengthIntUtils.encodeLong;
+
+/** Independently usable partition, row-id and bucket coverage for each manifest block. */
+public final class ManifestSidecar {
+
+ public static final String SUFFIX = ".avro.sidecar";
+ private static final Logger LOG = LoggerFactory.getLogger(ManifestSidecar.class);
+ private static final int MAGIC = 0x504d5343;
+ private static final int FORMAT_VERSION = 1;
+ private static final int MIN_HEADER_BYTES = 29;
+ private static final int MIN_BLOCK_BYTES = 6;
+ private static final byte[] EMPTY = new byte[0];
+ private static final int CHECKSUM_BYTES = Integer.BYTES;
+ private static final int SIDECAR_READ_BUFFER_BYTES = 1024 * 1024;
+ private static final int BLOCK_READ_BUFFER_BYTES = 4 * 1024 * 1024;
+ private static final ProjectedManifestEntry.Projection BLOCK_INDEX_PROJECTION =
+ createBlockIndexProjection();
+
+ private ManifestSidecar() {}
+
+ public static Path path(Path manifest) {
+ return new Path(manifest.toString() + SUFFIX);
+ }
+
+ @Nullable
+ public static String fileName(ManifestFileMeta manifest) {
+ if (manifest.extraFiles() != null) {
+ for (String extraFile : manifest.extraFiles()) {
+ if (extraFile.endsWith(SUFFIX)) {
+ return extraFile;
+ }
+ }
+ }
+ return null;
+ }
+
+ /** Original file offset/length and zero-based manifest entry ordinal, not table row id. */
+ public static final class Block {
+ public final long offset;
+ public final long length;
+ public final long firstRecord;
+ public final long recordCount;
+
+ public Block(long offset, long length, long firstRecord, long recordCount) {
+ this.offset = offset;
+ this.length = length;
+ this.firstRecord = firstRecord;
+ this.recordCount = recordCount;
+ }
+ }
+
+ /** Selected blocks in original file order. Empty means the manifest can be excluded. */
+ public static final class Selection {
+ private final byte[] header;
+ private final List<Block> blocks;
+
+ private Selection(byte[] header, List<Block> blocks) {
+ this.header = header;
+ this.blocks = Collections.unmodifiableList(blocks);
+ }
+
+ public List<Block> blocks() {
+ return blocks;
+ }
+ }
+
+ /** Builds a complete block directory with independently available coverage. */
+ public static final class Builder {
+ private final boolean rowIdEnabled;
+ private final boolean bucketEnabled;
+ private final byte[] header;
+ private final TreeMap<Long, Long> ranges = new TreeMap<>();
+ private final Map<ByteBuffer, Integer> dictionary = new LinkedHashMap<>();
+ private final TreeSet<Integer> partitionIds = new TreeSet<>();
+ private final TreeSet<Long> bucketPairs = new TreeSet<>();
+ private final List<IndexedBlock> blocks = new ArrayList<>();
+ private long nextOffset;
+ private long nextRecord;
+ private Block current;
+ private long entriesInBlock;
+ private boolean rowAvailable;
+ private boolean partitionAvailable;
+ private boolean bucketAvailable;
+
+ public Builder(byte[] header, boolean rowIdEnabled, boolean bucketEnabled) {
+ this.rowIdEnabled = rowIdEnabled;
+ this.bucketEnabled = bucketEnabled;
+ this.header = Objects.requireNonNull(header);
+ nextOffset = header.length;
+ }
+
+ public void beginBlock(long offset, long length, long records) throws IOException {
+ require(current == null && offset == nextOffset && length > 0 && records > 0);
+ current = new Block(offset, length, nextRecord, records);
+ entriesInBlock = 0;
+ rowAvailable = rowIdEnabled;
+ partitionAvailable = true;
+ bucketAvailable = bucketEnabled;
+ ranges.clear();
+ partitionIds.clear();
+ bucketPairs.clear();
+ }
+
+ @VisibleForTesting
+ public void add(@Nullable Long first, long count) {
+ add(first, count, null);
+ }
+
+ @VisibleForTesting
+ public void add(@Nullable Long first, long count, @Nullable byte[] partition) {
+ add(first, count, partition, null, null);
+ }
+
+ public void add(
+ @Nullable Long first,
+ long count,
+ @Nullable byte[] partition,
+ @Nullable Integer bucket,
+ @Nullable Integer totalBuckets) {
+ if (current == null) {
+ throw new IllegalStateException("No current Avro block");
+ }
+ entriesInBlock++;
+ addPartition(partition);
+ addBucket(bucket, totalBuckets);
+ if (!rowAvailable) {
+ return;
+ }
+ if (first == null || first < 0 || count <= 0 || count - 1 > Long.MAX_VALUE - first) {
+ rowAvailable = false;
+ ranges.clear();
+ return;
+ }
+ long start = first;
+ long end = first + (count - 1);
+ Map.Entry<Long, Long> before = ranges.floorEntry(start);
+ if (before != null && before.getValue() >= start - 1) {
+ start = before.getKey();
+ end = Math.max(end, before.getValue());
+ ranges.remove(before.getKey());
+ }
+ Map.Entry<Long, Long> next;
+ while ((next = ranges.ceilingEntry(start)) != null
+ && (next.getKey() <= end || next.getKey() - end == 1)) {
+ end = Math.max(end, next.getValue());
+ ranges.remove(next.getKey());
+ }
+ ranges.put(start, end);
+ }
+
+ private void addBucket(@Nullable Integer bucket, @Nullable Integer totalBuckets) {
+ if (!bucketAvailable) {
+ return;
+ }
+ if (bucket == null || totalBuckets == null || bucket < 0 || totalBuckets <= bucket) {
+ bucketAvailable = false;
+ bucketPairs.clear();
+ return;
+ }
+ bucketPairs.add(((long) bucket << 32) | totalBuckets);
+ }
+
+ private void addPartition(@Nullable byte[] bytes) {
+ if (!partitionAvailable) {
+ return;
+ }
+ if (bytes == null) {
+ partitionAvailable = false;
+ partitionIds.clear();
+ return;
+ }
+ Integer id = dictionary.get(ByteBuffer.wrap(bytes));
+ if (id == null) {
+ id = dictionary.size();
+ dictionary.put(ByteBuffer.wrap(bytes.clone()), id);
+ }
+ partitionIds.add(id);
+ }
+
+ public void endBlock() throws IOException {
+ require(current != null && entriesInBlock == current.recordCount);
+ blocks.add(
+ new IndexedBlock(
+ current,
+ partitionAvailable
+ ? encodeValues(partitionIds, partitionIds.size())
+ : EMPTY,
+ rowAvailable ? encodeRanges() : EMPTY,
+ bucketAvailable ? encodeBuckets() : EMPTY));
+ nextOffset = Math.addExact(current.offset, current.length);
+ nextRecord = Math.addExact(current.firstRecord, current.recordCount);
+ ranges.clear();
+ partitionIds.clear();
+ bucketPairs.clear();
+ current = null;
+ }
+
+ private byte[] encodeRanges() throws IOException {
+ ByteArrayOutputStream buffer = new ByteArrayOutputStream();
+ DataOutputStream out = new DataOutputStream(buffer);
+ long min = ranges.firstKey();
+ long max = ranges.lastEntry().getValue();
+ out.writeLong(min);
+ out.writeLong(max);
+ // The envelope supplies the first start and last end. Encode only interior endpoints.
+ DeltaVarintCodec.Writer encoder =
+ new DeltaVarintCodec.Writer(out, Math.multiplyExact(ranges.size() - 1, 2), min);
+ int index = 0;
+ for (Map.Entry<Long, Long> range : ranges.entrySet()) {
+ if (index > 0) {
+ encoder.write(range.getKey());
+ }
+ if (++index < ranges.size()) {
+ encoder.write(range.getValue());
+ }
+ }
+ return buffer.toByteArray();
+ }
+
+ private byte[] encodeBuckets() throws IOException {
+ ByteArrayOutputStream buffer = new ByteArrayOutputStream();
+ DataOutputStream out = new DataOutputStream(buffer);
+ DeltaVarintCodec.Writer buckets =
+ new DeltaVarintCodec.Writer(out, bucketPairs.size(), 0);
+ for (long pair : bucketPairs) {
+ buckets.write(pair >>> 32);
+ }
+ DeltaVarintCodec.Writer totals =
+ new DeltaVarintCodec.Writer(out, bucketPairs.size(), 0, true);
+ for (long pair : bucketPairs) {
+ totals.write((int) pair);
+ }
+ return buffer.toByteArray();
+ }
+
+ public byte[] serialize(long fileSize, long entryCount) throws IOException {
+ require(current == null && nextOffset == fileSize && nextRecord == entryCount);
+ ByteArrayOutputStream buffer = new ByteArrayOutputStream();
+ DataOutputStream out = new DataOutputStream(buffer);
+ out.writeInt(MAGIC);
+ encodeLong(out, FORMAT_VERSION);
+ encodeLong(out, header.length);
+ out.write(header);
+ encodeLong(out, dictionary.size());
+ for (ByteBuffer bytes : dictionary.keySet()) {
+ encodeLong(out, bytes.remaining());
+ out.write(bytes.array());
+ }
+ encodeLong(out, blocks.size());
+ for (IndexedBlock block : blocks) {
+ encodeLong(out, block.block.offset);
+ encodeLong(out, block.block.length);
+ encodeLong(out, block.block.recordCount);
+ writePayload(out, block.partitions);
+ writePayload(out, block.rowIds);
+ writePayload(out, block.buckets);
+ }
+ CRC32 crc = new CRC32();
+ crc.update(buffer.toByteArray());
+ out.writeInt((int) crc.getValue());
+ return buffer.toByteArray();
+ }
+
+ private static byte[] encodeValues(Iterable<? extends Number> values, int count)
+ throws IOException {
+ ByteArrayOutputStream buffer = new ByteArrayOutputStream();
+ DataOutputStream out = new DataOutputStream(buffer);
+ DeltaVarintCodec.Writer encoder = new DeltaVarintCodec.Writer(out, count, 0);
+ for (Number value : values) {
+ encoder.write(value.longValue());
+ }
+ return buffer.toByteArray();
+ }
+
+ private static void writePayload(DataOutputStream out, byte[] payload) throws IOException {
+ out.writeByte(payload.length == 0 ? 0 : 1);
+ if (payload.length > 0) {
+ encodeLong(out, payload.length);
+ out.write(payload);
+ }
+ }
+ }
+
+ private static final class IndexedBlock {
+ private final Block block;
+ private final byte[] partitions;
+ private final byte[] rowIds;
+ private final byte[] buckets;
+
+ private IndexedBlock(Block block, byte[] partitions, byte[] rowIds, byte[] buckets) {
+ this.block = block;
+ this.partitions = partitions;
+ this.rowIds = rowIds;
+ this.buckets = buckets;
+ }
+ }
+
+ private static ProjectedManifestEntry.Projection createBlockIndexProjection() {
+ List<DataField> fields =
+ new ArrayList<>(
+ ProjectedManifestEntry.ROW_RANGE_PROJECTION.projectedType().getFields());
+ fields.add(ManifestEntry.MANIFEST_ROW_TYPE.getField(ManifestEntry.BUCKET));
+ fields.add(ManifestEntry.MANIFEST_ROW_TYPE.getField(ManifestEntry.TOTAL_BUCKETS));
+ return ProjectedManifestEntry.Projection.create(new RowType(false, fields));
+ }
+
+ /**
+ * Rebuild from the final physical blocks, including raw-copy and encoded rewrite paths. Callers
+ * decide whether to build a sidecar. Partition coverage is always generated.
+ */
+ public static byte[] build(
+ FileIO io,
+ Path path,
+ long size,
+ long records,
+ boolean rowIdEnabled,
+ boolean bucketEnabled)
+ throws IOException {
+ try (ManifestAvroReader reader = new ManifestAvroReader(io.newInputStream(path))) {
+ Builder builder = new Builder(reader.headerBytes(), rowIdEnabled, bucketEnabled);
+ ProjectedManifestEntry.Projection projection = BLOCK_INDEX_PROJECTION;
+ ProjectedManifestEntry entry = projection.createEntry();
+ while (reader.hasNext()) {
+ ManifestAvroReader.RawBlock block = reader.next();
+ builder.beginBlock(reader.blockOffset(), reader.blockLength(), block.recordCount());
+ ManifestAvroReader.RowIterator rows = block.toRows(projection.projectedType());
+ while (rows.hasNext()) {
+ entry.replace(rows.next());
+ builder.add(
+ rowIdEnabled ? entry.file().firstRowId() : null,
+ rowIdEnabled ? entry.file().rowCount() : 0,
+ entry.partitionBytes(),
+ bucketEnabled ? entry.bucket() : null,
+ bucketEnabled ? entry.totalBuckets() : null);
+ }
+ builder.endBlock();
+ }
+ return builder.serialize(size, records);
+ }
+ }
+
+ /** Selects blocks using row-ID coverage. A null query retains every block after validation. */
+ public static Selection select(
+ byte[] data, ManifestFileMeta manifest, @Nullable RowRangeIndex query)
+ throws IOException {
+ return select(data, manifest, query, null, null);
+ }
+
+ /** Validates framing and tests row ID, partition, then bucket coverage. */
+ public static Selection select(
+ byte[] data,
+ ManifestFileMeta manifest,
+ @Nullable RowRangeIndex query,
+ @Nullable PartitionPredicate partitionFilter,
+ @Nullable RowType partitionType)
+ throws IOException {
+ return select(data, manifest, query, partitionFilter, partitionType, null);
+ }
+
+ /**
+ * Selects blocks using independent filters. The bucket predicate must conservatively test only
+ * the bucket and recorded total bucket count; omit it if filtering requires an entry partition.
+ */
+ public static Selection select(
+ byte[] data,
+ ManifestFileMeta manifest,
+ @Nullable RowRangeIndex query,
+ @Nullable PartitionPredicate partitionFilter,
+ @Nullable RowType partitionType,
+ @Nullable BiPredicate<Integer, Integer> bucketFilter)
+ throws IOException {
+ require(data.length >= MIN_HEADER_BYTES + CHECKSUM_BYTES);
+ int limit = data.length - CHECKSUM_BYTES;
+ CRC32 crc = new CRC32();
+ crc.update(data, 0, limit);
+ require((int) crc.getValue() == ByteBuffer.wrap(data, limit, CHECKSUM_BYTES).getInt());
+ ByteBuffer in = ByteBuffer.wrap(data, 0, limit).slice();
+ require(in.getInt() == MAGIC);
+ require(decodeInt(in) == FORMAT_VERSION);
+ long entries = Math.addExact(manifest.numAddedFiles(), manifest.numDeletedFiles());
+ require(entries >= 0);
+ int headerLength = decodeInt(in);
+ require(headerLength >= 21 && headerLength <= in.remaining() - 2);
+ require(headerLength <= manifest.fileSize());
+ byte[] header = new byte[headerLength];
+ in.get(header);
+ require(header[0] == 'O' && header[1] == 'b' && header[2] == 'j' && header[3] == 1);
+ int partitions = decodeInt(in);
+ require(partitions <= in.remaining() / 13);
+ boolean[] matches = partitionFilter == null ? null : new boolean[partitions];
+ Set<ByteBuffer> unique = new java.util.HashSet<>();
+ for (int id = 0; id < partitions; id++) {
+ int length = decodeInt(in);
+ require(length >= 12 && length <= in.remaining());
+ ByteBuffer encoded = in.slice();
+ encoded.limit(length);
+ int arity = encoded.getInt(0);
+ require(arity >= 0 && 4L + ((arity + 71L) / 64) * 8 + arity * 8L <= length);
+ require(partitionType == null || arity == partitionType.getFieldCount());
+ require(unique.add(encoded.asReadOnlyBuffer()));
+ if (partitionFilter != null) {
+ byte[] bytes = new byte[length];
+ encoded.get(bytes);
+ BinaryRow partition = SerializationUtils.deserializeBinaryRow(bytes);
+ matches[id] = partitionFilter.test(partition);
+ }
+ in.position(in.position() + length);
+ }
+ int count = decodeInt(in);
+ require(count <= in.remaining() / MIN_BLOCK_BYTES);
+ long nextOffset = headerLength;
+ long firstRecord = 0;
+ List<Block> selected = new ArrayList<>();
+ for (int i = 0; i < count; i++) {
+ require(in.remaining() >= MIN_BLOCK_BYTES);
+ long offset = VarLengthIntUtils.decodeLong(in);
+ long length = VarLengthIntUtils.decodeLong(in);
+ long records = VarLengthIntUtils.decodeLong(in);
+ require(offset == nextOffset && length > 0 && length <= manifest.fileSize() - offset);
+ require(records > 0 && records <= entries - firstRecord);
+ ByteBuffer partitionPayload = payload(in);
+ ByteBuffer rowPayload = payload(in);
+ ByteBuffer bucketPayload = payload(in);
+ DeltaVarintCodec.Reader ids = null;
+ if (partitionPayload != null) {
+ ids = new DeltaVarintCodec.Reader(partitionPayload, 0, partitions - 1L);
+ require(ids.count() > 0 && ids.count() <= records && ids.count() <= partitions);
+ }
+ long min = 0;
+ long max = 0;
+ DeltaVarintCodec.Reader endpoints = null;
+ if (rowPayload != null) {
+ require(rowPayload.remaining() >= 2 * Long.BYTES + 1);
+ min = rowPayload.getLong();
+ max = rowPayload.getLong();
+ endpoints = new DeltaVarintCodec.Reader(rowPayload, min, max);
+ require(endpoints.count() % 2 == 0 && endpoints.count() / 2L < records);
+ require(endpoints.count() != 0 || !rowPayload.hasRemaining());
+ }
+ if (bucketPayload != null) {
+ ByteBuffer prefix = bucketPayload.duplicate();
+ int pairs = decodeInt(prefix);
+ require(pairs > 0 && pairs <= records && 2L * pairs + 1 <= prefix.remaining());
+ }
+ long blockFirstRecord = firstRecord;
+ nextOffset = offset + length;
+ firstRecord += records;
+
+ if (query != null && rowPayload != null) {
+ if (!query.intersects(min, max)) {
+ continue;
+ }
+ int rangeCount = endpoints.count() / 2 + 1;
+ boolean rowHit = rangeCount == 1;
+ long start = min;
+ for (int range = 0; !rowHit && range < rangeCount; range++) {
+ long end = range + 1 == rangeCount ? max : endpoints.next();
+ require(end >= start);
+ rowHit = query.intersects(start, end);
+ if (!rowHit && range + 1 < rangeCount) {
+ start = endpoints.next();
+ require(start > end);
+ }
+ require(endpoints.hasNext() || !rowPayload.hasRemaining());
+ }
+ if (!rowHit) {
+ continue;
+ }
+ }
+
+ if (partitionFilter != null && partitionPayload != null) {
+ boolean partitionHit = false;
+ long previous = -1;
+ while (!partitionHit && ids.hasNext()) {
+ long id = ids.next();
+ require(id > previous);
+ require(ids.hasNext() || !partitionPayload.hasRemaining());
+ previous = id;
+ partitionHit = matches[(int) id];
+ }
+ if (!partitionHit) {
+ continue;
+ }
+ }
+
+ if (bucketFilter != null && bucketPayload != null) {
+ // Locate the second count-prefixed sequence without allocating value arrays.
+ ByteBuffer totalsData = bucketPayload.duplicate();
+ DeltaVarintCodec.Reader directory =
+ new DeltaVarintCodec.Reader(totalsData, 0, Integer.MAX_VALUE);
+ while (directory.hasNext()) {
+ directory.next();
+ }
+ bucketPayload.limit(totalsData.position());
+ DeltaVarintCodec.Reader buckets =
+ new DeltaVarintCodec.Reader(bucketPayload, 0, Integer.MAX_VALUE);
+ DeltaVarintCodec.Reader totals =
+ new DeltaVarintCodec.Reader(totalsData, 0, Integer.MAX_VALUE, true);
+ require(totals.count() == buckets.count());
+ boolean bucketHit = false;
+ int previousBucket = -1;
+ int previousTotal = -1;
+ while (!bucketHit && buckets.hasNext()) {
+ int bucket = (int) buckets.next();
+ int totalBuckets = (int) totals.next();
+ require(totalBuckets > bucket);
+ require(
+ bucket > previousBucket
+ || (bucket == previousBucket && totalBuckets > previousTotal));
+ require(
+ buckets.hasNext()
+ || (!bucketPayload.hasRemaining()
+ && !totalsData.hasRemaining()));
+ previousBucket = bucket;
+ previousTotal = totalBuckets;
+ bucketHit = bucketFilter.test(bucket, totalBuckets);
+ }
+ if (!bucketHit) {
+ continue;
+ }
+ }
+ selected.add(new Block(offset, length, blockFirstRecord, records));
+ }
+ require(!in.hasRemaining() && nextOffset == manifest.fileSize() && firstRecord == entries);
+ return new Selection(header, selected);
+ }
+
+ /** Reads framing without expanding the compressed contents. */
+ @Nullable
+ private static ByteBuffer payload(ByteBuffer in) throws IOException {
+ require(in.hasRemaining());
+ int encoding = Byte.toUnsignedInt(in.get());
+ if (encoding == 0) {
+ return null;
+ }
+ int length = decodeInt(in);
+ require(length <= in.remaining());
+ ByteBuffer result = in.slice();
+ result.limit(length);
+ in.position(in.position() + length);
+ if (encoding != 1) {
+ return null;
+ }
+ return result;
+ }
+
+ /** Reads the complete sidecar. Null means read the original manifest. */
+ @Nullable
+ public static Selection read(
+ FileIO io, Path path, ManifestFileMeta manifest, @Nullable RowRangeIndex query) {
+ return read(io, path, manifest, query, null, null);
+ }
+
+ @Nullable
+ public static Selection read(
+ FileIO io,
+ Path path,
+ ManifestFileMeta manifest,
+ @Nullable RowRangeIndex query,
+ @Nullable PartitionPredicate partitionFilter,
+ @Nullable RowType partitionType) {
+ return read(io, path, manifest, query, partitionFilter, partitionType, null, null);
+ }
+
+ @Nullable
+ public static Selection read(
+ FileIO io,
+ Path path,
+ ManifestFileMeta manifest,
+ @Nullable RowRangeIndex query,
+ @Nullable PartitionPredicate partitionFilter,
+ @Nullable RowType partitionType,
+ @Nullable BiPredicate<Integer, Integer> bucketFilter,
+ @Nullable SegmentsCache<Object> cache) {
+ String sidecarFileName = fileName(manifest);
+ if (sidecarFileName == null) {
+ return null;
+ }
+ try {
+ Path sidecarPath = new Path(path.getParent(), sidecarFileName);
+ Segments cached = cache == null ? null : cache.getIfPresents(sidecarPath);
+ boolean cacheHit = cached instanceof ManifestSidecarSegment;
+ byte[] data =
+ cacheHit
+ ? ((ManifestSidecarSegment) cached).bytes()
+ : readBytes(io, sidecarPath);
+ Selection selection =
+ select(data, manifest, query, partitionFilter, partitionType, bucketFilter);
+ if (cache != null && !cacheHit && data.length <= cache.maxElementSize()) {
+ cache.put(sidecarPath, new ManifestSidecarSegment(data));
+ }
+ return selection;
+ } catch (IOException failure) {
+ if (Thread.currentThread().isInterrupted()) {
+ throw new UncheckedIOException(failure);
+ }
+ LOG.debug("Cannot use manifest sidecar for {}; reading manifest", path, failure);
+ return null;
+ }
+ }
+
+ /** Complete sidecar bytes stored in the shared manifest cache. */
+ static final class ManifestSidecarSegment implements Segments {
+ private final byte[] bytes;
+
+ public ManifestSidecarSegment(byte[] bytes) {
+ this.bytes = bytes;
+ }
+
+ public byte[] bytes() {
+ return bytes;
+ }
+
+ @Override
+ public long totalMemorySize() {
+ return bytes.length;
+ }
+ }
+
+ private static byte[] readBytes(FileIO io, Path path) throws IOException {
+ try (InputStream in = io.newInputStream(path)) {
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ byte[] buffer = new byte[SIDECAR_READ_BUFFER_BYTES];
+ int n;
+ while ((n = in.read(buffer, 0, buffer.length)) != -1) {
+ out.write(buffer, 0, n);
+ }
+ return out.toByteArray();
+ }
+ }
+
+ static InputStream openManifest(FileIO io, Path path, @Nullable Selection selected)
+ throws IOException {
+ return openManifest(io, path, selected, null);
+ }
+
+ static InputStream openManifest(
+ FileIO io,
+ Path path,
+ @Nullable Selection selected,
+ @Nullable SegmentsCache<Object> cache)
+ throws IOException {
+ return selected == null
+ ? io.newInputStream(path)
+ : new SelectedBlockInput(io, path, selected, cache);
+ }
+
+ /** Separates physical byte ranges from whole-file cache keys. */
+ static final class BlockCacheKey {
+ private final Path path;
+ private final long offset;
+ private final long length;
+
+ BlockCacheKey(Path path, long offset, long length) {
+ this.path = Objects.requireNonNull(path);
+ this.offset = offset;
+ this.length = length;
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (!(other instanceof BlockCacheKey)) {
+ return false;
+ }
+ BlockCacheKey that = (BlockCacheKey) other;
+ return path.equals(that.path) && offset == that.offset && length == that.length;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(path, offset, length);
+ }
+ }
+
+ /** Complete encoded Avro blocks, distinct from cached manifest entries and sidecar bytes. */
+ private static final class ManifestBlockSegment implements Segments {
+ private final byte[] bytes;
+
+ private ManifestBlockSegment(byte[] bytes) {
+ this.bytes = bytes;
+ }
+
+ @Override
+ public long totalMemorySize() {
+ return bytes.length;
+ }
+ }
+
+ /** An OCF stream comprising the original header and selected complete compressed blocks. */
+ private static final class SelectedBlockInput extends InputStream {
+ private final FileIO io;
+ private final Path path;
+ private final Selection selected;
+ @Nullable private final SegmentsCache<Object> cache;
+ @Nullable private SeekableInputStream input;
+ private boolean closed;
+ private int headerPosition;
+ private int blockPosition;
+ private long remaining;
+ private byte[] buffer;
+ private int bufferPosition;
+ private int bufferLimit;
+
+ private SelectedBlockInput(
+ FileIO io, Path path, Selection selected, @Nullable SegmentsCache<Object> cache) {
+ this.io = io;
+ this.path = path;
+ this.selected = selected;
+ this.cache = cache;
+ }
+
+ @Override
+ public int read() throws IOException {
+ ensureOpen();
+ if (headerPosition < selected.header.length) {
+ return selected.header[headerPosition++] & 255;
+ }
+ return fillBuffer() ? buffer[bufferPosition++] & 255 : -1;
+ }
+
+ @Override
+ public int read(byte[] bytes, int offset, int length) throws IOException {
+ ensureOpen();
+ if (length == 0) {
+ return 0;
+ }
+ if (headerPosition < selected.header.length) {
+ int n = Math.min(length, selected.header.length - headerPosition);
+ System.arraycopy(selected.header, headerPosition, bytes, offset, n);
+ headerPosition += n;
+ return n;
+ }
+ if (!fillBuffer()) {
+ return -1;
+ }
+ int copied = Math.min(length, bufferLimit - bufferPosition);
+ System.arraycopy(buffer, bufferPosition, bytes, offset, copied);
+ bufferPosition += copied;
+ return copied;
+ }
+
+ private boolean fillBuffer() throws IOException {
+ if (bufferPosition < bufferLimit) {
+ return true;
+ }
+ if (remaining == 0) {
+ if (blockPosition == selected.blocks.size()) {
+ return false;
+ }
+ Block next = selected.blocks.get(blockPosition);
+ if (cache != null && next.length <= cache.maxElementSize()) {
+ readCachedBlocks(next);
+ return true;
+ }
+ Block block = selected.blocks.get(blockPosition++);
+ long end = block.offset + block.length;
+ while (blockPosition < selected.blocks.size()
+ && selected.blocks.get(blockPosition).offset == end
+ && (cache == null
+ || selected.blocks.get(blockPosition).length
+ > cache.maxElementSize())) {
+ end += selected.blocks.get(blockPosition++).length;
+ }
+ seekInput(block.offset);
+ remaining = end - block.offset;
+ }
+ int requested = (int) Math.min(BLOCK_READ_BUFFER_BYTES, remaining);
+ // A previous buffer may be shared with other readers through the block cache.
+ if (cache != null || buffer == null || buffer.length < requested) {
+ buffer = new byte[requested];
+ }
+ bufferPosition = 0;
+ bufferLimit = 0;
+ readFully(buffer, requested);
+ bufferLimit = requested;
+ remaining -= requested;
+ return true;
+ }
+
+ private void readCachedBlocks(Block first) throws IOException {
+ byte[] cached = cachedBlock(first);
+ if (cached != null) {
+ blockPosition++;
+ buffer = cached;
+ } else {
+ int firstPosition = blockPosition++;
+ long end = first.offset + first.length;
+ while (blockPosition < selected.blocks.size()) {
+ Block next = selected.blocks.get(blockPosition);
+ if (next.offset != end
+ || next.length > cache.maxElementSize()
+ || end - first.offset + next.length > BLOCK_READ_BUFFER_BYTES
+ || cachedBlock(next) != null) {
+ break;
+ }
+ end += next.length;
+ blockPosition++;
+ }
+
+ byte[] bytes = new byte[(int) (end - first.offset)];
+ seekInput(first.offset);
+ readFully(bytes, bytes.length);
+ // Publish only complete reads, and use individual block keys so overlapping
+ // selections can share data even when their coalesced read spans differ.
+ int offset = 0;
+ for (int i = firstPosition; i < blockPosition; i++) {
+ Block block = selected.blocks.get(i);
+ int length = (int) block.length;
+ byte[] blockBytes =
+ length == bytes.length
+ ? bytes
+ : Arrays.copyOfRange(bytes, offset, offset + length);
+ cache.put(
+ new BlockCacheKey(path, block.offset, block.length),
+ new ManifestBlockSegment(blockBytes));
+ offset += length;
+ }
+ buffer = bytes;
+ }
+ bufferPosition = 0;
+ bufferLimit = buffer.length;
+ }
+
+ @Nullable
+ private byte[] cachedBlock(Block block) {
+ Segments cached =
+ cache.getIfPresents(new BlockCacheKey(path, block.offset, block.length));
+ if (cached instanceof ManifestBlockSegment) {
+ byte[] bytes = ((ManifestBlockSegment) cached).bytes;
+ if (bytes.length == block.length) {
+ return bytes;
+ }
+ }
+ return null;
+ }
+
+ private void seekInput(long offset) throws IOException {
+ if (input == null) {
+ input = io.newInputStream(path);
+ }
+ input.seek(offset);
+ }
+
+ private void readFully(byte[] bytes, int length) throws IOException {
+ int position = 0;
+ while (position < length) {
+ int count =
+ input.read(
+ bytes,
+ position,
+ Math.min(BLOCK_READ_BUFFER_BYTES, length - position));
+ if (count < 0) {
+ throw new EOFException("Truncated manifest block");
+ }
+ position += count;
+ }
+ }
+
+ private void ensureOpen() throws IOException {
+ if (closed) {
+ throw new IOException("Manifest stream is closed");
+ }
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (!closed) {
+ closed = true;
+ if (input != null) {
+ input.close();
+ }
+ }
+ }
+ }
+
+ private static void require(boolean valid) throws IOException {
+ if (!valid) {
+ throw new IOException("Invalid, unsupported or mismatched manifest sidecar");
+ }
+ }
+}
diff --git a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestBlockIndexTest.java b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestBlockIndexTest.java
new file mode 100644
index 0000000000..77c3cc9c9e
--- /dev/null
+++ b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestBlockIndexTest.java
@@ -0,0 +1,891 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.manifest;
+
+import org.apache.paimon.data.BinaryRow;
+import org.apache.paimon.partition.PartitionPredicate;
+import org.apache.paimon.predicate.PredicateBuilder;
+import org.apache.paimon.types.DataTypes;
+import org.apache.paimon.types.RowType;
+import org.apache.paimon.utils.Range;
+import org.apache.paimon.utils.RowRangeIndex;
+import org.apache.paimon.utils.SerializationUtils;
+import org.apache.paimon.utils.VarLengthIntUtils;
+
+import org.junit.jupiter.api.Test;
+
+import java.io.ByteArrayOutputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.function.BiPredicate;
+import java.util.zip.CRC32;
+
+import static org.apache.paimon.manifest.ManifestSidecarTest.header;
+import static org.apache.paimon.manifest.ManifestSidecarTest.meta;
+import static org.apache.paimon.manifest.ManifestSidecarTest.partition;
+import static org.apache.paimon.manifest.ManifestSidecarTest.testSidecar;
+import static org.apache.paimon.manifest.ManifestSidecarTest.verifyGoldenFile;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoInteractions;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
+
+/** Complete sidecar payload format, compression and independent filtering. */
+class ManifestBlockIndexTest {
+
+ private final RowType type = RowType.of(DataTypes.INT(), DataTypes.STRING());
+
+ private RowRangeIndex query(long point) {
+ return RowRangeIndex.create(Collections.singletonList(new Range(point, point)));
+ }
+
+ private static BiPredicate<Integer, Integer> bucketFilter(int expected) {
+ return new BiPredicate<Integer, Integer>() {
+ @Override
+ public boolean test(Integer bucket, Integer totalBuckets) {
+ return bucket == expected;
+ }
+ };
+ }
+
+ private PartitionPredicate part(int value) {
+ return PartitionPredicate.fromPredicate(type, new PredicateBuilder(type).equal(0, value));
+ }
+
+ @Test
+ void testV1GoldenFile() throws Exception {
+ int blockCount = 10;
+ int entriesPerBlock = 16;
+ int blockLength = 1024 * 1024 + 17;
+ byte[] header = header();
+ ManifestSidecar.Builder builder = new ManifestSidecar.Builder(header, true, true);
+ for (int block = 0; block < blockCount; block++) {
+ builder.beginBlock(
+ header.length + (long) block * blockLength, blockLength, entriesPerBlock);
+ long rowBase = (long) block << 40;
+ int step = block % 4 == 0 ? 16400 : 16;
+ for (int entry = 0; entry < entriesPerBlock; entry++) {
+ int p = (block * 7 + entry * 3) % 16;
+ builder.add(
+ rowBase + (long) entry * step,
+ 3,
+ partition(p, p % 7 == 0 ? null : "partition-" + p),
+ block + entry * 13,
+ entry % 2 == 0 ? 512 : 1024);
+ }
+ builder.endBlock();
+ }
+ long fileSize = header.length + (long) blockCount * blockLength;
+ long entryCount = (long) blockCount * entriesPerBlock;
+ byte[] current = builder.serialize(fileSize, entryCount);
+ assertThat(current.length).isBetween(2 * 1024, 2 * 1024 + 256);
+ byte[] data = verifyGoldenFile(current);
+ ManifestFileMeta meta = meta("manifest-golden-v1", fileSize, entryCount);
+ assertThat(partitionCount(data)).isEqualTo(16);
+ List<ManifestSidecar.Block> blocks = ManifestSidecar.select(data, meta, null).blocks();
+ assertThat(blocks).hasSize(blockCount);
+ for (int block = 0; block < blockCount; block++) {
+ ManifestSidecar.Block expected = blocks.get(block);
+ assertThat(expected.offset).isEqualTo(header.length + (long) block * blockLength);
+ assertThat(expected.length).isEqualTo(blockLength);
+ assertThat(expected.firstRecord).isEqualTo((long) block * entriesPerBlock);
+ assertThat(expected.recordCount).isEqualTo(entriesPerBlock);
+ int step = block % 4 == 0 ? 16400 : 16;
+ for (int entry = 0; entry < entriesPerBlock; entry++) {
+ long rowId = ((long) block << 40) + (long) entry * step;
+ int p = (block * 7 + entry * 3) % 16;
+ int bucket = block + entry * 13;
+ int total = entry % 2 == 0 ? 512 : 1024;
+ assertThat(
+ ManifestSidecar.select(
+ data,
+ meta,
+ query(rowId),
+ part(p),
+ type,
+ (b, t) -> b == bucket && t == total)
+ .blocks())
+ .extracting(b -> b.firstRecord)
+ .containsExactly((long) block * entriesPerBlock);
+ assertThat(ManifestSidecar.select(data, meta, query(rowId + 3)).blocks()).isEmpty();
+ }
+ }
+ assertThat(ManifestSidecar.select(data, meta, null, part(99), type).blocks()).isEmpty();
+ assertThat(
+ ManifestSidecar.select(data, meta, null, null, type, (b, t) -> t == 4096)
+ .blocks())
+ .isEmpty();
+ }
+
+ @Test
+ void partitionCoverageIsAlwaysGenerated() throws Exception {
+ byte[] header = header();
+ for (int mask = 0; mask < 4; mask++) {
+ boolean rowIdEnabled = (mask & 1) != 0;
+ boolean bucketEnabled = (mask & 2) != 0;
+ ManifestSidecar.Builder builder =
+ new ManifestSidecar.Builder(header, rowIdEnabled, bucketEnabled);
+ for (int block = 0; block < 2; block++) {
+ builder.beginBlock(header.length + block * 100L, 100, 1);
+ builder.add(100L + block * 100L, 10, partition(7 + block, "p"), 1, 4);
+ builder.endBlock();
+ }
+ byte[] data = builder.serialize(header.length + 200, 2);
+ assertThat(partitionCount(data)).isEqualTo(2);
+ for (int[] position : positions(data)) {
+ assertThat(data[position[1]]).isEqualTo((byte) 1);
+ assertThat(data[position[2]]).isEqualTo((byte) (rowIdEnabled ? 1 : 0));
+ assertThat(data[position[3]]).isEqualTo((byte) (bucketEnabled ? 1 : 0));
+ }
+ ManifestFileMeta meta = meta("m", header.length + 200, 2);
+ assertThat(ManifestSidecar.select(data, meta, query(999)).blocks())
+ .hasSize(rowIdEnabled ? 0 : 2);
+ assertThat(ManifestSidecar.select(data, meta, null, part(99), type).blocks()).isEmpty();
+ BiPredicate<Integer, Integer> buckets = bucketFilter(99);
+ assertThat(ManifestSidecar.select(data, meta, null, null, type, buckets).blocks())
+ .hasSize(bucketEnabled ? 0 : 2);
+
+ // Generation settings do not disable payloads already stored in a sidecar.
+ byte[] existing = testSidecar();
+ ManifestFileMeta existingMeta = meta("manifest-golden", header.length + 400, 7);
+ assertThat(ManifestSidecar.select(existing, existingMeta, query(999)).blocks())
+ .isEmpty();
+ assertThat(
+ ManifestSidecar.select(existing, existingMeta, null, part(99), type)
+ .blocks())
+ .isEmpty();
+ assertThat(
+ ManifestSidecar.select(
+ existing, existingMeta, null, null, type, buckets)
+ .blocks())
+ .isEmpty();
+ }
+ }
+
+ private byte[] sidecarWithoutBuckets(boolean partitioned) throws IOException {
+ byte[] a = partitioned ? partition(7, "left") : null;
+ byte[] b = partitioned ? partition(9, null) : null;
+ byte[] header = header();
+ ManifestSidecar.Builder builder = new ManifestSidecar.Builder(header, true, false);
+ builder.beginBlock(header.length, 100, 3);
+ builder.add(0L, 10, a);
+ builder.add(5L, 5, a);
+ builder.add(20L, 5, b);
+ builder.endBlock();
+ builder.beginBlock(header.length + 100, 200, 2);
+ builder.add((1L << 32) - 2, 5, b);
+ builder.add(8254058425445L, 1, a);
+ builder.endBlock();
+ builder.beginBlock(header.length + 300, 100, 2);
+ builder.add(20L, 5, a);
+ builder.add(Long.MAX_VALUE, 1, b);
+ builder.endBlock();
+ return builder.serialize(header.length + 400, 7);
+ }
+
+ @Test
+ void partitionTuplesNullsAndDerivedOrdinals() throws Exception {
+ byte[] data = testSidecar();
+ ManifestFileMeta meta = testMeta();
+ PartitionPredicate filter = spy(part(7));
+ assertThat(ManifestSidecar.select(data, meta, query(20), filter, type).blocks())
+ .extracting(block -> block.firstRecord)
+ .containsExactly(0L, 5L);
+ verify(filter, times(2)).test(any(BinaryRow.class));
+ PartitionPredicate nullFilter =
+ PartitionPredicate.fromPredicate(type, new PredicateBuilder(type).isNull(1));
+ assertThat(ManifestSidecar.select(data, meta, null, nullFilter, type).blocks()).hasSize(3);
+ assertThat(ManifestSidecar.select(data, meta, null, part(99), type).blocks()).isEmpty();
+ // Missing partition payloads cannot be pruned by dictionary misses.
+ assertThat(
+ ManifestSidecar.select(
+ sidecarWithoutBuckets(false), meta, null, part(99), type)
+ .blocks())
+ .hasSize(3);
+ }
+
+ @Test
+ void unavailableDimensionsAreIndependentAndDoNotPoisonLaterBlocks() throws Exception {
+ byte[] header = header();
+ ManifestSidecar.Builder builder = new ManifestSidecar.Builder(header, true, true);
+ builder.beginBlock(header.length, 100, 1);
+ builder.add(null, 10, partition(7, "left"));
+ builder.endBlock();
+ builder.beginBlock(header.length + 100, 100, 1);
+ builder.add(200L, 10, null);
+ builder.endBlock();
+ builder.beginBlock(header.length + 200, 100, 1);
+ builder.add(300L, 10, partition(7, "left")); // an existing dictionary ID remains usable
+ builder.endBlock();
+ byte[] data = builder.serialize(header.length + 300, 3);
+ ManifestFileMeta meta = meta("m", header.length + 300, 3);
+ assertThat(ManifestSidecar.select(data, meta, null, part(9), type).blocks())
+ .extracting(block -> block.firstRecord)
+ .containsExactly(1L);
+ assertThat(ManifestSidecar.select(data, meta, query(999), part(7), type).blocks())
+ .extracting(block -> block.firstRecord)
+ .containsExactly(0L);
+ assertThat(ManifestSidecar.select(data, meta, query(200), part(9), type).blocks())
+ .extracting(block -> block.firstRecord)
+ .containsExactly(1L);
+ }
+
+ private int partitionCount(byte[] data) throws IOException {
+ ByteBuffer in = ByteBuffer.wrap(data);
+ in.position(4);
+ VarLengthIntUtils.decodeLong(in);
+ int header = (int) VarLengthIntUtils.decodeLong(in);
+ in.position(in.position() + header);
+ return (int) VarLengthIntUtils.decodeLong(in);
+ }
+
+ private List<int[]> positions(byte[] data) throws IOException {
+ ByteBuffer in = ByteBuffer.wrap(data);
+ in.position(4);
+ VarLengthIntUtils.decodeLong(in);
+ int header = (int) VarLengthIntUtils.decodeLong(in);
+ in.position(in.position() + header);
+ int partitions = (int) VarLengthIntUtils.decodeLong(in);
+ for (int i = 0; i < partitions; i++) {
+ int length = (int) VarLengthIntUtils.decodeLong(in);
+ in.position(in.position() + length);
+ }
+ int blocks = (int) VarLengthIntUtils.decodeLong(in);
+ List<int[]> result = new ArrayList<>();
+ for (int i = 0; i < blocks; i++) {
+ int block = in.position();
+ VarLengthIntUtils.decodeLong(in);
+ VarLengthIntUtils.decodeLong(in);
+ VarLengthIntUtils.decodeLong(in);
+ int partition = skipPayload(in);
+ int row = skipPayload(in);
+ int bucket = skipPayload(in);
+ result.add(new int[] {block, partition, row, bucket});
+ }
+ return result;
+ }
+
+ private int skipPayload(ByteBuffer in) throws IOException {
+ int position = in.position();
+ if (in.get() != 0) {
+ int length = (int) VarLengthIntUtils.decodeLong(in);
+ in.position(in.position() + length);
+ }
+ return position;
+ }
+
+ private byte[] checksum(byte[] data) {
+ int limit = data.length - Integer.BYTES;
+ CRC32 crc = new CRC32();
+ crc.update(data, 0, limit);
+ ByteBuffer.wrap(data, limit, Integer.BYTES).putInt((int) crc.getValue());
+ return data;
+ }
+
+ @Test
+ void absentRowOrBucketFiltersKeepRemainingDimensions() throws Exception {
+ byte[] data = testSidecar();
+ ManifestFileMeta meta = meta("manifest-golden", header().length + 400, 7);
+ assertThat(
+ ManifestSidecar.select(data, meta, null, part(7), type, bucketFilter(1))
+ .blocks())
+ .extracting(block -> block.firstRecord)
+ .containsExactly(0L);
+ assertThat(ManifestSidecar.select(data, meta, query(20), part(7), type, null).blocks())
+ .extracting(block -> block.firstRecord)
+ .containsExactly(0L, 5L);
+ assertThat(ManifestSidecar.select(data, meta, null, null, type, null).blocks())
+ .extracting(block -> block.firstRecord)
+ .containsExactly(0L, 3L, 5L);
+ }
+
+ @Test
+ void bucketPayloadAndTotalBucketsArePreserved() throws Exception {
+ byte[] header = header();
+ byte[] data = testSidecar();
+ ManifestFileMeta meta = meta("manifest-golden", header.length + 400, 7);
+ BiPredicate<Integer, Integer> bucket = bucketFilter(1);
+ assertThat(ManifestSidecar.select(data, meta, null, null, type, bucket).blocks())
+ .extracting(block -> block.firstRecord)
+ .containsExactly(0L);
+ // Existing row-id and partition coverage remains independently usable.
+ assertThat(
+ ManifestSidecar.select(data, meta, query(0), part(7), type, bucketFilter(2))
+ .blocks())
+ .isEmpty();
+ BiPredicate<Integer, Integer> filter = (bucketId, total) -> bucketId == 2 && total == 8;
+ assertThat(ManifestSidecar.select(data, meta, null, null, type, filter).blocks())
+ .extracting(block -> block.firstRecord)
+ .containsExactly(3L);
+ // The caller can omit a bucket filter which requires an actual entry partition.
+ assertThat(ManifestSidecar.select(data, meta, null, null, type, null).blocks()).hasSize(3);
+ for (boolean partitioned : new boolean[] {false, true}) {
+ assertThat(
+ ManifestSidecar.select(
+ sidecarWithoutBuckets(partitioned),
+ meta,
+ null,
+ null,
+ type,
+ bucketFilter(99))
+ .blocks())
+ .hasSize(3);
+ }
+ }
+
+ @Test
+ void splitBucketArraysPreserveDecreasingTotalsAndRescaling() throws Exception {
+ byte[] header = header();
+ ManifestSidecar.Builder builder = new ManifestSidecar.Builder(header, false, true);
+ builder.beginBlock(header.length, 100, 5);
+ for (int[] pair : new int[][] {{3, 4}, {2, 8}, {0, Integer.MAX_VALUE}, {2, 4}, {2, 4}}) {
+ builder.add(null, 0, partition(7, "left"), pair[0], pair[1]);
+ }
+ builder.endBlock();
+ byte[] data = builder.serialize(header.length + 100, 5);
+ ManifestFileMeta meta = meta("m", header.length + 100, 5);
+ List<String> pairs = new ArrayList<>();
+ assertThat(
+ ManifestSidecar.select(
+ data,
+ meta,
+ null,
+ null,
+ type,
+ (bucket, total) -> {
+ pairs.add(bucket + ":" + total);
+ return false;
+ })
+ .blocks())
+ .isEmpty();
+ assertThat(pairs).containsExactly("0:2147483647", "2:4", "2:8", "3:4");
+ assertThat(
+ ManifestSidecar.select(
+ data,
+ meta,
+ null,
+ null,
+ type,
+ (bucket, total) -> bucket == 2 && total == 4)
+ .blocks())
+ .hasSize(1);
+ assertThat(
+ ManifestSidecar.select(
+ data,
+ meta,
+ null,
+ null,
+ type,
+ (bucket, total) -> bucket == 3 && total == 8)
+ .blocks())
+ .isEmpty();
+ }
+
+ @Test
+ void mismatchedBucketCountsAndNegativeTotalsAreRejected() throws Exception {
+ for (byte[] payload :
+ Arrays.asList(
+ new byte[] {2, 1, 0, 1, 8, 8}, // Two buckets, one total, extra bytes.
+ bucketPayload(2, new long[] {1, 0}, 1, 0), // ZigZag -1 from zero.
+ bucketPayload(2, new long[] {1, 0}, 8, 0))) { // Repeated pair.
+ byte[] data = replacePayload(testSidecar(), 0, 3, payload);
+ assertThatThrownBy(
+ () ->
+ ManifestSidecar.select(
+ data, testMeta(), null, null, type, bucketFilter(99)))
+ .isInstanceOf(IOException.class);
+ }
+ }
+
+ @Test
+ void unknownOrInvalidBucketPayloadIsUnavailable() throws Exception {
+ byte[] header = header();
+ for (Integer[] pair :
+ Arrays.asList(
+ new Integer[] {null, null},
+ new Integer[] {-1, 4},
+ new Integer[] {4, 4},
+ new Integer[] {0, 0})) {
+ ManifestSidecar.Builder builder = new ManifestSidecar.Builder(header, true, true);
+ builder.beginBlock(header.length, 100, 2);
+ builder.add(100L, 10, partition(7, "left"), 1, 4);
+ builder.add(200L, 10, partition(7, "left"), pair[0], pair[1]);
+ builder.endBlock();
+ builder.beginBlock(header.length + 100, 100, 1);
+ builder.add(300L, 10, partition(7, "left"), 1, 4);
+ builder.endBlock();
+ byte[] data = builder.serialize(header.length + 200, 3);
+ int bucket = positions(data).get(0)[3];
+ assertThat(data[bucket]).isZero();
+ assertThat(positions(data).get(1)[0]).isEqualTo(bucket + 1);
+ ManifestFileMeta meta = meta("m", header.length + 200, 3);
+ assertThat(
+ ManifestSidecar.select(data, meta, null, null, type, bucketFilter(99))
+ .blocks())
+ .extracting(block -> block.firstRecord)
+ .containsExactly(0L);
+ assertThat(
+ ManifestSidecar.select(
+ data, meta, query(999), null, type, bucketFilter(99))
+ .blocks())
+ .isEmpty();
+ }
+ }
+
+ @Test
+ void largePayloadsKeepExactCoverage() throws Exception {
+ byte[] header = header();
+ ManifestSidecar.Builder builder = new ManifestSidecar.Builder(header, true, true);
+ int blocks = 33;
+ int entriesPerBlock = 4097;
+ int entries = blocks * entriesPerBlock;
+ int blockBytes = 1024 * 1024;
+ for (int block = 0; block < blocks; block++) {
+ builder.beginBlock(
+ header.length + (long) block * blockBytes, blockBytes, entriesPerBlock);
+ for (int i = 0; i < entriesPerBlock; i++) {
+ int entry = block * entriesPerBlock + i;
+ builder.add(entry * 2L, 1, partition(entry, null), i, entriesPerBlock + 1);
+ }
+ builder.endBlock();
+ }
+ long fileSize = header.length + (long) blocks * blockBytes;
+ byte[] data = builder.serialize(fileSize, entries);
+ ManifestFileMeta meta = meta("m", fileSize, entries);
+ long last = (entries - 1L) * 2;
+ assertThat(ManifestSidecar.select(data, meta, query(last)).blocks())
+ .extracting(block -> block.firstRecord)
+ .containsExactly((blocks - 1L) * entriesPerBlock);
+ assertThat(ManifestSidecar.select(data, meta, query(last - 1)).blocks()).isEmpty();
+ assertThat(ManifestSidecar.select(data, meta, null, part(entries), type).blocks())
+ .isEmpty();
+ assertThat(
+ ManifestSidecar.select(
+ data, meta, null, null, type, bucketFilter(entriesPerBlock))
+ .blocks())
+ .isEmpty();
+ }
+
+ @Test
+ void absentPayloadsOmitLengthFieldsForEveryDimensionCombination() throws Exception {
+ byte[] header = header();
+ ManifestSidecar.Builder builder = new ManifestSidecar.Builder(header, true, true);
+ for (int mask = 0; mask < 8; mask++) {
+ builder.beginBlock(header.length + mask * 100L, 100, 1);
+ builder.add(
+ (mask & 2) == 0 ? null : 100L + mask,
+ 1,
+ (mask & 1) == 0 ? null : partition(7, "left"),
+ (mask & 4) == 0 ? null : 0,
+ (mask & 4) == 0 ? null : 1);
+ builder.endBlock();
+ }
+ byte[] data = builder.serialize(header.length + 800, 8);
+ List<int[]> positions = positions(data);
+ int[] presentSizes = {4, 19, 6};
+ for (int mask = 0; mask < 8; mask++) {
+ for (int dimension = 0; dimension < 3; dimension++) {
+ int start = positions.get(mask)[dimension + 1];
+ int end =
+ dimension < 2
+ ? positions.get(mask)[dimension + 2]
+ : mask < 7
+ ? positions.get(mask + 1)[0]
+ : data.length - Integer.BYTES;
+ boolean present = (mask & (1 << dimension)) != 0;
+ assertThat(data[start]).isEqualTo((byte) (present ? 1 : 0));
+ assertThat(end - start).isEqualTo(present ? presentSizes[dimension] : 1);
+ }
+ }
+ ManifestFileMeta meta = meta("m", header.length + 800, 8);
+ assertThat(ManifestSidecar.select(data, meta, null, part(99), type).blocks())
+ .extracting(block -> block.firstRecord)
+ .containsExactly(0L, 2L, 4L, 6L);
+ assertThat(ManifestSidecar.select(data, meta, query(999)).blocks())
+ .extracting(block -> block.firstRecord)
+ .containsExactly(0L, 1L, 4L, 5L);
+ BiPredicate<Integer, Integer> buckets = bucketFilter(99);
+ assertThat(ManifestSidecar.select(data, meta, null, null, type, buckets).blocks())
+ .extracting(block -> block.firstRecord)
+ .containsExactly(0L, 1L, 2L, 3L);
+ assertThat(ManifestSidecar.select(data, meta, query(999), part(99), type, buckets).blocks())
+ .extracting(block -> block.firstRecord)
+ .containsExactly(0L);
+ }
+
+ @Test
+ void deltaVarintsCompressSortedPayloadsWithoutCoarseningRowIds() throws Exception {
+ byte[] header = header();
+ ManifestSidecar.Builder builder = new ManifestSidecar.Builder(header, true, true);
+ int count = 10000;
+ builder.beginBlock(header.length, 100, count);
+ for (int i = count - 1; i >= 0; i--) {
+ builder.add(i * 4L, 3, partition(i, null), i, count);
+ }
+ builder.endBlock();
+ byte[] data = builder.serialize(header.length + 100, count);
+ int[] block = positions(data).get(0);
+ assertThat(payloadLength(data, block[1])).isEqualTo(2 + count);
+ assertThat(payloadLength(data, block[2])).isEqualTo(16 + 3 + 2 * (count - 1));
+ assertThat(payloadLength(data, block[3])).isEqualTo(2 * count + 6);
+ ManifestFileMeta meta = meta("m", header.length + 100, count);
+ assertThat(ManifestSidecar.select(data, meta, query(3)).blocks()).isEmpty();
+ assertThat(
+ ManifestSidecar.select(
+ data, meta, query(4), part(9999), type, bucketFilter(9999))
+ .blocks())
+ .hasSize(1);
+ }
+
+ @Test
+ void unpartitionedTablesStillRecordTheEmptyPartition() throws Exception {
+ byte[] header = header();
+ ManifestSidecar.Builder builder = new ManifestSidecar.Builder(header, false, false);
+ builder.beginBlock(header.length, 100, 1);
+ builder.add(null, 0, SerializationUtils.serializeBinaryRow(BinaryRow.EMPTY_ROW));
+ builder.endBlock();
+ byte[] data = builder.serialize(header.length + 100, 1);
+ assertThat(partitionCount(data)).isEqualTo(1);
+ int[] block = positions(data).get(0);
+ assertThat(data[block[1]]).isEqualTo((byte) 1);
+ assertThat(data[block[2]]).isZero();
+ assertThat(data[block[3]]).isZero();
+ assertThat(
+ ManifestSidecar.select(
+ data,
+ meta("m", header.length + 100, 1),
+ null,
+ null,
+ RowType.of())
+ .blocks())
+ .hasSize(1);
+ }
+
+ @Test
+ void rowMissSkipsPartitionAndBucketDecoding() throws Exception {
+ byte[] data = replacePayload(testSidecar(), 0, 1, compressedPayload(2, 999, 1));
+ data = replacePayload(data, 0, 3, bucketPayload(2, new long[] {0, 0}, 0, 0));
+ BiPredicate<Integer, Integer> buckets = mock(BiPredicate.class);
+ assertThat(
+ ManifestSidecar.select(data, testMeta(), query(15), part(7), type, buckets)
+ .blocks())
+ .isEmpty();
+ verifyNoInteractions(buckets);
+ }
+
+ @Test
+ void partitionMissSkipsBucketDecodingWithOrWithoutRowQuery() throws Exception {
+ byte[] data =
+ replacePayload(testSidecar(), 0, 3, bucketPayload(2, new long[] {0, 0}, 0, 0));
+ for (RowRangeIndex rows : Arrays.asList(null, query(0))) {
+ BiPredicate<Integer, Integer> buckets = mock(BiPredicate.class);
+ assertThat(
+ ManifestSidecar.select(data, testMeta(), rows, part(99), type, buckets)
+ .blocks())
+ .isEmpty();
+ verifyNoInteractions(buckets);
+ }
+ }
+
+ @Test
+ void absentPartitionFilterDoesNotDecodePartitionIds() throws Exception {
+ byte[] data = replacePayload(testSidecar(), 0, 1, compressedPayload(2, 999, 1));
+ BiPredicate<Integer, Integer> buckets = spy(bucketFilter(1));
+ assertThat(
+ ManifestSidecar.select(data, testMeta(), query(20), null, type, buckets)
+ .blocks())
+ .extracting(block -> block.firstRecord)
+ .containsExactly(0L);
+ verify(buckets).test(1, 4);
+ verify(buckets).test(0, 1);
+ verify(buckets).test(3, 4);
+ verifyNoMoreInteractions(buckets);
+ }
+
+ @Test
+ void matchesSkipUnusedDeltas() throws Exception {
+ byte[] partitions = replacePayload(testSidecar(), 0, 1, compressedPayload(2, 0, 999));
+ assertThat(ManifestSidecar.select(partitions, testMeta(), query(0), part(7), type).blocks())
+ .hasSize(1);
+ assertThatThrownBy(
+ () ->
+ ManifestSidecar.select(
+ partitions, testMeta(), query(0), part(99), type))
+ .isInstanceOf(IOException.class);
+ byte[] buckets =
+ replacePayload(
+ testSidecar(),
+ 0,
+ 3,
+ bucketPayload(2, new long[] {1, 0}, 8, Long.MAX_VALUE));
+ assertThat(
+ ManifestSidecar.select(
+ buckets, testMeta(), query(0), null, type, bucketFilter(1))
+ .blocks())
+ .hasSize(1);
+ assertThatThrownBy(
+ () ->
+ ManifestSidecar.select(
+ buckets,
+ testMeta(),
+ query(0),
+ null,
+ type,
+ bucketFilter(99)))
+ .isInstanceOf(IOException.class);
+
+ byte[] header = header();
+ ManifestSidecar.Builder builder = new ManifestSidecar.Builder(header, true, true);
+ builder.beginBlock(header.length, 100, 3);
+ for (long first : new long[] {0, 20, 40}) {
+ builder.add(first, 10);
+ }
+ builder.endBlock();
+ byte[] rows =
+ replacePayload(
+ builder.serialize(header.length + 100, 3),
+ 0,
+ 2,
+ rowPayload(3, 0, 49, 9, 11, 9, 99));
+ ManifestFileMeta meta = meta("m", header.length + 100, 3);
+ assertThat(ManifestSidecar.select(rows, meta, query(0)).blocks()).hasSize(1);
+ assertThat(ManifestSidecar.select(rows, meta, query(20)).blocks()).hasSize(1);
+ assertThat(ManifestSidecar.select(rows, meta, query(100)).blocks()).isEmpty();
+ assertThatThrownBy(() -> ManifestSidecar.select(rows, meta, query(35)))
+ .isInstanceOf(IOException.class);
+ }
+
+ @Test
+ void malformedCompressedPayloadsFailWhenConsumed() throws Exception {
+ List<byte[]> badRows =
+ Arrays.asList(
+ rowPayload(2, 0, 24, 9), // Missing an endpoint.
+ rowPayload(2, 0, 24, 9, 0), // Overlapping intervals.
+ rowPayload(2, 0, 24, Long.MAX_VALUE, 0), // Exceeds the envelope.
+ rowPayload(2, 0, 24, 9, 11, 0), // More values than declared.
+ rowPayload(1, Long.MAX_VALUE, 1), // Reversed envelope.
+ rowPayload(1, -1, 24),
+ rowPayload(1, 0, -1),
+ Arrays.copyOf(rowPayload(1, 0, 24), 16), // Missing endpoint count.
+ rowPayload(1, 0, 24, 0)); // Unexpected value for a single interval.
+ for (byte[] payload : badRows) {
+ byte[] data = replacePayload(testSidecar(), 0, 2, payload);
+ assertThatThrownBy(() -> ManifestSidecar.select(data, testMeta(), query(15)))
+ .isInstanceOf(IOException.class);
+ }
+ for (byte[] payload :
+ Arrays.asList(compressedPayload(2, 999, 0), compressedPayload(2, 0, 0))) {
+ byte[] data = replacePayload(testSidecar(), 0, 1, payload);
+ assertThatThrownBy(
+ () ->
+ ManifestSidecar.select(
+ data, testMeta(), query(0), part(99), type))
+ .isInstanceOf(IOException.class);
+ }
+ for (byte[] payload :
+ Arrays.asList(
+ bucketPayload(2, new long[] {0, 4}, 2, 0),
+ bucketPayload(2, new long[] {1L << 31, 4}, 8, 0),
+ bucketPayload(2, new long[] {0, 0}, 2, 0))) {
+ byte[] data = replacePayload(testSidecar(), 0, 3, payload);
+ assertThatThrownBy(
+ () ->
+ ManifestSidecar.select(
+ data,
+ testMeta(),
+ query(0),
+ null,
+ type,
+ bucketFilter(99)))
+ .isInstanceOf(IOException.class);
+ }
+ }
+
+ @Test
+ void malformedDeltaVarintsFailWhenConsumed() throws Exception {
+ byte[] overlong = new byte[10];
+ Arrays.fill(overlong, (byte) 0x80);
+ for (byte[] deltas :
+ Arrays.asList(
+ new byte[] {(byte) 0x80},
+ new byte[] {(byte) 0x80, (byte) 0x80}, // Truncated delta.
+ new byte[] {(byte) 0x81, 0, 0}, // Noncanonical delta.
+ overlong)) {
+ for (int dimension = 1; dimension <= 3; dimension++) {
+ ByteArrayOutputStream payload = new ByteArrayOutputStream();
+ payload.write(
+ dimension == 2
+ ? rowPayload(2, 0, 24)
+ : dimension == 3
+ ? bucketPayload(2, new long[] {1, 0})
+ : compressedPayload(2));
+ payload.write(deltas);
+ byte[] data = replacePayload(testSidecar(), 0, dimension, payload.toByteArray());
+ int dim = dimension;
+ assertThatThrownBy(
+ () ->
+ ManifestSidecar.select(
+ data,
+ testMeta(),
+ query(0),
+ dim == 1 ? part(99) : null,
+ type,
+ dim == 3 ? bucketFilter(99) : null))
+ .isInstanceOf(IOException.class);
+ }
+ }
+ }
+
+ @Test
+ void invalidCountFramingAndDirectoryFailEvenWhenFiltersMiss() throws Exception {
+ List<byte[]> bad =
+ Arrays.asList(
+ new byte[0],
+ new byte[] {(byte) 0x80}, // Incomplete varint count.
+ compressedPayload(-1, 1, 1),
+ compressedPayload(0, 1, 1),
+ compressedPayload(4, 1, 1),
+ compressedPayload(Integer.MAX_VALUE, 1, 1),
+ compressedPayload(1), // Count without payload data.
+ compressedPayload(2, 1));
+ for (int dimension = 1; dimension <= 3; dimension++) {
+ for (byte[] payload : bad) {
+ byte[] data = replacePayload(testSidecar(), 0, dimension, payload);
+ assertThatThrownBy(() -> ManifestSidecar.select(data, testMeta(), query(999)))
+ .isInstanceOf(IOException.class);
+ }
+ byte[] data = testSidecar();
+ int position = positions(data).get(0)[dimension];
+ Arrays.fill(data, position + 1, position + 6, (byte) 0xff);
+ checksum(data);
+ assertThatThrownBy(() -> ManifestSidecar.select(data, testMeta(), query(999)))
+ .isInstanceOf(IOException.class);
+ }
+ byte[] missingEnvelope =
+ replacePayload(testSidecar(), 0, 2, Arrays.copyOf(rowPayload(1, 0, 24), 16));
+ assertThatThrownBy(() -> ManifestSidecar.select(missingEnvelope, testMeta(), null))
+ .isInstanceOf(IOException.class);
+ byte[] data = testSidecar();
+ int block = positions(data).get(0)[0];
+ ByteBuffer directory = ByteBuffer.wrap(data);
+ directory.position(block);
+ VarLengthIntUtils.decodeLong(directory);
+ VarLengthIntUtils.decodeLong(directory);
+ data[directory.position()] = 2;
+ checksum(data);
+ assertThatThrownBy(() -> ManifestSidecar.select(data, testMeta(), query(999)))
+ .isInstanceOf(IOException.class);
+ }
+
+ @Test
+ void unknownEncodingsRemainIndependent() throws Exception {
+ for (int dimension = 1; dimension <= 3; dimension++) {
+ byte[] data = replacePayload(testSidecar(), 0, dimension, new byte[] {(byte) 0x80});
+ data[positions(data).get(0)[dimension]] = (byte) 202;
+ checksum(data);
+ assertThat(
+ ManifestSidecar.select(
+ data,
+ testMeta(),
+ query(dimension == 2 ? 999 : 0),
+ part(dimension == 1 ? 99 : 7),
+ type,
+ bucketFilter(dimension == 3 ? 99 : 1))
+ .blocks())
+ .extracting(block -> block.firstRecord)
+ .containsExactly(0L);
+ }
+ }
+
+ private ManifestFileMeta testMeta() throws Exception {
+ return meta("manifest-golden", header().length + 400, 7);
+ }
+
+ private byte[] replacePayload(byte[] data, int block, int dimension, byte[] payload)
+ throws Exception {
+ int start = positions(data).get(block)[dimension];
+ ByteBuffer in = ByteBuffer.wrap(data);
+ in.position(start);
+ skipPayload(in);
+ int end = in.position();
+ ByteArrayOutputStream buffer = new ByteArrayOutputStream();
+ DataOutputStream out = new DataOutputStream(buffer);
+ out.write(data, 0, start);
+ out.writeByte(1);
+ VarLengthIntUtils.encodeLong(out, payload.length);
+ out.write(payload);
+ out.write(data, end, data.length - end);
+ return checksum(buffer.toByteArray());
+ }
+
+ private static byte[] compressedPayload(int count, long... values) throws IOException {
+ ByteArrayOutputStream buffer = new ByteArrayOutputStream();
+ DataOutputStream out = new DataOutputStream(buffer);
+ out.write(deltaBytes(count));
+ out.write(deltaBytes(values));
+ return buffer.toByteArray();
+ }
+
+ private static byte[] rowPayload(int count, long min, long max, long... values)
+ throws IOException {
+ ByteArrayOutputStream buffer = new ByteArrayOutputStream();
+ DataOutputStream out = new DataOutputStream(buffer);
+ out.writeLong(min);
+ out.writeLong(max);
+ out.write(deltaBytes(2L * (count - 1)));
+ out.write(deltaBytes(values));
+ return buffer.toByteArray();
+ }
+
+ private static int payloadLength(byte[] data, int position) throws IOException {
+ ByteBuffer buffer = ByteBuffer.wrap(data);
+ buffer.position(position + 1);
+ return (int) VarLengthIntUtils.decodeLong(buffer);
+ }
+
+ private static byte[] bucketPayload(int count, long[] bucketDeltas, long... totalDeltas)
+ throws IOException {
+ ByteArrayOutputStream buffer = new ByteArrayOutputStream();
+ buffer.write(compressedPayload(count, bucketDeltas));
+ buffer.write(compressedPayload(count, totalDeltas));
+ return buffer.toByteArray();
+ }
+
+ private static byte[] deltaBytes(long... values) {
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ for (long value : values) {
+ while ((value & ~0x7fL) != 0) {
+ out.write((int) (value & 0x7f) | 0x80);
+ value >>>= 7;
+ }
+ out.write((int) value);
+ }
+ return out.toByteArray();
+ }
+}
diff --git a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestSidecarTest.java b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestSidecarTest.java
new file mode 100644
index 0000000000..b4a7b3e29b
--- /dev/null
+++ b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestSidecarTest.java
@@ -0,0 +1,1121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.manifest;
+
+import org.apache.paimon.data.BinaryRow;
+import org.apache.paimon.data.BinaryRowWriter;
+import org.apache.paimon.data.BinaryString;
+import org.apache.paimon.data.InternalRow;
+import org.apache.paimon.data.SingleSegments;
+import org.apache.paimon.format.FileFormat;
+import org.apache.paimon.format.avro.AvroFileFormat;
+import org.apache.paimon.fs.ByteArraySeekableStream;
+import org.apache.paimon.fs.FileIO;
+import org.apache.paimon.fs.Path;
+import org.apache.paimon.fs.local.LocalFileIO;
+import org.apache.paimon.manifest.ManifestSidecar.ManifestSidecarSegment;
+import org.apache.paimon.memory.MemorySegment;
+import org.apache.paimon.options.MemorySize;
+import org.apache.paimon.options.Options;
+import org.apache.paimon.schema.FileSystemSchemaManager;
+import org.apache.paimon.utils.CloseableIterator;
+import org.apache.paimon.utils.CompatibilityUtils;
+import org.apache.paimon.utils.IOUtils;
+import org.apache.paimon.utils.PathFactory;
+import org.apache.paimon.utils.Range;
+import org.apache.paimon.utils.RowRangeIndex;
+import org.apache.paimon.utils.SegmentsCache;
+import org.apache.paimon.utils.SerializationUtils;
+
+import org.apache.avro.io.BinaryEncoder;
+import org.apache.avro.io.EncoderFactory;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import java.io.ByteArrayOutputStream;
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.CancellationException;
+import java.util.zip.CRC32;
+
+import static org.apache.paimon.TestKeyValueGenerator.DEFAULT_PART_TYPE;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoInteractions;
+import static org.mockito.Mockito.when;
+
+/** Golden file format, physical block positions, completeness and allocation bounds. */
+class ManifestSidecarTest {
+
+ private static final String GENERATE_GOLDEN_FILES_PROPERTY =
+ "generateManifestSidecarGoldenFiles";
+
+ @TempDir java.nio.file.Path temp;
+
+ static ManifestFileMeta meta(String name, long size, long entries) {
+ ManifestFileMeta meta = mock(ManifestFileMeta.class);
+ when(meta.fileName()).thenReturn(name);
+ when(meta.fileSize()).thenReturn(size);
+ when(meta.extraFiles())
+ .thenReturn(Collections.singletonList(name + ManifestSidecar.SUFFIX));
+ when(meta.numAddedFiles()).thenReturn(entries);
+ return meta;
+ }
+
+ static byte[] readGoldenFile() throws IOException {
+ String resource = "/compatibility/manifest-sidecar-v1";
+ try (InputStream input = ManifestSidecarTest.class.getResourceAsStream(resource)) {
+ assertThat(input).as("Golden file %s", resource).isNotNull();
+ return IOUtils.readFully(input, false);
+ }
+ }
+
+ static byte[] verifyGoldenFile(byte[] current) throws IOException {
+ if (Boolean.parseBoolean(
+ System.getProperties().getProperty(GENERATE_GOLDEN_FILES_PROPERTY))) {
+ CompatibilityUtils.writeCompatibilityFile("manifest-sidecar-v1", current);
+ return current;
+ }
+ byte[] golden = readGoldenFile();
+ assertThat(current).isEqualTo(golden);
+ return golden;
+ }
+
+ static byte[] header() throws IOException {
+ ByteArrayOutputStream output = new ByteArrayOutputStream();
+ BinaryEncoder encoder = EncoderFactory.get().directBinaryEncoder(output, null);
+ encoder.writeFixed(new byte[] {'O', 'b', 'j', 1});
+ // Fix both metadata order and sync marker for deterministic golden files.
+ encoder.writeMapStart();
+ encoder.setItemCount(2);
+ encoder.startItem();
+ encoder.writeString("avro.codec");
+ encoder.writeBytes("null".getBytes(StandardCharsets.UTF_8));
+ encoder.startItem();
+ encoder.writeString("avro.schema");
+ encoder.writeBytes("\"long\"".getBytes(StandardCharsets.UTF_8));
+ encoder.writeMapEnd();
+ encoder.writeFixed(new byte[16]);
+ encoder.flush();
+ return output.toByteArray();
+ }
+
+ static byte[] partition(int p, String q) {
+ BinaryRow row = new BinaryRow(2);
+ BinaryRowWriter writer = new BinaryRowWriter(row);
+ writer.writeInt(0, p);
+ if (q == null) {
+ writer.setNullAt(1);
+ } else {
+ writer.writeString(1, BinaryString.fromString(q));
+ }
+ writer.complete();
+ return SerializationUtils.serializeBinaryRow(row);
+ }
+
+ static byte[] testSidecar() throws IOException {
+ byte[] header = header();
+ byte[] a = partition(7, "left");
+ byte[] b = partition(9, null);
+ ManifestSidecar.Builder builder = new ManifestSidecar.Builder(header, true, true);
+ builder.beginBlock(header.length, 100, 3);
+ builder.add(0L, 10, a, 1, 4);
+ builder.add(5L, 5, a, 1, 4);
+ builder.add(20L, 5, b, 1, 8);
+ builder.endBlock();
+ builder.beginBlock(header.length + 100, 200, 2);
+ builder.add((1L << 32) - 2, 5, b, 2, 4);
+ builder.add(8254058425445L, 1, a, 2, 8);
+ builder.endBlock();
+ builder.beginBlock(header.length + 300, 100, 2);
+ builder.add(20L, 5, a, 0, 1);
+ builder.add(Long.MAX_VALUE, 1, b, 3, 4);
+ builder.endBlock();
+ return builder.serialize(header.length + 400, 7);
+ }
+
+ private ManifestFileMeta testMeta() throws IOException {
+ return meta("manifest-golden", header().length + 400, 7);
+ }
+
+ @Test
+ void emptyManifestHasACompleteSidecar() throws Exception {
+ byte[] header = header();
+ byte[] data = new ManifestSidecar.Builder(header, true, true).serialize(header.length, 0);
+ assertThat(ByteBuffer.wrap(data).getInt()).isEqualTo(0x504d5343);
+ assertThat(ManifestSidecar.select(data, meta("empty", header.length, 0), null).blocks())
+ .isEmpty();
+ }
+
+ @Test
+ void crc32FooterAndTruncatedSidecars() throws Exception {
+ byte[] good = testSidecar();
+ ManifestFileMeta meta = testMeta();
+ int limit = good.length - Integer.BYTES;
+ // Independently generated CRC32, stored in big-endian byte order.
+ assertThat(ByteBuffer.wrap(good, limit, Integer.BYTES).getInt()).isEqualTo(0xdf82cd30);
+ assertThat(ManifestSidecar.select(good, meta, null).blocks()).hasSize(3);
+ for (int position = limit; position < good.length; position++) {
+ byte[] bad = good.clone();
+ bad[position] ^= 1;
+ assertThatThrownBy(() -> ManifestSidecar.select(bad, meta, null))
+ .isInstanceOf(IOException.class);
+ }
+ for (int length = 0; length < good.length; length++) {
+ byte[] truncated = Arrays.copyOf(good, length);
+ assertThatThrownBy(() -> ManifestSidecar.select(truncated, meta, null))
+ .isInstanceOf(IOException.class);
+ }
+ }
+
+ @Test
+ void buildAndReadSelectedBlocksFromPhysicalManifests() throws Exception {
+ FileIO io = LocalFileIO.create();
+ ManifestTestDataGenerator generator = ManifestTestDataGenerator.builder().build();
+ List<ManifestEntry> entries = new ArrayList<>();
+ for (int i = 0; i < 4000; i++) {
+ ManifestEntry entry = generator.next();
+ entries.add(
+ ManifestEntry.create(
+ i % 2 == 0 ? FileKind.ADD : FileKind.DELETE,
+ entry.partition(),
+ 1,
+ 4,
+ entry.file().newFirstRowId(i * 1000000L)));
+ }
+ Path sourcePath = new Path(temp.toString(), "manifest-source");
+ ManifestFileMeta source;
+ try (ManifestAvroWriter writer = writer(io, sourcePath)) {
+ writer.write(entries);
+ writer.close();
+ source = writer.result().get(0);
+ }
+ Path rewrittenPath = new Path(temp.toString(), "manifest-rewritten");
+ ManifestFileMeta rewritten;
+ try (ManifestAvroWriter writer = writer(io, rewrittenPath);
+ ManifestAvroReader reader = new ManifestAvroReader(io.newInputStream(sourcePath))) {
+ writer.writeEncodedManifest(reader, source);
+ writer.close();
+ rewritten = writer.result().get(0);
+ }
+ for (ManifestFileMeta meta : Arrays.asList(source, rewritten)) {
+ Path path = new Path(temp.toString(), meta.fileName());
+ byte[] data =
+ ManifestSidecar.build(io, path, meta.fileSize(), entries.size(), true, true);
+ ManifestSidecar.Selection selected =
+ ManifestSidecar.select(
+ data,
+ meta,
+ RowRangeIndex.create(
+ Arrays.asList(
+ new Range(1000000000L, 1000000000L),
+ new Range(3000000000L, 3000000000L))),
+ null,
+ DEFAULT_PART_TYPE,
+ (bucket, totalBuckets) -> bucket == 1 && totalBuckets == 4);
+ assertThat(selected.blocks()).hasSize(2);
+ List<ManifestEntry> expected = new ArrayList<>();
+ for (ManifestSidecar.Block block : selected.blocks()) {
+ expected.addAll(
+ entries.subList(
+ (int) block.firstRecord,
+ (int) (block.firstRecord + block.recordCount)));
+ }
+ List<ManifestEntry> actual = new ArrayList<>();
+ try (ManifestAvroReader reader =
+ new ManifestAvroReader(
+ ManifestSidecar.openManifest(io, path, selected));
+ CloseableIterator<InternalRow> rows =
+ reader.read(ManifestEntry.MANIFEST_ROW_TYPE, null, null)) {
+ ManifestEntrySerializer serializer = new ManifestEntrySerializer();
+ while (rows.hasNext()) {
+ actual.add(serializer.fromRow(rows.next()));
+ }
+ }
+ assertThat(actual).containsExactlyElementsOf(expected);
+ }
+ }
+
+ private ManifestAvroWriter writer(FileIO io, Path path) {
+ PathFactory paths = mock(PathFactory.class);
+ when(paths.newPath()).thenReturn(path);
+ return new ManifestAvroWriter(
+ io,
+ new FileSystemSchemaManager(io, new Path(temp.toUri())),
+ DEFAULT_PART_TYPE,
+ (AvroFileFormat) FileFormat.fromIdentifier("avro", new Options()),
+ new ManifestEntrySerializer(),
+ "zstd",
+ paths,
+ Long.MAX_VALUE);
+ }
+
+ @Test
+ void rowIdCoverageAndBlockOrdinals() throws Exception {
+ byte[] header = header();
+ byte[] data = testSidecar();
+ ManifestFileMeta meta = testMeta();
+ for (long point :
+ new long[] {
+ 0,
+ 9,
+ 20,
+ 24,
+ (1L << 32) - 2,
+ 1L << 32,
+ (1L << 32) + 2,
+ 8254058425445L,
+ Long.MAX_VALUE
+ }) {
+ assertThat(select(data, meta, point).blocks()).as("row %s", point).isNotEmpty();
+ }
+ for (long point :
+ new long[] {
+ 10, 19, 25, (1L << 32) - 3, (1L << 32) + 3, 8254058425444L, Long.MAX_VALUE - 1
+ }) {
+ assertThat(select(data, meta, point).blocks()).as("row %s", point).isEmpty();
+ }
+ ManifestSidecar.Selection selected = select(data, meta, 20);
+ assertThat(selected.blocks()).extracting(b -> b.firstRecord).containsExactly(0L, 5L);
+ assertThat(selected.blocks())
+ .extracting(b -> b.offset)
+ .containsExactly((long) header.length, header.length + 300L);
+ assertThat(selected.blocks()).extracting(b -> b.length).containsExactly(100L, 100L);
+
+ ManifestSidecar.Selection gap = select(data, meta, 16);
+
+ assertThat(gap.blocks()).isEmpty();
+ RowRangeIndex query =
+ RowRangeIndex.create(Arrays.asList(new Range(10, 19), new Range(25, 40)));
+ assertThat(ManifestSidecar.select(data, meta, query).blocks()).isEmpty();
+ assertThat(query.ranges()).containsExactly(new Range(10, 19), new Range(25, 40));
+ }
+
+ @Test
+ void minMaxSkipsExactIntersectionChecksAndHandlesOneInterval() throws Exception {
+ byte[] header = header();
+ ManifestSidecar.Builder builder = new ManifestSidecar.Builder(header, true, true);
+ builder.beginBlock(header.length, 100, 2);
+ builder.add(0L, 10);
+ builder.add(20L, 10);
+ builder.endBlock();
+ builder.beginBlock(header.length + 100, 100, 2);
+ builder.add(100L, 10);
+ builder.add(200L, 10);
+ builder.endBlock();
+ builder.beginBlock(header.length + 200, 100, 1);
+ builder.add(1L << 32, 10);
+ builder.endBlock();
+ byte[] data = builder.serialize(header.length + 300, 5);
+ ManifestFileMeta meta = meta("m", header.length + 300, 5);
+ RowRangeIndex outside =
+ spy(RowRangeIndex.create(Collections.singletonList(new Range(50, 59))));
+ ManifestSidecar.Selection none = ManifestSidecar.select(data, meta, outside);
+ assertThat(none.blocks()).isEmpty();
+
+ // Only the three envelopes are tested; no individual interval intersection is evaluated.
+ verify(outside, times(3)).intersects(anyLong(), anyLong());
+ verify(outside).intersects(0, 29);
+ verify(outside).intersects(100, 209);
+ verify(outside).intersects(1L << 32, (1L << 32) + 9);
+
+ RowRangeIndex one =
+ spy(
+ RowRangeIndex.create(
+ Collections.singletonList(
+ new Range((1L << 32) + 9, (1L << 32) + 9))));
+ ManifestSidecar.Selection hit = ManifestSidecar.select(data, meta, one);
+ assertThat(hit.blocks()).extracting(b -> b.firstRecord).containsExactly(4L);
+
+ // A one-interval block needs no second intersection check after its envelope matches.
+ verify(one, times(3)).intersects(anyLong(), anyLong());
+ }
+
+ @Test
+ void singleIntervalHandlesBoundariesAndAbsentQueries() throws Exception {
+ byte[] header = header();
+ for (Range range :
+ Arrays.asList(
+ new Range(0, 0),
+ new Range(42, 51),
+ new Range(Long.MAX_VALUE, Long.MAX_VALUE))) {
+ ManifestSidecar.Builder builder = new ManifestSidecar.Builder(header, true, true);
+ builder.beginBlock(header.length, 100, 1);
+ builder.add(range.from, range.to - range.from + 1);
+ builder.endBlock();
+ byte[] data = builder.serialize(header.length + 100, 1);
+ ManifestFileMeta meta = meta("m", header.length + 100, 1);
+ assertThat(ManifestSidecar.select(data, meta, null).blocks()).hasSize(1);
+ assertThat(
+ ManifestSidecar.select(
+ data,
+ meta,
+ RowRangeIndex.create(Collections.emptyList()))
+ .blocks())
+ .isEmpty();
+ for (long point : new long[] {range.from, range.to}) {
+ RowRangeIndex query =
+ spy(
+ RowRangeIndex.create(
+ Collections.singletonList(new Range(point, point))));
+ assertThat(ManifestSidecar.select(data, meta, query).blocks()).hasSize(1);
+ verify(query).intersects(range.from, range.to);
+ }
+ long missing = range.from > 0 ? range.from - 1 : range.to + 1;
+ assertThat(select(data, meta, missing).blocks()).isEmpty();
+ }
+ }
+
+ @Test
+ void hugeRangesAreNotExpandedAndInvalidCoverageRetainsBlocks() throws Exception {
+ byte[] header = header();
+ ManifestSidecar.Builder builder = new ManifestSidecar.Builder(header, true, true);
+ builder.beginBlock(header.length, 100, 2);
+ builder.add(0L, Long.MAX_VALUE);
+ builder.add(Long.MAX_VALUE, 1);
+ builder.endBlock();
+ byte[] data = builder.serialize(header.length + 100, 2);
+ assertThat(data.length).isLessThan(512);
+ assertThat(select(data, meta("m", header.length + 100, 2), Long.MAX_VALUE).blocks())
+ .hasSize(1);
+ for (Long first : Arrays.asList(null, -1L, Long.MAX_VALUE)) {
+ builder = new ManifestSidecar.Builder(header, true, true);
+ builder.beginBlock(header.length, 100, 1);
+ builder.add(first, 2);
+ builder.endBlock();
+ assertThat(
+ select(
+ builder.serialize(header.length + 100, 1),
+ meta("m", header.length + 100, 1),
+ 100)
+ .blocks())
+ .hasSize(1);
+ }
+ for (long count : new long[] {0, -1}) {
+ builder = new ManifestSidecar.Builder(header, true, true);
+ builder.beginBlock(header.length, 100, 1);
+ builder.add(0L, count);
+ builder.endBlock();
+ assertThat(
+ select(
+ builder.serialize(header.length + 100, 1),
+ meta("m", header.length + 100, 1),
+ 100)
+ .blocks())
+ .hasSize(1);
+ }
+ builder = new ManifestSidecar.Builder(header, true, true);
+ builder.beginBlock(header.length, 100, 64);
+ for (int i = 0; i < 64; i++) {
+ builder.add(i * 10L, 1);
+ }
+ builder.endBlock();
+ assertThat(
+ select(
+ builder.serialize(header.length + 100, 64),
+ meta("m", header.length + 100, 64),
+ 5)
+ .blocks())
+ .isEmpty();
+ }
+
+ @Test
+ void cacheRespectsElementThreshold() throws Exception {
+ byte[] data = testSidecar();
+ Path path = new Path(temp.toString(), "manifest-golden");
+ Path sidecar = ManifestSidecar.path(path);
+ Files.write(temp.resolve(sidecar.getName()), data);
+ ManifestFileMeta meta = testMeta();
+ FileIO io = spy(LocalFileIO.create());
+ SegmentsCache<Object> tooSmall =
+ new SegmentsCache<>(1024, MemorySize.ofMebiBytes(1), data.length - 1L, null, false);
+ assertThat(readCached(io, path, meta, tooSmall).blocks()).hasSize(2);
+ assertThat(readCached(io, path, meta, tooSmall).blocks()).hasSize(2);
+ assertThat(tooSmall.getIfPresents(sidecar)).isNull();
+ verify(io, times(2)).newInputStream(sidecar);
+
+ SegmentsCache<Object> cache =
+ new SegmentsCache<>(1024, MemorySize.ofMebiBytes(1), data.length, null, false);
+ assertThat(readCached(io, path, meta, cache).blocks()).hasSize(2);
+ assertThat(readCached(io, path, meta, cache).blocks()).hasSize(2);
+ verify(io, times(3)).newInputStream(sidecar);
+ }
+
+ @Test
+ void cachedSegmentsRequireSidecarType() throws Exception {
+ byte[] data = testSidecar();
+ Path path = new Path(temp.toString(), "manifest-golden");
+ Path sidecar = ManifestSidecar.path(path);
+ Files.write(temp.resolve(sidecar.getName()), data);
+ SegmentsCache<Object> cache =
+ new SegmentsCache<>(1024, MemorySize.ofMebiBytes(1), Long.MAX_VALUE, null, false);
+ cache.put(sidecar, new SingleSegments(MemorySegment.wrap(data), data.length));
+ FileIO io = spy(LocalFileIO.create());
+
+ assertThat(readCached(io, path, testMeta(), cache).blocks()).hasSize(2);
+ assertThat(cache.getIfPresents(sidecar)).isInstanceOf(ManifestSidecarSegment.class);
+ ManifestSidecarSegment cached = (ManifestSidecarSegment) cache.getIfPresents(sidecar);
+ assertThat(cached.bytes()).containsExactly(data);
+ assertThat(cached.totalMemorySize()).isEqualTo(data.length);
+ assertThat(readCached(io, path, testMeta(), cache).blocks()).hasSize(2);
+ verify(io, times(1)).newInputStream(sidecar);
+ }
+
+ @Test
+ void missingAndInvalidSidecarsAreNotCached() throws Exception {
+ byte[] data = testSidecar();
+ Path path = new Path(temp.toString(), "manifest-golden");
+ Path sidecar = ManifestSidecar.path(path);
+ ManifestFileMeta meta = testMeta();
+ FileIO io = spy(LocalFileIO.create());
+ SegmentsCache<Object> cache =
+ new SegmentsCache<>(1024, MemorySize.ofMebiBytes(1), Long.MAX_VALUE, null, false);
+ assertThat(readCached(io, path, meta, cache)).isNull();
+ assertThat(cache.getIfPresents(sidecar)).isNull();
+ byte[] corrupt = data.clone();
+ corrupt[0] ^= 1;
+ Files.write(temp.resolve(sidecar.getName()), corrupt);
+ assertThat(readCached(io, path, meta, cache)).isNull();
+ assertThat(cache.getIfPresents(sidecar)).isNull();
+ Files.write(temp.resolve(sidecar.getName()), data);
+ assertThat(readCached(io, path, meta, cache).blocks()).hasSize(2);
+ assertThat(readCached(io, path, meta, cache).blocks()).hasSize(2);
+ verify(io, times(3)).newInputStream(sidecar);
+ }
+
+ @Test
+ void cachedBytesPreservePerQueryCancellation() throws Exception {
+ Path path = new Path(temp.toString(), "manifest-golden");
+ Path sidecar = ManifestSidecar.path(path);
+ Files.write(temp.resolve(sidecar.getName()), testSidecar());
+ ManifestFileMeta meta = testMeta();
+ FileIO io = spy(LocalFileIO.create());
+ SegmentsCache<Object> cache =
+ new SegmentsCache<>(1024, MemorySize.ofMebiBytes(1), Long.MAX_VALUE, null, false);
+ RowRangeIndex cancelled = mock(RowRangeIndex.class);
+ when(cancelled.intersects(anyLong(), anyLong()))
+ .thenThrow(new CancellationException("cancelled"));
+ assertThatThrownBy(
+ () ->
+ ManifestSidecar.read(
+ io, path, meta, cancelled, null, null, null, cache))
+ .isInstanceOf(CancellationException.class);
+ assertThat(cache.getIfPresents(sidecar)).isNull();
+
+ assertThat(readCached(io, path, meta, cache).blocks()).hasSize(2);
+ assertThatThrownBy(
+ () ->
+ ManifestSidecar.read(
+ io, path, meta, cancelled, null, null, null, cache))
+ .isInstanceOf(CancellationException.class);
+ assertThat(readCached(io, path, meta, cache).blocks()).hasSize(2);
+ verify(io, times(2)).newInputStream(sidecar);
+ }
+
+ private ManifestSidecar.Selection readCached(
+ FileIO io, Path path, ManifestFileMeta meta, SegmentsCache<Object> cache) {
+ return ManifestSidecar.read(
+ io,
+ path,
+ meta,
+ RowRangeIndex.create(Collections.singletonList(new Range(20, 20))),
+ null,
+ null,
+ null,
+ cache);
+ }
+
+ @Test
+ void corruptMissingIncompleteAndMismatchedIndexesFallback() throws Exception {
+ Path manifest = new Path(temp.toString(), "manifest-golden");
+ java.nio.file.Path index = temp.resolve("manifest-golden" + ManifestSidecar.SUFFIX);
+ ManifestFileMeta meta = testMeta();
+ RowRangeIndex query = RowRangeIndex.create(Collections.singletonList(new Range(11, 11)));
+
+ assertThat(ManifestSidecar.read(LocalFileIO.create(), manifest, meta, query)).isNull();
+ byte[] good = testSidecar();
+ for (int position : new int[] {0, 9, 11, 15, 16, 55, 63, 67, 75, good.length - 1}) {
+ byte[] bad = good.clone();
+ bad[position] ^= 2;
+ Files.write(index, bad);
+ assertThat(ManifestSidecar.read(LocalFileIO.create(), manifest, meta, query)).isNull();
+ }
+ // A valid checksum cannot make an unsupported container version readable.
+ for (int version : new int[] {0, 2, 99}) {
+ byte[] bad = good.clone();
+ bad[4] = (byte) version;
+ int limit = bad.length - Integer.BYTES;
+ CRC32 crc = new CRC32();
+ crc.update(bad, 0, limit);
+ ByteBuffer.wrap(bad, limit, Integer.BYTES).putInt((int) crc.getValue());
+ assertThatThrownBy(() -> ManifestSidecar.select(bad, meta, query))
+ .isInstanceOf(IOException.class);
+ }
+ Files.write(index, Arrays.copyOf(good, good.length - 1));
+ assertThat(ManifestSidecar.read(LocalFileIO.create(), manifest, meta, query)).isNull();
+ Files.write(index, good);
+ assertThat(ManifestSidecar.read(LocalFileIO.create(), manifest, meta, query).blocks())
+ .isEmpty();
+ // The sidecar is bound to physical coverage, not to a particular file name.
+ assertThat(
+ ManifestSidecar.select(
+ good,
+ meta("renamed", meta.fileSize(), 7),
+ RowRangeIndex.create(
+ Collections.singletonList(new Range(20, 20))))
+ .blocks())
+ .extracting(block -> block.firstRecord)
+ .containsExactly(0L, 5L);
+ assertThatThrownBy(
+ () ->
+ ManifestSidecar.select(
+ good, meta("renamed", meta.fileSize() + 1, 7), query))
+ .isInstanceOf(IOException.class);
+ assertThatThrownBy(
+ () ->
+ ManifestSidecar.select(
+ good, meta("renamed", meta.fileSize(), 8), query))
+ .isInstanceOf(IOException.class);
+ }
+
+ @Test
+ void ioFailuresFallBackWithoutInspectingNestedExceptions() throws Exception {
+ Path path = new Path(temp.toString(), "m");
+ IOException suppressed = new IOException("read failed");
+ suppressed.addSuppressed(new CancellationException("cancelled during close"));
+ for (IOException failure :
+ Arrays.asList(
+ new IOException("read failed"),
+ new java.net.SocketTimeoutException("timeout"),
+ new java.io.InterruptedIOException("no thread interruption flag"),
+ new IOException("wrapped", new InterruptedException("interrupted")),
+ suppressed)) {
+ FileIO fileIO = mock(FileIO.class);
+ when(fileIO.newInputStream(ManifestSidecar.path(path))).thenThrow(failure);
+ assertThat(ManifestSidecar.read(fileIO, path, meta("m", 1, 1), null)).isNull();
+ assertThat(Thread.currentThread().isInterrupted()).isFalse();
+ }
+ }
+
+ @Test
+ void interruptedThreadDoesNotFallBackOnIoFailure() throws Exception {
+ Path path = new Path(temp.toString(), "m");
+ IOException failure = new IOException("read failed");
+ FileIO fileIO = mock(FileIO.class);
+ when(fileIO.newInputStream(ManifestSidecar.path(path))).thenThrow(failure);
+ try {
+ Thread.currentThread().interrupt();
+ assertThatThrownBy(() -> ManifestSidecar.read(fileIO, path, meta("m", 1, 1), null))
+ .isInstanceOf(java.io.UncheckedIOException.class)
+ .hasCauseReference(failure);
+ assertThat(Thread.currentThread().isInterrupted()).isTrue();
+ } finally {
+ Thread.interrupted();
+ }
+ }
+
+ @Test
+ void uncheckedFailuresPropagateUnchanged() throws Exception {
+ Path path = new Path(temp.toString(), "m");
+ for (Throwable failure :
+ Arrays.asList(
+ new IllegalStateException("unexpected failure"),
+ new java.io.UncheckedIOException(new IOException("wrapped I/O")),
+ new CancellationException("cancelled"),
+ new AssertionError("error"))) {
+ FileIO fileIO = mock(FileIO.class);
+ when(fileIO.newInputStream(ManifestSidecar.path(path))).thenThrow(failure);
+ assertThatThrownBy(() -> ManifestSidecar.read(fileIO, path, meta("m", 1, 1), null))
+ .isSameAs(failure);
+ }
+ }
+
+ @Test
+ void indexReadsUseBoundedBulkRequests() throws Exception {
+ byte[] header = header();
+ for (int blockCount : new int[] {5000, 25000, 131073}) {
+ ManifestSidecar.Builder builder = new ManifestSidecar.Builder(header, true, true);
+ for (int blockNumber = 0; blockNumber < blockCount; blockNumber++) {
+ builder.beginBlock(header.length + blockNumber * 100L, 100, 1);
+ builder.add((long) blockNumber, 1);
+ builder.endBlock();
+ }
+ long size = header.length + blockCount * 100L;
+ byte[] data = builder.serialize(size, blockCount);
+ ManifestFileMeta meta = meta("manifest-large", size, blockCount);
+ CountingInput stream = new CountingInput(data, Integer.MAX_VALUE);
+ Path path = new Path(temp.toString(), meta.fileName());
+ FileIO io = mock(FileIO.class);
+ when(io.newInputStream(ManifestSidecar.path(path))).thenReturn(stream);
+ ManifestSidecar.Selection actual =
+ ManifestSidecar.read(
+ io,
+ path,
+ meta,
+ RowRangeIndex.create(Collections.singletonList(new Range(0, 0))));
+ assertThat(actual.blocks()).hasSize(1);
+ assertThat(actual.blocks().get(0).offset).isEqualTo(header.length);
+ assertThat(stream.readLengths).hasSize((data.length + (1 << 20) - 1) / (1 << 20));
+ assertThat(stream.requests).allMatch(request -> request <= 1 << 20);
+ assertThat(stream.closed).isTrue();
+ }
+ }
+
+ @Test
+ void sidecarsLargerThanTheFormerDefaultLimitAreReadCompletely() throws Exception {
+ byte[] header = header();
+ BinaryRow partition = new BinaryRow(1);
+ BinaryRowWriter rowWriter = new BinaryRowWriter(partition);
+ byte[] value = new byte[17 * 1024 * 1024];
+ rowWriter.writeBinary(0, value, 0, value.length);
+ rowWriter.complete();
+ ManifestSidecar.Builder builder = new ManifestSidecar.Builder(header, true, true);
+ builder.beginBlock(header.length, 100, 1);
+ builder.add(0L, 1, SerializationUtils.serializeBinaryRow(partition));
+ builder.endBlock();
+ byte[] data = builder.serialize(header.length + 100, 1);
+ assertThat(data.length).isGreaterThan(16 * 1024 * 1024);
+ Path path = new Path(temp.toString(), "manifest-large");
+ CountingInput stream = new CountingInput(data, Integer.MAX_VALUE);
+ FileIO io = mock(FileIO.class);
+ when(io.newInputStream(ManifestSidecar.path(path))).thenReturn(stream);
+ assertThat(
+ ManifestSidecar.read(
+ io,
+ path,
+ meta("manifest-large", header.length + 100, 1),
+ null)
+ .blocks())
+ .hasSize(1);
+ assertThat(stream.readLengths.stream().mapToInt(Integer::intValue).sum())
+ .isEqualTo(data.length);
+ assertThat(stream.requests).allMatch(request -> request <= 1 << 20);
+ assertThat(stream.closed).isTrue();
+ }
+
+ @Test
+ void indexShortReadsReadTheWholeFile() throws Exception {
+ byte[] data = testSidecar();
+ Path path = new Path(temp.toString(), "manifest-golden");
+ for (int maxRead : new int[] {Integer.MAX_VALUE, 7}) {
+ CountingInput stream = new CountingInput(data, maxRead);
+ FileIO io = mock(FileIO.class);
+ when(io.newInputStream(ManifestSidecar.path(path))).thenReturn(stream);
+ ManifestSidecar.Selection actual =
+ ManifestSidecar.read(
+ io,
+ path,
+ testMeta(),
+ RowRangeIndex.create(Collections.singletonList(new Range(20, 20))));
+ assertThat(actual.blocks())
+ .extracting(block -> block.firstRecord)
+ .containsExactly(0L, 5L);
+ assertThat(stream.closed).isTrue();
+ }
+ }
+
+ @Test
+ void adjacentBlocksShareReadsForSingleByteConsumers() throws Exception {
+ byte[] header = header();
+ byte[] body = new byte[400];
+ for (int position = 0; position < body.length; position++) {
+ body[position] = (byte) position;
+ }
+ ManifestSidecar.Selection selected =
+ ManifestSidecar.select(
+ testSidecar(),
+ testMeta(),
+ RowRangeIndex.create(
+ Arrays.asList(
+ new Range(0, 0),
+ new Range(8254058425445L, 8254058425445L))));
+ byte[] manifest = Arrays.copyOf(header, header.length + body.length);
+ System.arraycopy(body, 0, manifest, header.length, body.length);
+ CountingInput stream = new CountingInput(manifest, Integer.MAX_VALUE);
+ Path path = new Path(temp.toString(), "manifest-golden");
+ FileIO io = mock(FileIO.class);
+ when(io.newInputStream(path)).thenReturn(stream);
+ ByteArrayOutputStream actual = new ByteArrayOutputStream();
+ try (InputStream input = ManifestSidecar.openManifest(io, path, selected)) {
+ int value;
+ while ((value = input.read()) != -1) {
+ actual.write(value);
+ }
+ assertThat(input.read(new byte[1], 0, 0)).isZero();
+ }
+ assertThat(actual.toByteArray()).isEqualTo(Arrays.copyOf(manifest, header.length + 300));
+ assertThat(stream.readLengths).containsExactly(300);
+ assertThat(stream.seeks).containsExactly((long) header.length);
+ assertThat(stream.closed).isTrue();
+ }
+
+ @Test
+ void blockReadsSkipGapsAndEmptySelections() throws Exception {
+ byte[] header = header();
+ byte[] manifest = Arrays.copyOf(header, header.length + 400);
+ Arrays.fill(manifest, header.length + 100, header.length + 300, (byte) 7);
+ Path path = new Path(temp.toString(), "manifest-golden");
+ for (long point : new long[] {20, 16}) {
+ CountingInput stream = new CountingInput(manifest, Integer.MAX_VALUE);
+ FileIO io = mock(FileIO.class);
+ when(io.newInputStream(path)).thenReturn(stream);
+ byte[] actual;
+ try (InputStream input =
+ ManifestSidecar.openManifest(
+ io, path, select(testSidecar(), testMeta(), point))) {
+ actual = IOUtils.readFully(input, false);
+ }
+ if (point == 20) {
+ assertThat(actual).isEqualTo(Arrays.copyOf(header, header.length + 200));
+ assertThat(stream.readLengths).containsExactly(100, 100);
+ assertThat(stream.seeks)
+ .containsExactly((long) header.length, header.length + 300L);
+ } else {
+ assertThat(actual).isEqualTo(header);
+ assertThat(stream.readLengths).isEmpty();
+ assertThat(stream.seeks).isEmpty();
+ verifyNoInteractions(io);
+ }
+ assertThat(stream.closed).isEqualTo(point == 20);
+ }
+ }
+
+ @Test
+ void cachedBlocksAreSharedByDifferentSelectionsWithoutOpeningTheManifest() throws Exception {
+ byte[] header = header();
+ byte[] manifest = Arrays.copyOf(header, header.length + 400);
+ for (int i = header.length; i < manifest.length; i++) {
+ manifest[i] = (byte) i;
+ }
+ Path path = new Path(temp.toString(), "manifest-golden");
+ SegmentsCache<Object> cache =
+ new SegmentsCache<>(1024, MemorySize.ofMebiBytes(1), 400, null, false);
+ cache.put(
+ new ManifestSidecar.BlockCacheKey(path, header.length, 100),
+ new SingleSegments(MemorySegment.wrap(new byte[100]), 100));
+ FileIO io = mock(FileIO.class);
+ CountingInput stream = new CountingInput(manifest, Integer.MAX_VALUE);
+ when(io.newInputStream(path)).thenReturn(stream);
+ ManifestSidecar.Selection all =
+ ManifestSidecar.select(
+ testSidecar(),
+ testMeta(),
+ RowRangeIndex.create(
+ Collections.singletonList(new Range(0, Long.MAX_VALUE))));
+ try (InputStream in = ManifestSidecar.openManifest(io, path, all, cache)) {
+ assertThat(IOUtils.readFully(in, false)).isEqualTo(manifest);
+ }
+ assertThat(stream.readLengths).containsExactly(400);
+ assertThat(stream.seeks).containsExactly((long) header.length);
+ assertThat(cache.estimatedSize()).isEqualTo(3);
+ assertThat(cache.getIfPresents(path)).isNull();
+ when(io.newInputStream(path)).thenThrow(new IOException("Must use cached blocks"));
+ for (long point : new long[] {20, 8254058425445L, 0}) {
+ ManifestSidecar.Selection selected = select(testSidecar(), testMeta(), point);
+ ByteArrayOutputStream expected = new ByteArrayOutputStream();
+ expected.write(header);
+ for (ManifestSidecar.Block block : selected.blocks()) {
+ expected.write(manifest, (int) block.offset, (int) block.length);
+ }
+ try (InputStream in = ManifestSidecar.openManifest(io, path, selected, cache)) {
+ assertThat(IOUtils.readFully(in, false)).isEqualTo(expected.toByteArray());
+ }
+ }
+ verify(io, times(1)).newInputStream(path);
+
+ Path other = new Path(temp.toString(), "other/manifest-golden");
+ byte[] otherBytes = manifest.clone();
+ otherBytes[header.length] ^= 1;
+ when(io.newInputStream(other)).thenReturn(new CountingInput(otherBytes, Integer.MAX_VALUE));
+ try (InputStream in =
+ ManifestSidecar.openManifest(
+ io, other, select(testSidecar(), testMeta(), 0), cache)) {
+ assertThat(IOUtils.readFully(in, false))
+ .isEqualTo(Arrays.copyOf(otherBytes, header.length + 100));
+ }
+ verify(io).newInputStream(other);
+ }
+
+ @Test
+ void mixedHitsAndMissesReadOnlyUncachedBlocks() throws Exception {
+ byte[] header = header();
+ byte[] manifest = Arrays.copyOf(header, header.length + 400);
+ FileIO io = mock(FileIO.class);
+ Path path = new Path(temp.toString(), "manifest-golden");
+ CountingInput cold = new CountingInput(manifest, Integer.MAX_VALUE);
+ CountingInput mixed = new CountingInput(manifest, Integer.MAX_VALUE);
+ when(io.newInputStream(path)).thenReturn(cold, mixed);
+ SegmentsCache<Object> cache =
+ new SegmentsCache<>(1024, MemorySize.ofMebiBytes(1), 400, null, false);
+ try (InputStream in =
+ ManifestSidecar.openManifest(
+ io, path, select(testSidecar(), testMeta(), 8254058425445L), cache)) {
+ IOUtils.readFully(in, false);
+ }
+ ManifestSidecar.Selection all =
+ ManifestSidecar.select(
+ testSidecar(),
+ testMeta(),
+ RowRangeIndex.create(
+ Collections.singletonList(new Range(0, Long.MAX_VALUE))));
+ try (InputStream in = ManifestSidecar.openManifest(io, path, all, cache)) {
+ assertThat(IOUtils.readFully(in, false)).isEqualTo(manifest);
+ }
+ assertThat(mixed.readLengths).containsExactly(100, 100);
+ assertThat(mixed.seeks).containsExactly((long) header.length, header.length + 300L);
+ try (InputStream in = ManifestSidecar.openManifest(io, path, all, cache)) {
+ assertThat(IOUtils.readFully(in, false)).isEqualTo(manifest);
+ }
+ verify(io, times(2)).newInputStream(path);
+ }
+
+ @Test
+ void truncatedCoalescedReadsDoNotPopulateTheBlockCache() throws Exception {
+ byte[] header = header();
+ byte[] manifest = Arrays.copyOf(header, header.length + 400);
+ FileIO io = mock(FileIO.class);
+ Path path = new Path(temp.toString(), "manifest-golden");
+ CountingInput truncated =
+ new CountingInput(Arrays.copyOf(manifest, manifest.length - 1), 7);
+ CountingInput complete = new CountingInput(manifest, 7);
+ when(io.newInputStream(path)).thenReturn(truncated, complete);
+ SegmentsCache<Object> cache =
+ new SegmentsCache<>(1024, MemorySize.ofMebiBytes(1), 400, null, false);
+ ManifestSidecar.Selection all =
+ ManifestSidecar.select(
+ testSidecar(),
+ testMeta(),
+ RowRangeIndex.create(
+ Collections.singletonList(new Range(0, Long.MAX_VALUE))));
+ try (InputStream in = ManifestSidecar.openManifest(io, path, all, cache)) {
+ assertThatThrownBy(() -> IOUtils.readFully(in, false)).isInstanceOf(EOFException.class);
+ }
+ assertThat(truncated.closed).isTrue();
+ assertThat(cache.estimatedSize()).isZero();
+ try (InputStream in = ManifestSidecar.openManifest(io, path, all, cache)) {
+ assertThat(IOUtils.readFully(in, false)).isEqualTo(manifest);
+ }
+ assertThat(complete.closed).isTrue();
+ assertThat(cache.estimatedSize()).isEqualTo(3);
+ }
+
+ @Test
+ void evictedBlocksAreReadAgainWithinTheSharedBudget() throws Exception {
+ byte[] header = header();
+ byte[] manifest = Arrays.copyOf(header, header.length + 400);
+ Path path = new Path(temp.toString(), "manifest-golden");
+ FileIO io = mock(FileIO.class);
+ when(io.newInputStream(path))
+ .thenAnswer(ignored -> new CountingInput(manifest, Integer.MAX_VALUE));
+ SegmentsCache<Object> cache =
+ new SegmentsCache<>(1024, MemorySize.ofBytes(1300), 400, null, false);
+ ManifestSidecar.Selection all =
+ ManifestSidecar.select(
+ testSidecar(),
+ testMeta(),
+ RowRangeIndex.create(
+ Collections.singletonList(new Range(0, Long.MAX_VALUE))));
+ for (int round = 0; round < 2; round++) {
+ try (InputStream in = ManifestSidecar.openManifest(io, path, all, cache)) {
+ assertThat(IOUtils.readFully(in, false)).isEqualTo(manifest);
+ }
+ assertThat(cache.totalCacheBytes()).isLessThanOrEqualTo(1300);
+ assertThat(cache.estimatedSize()).isEqualTo(1);
+ }
+ verify(io, times(2)).newInputStream(path);
+ }
+
+ @Test
+ void oversizedBlocksUseBoundedReadsWithoutModifyingPreviouslyCachedBytes() throws Exception {
+ byte[] header = header();
+ int cachedLength = (4 << 20) + 17;
+ int uncachedLength = 2 * (4 << 20) + 31;
+ ManifestSidecar.Builder builder = new ManifestSidecar.Builder(header, true, true);
+ builder.beginBlock(header.length, cachedLength, 1);
+ builder.add(0L, 1);
+ builder.endBlock();
+ builder.beginBlock(header.length + cachedLength, uncachedLength, 1);
+ builder.add(100L, 1);
+ builder.endBlock();
+ byte[] manifest = Arrays.copyOf(header, header.length + cachedLength + uncachedLength);
+ Arrays.fill(manifest, header.length, header.length + cachedLength, (byte) 7);
+ Arrays.fill(manifest, header.length + cachedLength, manifest.length, (byte) 9);
+ byte[] data = builder.serialize(manifest.length, 2);
+ ManifestFileMeta meta = meta("large", manifest.length, 2);
+ Path path = new Path(temp.toString(), "large");
+ FileIO io = mock(FileIO.class);
+ CountingInput cold = new CountingInput(manifest, Integer.MAX_VALUE);
+ CountingInput mixed = new CountingInput(manifest, Integer.MAX_VALUE);
+ when(io.newInputStream(path)).thenReturn(cold, mixed);
+ SegmentsCache<Object> cache =
+ new SegmentsCache<>(1024, MemorySize.ofMebiBytes(8), cachedLength, null, false);
+ ManifestSidecar.Selection first = select(data, meta, 0);
+ byte[] expected = Arrays.copyOf(manifest, header.length + cachedLength);
+ try (InputStream in = ManifestSidecar.openManifest(io, path, first, cache)) {
+ assertThat(IOUtils.readFully(in, false)).isEqualTo(expected);
+ }
+ assertThat(cold.readLengths).containsExactly(4 << 20, 17);
+ ManifestSidecar.Selection all =
+ ManifestSidecar.select(
+ data,
+ meta,
+ RowRangeIndex.create(
+ Collections.singletonList(new Range(0, Long.MAX_VALUE))));
+ try (InputStream in = ManifestSidecar.openManifest(io, path, all, cache)) {
+ assertThat(IOUtils.readFully(in, false)).isEqualTo(manifest);
+ }
+ assertThat(mixed.readLengths).containsExactly(4 << 20, 4 << 20, 31);
+ assertThat(cache.estimatedSize()).isEqualTo(1);
+ assertThat(
+ cache.getIfPresents(
+ new ManifestSidecar.BlockCacheKey(
+ path, header.length + cachedLength, uncachedLength)))
+ .isNull();
+ try (InputStream in = ManifestSidecar.openManifest(io, path, first, cache)) {
+ assertThat(IOUtils.readFully(in, false)).isEqualTo(expected);
+ }
+ verify(io, times(2)).newInputStream(path);
+ }
+
+ @Test
+ void largeBlockSpansUseBoundedReads() throws Exception {
+ byte[] header = header();
+ ManifestSidecar.Builder builder = new ManifestSidecar.Builder(header, true, true);
+ long offset = header.length;
+ for (int length : new int[] {2 << 20, 2 << 20, 2 << 20, 2 << 20, 1 << 20}) {
+ builder.beginBlock(offset, length, 1);
+ builder.add(20L, 1);
+ builder.endBlock();
+ offset += length;
+ }
+ byte[] data = builder.serialize(offset, 5);
+ byte[] manifest = Arrays.copyOf(header, (int) offset);
+ Path path = new Path(temp.toString(), "manifest-large");
+ for (boolean withCache : new boolean[] {false, true}) {
+ CountingInput stream = new CountingInput(manifest, Integer.MAX_VALUE);
+ FileIO io = mock(FileIO.class);
+ when(io.newInputStream(path)).thenReturn(stream);
+ SegmentsCache<Object> cache =
+ withCache
+ ? new SegmentsCache<>(
+ 1024, MemorySize.ofMebiBytes(16), 4 << 20, null, false)
+ : null;
+ try (InputStream input =
+ ManifestSidecar.openManifest(
+ io, path, select(data, meta("manifest-large", offset, 5), 20), cache)) {
+ assertThat(IOUtils.readFully(input, false)).isEqualTo(manifest);
+ }
+ assertThat(stream.readLengths).containsExactly(4 << 20, 4 << 20, 1 << 20);
+ if (withCache) {
+ assertThat(stream.seeks)
+ .containsExactly(
+ (long) header.length,
+ header.length + (4L << 20),
+ header.length + (8L << 20));
+ assertThat(cache.estimatedSize()).isEqualTo(5);
+ } else {
+ assertThat(stream.seeks).containsExactly((long) header.length);
+ }
+ assertThat(stream.closed).isTrue();
+ }
+ }
+
+ @Test
+ void blockShortReadsAndTruncation() throws Exception {
+ byte[] header = header();
+ Path path = new Path(temp.toString(), "manifest-golden");
+ ManifestSidecar.Selection selected =
+ ManifestSidecar.select(
+ testSidecar(),
+ testMeta(),
+ RowRangeIndex.create(
+ Collections.singletonList(new Range(0, Long.MAX_VALUE))));
+ for (int bodyLength : new int[] {400, 399}) {
+ byte[] manifest = Arrays.copyOf(header, header.length + bodyLength);
+ CountingInput stream = new CountingInput(manifest, 7);
+ FileIO io = mock(FileIO.class);
+ when(io.newInputStream(path)).thenReturn(stream);
+ try (InputStream input = ManifestSidecar.openManifest(io, path, selected)) {
+ if (bodyLength == 400) {
+ assertThat(IOUtils.readFully(input, false)).isEqualTo(manifest);
+ } else {
+ assertThatThrownBy(() -> IOUtils.readFully(input, false))
+ .isInstanceOf(EOFException.class);
+ }
+ }
+ assertThat(stream.closed).isTrue();
+ }
+ }
+
+ private static class CountingInput extends ByteArraySeekableStream {
+ private final int maxRead;
+ private final List<Integer> requests = new ArrayList<>();
+ private final List<Integer> readLengths = new ArrayList<>();
+ private final List<Long> seeks = new ArrayList<>();
+ private boolean closed;
+
+ private CountingInput(byte[] data, int maxRead) {
+ super(data);
+ this.maxRead = maxRead;
+ }
+
+ @Override
+ public int read(byte[] bytes, int offset, int length) throws IOException {
+ requests.add(length);
+ int count = super.read(bytes, offset, Math.min(length, maxRead));
+ if (count > 0) {
+ readLengths.add(count);
+ }
+ return count;
+ }
+
+ @Override
+ public void seek(long position) throws IOException {
+ seeks.add(position);
+ super.seek(position);
+ }
+
+ @Override
+ public void close() throws IOException {
+ closed = true;
+ super.close();
+ }
+ }
+
+ private ManifestSidecar.Selection select(byte[] data, ManifestFileMeta meta, long point)
+ throws IOException {
+ return ManifestSidecar.select(
+ data,
+ meta,
+ RowRangeIndex.create(Collections.singletonList(new Range(point, point))));
+ }
+}
diff --git a/paimon-core/src/test/resources/compatibility/manifest-sidecar-v1 b/paimon-core/src/test/resources/compatibility/manifest-sidecar-v1
new file mode 100644
index 0000000000..53bf4c421e
Binary files /dev/null and b/paimon-core/src/test/resources/compatibility/manifest-sidecar-v1 differ
(paimon) 12/20: [spark][core] Support more expressions in V2 filter pushdown (#9891)
Posted by ho...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
XiaoHongbo-Hope pushed a commit to branch release-2.1
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit ce81e89b9d390c09bc72701abf0d9a08bdcf3b35
Author: Jingsong Lee <ji...@gmail.com>
AuthorDate: Wed Sep 16 23:15:17 2026 +0800
[spark][core] Support more expressions in V2 filter pushdown (#9891)
---
.../apache/paimon/fileindex/FileIndexReader.java | 20 +++
.../paimon/globalindex/GlobalIndexReader.java | 24 ++++
.../java/org/apache/paimon/predicate/Contains.java | 2 +-
.../paimon/predicate/DateExtractTransform.java | 145 +++++++++++++++++++
.../org/apache/paimon/predicate/DayTransform.java | 57 ++++++++
.../java/org/apache/paimon/predicate/EndsWith.java | 2 +-
.../apache/paimon/predicate/FunctionVisitor.java | 16 +++
.../org/apache/paimon/predicate/HourTransform.java | 57 ++++++++
.../org/apache/paimon/predicate/LeafFunction.java | 4 +
.../apache/paimon/predicate/LengthTransform.java | 129 +++++++++++++++++
.../java/org/apache/paimon/predicate/Like.java | 2 +-
.../apache/paimon/predicate/MinuteTransform.java | 57 ++++++++
.../apache/paimon/predicate/MonthTransform.java | 57 ++++++++
.../predicate/{Contains.java => NotContains.java} | 20 +--
.../predicate/{Contains.java => NotEndsWith.java} | 20 +--
.../predicate/{Contains.java => NotLike.java} | 27 ++--
.../{Contains.java => NotStartsWith.java} | 20 +--
.../predicate/OnlyPartitionKeyEqualVisitor.java | 20 +++
.../apache/paimon/predicate/PredicateBuilder.java | 8 ++
.../apache/paimon/predicate/SecondTransform.java | 57 ++++++++
.../org/apache/paimon/predicate/StartsWith.java | 2 +-
.../org/apache/paimon/predicate/Transform.java | 7 +
.../org/apache/paimon/predicate/YearTransform.java | 57 ++++++++
.../paimon/predicate/DateExtractTransformTest.java | 133 ++++++++++++++++++
.../paimon/predicate/LengthTransformTest.java | 82 +++++++++++
.../paimon/predicate/PredicateJsonSerdeTest.java | 31 ++++-
.../org/apache/paimon/predicate/PredicateTest.java | 60 ++++++++
.../paimon/predicate/TransformJsonSerdeTest.java | 40 +++++-
.../orc/filter/OrcPredicateFunctionVisitor.java | 22 +++
.../format/orc/filter/OrcFilterConverterTest.java | 15 ++
.../paimon/format/parquet/ParquetFiltersTest.java | 15 ++
.../paimon/spark/SparkV2FilterConverter.scala | 9 ++
.../spark/util/SparkExpressionConverter.scala | 43 +++++-
.../paimon/spark/SparkFilterConverterTest.java | 17 ++-
.../spark/sql/SparkV2FilterConverterTestBase.scala | 153 ++++++++++++++++++++-
35 files changed, 1370 insertions(+), 60 deletions(-)
diff --git a/paimon-common/src/main/java/org/apache/paimon/fileindex/FileIndexReader.java b/paimon-common/src/main/java/org/apache/paimon/fileindex/FileIndexReader.java
index b67dcaefd4..5dba86fbbb 100644
--- a/paimon-common/src/main/java/org/apache/paimon/fileindex/FileIndexReader.java
+++ b/paimon-common/src/main/java/org/apache/paimon/fileindex/FileIndexReader.java
@@ -53,16 +53,31 @@ public abstract class FileIndexReader implements FunctionVisitor<FileIndexResult
return REMAIN;
}
+ @Override
+ public FileIndexResult visitNotStartsWith(FieldRef fieldRef, Object literal) {
+ return REMAIN;
+ }
+
@Override
public FileIndexResult visitEndsWith(FieldRef fieldRef, Object literal) {
return REMAIN;
}
+ @Override
+ public FileIndexResult visitNotEndsWith(FieldRef fieldRef, Object literal) {
+ return REMAIN;
+ }
+
@Override
public FileIndexResult visitContains(FieldRef fieldRef, Object literal) {
return REMAIN;
}
+ @Override
+ public FileIndexResult visitNotContains(FieldRef fieldRef, Object literal) {
+ return REMAIN;
+ }
+
@Override
public FileIndexResult visitArrayContains(FieldRef fieldRef, Object literal) {
return REMAIN;
@@ -83,6 +98,11 @@ public abstract class FileIndexReader implements FunctionVisitor<FileIndexResult
return REMAIN;
}
+ @Override
+ public FileIndexResult visitNotLike(FieldRef fieldRef, Object literal) {
+ return REMAIN;
+ }
+
@Override
public FileIndexResult visitLessThan(FieldRef fieldRef, Object literal) {
return REMAIN;
diff --git a/paimon-common/src/main/java/org/apache/paimon/globalindex/GlobalIndexReader.java b/paimon-common/src/main/java/org/apache/paimon/globalindex/GlobalIndexReader.java
index f1098cf68e..c6901c176b 100644
--- a/paimon-common/src/main/java/org/apache/paimon/globalindex/GlobalIndexReader.java
+++ b/paimon-common/src/main/java/org/apache/paimon/globalindex/GlobalIndexReader.java
@@ -41,6 +41,24 @@ public interface GlobalIndexReader
return CompletableFuture.completedFuture(Optional.empty());
}
+ @Override
+ default CompletableFuture<Optional<GlobalIndexResult>> visitNotStartsWith(
+ FieldRef fieldRef, Object literal) {
+ return CompletableFuture.completedFuture(Optional.empty());
+ }
+
+ @Override
+ default CompletableFuture<Optional<GlobalIndexResult>> visitNotEndsWith(
+ FieldRef fieldRef, Object literal) {
+ return CompletableFuture.completedFuture(Optional.empty());
+ }
+
+ @Override
+ default CompletableFuture<Optional<GlobalIndexResult>> visitNotContains(
+ FieldRef fieldRef, Object literal) {
+ return CompletableFuture.completedFuture(Optional.empty());
+ }
+
@Override
default CompletableFuture<Optional<GlobalIndexResult>> visitArrayContains(
FieldRef fieldRef, Object literal) {
@@ -59,6 +77,12 @@ public interface GlobalIndexReader
return CompletableFuture.completedFuture(Optional.empty());
}
+ @Override
+ default CompletableFuture<Optional<GlobalIndexResult>> visitNotLike(
+ FieldRef fieldRef, Object literal) {
+ return CompletableFuture.completedFuture(Optional.empty());
+ }
+
@Override
default CompletableFuture<Optional<GlobalIndexResult>> visitBetween(
FieldRef fieldRef, Object from, Object to) {
diff --git a/paimon-common/src/main/java/org/apache/paimon/predicate/Contains.java b/paimon-common/src/main/java/org/apache/paimon/predicate/Contains.java
index aa4c0d4c21..9cf65896a5 100644
--- a/paimon-common/src/main/java/org/apache/paimon/predicate/Contains.java
+++ b/paimon-common/src/main/java/org/apache/paimon/predicate/Contains.java
@@ -55,7 +55,7 @@ public class Contains extends LeafBinaryFunction {
@Override
public Optional<LeafFunction> negate() {
- return Optional.empty();
+ return Optional.of(NotContains.INSTANCE);
}
@Override
diff --git a/paimon-common/src/main/java/org/apache/paimon/predicate/DateExtractTransform.java b/paimon-common/src/main/java/org/apache/paimon/predicate/DateExtractTransform.java
new file mode 100644
index 0000000000..70dfa9e040
--- /dev/null
+++ b/paimon-common/src/main/java/org/apache/paimon/predicate/DateExtractTransform.java
@@ -0,0 +1,145 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.predicate;
+
+import org.apache.paimon.data.InternalRow;
+import org.apache.paimon.data.Timestamp;
+import org.apache.paimon.types.DataType;
+import org.apache.paimon.types.DataTypes;
+
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter;
+
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.util.Collections;
+import java.util.List;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.function.Function;
+
+import static org.apache.paimon.utils.InternalRowUtils.get;
+import static org.apache.paimon.utils.Preconditions.checkArgument;
+import static org.apache.paimon.utils.Preconditions.checkNotNull;
+
+/**
+ * Base {@link Transform} that extracts a calendar field from a {@code DATE} or {@code TIMESTAMP}
+ * field, like SQL {@code EXTRACT(YEAR FROM d)} and the {@code year}, {@code month}, ... functions.
+ * See the subclasses {@link YearTransform}, {@link MonthTransform}, {@link DayTransform}, {@link
+ * HourTransform}, {@link MinuteTransform} and {@link SecondTransform}.
+ *
+ * <p>{@code TIMESTAMP WITH LOCAL TIME ZONE} is deliberately not supported: extracting a calendar
+ * field from it depends on a session time zone, which the reader does not know, so such a predicate
+ * must stay in the query engine.
+ */
+public abstract class DateExtractTransform implements Transform {
+
+ private static final long serialVersionUID = 1L;
+
+ public static final String FIELD_FIELD_REF = "fieldRef";
+
+ private final FieldRef fieldRef;
+
+ protected DateExtractTransform(FieldRef fieldRef) {
+ this.fieldRef = checkNotNull(fieldRef, "fieldRef must not be null");
+ checkArgument(
+ supported(fieldRef.type()),
+ "%s requires a DATE or TIMESTAMP field, found %s",
+ name(),
+ fieldRef.type());
+ }
+
+ /** Creates a transform if {@code fieldRef} is a DATE or TIMESTAMP field. */
+ protected static Optional<Transform> tryCreate(
+ FieldRef fieldRef, Function<FieldRef, Transform> factory) {
+ if (fieldRef == null || !supported(fieldRef.type())) {
+ return Optional.empty();
+ }
+ return Optional.of(factory.apply(fieldRef));
+ }
+
+ private static boolean supported(DataType type) {
+ switch (type.getTypeRoot()) {
+ case DATE:
+ case TIMESTAMP_WITHOUT_TIME_ZONE:
+ return true;
+ default:
+ return false;
+ }
+ }
+
+ @JsonGetter(FIELD_FIELD_REF)
+ public FieldRef fieldRef() {
+ return fieldRef;
+ }
+
+ @Override
+ public List<Object> inputs() {
+ return Collections.singletonList(fieldRef);
+ }
+
+ @Override
+ public DataType outputType() {
+ return DataTypes.INT();
+ }
+
+ @Override
+ public Object transform(InternalRow row) {
+ Object value = get(row, fieldRef.index(), fieldRef.type());
+ if (value == null) {
+ return null;
+ }
+ LocalDateTime dateTime;
+ if (value instanceof Timestamp) {
+ dateTime = ((Timestamp) value).toLocalDateTime();
+ } else {
+ dateTime = LocalDate.ofEpochDay((Integer) value).atStartOfDay();
+ }
+ return extract(dateTime);
+ }
+
+ /** Extracts the calendar field this transform stands for. */
+ protected abstract Integer extract(LocalDateTime dateTime);
+
+ @Override
+ public Transform copyWithNewInputs(List<Object> inputs) {
+ checkArgument(inputs.size() == 1, "%s requires exactly one input", name());
+ return copy((FieldRef) inputs.get(0));
+ }
+
+ /** Rebuilds this transform over a new field reference. */
+ protected abstract Transform copy(FieldRef fieldRef);
+
+ @Override
+ public boolean equals(Object o) {
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ return Objects.equals(fieldRef, ((DateExtractTransform) o).fieldRef);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(fieldRef);
+ }
+
+ @Override
+ public String toString() {
+ return name() + "(" + fieldRef + ")";
+ }
+}
diff --git a/paimon-common/src/main/java/org/apache/paimon/predicate/DayTransform.java b/paimon-common/src/main/java/org/apache/paimon/predicate/DayTransform.java
new file mode 100644
index 0000000000..9b7e60aa84
--- /dev/null
+++ b/paimon-common/src/main/java/org/apache/paimon/predicate/DayTransform.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.predicate;
+
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator;
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty;
+
+import java.time.LocalDateTime;
+import java.util.Optional;
+
+/** Extracts the day of month, like SQL {@code EXTRACT(DAY FROM d)}. */
+public class DayTransform extends DateExtractTransform {
+
+ private static final long serialVersionUID = 1L;
+
+ public static final String NAME = "DAY";
+
+ @JsonCreator
+ public DayTransform(@JsonProperty(DateExtractTransform.FIELD_FIELD_REF) FieldRef fieldRef) {
+ super(fieldRef);
+ }
+
+ public static Optional<Transform> tryCreate(FieldRef fieldRef) {
+ return DateExtractTransform.tryCreate(fieldRef, DayTransform::new);
+ }
+
+ @Override
+ public String name() {
+ return NAME;
+ }
+
+ @Override
+ protected Integer extract(LocalDateTime dateTime) {
+ return dateTime.getDayOfMonth();
+ }
+
+ @Override
+ protected Transform copy(FieldRef fieldRef) {
+ return new DayTransform(fieldRef);
+ }
+}
diff --git a/paimon-common/src/main/java/org/apache/paimon/predicate/EndsWith.java b/paimon-common/src/main/java/org/apache/paimon/predicate/EndsWith.java
index 2757ade2d5..676e1ffbbb 100644
--- a/paimon-common/src/main/java/org/apache/paimon/predicate/EndsWith.java
+++ b/paimon-common/src/main/java/org/apache/paimon/predicate/EndsWith.java
@@ -55,7 +55,7 @@ public class EndsWith extends LeafBinaryFunction {
@Override
public Optional<LeafFunction> negate() {
- return Optional.empty();
+ return Optional.of(NotEndsWith.INSTANCE);
}
@Override
diff --git a/paimon-common/src/main/java/org/apache/paimon/predicate/FunctionVisitor.java b/paimon-common/src/main/java/org/apache/paimon/predicate/FunctionVisitor.java
index 6ffa97cee2..a8b0440bbf 100644
--- a/paimon-common/src/main/java/org/apache/paimon/predicate/FunctionVisitor.java
+++ b/paimon-common/src/main/java/org/apache/paimon/predicate/FunctionVisitor.java
@@ -62,10 +62,22 @@ public interface FunctionVisitor<T> extends PredicateVisitor<T> {
T visitStartsWith(FieldRef fieldRef, Object literal);
+ default T visitNotStartsWith(FieldRef fieldRef, Object literal) {
+ throw new UnsupportedOperationException();
+ }
+
T visitEndsWith(FieldRef fieldRef, Object literal);
+ default T visitNotEndsWith(FieldRef fieldRef, Object literal) {
+ throw new UnsupportedOperationException();
+ }
+
T visitContains(FieldRef fieldRef, Object literal);
+ default T visitNotContains(FieldRef fieldRef, Object literal) {
+ throw new UnsupportedOperationException();
+ }
+
default T visitArrayContains(FieldRef fieldRef, Object literal) {
throw new UnsupportedOperationException();
}
@@ -80,6 +92,10 @@ public interface FunctionVisitor<T> extends PredicateVisitor<T> {
T visitLike(FieldRef fieldRef, Object literal);
+ default T visitNotLike(FieldRef fieldRef, Object literal) {
+ throw new UnsupportedOperationException();
+ }
+
T visitLessThan(FieldRef fieldRef, Object literal);
T visitGreaterOrEqual(FieldRef fieldRef, Object literal);
diff --git a/paimon-common/src/main/java/org/apache/paimon/predicate/HourTransform.java b/paimon-common/src/main/java/org/apache/paimon/predicate/HourTransform.java
new file mode 100644
index 0000000000..86edcd344b
--- /dev/null
+++ b/paimon-common/src/main/java/org/apache/paimon/predicate/HourTransform.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.predicate;
+
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator;
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty;
+
+import java.time.LocalDateTime;
+import java.util.Optional;
+
+/** Extracts the hour of day, like SQL {@code EXTRACT(HOUR FROM t)}. */
+public class HourTransform extends DateExtractTransform {
+
+ private static final long serialVersionUID = 1L;
+
+ public static final String NAME = "HOUR";
+
+ @JsonCreator
+ public HourTransform(@JsonProperty(DateExtractTransform.FIELD_FIELD_REF) FieldRef fieldRef) {
+ super(fieldRef);
+ }
+
+ public static Optional<Transform> tryCreate(FieldRef fieldRef) {
+ return DateExtractTransform.tryCreate(fieldRef, HourTransform::new);
+ }
+
+ @Override
+ public String name() {
+ return NAME;
+ }
+
+ @Override
+ protected Integer extract(LocalDateTime dateTime) {
+ return dateTime.getHour();
+ }
+
+ @Override
+ protected Transform copy(FieldRef fieldRef) {
+ return new HourTransform(fieldRef);
+ }
+}
diff --git a/paimon-common/src/main/java/org/apache/paimon/predicate/LeafFunction.java b/paimon-common/src/main/java/org/apache/paimon/predicate/LeafFunction.java
index 77326abd32..5c88bb2208 100644
--- a/paimon-common/src/main/java/org/apache/paimon/predicate/LeafFunction.java
+++ b/paimon-common/src/main/java/org/apache/paimon/predicate/LeafFunction.java
@@ -61,12 +61,16 @@ public abstract class LeafFunction implements Serializable {
registry.put(IsNull.NAME, IsNull.INSTANCE);
registry.put(IsNotNull.NAME, IsNotNull.INSTANCE);
registry.put(StartsWith.NAME, StartsWith.INSTANCE);
+ registry.put(NotStartsWith.NAME, NotStartsWith.INSTANCE);
registry.put(EndsWith.NAME, EndsWith.INSTANCE);
+ registry.put(NotEndsWith.NAME, NotEndsWith.INSTANCE);
registry.put(Contains.NAME, Contains.INSTANCE);
+ registry.put(NotContains.NAME, NotContains.INSTANCE);
registry.put(ArrayContains.NAME, ArrayContains.INSTANCE);
registry.put(ArraysOverlap.NAME, ArraysOverlap.INSTANCE);
registry.put(ArrayContainsAll.NAME, ArrayContainsAll.INSTANCE);
registry.put(Like.NAME, Like.INSTANCE);
+ registry.put(NotLike.NAME, NotLike.INSTANCE);
registry.put(In.NAME, In.INSTANCE);
registry.put(NotIn.NAME, NotIn.INSTANCE);
registry.put(Between.NAME, Between.INSTANCE);
diff --git a/paimon-common/src/main/java/org/apache/paimon/predicate/LengthTransform.java b/paimon-common/src/main/java/org/apache/paimon/predicate/LengthTransform.java
new file mode 100644
index 0000000000..b4fc6e0572
--- /dev/null
+++ b/paimon-common/src/main/java/org/apache/paimon/predicate/LengthTransform.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.predicate;
+
+import org.apache.paimon.data.BinaryString;
+import org.apache.paimon.data.InternalRow;
+import org.apache.paimon.types.DataType;
+import org.apache.paimon.types.DataTypes;
+
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator;
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter;
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnore;
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty;
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+
+import java.util.List;
+import java.util.Objects;
+
+import static org.apache.paimon.types.DataTypeFamily.CHARACTER_STRING;
+import static org.apache.paimon.utils.Preconditions.checkArgument;
+
+/**
+ * Transform that returns the number of characters of a string, like SQL {@code CHAR_LENGTH}.
+ *
+ * <p>Unlike {@link StringTransform} its output is an {@code INT}, so it is a standalone {@link
+ * Transform}. Input and JSON conventions follow {@link StringTransform}.
+ */
+public class LengthTransform implements Transform {
+
+ private static final long serialVersionUID = 1L;
+
+ public static final String NAME = "LENGTH";
+
+ private final List<Object> inputs;
+
+ @JsonCreator
+ public LengthTransform(
+ @JsonProperty(StringTransform.FIELD_INPUTS)
+ @JsonDeserialize(contentUsing = StringTransform.InputDeserializer.class)
+ List<Object> inputs) {
+ checkArgument(inputs.size() == 1, "LENGTH requires exactly one input");
+ Object input = inputs.get(0);
+ if (input instanceof FieldRef) {
+ checkArgument(
+ ((FieldRef) input).type().is(CHARACTER_STRING),
+ "LENGTH input must be a string field");
+ } else {
+ checkArgument(
+ input == null || input instanceof BinaryString,
+ "LENGTH input literal must be a string");
+ }
+ this.inputs = inputs;
+ }
+
+ @Override
+ public String name() {
+ return NAME;
+ }
+
+ @Override
+ @JsonIgnore
+ public List<Object> inputs() {
+ return inputs;
+ }
+
+ @JsonGetter(StringTransform.FIELD_INPUTS)
+ public List<Object> inputsForJson() {
+ return StringTransform.inputsForJson(inputs);
+ }
+
+ @Override
+ public DataType outputType() {
+ return DataTypes.INT();
+ }
+
+ @Override
+ public Object transform(InternalRow row) {
+ Object input = inputs.get(0);
+ BinaryString value;
+ if (input instanceof FieldRef) {
+ FieldRef ref = (FieldRef) input;
+ int i = ref.index();
+ value = row.isNullAt(i) ? null : row.getString(i);
+ } else {
+ value = (BinaryString) input;
+ }
+ return value == null ? null : value.numChars();
+ }
+
+ @Override
+ public Transform copyWithNewInputs(List<Object> inputs) {
+ return new LengthTransform(inputs);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ LengthTransform that = (LengthTransform) o;
+ return Objects.equals(inputs, that.inputs);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(inputs);
+ }
+
+ @Override
+ public String toString() {
+ return StringTransform.formatCall(name(), inputs);
+ }
+}
diff --git a/paimon-common/src/main/java/org/apache/paimon/predicate/Like.java b/paimon-common/src/main/java/org/apache/paimon/predicate/Like.java
index 29281fe3b4..8737fb8afe 100644
--- a/paimon-common/src/main/java/org/apache/paimon/predicate/Like.java
+++ b/paimon-common/src/main/java/org/apache/paimon/predicate/Like.java
@@ -148,7 +148,7 @@ public class Like extends LeafBinaryFunction {
@Override
public Optional<LeafFunction> negate() {
- return Optional.empty();
+ return Optional.of(NotLike.INSTANCE);
}
@Override
diff --git a/paimon-common/src/main/java/org/apache/paimon/predicate/MinuteTransform.java b/paimon-common/src/main/java/org/apache/paimon/predicate/MinuteTransform.java
new file mode 100644
index 0000000000..431231abb9
--- /dev/null
+++ b/paimon-common/src/main/java/org/apache/paimon/predicate/MinuteTransform.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.predicate;
+
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator;
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty;
+
+import java.time.LocalDateTime;
+import java.util.Optional;
+
+/** Extracts the minute of hour, like SQL {@code EXTRACT(MINUTE FROM t)}. */
+public class MinuteTransform extends DateExtractTransform {
+
+ private static final long serialVersionUID = 1L;
+
+ public static final String NAME = "MINUTE";
+
+ @JsonCreator
+ public MinuteTransform(@JsonProperty(DateExtractTransform.FIELD_FIELD_REF) FieldRef fieldRef) {
+ super(fieldRef);
+ }
+
+ public static Optional<Transform> tryCreate(FieldRef fieldRef) {
+ return DateExtractTransform.tryCreate(fieldRef, MinuteTransform::new);
+ }
+
+ @Override
+ public String name() {
+ return NAME;
+ }
+
+ @Override
+ protected Integer extract(LocalDateTime dateTime) {
+ return dateTime.getMinute();
+ }
+
+ @Override
+ protected Transform copy(FieldRef fieldRef) {
+ return new MinuteTransform(fieldRef);
+ }
+}
diff --git a/paimon-common/src/main/java/org/apache/paimon/predicate/MonthTransform.java b/paimon-common/src/main/java/org/apache/paimon/predicate/MonthTransform.java
new file mode 100644
index 0000000000..cd4a37458f
--- /dev/null
+++ b/paimon-common/src/main/java/org/apache/paimon/predicate/MonthTransform.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.predicate;
+
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator;
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty;
+
+import java.time.LocalDateTime;
+import java.util.Optional;
+
+/** Extracts the month of year, like SQL {@code EXTRACT(MONTH FROM d)}. */
+public class MonthTransform extends DateExtractTransform {
+
+ private static final long serialVersionUID = 1L;
+
+ public static final String NAME = "MONTH";
+
+ @JsonCreator
+ public MonthTransform(@JsonProperty(DateExtractTransform.FIELD_FIELD_REF) FieldRef fieldRef) {
+ super(fieldRef);
+ }
+
+ public static Optional<Transform> tryCreate(FieldRef fieldRef) {
+ return DateExtractTransform.tryCreate(fieldRef, MonthTransform::new);
+ }
+
+ @Override
+ public String name() {
+ return NAME;
+ }
+
+ @Override
+ protected Integer extract(LocalDateTime dateTime) {
+ return dateTime.getMonthValue();
+ }
+
+ @Override
+ protected Transform copy(FieldRef fieldRef) {
+ return new MonthTransform(fieldRef);
+ }
+}
diff --git a/paimon-common/src/main/java/org/apache/paimon/predicate/Contains.java b/paimon-common/src/main/java/org/apache/paimon/predicate/NotContains.java
similarity index 75%
copy from paimon-common/src/main/java/org/apache/paimon/predicate/Contains.java
copy to paimon-common/src/main/java/org/apache/paimon/predicate/NotContains.java
index aa4c0d4c21..add19d04e1 100644
--- a/paimon-common/src/main/java/org/apache/paimon/predicate/Contains.java
+++ b/paimon-common/src/main/java/org/apache/paimon/predicate/NotContains.java
@@ -18,7 +18,6 @@
package org.apache.paimon.predicate;
-import org.apache.paimon.data.BinaryString;
import org.apache.paimon.types.DataType;
import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator;
@@ -26,20 +25,21 @@ import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCre
import java.util.List;
import java.util.Optional;
-/** A {@link LeafBinaryFunction} to evaluate {@code filter like '%abc%'}. */
-public class Contains extends LeafBinaryFunction {
+/** A {@link LeafBinaryFunction} to evaluate {@code NOT CONTAINS(field, literal)}. */
+public class NotContains extends LeafBinaryFunction {
- public static final String NAME = "CONTAINS";
+ private static final long serialVersionUID = 1L;
- public static final Contains INSTANCE = new Contains();
+ public static final String NAME = "NOT_CONTAINS";
+
+ public static final NotContains INSTANCE = new NotContains();
@JsonCreator
- private Contains() {}
+ private NotContains() {}
@Override
public boolean test(DataType type, Object field, Object patternLiteral) {
- BinaryString fieldString = (BinaryString) field;
- return fieldString.contains((BinaryString) patternLiteral);
+ return !Contains.INSTANCE.test(type, field, patternLiteral);
}
@Override
@@ -55,12 +55,12 @@ public class Contains extends LeafBinaryFunction {
@Override
public Optional<LeafFunction> negate() {
- return Optional.empty();
+ return Optional.of(Contains.INSTANCE);
}
@Override
public <T> T visit(FunctionVisitor<T> visitor, FieldRef fieldRef, List<Object> literals) {
- return visitor.visitContains(fieldRef, literals.get(0));
+ return visitor.visitNotContains(fieldRef, literals.get(0));
}
@Override
diff --git a/paimon-common/src/main/java/org/apache/paimon/predicate/Contains.java b/paimon-common/src/main/java/org/apache/paimon/predicate/NotEndsWith.java
similarity index 75%
copy from paimon-common/src/main/java/org/apache/paimon/predicate/Contains.java
copy to paimon-common/src/main/java/org/apache/paimon/predicate/NotEndsWith.java
index aa4c0d4c21..98e37959e8 100644
--- a/paimon-common/src/main/java/org/apache/paimon/predicate/Contains.java
+++ b/paimon-common/src/main/java/org/apache/paimon/predicate/NotEndsWith.java
@@ -18,7 +18,6 @@
package org.apache.paimon.predicate;
-import org.apache.paimon.data.BinaryString;
import org.apache.paimon.types.DataType;
import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator;
@@ -26,20 +25,21 @@ import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCre
import java.util.List;
import java.util.Optional;
-/** A {@link LeafBinaryFunction} to evaluate {@code filter like '%abc%'}. */
-public class Contains extends LeafBinaryFunction {
+/** A {@link LeafBinaryFunction} to evaluate {@code NOT ENDS_WITH(field, literal)}. */
+public class NotEndsWith extends LeafBinaryFunction {
- public static final String NAME = "CONTAINS";
+ private static final long serialVersionUID = 1L;
- public static final Contains INSTANCE = new Contains();
+ public static final String NAME = "NOT_ENDS_WITH";
+
+ public static final NotEndsWith INSTANCE = new NotEndsWith();
@JsonCreator
- private Contains() {}
+ private NotEndsWith() {}
@Override
public boolean test(DataType type, Object field, Object patternLiteral) {
- BinaryString fieldString = (BinaryString) field;
- return fieldString.contains((BinaryString) patternLiteral);
+ return !EndsWith.INSTANCE.test(type, field, patternLiteral);
}
@Override
@@ -55,12 +55,12 @@ public class Contains extends LeafBinaryFunction {
@Override
public Optional<LeafFunction> negate() {
- return Optional.empty();
+ return Optional.of(EndsWith.INSTANCE);
}
@Override
public <T> T visit(FunctionVisitor<T> visitor, FieldRef fieldRef, List<Object> literals) {
- return visitor.visitContains(fieldRef, literals.get(0));
+ return visitor.visitNotEndsWith(fieldRef, literals.get(0));
}
@Override
diff --git a/paimon-common/src/main/java/org/apache/paimon/predicate/Contains.java b/paimon-common/src/main/java/org/apache/paimon/predicate/NotLike.java
similarity index 67%
copy from paimon-common/src/main/java/org/apache/paimon/predicate/Contains.java
copy to paimon-common/src/main/java/org/apache/paimon/predicate/NotLike.java
index aa4c0d4c21..c9c6cf376d 100644
--- a/paimon-common/src/main/java/org/apache/paimon/predicate/Contains.java
+++ b/paimon-common/src/main/java/org/apache/paimon/predicate/NotLike.java
@@ -18,7 +18,6 @@
package org.apache.paimon.predicate;
-import org.apache.paimon.data.BinaryString;
import org.apache.paimon.types.DataType;
import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator;
@@ -26,41 +25,37 @@ import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCre
import java.util.List;
import java.util.Optional;
-/** A {@link LeafBinaryFunction} to evaluate {@code filter like '%abc%'}. */
-public class Contains extends LeafBinaryFunction {
+/** A {@link LeafBinaryFunction} to evaluate {@code field not like pattern}. */
+public class NotLike extends LeafBinaryFunction {
- public static final String NAME = "CONTAINS";
+ private static final long serialVersionUID = 1L;
- public static final Contains INSTANCE = new Contains();
+ public static final String NAME = "NOT_LIKE";
+
+ public static final NotLike INSTANCE = new NotLike();
@JsonCreator
- private Contains() {}
+ private NotLike() {}
@Override
public boolean test(DataType type, Object field, Object patternLiteral) {
- BinaryString fieldString = (BinaryString) field;
- return fieldString.contains((BinaryString) patternLiteral);
+ return !Like.INSTANCE.test(type, field, patternLiteral);
}
@Override
public boolean test(
- DataType type,
- long rowCount,
- Object min,
- Object max,
- Long nullCount,
- Object patternLiteral) {
+ DataType type, long rowCount, Object min, Object max, Long nullCount, Object literal) {
return true;
}
@Override
public Optional<LeafFunction> negate() {
- return Optional.empty();
+ return Optional.of(Like.INSTANCE);
}
@Override
public <T> T visit(FunctionVisitor<T> visitor, FieldRef fieldRef, List<Object> literals) {
- return visitor.visitContains(fieldRef, literals.get(0));
+ return visitor.visitNotLike(fieldRef, literals.get(0));
}
@Override
diff --git a/paimon-common/src/main/java/org/apache/paimon/predicate/Contains.java b/paimon-common/src/main/java/org/apache/paimon/predicate/NotStartsWith.java
similarity index 74%
copy from paimon-common/src/main/java/org/apache/paimon/predicate/Contains.java
copy to paimon-common/src/main/java/org/apache/paimon/predicate/NotStartsWith.java
index aa4c0d4c21..6817f5a3ec 100644
--- a/paimon-common/src/main/java/org/apache/paimon/predicate/Contains.java
+++ b/paimon-common/src/main/java/org/apache/paimon/predicate/NotStartsWith.java
@@ -18,7 +18,6 @@
package org.apache.paimon.predicate;
-import org.apache.paimon.data.BinaryString;
import org.apache.paimon.types.DataType;
import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator;
@@ -26,20 +25,21 @@ import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCre
import java.util.List;
import java.util.Optional;
-/** A {@link LeafBinaryFunction} to evaluate {@code filter like '%abc%'}. */
-public class Contains extends LeafBinaryFunction {
+/** A {@link LeafBinaryFunction} to evaluate {@code NOT STARTS_WITH(field, literal)}. */
+public class NotStartsWith extends LeafBinaryFunction {
- public static final String NAME = "CONTAINS";
+ private static final long serialVersionUID = 1L;
- public static final Contains INSTANCE = new Contains();
+ public static final String NAME = "NOT_STARTS_WITH";
+
+ public static final NotStartsWith INSTANCE = new NotStartsWith();
@JsonCreator
- private Contains() {}
+ private NotStartsWith() {}
@Override
public boolean test(DataType type, Object field, Object patternLiteral) {
- BinaryString fieldString = (BinaryString) field;
- return fieldString.contains((BinaryString) patternLiteral);
+ return !StartsWith.INSTANCE.test(type, field, patternLiteral);
}
@Override
@@ -55,12 +55,12 @@ public class Contains extends LeafBinaryFunction {
@Override
public Optional<LeafFunction> negate() {
- return Optional.empty();
+ return Optional.of(StartsWith.INSTANCE);
}
@Override
public <T> T visit(FunctionVisitor<T> visitor, FieldRef fieldRef, List<Object> literals) {
- return visitor.visitContains(fieldRef, literals.get(0));
+ return visitor.visitNotStartsWith(fieldRef, literals.get(0));
}
@Override
diff --git a/paimon-common/src/main/java/org/apache/paimon/predicate/OnlyPartitionKeyEqualVisitor.java b/paimon-common/src/main/java/org/apache/paimon/predicate/OnlyPartitionKeyEqualVisitor.java
index fb6e6e1c85..57bd3bd505 100644
--- a/paimon-common/src/main/java/org/apache/paimon/predicate/OnlyPartitionKeyEqualVisitor.java
+++ b/paimon-common/src/main/java/org/apache/paimon/predicate/OnlyPartitionKeyEqualVisitor.java
@@ -57,16 +57,31 @@ public class OnlyPartitionKeyEqualVisitor implements FunctionVisitor<Boolean> {
return false;
}
+ @Override
+ public Boolean visitNotStartsWith(FieldRef fieldRef, Object literal) {
+ return false;
+ }
+
@Override
public Boolean visitEndsWith(FieldRef fieldRef, Object literal) {
return false;
}
+ @Override
+ public Boolean visitNotEndsWith(FieldRef fieldRef, Object literal) {
+ return false;
+ }
+
@Override
public Boolean visitContains(FieldRef fieldRef, Object literal) {
return false;
}
+ @Override
+ public Boolean visitNotContains(FieldRef fieldRef, Object literal) {
+ return false;
+ }
+
@Override
public Boolean visitArrayContains(FieldRef fieldRef, Object literal) {
return false;
@@ -87,6 +102,11 @@ public class OnlyPartitionKeyEqualVisitor implements FunctionVisitor<Boolean> {
return false;
}
+ @Override
+ public Boolean visitNotLike(FieldRef fieldRef, Object literal) {
+ return false;
+ }
+
@Override
public Boolean visitLessThan(FieldRef fieldRef, Object literal) {
return false;
diff --git a/paimon-common/src/main/java/org/apache/paimon/predicate/PredicateBuilder.java b/paimon-common/src/main/java/org/apache/paimon/predicate/PredicateBuilder.java
index 04e813cdcf..f576e48fc1 100644
--- a/paimon-common/src/main/java/org/apache/paimon/predicate/PredicateBuilder.java
+++ b/paimon-common/src/main/java/org/apache/paimon/predicate/PredicateBuilder.java
@@ -217,6 +217,14 @@ public class PredicateBuilder {
return leaf(optimized.getKey(), transform, optimized.getValue());
}
+ public Predicate notLike(int idx, Object patternLiteral) {
+ return leaf(NotLike.INSTANCE, idx, patternLiteral);
+ }
+
+ public Predicate notLike(Transform transform, Object patternLiteral) {
+ return leaf(NotLike.INSTANCE, transform, patternLiteral);
+ }
+
private Predicate leaf(LeafFunction function, int idx, Object literal) {
DataField field = rowType.getFields().get(idx);
return new LeafPredicate(function, field.type(), idx, field.name(), singletonList(literal));
diff --git a/paimon-common/src/main/java/org/apache/paimon/predicate/SecondTransform.java b/paimon-common/src/main/java/org/apache/paimon/predicate/SecondTransform.java
new file mode 100644
index 0000000000..ca676dd43a
--- /dev/null
+++ b/paimon-common/src/main/java/org/apache/paimon/predicate/SecondTransform.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.predicate;
+
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator;
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty;
+
+import java.time.LocalDateTime;
+import java.util.Optional;
+
+/** Extracts the second of minute, like SQL {@code EXTRACT(SECOND FROM t)}. */
+public class SecondTransform extends DateExtractTransform {
+
+ private static final long serialVersionUID = 1L;
+
+ public static final String NAME = "SECOND";
+
+ @JsonCreator
+ public SecondTransform(@JsonProperty(DateExtractTransform.FIELD_FIELD_REF) FieldRef fieldRef) {
+ super(fieldRef);
+ }
+
+ public static Optional<Transform> tryCreate(FieldRef fieldRef) {
+ return DateExtractTransform.tryCreate(fieldRef, SecondTransform::new);
+ }
+
+ @Override
+ public String name() {
+ return NAME;
+ }
+
+ @Override
+ protected Integer extract(LocalDateTime dateTime) {
+ return dateTime.getSecond();
+ }
+
+ @Override
+ protected Transform copy(FieldRef fieldRef) {
+ return new SecondTransform(fieldRef);
+ }
+}
diff --git a/paimon-common/src/main/java/org/apache/paimon/predicate/StartsWith.java b/paimon-common/src/main/java/org/apache/paimon/predicate/StartsWith.java
index 26b9689006..ba855e85d7 100644
--- a/paimon-common/src/main/java/org/apache/paimon/predicate/StartsWith.java
+++ b/paimon-common/src/main/java/org/apache/paimon/predicate/StartsWith.java
@@ -59,7 +59,7 @@ public class StartsWith extends LeafBinaryFunction {
@Override
public Optional<LeafFunction> negate() {
- return Optional.empty();
+ return Optional.of(NotStartsWith.INSTANCE);
}
@Override
diff --git a/paimon-common/src/main/java/org/apache/paimon/predicate/Transform.java b/paimon-common/src/main/java/org/apache/paimon/predicate/Transform.java
index ad01afcfb7..dffa541397 100644
--- a/paimon-common/src/main/java/org/apache/paimon/predicate/Transform.java
+++ b/paimon-common/src/main/java/org/apache/paimon/predicate/Transform.java
@@ -37,10 +37,17 @@ import java.util.List;
@JsonSubTypes.Type(value = CastTransform.class, name = CastTransform.NAME),
@JsonSubTypes.Type(value = ConcatTransform.class, name = ConcatTransform.NAME),
@JsonSubTypes.Type(value = ConcatWsTransform.class, name = ConcatWsTransform.NAME),
+ @JsonSubTypes.Type(value = YearTransform.class, name = YearTransform.NAME),
+ @JsonSubTypes.Type(value = MonthTransform.class, name = MonthTransform.NAME),
+ @JsonSubTypes.Type(value = DayTransform.class, name = DayTransform.NAME),
+ @JsonSubTypes.Type(value = HourTransform.class, name = HourTransform.NAME),
+ @JsonSubTypes.Type(value = MinuteTransform.class, name = MinuteTransform.NAME),
+ @JsonSubTypes.Type(value = SecondTransform.class, name = SecondTransform.NAME),
@JsonSubTypes.Type(value = UpperTransform.class, name = UpperTransform.NAME),
@JsonSubTypes.Type(value = LowerTransform.class, name = LowerTransform.NAME),
@JsonSubTypes.Type(value = SubstringTransform.class, name = SubstringTransform.NAME),
@JsonSubTypes.Type(value = TrimTransform.class, name = TrimTransform.NAME),
+ @JsonSubTypes.Type(value = LengthTransform.class, name = LengthTransform.NAME),
@JsonSubTypes.Type(value = NullTransform.class, name = NullTransform.NAME)
})
public interface Transform extends Serializable {
diff --git a/paimon-common/src/main/java/org/apache/paimon/predicate/YearTransform.java b/paimon-common/src/main/java/org/apache/paimon/predicate/YearTransform.java
new file mode 100644
index 0000000000..aa9460e09e
--- /dev/null
+++ b/paimon-common/src/main/java/org/apache/paimon/predicate/YearTransform.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.predicate;
+
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator;
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty;
+
+import java.time.LocalDateTime;
+import java.util.Optional;
+
+/** Extracts the year, like SQL {@code EXTRACT(YEAR FROM d)}. */
+public class YearTransform extends DateExtractTransform {
+
+ private static final long serialVersionUID = 1L;
+
+ public static final String NAME = "YEAR";
+
+ @JsonCreator
+ public YearTransform(@JsonProperty(DateExtractTransform.FIELD_FIELD_REF) FieldRef fieldRef) {
+ super(fieldRef);
+ }
+
+ public static Optional<Transform> tryCreate(FieldRef fieldRef) {
+ return DateExtractTransform.tryCreate(fieldRef, YearTransform::new);
+ }
+
+ @Override
+ public String name() {
+ return NAME;
+ }
+
+ @Override
+ protected Integer extract(LocalDateTime dateTime) {
+ return dateTime.getYear();
+ }
+
+ @Override
+ protected Transform copy(FieldRef fieldRef) {
+ return new YearTransform(fieldRef);
+ }
+}
diff --git a/paimon-common/src/test/java/org/apache/paimon/predicate/DateExtractTransformTest.java b/paimon-common/src/test/java/org/apache/paimon/predicate/DateExtractTransformTest.java
new file mode 100644
index 0000000000..0a472c78e4
--- /dev/null
+++ b/paimon-common/src/test/java/org/apache/paimon/predicate/DateExtractTransformTest.java
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.predicate;
+
+import org.apache.paimon.data.GenericRow;
+import org.apache.paimon.data.Timestamp;
+import org.apache.paimon.types.DataType;
+import org.apache.paimon.types.DataTypes;
+import org.apache.paimon.types.RowType;
+
+import org.junit.jupiter.api.Test;
+
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.util.Collections;
+import java.util.Optional;
+import java.util.function.Function;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+class DateExtractTransformTest {
+
+ private static int epochDay(int year, int month, int day) {
+ return (int) LocalDate.of(year, month, day).toEpochDay();
+ }
+
+ private static Object extract(
+ DataType type, Function<FieldRef, DateExtractTransform> factory, Object value) {
+ return factory.apply(new FieldRef(0, "f0", type)).transform(GenericRow.of(value));
+ }
+
+ @Test
+ public void testExtractFromDate() {
+ int value = epochDay(2023, 7, 15);
+ assertThat(extract(DataTypes.DATE(), YearTransform::new, value)).isEqualTo(2023);
+ assertThat(extract(DataTypes.DATE(), MonthTransform::new, value)).isEqualTo(7);
+ assertThat(extract(DataTypes.DATE(), DayTransform::new, value)).isEqualTo(15);
+ }
+
+ @Test
+ public void testExtractFromTimestamp() {
+ Timestamp value = Timestamp.fromLocalDateTime(LocalDateTime.of(2024, 3, 5, 14, 30, 45));
+ assertThat(extract(DataTypes.TIMESTAMP(3), YearTransform::new, value)).isEqualTo(2024);
+ assertThat(extract(DataTypes.TIMESTAMP(3), MonthTransform::new, value)).isEqualTo(3);
+ assertThat(extract(DataTypes.TIMESTAMP(3), DayTransform::new, value)).isEqualTo(5);
+ assertThat(extract(DataTypes.TIMESTAMP(3), HourTransform::new, value)).isEqualTo(14);
+ assertThat(extract(DataTypes.TIMESTAMP(3), MinuteTransform::new, value)).isEqualTo(30);
+ assertThat(extract(DataTypes.TIMESTAMP(3), SecondTransform::new, value)).isEqualTo(45);
+ }
+
+ @Test
+ public void testTimeFieldsOfDateAreZero() {
+ int value = epochDay(2024, 1, 1);
+ assertThat(extract(DataTypes.DATE(), HourTransform::new, value)).isEqualTo(0);
+ assertThat(extract(DataTypes.DATE(), MinuteTransform::new, value)).isEqualTo(0);
+ assertThat(extract(DataTypes.DATE(), SecondTransform::new, value)).isEqualTo(0);
+ }
+
+ @Test
+ public void testNullYieldsNull() {
+ DateExtractTransform transform = new YearTransform(new FieldRef(0, "d0", DataTypes.DATE()));
+ assertThat(transform.transform(GenericRow.of((Object) null))).isNull();
+ }
+
+ @Test
+ public void testOutputTypeIsInt() {
+ DateExtractTransform transform =
+ new HourTransform(new FieldRef(0, "t0", DataTypes.TIMESTAMP(6)));
+ assertThat(transform.outputType()).isEqualTo(DataTypes.INT());
+ }
+
+ @Test
+ public void testExtractPredicateFiltersRows() {
+ PredicateBuilder builder =
+ new PredicateBuilder(RowType.of(DataTypes.DATE(), DataTypes.TIMESTAMP(3)));
+ Predicate predicate =
+ builder.equal(new YearTransform(new FieldRef(0, "d0", DataTypes.DATE())), 2023);
+
+ assertThat(predicate.test(GenericRow.of(epochDay(2023, 7, 15), null))).isTrue();
+ assertThat(predicate.test(GenericRow.of(epochDay(2024, 1, 1), null))).isFalse();
+ assertThat(predicate.test(GenericRow.of((Object) null, null))).isFalse();
+ }
+
+ @Test
+ public void testCopyWithNewInputsRemapsField() {
+ DateExtractTransform transform =
+ new MonthTransform(new FieldRef(0, "d0", DataTypes.DATE()));
+ DateExtractTransform copied =
+ (DateExtractTransform)
+ transform.copyWithNewInputs(
+ Collections.singletonList(new FieldRef(1, "d1", DataTypes.DATE())));
+ assertThat(copied.transform(GenericRow.of((Object) null, epochDay(2024, 5, 20))))
+ .isEqualTo(5);
+ }
+
+ @Test
+ public void testUnsupportedFieldTypeIsRejected() {
+ for (DataType type :
+ new DataType[] {DataTypes.STRING(), DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE(3)}) {
+ FieldRef fieldRef = new FieldRef(0, "f0", type);
+ assertThatThrownBy(() -> new YearTransform(fieldRef))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("YEAR requires a DATE or TIMESTAMP field, found " + type);
+ assertThat(YearTransform.tryCreate(fieldRef)).isEqualTo(Optional.empty());
+ }
+
+ FieldRef dateRef = new FieldRef(0, "d0", DataTypes.DATE());
+ assertThat(YearTransform.tryCreate(dateRef).isPresent()).isTrue();
+ }
+
+ @Test
+ public void testToString() {
+ DateExtractTransform transform = new YearTransform(new FieldRef(0, "d0", DataTypes.DATE()));
+ assertThat(transform.toString()).isEqualTo("YEAR(d0)");
+ }
+}
diff --git a/paimon-common/src/test/java/org/apache/paimon/predicate/LengthTransformTest.java b/paimon-common/src/test/java/org/apache/paimon/predicate/LengthTransformTest.java
new file mode 100644
index 0000000000..9abf4cc3ed
--- /dev/null
+++ b/paimon-common/src/test/java/org/apache/paimon/predicate/LengthTransformTest.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.predicate;
+
+import org.apache.paimon.data.BinaryString;
+import org.apache.paimon.data.GenericRow;
+import org.apache.paimon.types.DataTypes;
+
+import org.junit.jupiter.api.Test;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+class LengthTransformTest {
+
+ @Test
+ public void testLiteralInput() {
+ List<Object> inputs = new ArrayList<>();
+ inputs.add(BinaryString.fromString("hello"));
+ LengthTransform transform = new LengthTransform(inputs);
+ assertThat(transform.transform(GenericRow.of())).isEqualTo(5);
+ assertThat(transform.outputType()).isEqualTo(DataTypes.INT());
+ }
+
+ @Test
+ public void testFieldInput() {
+ List<Object> inputs = new ArrayList<>();
+ inputs.add(new FieldRef(0, "f0", DataTypes.STRING()));
+ LengthTransform transform = new LengthTransform(inputs);
+ assertThat(transform.transform(GenericRow.of(BinaryString.fromString("paimon"))))
+ .isEqualTo(6);
+ assertThat(transform.transform(GenericRow.of((Object) null))).isNull();
+ }
+
+ @Test
+ public void testNullInputSlot() {
+ LengthTransform transform = new LengthTransform(Collections.singletonList(null));
+ assertThat(transform.transform(GenericRow.of())).isNull();
+ }
+
+ @Test
+ public void testIllegalInputs() {
+ // wrong arity
+ List<Object> two = new ArrayList<>();
+ two.add(BinaryString.fromString("hello"));
+ two.add(BinaryString.fromString("hi"));
+ assertThatThrownBy(() -> new LengthTransform(two))
+ .isInstanceOf(IllegalArgumentException.class);
+
+ // non-string field
+ assertThatThrownBy(
+ () ->
+ new LengthTransform(
+ Collections.singletonList(
+ new FieldRef(0, "f0", DataTypes.INT()))))
+ .isInstanceOf(IllegalArgumentException.class);
+
+ // non-string literal
+ assertThatThrownBy(() -> new LengthTransform(Collections.singletonList((Object) 5)))
+ .isInstanceOf(IllegalArgumentException.class);
+ }
+}
diff --git a/paimon-common/src/test/java/org/apache/paimon/predicate/PredicateJsonSerdeTest.java b/paimon-common/src/test/java/org/apache/paimon/predicate/PredicateJsonSerdeTest.java
index 4a62f0008e..a0ff9592e2 100644
--- a/paimon-common/src/test/java/org/apache/paimon/predicate/PredicateJsonSerdeTest.java
+++ b/paimon-common/src/test/java/org/apache/paimon/predicate/PredicateJsonSerdeTest.java
@@ -158,11 +158,22 @@ class PredicateJsonSerdeTest {
.expectJson(
"{\"kind\":\"LEAF\",\"transform\":{\"name\":\"CONCAT_WS\",\"inputs\":[\"|\",{\"index\":1,\"name\":\"f1\",\"type\":\"STRING\"},\"X\",null,{\"index\":2,\"name\":\"f2\",\"type\":\"STRING\"}]},\"function\":\"ENDS_WITH\",\"literals\":[\"z\"]}"),
- // LeafPredicate - Like (non-negatable)
+ // LeafPredicate - Like
TestSpec.forPredicate(builder.like(2, BinaryString.fromString("%a%b%")))
.expectJson(
"{\"kind\":\"LEAF\",\"transform\":{\"name\":\"FIELD_REF\",\"fieldRef\":{\"index\":2,\"name\":\"f2\",\"type\":\"STRING\"}},\"function\":\"LIKE\",\"literals\":[\"%a%b%\"]}"),
+ // LeafPredicate - NotLike
+ TestSpec.forPredicate(builder.notLike(2, BinaryString.fromString("%a%b%")))
+ .expectJson(
+ "{\"kind\":\"LEAF\",\"transform\":{\"name\":\"FIELD_REF\",\"fieldRef\":{\"index\":2,\"name\":\"f2\",\"type\":\"STRING\"}},\"function\":\"NOT_LIKE\",\"literals\":[\"%a%b%\"]}"),
+
+ // LeafPredicate - NotLike (negate of Like)
+ TestSpec.forPredicate(
+ builder.like(2, BinaryString.fromString("%a%b%")).negate().get())
+ .expectJson(
+ "{\"kind\":\"LEAF\",\"transform\":{\"name\":\"FIELD_REF\",\"fieldRef\":{\"index\":2,\"name\":\"f2\",\"type\":\"STRING\"}},\"function\":\"NOT_LIKE\",\"literals\":[\"%a%b%\"]}"),
+
// LeafPredicate - StartsWith (field index)
TestSpec.forPredicate(builder.startsWith(2, BinaryString.fromString("hello")))
.expectJson(
@@ -178,6 +189,24 @@ class PredicateJsonSerdeTest {
.expectJson(
"{\"kind\":\"LEAF\",\"transform\":{\"name\":\"FIELD_REF\",\"fieldRef\":{\"index\":2,\"name\":\"f2\",\"type\":\"STRING\"}},\"function\":\"CONTAINS\",\"literals\":[\"foo\"]}"),
+ // LeafPredicate - negated string predicates
+ TestSpec.forPredicate(
+ builder.startsWith(2, BinaryString.fromString("hello"))
+ .negate()
+ .get())
+ .expectJson(
+ "{\"kind\":\"LEAF\",\"transform\":{\"name\":\"FIELD_REF\",\"fieldRef\":{\"index\":2,\"name\":\"f2\",\"type\":\"STRING\"}},\"function\":\"NOT_STARTS_WITH\",\"literals\":[\"hello\"]}"),
+ TestSpec.forPredicate(
+ builder.endsWith(2, BinaryString.fromString("world"))
+ .negate()
+ .get())
+ .expectJson(
+ "{\"kind\":\"LEAF\",\"transform\":{\"name\":\"FIELD_REF\",\"fieldRef\":{\"index\":2,\"name\":\"f2\",\"type\":\"STRING\"}},\"function\":\"NOT_ENDS_WITH\",\"literals\":[\"world\"]}"),
+ TestSpec.forPredicate(
+ builder.contains(2, BinaryString.fromString("foo")).negate().get())
+ .expectJson(
+ "{\"kind\":\"LEAF\",\"transform\":{\"name\":\"FIELD_REF\",\"fieldRef\":{\"index\":2,\"name\":\"f2\",\"type\":\"STRING\"}},\"function\":\"NOT_CONTAINS\",\"literals\":[\"foo\"]}"),
+
// LeafPredicate - ArrayContains uses the element type for literal serde
TestSpec.forPredicate(builder.arrayContains(4, BinaryString.fromString("vip")))
.expectJson(
diff --git a/paimon-common/src/test/java/org/apache/paimon/predicate/PredicateTest.java b/paimon-common/src/test/java/org/apache/paimon/predicate/PredicateTest.java
index 9585bb7199..f87ba4095a 100644
--- a/paimon-common/src/test/java/org/apache/paimon/predicate/PredicateTest.java
+++ b/paimon-common/src/test/java/org/apache/paimon/predicate/PredicateTest.java
@@ -449,6 +449,38 @@ public class PredicateTest {
assertThat(predicate2.test(10, min, max, new GenericArray(nullCount))).isEqualTo(true);
}
+ @Test
+ public void testNegatedStringPredicates() {
+ PredicateBuilder builder = new PredicateBuilder(RowType.of(new VarCharType()));
+ List<Predicate> positives =
+ Arrays.asList(
+ builder.startsWith(0, fromString("he")),
+ builder.endsWith(0, fromString("lo")),
+ builder.contains(0, fromString("ell")));
+
+ for (Predicate positive : positives) {
+ Predicate negative = positive.negate().get();
+ assertThat(negative.test(GenericRow.of(fromString("hello")))).isFalse();
+ assertThat(negative.test(GenericRow.of(fromString("world")))).isTrue();
+ assertThat(negative.test(GenericRow.of((Object) null))).isFalse();
+ assertThat(negative.negate()).contains(positive);
+
+ // Negative string predicates cannot use min/max statistics, but an all-null column
+ // still cannot contain a matching row.
+ assertThat(
+ test(
+ negative,
+ 3,
+ new SimpleColStats[] {
+ new SimpleColStats(
+ fromString("hello"), fromString("world"), 0L)
+ }))
+ .isTrue();
+ assertThat(test(negative, 1, new SimpleColStats[] {new SimpleColStats(null, null, 1L)}))
+ .isFalse();
+ }
+ }
+
@Test
public void testLargeIn() {
PredicateBuilder builder = new PredicateBuilder(RowType.of(new IntType()));
@@ -622,6 +654,34 @@ public class PredicateTest {
.isTrue();
}
+ @Test
+ public void testNotLike() {
+ PredicateBuilder builder = new PredicateBuilder(RowType.of(new VarCharType()));
+ Predicate predicate = builder.notLike(0, fromString("h%"));
+
+ assertThat(predicate.test(GenericRow.of(fromString("hello")))).isEqualTo(false);
+ assertThat(predicate.test(GenericRow.of(fromString("world")))).isEqualTo(true);
+ assertThat(predicate.test(GenericRow.of((Object) null))).isEqualTo(false);
+
+ // unknown stats cannot prune
+ assertThat(test(predicate, 3, new SimpleColStats[] {new SimpleColStats(null, null, 1L)}))
+ .isEqualTo(true);
+ assertThat(
+ test(
+ predicate,
+ 3,
+ new SimpleColStats[] {
+ new SimpleColStats(fromString("a"), fromString("z"), 0L)
+ }))
+ .isEqualTo(true);
+
+ // like and not like negate each other, 'a_c' cannot be optimized to starts/ends/contains
+ assertThat(builder.like(0, fromString("a_c")).negate().orElse(null))
+ .isEqualTo(builder.notLike(0, fromString("a_c")));
+ assertThat(builder.notLike(0, fromString("a_c")).negate().orElse(null))
+ .isEqualTo(builder.like(0, fromString("a_c")));
+ }
+
private boolean executeLike(String s, String pattern) {
ThreadLocalRandom rnd = ThreadLocalRandom.current();
if (rnd.nextBoolean()) {
diff --git a/paimon-common/src/test/java/org/apache/paimon/predicate/TransformJsonSerdeTest.java b/paimon-common/src/test/java/org/apache/paimon/predicate/TransformJsonSerdeTest.java
index 3e6b97a2a2..f2a711e01e 100644
--- a/paimon-common/src/test/java/org/apache/paimon/predicate/TransformJsonSerdeTest.java
+++ b/paimon-common/src/test/java/org/apache/paimon/predicate/TransformJsonSerdeTest.java
@@ -122,6 +122,15 @@ class TransformJsonSerdeTest {
new FieldRef(2, "f2", DataTypes.STRING()))))
.expectJson(
"{\"name\":\"CONCAT_WS\",\"inputs\":[\"|\",{\"index\":1,\"name\":\"f1\",\"type\":\"STRING\"},\"X\",null,{\"index\":2,\"name\":\"f2\",\"type\":\"STRING\"}]}"),
+
+ // DateExtractTransform - YEAR on DATE, MINUTE on TIMESTAMP
+ TestSpec.forTransform(new YearTransform(new FieldRef(0, "d0", DataTypes.DATE())))
+ .expectJson(
+ "{\"name\":\"YEAR\",\"fieldRef\":{\"index\":0,\"name\":\"d0\",\"type\":\"DATE\"}}"),
+ TestSpec.forTransform(
+ new MinuteTransform(new FieldRef(1, "t1", DataTypes.TIMESTAMP(3))))
+ .expectJson(
+ "{\"name\":\"MINUTE\",\"fieldRef\":{\"index\":1,\"name\":\"t1\",\"type\":\"TIMESTAMP(3)\"}}"),
TestSpec.forTransform(
new SubstringTransform(
Arrays.asList(
@@ -171,6 +180,21 @@ class TransformJsonSerdeTest {
.expectJson(
"{\"name\":\"TRIM\",\"inputs\":[{\"index\":1,\"name\":\"f1\",\"type\":\"STRING\"},\"x\"],\"trimFlag\":\"TRAILING\"}"),
+ // LengthTransform
+ TestSpec.forTransform(
+ new LengthTransform(
+ Collections.singletonList(
+ new FieldRef(1, "f1", DataTypes.STRING()))))
+ .expectJson(
+ "{\"name\":\"LENGTH\",\"inputs\":[{\"index\":1,\"name\":\"f1\",\"type\":\"STRING\"}]}"),
+ TestSpec.forTransform(
+ new LengthTransform(
+ Collections.singletonList(
+ BinaryString.fromString("hello"))))
+ .expectJson("{\"name\":\"LENGTH\",\"inputs\":[\"hello\"]}"),
+ TestSpec.forTransform(new LengthTransform(Collections.singletonList(null)))
+ .expectJson("{\"name\":\"LENGTH\",\"inputs\":[null]}"),
+
// error message testing
TestSpec.forJson("{\"name\":\"invalid\"}")
.expectErrorMessage("Could not resolve type id 'invalid'"),
@@ -184,7 +208,21 @@ class TransformJsonSerdeTest {
.expectErrorMessage("position must be an integer"),
TestSpec.forJson("{\"name\":\"SUBSTRING\",\"inputs\":[123,1,1]}")
.expectErrorMessage(
- "SUBSTRING source must be a string or a field reference"));
+ "SUBSTRING source must be a string or a field reference"),
+ TestSpec.forJson(
+ "{\"name\":\"YEAR\",\"fieldRef\":{\"index\":0,\"name\":\"f0\",\"type\":\"STRING\"}}")
+ .expectErrorMessage(
+ "YEAR requires a DATE or TIMESTAMP field, found STRING"),
+ TestSpec.forJson("{\"name\":\"LENGTH\",\"inputs\":[]}")
+ .expectErrorMessage("LENGTH requires exactly one input"),
+ TestSpec.forJson(
+ "{\"name\":\"LENGTH\",\"inputs\":[{\"index\":0,\"name\":\"f0\",\"type\":\"INT\"}]}")
+ .expectErrorMessage("LENGTH input must be a string field"),
+ TestSpec.forJson(
+ "{\"name\":\"LENGTH\",\"inputs\":[{\"index\":0,\"name\":\"f0\",\"type\":\"INT\"},{\"index\":1,\"name\":\"f1\",\"type\":\"STRING\"}]}")
+ .expectErrorMessage("LENGTH requires exactly one input"),
+ TestSpec.forJson("{\"name\":\"LENGTH\",\"inputs\":[5]}")
+ .expectErrorMessage("Unsupported StringTransform input JSON"));
}
@ParameterizedTest(name = "{index}: {0}")
diff --git a/paimon-format/src/main/java/org/apache/paimon/format/orc/filter/OrcPredicateFunctionVisitor.java b/paimon-format/src/main/java/org/apache/paimon/format/orc/filter/OrcPredicateFunctionVisitor.java
index f72478c8a1..3ee59e2c50 100644
--- a/paimon-format/src/main/java/org/apache/paimon/format/orc/filter/OrcPredicateFunctionVisitor.java
+++ b/paimon-format/src/main/java/org/apache/paimon/format/orc/filter/OrcPredicateFunctionVisitor.java
@@ -72,16 +72,31 @@ public class OrcPredicateFunctionVisitor
return Optional.empty();
}
+ @Override
+ public Optional<OrcFilters.Predicate> visitNotStartsWith(FieldRef fieldRef, Object literal) {
+ return Optional.empty();
+ }
+
@Override
public Optional<OrcFilters.Predicate> visitEndsWith(FieldRef fieldRef, Object literal) {
return Optional.empty();
}
+ @Override
+ public Optional<OrcFilters.Predicate> visitNotEndsWith(FieldRef fieldRef, Object literal) {
+ return Optional.empty();
+ }
+
@Override
public Optional<OrcFilters.Predicate> visitContains(FieldRef fieldRef, Object literal) {
return Optional.empty();
}
+ @Override
+ public Optional<OrcFilters.Predicate> visitNotContains(FieldRef fieldRef, Object literal) {
+ return Optional.empty();
+ }
+
@Override
public Optional<OrcFilters.Predicate> visitArrayContains(FieldRef fieldRef, Object literal) {
return Optional.empty();
@@ -104,6 +119,13 @@ public class OrcPredicateFunctionVisitor
return Optional.empty();
}
+ @Override
+ public Optional<OrcFilters.Predicate> visitNotLike(FieldRef fieldRef, Object literal) {
+ // ORC SearchArgument has no not-like leaf, so skip push-down and let the engine
+ // evaluate the filter (consistent with the Parquet path).
+ return Optional.empty();
+ }
+
@Override
public Optional<OrcFilters.Predicate> visitLessThan(FieldRef fieldRef, Object literal) {
return convertBinary(fieldRef, literal, OrcFilters.LessThan::new);
diff --git a/paimon-format/src/test/java/org/apache/paimon/format/orc/filter/OrcFilterConverterTest.java b/paimon-format/src/test/java/org/apache/paimon/format/orc/filter/OrcFilterConverterTest.java
index 40ceaaa0b3..03b3e88727 100644
--- a/paimon-format/src/test/java/org/apache/paimon/format/orc/filter/OrcFilterConverterTest.java
+++ b/paimon-format/src/test/java/org/apache/paimon/format/orc/filter/OrcFilterConverterTest.java
@@ -18,6 +18,7 @@
package org.apache.paimon.format.orc.filter;
+import org.apache.paimon.data.BinaryString;
import org.apache.paimon.data.Decimal;
import org.apache.paimon.predicate.LeafPredicate;
import org.apache.paimon.predicate.Predicate;
@@ -253,6 +254,20 @@ public class OrcFilterConverterTest {
.isEqualTo(Optional.empty());
}
+ @Test
+ public void testNotLike() {
+ PredicateBuilder builder =
+ new PredicateBuilder(
+ new RowType(
+ Collections.singletonList(
+ new DataField(0, "stringField", new VarCharType()))));
+
+ assertThat(
+ builder.notLike(0, BinaryString.fromString("%value%"))
+ .visit(OrcPredicateFunctionVisitor.VISITOR))
+ .isEqualTo(Optional.empty());
+ }
+
private void test(Predicate predicate, OrcFilters.Predicate orcPredicate, boolean canPushDown) {
Optional<OrcFilters.Predicate> optionalPredicate =
predicate.visit(OrcPredicateFunctionVisitor.VISITOR);
diff --git a/paimon-format/src/test/java/org/apache/paimon/format/parquet/ParquetFiltersTest.java b/paimon-format/src/test/java/org/apache/paimon/format/parquet/ParquetFiltersTest.java
index fd677c3849..b1119bc345 100644
--- a/paimon-format/src/test/java/org/apache/paimon/format/parquet/ParquetFiltersTest.java
+++ b/paimon-format/src/test/java/org/apache/paimon/format/parquet/ParquetFiltersTest.java
@@ -18,6 +18,7 @@
package org.apache.paimon.format.parquet;
+import org.apache.paimon.data.BinaryString;
import org.apache.paimon.data.Decimal;
import org.apache.paimon.data.Timestamp;
import org.apache.paimon.predicate.Predicate;
@@ -70,6 +71,20 @@ import static org.assertj.core.api.Assertions.assertThat;
class ParquetFiltersTest {
+ @Test
+ public void testNotLikeIsNotPushedDown() {
+ RowType rowType =
+ new RowType(
+ Collections.singletonList(new DataField(0, "string1", new VarCharType())));
+ MessageType schema = ParquetSchemaConverter.convertToParquetMessageType(rowType);
+ Predicate predicate =
+ new PredicateBuilder(rowType).notLike(0, BinaryString.fromString("%unsupported%"));
+
+ FilterCompat.Filter filter =
+ ParquetFilters.convert(PredicateBuilder.splitAnd(predicate), schema, true);
+ assertThat(filter).isEqualTo(FilterCompat.NOOP);
+ }
+
@Test
public void testBoolean() {
RowType rowType =
diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/SparkV2FilterConverter.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/SparkV2FilterConverter.scala
index 41e5c47c4a..6f5d215130 100644
--- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/SparkV2FilterConverter.scala
+++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/SparkV2FilterConverter.scala
@@ -73,6 +73,14 @@ case class SparkV2FilterConverter(rowType: RowType) extends Logging {
throw new UnsupportedOperationException(s"Convert $sparkPredicate is unsupported.")
}
+ case NOT_EQUAL =>
+ sparkPredicate match {
+ case BinaryPredicate(transform, literal) =>
+ builder.notEqual(transform, literal)
+ case _ =>
+ throw new UnsupportedOperationException(s"Convert $sparkPredicate is unsupported.")
+ }
+
case GREATER_THAN =>
sparkPredicate match {
case BinaryPredicate(transform, literal) =>
@@ -242,6 +250,7 @@ object SparkV2FilterConverter extends Logging {
private val EQUAL_TO = "="
private val EQUAL_NULL_SAFE = "<=>"
+ private val NOT_EQUAL = "<>"
private val GREATER_THAN = ">"
private val GREATER_THAN_OR_EQUAL = ">="
private val LESS_THAN = "<"
diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/util/SparkExpressionConverter.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/util/SparkExpressionConverter.scala
index 00898c98ca..b90f4902c4 100644
--- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/util/SparkExpressionConverter.scala
+++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/util/SparkExpressionConverter.scala
@@ -26,7 +26,7 @@ import org.apache.paimon.types.{DecimalType, RowType}
import org.apache.paimon.types.DataTypeRoot._
import org.apache.spark.sql.catalyst.util.{ArrayData, DateTimeUtils}
-import org.apache.spark.sql.connector.expressions.{Cast, Expression, GeneralScalarExpression, Literal, NamedReference}
+import org.apache.spark.sql.connector.expressions.{Cast, Expression, Extract, GeneralScalarExpression, Literal, NamedReference}
import org.apache.spark.sql.types.{ArrayType => SparkArrayType, DataType => SparkDataType}
import scala.collection.JavaConverters._
@@ -40,10 +40,19 @@ object SparkExpressionConverter {
private val UPPER = "UPPER"
private val LOWER = "LOWER"
private val SUBSTRING = "SUBSTRING"
+ private val CHAR_LENGTH = "CHAR_LENGTH"
private val TRIM = "TRIM"
private val LTRIM = "LTRIM"
private val RTRIM = "RTRIM"
+ // Supported fields of the EXTRACT expression
+ private val EXTRACT_YEAR = "YEAR"
+ private val EXTRACT_MONTH = "MONTH"
+ private val EXTRACT_DAY = "DAY"
+ private val EXTRACT_HOUR = "HOUR"
+ private val EXTRACT_MINUTE = "MINUTE"
+ private val EXTRACT_SECOND = "SECOND"
+
/** Convert Spark [[Expression]] to Paimon [[Transform]], return None if not supported. */
def toPaimonTransform(exp: Expression, rowType: RowType): Option[Transform] = {
@@ -68,6 +77,7 @@ object SparkExpressionConverter {
case UPPER => convertChildren(s.children()).map(i => new UpperTransform(i))
case LOWER => convertChildren(s.children()).map(i => new LowerTransform(i))
case SUBSTRING => convertChildren(s.children()).map(i => new SubstringTransform(i))
+ case CHAR_LENGTH => convertChildren(s.children()).map(i => new LengthTransform(i))
case TRIM =>
convertChildren(s.children()).map(i => new TrimTransform(i, TrimTransform.Flag.BOTH))
case LTRIM =>
@@ -85,6 +95,37 @@ object SparkExpressionConverter {
SparkTypeUtils.toPaimonType(c.dataType()))
case _ => None
}
+ // The connector `Extract` expression was added in Spark 3.4 and does not exist on
+ // Spark 3.2/3.3 runtimes, so its type test must stay behind this version gate to avoid
+ // a NoClassDefFoundError when linking the class there.
+ case e if org.apache.spark.SPARK_VERSION >= "3.4" =>
+ e match {
+ case extract: Extract =>
+ extract.source() match {
+ case n: NamedReference =>
+ val fieldRef = toPaimonFieldRef(n, rowType)
+ if (
+ fieldRef.`type`().getTypeRoot == TIMESTAMP_WITHOUT_TIME_ZONE &&
+ treatPaimonTimestampTypeAsSparkTimestampType()
+ ) {
+ // Legacy mapping exposes this Paimon type as Spark TIMESTAMP, whose extract
+ // semantics depend on the Spark session time zone.
+ None
+ } else {
+ extract.field() match {
+ case EXTRACT_YEAR => YearTransform.tryCreate(fieldRef)
+ case EXTRACT_MONTH => MonthTransform.tryCreate(fieldRef)
+ case EXTRACT_DAY => DayTransform.tryCreate(fieldRef)
+ case EXTRACT_HOUR => HourTransform.tryCreate(fieldRef)
+ case EXTRACT_MINUTE => MinuteTransform.tryCreate(fieldRef)
+ case EXTRACT_SECOND => SecondTransform.tryCreate(fieldRef)
+ case _ => None
+ }
+ }
+ case _ => None
+ }
+ case _ => None
+ }
case _ => None
}
}
diff --git a/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkFilterConverterTest.java b/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkFilterConverterTest.java
index 8b5457c9df..2e5cec362e 100644
--- a/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkFilterConverterTest.java
+++ b/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkFilterConverterTest.java
@@ -266,15 +266,20 @@ public class SparkFilterConverterTest {
@Test
public void testIgnoreFailure() {
List<DataField> dataFields = new ArrayList<>();
- dataFields.add(new DataField(0, "id", new IntType()));
+ dataFields.add(new DataField(0, "id", new FloatType()));
dataFields.add(new DataField(1, "name", new VarCharType(VarCharType.MAX_LENGTH)));
RowType rowType = new RowType(dataFields);
SparkFilterConverter converter = new SparkFilterConverter(rowType);
+ PredicateBuilder builder = new PredicateBuilder(rowType);
+
+ Not notStartsWith = Not.apply(StringStartsWith.apply("name", "paimon"));
+ assertThat(converter.convert(notStartsWith))
+ .isEqualTo(builder.startsWith(1, fromString("paimon")).negate().get());
- Not not = Not.apply(StringStartsWith.apply("name", "paimon"));
- assertThatThrownBy(() -> converter.convert(not, false))
- .hasMessageContaining("Not(StringStartsWith(name,paimon)) is unsupported.");
- assertThat(converter.convert(not, true)).isNull();
- assertThat(converter.convertIgnoreFailure(not)).isNull();
+ Not unsupported = Not.apply(EqualTo.apply("id", Float.NaN));
+ assertThatThrownBy(() -> converter.convert(unsupported, false))
+ .hasMessageContaining("is unsupported.");
+ assertThat(converter.convert(unsupported, true)).isNull();
+ assertThat(converter.convertIgnoreFailure(unsupported)).isNull();
}
}
diff --git a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/SparkV2FilterConverterTestBase.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/SparkV2FilterConverterTestBase.scala
index a865bce9ec..7a87bd0666 100644
--- a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/SparkV2FilterConverterTestBase.scala
+++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/SparkV2FilterConverterTestBase.scala
@@ -19,7 +19,7 @@
package org.apache.paimon.spark.sql
import org.apache.paimon.data.{BinaryString, Decimal, Timestamp}
-import org.apache.paimon.predicate.PredicateBuilder
+import org.apache.paimon.predicate.{DayTransform, FieldRef, HourTransform, LengthTransform, MinuteTransform, MonthTransform, PredicateBuilder, SecondTransform, YearTransform}
import org.apache.paimon.spark.{PaimonSparkTestBase, SparkV2FilterConverter}
import org.apache.paimon.spark.util.shim.TypeUtils.treatPaimonTimestampTypeAsSparkTimestampType
import org.apache.paimon.table.source.DataSplit
@@ -90,6 +90,34 @@ abstract class SparkV2FilterConverterTestBase extends PaimonSparkTestBase {
lazy val converter: SparkV2FilterConverter = SparkV2FilterConverter(rowType)
+ test("V2Filter: legacy timestamp mapping does not push down extract") {
+ if (gteqSpark3_4) {
+ withTimeZone("UTC") {
+ withSparkSQLConf(
+ "spark.paimon.legacy-timestamp-mapping.enabled" -> "true",
+ "spark.sql.session.timeZone" -> "America/Los_Angeles") {
+ withTable("legacy_extract") {
+ sql("""
+ |CREATE TABLE legacy_extract (id INT, ts TIMESTAMP)
+ |USING paimon PARTITIONED BY (ts)
+ |""".stripMargin)
+
+ val legacyRowType = loadTable("legacy_extract").rowType()
+ val legacyConverter = SparkV2FilterConverter(legacyRowType)
+ val filter = "hour(ts) = 1"
+ val condition =
+ sql(s"SELECT * FROM legacy_extract WHERE $filter").queryExecution.analyzed
+ .collectFirst { case f: Filter => f }
+ .get
+ .condition
+ val sparkPredicate = translateFilterV2(condition).get
+ assert(legacyConverter.convert(sparkPredicate).isEmpty)
+ }
+ }
+ }
+ }
+ }
+
test("V2Filter: all types") {
var filter = "string_col = 'hello'"
var actual = converter.convert(v2Filter(filter)).get
@@ -380,6 +408,106 @@ abstract class SparkV2FilterConverterTestBase extends PaimonSparkTestBase {
assert(scanFilesCount(filter) == 2)
}
+ test("V2Filter: NotEqual") {
+ val filter = "int_col <> 1"
+ val actual = converter.convert(v2Filter(filter)).get
+ assert(actual.equals(builder.notEqual(3, 1)))
+ checkAnswer(
+ sql(s"SELECT int_col from test_tbl WHERE $filter ORDER BY int_col"),
+ Seq(Row(2), Row(3)))
+ assert(scanFilesCount(filter) == 2)
+ }
+
+ test("V2Filter: CharLength") {
+ if (gteqSpark3_4) {
+ val filter = "char_length(string_col) = 2"
+ val transform = new LengthTransform(
+ List[Object](new FieldRef(0, "string_col", rowType.getTypeAt(0))).asJava)
+ val actual = converter.convert(v2Filter(filter)).get
+ assert(actual.equals(builder.equal(transform, 2)))
+ checkAnswer(sql(s"SELECT string_col from test_tbl WHERE $filter"), Seq(Row("hi")))
+ // CHAR_LENGTH cannot prune files by column stats.
+ assert(scanFilesCount(filter) == 4)
+ }
+ }
+
+ test("V2Filter: Year, Month and Day") {
+ if (gteqSpark3_4) {
+ val dateFieldRef = new FieldRef(9, "date_col", rowType.getTypeAt(9))
+
+ var filter = "year(date_col) = 2025"
+ var actual = converter.convert(v2Filter(filter)).get
+ assert(actual.equals(builder.equal(new YearTransform(dateFieldRef), 2025)))
+ checkAnswer(
+ sql(s"SELECT date_col from test_tbl WHERE $filter"),
+ sql("SELECT date_col from test_tbl"))
+ // Extracted fields cannot prune files by column stats.
+ assert(scanFilesCount(filter) == 4)
+
+ filter = "month(date_col) = 1"
+ actual = converter.convert(v2Filter(filter)).get
+ assert(actual.equals(builder.equal(new MonthTransform(dateFieldRef), 1)))
+ checkAnswer(
+ sql(s"SELECT date_col from test_tbl WHERE $filter"),
+ sql("SELECT date_col from test_tbl"))
+ assert(scanFilesCount(filter) == 4)
+
+ filter = "day(date_col) = 15"
+ actual = converter.convert(v2Filter(filter)).get
+ assert(actual.equals(builder.equal(new DayTransform(dateFieldRef), 15)))
+ checkAnswer(
+ sql(s"SELECT date_col from test_tbl WHERE $filter"),
+ sql("SELECT date('2025-01-15')"))
+ assert(scanFilesCount(filter) == 4)
+ }
+ }
+
+ test("V2Filter: Hour, Minute and Second") {
+ if (gteqSpark3_4) {
+ withTable("extract_tbl", "extract_ltz_tbl") {
+ sql("CREATE TABLE extract_tbl (ts_col TIMESTAMP_NTZ) USING paimon")
+ sql("INSERT INTO extract_tbl VALUES (timestamp_ntz'2025-01-15 01:02:03')")
+ sql("INSERT INTO extract_tbl VALUES (timestamp_ntz'2025-01-16 04:05:06')")
+
+ val ntzRowType = loadTable("extract_tbl").rowType()
+ val ntzBuilder = new PredicateBuilder(ntzRowType)
+ val ntzConverter = SparkV2FilterConverter(ntzRowType)
+ val tsFieldRef = new FieldRef(0, "ts_col", ntzRowType.getTypeAt(0))
+
+ var filter = "hour(ts_col) = 1"
+ var actual = ntzConverter.convert(v2Filter(filter, "extract_tbl")).get
+ assert(actual.equals(ntzBuilder.equal(new HourTransform(tsFieldRef), 1)))
+ checkAnswer(
+ sql(s"SELECT ts_col from extract_tbl WHERE $filter"),
+ sql("SELECT timestamp_ntz'2025-01-15 01:02:03'"))
+ assert(scanFilesCount(filter, "extract_tbl") == 2)
+
+ filter = "minute(ts_col) = 2"
+ actual = ntzConverter.convert(v2Filter(filter, "extract_tbl")).get
+ assert(actual.equals(ntzBuilder.equal(new MinuteTransform(tsFieldRef), 2)))
+ checkAnswer(
+ sql(s"SELECT ts_col from extract_tbl WHERE $filter"),
+ sql("SELECT timestamp_ntz'2025-01-15 01:02:03'"))
+ assert(scanFilesCount(filter, "extract_tbl") == 2)
+
+ filter = "second(ts_col) = 3"
+ actual = ntzConverter.convert(v2Filter(filter, "extract_tbl")).get
+ assert(actual.equals(ntzBuilder.equal(new SecondTransform(tsFieldRef), 3)))
+ checkAnswer(
+ sql(s"SELECT ts_col from extract_tbl WHERE $filter"),
+ sql("SELECT timestamp_ntz'2025-01-15 01:02:03'"))
+ assert(scanFilesCount(filter, "extract_tbl") == 2)
+
+ // Spark TIMESTAMP maps to a Paimon local-time-zone timestamp by default, which the
+ // date/time extract transforms do not support, so the conversion degrades.
+ sql("CREATE TABLE extract_ltz_tbl (ts_col TIMESTAMP) USING paimon")
+ val ltzRowType = loadTable("extract_ltz_tbl").rowType()
+ val ltzConverter = SparkV2FilterConverter(ltzRowType)
+ assert(ltzConverter.convert(v2Filter("hour(ts_col) = 1", "extract_ltz_tbl")).isEmpty)
+ }
+ }
+ }
+
test("V2Filter: StartWith") {
val filter = "string_col LIKE 'h%'"
val actual = converter.convert(v2Filter(filter)).get
@@ -412,6 +540,29 @@ abstract class SparkV2FilterConverterTestBase extends PaimonSparkTestBase {
assert(scanFilesCount(filter) == 4)
}
+ test("V2Filter: Not string predicates") {
+ Seq(
+ (
+ "string_col NOT LIKE 'h%'",
+ builder.startsWith(0, BinaryString.fromString("h")).negate().get(),
+ Seq(Row("paimon"), Row("world"))),
+ (
+ "string_col NOT LIKE '%d'",
+ builder.endsWith(0, BinaryString.fromString("d")).negate().get(),
+ Seq(Row("hello"), Row("hi"), Row("paimon"))),
+ (
+ "string_col NOT LIKE '%orl%'",
+ builder.contains(0, BinaryString.fromString("orl")).negate().get(),
+ Seq(Row("hello"), Row("hi"), Row("paimon")))
+ ).foreach {
+ case (filter, expectedPredicate, expectedRows) =>
+ assert(converter.convert(v2Filter(filter)).contains(expectedPredicate))
+ checkAnswer(
+ sql(s"SELECT string_col from test_tbl WHERE $filter ORDER BY string_col"),
+ expectedRows)
+ }
+ }
+
private def paimonAlwaysTrue: org.apache.paimon.predicate.Predicate =
PredicateBuilder.alwaysTrue()
(paimon) 16/20: [python] Make BLOB I/O merge parameters configurable (#9890)
Posted by ho...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
XiaoHongbo-Hope pushed a commit to branch release-2.1
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit d6644bbc979af08e343b36fe5daad95d16c651c2
Author: XiaoHongbo <xi...@alibaba-inc.com>
AuthorDate: Thu Sep 17 11:31:27 2026 +0800
[python] Make BLOB I/O merge parameters configurable (#9890)
---
paimon-python/README.md | 19 ++++
paimon-python/pypaimon/common/file_io.py | 22 ++--
paimon-python/pypaimon/common/options/config.py | 20 ++++
paimon-python/pypaimon/tests/blob_test.py | 132 +++++++++++++++++-------
4 files changed, 150 insertions(+), 43 deletions(-)
diff --git a/paimon-python/README.md b/paimon-python/README.md
index c745cdc261..b3541b1bdb 100644
--- a/paimon-python/README.md
+++ b/paimon-python/README.md
@@ -207,6 +207,25 @@ and precomputed primary-key global-index results still use the Python planner.
Continuous streaming and write planning also retain their Python entrypoints.
Native planning remains optional and is disabled by default.
+# Coalesced BLOB reads
+
+FileIO merges nearby BLOB ranges before reading. Set
+`file-io.read-coalesce.max-gap` and `file-io.read-coalesce.max-block` in the
+catalog or connection options to tune the 1 MiB and 8 MiB defaults:
+
+```python
+import pypaimon.multimodal as pmm
+
+connection = pmm.connect(options={
+ "warehouse": "/tmp/warehouse",
+ "file-io.read-coalesce.max-gap": "64 kb",
+ "file-io.read-coalesce.max-block": "16 mb",
+})
+```
+
+`max-block` constrains coalescing, but does not split an individual BLOB range.
+A single read can therefore exceed this value.
+
# Load LeRobot Dataset v3
Install the optional dependency, then import a local directory, FileIO URI, or
diff --git a/paimon-python/pypaimon/common/file_io.py b/paimon-python/pypaimon/common/file_io.py
index 720f9ae510..3baf4813a5 100644
--- a/paimon-python/pypaimon/common/file_io.py
+++ b/paimon-python/pypaimon/common/file_io.py
@@ -50,10 +50,6 @@ def pread(stream, length: int, offset: int) -> bytes:
return os.pread(stream.fileno(), length, offset)
-# Coalescing bounds: merge same-file ranges whose gap is within GAP, capping a
-# merged read at SPAN so threads stay busy and memory stays bounded.
-_COALESCE_GAP = 1 << 20
-_COALESCE_SPAN = 8 << 20
_COALESCE_VIEW_MAX_RETAINED_AMPLIFICATION = 2.0
# Bound per-object opens; 16 cuts them by 75% for default 64-range batches.
_MAX_RANGE_LANES_PER_PATH = 16
@@ -187,8 +183,7 @@ class FileIO(ABC):
finally:
stream.close()
- def read_ranges_coalesced(self, ranges, parallelism,
- max_gap=_COALESCE_GAP, max_span=_COALESCE_SPAN):
+ def read_ranges_coalesced(self, ranges, parallelism):
"""Read ``ranges`` (each ``None`` or ``(path, offset, length)``), returning
bytes in the same order. Same-file nearby ranges are merged into one read
to cut round trips, then sliced. Each worker lane reuses one exclusive
@@ -198,12 +193,12 @@ class FileIO(ABC):
A failed read propagates and aborts the whole batch (unlike a per-row
``file.open()`` loop that fails one row at a time).
"""
+ max_gap, max_span = self._resolve_coalesce_limits()
return self._read_ranges_coalesced(
ranges, parallelism, max_gap, max_span,
max_retained_amplification=0, return_views=False)
- def read_ranges_coalesced_views(self, ranges, parallelism,
- max_gap=_COALESCE_GAP, max_span=_COALESCE_SPAN,
+ def read_ranges_coalesced_views(self, ranges, parallelism, *,
max_retained_amplification=(
_COALESCE_VIEW_MAX_RETAINED_AMPLIFICATION)):
"""Read coalesced ranges as zero-copy ``memoryview`` slices.
@@ -216,10 +211,21 @@ class FileIO(ABC):
excessive gap bytes; set ``max_retained_amplification`` to a non-positive
value to always share the merged buffer.
"""
+ max_gap, max_span = self._resolve_coalesce_limits()
return self._read_ranges_coalesced(
ranges, parallelism, max_gap, max_span, max_retained_amplification,
return_views=True)
+ def _resolve_coalesce_limits(self):
+ from pypaimon.common.options.config import FileIOOptions
+ properties = getattr(self, "properties", None)
+ if not isinstance(properties, Options):
+ properties = Options({})
+ return (
+ properties.get(FileIOOptions.READ_COALESCE_MAX_GAP).get_bytes(),
+ properties.get(FileIOOptions.READ_COALESCE_MAX_BLOCK).get_bytes(),
+ )
+
def _read_ranges_coalesced(self, ranges, parallelism, max_gap, max_span,
max_retained_amplification, return_views):
from concurrent.futures import ThreadPoolExecutor
diff --git a/paimon-python/pypaimon/common/options/config.py b/paimon-python/pypaimon/common/options/config.py
index 98a466496b..80095404ff 100644
--- a/paimon-python/pypaimon/common/options/config.py
+++ b/paimon-python/pypaimon/common/options/config.py
@@ -146,6 +146,26 @@ class CatalogOptions:
BLOB_FILE_IO_DEFAULT_CACHE_SIZE = 2 ** 31 - 1
+class FileIOOptions:
+ READ_COALESCE_MAX_GAP = (
+ ConfigOptions.key("file-io.read-coalesce.max-gap")
+ .memory_type()
+ .default_value(MemorySize.of_mebi_bytes(1))
+ .with_description(
+ "Maximum gap between same-file ranges merged into one read."
+ )
+ )
+ READ_COALESCE_MAX_BLOCK = (
+ ConfigOptions.key("file-io.read-coalesce.max-block")
+ .memory_type()
+ .default_value(MemorySize.of_mebi_bytes(8))
+ .with_description(
+ "Maximum span for coalescing same-file ranges, except when an "
+ "individual range is larger. Individual ranges are not split."
+ )
+ )
+
+
class HdfsOptions:
HDFS_CLIENT_IMPL = (
ConfigOptions.key("hdfs.client.impl")
diff --git a/paimon-python/pypaimon/tests/blob_test.py b/paimon-python/pypaimon/tests/blob_test.py
index 456d8dc2b2..00a721105e 100644
--- a/paimon-python/pypaimon/tests/blob_test.py
+++ b/paimon-python/pypaimon/tests/blob_test.py
@@ -4135,6 +4135,68 @@ class CoalesceRangesTest(unittest.TestCase):
self.assertEqual(got[4], data[100:]) # length -1 => read to EOF
self.assertIsNone(got[5]) # None offset/length => skipped
+ def test_coalesce_limits_from_file_io_options(self):
+ from pypaimon.common.options.config import FileIOOptions
+ self.assertEqual(1 << 20, Options({}).get(
+ FileIOOptions.READ_COALESCE_MAX_GAP).get_bytes())
+ self.assertEqual(8 << 20, Options({}).get(
+ FileIOOptions.READ_COALESCE_MAX_BLOCK).get_bytes())
+ data = bytes(range(256))
+ with tempfile.TemporaryDirectory() as tmp_dir:
+ path = os.path.join(tmp_dir, "f.bin")
+ with open(path, "wb") as output:
+ output.write(data)
+ file_io = FileIO.get(
+ "file://" + tmp_dir,
+ Options({
+ "file-io.read-coalesce.max-gap": "64 b",
+ "file-io.read-coalesce.max-block": "100 b",
+ }),
+ )
+ reads = []
+ original_open = file_io.new_input_stream
+
+ def new_input_stream(file_path):
+ stream = original_open(file_path)
+
+ class TrackingStream:
+ def read_at(self, length, offset):
+ reads.append((offset, length))
+ return os.pread(stream.fileno(), length, offset)
+
+ def close(self):
+ stream.close()
+
+ return TrackingStream()
+
+ file_io.new_input_stream = new_input_stream
+ ranges = [(path, 0, 10), (path, 60, 10), (path, 120, 10)]
+
+ self.assertEqual(
+ [data[0:10], data[60:70], data[120:130]],
+ file_io.read_ranges_coalesced(ranges, parallelism=3),
+ )
+ self.assertEqual([(0, 70), (120, 10)], sorted(reads))
+
+ reads.clear()
+ views = file_io.read_ranges_coalesced_views(ranges, parallelism=3)
+ self.assertEqual([data[0:10], data[60:70], data[120:130]],
+ [bytes(view) for view in views])
+ self.assertEqual([(0, 70), (120, 10)], sorted(reads))
+
+ reads.clear()
+ file_io.properties.set(
+ FileIOOptions.READ_COALESCE_MAX_GAP, "0 b")
+ file_io.read_ranges_coalesced(
+ ranges, parallelism=3)
+ self.assertEqual([(0, 10), (60, 10), (120, 10)], sorted(reads))
+
+ reads.clear()
+ file_io.properties.set(
+ FileIOOptions.READ_COALESCE_MAX_GAP, "64 b")
+ file_io.read_ranges_coalesced(ranges, parallelism=3)
+ self.assertEqual([(0, 70), (120, 10)], sorted(reads))
+
def test_read_ranges_coalesced_views(self):
from pypaimon.common.file_io import FileIO
data = bytes(range(256)) * 4
@@ -4142,11 +4204,13 @@ class CoalesceRangesTest(unittest.TestCase):
path = os.path.join(tmp_dir, "f.bin")
with open(path, 'wb') as output:
output.write(data)
- file_io = FileIO.get(f"file://{tmp_dir}", {})
+ file_io = FileIO.get(
+ f"file://{tmp_dir}",
+ Options({"file-io.read-coalesce.max-gap": "100 b"}))
ranges = [(path, 0, 10), (path, 10, 10), None,
(path, 500, 20), (path, 100, -1)]
got = file_io.read_ranges_coalesced_views(
- ranges, parallelism=4, max_gap=100)
+ ranges, parallelism=4)
self.assertIsInstance(got[0], memoryview)
self.assertIsInstance(got[1], memoryview)
@@ -4170,7 +4234,10 @@ class CoalesceRangesTest(unittest.TestCase):
path = os.path.join(tmp_dir, "f.bin")
with open(path, 'wb') as output:
output.write(data)
- file_io = FileIO.get(f"file://{tmp_dir}", {})
+ file_io = FileIO.get(
+ f"file://{tmp_dir}",
+ Options({"file-io.read-coalesce.max-gap": "1000 b",
+ "file-io.read-coalesce.max-block": "1 mb"}))
reads = []
original_open = file_io.new_input_stream
@@ -4191,8 +4258,6 @@ class CoalesceRangesTest(unittest.TestCase):
got = file_io.read_ranges_coalesced_views(
[(path, 0, 10), (path, 1000, 10)],
parallelism=4,
- max_gap=1000,
- max_span=1 << 20,
)
self.assertEqual(reads, [(path, 0, 1010)])
@@ -4204,8 +4269,6 @@ class CoalesceRangesTest(unittest.TestCase):
shared = file_io.read_ranges_coalesced_views(
[(path, 0, 10), (path, 1000, 10)],
parallelism=4,
- max_gap=1000,
- max_span=1 << 20,
max_retained_amplification=0,
)
self.assertEqual(reads, [(path, 0, 1010)])
@@ -4219,7 +4282,9 @@ class CoalesceRangesTest(unittest.TestCase):
path = os.path.join(tmp_dir, "blob.bin")
with open(path, "wb") as output:
output.write(data)
- file_io = FileIO.get(f"file://{tmp_dir}", {})
+ file_io = FileIO.get(
+ f"file://{tmp_dir}",
+ Options({"file-io.read-coalesce.max-gap": "0 b"}))
fallbacks = []
class FailingStream:
@@ -4239,8 +4304,7 @@ class CoalesceRangesTest(unittest.TestCase):
self.assertEqual(
[data[0:4], data[16:20]],
- file_io.read_ranges_coalesced(
- ranges, parallelism=2, max_gap=0),
+ file_io.read_ranges_coalesced(ranges, parallelism=2),
)
self.assertEqual(2, len(fallbacks))
@@ -4297,7 +4361,7 @@ class CoalesceRangesTest(unittest.TestCase):
self.assertEqual(
[b"ok"] * parallelism,
file_io.read_ranges_coalesced(
- ranges, parallelism=parallelism, max_gap=0),
+ ranges, parallelism=parallelism),
)
self.assertLessEqual(max_open_streams, parallelism)
self.assertEqual(0, open_streams)
@@ -4344,7 +4408,6 @@ class CoalesceRangesTest(unittest.TestCase):
file_io.read_ranges_coalesced(
[("blob", 0, 5), ("blob", 5, -1)],
parallelism=1,
- max_gap=0,
),
)
self.assertEqual([
@@ -4359,7 +4422,8 @@ class CoalesceRangesTest(unittest.TestCase):
from pypaimon.common.file_io import FileIO
data = bytes(range(128))
- file_io = FileIO.get("file:///tmp", {})
+ file_io = FileIO.get(
+ "file:///tmp", Options({"file-io.read-coalesce.max-gap": "0 b"}))
class SerialStream:
def __init__(self):
@@ -4396,8 +4460,7 @@ class CoalesceRangesTest(unittest.TestCase):
self.assertEqual(
[data[i * 4:i * 4 + 2] for i in range(16)],
- file_io.read_ranges_coalesced(
- ranges, parallelism=8, max_gap=0),
+ file_io.read_ranges_coalesced(ranges, parallelism=8),
)
self.assertGreater(len(streams), 1)
self.assertLessEqual(len(streams), 8)
@@ -4406,7 +4469,8 @@ class CoalesceRangesTest(unittest.TestCase):
from pypaimon.common.file_io import FileIO
data = bytes(range(256)) * 64
- file_io = FileIO.get("file:///tmp", {})
+ file_io = FileIO.get(
+ "file:///tmp", Options({"file-io.read-coalesce.max-gap": "0 b"}))
streams = []
class PositionalStream:
@@ -4440,8 +4504,7 @@ class CoalesceRangesTest(unittest.TestCase):
self.assertEqual(
[data[offset:offset + length]
for _, offset, length in ranges],
- file_io.read_ranges_coalesced(
- ranges, parallelism=64, max_gap=0),
+ file_io.read_ranges_coalesced(ranges, parallelism=64),
)
self.assertEqual(16, len(streams))
self.assertTrue(all(stream.reads == 4 for stream in streams))
@@ -4452,7 +4515,8 @@ class CoalesceRangesTest(unittest.TestCase):
from pypaimon.common.file_io import FileIO
large = 8 << 20
- file_io = FileIO.get("file:///tmp", {})
+ file_io = FileIO.get(
+ "file:///tmp", Options({"file-io.read-coalesce.max-gap": "0 b"}))
streams = []
class PositionalStream:
@@ -4479,8 +4543,7 @@ class CoalesceRangesTest(unittest.TestCase):
ranges.append(("blob", offset, length))
offset += length + 1
- file_io.read_ranges_coalesced(
- ranges, parallelism=64, max_gap=0)
+ file_io.read_ranges_coalesced(ranges, parallelism=64)
self.assertEqual(16, len(streams))
self.assertEqual(
@@ -4493,7 +4556,8 @@ class CoalesceRangesTest(unittest.TestCase):
from pypaimon.common.file_io import FileIO
- file_io = FileIO.get("file:///tmp", {})
+ file_io = FileIO.get(
+ "file:///tmp", Options({"file-io.read-coalesce.max-gap": "0 b"}))
streams = Counter()
lock = threading.Lock()
@@ -4518,8 +4582,7 @@ class CoalesceRangesTest(unittest.TestCase):
+ [("cold", index * 2, 1) for index in range(100)]
)
- result = file_io.read_ranges_coalesced(
- ranges, parallelism=64, max_gap=0)
+ result = file_io.read_ranges_coalesced(ranges, parallelism=64)
self.assertEqual([b"h"] * 9900 + [b"c"] * 100, result)
self.assertEqual(Counter({"hot": 16, "cold": 16}), streams)
@@ -4529,7 +4592,8 @@ class CoalesceRangesTest(unittest.TestCase):
from pypaimon.common.file_io import FileIO
- file_io = FileIO.get("file:///tmp", {})
+ file_io = FileIO.get(
+ "file:///tmp", Options({"file-io.read-coalesce.max-gap": "0 b"}))
streams = Counter()
lock = threading.Lock()
active_streams = 0
@@ -4566,8 +4630,7 @@ class CoalesceRangesTest(unittest.TestCase):
+ [("cold-%d" % index, 0, 1) for index in range(63)]
)
- result = file_io.read_ranges_coalesced(
- ranges, parallelism=64, max_gap=0)
+ result = file_io.read_ranges_coalesced(ranges, parallelism=64)
self.assertEqual(10063, len(result))
self.assertEqual(16, streams["hot"])
@@ -4577,7 +4640,8 @@ class CoalesceRangesTest(unittest.TestCase):
def test_stream_count_is_bounded_across_paths(self):
from pypaimon.common.file_io import FileIO
- file_io = FileIO.get("file:///tmp", {})
+ file_io = FileIO.get(
+ "file:///tmp", Options({"file-io.read-coalesce.max-gap": "0 b"}))
lock = threading.Lock()
open_streams = 0
max_open_streams = 0
@@ -4610,8 +4674,7 @@ class CoalesceRangesTest(unittest.TestCase):
self.assertEqual(
[bytes([offset]) * length for _, offset, length in ranges],
- file_io.read_ranges_coalesced(
- ranges, parallelism=4, max_gap=0),
+ file_io.read_ranges_coalesced(ranges, parallelism=4),
)
self.assertLessEqual(max_open_streams, 4)
self.assertEqual(4, total_streams)
@@ -4621,7 +4684,8 @@ class CoalesceRangesTest(unittest.TestCase):
from pypaimon.common.file_io import FileIO
data = bytes(range(128))
- file_io = FileIO.get("file:///tmp", {})
+ file_io = FileIO.get(
+ "file:///tmp", Options({"file-io.read-coalesce.max-gap": "0 b"}))
streams = []
class CloseStream:
@@ -4647,8 +4711,7 @@ class CoalesceRangesTest(unittest.TestCase):
ranges = [("blob", offset, 4) for offset in range(0, 64, 8)]
with self.assertRaisesRegex(IOError, "first close failed"):
- file_io.read_ranges_coalesced(
- ranges, parallelism=4, max_gap=0)
+ file_io.read_ranges_coalesced(ranges, parallelism=4)
self.assertGreater(len(streams), 1)
self.assertTrue(all(stream.closed for stream in streams))
@@ -4680,7 +4743,6 @@ class CoalesceRangesTest(unittest.TestCase):
file_io.read_ranges_coalesced(
[("close-error", 0, 2), ("read-error", 0, 2)],
parallelism=2,
- max_gap=0,
)
def test_failed_stream_close_stops_before_fallback(self):
@@ -4705,7 +4767,7 @@ class CoalesceRangesTest(unittest.TestCase):
with self.assertRaisesRegex(IOError, "pooled read failed"):
file_io.read_ranges_coalesced(
- [("blob", 0, 2)], parallelism=1, max_gap=0)
+ [("blob", 0, 2)], parallelism=1)
self.assertEqual([], fallbacks)
(paimon) 08/20: [core] Manage manifest extra file lifecycle (#9886)
Posted by ho...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
XiaoHongbo-Hope pushed a commit to branch release-2.1
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit 301713fe6cb8106be9caf353048aff6443f81bb0
Author: YeJunHao <41...@users.noreply.github.com>
AuthorDate: Wed Sep 16 17:25:40 2026 +0800
[core] Manage manifest extra file lifecycle (#9886)
---
.../org/apache/paimon/manifest/ManifestFile.java | 8 ++
.../apache/paimon/operation/ChangelogDeletion.java | 13 ++-
.../apache/paimon/operation/FileDeletionBase.java | 20 +++-
.../paimon/operation/ManifestFileMerger.java | 2 +-
.../paimon/operation/commit/CommitCleaner.java | 6 +-
.../manifest/ManifestFileMetaSerializerTest.java | 3 +-
.../apache/paimon/manifest/ManifestFileTest.java | 22 +++++
.../paimon/manifest/ManifestIndexTestUtils.java | 99 +++++++++++++++++++
.../paimon/operation/ExpireSnapshotsTest.java | 72 ++++++++++++++
.../operation/LocalOrphanFilesCleanTest.java | 35 +++++++
.../operation/ManifestSidecarLifecycleTest.java | 106 +++++++++++++++++++++
11 files changed, 370 insertions(+), 16 deletions(-)
diff --git a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFile.java b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFile.java
index 0dc99a0470..9781be4b90 100644
--- a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFile.java
+++ b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFile.java
@@ -339,6 +339,14 @@ public class ManifestFile extends ObjectsFile<ManifestEntry> {
};
}
+ /** Deletes an unreferenced manifest and its explicitly referenced extra files. */
+ public void delete(ManifestFileMeta manifest) {
+ delete(manifest.fileName());
+ if (manifest.extraFiles() != null) {
+ manifest.extraFiles().forEach(this::delete);
+ }
+ }
+
/** Creator of {@link ManifestFile}. */
public static class Factory {
diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/ChangelogDeletion.java b/paimon-core/src/main/java/org/apache/paimon/operation/ChangelogDeletion.java
index 9689f272e2..eec6090a36 100644
--- a/paimon-core/src/main/java/org/apache/paimon/operation/ChangelogDeletion.java
+++ b/paimon-core/src/main/java/org/apache/paimon/operation/ChangelogDeletion.java
@@ -26,7 +26,6 @@ import org.apache.paimon.index.IndexFileMeta;
import org.apache.paimon.manifest.ExpireFileEntry;
import org.apache.paimon.manifest.IndexManifestEntry;
import org.apache.paimon.manifest.ManifestFile;
-import org.apache.paimon.manifest.ManifestFileMeta;
import org.apache.paimon.manifest.ManifestList;
import org.apache.paimon.stats.StatsFileHandler;
import org.apache.paimon.utils.FileStorePathFactory;
@@ -100,17 +99,17 @@ public class ChangelogDeletion extends FileDeletionBase<Changelog> {
// base manifests
if (manifestList.exists(skippingSnapshot.baseManifestList())) {
skippingSet.add(skippingSnapshot.baseManifestList());
- manifestList.read(skippingSnapshot.baseManifestList()).stream()
- .map(ManifestFileMeta::fileName)
- .forEach(skippingSet::add);
+ manifestList
+ .read(skippingSnapshot.baseManifestList())
+ .forEach(manifest -> addManifestToSkippingSet(skippingSet, manifest));
}
// delta manifests
if (manifestList.exists(skippingSnapshot.deltaManifestList())) {
skippingSet.add(skippingSnapshot.deltaManifestList());
- manifestList.read(skippingSnapshot.deltaManifestList()).stream()
- .map(ManifestFileMeta::fileName)
- .forEach(skippingSet::add);
+ manifestList
+ .read(skippingSnapshot.deltaManifestList())
+ .forEach(manifest -> addManifestToSkippingSet(skippingSet, manifest));
}
// index manifests
diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/FileDeletionBase.java b/paimon-core/src/main/java/org/apache/paimon/operation/FileDeletionBase.java
index 6494411a8c..ad544af0ea 100644
--- a/paimon-core/src/main/java/org/apache/paimon/operation/FileDeletionBase.java
+++ b/paimon-core/src/main/java/org/apache/paimon/operation/FileDeletionBase.java
@@ -327,7 +327,11 @@ public abstract class FileDeletionBase<T extends Snapshot> {
if (skippingSet.add(fileName)) {
manifests.add(fileName);
if (manifest.extraFiles() != null) {
- manifests.addAll(manifest.extraFiles());
+ for (String extraFile : manifest.extraFiles()) {
+ if (skippingSet.add(extraFile)) {
+ manifests.add(extraFile);
+ }
+ }
}
}
}
@@ -489,9 +493,9 @@ public abstract class FileDeletionBase<T extends Snapshot> {
// data manifests
skippingSet.add(skippingSnapshot.baseManifestList());
skippingSet.add(skippingSnapshot.deltaManifestList());
- manifestList.readDataManifests(skippingSnapshot).stream()
- .map(ManifestFileMeta::fileName)
- .forEach(skippingSet::add);
+ manifestList
+ .readDataManifests(skippingSnapshot)
+ .forEach(manifest -> addManifestToSkippingSet(skippingSet, manifest));
// index manifests
String indexManifest = skippingSnapshot.indexManifest();
@@ -511,6 +515,14 @@ public abstract class FileDeletionBase<T extends Snapshot> {
return skippingSet;
}
+ protected static void addManifestToSkippingSet(
+ Set<String> skippingSet, ManifestFileMeta manifest) {
+ skippingSet.add(manifest.fileName());
+ if (manifest.extraFiles() != null) {
+ skippingSet.addAll(manifest.extraFiles());
+ }
+ }
+
private boolean tryDeleteEmptyDirectory(Path path) {
try {
fileIO.delete(path, false);
diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/ManifestFileMerger.java b/paimon-core/src/main/java/org/apache/paimon/operation/ManifestFileMerger.java
index 79d21e025e..eff0d00908 100644
--- a/paimon-core/src/main/java/org/apache/paimon/operation/ManifestFileMerger.java
+++ b/paimon-core/src/main/java/org/apache/paimon/operation/ManifestFileMerger.java
@@ -95,7 +95,7 @@ public class ManifestFileMerger {
// exception occurs, clean up and rethrow
for (ManifestFileMeta manifest : newFilesForAbort) {
try {
- manifestFile.delete(manifest.fileName());
+ manifestFile.delete(manifest);
} catch (Throwable cleanupFailure) {
primaryFailure =
ExceptionUtils.firstOrSuppressed(cleanupFailure, primaryFailure);
diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/commit/CommitCleaner.java b/paimon-core/src/main/java/org/apache/paimon/operation/commit/CommitCleaner.java
index a24b5c4c6e..735a706937 100644
--- a/paimon-core/src/main/java/org/apache/paimon/operation/commit/CommitCleaner.java
+++ b/paimon-core/src/main/java/org/apache/paimon/operation/commit/CommitCleaner.java
@@ -52,14 +52,14 @@ public class CommitCleaner {
String newIndexManifest) {
if (deltaManifestList != null) {
for (ManifestFileMeta manifest : manifestList.read(deltaManifestList.getKey())) {
- manifestFile.delete(manifest.fileName());
+ manifestFile.delete(manifest);
}
manifestList.delete(deltaManifestList.getKey());
}
if (changelogManifestList != null) {
for (ManifestFileMeta manifest : manifestList.read(changelogManifestList.getKey())) {
- manifestFile.delete(manifest.fileName());
+ manifestFile.delete(manifest);
}
manifestList.delete(changelogManifestList.getKey());
}
@@ -80,7 +80,7 @@ public class CommitCleaner {
.collect(Collectors.toSet());
for (ManifestFileMeta suspect : mergeAfterManifests) {
if (!oldMetaSet.contains(suspect.fileName())) {
- manifestFile.delete(suspect.fileName());
+ manifestFile.delete(suspect);
}
}
}
diff --git a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaSerializerTest.java b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaSerializerTest.java
index 46ff2067d2..04f41aa820 100644
--- a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaSerializerTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaSerializerTest.java
@@ -53,7 +53,8 @@ public class ManifestFileMetaSerializerTest extends ObjectSerializerTestBase<Man
Arrays.asList(
null,
Collections.<String>emptyList(),
- Arrays.asList("extra-1", "extra-2"))) {
+ Arrays.asList("extra-1", "extra-2"),
+ Arrays.asList("partition-index", "manifest" + ManifestSidecar.SUFFIX))) {
ManifestFileMeta meta =
new ManifestFileMeta(
original.fileName(),
diff --git a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileTest.java b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileTest.java
index 5fb0a7e049..50097b041a 100644
--- a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileTest.java
@@ -100,6 +100,28 @@ public class ManifestFileTest {
assertThat(actualEntries).isEqualTo(entries);
}
+ @Test
+ void testDeleteManifestAndOnlyReferencedExtraFiles() throws Exception {
+ ManifestFile manifests = createManifestFile(tempDir.toString());
+ ManifestFileMeta meta = manifests.write(Collections.singletonList(gen.next())).get(0);
+ java.nio.file.Path manifestDir = tempDir.resolve("manifest");
+ String sidecar = "custom-index" + ManifestSidecar.SUFFIX;
+ String extra = "other-extra";
+ String unreferenced = meta.fileName() + ManifestSidecar.SUFFIX;
+ for (String name : Arrays.asList(sidecar, extra, unreferenced)) {
+ Files.createFile(manifestDir.resolve(name));
+ }
+ manifests.delete(
+ ManifestIndexTestUtils.withExtraFiles(meta, Arrays.asList(sidecar, extra)));
+ assertThat(Files.exists(manifestDir.resolve(meta.fileName()))).isFalse();
+ assertThat(Files.exists(manifestDir.resolve(sidecar))).isFalse();
+ assertThat(Files.exists(manifestDir.resolve(extra))).isFalse();
+ assertThat(Files.exists(manifestDir.resolve(unreferenced))).isTrue();
+ // Deleting already removed files is harmless.
+ manifests.delete(
+ ManifestIndexTestUtils.withExtraFiles(meta, Arrays.asList(sidecar, extra)));
+ }
+
@Test
void testWriteManifestFileToExplicitPath() throws Exception {
List<ManifestEntry> entries = generateData();
diff --git a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestIndexTestUtils.java b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestIndexTestUtils.java
new file mode 100644
index 0000000000..32a94c9a5b
--- /dev/null
+++ b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestIndexTestUtils.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.manifest;
+
+import org.apache.paimon.FileStore;
+import org.apache.paimon.fs.FileIO;
+import org.apache.paimon.fs.Path;
+import org.apache.paimon.utils.JsonSerdeUtil;
+import org.apache.paimon.utils.Pair;
+import org.apache.paimon.utils.SnapshotManager;
+
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.JsonNode;
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.node.ObjectNode;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+/** Synthetic index references for manifest serialization and lifecycle tests. */
+public final class ManifestIndexTestUtils {
+ private ManifestIndexTestUtils() {}
+
+ public static ManifestFileMeta withIndexFileName(ManifestFileMeta meta, String indexFileName) {
+ return withExtraFiles(
+ meta, indexFileName == null ? null : Collections.singletonList(indexFileName));
+ }
+
+ public static ManifestFileMeta withExtraFiles(ManifestFileMeta meta, List<String> extraFiles) {
+ return new ManifestFileMeta(
+ meta.fileName(),
+ meta.fileSize(),
+ meta.numAddedFiles(),
+ meta.numDeletedFiles(),
+ meta.partitionStats(),
+ meta.schemaId(),
+ meta.minBucket(),
+ meta.maxBucket(),
+ meta.minLevel(),
+ meta.maxLevel(),
+ meta.minRowId(),
+ meta.maxRowId(),
+ meta.totalBuckets(),
+ extraFiles);
+ }
+
+ /** Replaces only synthetic snapshot fixtures, using newly written manifest lists. */
+ public static void registerIndexReferences(FileStore<?> store, long snapshotId)
+ throws IOException {
+ SnapshotManager manager = store.snapshotManager();
+ FileIO io = manager.fileIO();
+ Path snapshotPath = manager.snapshotPath(snapshotId);
+ ObjectNode snapshot =
+ (ObjectNode)
+ JsonSerdeUtil.OBJECT_MAPPER_INSTANCE.readTree(
+ io.readFileUtf8(snapshotPath));
+ ManifestList lists = store.manifestListFactory().create();
+ for (String field :
+ new String[] {"baseManifestList", "deltaManifestList", "changelogManifestList"}) {
+ JsonNode value = snapshot.get(field);
+ if (value == null || value.isNull()) {
+ continue;
+ }
+ List<ManifestFileMeta> indexed = new ArrayList<>();
+ for (ManifestFileMeta meta : lists.read(value.asText())) {
+ // Deliberately use a name which cannot be derived by appending the sidecar suffix.
+ String name = "index-for-" + meta.fileName() + ManifestSidecar.SUFFIX;
+ Path index = store.pathFactory().toManifestFilePath(name);
+ if (!io.exists(index)) {
+ // GC treats index bytes as opaque; unsupported/partial files are still owned.
+ io.newOutputStream(index, false).close();
+ }
+ indexed.add(withIndexFileName(meta, name));
+ }
+ Pair<String, Long> replacement = lists.write(indexed);
+ snapshot.put(field, replacement.getLeft());
+ snapshot.put(field + "Size", replacement.getRight());
+ }
+ io.overwriteFileUtf8(
+ snapshotPath, JsonSerdeUtil.OBJECT_MAPPER_INSTANCE.writeValueAsString(snapshot));
+ manager.invalidateCache();
+ }
+}
diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/ExpireSnapshotsTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/ExpireSnapshotsTest.java
index 24a6e0eaea..957e4b7317 100644
--- a/paimon-core/src/test/java/org/apache/paimon/operation/ExpireSnapshotsTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/operation/ExpireSnapshotsTest.java
@@ -38,8 +38,11 @@ import org.apache.paimon.manifest.FileKind;
import org.apache.paimon.manifest.FileSource;
import org.apache.paimon.manifest.ManifestEntry;
import org.apache.paimon.manifest.ManifestFileMeta;
+import org.apache.paimon.manifest.ManifestIndexTestUtils;
+import org.apache.paimon.manifest.ManifestSidecar;
import org.apache.paimon.mergetree.compact.DeduplicateMergeFunction;
import org.apache.paimon.options.ExpireConfig;
+import org.apache.paimon.options.MemorySize;
import org.apache.paimon.schema.FileSystemSchemaManager;
import org.apache.paimon.schema.Schema;
import org.apache.paimon.schema.SchemaManager;
@@ -854,6 +857,75 @@ public class ExpireSnapshotsTest {
store.assertCleaned();
}
+ @Test
+ void testSidecarsFollowSnapshotAndTagRetention() throws Exception {
+ store.options().toConfiguration().set(CoreOptions.MANIFEST_MERGE_MIN_COUNT, 2);
+ store.options()
+ .toConfiguration()
+ .set(CoreOptions.MANIFEST_TARGET_FILE_SIZE, MemorySize.parse("8 mb"));
+ List<KeyValue> allData = new ArrayList<>();
+ List<Integer> snapshotPositions = new ArrayList<>();
+ commit(8, allData, snapshotPositions);
+ int latest = requireNonNull(snapshotManager.latestSnapshotId()).intValue();
+ Set<Path> manifests = new HashSet<>();
+ Set<Path> retainedManifests = new HashSet<>();
+ for (int snapshotId = 1; snapshotId <= latest; snapshotId++) {
+ rewriteSnapshotTime(snapshotId, 0);
+ ManifestIndexTestUtils.registerIndexReferences(store, snapshotId);
+ snapshotManager.invalidateCache();
+ for (ManifestFileMeta meta :
+ store.manifestListFactory()
+ .create()
+ .readDataManifests(snapshotManager.snapshot(snapshotId))) {
+ Path manifest = store.pathFactory().toManifestFilePath(meta.fileName());
+ manifests.add(manifest);
+ if (snapshotId == 3 || snapshotId == latest) {
+ retainedManifests.add(manifest);
+ }
+ }
+ }
+ store.newTagManager()
+ .createTag(
+ snapshotManager.snapshot(3),
+ "keep-sidecars",
+ store.options().tagDefaultTimeRetained(),
+ Collections.emptyList(),
+ false);
+ Set<Path> expiredManifests = new HashSet<>(manifests);
+ expiredManifests.removeAll(retainedManifests);
+ assertThat(expiredManifests).isNotEmpty();
+ ExpireSnapshotsImpl expire =
+ (ExpireSnapshotsImpl) store.newExpire(expireAllButLatestConfig());
+ expire.setCurrentTimeMillis(() -> 1000L);
+ expire.expire();
+ for (Path manifest : manifests) {
+ boolean retained = retainedManifests.contains(manifest);
+ assertThat(fileIO.exists(manifest)).as("manifest %s", manifest).isEqualTo(retained);
+ assertThat(
+ fileIO.exists(
+ new Path(
+ manifest.getParent(),
+ "index-for-"
+ + manifest.getName()
+ + ManifestSidecar.SUFFIX)))
+ .as("sidecar for %s", manifest)
+ .isEqualTo(retained);
+ }
+ for (ManifestFileMeta meta :
+ store.manifestListFactory()
+ .create()
+ .readDataManifests(
+ store.newTagManager()
+ .getOrThrow("keep-sidecars")
+ .trimToSnapshot())) {
+ assertThat(
+ fileIO.exists(
+ store.pathFactory()
+ .toManifestFilePath(ManifestSidecar.fileName(meta))))
+ .isTrue();
+ }
+ }
+
@Test
public void testExpireWithTagsAndConcurrentPlanningKeepsTaggedSnapshotsReadable()
throws Exception {
diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/LocalOrphanFilesCleanTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/LocalOrphanFilesCleanTest.java
index 9e3bf05b69..49e22ee551 100644
--- a/paimon-core/src/test/java/org/apache/paimon/operation/LocalOrphanFilesCleanTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/operation/LocalOrphanFilesCleanTest.java
@@ -32,7 +32,9 @@ import org.apache.paimon.fs.FileStatus;
import org.apache.paimon.fs.Path;
import org.apache.paimon.fs.local.LocalFileIO;
import org.apache.paimon.manifest.ManifestFileMeta;
+import org.apache.paimon.manifest.ManifestIndexTestUtils;
import org.apache.paimon.manifest.ManifestList;
+import org.apache.paimon.manifest.ManifestSidecar;
import org.apache.paimon.mergetree.compact.ConcatRecordReader;
import org.apache.paimon.options.Options;
import org.apache.paimon.reader.ReaderSupplier;
@@ -158,6 +160,39 @@ public class LocalOrphanFilesCleanTest {
normallyRemoving(tablePath);
}
+ @Test
+ void testOrphanCleanupProtectsReferencedSidecars() throws Exception {
+ commit(Collections.singletonList(TestPojo.next()));
+ ManifestIndexTestUtils.registerIndexReferences(
+ table.store(), table.snapshotManager().latestSnapshotId());
+ table.snapshotManager().invalidateCache();
+ table.createTag("sidecar-tag", table.snapshotManager().latestSnapshotId());
+ List<Path> sidecars = new ArrayList<>();
+ List<Path> unreferenced = new ArrayList<>();
+ for (ManifestFileMeta meta :
+ table.store()
+ .manifestListFactory()
+ .create()
+ .readDataManifests(table.snapshotManager().latestSnapshot())) {
+ Path sidecar = new Path(manifestDir, ManifestSidecar.fileName(meta));
+ sidecars.add(sidecar);
+ Path guessed = new Path(manifestDir, meta.fileName() + ManifestSidecar.SUFFIX);
+ fileIO.newOutputStream(guessed, false).close();
+ unreferenced.add(guessed);
+ }
+ Path orphan = new Path(manifestDir, "manifest-orphan" + ManifestSidecar.SUFFIX);
+ fileIO.newOutputStream(orphan, false).close();
+ new LocalOrphanFilesClean(table, System.currentTimeMillis() + 2000).clean();
+ assertThat(fileIO.exists(orphan)).isFalse();
+ assertThat(sidecars).isNotEmpty();
+ for (Path sidecar : sidecars) {
+ assertThat(fileIO.exists(sidecar)).isTrue();
+ }
+ for (Path guessed : unreferenced) {
+ assertThat(fileIO.exists(guessed)).isFalse();
+ }
+ }
+
@Test
public void testKeepManagedBlobPack() throws Exception {
commit(Collections.singletonList(TestPojo.next()));
diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/ManifestSidecarLifecycleTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/ManifestSidecarLifecycleTest.java
new file mode 100644
index 0000000000..be9e199427
--- /dev/null
+++ b/paimon-core/src/test/java/org/apache/paimon/operation/ManifestSidecarLifecycleTest.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.operation;
+
+import org.apache.paimon.manifest.IndexManifestFile;
+import org.apache.paimon.manifest.ManifestFile;
+import org.apache.paimon.manifest.ManifestFileMeta;
+import org.apache.paimon.manifest.ManifestList;
+import org.apache.paimon.operation.commit.CommitCleaner;
+import org.apache.paimon.utils.Pair;
+
+import org.junit.jupiter.api.Test;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Mockito.CALLS_REAL_METHODS;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoInteractions;
+import static org.mockito.Mockito.when;
+
+/** Tests ownership of manifest extra files during expiration and aborted commits. */
+class ManifestSidecarLifecycleTest {
+
+ @Test
+ void retainedManifestProtectsSharedExtraFiles() {
+ ManifestFileMeta retained = manifest("retained", "shared.avro.sidecar", "retained-extra");
+ ManifestFileMeta expired = manifest("expired", "shared.avro.sidecar", "expired-extra");
+ FileDeletionBase<?> deletion = mock(FileDeletionBase.class, CALLS_REAL_METHODS);
+ doReturn(Arrays.asList(retained, expired)).when(deletion).tryReadManifestList("old-list");
+ Set<String> skipping = new HashSet<>();
+ FileDeletionBase.addManifestToSkippingSet(skipping, retained);
+ Set<String> removed = new HashSet<>();
+ deletion.collectUnusedManifestList("old-list", skipping, removed);
+ assertThat(removed).containsExactlyInAnyOrder("expired", "expired-extra", "old-list");
+ }
+
+ @Test
+ void failedCommitDeletesOnlyNewManifestsWithTheirMetadata() {
+ ManifestList lists = mock(ManifestList.class);
+ ManifestFile files = mock(ManifestFile.class);
+ IndexManifestFile indexes = mock(IndexManifestFile.class);
+ ManifestFileMeta reused = manifest("reused", "reused.avro.sidecar");
+ ManifestFileMeta added = manifest("added", "added.avro.sidecar");
+ CommitCleaner cleaner = new CommitCleaner(lists, files, indexes);
+ cleaner.cleanUpNoReuseTmpManifests(
+ Pair.of("new-base-list", 1L),
+ Collections.singletonList(reused),
+ Arrays.asList(reused, added));
+ verify(files).delete(added);
+ verify(files, never()).delete(reused);
+ verify(lists).delete("new-base-list");
+ verifyNoInteractions(indexes);
+ }
+
+ @Test
+ void failedCommitDeletesDeltaAndChangelogManifestExtras() {
+ ManifestList lists = mock(ManifestList.class);
+ ManifestFile files = mock(ManifestFile.class);
+ IndexManifestFile indexes = mock(IndexManifestFile.class);
+ ManifestFileMeta delta = manifest("delta", "delta.avro.sidecar");
+ ManifestFileMeta changelog = manifest("changelog", "changelog.avro.sidecar");
+ when(lists.read("delta-list")).thenReturn(Collections.singletonList(delta));
+ when(lists.read("changelog-list")).thenReturn(Collections.singletonList(changelog));
+ new CommitCleaner(lists, files, indexes)
+ .cleanUpReuseTmpManifests(
+ Pair.of("delta-list", 1L),
+ Pair.of("changelog-list", 1L),
+ "same-index",
+ "same-index");
+ verify(files).delete(delta);
+ verify(files).delete(changelog);
+ verify(lists).delete("delta-list");
+ verify(lists).delete("changelog-list");
+ verifyNoInteractions(indexes);
+ }
+
+ private static ManifestFileMeta manifest(String name, String... extras) {
+ ManifestFileMeta meta = mock(ManifestFileMeta.class);
+ when(meta.fileName()).thenReturn(name);
+ when(meta.extraFiles()).thenReturn(Arrays.asList(extras));
+ return meta;
+ }
+}
(paimon) 06/20: [python] Support writing shared-shredding MAPs in Parquet (#9837)
Posted by ho...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
XiaoHongbo-Hope pushed a commit to branch release-2.1
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit 478a7201c451fa8bc96f81185457f18c062f8c18
Author: XiaoHongbo <xi...@alibaba-inc.com>
AuthorDate: Wed Sep 16 15:22:02 2026 +0800
[python] Support writing shared-shredding MAPs in Parquet (#9837)
---
.../test/java/org/apache/paimon/JavaPyE2ETest.java | 61 +++
paimon-python/dev/run_mixed_tests.sh | 15 +
.../pypaimon/common/options/core_options.py | 44 ++
.../pypaimon/data/map_shared_shredding.py | 51 +-
.../pypaimon/tests/e2e/java_py_read_write_test.py | 46 ++
.../tests/map_shared_shredding_write_test.py | 385 ++++++++++++++
.../pypaimon/write/map_shared_shredding_writer.py | 573 +++++++++++++++++++++
.../pypaimon/write/writer/data_vector_writer.py | 8 +-
paimon-python/pypaimon/write/writer/data_writer.py | 34 +-
.../write/writer/dedicated_format_writer.py | 8 +-
10 files changed, 1212 insertions(+), 13 deletions(-)
diff --git a/paimon-core/src/test/java/org/apache/paimon/JavaPyE2ETest.java b/paimon-core/src/test/java/org/apache/paimon/JavaPyE2ETest.java
index 524344ed54..8a59c43326 100644
--- a/paimon-core/src/test/java/org/apache/paimon/JavaPyE2ETest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/JavaPyE2ETest.java
@@ -1620,6 +1620,67 @@ public class JavaPyE2ETest {
}
}
+ /** Java reads shared-shredding MAP columns written by Python. */
+ @Test
+ @EnabledIfSystemProperty(named = "run.e2e.tests", matches = "true")
+ public void testJavaReadSharedShreddingMapTable() throws Exception {
+ FileStoreTable table =
+ (FileStoreTable)
+ catalog.getTable(identifier("shared_shredding_map_python_test_parquet"));
+ Map<Integer, Map<String, Long>> rows = new HashMap<>();
+ List<Split> splits = new ArrayList<>(table.newSnapshotReader().read().dataSplits());
+ try (org.apache.paimon.reader.RecordReader<InternalRow> reader =
+ table.newRead().createReader(splits)) {
+ reader.forEachRemaining(
+ row -> {
+ int id = row.getInt(0);
+ for (int column = 2; column <= 3; column++) {
+ if (id == 3) {
+ assertThat(row.isNullAt(column)).isTrue();
+ } else {
+ InternalMap required = row.getMap(column);
+ assertThat(required.size()).isEqualTo(id == 2 ? 0 : 1);
+ if (id != 2) {
+ InternalArray values = required.valueArray();
+ long value =
+ column == 2
+ ? values.getLong(0)
+ : values.getRow(0, 1).getLong(0);
+ assertThat(value).isEqualTo(id == 1 ? 1L : 2L);
+ }
+ }
+ }
+ if (row.isNullAt(1)) {
+ rows.put(id, null);
+ return;
+ }
+ InternalMap map = row.getMap(1);
+ InternalArray keys = map.keyArray();
+ InternalArray values = map.valueArray();
+ Map<String, Long> converted = new LinkedHashMap<>();
+ for (int i = 0; i < map.size(); i++) {
+ converted.put(
+ keys.getString(i).toString(),
+ values.isNullAt(i) ? null : values.getLong(i));
+ }
+ rows.put(id, converted);
+ });
+ }
+
+ assertThat(rows).containsOnlyKeys(1, 2, 3, 4);
+ assertThat(rows.get(1))
+ .containsOnlyKeys("hot", "warm", "overflow")
+ .containsEntry("hot", 10L)
+ .containsEntry("warm", 20L)
+ .containsEntry("overflow", 30L);
+ assertThat(rows.get(2))
+ .containsOnlyKeys("hot", "new")
+ .containsEntry("hot", null)
+ .containsEntry("new", 40L);
+ assertThat(rows.get(3)).isEmpty();
+ assertThat(rows.get(4)).isNull();
+ }
+
private Map<Integer, Map<Integer, byte[]>> readMapBlobRows(FileStoreTable table)
throws Exception {
Map<Integer, Map<Integer, byte[]>> rows = new HashMap<>();
diff --git a/paimon-python/dev/run_mixed_tests.sh b/paimon-python/dev/run_mixed_tests.sh
index 620c4d989a..54fef2a4e1 100755
--- a/paimon-python/dev/run_mixed_tests.sh
+++ b/paimon-python/dev/run_mixed_tests.sh
@@ -1033,6 +1033,21 @@ run_shared_shredding_map_test() {
return 1
fi
echo -e "${GREEN}✓ Python shared-shredding MAP read test completed successfully${NC}"
+
+ echo "Running Python shared-shredding MAP write test..."
+ if ! python -m pytest java_py_read_write_test.py::JavaPyReadWriteTest::test_write_shared_shredding_map_for_java -v; then
+ echo -e "${RED}✗ Python shared-shredding MAP write test failed${NC}"
+ return 1
+ fi
+ echo -e "${GREEN}✓ Python shared-shredding MAP write test completed successfully${NC}"
+
+ cd "$PROJECT_ROOT"
+ echo "Running Maven test for JavaPyE2ETest.testJavaReadSharedShreddingMapTable..."
+ if ! mvn test -Dtest=org.apache.paimon.JavaPyE2ETest#testJavaReadSharedShreddingMapTable -pl paimon-core -q -Drun.e2e.tests=true; then
+ echo -e "${RED}✗ Java shared-shredding MAP read test failed${NC}"
+ return 1
+ fi
+ echo -e "${GREEN}✓ Java shared-shredding MAP read test completed successfully${NC}"
}
# Function to run VARIANT test (Java write, Python read)
diff --git a/paimon-python/pypaimon/common/options/core_options.py b/paimon-python/pypaimon/common/options/core_options.py
index 6586e051a5..2fe0d039fd 100644
--- a/paimon-python/pypaimon/common/options/core_options.py
+++ b/paimon-python/pypaimon/common/options/core_options.py
@@ -158,6 +158,10 @@ class CoreOptions:
NESTED_SEQUENCE_FIELD = "nested-sequence-field"
COUNT_LIMIT = "count-limit"
MERGE_MAP_TS_FIELD = "ts-field"
+ MAP_STORAGE_LAYOUT = "map.storage-layout"
+ MAP_SHARED_SHREDDING_MAX_COLUMNS = "map.shared-shredding.max-columns"
+ MAP_SHARED_SHREDDING_COLUMN_PLACEMENT_POLICY = \
+ "map.shared-shredding.column-placement-policy"
# Basic options
AUTO_CREATE: ConfigOption[bool] = (
@@ -1894,6 +1898,46 @@ class CoreOptions:
.no_default_value()
)
+ def map_storage_layout(self, field_name: str) -> str:
+ return self.options.get(
+ ConfigOptions.key(
+ f'{CoreOptions.FIELDS_PREFIX}.{field_name}.{CoreOptions.MAP_STORAGE_LAYOUT}'
+ )
+ .string_type()
+ .default_value('default')
+ ).lower()
+
+ def map_shared_shredding_max_columns(self, field_name: str) -> int:
+ value = self.options.get(
+ ConfigOptions.key(
+ f'{CoreOptions.FIELDS_PREFIX}.{field_name}.'
+ f'{CoreOptions.MAP_SHARED_SHREDDING_MAX_COLUMNS}'
+ )
+ .int_type()
+ .default_value(256)
+ )
+ if value <= 0:
+ raise ValueError(
+ '{} must be greater than 0'.format(
+ CoreOptions.MAP_SHARED_SHREDDING_MAX_COLUMNS))
+ return value
+
+ def map_shared_shredding_column_placement_policy(
+ self, field_name: str) -> str:
+ value = self.options.get(
+ ConfigOptions.key(
+ f'{CoreOptions.FIELDS_PREFIX}.{field_name}.'
+ f'{CoreOptions.MAP_SHARED_SHREDDING_COLUMN_PLACEMENT_POLICY}'
+ )
+ .string_type()
+ .default_value('lru')
+ ).lower()
+ if value not in ('plain', 'sequential', 'lru'):
+ raise ValueError(
+ "Unsupported shared-shredding column placement policy: {}".format(
+ value))
+ return value
+
@property
def query_auth_enabled(self) -> bool:
return self.options.get(CoreOptions.QUERY_AUTH_ENABLED)
diff --git a/paimon-python/pypaimon/data/map_shared_shredding.py b/paimon-python/pypaimon/data/map_shared_shredding.py
index a3d6c58bac..13c15df75f 100644
--- a/paimon-python/pypaimon/data/map_shared_shredding.py
+++ b/paimon-python/pypaimon/data/map_shared_shredding.py
@@ -14,7 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-"""Read support for Paimon's shared-shredding MAP storage layout."""
+"""Metadata and read support for the shared-shredding MAP layout."""
import json
import struct
@@ -33,6 +33,7 @@ _FIELD_DICT_ORIGINAL_SIZE = b"paimon.map.shared-shredding.field-dict-original-si
_NUM_COLUMNS = b"paimon.map.shared-shredding.num-columns"
_FIELD_COLUMNS = b"paimon.map.shared-shredding.field-columns"
_OVERFLOW_SET = b"paimon.map.shared-shredding.overflow-set"
+_MAX_ROW_WIDTH = b"paimon.map.shared-shredding.max-row-width"
_FIELD_MAPPING = "__field_mapping"
_OVERFLOW = "__overflow"
_PHYSICAL_COLUMN_PREFIX = "__col_"
@@ -104,6 +105,40 @@ def parse_shared_shredding_selection_metadata(field: pa.Field):
return name_by_id, field_to_columns, set(overflow_json), num_columns
+def shared_shredding_metadata(
+ name_to_id, field_to_columns, overflow_set, num_columns,
+ max_row_width, compression):
+ """Build Java-compatible Arrow field metadata for one data file."""
+ compression = compression.lower()
+ if compression not in ("none", "lz4", "zstd"):
+ raise ValueError(
+ "MAP shared-shredding only supports none/lz4/zstd compression, "
+ "but is {}.".format(compression))
+ field_dict = json.dumps(
+ dict(sorted(name_to_id.items())),
+ separators=(",", ":"),
+ sort_keys=True,
+ ).encode("utf-8")
+ encoded_dict = _compress(field_dict, compression)
+ columns = {
+ str(field_id): sorted(column_ids)
+ for field_id, column_ids in sorted(field_to_columns.items())
+ }
+ return {
+ _STORAGE_LAYOUT: b"shared-shredding",
+ _VERSION: b"1",
+ _FIELD_DICT: encoded_dict.decode("latin-1").encode("utf-8"),
+ _FIELD_DICT_COMPRESSION: compression.encode("utf-8"),
+ _FIELD_DICT_ORIGINAL_SIZE: str(len(field_dict)).encode("utf-8"),
+ _FIELD_COLUMNS: json.dumps(
+ columns, separators=(",", ":"), sort_keys=True).encode("utf-8"),
+ _OVERFLOW_SET: json.dumps(
+ sorted(overflow_set), separators=(",", ":")).encode("utf-8"),
+ _NUM_COLUMNS: str(num_columns).encode("utf-8"),
+ _MAX_ROW_WIDTH: str(max_row_width).encode("utf-8"),
+ }
+
+
def map_selected_keys(description: str) -> List[str]:
if not description or not description.startswith(_SELECTED_KEYS_PREFIX):
raise ValueError("Invalid selected-key MAP metadata: {}".format(
@@ -589,6 +624,20 @@ def _decompress(data: bytes, original_size: int, compression: str) -> bytes:
return result
+def _compress(data: bytes, compression: str) -> bytes:
+ if compression == "none":
+ return data
+ if compression == "zstd":
+ import zstandard as zstd
+ return zstd.ZstdCompressor(level=1).compress(data)
+ if compression == "lz4":
+ payload = bytes(pa.Codec("lz4_raw").compress(data))
+ return struct.pack("<ii", len(payload), len(data)) + payload
+ raise ValueError(
+ "Unsupported shared-shredding dictionary compression: {}".format(
+ compression))
+
+
def _required(metadata, key):
try:
return metadata[key]
diff --git a/paimon-python/pypaimon/tests/e2e/java_py_read_write_test.py b/paimon-python/pypaimon/tests/e2e/java_py_read_write_test.py
index 86d1255ecf..d850206277 100644
--- a/paimon-python/pypaimon/tests/e2e/java_py_read_write_test.py
+++ b/paimon-python/pypaimon/tests/e2e/java_py_read_write_test.py
@@ -1636,6 +1636,52 @@ class JavaPyReadWriteTest(unittest.TestCase):
self.assertEqual([None] * 5,
result.column('metrics_missing').to_pylist())
+ def test_write_shared_shredding_map_for_java(self):
+ schema = pa.schema([
+ pa.field('id', pa.int32()),
+ pa.field('metrics', pa.map_(pa.string(), pa.int64())),
+ pa.field('required_values', pa.map_(pa.string(), pa.field(
+ 'value', pa.int64(), nullable=False))),
+ pa.field('required_rows', pa.map_(pa.string(), pa.field(
+ 'value', pa.struct([pa.field('score', pa.int64(), nullable=False)]),
+ nullable=False))),
+ ])
+ rows = pa.Table.from_pydict({
+ 'id': [1, 2, 3, 4],
+ 'metrics': [
+ [('hot', 10), ('warm', 20), ('overflow', 30)],
+ [('hot', None), ('new', 40)],
+ [],
+ None,
+ ],
+ 'required_values': [[('a', 1)], [], None, [('b', 2)]],
+ 'required_rows': [[('a', {'score': 1})], [], None, [('b', {'score': 2})]],
+ }, schema=schema)
+
+ table_name = 'default.shared_shredding_map_python_test_parquet'
+ self.catalog.drop_table(table_name, True)
+ self.catalog.create_table(
+ table_name,
+ Schema.from_pyarrow_schema(schema, options={
+ 'bucket': '-1',
+ 'file.format': 'parquet',
+ 'write-only': 'true',
+ 'fields.metrics.map.storage-layout': 'shared-shredding',
+ 'fields.metrics.map.shared-shredding.max-columns': '2',
+ 'fields.required_values.map.storage-layout': 'shared-shredding',
+ 'fields.required_values.map.shared-shredding.max-columns': '2',
+ 'fields.required_rows.map.storage-layout': 'shared-shredding',
+ 'fields.required_rows.map.shared-shredding.max-columns': '2',
+ }),
+ False,
+ )
+ table = self.catalog.get_table(table_name)
+ builder = table.new_batch_write_builder()
+ writer = builder.new_write()
+ writer.write_arrow(rows)
+ builder.new_commit().commit(writer.prepare_commit())
+ writer.close()
+
def test_write_map_blob_for_java(self):
map_blob_type = pa.map_(pa.int32(), pa.large_binary())
boolean_map_blob_type = pa.map_(pa.bool_(), pa.large_binary())
diff --git a/paimon-python/pypaimon/tests/map_shared_shredding_write_test.py b/paimon-python/pypaimon/tests/map_shared_shredding_write_test.py
new file mode 100644
index 0000000000..eba20b4270
--- /dev/null
+++ b/paimon-python/pypaimon/tests/map_shared_shredding_write_test.py
@@ -0,0 +1,385 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import shutil
+import subprocess
+import sys
+import tempfile
+import unittest
+from unittest.mock import patch
+
+import pyarrow as pa
+import pyarrow.parquet as pq
+
+from pypaimon import CatalogFactory, Schema
+from pypaimon.data.map_shared_shredding import (
+ is_shared_shredding,
+ parse_shared_shredding_metadata,
+)
+
+
+class MapSharedShreddingWriteTest(unittest.TestCase):
+
+ def setUp(self):
+ self.temp_dir = tempfile.mkdtemp()
+ self.catalog = CatalogFactory.create({"warehouse": self.temp_dir})
+ self.catalog.create_database("default", True)
+ self.arrow_schema = pa.schema([
+ pa.field("id", pa.int32()),
+ pa.field("metrics", pa.map_(pa.string(), pa.int64())),
+ ])
+
+ def tearDown(self):
+ shutil.rmtree(self.temp_dir, ignore_errors=True)
+
+ def test_write_and_read_parquet(self):
+ expected = [
+ [("hot", 10), ("warm", 20), ("overflow", 30)],
+ [("hot", None), ("new", 40)],
+ [],
+ None,
+ ]
+ data = pa.Table.from_pydict({
+ "id": [1, 2, 3, 4],
+ "metrics": expected,
+ }, schema=self.arrow_schema)
+
+ table = self._create_table("parquet", max_columns=2)
+ messages = self._write(table, data)
+
+ physical_field = pq.read_schema(
+ messages[0].new_files[0].file_path).field("metrics")
+ self.assertTrue(pa.types.is_struct(physical_field.type))
+ self.assertTrue(is_shared_shredding(physical_field))
+ name_by_id, num_columns = \
+ parse_shared_shredding_metadata(physical_field)
+ self.assertEqual(2, num_columns)
+ self.assertEqual(
+ {"hot", "warm", "overflow", "new"},
+ set(name_by_id.values()),
+ )
+
+ read_builder = table.new_read_builder()
+ result = read_builder.new_read().to_arrow(
+ read_builder.new_scan().plan().splits())
+ self.assertEqual(expected, result.column("metrics").to_pylist())
+
+ selected = table.new_read_builder().with_projection([
+ "id", "metrics['hot']", "metrics['overflow']",
+ ])
+ result = selected.new_read().to_arrow(
+ selected.new_scan().plan().splits())
+ self.assertEqual(
+ [10, None, None, None],
+ result.column("metrics_hot").to_pylist(),
+ )
+ self.assertEqual(
+ [30, None, None, None],
+ result.column("metrics_overflow").to_pylist(),
+ )
+
+ def test_reject_orc(self):
+ with self.assertRaisesRegex(
+ ValueError,
+ "PyPaimon MAP shared-shredding writes only support parquet"):
+ writer = self._create_table(
+ "orc", max_columns=2).new_batch_write_builder().new_write()
+ writer.write_arrow(pa.Table.from_pydict({
+ "id": [1],
+ "metrics": [[("key", 1)]],
+ }, schema=self.arrow_schema))
+
+ def test_non_nullable_values(self):
+ for value_type, value in [
+ (pa.int64(), 1),
+ (pa.struct([pa.field('score', pa.int64(), nullable=False)]),
+ {'score': 1})]:
+ with self.subTest(value_type=value_type):
+ self.arrow_schema = pa.schema([
+ pa.field('id', pa.int32()),
+ pa.field('metrics', pa.map_(
+ pa.string(), pa.field('value', value_type, nullable=False))),
+ ])
+ expected = [[('a', value)], [], None]
+ data = pa.Table.from_pydict({
+ 'id': [1, 2, 3], 'metrics': expected,
+ }, schema=self.arrow_schema)
+ table = self._create_table('parquet', 2)
+ messages = self._write(table, data)
+ field = pq.read_schema(messages[0].new_files[0].file_path).field('metrics')
+ self.assertTrue(field.type['__col_0'].nullable)
+ self.assertTrue(field.type['__col_1'].nullable)
+ reader = table.new_read_builder()
+ result = reader.new_read().to_arrow(reader.new_scan().plan().splits())
+ # Arrow 6 cannot convert non-nullable MAP values to scalars.
+ actual = result.column('metrics').cast(pa.map_(pa.string(), value_type))
+ self.assertEqual(expected, actual.to_pylist())
+
+ def test_nested_non_nullable_map_values_in_subprocess(self):
+ # A native Arrow assertion aborts the process, not a Python exception.
+ result = subprocess.run([
+ sys.executable, '-c',
+ 'from pypaimon.tests.map_shared_shredding_write_test import '
+ 'MapSharedShreddingWriteTest\n'
+ 'test = MapSharedShreddingWriteTest()\n'
+ 'test.setUp()\n'
+ 'try:\n'
+ ' test.check_nested_non_nullable_map_values()\n'
+ 'finally:\n'
+ ' test.tearDown()\n',
+ ], stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=60)
+ self.assertEqual(0, result.returncode, result.stderr.decode('utf-8', 'replace'))
+
+ def check_nested_non_nullable_map_values(self):
+ from pypaimon.write.map_shared_shredding_writer import _to_python_values
+
+ inner = pa.map_(pa.string(), pa.field('value', pa.int64(), nullable=False))
+ for value_type, value in [
+ (inner, [('x', 1)]),
+ (pa.struct([pa.field('nested', inner)]), {'nested': [('x', 1)]}),
+ (pa.list_(inner), [[('x', 1)], [], None])]:
+ self.arrow_schema = pa.schema([
+ pa.field('id', pa.int32()),
+ pa.field('metrics', pa.map_(pa.string(), value_type)),
+ ])
+ expected = [[('a', value)], [], None, [('b', value)]]
+ array = pa.array([[('unused', value)]] + expected + [[]],
+ type=self.arrow_schema.field('metrics').type)
+ # Keep nonzero offsets and multiple chunks in the conversion check.
+ chunks = [array.slice(1, 2), array.slice(3, 2)]
+ self.assertEqual(expected, [v for chunk in chunks
+ for v in _to_python_values(chunk)])
+ data = pa.Table.from_arrays([
+ pa.array([1, 2, 3, 4], type=pa.int32()),
+ pa.chunked_array(chunks),
+ ], schema=self.arrow_schema)
+ table = self._create_table('parquet', 2)
+ messages = self._write(table, data)
+ self.assertTrue(is_shared_shredding(pq.read_schema(
+ messages[0].new_files[0].file_path).field('metrics')))
+ reader = table.new_read_builder()
+ result = reader.new_read().to_arrow(reader.new_scan().plan().splits())
+ self.assertEqual(expected, [v for chunk in result.column('metrics').chunks
+ for v in _to_python_values(chunk)])
+
+ def test_dedicated_columns_preserve_shredding(self):
+ for blob, vector in [(True, False), (False, True), (True, True)]:
+ with self.subTest(blob=blob, vector=vector):
+ fields = [pa.field('id', pa.int32()),
+ pa.field('metrics', pa.map_(pa.string(), pa.int64()))]
+ values = {'id': [1, 2], 'metrics': [[('a', 1)], []]}
+ options = {'row-tracking.enabled': 'true',
+ 'data-evolution.enabled': 'true'}
+ if blob:
+ fields.append(pa.field('payload', pa.large_binary()))
+ values['payload'] = [b'one', b'two']
+ if vector:
+ fields.append(pa.field('embedding', pa.list_(pa.float32(), 2)))
+ values['embedding'] = [[1., 2.], [3., 4.]]
+ options['vector.file.format'] = 'parquet'
+ self.arrow_schema = pa.schema(fields)
+ table = self._create_table('parquet', 2, options)
+ messages = self._write(table, pa.Table.from_pydict(
+ values, schema=self.arrow_schema))
+ normal_files = [f for m in messages for f in m.new_files
+ if f.file_name.endswith('.parquet')
+ and '.vector.' not in f.file_name]
+ self.assertTrue(normal_files)
+ for file in normal_files:
+ self.assertTrue(is_shared_shredding(
+ pq.read_schema(file.file_path).field('metrics')))
+ reader = table.new_read_builder().with_projection(["metrics['a']"])
+ result = reader.new_read().to_arrow(reader.new_scan().plan().splits())
+ self.assertEqual([1, None], result.column('metrics_a').to_pylist())
+
+ def test_reject_default_layout_on_non_map(self):
+ table = self._create_table('parquet', 2, {
+ 'fields.metrics.map.storage-layout': 'default',
+ 'fields.id.map.storage-layout': 'default',
+ })
+ with self.assertRaisesRegex(ValueError, 'its type is not MAP'):
+ self._write(table, pa.Table.from_pydict({
+ 'id': [1], 'metrics': [[('a', 1)]],
+ }, schema=self.arrow_schema))
+
+ def test_reject_char_keys(self):
+ from pypaimon.schema.data_types import AtomicType, DataField, MapType
+ from pypaimon.write.map_shared_shredding_writer import MapSharedShreddingWriter
+
+ with self.assertRaisesRegex(ValueError, 'STRING keys'):
+ MapSharedShreddingWriter._validate_field(DataField(
+ 0, 'metrics', MapType(True, AtomicType('CHAR(4)', False), AtomicType('BIGINT'))))
+
+ def test_streams_physical_batches(self):
+ from pypaimon.write.map_shared_shredding_writer import _MapFieldConverter
+
+ table = self._create_table('parquet', 256)
+ data = pa.Table.from_pydict({
+ 'id': list(range(5000)), 'metrics': [[('a', 1)]] * 4999 + [[('late', 2)]],
+ }, schema=self.arrow_schema)
+ convert = _MapFieldConverter.convert
+ write_table = pq.ParquetWriter.write_table
+ pending = []
+ sizes = []
+
+ def convert_batch(converter, column):
+ self.assertFalse(pending, 'physical batches were retained before writing')
+ pending.append(len(column))
+ sizes.append(len(column))
+ return convert(converter, column)
+
+ def write_batch(writer, physical, *args, **kwargs):
+ self.assertEqual([physical.num_rows], pending)
+ pending.clear()
+ return write_table(writer, physical, *args, **kwargs)
+
+ with patch.object(_MapFieldConverter, 'convert', convert_batch), \
+ patch.object(pq.ParquetWriter, 'write_table', write_batch):
+ messages = self._write(table, data)
+ self.assertEqual(5000, sum(sizes))
+ self.assertLessEqual(max(sizes), 1024)
+ self.assertEqual(1, len(messages[0].new_files))
+ reader = table.new_read_builder().with_projection(["metrics['a']", "metrics['late']"])
+ result = reader.new_read().to_arrow(reader.new_scan().plan().splits())
+ self.assertEqual([1] * 4999 + [None], result.column('metrics_a').to_pylist())
+ self.assertEqual([None] * 4999 + [2], result.column('metrics_late').to_pylist())
+
+ def test_failed_stream_removes_partial_file(self):
+ from pypaimon.write.map_shared_shredding_writer import MapSharedShreddingWriter
+
+ table = self._create_table('parquet', 2)
+ converter = MapSharedShreddingWriter(table.fields, table.options, 'parquet', None)
+ data = pa.Table.from_pydict({
+ 'id': list(range(2048)), 'metrics': [[('a', 1)]] * 2048,
+ }, schema=self.arrow_schema)
+ path = self.temp_dir + '/partial.parquet'
+ write_table = pq.ParquetWriter.write_table
+ calls = []
+
+ def fail_second_batch(writer, physical, *args, **kwargs):
+ calls.append(physical.num_rows)
+ if len(calls) == 2:
+ raise OSError('injected write failure')
+ return write_table(writer, physical, *args, **kwargs)
+
+ with patch.object(pq.ParquetWriter, 'write_table', fail_second_batch):
+ with self.assertRaisesRegex(OSError, 'injected write failure'):
+ converter.write_parquet(table.file_io, path, data, 'zstd', 1)
+ self.assertEqual(2, len(calls))
+ self.assertFalse(table.file_io.exists(path))
+
+ def test_row_groups_do_not_follow_input_calls(self):
+ from pypaimon.table.row.generic_row import GenericRow
+
+ for count in (1000, 2500):
+ layouts = []
+ for by_row in (False, True):
+ table = self._create_table('parquet', 256)
+ builder = table.new_batch_write_builder()
+ writer = builder.new_write()
+ if by_row:
+ for i in range(count):
+ writer.write_row(GenericRow([i, [('a', i)]], table.fields))
+ else:
+ writer.write_arrow(pa.Table.from_pydict({
+ 'id': list(range(count)),
+ 'metrics': [[('a', i)] for i in range(count)],
+ }, schema=self.arrow_schema))
+ messages = writer.prepare_commit()
+ builder.new_commit().commit(messages)
+ writer.close()
+ files = [f for m in messages for f in m.new_files]
+ self.assertEqual(1, len(files))
+ metadata = pq.read_metadata(files[0].file_path)
+ layouts.append([metadata.row_group(i).num_rows
+ for i in range(metadata.num_row_groups)])
+ self.assertEqual((count + 1023) // 1024, metadata.num_row_groups)
+ reader = table.new_read_builder().with_projection(['id', "metrics['a']"])
+ result = reader.new_read().to_arrow(reader.new_scan().plan().splits())
+ self.assertEqual(list(range(count)), result.column('id').to_pylist())
+ self.assertEqual(list(range(count)), result.column('metrics_a').to_pylist())
+ self.assertEqual(layouts[0], layouts[1])
+
+ def test_reject_postpone_with_fixed_output_bucket(self):
+ from pypaimon.write.writer.append_only_data_writer import AppendOnlyDataWriter
+
+ table = self._create_table('parquet', 2, {'bucket': '-2'})
+ with self.assertRaisesRegex(ValueError, 'postpone bucket'):
+ AppendOnlyDataWriter(table, (), 0, 0, table.options)
+
+ def test_adapts_physical_column_count_between_files(self):
+ table = self._create_table(
+ "parquet", max_columns=4,
+ extra_options={
+ "data-evolution.enabled": "true",
+ "row-tracking.enabled": "true",
+ "target-file-row-num": "2",
+ },
+ )
+ data = pa.Table.from_pydict({
+ "id": [1, 2, 3, 4],
+ "metrics": [
+ [("a", 1)],
+ [("a", 2)],
+ [("a", 3), ("b", 4)],
+ [("b", 5)],
+ ],
+ }, schema=self.arrow_schema)
+
+ messages = self._write(table, data)
+ files = [file for message in messages for file in message.new_files]
+ self.assertEqual(2, len(files))
+ counts = []
+ for file in files:
+ field = pq.read_schema(file.file_path).field("metrics")
+ counts.append(parse_shared_shredding_metadata(field)[1])
+ self.assertEqual([4, 1], counts)
+
+ read_builder = table.new_read_builder()
+ result = read_builder.new_read().to_arrow(
+ read_builder.new_scan().plan().splits())
+ self.assertEqual(4, result.num_rows)
+
+ def _create_table(
+ self, file_format, max_columns, extra_options=None):
+ options = {
+ "file.format": file_format,
+ "fields.metrics.map.storage-layout": "shared-shredding",
+ "fields.metrics.map.shared-shredding.max-columns": str(max_columns),
+ }
+ options.update(extra_options or {})
+ name = "default.map_write_{}_{}".format(
+ file_format, len(self.catalog.list_tables("default")))
+ self.catalog.create_table(
+ name,
+ Schema.from_pyarrow_schema(self.arrow_schema, options=options),
+ False,
+ )
+ return self.catalog.get_table(name)
+
+ @staticmethod
+ def _write(table, data):
+ builder = table.new_batch_write_builder()
+ writer = builder.new_write()
+ writer.write_arrow(data)
+ messages = writer.prepare_commit()
+ builder.new_commit().commit(messages)
+ writer.close()
+ return messages
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/paimon-python/pypaimon/write/map_shared_shredding_writer.py b/paimon-python/pypaimon/write/map_shared_shredding_writer.py
new file mode 100644
index 0000000000..07f21edf5c
--- /dev/null
+++ b/paimon-python/pypaimon/write/map_shared_shredding_writer.py
@@ -0,0 +1,573 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Java-compatible shared-shredding MAP conversion for data files."""
+
+import math
+from array import array
+from collections import deque
+
+import pyarrow as pa
+
+from pypaimon.data.map_shared_shredding import (
+ _normalized_offsets,
+ shared_shredding_metadata,
+)
+from pypaimon.schema.data_types import (
+ ArrayType,
+ AtomicType,
+ MapType,
+ MultisetType,
+ RowType,
+ VectorType,
+)
+from pypaimon.table.bucket_mode import BucketMode
+
+
+_FIELD_MAPPING = "__field_mapping"
+_OVERFLOW = "__overflow"
+_PHYSICAL_COLUMN_PREFIX = "__col_"
+_FIELD_ID_BASE = 2147483647 // 4
+_FIELD_ID_DEPTH_LIMIT = 1 << 10
+_CONVERSION_BYTES = 8 * 1024 * 1024
+
+
+class MapSharedShreddingWriter:
+ """Converts configured logical MAP columns before each file write."""
+
+ def __init__(self, fields, options, file_format, changelog_format):
+ self._options = options
+ field_by_name = {field.name: field for field in fields}
+ configured = self._configured_fields()
+ self._fields = {}
+ self._max_columns = {}
+ self._policies = {}
+ self._recent_widths = {}
+
+ for name, field in field_by_name.items():
+ if name in configured and not isinstance(field.type, MapType):
+ raise ValueError(
+ "Column '{}' is configured with map.storage-layout but "
+ "its type is not MAP.".format(name))
+ layout = options.map_storage_layout(name)
+ if layout not in ("default", "shared-shredding"):
+ raise ValueError(
+ "Unsupported MAP storage layout for '{}': {}".format(
+ name, layout))
+ if layout != "shared-shredding":
+ continue
+ self._validate_field(field)
+ self._fields[name] = field
+ self._max_columns[name] = \
+ options.map_shared_shredding_max_columns(name)
+ self._policies[name] = \
+ options.map_shared_shredding_column_placement_policy(name)
+ self._recent_widths[name] = deque(maxlen=20)
+
+ if configured.difference(field_by_name):
+ name = sorted(configured.difference(field_by_name))[0]
+ raise ValueError(
+ "Column '{}' is configured with map.storage-layout but does "
+ "not exist in table schema.".format(name))
+ if not self._fields:
+ return
+
+ self._validate_format("file.format", file_format)
+ self._validate_format("changelog.file.format", changelog_format)
+ self._validate_compression("file.compression", options.file_compression())
+ if options.bucket() == BucketMode.POSTPONE_BUCKET.value:
+ raise ValueError(
+ "MAP shared-shredding does not support postpone bucket mode.")
+ if any(_contains_type(field.type, _is_variant)
+ for field in fields):
+ raise ValueError(
+ "MAP shared-shredding cannot be used with VARIANT fields.")
+ if any(_contains_type(field.type, lambda t: isinstance(t, MultisetType))
+ for field in fields):
+ raise ValueError(
+ "MAP shared-shredding cannot be used with MULTISET fields.")
+
+ def is_active(self):
+ return bool(self._fields)
+
+ def write_parquet(self, file_io, path, data, compression, zstd_level):
+ """Plan key metadata, then write bounded physical batches without retaining them."""
+ import pyarrow.parquet as pq
+
+ fields = list(data.schema)
+ completed = {}
+ converters = {}
+ field_index = {field.name: index for index, field in enumerate(fields)}
+
+ for name, logical_field in self._fields.items():
+ index = field_index.get(name)
+ if index is None:
+ continue
+ input_field = fields[index]
+ if not pa.types.is_map(input_field.type):
+ raise TypeError(
+ "Shared-shredding column '{}' must be a MAP.".format(name))
+ num_columns = self._next_num_columns(name)
+ converter = _MapFieldConverter(
+ num_columns,
+ self._policies[name],
+ input_field.type.item_type,
+ logical_field.type.value,
+ )
+ # Parquet's Arrow schema metadata must be known before opening the
+ # writer. Scan only keys; reset placement for the actual write pass.
+ for chunk in data.column(index).chunks:
+ for start in range(0, len(chunk), 1024):
+ converter.analyze(chunk.slice(start, 1024))
+ metadata = dict(input_field.metadata or {})
+ metadata.update(shared_shredding_metadata(
+ converter.name_to_id,
+ converter.field_to_columns,
+ converter.overflow_set,
+ num_columns,
+ converter.max_row_width,
+ self._options.file_compression(),
+ ))
+ metadata.update(_field_id_metadata(logical_field.id))
+ fields[index] = pa.field(
+ name,
+ converter.physical_type,
+ nullable=input_field.nullable,
+ metadata=metadata,
+ )
+ completed[name] = converter.max_row_width
+ converters[index] = _MapFieldConverter(
+ num_columns, self._policies[name], input_field.type.item_type,
+ logical_field.type.value)
+
+ schema = pa.schema(fields, metadata=data.schema.metadata)
+ # Bound scratch space for slot mappings/indices as well as logical values.
+ slots = sum(c.num_columns for c in converters.values())
+ batch_rows = max(1, min(1024, _CONVERSION_BYTES // max(1, slots * 16)))
+ kwargs = {'compression': compression}
+ if compression.lower() == 'zstd':
+ kwargs['compression_level'] = zstd_level
+ try:
+ with file_io.new_output_stream(path) as stream:
+ with pq.ParquetWriter(stream, schema, **kwargs) as writer:
+ for offset in range(0, data.num_rows, batch_rows):
+ for bounded in _bounded_batches(data.slice(offset, batch_rows)):
+ # Coalesce only this bounded logical window, not the
+ # file. Input calls must not define row groups.
+ batch = bounded.combine_chunks().to_batches()[0]
+ columns = list(batch.columns)
+ for index, converter in converters.items():
+ columns[index] = converter.convert(columns[index])
+ physical = pa.Table.from_arrays(columns, schema=schema)
+ writer.write_table(physical)
+ del physical, columns
+ except Exception:
+ file_io.delete_quietly(path)
+ raise
+ return completed
+
+ def file_completed(self, completed):
+ for name, max_row_width in completed.items():
+ self._recent_widths[name].append(max_row_width)
+
+ def _next_num_columns(self, name):
+ widths = self._recent_widths[name]
+ max_columns = self._max_columns[name]
+ if not widths:
+ return max_columns
+ ordered = sorted(widths)
+ rank = max(1, min(int(math.ceil(0.90 * len(ordered))), len(ordered)))
+ percentile = ordered[rank - 1]
+ maximum = ordered[-1]
+ if (maximum - percentile <= 4
+ or maximum <= int(math.ceil(percentile * 1.25))):
+ width = maximum
+ else:
+ width = percentile
+ return max(1, min(width, max_columns))
+
+ def _configured_fields(self):
+ prefix = "fields."
+ suffix = ".map.storage-layout"
+ result = set()
+ for key in self._options.options.to_map():
+ if key.startswith(prefix) and key.endswith(suffix):
+ result.add(key[len(prefix):-len(suffix)])
+ return result
+
+ @staticmethod
+ def _validate_field(field):
+ if not isinstance(field.type, MapType):
+ raise ValueError(
+ "Column '{}' is configured with map.storage-layout="
+ "shared-shredding but its type is not MAP.".format(field.name))
+ key_type = field.type.key
+ if (not isinstance(key_type, AtomicType)
+ or not key_type.type.upper().startswith(
+ ("STRING", "VARCHAR"))):
+ raise ValueError(
+ "Shared-shredding column '{}' must use STRING keys.".format(
+ field.name))
+ if key_type.nullable:
+ raise ValueError(
+ "Shared-shredding MAP keys cannot be nullable for column '{}'."
+ .format(field.name))
+ if _contains_type(field.type.value, _is_blob):
+ raise ValueError(
+ "MAP shared-shredding cannot contain BLOB fields.")
+ if _contains_type(field.type.value, lambda t: isinstance(t, VectorType)):
+ raise ValueError(
+ "MAP shared-shredding cannot contain VECTOR fields.")
+
+ @staticmethod
+ def _validate_format(option, file_format):
+ if file_format and file_format.lower() != "parquet":
+ raise ValueError(
+ "PyPaimon MAP shared-shredding writes only support parquet, "
+ "but {} is {}.".format(option, file_format))
+
+ @staticmethod
+ def _validate_compression(option, compression):
+ if compression and compression.lower() not in ("none", "lz4", "zstd"):
+ raise ValueError(
+ "MAP shared-shredding only supports none/lz4/zstd compression, "
+ "but {} is {}.".format(option, compression))
+
+
+class _MapFieldConverter:
+
+ def __init__(self, num_columns, policy, item_type, logical_item_type):
+ self.num_columns = num_columns
+ self.policy = policy
+ self.name_to_id = {}
+ self.field_to_columns = {}
+ self.overflow_set = set()
+ self.max_row_width = 0
+ self._resident = [-1] * num_columns
+ self._last_used = [0] * num_columns
+ self._clock = 0
+ self.physical_type = _physical_struct_type(
+ num_columns, item_type, logical_item_type)
+
+ def convert(self, column):
+ offsets, start, end = _normalized_offsets(column)
+ keys = column.keys.slice(start, end - start).to_pylist()
+ values = _to_python_values(column.items.slice(start, end - start))
+ nulls = column.is_null().to_pylist()
+ mappings = array('i')
+ mapping_offsets = [0]
+ slots = {}
+ overflows = []
+ for row, is_null in enumerate(nulls):
+ start, end = offsets[row:row + 2]
+ if is_null:
+ mapping_offsets.append(len(mappings))
+ overflows.append(None)
+ continue
+ field_ids, mapping, overflow = self._place(keys[start:end])
+ items = dict(zip(field_ids, values[start:end]))
+ mappings.extend(mapping)
+ mapping_offsets.append(len(mappings))
+ for column_id, field_id in enumerate(mapping):
+ if field_id >= 0:
+ if column_id not in slots:
+ slots[column_id] = [None] * len(column)
+ slots[column_id][row] = items[field_id]
+ overflows.append([(field_id, items[field_id]) for field_id in overflow]
+ if overflow else None)
+ children = [pa.ListArray.from_arrays(
+ pa.array(mapping_offsets, type=pa.int32()),
+ pa.array(mappings, type=pa.int32()))]
+ empty = None
+ for column_id in range(self.num_columns):
+ value_type = self.physical_type[column_id + 1].type
+ if column_id in slots:
+ children.append(pa.array(slots.pop(column_id), type=value_type))
+ else:
+ if empty is None:
+ empty = pa.array([None] * len(column), type=value_type)
+ children.append(empty)
+ children.append(pa.array(overflows, type=self.physical_type[-1].type))
+ return pa.StructArray.from_arrays(
+ children, fields=list(self.physical_type),
+ mask=column.is_null() if column.null_count else None)
+
+ def analyze(self, column):
+ offsets, start, end = _normalized_offsets(column)
+ keys = column.keys.slice(start, end - start).to_pylist()
+ for row, is_null in enumerate(column.is_null().to_pylist()):
+ if not is_null:
+ self._place(keys[offsets[row]:offsets[row + 1]])
+
+ def _place(self, keys):
+ field_ids = []
+ for key in keys:
+ if key is None:
+ raise ValueError("Shared-shredding MAP keys cannot be null")
+ if not isinstance(key, str):
+ raise TypeError("Shared-shredding MAP keys must be strings")
+ field_id = self.name_to_id.setdefault(key, len(self.name_to_id))
+ field_ids.append(field_id)
+
+ mapping, overflow = self._allocate(field_ids)
+ for column_id, field_id in enumerate(mapping):
+ if field_id >= 0:
+ self.field_to_columns.setdefault(field_id, set()).add(column_id)
+ self.overflow_set.update(overflow)
+ self.max_row_width = max(self.max_row_width, len(field_ids))
+ return field_ids, mapping, overflow
+
+ def _allocate(self, field_ids):
+ if self.policy == "plain":
+ ordered = field_ids
+ return self._leading(ordered)
+ if self.policy == "sequential":
+ return self._leading(sorted(field_ids))
+ return self._lru(field_ids)
+
+ def _leading(self, field_ids):
+ mapping = [-1] * self.num_columns
+ for index, field_id in enumerate(field_ids[:self.num_columns]):
+ mapping[index] = field_id
+ return mapping, list(field_ids[self.num_columns:])
+
+ def _lru(self, field_ids):
+ mapping = [-1] * self.num_columns
+ next_resident = list(self._resident)
+ used = [False] * self.num_columns
+ unassigned = []
+ for field_id in sorted(field_ids):
+ try:
+ column_id = self._resident.index(field_id)
+ except ValueError:
+ unassigned.append(field_id)
+ continue
+ used[column_id] = True
+ mapping[column_id] = field_id
+
+ overflow = []
+ for field_id in unassigned:
+ column_id = self._select_lru_column(used, next_resident)
+ if column_id < 0:
+ overflow.append(field_id)
+ continue
+ used[column_id] = True
+ mapping[column_id] = field_id
+ next_resident[column_id] = field_id
+
+ touched = False
+ for column_id, field_id in enumerate(mapping):
+ if field_id >= 0:
+ self._last_used[column_id] = self._clock
+ touched = True
+ if touched:
+ self._clock += 1
+ self._resident = next_resident
+ return mapping, overflow
+
+ def _select_lru_column(self, used, resident):
+ selected = -1
+ selected_last_used = None
+ for column_id in range(self.num_columns):
+ if used[column_id]:
+ continue
+ if resident[column_id] < 0:
+ return column_id
+ last_used = self._last_used[column_id]
+ if selected < 0 or last_used < selected_last_used:
+ selected = column_id
+ selected_last_used = last_used
+ return selected
+
+
+def _bounded_batches(batch):
+ # One oversized row is indivisible; bound the other logical batches before
+ # expanding values into Python objects. The caller's input buffer is unchanged.
+ if batch.nbytes > _CONVERSION_BYTES and batch.num_rows > 1:
+ middle = batch.num_rows // 2
+ yield from _bounded_batches(batch.slice(0, middle))
+ yield from _bounded_batches(batch.slice(middle))
+ else:
+ yield batch
+
+
+def _to_python_values(column):
+ """Avoid Arrow 6 MAP scalars, including MAPs nested in ROW/ARRAY values."""
+ data_type = column.type
+ if pa.types.is_struct(data_type):
+ children = [_to_python_values(column.field(i))
+ for i in range(len(data_type))]
+ names = [field.name for field in data_type]
+ values = [dict(zip(names, (child[i] for child in children)))
+ for i in range(len(column))]
+ elif (pa.types.is_map(data_type) or pa.types.is_list(data_type)
+ or pa.types.is_large_list(data_type)):
+ offsets, start, end = _normalized_offsets(column)
+ if pa.types.is_map(data_type):
+ keys = _to_python_values(column.keys.slice(start, end - start))
+ items = _to_python_values(column.items.slice(start, end - start))
+ children = list(zip(keys, items))
+ else:
+ children = _to_python_values(column.values.slice(start, end - start))
+ values = [children[offsets[i]:offsets[i + 1]] for i in range(len(column))]
+ elif pa.types.is_fixed_size_list(data_type):
+ size = data_type.list_size
+ children = _to_python_values(column.values.slice(
+ column.offset * size, len(column) * size))
+ values = [children[i * size:(i + 1) * size] for i in range(len(column))]
+ else:
+ return column.to_pylist()
+ return [None if is_null else value
+ for is_null, value in zip(column.is_null().to_pylist(), values)]
+
+
+def _physical_struct_type(num_columns, item_type, logical_item_type):
+ mapping_type = pa.list_(pa.field(
+ "item",
+ pa.int32(),
+ metadata=_field_id_metadata(_array_element_id(0, 1)),
+ ))
+ fields = [pa.field(
+ _FIELD_MAPPING,
+ mapping_type,
+ metadata=_field_id_metadata(0),
+ )]
+ for column_id in range(num_columns):
+ fields.append(_field_with_ids(
+ _PHYSICAL_COLUMN_PREFIX + str(column_id),
+ item_type,
+ True,
+ logical_item_type,
+ column_id + 1,
+ ))
+ overflow_id = num_columns + 1
+ overflow_type = _map_type_with_ids(
+ pa.int32(), item_type, logical_item_type, overflow_id, 0)
+ fields.append(pa.field(
+ _OVERFLOW,
+ overflow_type,
+ metadata=_field_id_metadata(overflow_id),
+ ))
+ return pa.struct(fields)
+
+
+def _field_with_ids(name, arrow_type, nullable, logical_type, field_id):
+ return pa.field(
+ name,
+ _type_with_ids(arrow_type, logical_type, field_id, 0),
+ nullable=nullable,
+ metadata=_field_id_metadata(field_id),
+ )
+
+
+def _type_with_ids(arrow_type, logical_type, field_id, depth):
+ if isinstance(logical_type, RowType) and pa.types.is_struct(arrow_type):
+ arrow_by_name = {field.name: field for field in arrow_type}
+ return pa.struct([
+ _field_with_ids(
+ field.name,
+ arrow_by_name[field.name].type,
+ arrow_by_name[field.name].nullable,
+ field.type,
+ field.id,
+ )
+ for field in logical_type.fields
+ ])
+ if isinstance(logical_type, (ArrayType, VectorType)):
+ child_id = _array_element_id(field_id, depth + 1)
+ value_field = pa.field(
+ "item",
+ _type_with_ids(
+ arrow_type.value_type,
+ logical_type.element,
+ field_id,
+ depth + 1,
+ ),
+ nullable=logical_type.element.nullable,
+ metadata=_field_id_metadata(child_id),
+ )
+ if isinstance(logical_type, VectorType):
+ return pa.list_(value_field, logical_type.length)
+ return pa.list_(value_field)
+ if isinstance(logical_type, MapType):
+ return _map_type_with_ids(
+ arrow_type.key_type,
+ arrow_type.item_type,
+ logical_type.value,
+ field_id,
+ depth,
+ )
+ return arrow_type
+
+
+def _map_type_with_ids(
+ key_type, item_type, logical_item_type, field_id, depth):
+ key = pa.field(
+ "key",
+ key_type,
+ nullable=False,
+ metadata=_field_id_metadata(_map_key_id(field_id, depth + 1)),
+ )
+ item = pa.field(
+ "value",
+ _type_with_ids(
+ item_type, logical_item_type, field_id, depth + 1),
+ nullable=logical_item_type.nullable,
+ metadata=_field_id_metadata(_map_value_id(field_id, depth + 1)),
+ )
+ return pa.map_(key, item)
+
+
+def _field_id_metadata(field_id):
+ value = str(field_id).encode("utf-8")
+ return {b"PARQUET:field_id": value, b"paimon.id": value}
+
+
+def _array_element_id(field_id, depth):
+ return _FIELD_ID_BASE + field_id * _FIELD_ID_DEPTH_LIMIT + depth
+
+
+def _map_key_id(field_id, depth):
+ return _FIELD_ID_BASE - field_id * _FIELD_ID_DEPTH_LIMIT - depth
+
+
+def _map_value_id(field_id, depth):
+ return _FIELD_ID_BASE + field_id * _FIELD_ID_DEPTH_LIMIT + depth
+
+
+def _contains_type(data_type, predicate):
+ if predicate(data_type):
+ return True
+ if isinstance(data_type, RowType):
+ return any(_contains_type(field.type, predicate)
+ for field in data_type.fields)
+ if isinstance(data_type, (ArrayType, VectorType, MultisetType)):
+ return _contains_type(data_type.element, predicate)
+ if isinstance(data_type, MapType):
+ return (_contains_type(data_type.key, predicate)
+ or _contains_type(data_type.value, predicate))
+ return False
+
+
+def _is_variant(data_type):
+ return (isinstance(data_type, AtomicType)
+ and data_type.type.upper() == "VARIANT")
+
+
+def _is_blob(data_type):
+ return (isinstance(data_type, AtomicType)
+ and data_type.type.upper() == "BLOB")
diff --git a/paimon-python/pypaimon/write/writer/data_vector_writer.py b/paimon-python/pypaimon/write/writer/data_vector_writer.py
index 9294f67d92..4825b22af1 100644
--- a/paimon-python/pypaimon/write/writer/data_vector_writer.py
+++ b/paimon-python/pypaimon/write/writer/data_vector_writer.py
@@ -261,11 +261,13 @@ class DataVectorWriter(DataWriter):
if data.num_rows == 0:
return None
+ shredding_stats = {}
+
file_name = f"{CoreOptions.data_file_prefix(self.options)}{uuid.uuid4()}-0.{self.file_format}"
file_path = self._generate_file_path(file_name)
if self.file_format == CoreOptions.FILE_FORMAT_PARQUET:
- self.file_io.write_parquet(file_path, data, compression=self.compression, zstd_level=self.zstd_level)
+ shredding_stats = self._write_parquet_data(file_path, data)
elif self.file_format == CoreOptions.FILE_FORMAT_ORC:
self.file_io.write_orc(file_path, data, compression=self.compression, zstd_level=self.zstd_level)
elif self.file_format == CoreOptions.FILE_FORMAT_AVRO:
@@ -290,7 +292,7 @@ class DataVectorWriter(DataWriter):
min_seq, max_seq = self._append_file_sequence_range(data.num_rows)
- return DataFileMeta.create(
+ meta = DataFileMeta.create(
file_name=file_name,
file_size=self.file_io.get_file_size(file_path),
row_count=data.num_rows,
@@ -311,6 +313,8 @@ class DataVectorWriter(DataWriter):
file_path=file_path,
write_cols=self.write_cols,
)
+ self._map_shared_shredding.file_completed(shredding_stats)
+ return meta
def _validate_consistency(
self, normal_meta: DataFileMeta, vector_metas: List[DataFileMeta]):
diff --git a/paimon-python/pypaimon/write/writer/data_writer.py b/paimon-python/pypaimon/write/writer/data_writer.py
index 24a67186c3..46db0578ce 100644
--- a/paimon-python/pypaimon/write/writer/data_writer.py
+++ b/paimon-python/pypaimon/write/writer/data_writer.py
@@ -29,6 +29,7 @@ from pypaimon.manifest.schema.simple_stats import SimpleStats
from pypaimon.schema.data_types import PyarrowFieldParser
from pypaimon.table.bucket_mode import BucketMode
from pypaimon.table.row.generic_row import GenericRow
+from pypaimon.write.map_shared_shredding_writer import MapSharedShreddingWriter
from pypaimon.write.writer.mosaic_writer_options import create_mosaic_writer_options
from pypaimon.write.writer.write_buffer import WriteBuffer
@@ -104,6 +105,13 @@ class DataWriter(ABC):
# Paimon field id map, used by _apply_variant_shredding; built once since
# the table schema is fixed for the lifetime of this writer.
self._paimon_field_id: Dict[str, int] = {pf.name: pf.id for pf in self.table.fields}
+ self._map_shared_shredding = MapSharedShreddingWriter(
+ self.table.fields,
+ self.options,
+ self.file_format,
+ self.changelog_file_format
+ if self.changelog_producer == ChangelogProducer.INPUT else None,
+ )
# Set by the composite writers when a flush landed its normal data file but a
# later phase of the same flush failed; see their ``_close_current_writers``.
@@ -259,6 +267,7 @@ class DataWriter(ABC):
extra_files = []
row_sidecar_path = None
changelog_meta = None
+ shared_shredding_stats = {}
if self._variant_shredding:
data = self._apply_variant_shredding(data)
@@ -270,7 +279,7 @@ class DataWriter(ABC):
# already covers.
try:
if self.file_format == CoreOptions.FILE_FORMAT_PARQUET:
- self.file_io.write_parquet(file_path, data, compression=self.compression, zstd_level=self.zstd_level)
+ shared_shredding_stats = self._write_parquet_data(file_path, data)
elif self.file_format == CoreOptions.FILE_FORMAT_ORC:
self.file_io.write_orc(file_path, data, compression=self.compression, zstd_level=self.zstd_level)
elif self.file_format == CoreOptions.FILE_FORMAT_AVRO:
@@ -300,7 +309,7 @@ class DataWriter(ABC):
# min key & max key
- selected_table = data.select(self.trimmed_primary_keys)
+ selected_table = logical_data.select(self.trimmed_primary_keys)
key_columns_batch = selected_table.to_batches()[0]
min_key_row_batch = key_columns_batch.slice(0, 1)
max_key_row_batch = key_columns_batch.slice(key_columns_batch.num_rows - 1, 1)
@@ -311,21 +320,23 @@ class DataWriter(ABC):
value_stats_enabled = self.options.metadata_stats_enabled()
if value_stats_enabled:
stats_fields = self.table.fields if self.table.is_primary_key_table \
- else PyarrowFieldParser.to_paimon_schema(data.schema)
+ else PyarrowFieldParser.to_paimon_schema(logical_data.schema)
else:
stats_fields = self.table.trimmed_primary_keys_fields
column_stats = {
- field.name: self._get_column_stats(data, field.name)
+ field.name: self._get_column_stats(logical_data, field.name)
for field in stats_fields
}
key_fields = self.trimmed_primary_keys_fields
- key_stats = self._collect_value_stats(data, key_fields, column_stats)
+ key_stats = self._collect_value_stats(
+ logical_data, key_fields, column_stats)
if not self.options.primary_key_nullable() and not all(
count == 0 for count in key_stats.null_counts):
raise RuntimeError("Primary key should not be null")
value_fields = stats_fields if value_stats_enabled else []
- value_stats = self._collect_value_stats(data, value_fields, column_stats)
+ value_stats = self._collect_value_stats(
+ logical_data, value_fields, column_stats)
# Read the range without advancing it: the advance belongs with the
# append below, so a retried flush derives the same range.
@@ -368,10 +379,18 @@ class DataWriter(ABC):
raise
self.sequence_generator.start = self.sequence_generator.current
+ self._map_shared_shredding.file_completed(shared_shredding_stats)
self.committed_files.append(data_meta)
if changelog_meta is not None:
self.committed_changelog_files.append(changelog_meta)
+ def _write_parquet_data(self, path, data):
+ if self._map_shared_shredding.is_active():
+ return self._map_shared_shredding.write_parquet(
+ self.file_io, path, data, self.compression, self.zstd_level)
+ self.file_io.write_parquet(path, data, compression=self.compression, zstd_level=self.zstd_level)
+ return {}
+
def _apply_variant_shredding(self, data: pa.Table) -> pa.Table:
"""Transform VARIANT columns into shredded Parquet format.
@@ -414,8 +433,7 @@ class DataWriter(ABC):
try:
if cl_fmt == CoreOptions.FILE_FORMAT_PARQUET:
- self.file_io.write_parquet(changelog_file_path, data, compression=self.compression,
- zstd_level=self.zstd_level)
+ self._write_parquet_data(changelog_file_path, data)
elif cl_fmt == CoreOptions.FILE_FORMAT_ORC:
self.file_io.write_orc(changelog_file_path, data, compression=self.compression,
zstd_level=self.zstd_level)
diff --git a/paimon-python/pypaimon/write/writer/dedicated_format_writer.py b/paimon-python/pypaimon/write/writer/dedicated_format_writer.py
index c894182fbe..f79369d1cd 100644
--- a/paimon-python/pypaimon/write/writer/dedicated_format_writer.py
+++ b/paimon-python/pypaimon/write/writer/dedicated_format_writer.py
@@ -703,12 +703,14 @@ class DedicatedFormatWriter(DataWriter):
if data.num_rows == 0:
return None
+ shredding_stats = {}
+
file_name = f"{CoreOptions.data_file_prefix(self.options)}{uuid.uuid4()}-0.{self.file_format}"
file_path = self._generate_file_path(file_name)
# Write file based on format
if self.file_format == CoreOptions.FILE_FORMAT_PARQUET:
- self.file_io.write_parquet(file_path, data, compression=self.compression, zstd_level=self.zstd_level)
+ shredding_stats = self._write_parquet_data(file_path, data)
elif self.file_format == CoreOptions.FILE_FORMAT_ORC:
self.file_io.write_orc(file_path, data, compression=self.compression, zstd_level=self.zstd_level)
elif self.file_format == CoreOptions.FILE_FORMAT_AVRO:
@@ -728,7 +730,9 @@ class DedicatedFormatWriter(DataWriter):
is_external_path = self.external_path_provider is not None
external_path_str = file_path if is_external_path else None
- return self._create_data_file_meta(file_name, file_path, data, external_path_str)
+ meta = self._create_data_file_meta(file_name, file_path, data, external_path_str)
+ self._map_shared_shredding.file_completed(shredding_stats)
+ return meta
def _create_data_file_meta(self, file_name: str, file_path: str, data: pa.Table,
external_path: Optional[str] = None) -> DataFileMeta:
(paimon) 15/20: [python] Resolve value stats columns using file schema (#9906)
Posted by ho...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
XiaoHongbo-Hope pushed a commit to branch release-2.1
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit 4151992545117ea5d86394fb886c65eb48addfcc
Author: XiaoHongbo <xi...@alibaba-inc.com>
AuthorDate: Thu Sep 17 11:30:10 2026 +0800
[python] Resolve value stats columns using file schema (#9906)
---
.../pypaimon/manifest/manifest_file_manager.py | 3 ++-
paimon-python/pypaimon/tests/reader_base_test.py | 23 ++++++++++++++++++++++
2 files changed, 25 insertions(+), 1 deletion(-)
diff --git a/paimon-python/pypaimon/manifest/manifest_file_manager.py b/paimon-python/pypaimon/manifest/manifest_file_manager.py
index 7b438da88f..044d8985f9 100644
--- a/paimon-python/pypaimon/manifest/manifest_file_manager.py
+++ b/paimon-python/pypaimon/manifest/manifest_file_manager.py
@@ -317,7 +317,8 @@ class ManifestFileManager:
elif not file_dict['_VALUE_STATS_COLS']:
fields = []
else:
- fields = [self.table.field_dict[col] for col in file_dict['_VALUE_STATS_COLS']]
+ data_field_dict = {f.name: f for f in schema_fields}
+ fields = [data_field_dict[col] for col in file_dict['_VALUE_STATS_COLS']]
return fields
def write(self, file_name, entries: List[ManifestEntry]):
diff --git a/paimon-python/pypaimon/tests/reader_base_test.py b/paimon-python/pypaimon/tests/reader_base_test.py
index 27c2413bb5..6faf40851d 100644
--- a/paimon-python/pypaimon/tests/reader_base_test.py
+++ b/paimon-python/pypaimon/tests/reader_base_test.py
@@ -37,6 +37,7 @@ from pypaimon.manifest.schema.manifest_entry import ManifestEntry
from pypaimon.manifest.schema.simple_stats import SimpleStats
from pypaimon.schema.data_types import (ArrayType, AtomicType, DataField,
MapType, PyarrowFieldParser)
+from pypaimon.schema.schema_change import SchemaChange
from pypaimon.schema.table_schema import TableSchema
from pypaimon.table.row.generic_row import GenericRow, GenericRowDeserializer
from pypaimon.write.file_store_commit import FileStoreCommit
@@ -585,6 +586,28 @@ class ReaderBasicTest(unittest.TestCase):
table_with_stats = catalog.get_table("test_db.test_value_stats_cols_schema_match")
self._test_append_only_schema_match_case(table_with_stats, pa_schema)
+ def test_value_stats_cols_use_file_schema_after_schema_evolution(self):
+ table_name = 'default.test_value_stats_schema_evolution'
+ schema = Schema.from_pyarrow_schema(pa.schema([
+ ('id', pa.int64()), ('name', pa.string())]))
+ self.catalog.create_table(table_name, schema, False)
+ table = self.catalog.get_table(table_name)
+ self._test_value_stats_cols_case(
+ ManifestFileManager(table), table, ['id', 'name'], 2, 'schema_evolution')
+
+ for change in (SchemaChange.rename_column('name', 'renamed'),
+ SchemaChange.drop_column('renamed'),
+ SchemaChange.add_column('name', AtomicType('INT'))):
+ self.catalog.alter_table(table_name, [change], False)
+ table = self.catalog.get_table(table_name)
+ entries = ManifestFileManager(table).read(
+ 'manifest-test-schema_evolution', drop_stats=False)
+ stats = entries[0].file.value_stats
+ self.assertEqual([field.name for field in stats.min_values.fields],
+ ['id', 'name'])
+ self.assertEqual(stats.min_values.get_field(1), 'apple')
+ self.assertEqual(stats.max_values.get_field(1), 'zebra')
+
def test_primary_key_value_stats_excludes_system_fields(self):
catalog = CatalogFactory.create({
"warehouse": self.warehouse
(paimon) 13/20: [core] Prune manifest blocks with sidecars during scans (#9897)
Posted by ho...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
XiaoHongbo-Hope pushed a commit to branch release-2.1
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit 084cde4772de3ff587e3747f3771f75294fbf41c
Author: YeJunHao <41...@users.noreply.github.com>
AuthorDate: Thu Sep 17 11:04:01 2026 +0800
[core] Prune manifest blocks with sidecars during scans (#9897)
---
docs/docs/concepts/spec/manifest.md | 20 +-
docs/generated/catalog_configuration.html | 6 +
.../org/apache/paimon/options/CatalogOptions.java | 10 +
.../benchmark/ManifestFileSorterBenchmark.java | 1 +
.../java/org/apache/paimon/AbstractFileStore.java | 7 +
.../src/main/java/org/apache/paimon/FileStore.java | 2 +
.../org/apache/paimon/catalog/CachingCatalog.java | 21 +
.../org/apache/paimon/manifest/BucketFilter.java | 16 +
.../org/apache/paimon/manifest/ManifestFile.java | 85 ++-
.../apache/paimon/manifest/ManifestSidecar.java | 4 +-
.../paimon/operation/AbstractFileStoreScan.java | 17 +-
.../paimon/table/AbstractFileStoreTable.java | 16 +
.../paimon/table/DelegatedFileStoreTable.java | 11 +
.../paimon/table/FallbackReadFileStoreTable.java | 6 +
.../org/apache/paimon/table/FileStoreTable.java | 5 +
.../java/org/apache/paimon/utils/ObjectsCache.java | 6 +
.../apache/paimon/catalog/CachingCatalogTest.java | 142 +++++
.../paimon/manifest/ManifestFileMetaTest.java | 2 +
.../paimon/manifest/ManifestFileMetaTestBase.java | 1 +
.../apache/paimon/manifest/ManifestFileTest.java | 667 ++++++++++++++++++++-
.../paimon/manifest/ManifestSidecarTest.java | 12 +-
.../paimon/manifest/ManifestSidecarWriteTest.java | 1 +
.../operation/ManifestRewriteCleanupTest.java | 1 +
23 files changed, 1022 insertions(+), 37 deletions(-)
diff --git a/docs/docs/concepts/spec/manifest.md b/docs/docs/concepts/spec/manifest.md
index 29466c1ad6..7f8bf7a8a5 100644
--- a/docs/docs/concepts/spec/manifest.md
+++ b/docs/docs/concepts/spec/manifest.md
@@ -72,20 +72,28 @@ using independent partition, row-ID and bucket coverage. A sidecar uses the
derived file name. The Avro schemas and `_VERSION` identifiers remain unchanged.
The utility includes construction, validation, block selection and optional caching. Java table
-writers generate sidecars when `manifest.sidecar.enabled` is true; when unset, it inherits
+writers and scans use sidecars when `manifest.sidecar.enabled` is true; when unset, it inherits
`manifest-sort.enabled`. Both ordinary writes and raw manifest rewrites build the sidecar from
the completed output manifest and publish its `_EXTRA_FILES` reference only after both files
close successfully. Failed writes and aborted writers clean up their own manifest/sidecar pairs.
-Scans do not yet invoke sidecar pruning automatically. Callers remain responsible for applying
-entry filters and reconciling ADD/DELETE entries after block selection. The low-level `build`
-method returns sidecar bytes without writing or publishing another file.
+Scans with partition, row-ID or bucket filters select blocks before reading manifest entries.
+Normal entry filtering and ADD/DELETE reconciliation still apply. Missing or unusable sidecars
+fall back to normal manifest reads; disabled sidecars and scans without these filters do not
+perform sidecar I/O. Sidecar caching is controlled by the catalog option
+`cache.manifest-sidecar.max-memory` (64 MiB by default). A positive value supplies an
+additional budget independent of the manifest content cache. When set to 0, sidecars reuse
+the manifest content cache, or remain uncached if that cache is disabled. Sidecar caching
+uses the catalog's `cache.expire-after-access` and `cache.manifest.soft-values` policies.
+Selected block bytes still share the manifest content cache without populating the
+whole-manifest entry cache with partial results. The low-level `build` method returns
+sidecar bytes without writing or publishing another file.
Callers decide whether to invoke `build` and `read`; these utilities have no read/write switches.
`build` and `Builder` accept `rowIdEnabled` and `bucketEnabled` arguments for independent
payload generation. Partition generation is always enabled,
including the empty partition tuple for unpartitioned tables. Missing or invalid
-metadata makes only the affected block's dimension unavailable. There is no sidecar byte budget:
-construction keeps complete coverage and `read` consumes the entire file once it is opened.
+metadata makes only the affected block's dimension unavailable. Cache limits do not truncate
+sidecars: construction keeps complete coverage and `read` consumes the entire file once it is opened.
`read` returns null for an absent sidecar reference or an `IOException`, allowing the caller
to fall back to the manifest. If the thread is interrupted, the I/O failure is propagated as
diff --git a/docs/generated/catalog_configuration.html b/docs/generated/catalog_configuration.html
index 806edac3f8..bc7226d1b0 100644
--- a/docs/generated/catalog_configuration.html
+++ b/docs/generated/catalog_configuration.html
@@ -50,6 +50,12 @@ under the License.
<td>Duration</td>
<td>Cache expiration policy: marks cache entries to expire after a specified duration has passed since their last refresh.</td>
</tr>
+ <tr>
+ <td><h5>cache.manifest-sidecar.max-memory</h5></td>
+ <td style="word-wrap: break-word;">64 mb</td>
+ <td>MemorySize</td>
+ <td>Controls the maximum memory for the separate manifest sidecar cache. This budget is additional to the manifest content cache. Set to 0 to reuse the manifest content cache. It uses 'cache.expire-after-access' and 'cache.manifest.soft-values'.</td>
+ </tr>
<tr>
<td><h5>cache.manifest.max-memory</h5></td>
<td style="word-wrap: break-word;">(none)</td>
diff --git a/paimon-api/src/main/java/org/apache/paimon/options/CatalogOptions.java b/paimon-api/src/main/java/org/apache/paimon/options/CatalogOptions.java
index b1ccfe1caa..bda512a944 100644
--- a/paimon-api/src/main/java/org/apache/paimon/options/CatalogOptions.java
+++ b/paimon-api/src/main/java/org/apache/paimon/options/CatalogOptions.java
@@ -130,6 +130,16 @@ public class CatalogOptions {
.noDefaultValue()
.withDescription("Controls the maximum memory to cache manifest content.");
+ public static final ConfigOption<MemorySize> CACHE_MANIFEST_SIDECAR_MAX_MEMORY =
+ key("cache.manifest-sidecar.max-memory")
+ .memoryType()
+ .defaultValue(MemorySize.ofMebiBytes(64))
+ .withDescription(
+ "Controls the maximum memory for the separate manifest sidecar cache. "
+ + "This budget is additional to the manifest content cache. "
+ + "Set to 0 to reuse the manifest content cache. It uses "
+ + "'cache.expire-after-access' and 'cache.manifest.soft-values'.");
+
public static final ConfigOption<Boolean> CACHE_MANIFEST_SOFT_VALUES =
key("cache.manifest.soft-values")
.booleanType()
diff --git a/paimon-benchmark/paimon-micro-benchmarks/src/test/java/org/apache/paimon/benchmark/ManifestFileSorterBenchmark.java b/paimon-benchmark/paimon-micro-benchmarks/src/test/java/org/apache/paimon/benchmark/ManifestFileSorterBenchmark.java
index e805efb694..1f78ff8d45 100644
--- a/paimon-benchmark/paimon-micro-benchmarks/src/test/java/org/apache/paimon/benchmark/ManifestFileSorterBenchmark.java
+++ b/paimon-benchmark/paimon-micro-benchmarks/src/test/java/org/apache/paimon/benchmark/ManifestFileSorterBenchmark.java
@@ -301,6 +301,7 @@ public class ManifestFileSorterBenchmark {
pathFactory,
TARGET_MANIFEST_SIZE,
null,
+ null,
new CoreOptions(new Options()))
.create();
}
diff --git a/paimon-core/src/main/java/org/apache/paimon/AbstractFileStore.java b/paimon-core/src/main/java/org/apache/paimon/AbstractFileStore.java
index 29c7ab8de0..86839355a7 100644
--- a/paimon-core/src/main/java/org/apache/paimon/AbstractFileStore.java
+++ b/paimon-core/src/main/java/org/apache/paimon/AbstractFileStore.java
@@ -109,6 +109,7 @@ abstract class AbstractFileStore<T> implements FileStore<T> {
protected final CatalogEnvironment catalogEnvironment;
@Nullable private SegmentsCache<Path> readManifestCache;
+ @Nullable private SegmentsCache<Path> manifestSidecarCache;
@Nullable private Cache<Path, Snapshot> snapshotCache;
protected AbstractFileStore(
@@ -212,6 +213,7 @@ abstract class AbstractFileStore<T> implements FileStore<T> {
pathFactory(),
options.manifestTargetSize().getBytes(),
readManifestCache,
+ manifestSidecarCache,
options);
}
@@ -616,6 +618,11 @@ abstract class AbstractFileStore<T> implements FileStore<T> {
this.readManifestCache = manifestCache;
}
+ @Override
+ public void setManifestSidecarCache(SegmentsCache<Path> manifestSidecarCache) {
+ this.manifestSidecarCache = manifestSidecarCache;
+ }
+
@Override
public void setSnapshotCache(Cache<Path, Snapshot> cache) {
this.snapshotCache = cache;
diff --git a/paimon-core/src/main/java/org/apache/paimon/FileStore.java b/paimon-core/src/main/java/org/apache/paimon/FileStore.java
index 1714eec2a9..d9b143f75f 100644
--- a/paimon-core/src/main/java/org/apache/paimon/FileStore.java
+++ b/paimon-core/src/main/java/org/apache/paimon/FileStore.java
@@ -127,5 +127,7 @@ public interface FileStore<T> {
void setManifestCache(SegmentsCache<Path> manifestCache);
+ void setManifestSidecarCache(SegmentsCache<Path> manifestSidecarCache);
+
void setSnapshotCache(Cache<Path, Snapshot> cache);
}
diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/CachingCatalog.java b/paimon-core/src/main/java/org/apache/paimon/catalog/CachingCatalog.java
index 7fce5edf0e..037f7ea2bf 100644
--- a/paimon-core/src/main/java/org/apache/paimon/catalog/CachingCatalog.java
+++ b/paimon-core/src/main/java/org/apache/paimon/catalog/CachingCatalog.java
@@ -51,6 +51,7 @@ import static org.apache.paimon.options.CatalogOptions.CACHE_ENABLED;
import static org.apache.paimon.options.CatalogOptions.CACHE_EXPIRE_AFTER_ACCESS;
import static org.apache.paimon.options.CatalogOptions.CACHE_EXPIRE_AFTER_WRITE;
import static org.apache.paimon.options.CatalogOptions.CACHE_MANIFEST_MAX_MEMORY;
+import static org.apache.paimon.options.CatalogOptions.CACHE_MANIFEST_SIDECAR_MAX_MEMORY;
import static org.apache.paimon.options.CatalogOptions.CACHE_MANIFEST_SMALL_FILE_MEMORY;
import static org.apache.paimon.options.CatalogOptions.CACHE_MANIFEST_SMALL_FILE_THRESHOLD;
import static org.apache.paimon.options.CatalogOptions.CACHE_MANIFEST_SOFT_VALUES;
@@ -71,6 +72,7 @@ public class CachingCatalog extends DelegateCatalog {
protected Cache<String, Database> databaseCache;
protected Cache<Identifier, Table> tableCache;
@Nullable protected final SegmentsCache<Path> manifestCache;
+ @Nullable protected final SegmentsCache<Path> manifestSidecarCache;
// partition cache will affect data latency
@Nullable protected Cache<Identifier, List<Partition>> partitionCache;
@Nullable protected DVMetaCache dvMetaCache;
@@ -108,6 +110,17 @@ public class CachingCatalog extends DelegateCatalog {
expireAfterAccess,
manifestCacheSoftValues);
+ MemorySize sidecarMaxMemory = options.get(CACHE_MANIFEST_SIDECAR_MAX_MEMORY);
+ this.manifestSidecarCache =
+ sidecarMaxMemory.getBytes() == 0
+ ? manifestCache
+ : SegmentsCache.create(
+ (int) CoreOptions.PAGE_SIZE.defaultValue().getBytes(),
+ sidecarMaxMemory,
+ sidecarMaxMemory.getBytes(),
+ expireAfterAccess,
+ manifestCacheSoftValues);
+
this.cachedPartitionMaxNum = options.get(CACHE_PARTITION_MAX_NUM);
int cacheDvMaxNum = options.get(CACHE_DV_MAX_NUM);
@@ -299,6 +312,9 @@ public class CachingCatalog extends DelegateCatalog {
if (manifestCache != null) {
storeTable.setManifestCache(manifestCache);
}
+ if (manifestSidecarCache != null) {
+ storeTable.setManifestSidecarCache(manifestSidecarCache);
+ }
if (dvMetaCache != null) {
storeTable.setDVMetaCache(dvMetaCache);
}
@@ -425,6 +441,11 @@ public class CachingCatalog extends DelegateCatalog {
manifestCacheSize = manifestCache.estimatedSize();
manifestCacheBytes = manifestCache.totalCacheBytes();
}
+ // Keep reporting total manifest metadata usage even though sidecars have their own budget.
+ if (manifestSidecarCache != null && manifestSidecarCache != manifestCache) {
+ manifestCacheSize += manifestSidecarCache.estimatedSize();
+ manifestCacheBytes += manifestSidecarCache.totalCacheBytes();
+ }
long partitionCacheSize = 0L;
if (partitionCache != null) {
for (Map.Entry<Identifier, List<Partition>> entry : partitionCache.asMap().entrySet()) {
diff --git a/paimon-core/src/main/java/org/apache/paimon/manifest/BucketFilter.java b/paimon-core/src/main/java/org/apache/paimon/manifest/BucketFilter.java
index 536be456ee..3e564a60f9 100644
--- a/paimon-core/src/main/java/org/apache/paimon/manifest/BucketFilter.java
+++ b/paimon-core/src/main/java/org/apache/paimon/manifest/BucketFilter.java
@@ -78,6 +78,22 @@ public class BucketFilter {
|| totalAwareBucketFilter.test(partition, bucket, totalBucket);
}
+ /** Conservatively checks an indexed pair without inventing a partition for custom filters. */
+ public boolean mayContain(int bucket, int totalBuckets) {
+ if (onlyReadRealBuckets && bucket < 0) {
+ return false;
+ }
+ if (specifiedBucket != null && bucket != specifiedBucket) {
+ return false;
+ }
+ if (bucketFilter != null && !bucketFilter.test(bucket)) {
+ return false;
+ }
+ return !(totalAwareBucketFilter instanceof ManifestBucketFilter)
+ || ((ManifestBucketFilter) totalAwareBucketFilter)
+ .mayContain(bucket, bucket, totalBuckets);
+ }
+
/** Conservatively tests whether a manifest's bucket metadata can contain a matching entry. */
public boolean mayContain(ManifestFileMeta manifest) {
Integer minBucket = manifest.minBucket();
diff --git a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFile.java b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFile.java
index 30e0adcf1b..9740961e73 100644
--- a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFile.java
+++ b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFile.java
@@ -37,6 +37,7 @@ import org.apache.paimon.utils.FileUtils;
import org.apache.paimon.utils.Filter;
import org.apache.paimon.utils.ObjectsFile;
import org.apache.paimon.utils.PathFactory;
+import org.apache.paimon.utils.RowRangeIndex;
import org.apache.paimon.utils.SegmentsCache;
import javax.annotation.Nullable;
@@ -61,6 +62,7 @@ public class ManifestFile extends ObjectsFile<ManifestEntry> {
private final AvroFileFormat avroFileFormat;
private final long suggestedFileSize;
private final CoreOptions options;
+ @Nullable private final SegmentsCache<Path> sidecarCache;
private ManifestFile(
FileIO fileIO,
@@ -72,6 +74,7 @@ public class ManifestFile extends ObjectsFile<ManifestEntry> {
PathFactory pathFactory,
long suggestedFileSize,
@Nullable SegmentsCache<Path> cache,
+ @Nullable SegmentsCache<Path> sidecarCache,
CoreOptions options) {
super(
fileIO,
@@ -89,6 +92,7 @@ public class ManifestFile extends ObjectsFile<ManifestEntry> {
this.avroFileFormat = avroFileFormat;
this.suggestedFileSize = suggestedFileSize;
this.options = options;
+ this.sidecarCache = sidecarCache == null ? cache : sidecarCache;
}
@Override
@@ -140,9 +144,33 @@ public class ManifestFile extends ObjectsFile<ManifestEntry> {
Filter<InternalRow> readFilter,
Filter<ManifestEntry> readTFilter,
Function<ManifestEntry, T> convertor) {
+ return read(
+ fileName,
+ fileSize,
+ partitionFilter,
+ bucketFilter,
+ readFilter,
+ readTFilter,
+ convertor,
+ null);
+ }
+
+ public <T> List<T> read(
+ String fileName,
+ @Nullable Long fileSize,
+ @Nullable PartitionPredicate partitionFilter,
+ @Nullable BucketFilter bucketFilter,
+ Filter<InternalRow> readFilter,
+ Filter<ManifestEntry> readTFilter,
+ Function<ManifestEntry, T> convertor,
+ @Nullable ManifestSidecar.Selection selected) {
+ if (selected != null && selected.blocks().isEmpty()) {
+ return java.util.Collections.emptyList();
+ }
try {
Path path = pathFactory.toPath(fileName);
- if (cache != null) {
+ // Sidecar selections use the block cache, even when every block is selected.
+ if (cache != null && selected == null) {
ManifestEntryFilters filters =
new ManifestEntryFilters(
partitionFilter, bucketFilter, readFilter, readTFilter);
@@ -155,7 +183,9 @@ public class ManifestFile extends ObjectsFile<ManifestEntry> {
path,
ManifestEntry.MANIFEST_ROW_TYPE,
partitionFilter,
- bucketFilter);
+ bucketFilter,
+ selected,
+ cache == null ? null : cache.segmentsCache());
return readFromIterator(iterator, serializer, readFilter, readTFilter, convertor);
} catch (IOException e) {
throw new UncheckedIOException(e);
@@ -209,8 +239,23 @@ public class ManifestFile extends ObjectsFile<ManifestEntry> {
@Nullable PartitionPredicate partitionFilter,
@Nullable BucketFilter bucketFilter)
throws IOException {
+ return createManifestIterator(
+ fileIO, path, projectedType, partitionFilter, bucketFilter, null, null);
+ }
+
+ private static CloseableIterator<InternalRow> createManifestIterator(
+ FileIO fileIO,
+ Path path,
+ RowType projectedType,
+ @Nullable PartitionPredicate partitionFilter,
+ @Nullable BucketFilter bucketFilter,
+ @Nullable ManifestSidecar.Selection selected,
+ @Nullable SegmentsCache<Object> cache)
+ throws IOException {
try {
- ManifestAvroReader reader = new ManifestAvroReader(fileIO.newInputStream(path));
+ ManifestAvroReader reader =
+ new ManifestAvroReader(
+ ManifestSidecar.openManifest(fileIO, path, selected, cache));
return reader.read(projectedType, partitionFilter, bucketFilter);
} catch (IOException e) {
FileUtils.checkExists(fileIO, path);
@@ -345,6 +390,32 @@ public class ManifestFile extends ObjectsFile<ManifestEntry> {
};
}
+ @Nullable
+ public ManifestSidecar.Selection selectBlocks(
+ ManifestFileMeta manifest, @Nullable RowRangeIndex query) {
+ return selectBlocks(manifest, query, null, null);
+ }
+
+ @Nullable
+ public ManifestSidecar.Selection selectBlocks(
+ ManifestFileMeta manifest,
+ @Nullable RowRangeIndex query,
+ @Nullable PartitionPredicate partitionFilter,
+ @Nullable BucketFilter bucketFilter) {
+ return !options.manifestSidecarEnabled()
+ || (query == null && partitionFilter == null && bucketFilter == null)
+ ? null
+ : ManifestSidecar.read(
+ fileIO,
+ pathFactory.toPath(manifest.fileName()),
+ manifest,
+ query,
+ partitionFilter,
+ partitionType,
+ bucketFilter == null ? null : bucketFilter::mayContain,
+ sidecarCache);
+ }
+
/** Deletes an unreferenced manifest and its explicitly referenced extra files. */
public void delete(ManifestFileMeta manifest) {
delete(manifest.fileName());
@@ -365,6 +436,7 @@ public class ManifestFile extends ObjectsFile<ManifestEntry> {
private final long suggestedFileSize;
private final CoreOptions options;
@Nullable private final SegmentsCache<Path> cache;
+ @Nullable private final SegmentsCache<Path> sidecarCache;
public Factory(
FileIO fileIO,
@@ -375,6 +447,7 @@ public class ManifestFile extends ObjectsFile<ManifestEntry> {
FileStorePathFactory pathFactory,
long suggestedFileSize,
@Nullable SegmentsCache<Path> cache,
+ @Nullable SegmentsCache<Path> sidecarCache,
CoreOptions options) {
this.fileIO = fileIO;
this.schemaManager = schemaManager;
@@ -384,13 +457,10 @@ public class ManifestFile extends ObjectsFile<ManifestEntry> {
this.pathFactory = pathFactory;
this.suggestedFileSize = suggestedFileSize;
this.cache = cache;
+ this.sidecarCache = sidecarCache;
this.options = options;
}
- public boolean isCacheEnabled() {
- return cache != null;
- }
-
public ManifestFile create() {
return new ManifestFile(
fileIO,
@@ -402,6 +472,7 @@ public class ManifestFile extends ObjectsFile<ManifestEntry> {
pathFactory.manifestFileFactory(),
suggestedFileSize,
cache,
+ sidecarCache,
options);
}
}
diff --git a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestSidecar.java b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestSidecar.java
index c6ee9e9f56..6a8ce2b143 100644
--- a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestSidecar.java
+++ b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestSidecar.java
@@ -632,7 +632,7 @@ public final class ManifestSidecar {
@Nullable PartitionPredicate partitionFilter,
@Nullable RowType partitionType,
@Nullable BiPredicate<Integer, Integer> bucketFilter,
- @Nullable SegmentsCache<Object> cache) {
+ @Nullable SegmentsCache<Path> cache) {
String sidecarFileName = fileName(manifest);
if (sidecarFileName == null) {
return null;
@@ -660,7 +660,7 @@ public final class ManifestSidecar {
}
}
- /** Complete sidecar bytes stored in the shared manifest cache. */
+ /** Complete sidecar bytes stored in the configured sidecar cache. */
static final class ManifestSidecarSegment implements Segments {
private final byte[] bytes;
diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/AbstractFileStoreScan.java b/paimon-core/src/main/java/org/apache/paimon/operation/AbstractFileStoreScan.java
index 82923ef9d8..e40d27e789 100644
--- a/paimon-core/src/main/java/org/apache/paimon/operation/AbstractFileStoreScan.java
+++ b/paimon-core/src/main/java/org/apache/paimon/operation/AbstractFileStoreScan.java
@@ -29,6 +29,7 @@ import org.apache.paimon.manifest.ManifestEntry;
import org.apache.paimon.manifest.ManifestEntrySerializer;
import org.apache.paimon.manifest.ManifestFile;
import org.apache.paimon.manifest.ManifestFileMeta;
+import org.apache.paimon.manifest.ManifestSidecar;
import org.apache.paimon.manifest.PartitionEntry;
import org.apache.paimon.manifest.SimpleFileEntry;
import org.apache.paimon.operation.metrics.ScanMetrics;
@@ -498,27 +499,35 @@ public abstract class AbstractFileStoreScan implements FileStoreScan {
@Nullable Filter<InternalRow> additionalFilter,
@Nullable Filter<ManifestEntry> additionalTFilter) {
+ ManifestFile manifestFile = manifestFileFactory.create();
+ BucketFilter bucketFilter = createBucketFilter();
+ ManifestSidecar.Selection selected =
+ manifestFile.selectBlocks(
+ manifest, rowRangeIndex, manifestsReader.partitionFilter(), bucketFilter);
+ if (selected != null && selected.blocks().isEmpty()) {
+ return Collections.emptyList();
+ }
Filter<InternalRow> entryRowFilter = createEntryRowFilter();
Function<ManifestEntry, T> finalConverter =
dropStats ? e -> converter.apply(dropStats(e)) : converter;
List<T> entries =
- manifestFileFactory
- .create()
+ manifestFile
.withCacheMetrics(
scanMetrics != null ? scanMetrics.getCacheMetrics() : null)
.read(
manifest.fileName(),
manifest.fileSize(),
manifestsReader.partitionFilter(),
- createBucketFilter(),
+ bucketFilter,
entryRowFilter.and(additionalFilter),
entry ->
(additionalTFilter == null || additionalTFilter.test(entry))
&& (manifestEntryFilter == null
|| manifestEntryFilter.test(entry))
&& filterByStats(entry),
- finalConverter);
+ finalConverter,
+ selected);
LOG.info("Read {} manifest entries from {}", entries.size(), manifest.fileName());
return entries;
}
diff --git a/paimon-core/src/main/java/org/apache/paimon/table/AbstractFileStoreTable.java b/paimon-core/src/main/java/org/apache/paimon/table/AbstractFileStoreTable.java
index e9d9603f5a..93ecc173d5 100644
--- a/paimon-core/src/main/java/org/apache/paimon/table/AbstractFileStoreTable.java
+++ b/paimon-core/src/main/java/org/apache/paimon/table/AbstractFileStoreTable.java
@@ -107,6 +107,7 @@ abstract class AbstractFileStoreTable implements FileStoreTable {
@Nullable private Set<String> appliedDynamicOptionKeys;
@Nullable protected transient SegmentsCache<Path> manifestCache;
+ @Nullable protected transient SegmentsCache<Path> manifestSidecarCache;
@Nullable protected transient Cache<Path, Snapshot> snapshotCache;
@Nullable protected transient Cache<String, Statistics> statsCache;
@Nullable protected transient DVMetaCache dvmetaCache;
@@ -144,6 +145,18 @@ abstract class AbstractFileStoreTable implements FileStoreTable {
return manifestCache;
}
+ @Override
+ public void setManifestSidecarCache(SegmentsCache<Path> manifestSidecarCache) {
+ this.manifestSidecarCache = manifestSidecarCache;
+ store().setManifestSidecarCache(manifestSidecarCache);
+ }
+
+ @Nullable
+ @Override
+ public SegmentsCache<Path> getManifestSidecarCache() {
+ return manifestSidecarCache;
+ }
+
@Override
public void setSnapshotCache(Cache<Path, Snapshot> cache) {
this.snapshotCache = cache;
@@ -432,6 +445,9 @@ abstract class AbstractFileStoreTable implements FileStoreTable {
if (manifestCache != null) {
copied.setManifestCache(manifestCache);
}
+ if (manifestSidecarCache != null) {
+ copied.setManifestSidecarCache(manifestSidecarCache);
+ }
if (statsCache != null) {
copied.setStatsCache(statsCache);
}
diff --git a/paimon-core/src/main/java/org/apache/paimon/table/DelegatedFileStoreTable.java b/paimon-core/src/main/java/org/apache/paimon/table/DelegatedFileStoreTable.java
index 9a5e81bcb6..ed541721fc 100644
--- a/paimon-core/src/main/java/org/apache/paimon/table/DelegatedFileStoreTable.java
+++ b/paimon-core/src/main/java/org/apache/paimon/table/DelegatedFileStoreTable.java
@@ -145,6 +145,17 @@ public abstract class DelegatedFileStoreTable implements FileStoreTable {
return wrapped.getManifestCache();
}
+ @Override
+ public void setManifestSidecarCache(SegmentsCache<Path> manifestSidecarCache) {
+ wrapped.setManifestSidecarCache(manifestSidecarCache);
+ }
+
+ @Nullable
+ @Override
+ public SegmentsCache<Path> getManifestSidecarCache() {
+ return wrapped.getManifestSidecarCache();
+ }
+
@Override
public void setSnapshotCache(Cache<Path, Snapshot> cache) {
wrapped.setSnapshotCache(cache);
diff --git a/paimon-core/src/main/java/org/apache/paimon/table/FallbackReadFileStoreTable.java b/paimon-core/src/main/java/org/apache/paimon/table/FallbackReadFileStoreTable.java
index da37e07808..940dd38b2b 100644
--- a/paimon-core/src/main/java/org/apache/paimon/table/FallbackReadFileStoreTable.java
+++ b/paimon-core/src/main/java/org/apache/paimon/table/FallbackReadFileStoreTable.java
@@ -149,6 +149,12 @@ public class FallbackReadFileStoreTable extends DelegatedFileStoreTable {
other.setManifestCache(manifestCache);
}
+ @Override
+ public void setManifestSidecarCache(SegmentsCache<Path> manifestSidecarCache) {
+ super.setManifestSidecarCache(manifestSidecarCache);
+ other.setManifestSidecarCache(manifestSidecarCache);
+ }
+
protected FileStoreTable switchWrappedToBranch(String branchName) {
Optional<TableSchema> optionalSchema =
wrapped.schemaManager().copyWithBranch(branchName).latest();
diff --git a/paimon-core/src/main/java/org/apache/paimon/table/FileStoreTable.java b/paimon-core/src/main/java/org/apache/paimon/table/FileStoreTable.java
index 9704934305..ddaa174076 100644
--- a/paimon-core/src/main/java/org/apache/paimon/table/FileStoreTable.java
+++ b/paimon-core/src/main/java/org/apache/paimon/table/FileStoreTable.java
@@ -70,6 +70,11 @@ public interface FileStoreTable extends DataTable {
@Nullable
SegmentsCache<Path> getManifestCache();
+ void setManifestSidecarCache(SegmentsCache<Path> manifestSidecarCache);
+
+ @Nullable
+ SegmentsCache<Path> getManifestSidecarCache();
+
void setSnapshotCache(Cache<Path, Snapshot> cache);
void setStatsCache(Cache<String, Statistics> cache);
diff --git a/paimon-core/src/main/java/org/apache/paimon/utils/ObjectsCache.java b/paimon-core/src/main/java/org/apache/paimon/utils/ObjectsCache.java
index 4530f698e2..59106c5a2a 100644
--- a/paimon-core/src/main/java/org/apache/paimon/utils/ObjectsCache.java
+++ b/paimon-core/src/main/java/org/apache/paimon/utils/ObjectsCache.java
@@ -63,6 +63,12 @@ public abstract class ObjectsCache<K, V, S extends Segments> {
this.cacheMetrics = cacheMetrics;
}
+ /** Shares the byte cache with consumers using distinct whole-file and block keys. */
+ @SuppressWarnings("unchecked")
+ public SegmentsCache<Object> segmentsCache() {
+ return (SegmentsCache<Object>) (SegmentsCache<?>) cache;
+ }
+
public List<V> read(K key, @Nullable Long fileSize, Filters<V> filters) throws IOException {
return read(key, fileSize, filters, Function.identity());
}
diff --git a/paimon-core/src/test/java/org/apache/paimon/catalog/CachingCatalogTest.java b/paimon-core/src/test/java/org/apache/paimon/catalog/CachingCatalogTest.java
index 3de7bbf73f..71427641b7 100644
--- a/paimon-core/src/test/java/org/apache/paimon/catalog/CachingCatalogTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/catalog/CachingCatalogTest.java
@@ -22,12 +22,17 @@ import org.apache.paimon.CoreOptions;
import org.apache.paimon.Snapshot;
import org.apache.paimon.data.GenericRow;
import org.apache.paimon.fs.Path;
+import org.apache.paimon.manifest.ManifestFileMeta;
+import org.apache.paimon.manifest.ManifestSidecar;
import org.apache.paimon.options.MemorySize;
import org.apache.paimon.options.Options;
import org.apache.paimon.partition.Partition;
+import org.apache.paimon.partition.PartitionPredicate;
import org.apache.paimon.partition.PartitionStatistics;
import org.apache.paimon.schema.Schema;
import org.apache.paimon.schema.SchemaChange;
+import org.apache.paimon.table.FallbackReadFileStoreTable;
+import org.apache.paimon.table.FileStoreTable;
import org.apache.paimon.table.Table;
import org.apache.paimon.table.sink.BatchTableCommit;
import org.apache.paimon.table.sink.BatchTableWrite;
@@ -48,6 +53,8 @@ import org.apache.paimon.shade.guava30.com.google.common.collect.Lists;
import org.assertj.core.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
import org.mockito.Mockito;
import java.io.FileNotFoundException;
@@ -71,6 +78,7 @@ import static java.util.Collections.singletonMap;
import static org.apache.paimon.data.BinaryString.fromString;
import static org.apache.paimon.options.CatalogOptions.CACHE_EXPIRE_AFTER_ACCESS;
import static org.apache.paimon.options.CatalogOptions.CACHE_MANIFEST_MAX_MEMORY;
+import static org.apache.paimon.options.CatalogOptions.CACHE_MANIFEST_SIDECAR_MAX_MEMORY;
import static org.apache.paimon.options.CatalogOptions.CACHE_MANIFEST_SMALL_FILE_MEMORY;
import static org.apache.paimon.options.CatalogOptions.CACHE_MANIFEST_SMALL_FILE_THRESHOLD;
import static org.apache.paimon.options.CatalogOptions.CACHE_MANIFEST_SOFT_VALUES;
@@ -633,4 +641,138 @@ class CachingCatalogTest extends CatalogTestBase {
assertThat(caching.manifestCache.softValues()).isFalse();
assertThat(caching.manifestCache.ttl()).isEqualTo(CACHE_EXPIRE_AFTER_ACCESS.defaultValue());
}
+
+ @Test
+ public void testManifestSidecarCacheOptions() {
+ Options options = new Options();
+ CachingCatalog caching = new CachingCatalog(catalog, options);
+ assertThat(CACHE_MANIFEST_SIDECAR_MAX_MEMORY.defaultValue())
+ .isEqualTo(MemorySize.ofMebiBytes(64));
+ assertThat(caching.manifestSidecarCache).isNotSameAs(caching.manifestCache);
+ assertThat(caching.manifestSidecarCache.maxMemorySize())
+ .isEqualTo(MemorySize.ofMebiBytes(64));
+ assertThat(caching.manifestSidecarCache.maxElementSize())
+ .isEqualTo(MemorySize.ofMebiBytes(64).getBytes());
+
+ options.set(CACHE_MANIFEST_SIDECAR_MAX_MEMORY, MemorySize.ofMebiBytes(8));
+ caching = new CachingCatalog(catalog, options);
+ assertThat(caching.manifestSidecarCache).isNotSameAs(caching.manifestCache);
+ assertThat(caching.manifestSidecarCache.maxMemorySize())
+ .isEqualTo(MemorySize.ofMebiBytes(8));
+ assertThat(caching.manifestSidecarCache.maxElementSize())
+ .isEqualTo(MemorySize.ofMebiBytes(8).getBytes());
+ assertThat(caching.manifestSidecarCache.ttl())
+ .isEqualTo(CACHE_EXPIRE_AFTER_ACCESS.defaultValue());
+ assertThat(caching.manifestSidecarCache.softValues()).isTrue();
+
+ options.set(CACHE_MANIFEST_SMALL_FILE_MEMORY, MemorySize.ofBytes(0));
+ options.set(CACHE_EXPIRE_AFTER_ACCESS, Duration.ofMinutes(2));
+ options.set(CACHE_MANIFEST_SOFT_VALUES, false);
+ caching = new CachingCatalog(catalog, options);
+ assertThat(caching.manifestCache).isNull();
+ assertThat(caching.manifestSidecarCache.maxMemorySize())
+ .isEqualTo(MemorySize.ofMebiBytes(8));
+ assertThat(caching.manifestSidecarCache.ttl()).isEqualTo(Duration.ofMinutes(2));
+ assertThat(caching.manifestSidecarCache.softValues()).isFalse();
+
+ options.set(CACHE_MANIFEST_SIDECAR_MAX_MEMORY, MemorySize.ofBytes(0));
+ options.set(CACHE_MANIFEST_SMALL_FILE_MEMORY, MemorySize.ofMebiBytes(1));
+ caching = new CachingCatalog(catalog, options);
+ assertThat(caching.manifestCache).isNotNull();
+ assertThat(caching.manifestSidecarCache).isSameAs(caching.manifestCache);
+
+ options.set(CACHE_MANIFEST_SMALL_FILE_MEMORY, MemorySize.ofBytes(0));
+ caching = new CachingCatalog(catalog, options);
+ assertThat(caching.manifestCache).isNull();
+ assertThat(caching.manifestSidecarCache).isNull();
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {false, true})
+ public void testManifestSidecarCacheSharedAcrossTableCopies(boolean separate) throws Exception {
+ Options options = new Options();
+ options.set(CACHE_MANIFEST_SOFT_VALUES, false);
+ options.set(
+ CACHE_MANIFEST_SIDECAR_MAX_MEMORY,
+ separate ? MemorySize.ofMebiBytes(1) : MemorySize.ofBytes(0));
+ CachingCatalog caching = new CachingCatalog(catalog, options);
+ Identifier id = Identifier.create("db", "sidecar_cache");
+ caching.createTable(id, DEFAULT_TABLE_SCHEMA, false);
+ caching.alterTable(
+ id,
+ SchemaChange.setOption(CoreOptions.MANIFEST_SIDECAR_ENABLED.key(), "true"),
+ false);
+ FileStoreTable table = (FileStoreTable) caching.getTable(id);
+ assertThat(table.getManifestSidecarCache()).isSameAs(caching.manifestSidecarCache);
+ writeTableForTestManifestCache(table);
+ ManifestFileMeta meta =
+ table.store()
+ .manifestListFactory()
+ .create()
+ .readDataManifests(table.latestSnapshot().get())
+ .get(0);
+ PartitionPredicate partition = PartitionPredicate.ALWAYS_TRUE;
+ assertThat(
+ table.store()
+ .manifestFileFactory()
+ .create()
+ .selectBlocks(meta, null, partition, null))
+ .isNotNull();
+ Path sidecar = new Path(table.location(), "manifest/" + ManifestSidecar.fileName(meta));
+ assertThat(caching.manifestSidecarCache.getIfPresents(sidecar)).isNotNull();
+ if (separate) {
+ assertThat(caching.manifestCache.getIfPresents(sidecar)).isNull();
+ } else {
+ assertThat(caching.manifestCache.getIfPresents(sidecar)).isNotNull();
+ }
+ assertThat(caching.estimatedCacheSizes().manifestCacheSize())
+ .isEqualTo(
+ caching.manifestCache.estimatedSize()
+ + (separate ? caching.manifestSidecarCache.estimatedSize() : 0));
+ assertThat(caching.estimatedCacheSizes().manifestCacheBytes())
+ .isEqualTo(
+ caching.manifestCache.totalCacheBytes()
+ + (separate ? caching.manifestSidecarCache.totalCacheBytes() : 0));
+
+ // A copied table and a new ManifestFile must reuse cached bytes even after the file is
+ // gone.
+ fileIO.delete(sidecar, false);
+ for (FileStoreTable copy :
+ new FileStoreTable[] {
+ table.copy(singletonMap("a", "b")),
+ table.copy(table.schema()),
+ (FileStoreTable) caching.getTable(id)
+ }) {
+ assertThat(copy.getManifestSidecarCache()).isSameAs(caching.manifestSidecarCache);
+ assertThat(
+ copy.store()
+ .manifestFileFactory()
+ .create()
+ .selectBlocks(meta, null, partition, null))
+ .isNotNull();
+ }
+ }
+
+ @Test
+ public void testManifestSidecarCachePropagatesToFallbackTable() throws Exception {
+ CachingCatalog caching = new CachingCatalog(catalog, new Options());
+ Identifier id = Identifier.create("db", "sidecar_fallback");
+ caching.createTable(id, DEFAULT_TABLE_SCHEMA, false);
+ caching.getTable(id).createBranch("fallback");
+ caching.alterTable(
+ id,
+ SchemaChange.setOption(CoreOptions.SCAN_FALLBACK_BRANCH.key(), "fallback"),
+ false);
+ FallbackReadFileStoreTable table = (FallbackReadFileStoreTable) caching.getTable(id);
+ for (FallbackReadFileStoreTable copy :
+ new FallbackReadFileStoreTable[] {
+ table, (FallbackReadFileStoreTable) table.copy(singletonMap("a", "b"))
+ }) {
+ assertThat(copy.getManifestSidecarCache()).isSameAs(caching.manifestSidecarCache);
+ assertThat(copy.wrapped().getManifestSidecarCache())
+ .isSameAs(caching.manifestSidecarCache);
+ assertThat(copy.other().getManifestSidecarCache())
+ .isSameAs(caching.manifestSidecarCache);
+ }
+ }
}
diff --git a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaTest.java b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaTest.java
index 16f33f4cee..161591a48e 100644
--- a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaTest.java
@@ -2745,6 +2745,7 @@ public class ManifestFileMetaTest extends ManifestFileMetaTestBase {
null),
Long.MAX_VALUE,
null,
+ null,
new CoreOptions(new Options()))
.create();
@@ -3206,6 +3207,7 @@ public class ManifestFileMetaTest extends ManifestFileMetaTestBase {
null),
Long.MAX_VALUE,
null,
+ null,
new CoreOptions(new Options()))
.create();
}
diff --git a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaTestBase.java b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaTestBase.java
index a4516015c2..4e72f58e73 100644
--- a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaTestBase.java
+++ b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaTestBase.java
@@ -167,6 +167,7 @@ public abstract class ManifestFileMetaTestBase {
null),
Long.MAX_VALUE,
null,
+ null,
new CoreOptions(new Options()))
.create();
}
diff --git a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileTest.java b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileTest.java
index 0362871ef8..db914239ee 100644
--- a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileTest.java
@@ -19,6 +19,7 @@
package org.apache.paimon.manifest;
import org.apache.paimon.CoreOptions;
+import org.apache.paimon.data.BinaryRow;
import org.apache.paimon.data.GenericRow;
import org.apache.paimon.data.InternalRow;
import org.apache.paimon.format.FileFormat;
@@ -28,13 +29,18 @@ import org.apache.paimon.fs.FileIO;
import org.apache.paimon.fs.FileIOFinder;
import org.apache.paimon.fs.Path;
import org.apache.paimon.fs.PositionOutputStream;
+import org.apache.paimon.fs.SeekableInputStream;
+import org.apache.paimon.fs.SeekableInputStreamWrapper;
import org.apache.paimon.fs.local.LocalFileIO;
import org.apache.paimon.io.DataFileMeta;
import org.apache.paimon.io.DataFileMetaWriteColsLegacySerializer;
+import org.apache.paimon.operation.AppendOnlyFileStoreScan;
+import org.apache.paimon.operation.ManifestsReader;
import org.apache.paimon.options.MemorySize;
import org.apache.paimon.options.Options;
import org.apache.paimon.partition.PartitionPredicate;
import org.apache.paimon.schema.FileSystemSchemaManager;
+import org.apache.paimon.schema.TableSchema;
import org.apache.paimon.stats.StatsTestUtils;
import org.apache.paimon.types.DataField;
import org.apache.paimon.types.DataType;
@@ -44,6 +50,8 @@ import org.apache.paimon.utils.CloseableIterator;
import org.apache.paimon.utils.FailingFileIO;
import org.apache.paimon.utils.FileStorePathFactory;
import org.apache.paimon.utils.Filter;
+import org.apache.paimon.utils.Range;
+import org.apache.paimon.utils.RowRangeIndex;
import org.apache.paimon.utils.SegmentsCache;
import org.junit.jupiter.api.RepeatedTest;
@@ -68,13 +76,18 @@ import java.util.List;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.atomic.AtomicLong;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import static org.apache.paimon.TestKeyValueGenerator.DEFAULT_PART_TYPE;
+import static org.apache.paimon.manifest.ManifestIndexTestUtils.withExtraFiles;
import static org.apache.paimon.stats.StatsTestUtils.convertWithoutSchemaEvolution;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verifyNoInteractions;
+import static org.mockito.Mockito.when;
/** Tests for {@link ManifestFile}. */
public class ManifestFileTest {
@@ -1405,16 +1418,646 @@ public class ManifestFileTest {
return -1;
}
+ @Test
+ void testSidecarCacheUsesExplicitPathsAndIsSeparateFromManifestCache() throws Exception {
+ Options options = new Options();
+ options.set(CoreOptions.DATA_EVOLUTION_ENABLED, true);
+ options.set(CoreOptions.MANIFEST_SIDECAR_ENABLED, true);
+ RecordingFileIO io = new RecordingFileIO();
+ SegmentsCache<Path> cache =
+ new SegmentsCache<>(1024, MemorySize.ofMebiBytes(1), Long.MAX_VALUE, null, false);
+ SegmentsCache<Path> sidecarCache =
+ new SegmentsCache<>(1024, MemorySize.ofMebiBytes(1), Long.MAX_VALUE, null, false);
+ ManifestFile.Factory factory =
+ createManifestFileFactory(
+ tempDir.toString(), Long.MAX_VALUE, options, io, cache, sidecarCache);
+ List<ManifestEntry> entries = new ArrayList<>();
+ for (int i = 0; i < 2; i++) {
+ ManifestEntry entry = gen.next();
+ entries.add(
+ ManifestEntry.create(
+ FileKind.ADD,
+ entry.partition(),
+ entry.bucket(),
+ entry.totalBuckets(),
+ entry.file().newFirstRowId(i * 1000000L)));
+ }
+ ManifestFileMeta written = factory.create().write(entries).get(0);
+ String sidecarName = "cached-explicit" + ManifestSidecar.SUFFIX;
+ java.nio.file.Path sidecar = tempDir.resolve("manifest").resolve(sidecarName);
+ java.nio.file.Files.move(
+ tempDir.resolve("manifest").resolve(ManifestSidecar.fileName(written)), sidecar);
+ ManifestFileMeta meta = withExtraFiles(written, Collections.singletonList(sidecarName));
+ Path sidecarPath = new Path(tempDir.toString(), "manifest/" + sidecarName);
+
+ io.reset();
+ assertThat(
+ factory.create()
+ .selectBlocks(
+ meta,
+ RowRangeIndex.create(
+ Collections.singletonList(
+ new Range(Long.MAX_VALUE, Long.MAX_VALUE))))
+ .blocks())
+ .isEmpty();
+ assertThat(io.opened).containsExactly(sidecarPath);
+ assertThat(sidecarCache.getIfPresents(sidecarPath).totalMemorySize())
+ .isEqualTo(java.nio.file.Files.size(sidecar));
+ assertThat(cache.estimatedSize()).isZero();
+
+ io.reset();
+ RowRangeIndex hit = RowRangeIndex.create(Collections.singletonList(new Range(0, 0)));
+ assertThat(factory.create().selectBlocks(meta, hit).blocks()).isNotEmpty();
+ assertThat(io.opened).isEmpty();
+ assertThat(factory.create().read(meta.fileName()))
+ .containsExactlyInAnyOrderElementsOf(entries);
+ assertThat(cache.estimatedSize()).isEqualTo(1);
+ assertThat(sidecarCache.estimatedSize()).isEqualTo(1);
+ assertThat(cache.getIfPresents(sidecarPath)).isNull();
+ assertThat(
+ sidecarCache.getIfPresents(
+ new Path(tempDir.toString(), "manifest/" + meta.fileName())))
+ .isNull();
+
+ io.reset();
+ assertThat(factory.create().selectBlocks(meta, hit).blocks()).isNotEmpty();
+ assertThat(factory.create().read(meta.fileName()))
+ .containsExactlyInAnyOrderElementsOf(entries);
+ assertThat(io.opened).isEmpty();
+ }
+
+ @Test
+ void testDedicatedSidecarCacheAndManifestCacheFallback() {
+ Options options = new Options();
+ options.set(CoreOptions.DATA_EVOLUTION_ENABLED, true);
+ options.set(CoreOptions.MANIFEST_SIDECAR_ENABLED, true);
+ for (boolean cacheManifest : new boolean[] {false, true}) {
+ for (boolean cacheSidecar : new boolean[] {false, true}) {
+ RecordingFileIO io = new RecordingFileIO();
+ SegmentsCache<Path> manifestCache =
+ cacheManifest
+ ? new SegmentsCache<>(
+ 1024,
+ MemorySize.ofMebiBytes(1),
+ Long.MAX_VALUE,
+ null,
+ false)
+ : null;
+ SegmentsCache<Path> sidecarCache =
+ cacheSidecar
+ ? new SegmentsCache<>(
+ 1024,
+ MemorySize.ofMebiBytes(1),
+ Long.MAX_VALUE,
+ null,
+ false)
+ : null;
+ ManifestFile.Factory factory =
+ createManifestFileFactory(
+ tempDir.resolve(cacheManifest + "-" + cacheSidecar).toString(),
+ Long.MAX_VALUE,
+ options,
+ io,
+ manifestCache,
+ sidecarCache);
+ ManifestEntry entry = gen.next();
+ ManifestEntry added =
+ ManifestEntry.create(
+ FileKind.ADD,
+ entry.partition(),
+ entry.bucket(),
+ entry.totalBuckets(),
+ entry.file().newFirstRowId(0L));
+ ManifestFileMeta meta =
+ factory.create().write(Collections.singletonList(added)).get(0);
+ RowRangeIndex query =
+ RowRangeIndex.create(Collections.singletonList(new Range(0, 0)));
+
+ for (int round = 0; round < 2; round++) {
+ io.reset();
+ ManifestFile manifest = factory.create();
+ ManifestSidecar.Selection selected = manifest.selectBlocks(meta, query);
+ assertThat(readSelectedEntries(manifest, meta, selected))
+ .containsExactly(added);
+ assertThat(
+ io.opened.stream()
+ .filter(
+ path ->
+ path.getName()
+ .endsWith(
+ ManifestSidecar
+ .SUFFIX)))
+ .hasSize(round == 0 || (!cacheSidecar && !cacheManifest) ? 1 : 0);
+ assertThat(
+ io.opened.stream()
+ .filter(path -> path.getName().equals(meta.fileName())))
+ .hasSize(round == 0 || !cacheManifest ? 1 : 0);
+ }
+ if (manifestCache != null) {
+ // Blocks stay in the manifest cache; sidecar bytes only join them on fallback.
+ assertThat(manifestCache.estimatedSize()).isEqualTo(cacheSidecar ? 1 : 2);
+ }
+ if (sidecarCache != null) {
+ assertThat(sidecarCache.estimatedSize()).isEqualTo(1);
+ }
+ }
+ }
+ }
+
+ @Test
+ void testReadsOnlySelectedBlocksAndPreservesPhysicalOrdinals() throws Exception {
+ Options options = new Options();
+ options.set(CoreOptions.DATA_EVOLUTION_ENABLED, true);
+ options.set(CoreOptions.MANIFEST_SIDECAR_ENABLED, true);
+ RecordingFileIO fileIO = new RecordingFileIO();
+ SegmentsCache<Path> cache =
+ new SegmentsCache<>(1024, MemorySize.ofMebiBytes(16), Long.MAX_VALUE);
+ ManifestFile.Factory factory =
+ createManifestFileFactory(
+ tempDir.toString(), Long.MAX_VALUE, options, fileIO, cache);
+ ManifestFile manifests = factory.create();
+ List<ManifestEntry> entries = new ArrayList<>();
+ for (int i = 0; i < 4000; i++) {
+ ManifestEntry entry = gen.next();
+ entries.add(
+ ManifestEntry.create(
+ FileKind.ADD,
+ entry.partition(),
+ entry.bucket(),
+ entry.totalBuckets(),
+ entry.file().newFirstRowId(i * 1000000L)));
+ }
+ ManifestFileMeta meta = manifests.write(entries).get(0);
+ Path manifestPath = new Path(tempDir.toString(), "manifest/" + meta.fileName());
+ RowRangeIndex query =
+ RowRangeIndex.create(
+ Arrays.asList(
+ new Range(1000000000L, 1000000000L),
+ new Range(3000000000L, 3000000000L)));
+
+ ManifestSidecar.Selection selected = manifests.selectBlocks(meta, query);
+ assertThat(selected.blocks()).hasSize(2);
+
+ fileIO.reset();
+ List<ManifestEntry> actual = readSelectedEntries(factory.create(), meta, selected);
+ List<ManifestEntry> expected = new ArrayList<>();
+ for (ManifestSidecar.Block block : selected.blocks()) {
+ expected.addAll(
+ entries.subList(
+ (int) block.firstRecord,
+ (int) (block.firstRecord + block.recordCount)));
+ }
+ assertThat(actual).containsExactlyElementsOf(expected);
+ assertThat(actual).contains(entries.get(1000), entries.get(3000));
+ assertThat(fileIO.bytes.get()).isLessThan(meta.fileSize() / 4);
+ assertThat(fileIO.seeks)
+ .containsExactlyElementsOf(
+ selected.blocks().stream()
+ .map(block -> block.offset)
+ .collect(Collectors.toList()));
+ assertThat(fileIO.opened).containsExactly(manifestPath);
+ assertThat(cache.getIfPresents(manifestPath)).isNull();
+ fileIO.reset();
+ assertThat(readSelectedEntries(factory.create(), meta, selected))
+ .containsExactlyElementsOf(expected);
+ assertThat(fileIO.opened).isEmpty();
+ assertThat(fileIO.bytes.get()).isZero();
+ ManifestSidecar.Selection allBlocks =
+ manifests.selectBlocks(
+ meta,
+ RowRangeIndex.create(
+ Collections.singletonList(new Range(0, Long.MAX_VALUE))));
+ fileIO.reset();
+ assertThat(readSelectedEntries(factory.create(), meta, allBlocks))
+ .containsExactlyInAnyOrderElementsOf(entries);
+ assertThat(fileIO.opened).containsExactly(manifestPath);
+ assertThat(cache.getIfPresents(manifestPath)).isNull();
+
+ fileIO.reset();
+ assertThat(readSelectedEntries(factory.create(), meta, allBlocks))
+ .containsExactlyInAnyOrderElementsOf(entries);
+ assertThat(fileIO.opened).isEmpty();
+ assertThat(cache.getIfPresents(manifestPath)).isNull();
+
+ // Reads without a sidecar selection populate and reuse the full-manifest cache.
+ assertThat(manifests.read(meta.fileName())).containsExactlyInAnyOrderElementsOf(entries);
+ assertThat(fileIO.opened).containsExactly(manifestPath);
+ assertThat(cache.getIfPresents(manifestPath)).isNotNull();
+
+ fileIO.reset();
+ assertThat(manifests.read(meta.fileName())).containsExactlyInAnyOrderElementsOf(entries);
+ assertThat(fileIO.opened).isEmpty();
+
+ long largestBlock =
+ allBlocks.blocks().stream().mapToLong(block -> block.length).max().getAsLong();
+ assertThat(meta.fileSize()).isGreaterThan(largestBlock);
+ SegmentsCache<Path> blockCache =
+ new SegmentsCache<>(1024, MemorySize.ofMebiBytes(16), largestBlock, null, false);
+ ManifestFile.Factory limitedFactory =
+ createManifestFileFactory(
+ tempDir.toString(), Long.MAX_VALUE, options, fileIO, blockCache);
+ for (int round = 0; round < 2; round++) {
+ fileIO.reset();
+ assertThat(readSelectedEntries(limitedFactory.create(), meta, allBlocks))
+ .containsExactlyElementsOf(entries);
+ if (round == 1) {
+ assertThat(fileIO.opened).isEmpty();
+ }
+ }
+ assertThat(
+ blockCache.getIfPresents(
+ new Path(tempDir.toString(), "manifest/" + meta.fileName())))
+ .isNull();
+ }
+
+ @Test
+ void testScannerPreservesDeletesAndColumnGroups() throws Exception {
+ Options options = new Options();
+ options.set(CoreOptions.DATA_EVOLUTION_ENABLED, true);
+ options.set(CoreOptions.MANIFEST_SIDECAR_ENABLED, true);
+ RecordingFileIO fileIO = new RecordingFileIO();
+ ManifestFile.Factory factory =
+ createManifestFileFactory(tempDir.toString(), Long.MAX_VALUE, options, fileIO);
+ ManifestFile manifests = factory.create();
+ ManifestEntry entry = gen.next();
+ ManifestEntry add =
+ ManifestEntry.create(
+ FileKind.ADD,
+ entry.partition(),
+ entry.bucket(),
+ entry.totalBuckets(),
+ entry.file().newFirstRowId(100L));
+ ManifestEntry delete =
+ ManifestEntry.create(
+ FileKind.DELETE,
+ entry.partition(),
+ entry.bucket(),
+ entry.totalBuckets(),
+ add.file());
+ ManifestEntry other = gen.next();
+ ManifestEntry live =
+ ManifestEntry.create(
+ FileKind.ADD,
+ other.partition(),
+ other.bucket(),
+ other.totalBuckets(),
+ other.file().newFirstRowId(100L));
+ List<ManifestFileMeta> metas = new ArrayList<>();
+ metas.addAll(manifests.write(Arrays.asList(add, live)));
+ metas.addAll(manifests.write(Collections.singletonList(delete)));
+ metas.addAll(
+ manifests.write(
+ Collections.singletonList(
+ ManifestEntry.create(
+ FileKind.ADD,
+ entry.partition(),
+ entry.bucket(),
+ entry.totalBuckets(),
+ entry.file().newFirstRowId(0L)))));
+ AppendOnlyFileStoreScan scan =
+ new AppendOnlyFileStoreScan(
+ mock(ManifestsReader.class),
+ null,
+ null,
+ null,
+ mock(TableSchema.class),
+ factory,
+ 2,
+ false,
+ false,
+ false);
+ scan.withRowRanges(Collections.singletonList(new Range(100, 100)));
+ fileIO.reset();
+ List<ManifestEntry> result = new ArrayList<>();
+ scan.readManifestEntries(metas, false).forEachRemaining(result::add);
+ assertThat(result).containsExactly(live);
+ assertThat(
+ fileIO.opened.stream()
+ .filter(path -> !path.getName().endsWith(ManifestSidecar.SUFFIX))
+ .map(Path::getName))
+ .containsExactlyInAnyOrder(metas.get(0).fileName(), metas.get(1).fileName());
+
+ for (boolean missing : new boolean[] {false, true}) {
+ for (ManifestFileMeta manifest : metas) {
+ Path sidecar =
+ new Path(
+ tempDir.toString(),
+ "manifest/" + ManifestSidecar.fileName(manifest));
+ if (missing) {
+ fileIO.delete(sidecar, false);
+ } else {
+ fileIO.overwriteFileUtf8(sidecar, "corrupt sidecar");
+ }
+ }
+ List<ManifestEntry> fallback = new ArrayList<>();
+ scan.readManifestEntries(metas, false).forEachRemaining(fallback::add);
+ assertThat(fallback).containsExactly(live);
+ }
+ }
+
+ @Test
+ void testBucketOnlyPlanningAndRawRewriteUseNullableBucketPayload() throws Exception {
+ Options options = new Options();
+ options.set(CoreOptions.BUCKET, 4);
+ options.set(CoreOptions.MANIFEST_SIDECAR_ENABLED, true);
+ RecordingFileIO io = new RecordingFileIO();
+ ManifestFile.Factory factory =
+ createManifestFileFactory(tempDir.toString(), Long.MAX_VALUE, options, io);
+ ManifestFile manifests = factory.create();
+ List<ManifestEntry> entries = new ArrayList<>();
+ for (int i = 0; i < 4000; i++) {
+ ManifestEntry entry = gen.next();
+ entries.add(
+ ManifestEntry.create(
+ FileKind.ADD, entry.partition(), i / 1000, 4, entry.file()));
+ }
+ ManifestFileMeta meta = manifests.write(entries).get(0);
+ AppendOnlyFileStoreScan scan =
+ new AppendOnlyFileStoreScan(
+ mock(ManifestsReader.class),
+ null,
+ null,
+ null,
+ mock(TableSchema.class),
+ factory,
+ 2,
+ false,
+ false,
+ false);
+ scan.withBucket(1);
+ io.reset();
+ assertThat(scan.readManifest(meta)).containsExactlyElementsOf(entries.subList(1000, 2000));
+ assertThat(io.bytes.get()).isLessThan(meta.fileSize());
+ assertThat(io.seeks).isNotEmpty();
+ ManifestAvroWriter writer = manifests.createAvroWriter();
+ try (ManifestAvroReader reader =
+ manifests.scanAvroBlocks(meta.fileName(), meta.fileSize())) {
+ writer.writeEncodedManifest(reader, meta);
+ }
+ writer.close();
+ assertThat(scan.readManifest(writer.result().get(0)))
+ .containsExactlyElementsOf(entries.subList(1000, 2000));
+
+ ManifestEntry added = entries.get(1000);
+ ManifestEntry deleted =
+ ManifestEntry.create(FileKind.DELETE, added.partition(), 1, 4, added.file());
+ List<ManifestFileMeta> changes = new ArrayList<>();
+ changes.addAll(manifests.write(Collections.singletonList(added)));
+ changes.addAll(manifests.write(Collections.singletonList(deleted)));
+ assertThat(scan.readManifestEntries(changes, false)).isExhausted();
+ }
+
+ @Test
+ void testPartitionOnlyPlanningUsesBlocksWithoutRowIds() {
+ Options options = new Options();
+ options.set(CoreOptions.MANIFEST_SIDECAR_ENABLED, true);
+ RecordingFileIO io = new RecordingFileIO();
+ ManifestFile.Factory factory =
+ createManifestFileFactory(tempDir.toString(), Long.MAX_VALUE, options, io);
+ BinaryRow first = gen.next().partition();
+ BinaryRow second = gen.next().partition();
+ while (second.equals(first)) {
+ second = gen.next().partition();
+ }
+ List<ManifestEntry> entries = new ArrayList<>();
+ for (int i = 0; i < 4000; i++) {
+ ManifestEntry entry = gen.next();
+ entries.add(
+ ManifestEntry.create(
+ FileKind.ADD,
+ i < 1000 ? first : second,
+ entry.bucket(),
+ entry.totalBuckets(),
+ entry.file()));
+ }
+ ManifestFileMeta meta = factory.create().write(entries).get(0);
+ ManifestsReader lists = mock(ManifestsReader.class);
+ when(lists.partitionFilter())
+ .thenReturn(
+ PartitionPredicate.fromMultiple(
+ DEFAULT_PART_TYPE, Collections.singletonList(first)));
+ AppendOnlyFileStoreScan scan =
+ new AppendOnlyFileStoreScan(
+ lists,
+ null,
+ null,
+ null,
+ mock(TableSchema.class),
+ factory,
+ 2,
+ false,
+ false,
+ false);
+ io.reset();
+ List<ManifestEntry> actual = scan.readManifest(meta);
+ assertThat(actual).containsExactlyElementsOf(entries.subList(0, 1000));
+ assertThat(io.bytes.get()).isLessThan(meta.fileSize());
+ assertThat(io.opened).hasSize(2);
+ }
+
+ @Test
+ void testUnknownRowIdKeepsPartitionIndexAndNoQueryDoesNotReadSidecar() {
+ Options options = new Options();
+ options.set(CoreOptions.DATA_EVOLUTION_ENABLED, true);
+ options.set(CoreOptions.MANIFEST_SIDECAR_ENABLED, true);
+ RecordingFileIO fileIO = new RecordingFileIO();
+ ManifestFile manifests =
+ createManifestFileFactory(tempDir.toString(), Long.MAX_VALUE, options, fileIO)
+ .create();
+ ManifestFileMeta meta = manifests.write(Collections.singletonList(gen.next())).get(0);
+ assertThat(ManifestSidecar.fileName(meta)).isNotNull();
+ assertThat(
+ java.nio.file.Files.exists(
+ tempDir.resolve("manifest")
+ .resolve(meta.fileName() + ManifestSidecar.SUFFIX)))
+ .isTrue();
+
+ fileIO.reset();
+ assertThat(manifests.selectBlocks(meta, null)).isNull();
+ assertThat(fileIO.opened).isEmpty();
+ }
+
+ @Test
+ void testExplicitIndexReferenceAndNullDoesNotProbe() throws Exception {
+ Options options = new Options();
+ options.set(CoreOptions.DATA_EVOLUTION_ENABLED, true);
+ options.set(CoreOptions.MANIFEST_SIDECAR_ENABLED, true);
+ RecordingFileIO io = new RecordingFileIO();
+ ManifestFile manifests =
+ createManifestFileFactory(tempDir.toString(), Long.MAX_VALUE, options, io).create();
+ ManifestEntry original = gen.next();
+ ManifestEntry entry =
+ ManifestEntry.create(
+ FileKind.ADD,
+ original.partition(),
+ original.bucket(),
+ original.totalBuckets(),
+ original.file().newFirstRowId(100L));
+ ManifestFileMeta written = manifests.write(Collections.singletonList(entry)).get(0);
+ assertThat(ManifestSidecar.fileName(written)).isNotNull();
+ RowRangeIndex query = RowRangeIndex.create(Collections.singletonList(new Range(0, 0)));
+ io.reset();
+ for (List<String> extraFiles :
+ Arrays.asList(
+ null,
+ Collections.<String>emptyList(),
+ Collections.singletonList("other-partition-index"))) {
+ ManifestFileMeta unindexed = withExtraFiles(written, extraFiles);
+ assertThat(manifests.selectBlocks(unindexed, query)).isNull();
+ assertThat(io.opened).isEmpty();
+ }
+ // An existing suffix-named object must not be inferred as a reference.
+ assertThat(
+ java.nio.file.Files.exists(
+ tempDir.resolve("manifest")
+ .resolve(ManifestSidecar.fileName(written))))
+ .isTrue();
+ String explicitName = "custom-index-name" + ManifestSidecar.SUFFIX;
+ java.nio.file.Files.move(
+ tempDir.resolve("manifest").resolve(ManifestSidecar.fileName(written)),
+ tempDir.resolve("manifest").resolve(explicitName));
+ String otherName = "other-partition-index";
+ java.nio.file.Path otherPath = tempDir.resolve("manifest").resolve(otherName);
+ java.nio.file.Files.write(otherPath, new byte[] {1, 2, 3});
+ ManifestFileMeta indexed = withExtraFiles(written, Arrays.asList(otherName, explicitName));
+ assertThat(manifests.selectBlocks(indexed, query).blocks()).isEmpty();
+ assertThat(io.opened)
+ .containsExactly(new Path(tempDir.toString(), "manifest/" + explicitName));
+ manifests.delete(indexed);
+ assertThat(java.nio.file.Files.exists(otherPath)).isFalse();
+ assertThat(java.nio.file.Files.exists(tempDir.resolve("manifest").resolve(explicitName)))
+ .isFalse();
+ assertThat(
+ java.nio.file.Files.exists(
+ tempDir.resolve("manifest").resolve(written.fileName())))
+ .isFalse();
+ }
+
+ @Test
+ void testDisabledOrUnfilteredReadsSkipSidecarMetadata() {
+ for (boolean enabled : new boolean[] {false, true}) {
+ Options options = new Options();
+ options.set(CoreOptions.MANIFEST_SORT_ENABLED, true);
+ options.set(CoreOptions.MANIFEST_SIDECAR_ENABLED, enabled);
+ RecordingFileIO io = new RecordingFileIO();
+ ManifestFile manifests =
+ createManifestFileFactory(tempDir.toString(), Long.MAX_VALUE, options, io)
+ .create();
+ ManifestFileMeta meta = mock(ManifestFileMeta.class);
+ RowRangeIndex rows =
+ enabled
+ ? null
+ : RowRangeIndex.create(Collections.singletonList(new Range(1, 1)));
+ assertThat(manifests.selectBlocks(meta, rows)).isNull();
+ verifyNoInteractions(meta);
+ assertThat(io.opened).isEmpty();
+ }
+ }
+
+ private List<ManifestEntry> readSelectedEntries(
+ ManifestFile manifests, ManifestFileMeta meta, ManifestSidecar.Selection selected) {
+ return manifests.read(
+ meta.fileName(),
+ meta.fileSize(),
+ null,
+ null,
+ row -> true,
+ entry -> true,
+ java.util.function.Function.identity(),
+ selected);
+ }
+
+ /** Observes actual file access without adding counters to production readers. */
+ private static final class RecordingFileIO extends LocalFileIO {
+
+ private final List<Path> opened = Collections.synchronizedList(new ArrayList<>());
+ private final List<Long> seeks = Collections.synchronizedList(new ArrayList<>());
+ private final AtomicLong bytes = new AtomicLong();
+
+ private void reset() {
+ opened.clear();
+ seeks.clear();
+ bytes.set(0);
+ }
+
+ @Override
+ public SeekableInputStream newInputStream(Path path) throws IOException {
+ opened.add(path);
+ return new SeekableInputStreamWrapper(super.newInputStream(path)) {
+ @Override
+ public void seek(long desired) throws IOException {
+ seeks.add(desired);
+ super.seek(desired);
+ }
+
+ @Override
+ public int read() throws IOException {
+ int value = super.read();
+ if (value >= 0) {
+ bytes.incrementAndGet();
+ }
+ return value;
+ }
+
+ @Override
+ public int read(byte[] buffer, int offset, int length) throws IOException {
+ int n = super.read(buffer, offset, length);
+ if (n > 0) {
+ bytes.addAndGet(n);
+ }
+ return n;
+ }
+ };
+ }
+ }
+
private ManifestFile createManifestFile(String pathStr) {
return createManifestFile(pathStr, ThreadLocalRandom.current().nextInt(8192) + 1024);
}
private ManifestFile createManifestFile(String pathStr, long suggestedFileSize) {
- return createManifestFile(pathStr, suggestedFileSize, null);
+ return createManifestFile(pathStr, suggestedFileSize, new Options());
}
private ManifestFile createManifestFile(
String pathStr, long suggestedFileSize, @Nullable SegmentsCache<Path> cache) {
+ return createManifestFileFactory(
+ pathStr,
+ suggestedFileSize,
+ new Options(),
+ FileIOFinder.find(new Path(pathStr)),
+ cache)
+ .create();
+ }
+
+ private ManifestFile createManifestFile(
+ String pathStr, long suggestedFileSize, Options options) {
+ return createManifestFileFactory(
+ pathStr, suggestedFileSize, options, FileIOFinder.find(new Path(pathStr)))
+ .create();
+ }
+
+ private ManifestFile.Factory createManifestFileFactory(
+ String pathStr, long suggestedFileSize, Options options, FileIO fileIO) {
+ return createManifestFileFactory(pathStr, suggestedFileSize, options, fileIO, null);
+ }
+
+ private ManifestFile.Factory createManifestFileFactory(
+ String pathStr,
+ long suggestedFileSize,
+ Options options,
+ FileIO fileIO,
+ @Nullable SegmentsCache<Path> cache) {
+ return createManifestFileFactory(pathStr, suggestedFileSize, options, fileIO, cache, null);
+ }
+
+ private ManifestFile.Factory createManifestFileFactory(
+ String pathStr,
+ long suggestedFileSize,
+ Options options,
+ FileIO fileIO,
+ @Nullable SegmentsCache<Path> cache,
+ @Nullable SegmentsCache<Path> sidecarCache) {
Path path = new Path(pathStr);
FileStorePathFactory pathFactory =
new FileStorePathFactory(
@@ -1433,18 +2076,18 @@ public class ManifestFileTest {
null,
false,
null);
- FileIO fileIO = FileIOFinder.find(path);
+ CoreOptions coreOptions = new CoreOptions(options);
return new ManifestFile.Factory(
- fileIO,
- new FileSystemSchemaManager(fileIO, path),
- DEFAULT_PART_TYPE,
- avro,
- "zstd",
- pathFactory,
- suggestedFileSize,
- cache,
- new CoreOptions(new Options()))
- .create();
+ fileIO,
+ new FileSystemSchemaManager(fileIO, path),
+ DEFAULT_PART_TYPE,
+ avro,
+ "zstd",
+ pathFactory,
+ suggestedFileSize,
+ cache,
+ sidecarCache,
+ coreOptions);
}
@Test
diff --git a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestSidecarTest.java b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestSidecarTest.java
index 1110c70158..1b3f9d258e 100644
--- a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestSidecarTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestSidecarTest.java
@@ -458,14 +458,14 @@ class ManifestSidecarTest {
Files.write(temp.resolve(sidecar.getName()), data);
ManifestFileMeta meta = testMeta();
FileIO io = spy(LocalFileIO.create());
- SegmentsCache<Object> tooSmall =
+ SegmentsCache<Path> tooSmall =
new SegmentsCache<>(1024, MemorySize.ofMebiBytes(1), data.length - 1L, null, false);
assertThat(readCached(io, path, meta, tooSmall).blocks()).hasSize(2);
assertThat(readCached(io, path, meta, tooSmall).blocks()).hasSize(2);
assertThat(tooSmall.getIfPresents(sidecar)).isNull();
verify(io, times(2)).newInputStream(sidecar);
- SegmentsCache<Object> cache =
+ SegmentsCache<Path> cache =
new SegmentsCache<>(1024, MemorySize.ofMebiBytes(1), data.length, null, false);
assertThat(readCached(io, path, meta, cache).blocks()).hasSize(2);
assertThat(readCached(io, path, meta, cache).blocks()).hasSize(2);
@@ -478,7 +478,7 @@ class ManifestSidecarTest {
Path path = new Path(temp.toString(), "manifest-golden");
Path sidecar = ManifestSidecar.path(path);
Files.write(temp.resolve(sidecar.getName()), data);
- SegmentsCache<Object> cache =
+ SegmentsCache<Path> cache =
new SegmentsCache<>(1024, MemorySize.ofMebiBytes(1), Long.MAX_VALUE, null, false);
cache.put(sidecar, new SingleSegments(MemorySegment.wrap(data), data.length));
FileIO io = spy(LocalFileIO.create());
@@ -499,7 +499,7 @@ class ManifestSidecarTest {
Path sidecar = ManifestSidecar.path(path);
ManifestFileMeta meta = testMeta();
FileIO io = spy(LocalFileIO.create());
- SegmentsCache<Object> cache =
+ SegmentsCache<Path> cache =
new SegmentsCache<>(1024, MemorySize.ofMebiBytes(1), Long.MAX_VALUE, null, false);
assertThat(readCached(io, path, meta, cache)).isNull();
assertThat(cache.getIfPresents(sidecar)).isNull();
@@ -521,7 +521,7 @@ class ManifestSidecarTest {
Files.write(temp.resolve(sidecar.getName()), testSidecar());
ManifestFileMeta meta = testMeta();
FileIO io = spy(LocalFileIO.create());
- SegmentsCache<Object> cache =
+ SegmentsCache<Path> cache =
new SegmentsCache<>(1024, MemorySize.ofMebiBytes(1), Long.MAX_VALUE, null, false);
RowRangeIndex cancelled = mock(RowRangeIndex.class);
when(cancelled.intersects(anyLong(), anyLong()))
@@ -544,7 +544,7 @@ class ManifestSidecarTest {
}
private ManifestSidecar.Selection readCached(
- FileIO io, Path path, ManifestFileMeta meta, SegmentsCache<Object> cache) {
+ FileIO io, Path path, ManifestFileMeta meta, SegmentsCache<Path> cache) {
return ManifestSidecar.read(
io,
path,
diff --git a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestSidecarWriteTest.java b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestSidecarWriteTest.java
index cea952943f..3549bf6dcd 100644
--- a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestSidecarWriteTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestSidecarWriteTest.java
@@ -341,6 +341,7 @@ class ManifestSidecarWriteTest {
paths,
targetSize,
null,
+ null,
new CoreOptions(options))
.create();
}
diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/ManifestRewriteCleanupTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/ManifestRewriteCleanupTest.java
index 20a1b26bee..dd1a1b176b 100644
--- a/paimon-core/src/test/java/org/apache/paimon/operation/ManifestRewriteCleanupTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/operation/ManifestRewriteCleanupTest.java
@@ -669,6 +669,7 @@ class ManifestRewriteCleanupTest extends ManifestFileMetaTestBase {
null),
suggestedFileSize,
null,
+ null,
new CoreOptions(new Options()))
.create();
}
(paimon) 14/20: [spark][core] Support more scalar transforms in V2 filter pushdown (#9905)
Posted by ho...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
XiaoHongbo-Hope pushed a commit to branch release-2.1
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit a5fb087bdeb500e68513e5a79f2a7fd0614fb2f4
Author: Jingsong Lee <ji...@gmail.com>
AuthorDate: Thu Sep 17 11:23:09 2026 +0800
[spark][core] Support more scalar transforms in V2 filter pushdown (#9905)
---
.../paimon/predicate/BitLengthTransform.java | 104 +++++++++++++
.../apache/paimon/predicate/DateAddTransform.java | 105 +++++++++++++
.../apache/paimon/predicate/DateDiffTransform.java | 105 +++++++++++++
.../paimon/predicate/DateExtractTransform.java | 4 +-
.../paimon/predicate/DateTruncTransform.java | 135 ++++++++++++++++
.../paimon/predicate/DayOfWeekTransform.java | 58 +++++++
.../paimon/predicate/DayOfYearTransform.java | 58 +++++++
.../paimon/predicate/IsoDayOfWeekTransform.java | 58 +++++++
.../apache/paimon/predicate/OverlayTransform.java | 124 +++++++++++++++
.../org/apache/paimon/predicate/PadTransform.java | 145 +++++++++++++++++
.../apache/paimon/predicate/QuarterTransform.java | 57 +++++++
.../org/apache/paimon/predicate/Transform.java | 14 ++
.../paimon/predicate/TransformInputUtils.java | 77 ++++++++++
.../paimon/predicate/TranslateTransform.java | 95 ++++++++++++
.../org/apache/paimon/predicate/WeekTransform.java | 58 +++++++
.../apache/paimon/predicate/WeekdayTransform.java | 57 +++++++
.../paimon/predicate/YearOfWeekTransform.java | 59 +++++++
.../paimon/predicate/DateExtractTransformTest.java | 17 ++
.../paimon/predicate/ScalarTransformTest.java | 171 +++++++++++++++++++++
.../paimon/predicate/TransformJsonSerdeTest.java | 78 ++++++++++
.../spark/util/SparkExpressionConverter.scala | 78 ++++++++++
.../spark/sql/SparkV2FilterConverterTestBase.scala | 133 +++++++++++++++-
22 files changed, 1788 insertions(+), 2 deletions(-)
diff --git a/paimon-common/src/main/java/org/apache/paimon/predicate/BitLengthTransform.java b/paimon-common/src/main/java/org/apache/paimon/predicate/BitLengthTransform.java
new file mode 100644
index 0000000000..08baec3242
--- /dev/null
+++ b/paimon-common/src/main/java/org/apache/paimon/predicate/BitLengthTransform.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.predicate;
+
+import org.apache.paimon.data.BinaryString;
+import org.apache.paimon.data.InternalRow;
+import org.apache.paimon.types.DataType;
+import org.apache.paimon.types.DataTypes;
+
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator;
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter;
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnore;
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty;
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+
+import java.util.List;
+import java.util.Objects;
+
+import static org.apache.paimon.utils.Preconditions.checkArgument;
+
+/** Transform that returns the UTF-8 bit length of a string. */
+public class BitLengthTransform implements Transform {
+
+ private static final long serialVersionUID = 1L;
+
+ public static final String NAME = "BIT_LENGTH";
+
+ private final List<Object> inputs;
+
+ @JsonCreator
+ public BitLengthTransform(
+ @JsonProperty(StringTransform.FIELD_INPUTS)
+ @JsonDeserialize(contentUsing = StringTransform.InputDeserializer.class)
+ List<Object> inputs) {
+ checkArgument(inputs.size() == 1, "BIT_LENGTH requires exactly one input");
+ TransformInputUtils.checkString(inputs.get(0), "BIT_LENGTH input must be a string");
+ this.inputs = inputs;
+ }
+
+ @Override
+ public String name() {
+ return NAME;
+ }
+
+ @Override
+ @JsonIgnore
+ public List<Object> inputs() {
+ return inputs;
+ }
+
+ @JsonGetter(StringTransform.FIELD_INPUTS)
+ public List<Object> inputsForJson() {
+ return StringTransform.inputsForJson(inputs);
+ }
+
+ @Override
+ public DataType outputType() {
+ return DataTypes.INT();
+ }
+
+ @Override
+ public Object transform(InternalRow row) {
+ BinaryString value = TransformInputUtils.string(inputs.get(0), row);
+ return value == null ? null : value.getSizeInBytes() * 8;
+ }
+
+ @Override
+ public Transform copyWithNewInputs(List<Object> inputs) {
+ return new BitLengthTransform(inputs);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ return o != null
+ && getClass() == o.getClass()
+ && Objects.equals(inputs, ((BitLengthTransform) o).inputs);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(inputs);
+ }
+
+ @Override
+ public String toString() {
+ return StringTransform.formatCall(name(), inputs);
+ }
+}
diff --git a/paimon-common/src/main/java/org/apache/paimon/predicate/DateAddTransform.java b/paimon-common/src/main/java/org/apache/paimon/predicate/DateAddTransform.java
new file mode 100644
index 0000000000..c663e04a6d
--- /dev/null
+++ b/paimon-common/src/main/java/org/apache/paimon/predicate/DateAddTransform.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.predicate;
+
+import org.apache.paimon.data.InternalRow;
+import org.apache.paimon.types.DataType;
+import org.apache.paimon.types.DataTypes;
+
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator;
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter;
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnore;
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty;
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+
+import java.util.List;
+import java.util.Objects;
+
+import static org.apache.paimon.utils.Preconditions.checkArgument;
+
+/** SQL {@code DATE_ADD(date, days)} transform. */
+public class DateAddTransform implements Transform {
+
+ private static final long serialVersionUID = 1L;
+
+ public static final String NAME = "DATE_ADD";
+
+ private final List<Object> inputs;
+
+ @JsonCreator
+ public DateAddTransform(
+ @JsonProperty(StringTransform.FIELD_INPUTS)
+ @JsonDeserialize(contentUsing = SubstringTransform.InputDeserializer.class)
+ List<Object> inputs) {
+ checkArgument(inputs.size() == 2, "DATE_ADD requires exactly two inputs");
+ TransformInputUtils.checkDate(inputs.get(0), "DATE_ADD first input must be a date");
+ TransformInputUtils.checkInteger(inputs.get(1), "DATE_ADD second input must be an integer");
+ this.inputs = inputs;
+ }
+
+ @Override
+ public String name() {
+ return NAME;
+ }
+
+ @Override
+ @JsonIgnore
+ public List<Object> inputs() {
+ return inputs;
+ }
+
+ @JsonGetter(StringTransform.FIELD_INPUTS)
+ public List<Object> inputsForJson() {
+ return StringTransform.inputsForJson(inputs);
+ }
+
+ @Override
+ public DataType outputType() {
+ return DataTypes.DATE();
+ }
+
+ @Override
+ public Object transform(InternalRow row) {
+ Integer date = TransformInputUtils.integer(inputs.get(0), row);
+ Integer days = TransformInputUtils.integer(inputs.get(1), row);
+ return date == null || days == null ? null : date + days;
+ }
+
+ @Override
+ public Transform copyWithNewInputs(List<Object> inputs) {
+ return new DateAddTransform(inputs);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ return o != null
+ && getClass() == o.getClass()
+ && Objects.equals(inputs, ((DateAddTransform) o).inputs);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(inputs);
+ }
+
+ @Override
+ public String toString() {
+ return StringTransform.formatCall(name(), inputs);
+ }
+}
diff --git a/paimon-common/src/main/java/org/apache/paimon/predicate/DateDiffTransform.java b/paimon-common/src/main/java/org/apache/paimon/predicate/DateDiffTransform.java
new file mode 100644
index 0000000000..268822fe1e
--- /dev/null
+++ b/paimon-common/src/main/java/org/apache/paimon/predicate/DateDiffTransform.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.predicate;
+
+import org.apache.paimon.data.InternalRow;
+import org.apache.paimon.types.DataType;
+import org.apache.paimon.types.DataTypes;
+
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator;
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter;
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnore;
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty;
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+
+import java.util.List;
+import java.util.Objects;
+
+import static org.apache.paimon.utils.Preconditions.checkArgument;
+
+/** SQL {@code DATE_DIFF(end_date, start_date)} transform. */
+public class DateDiffTransform implements Transform {
+
+ private static final long serialVersionUID = 1L;
+
+ public static final String NAME = "DATE_DIFF";
+
+ private final List<Object> inputs;
+
+ @JsonCreator
+ public DateDiffTransform(
+ @JsonProperty(StringTransform.FIELD_INPUTS)
+ @JsonDeserialize(contentUsing = SubstringTransform.InputDeserializer.class)
+ List<Object> inputs) {
+ checkArgument(inputs.size() == 2, "DATE_DIFF requires exactly two inputs");
+ TransformInputUtils.checkDate(inputs.get(0), "DATE_DIFF first input must be a date");
+ TransformInputUtils.checkDate(inputs.get(1), "DATE_DIFF second input must be a date");
+ this.inputs = inputs;
+ }
+
+ @Override
+ public String name() {
+ return NAME;
+ }
+
+ @Override
+ @JsonIgnore
+ public List<Object> inputs() {
+ return inputs;
+ }
+
+ @JsonGetter(StringTransform.FIELD_INPUTS)
+ public List<Object> inputsForJson() {
+ return StringTransform.inputsForJson(inputs);
+ }
+
+ @Override
+ public DataType outputType() {
+ return DataTypes.INT();
+ }
+
+ @Override
+ public Object transform(InternalRow row) {
+ Integer end = TransformInputUtils.integer(inputs.get(0), row);
+ Integer start = TransformInputUtils.integer(inputs.get(1), row);
+ return end == null || start == null ? null : end - start;
+ }
+
+ @Override
+ public Transform copyWithNewInputs(List<Object> inputs) {
+ return new DateDiffTransform(inputs);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ return o != null
+ && getClass() == o.getClass()
+ && Objects.equals(inputs, ((DateDiffTransform) o).inputs);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(inputs);
+ }
+
+ @Override
+ public String toString() {
+ return StringTransform.formatCall(name(), inputs);
+ }
+}
diff --git a/paimon-common/src/main/java/org/apache/paimon/predicate/DateExtractTransform.java b/paimon-common/src/main/java/org/apache/paimon/predicate/DateExtractTransform.java
index 70dfa9e040..5da9fddcdd 100644
--- a/paimon-common/src/main/java/org/apache/paimon/predicate/DateExtractTransform.java
+++ b/paimon-common/src/main/java/org/apache/paimon/predicate/DateExtractTransform.java
@@ -41,7 +41,9 @@ import static org.apache.paimon.utils.Preconditions.checkNotNull;
* Base {@link Transform} that extracts a calendar field from a {@code DATE} or {@code TIMESTAMP}
* field, like SQL {@code EXTRACT(YEAR FROM d)} and the {@code year}, {@code month}, ... functions.
* See the subclasses {@link YearTransform}, {@link MonthTransform}, {@link DayTransform}, {@link
- * HourTransform}, {@link MinuteTransform} and {@link SecondTransform}.
+ * HourTransform}, {@link MinuteTransform}, {@link SecondTransform}, {@link QuarterTransform},
+ * {@link IsoDayOfWeekTransform}, {@link DayOfWeekTransform}, {@link WeekdayTransform}, {@link
+ * DayOfYearTransform}, {@link WeekTransform} and {@link YearOfWeekTransform}.
*
* <p>{@code TIMESTAMP WITH LOCAL TIME ZONE} is deliberately not supported: extracting a calendar
* field from it depends on a session time zone, which the reader does not know, so such a predicate
diff --git a/paimon-common/src/main/java/org/apache/paimon/predicate/DateTruncTransform.java b/paimon-common/src/main/java/org/apache/paimon/predicate/DateTruncTransform.java
new file mode 100644
index 0000000000..cb0125c5c9
--- /dev/null
+++ b/paimon-common/src/main/java/org/apache/paimon/predicate/DateTruncTransform.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.predicate;
+
+import org.apache.paimon.data.BinaryString;
+import org.apache.paimon.data.InternalRow;
+import org.apache.paimon.types.DataType;
+import org.apache.paimon.types.DataTypes;
+
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator;
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter;
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnore;
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty;
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+
+import java.time.DayOfWeek;
+import java.time.LocalDate;
+import java.time.temporal.TemporalAdjusters;
+import java.util.List;
+import java.util.Locale;
+import java.util.Objects;
+
+import static org.apache.paimon.utils.Preconditions.checkArgument;
+
+/** SQL {@code TRUNC(date, format)} transform. */
+public class DateTruncTransform implements Transform {
+
+ private static final long serialVersionUID = 1L;
+
+ public static final String NAME = "DATE_TRUNC";
+
+ private final List<Object> inputs;
+
+ @JsonCreator
+ public DateTruncTransform(
+ @JsonProperty(StringTransform.FIELD_INPUTS)
+ @JsonDeserialize(contentUsing = SubstringTransform.InputDeserializer.class)
+ List<Object> inputs) {
+ checkArgument(inputs.size() == 2, "DATE_TRUNC requires exactly two inputs");
+ TransformInputUtils.checkDate(inputs.get(0), "DATE_TRUNC first input must be a date");
+ TransformInputUtils.checkString(inputs.get(1), "DATE_TRUNC format must be a string");
+ this.inputs = inputs;
+ }
+
+ @Override
+ public String name() {
+ return NAME;
+ }
+
+ @Override
+ @JsonIgnore
+ public List<Object> inputs() {
+ return inputs;
+ }
+
+ @JsonGetter(StringTransform.FIELD_INPUTS)
+ public List<Object> inputsForJson() {
+ return StringTransform.inputsForJson(inputs);
+ }
+
+ @Override
+ public DataType outputType() {
+ return DataTypes.DATE();
+ }
+
+ @Override
+ public Object transform(InternalRow row) {
+ Integer epochDay = TransformInputUtils.integer(inputs.get(0), row);
+ BinaryString format = TransformInputUtils.string(inputs.get(1), row);
+ if (epochDay == null || format == null) {
+ return null;
+ }
+
+ LocalDate date = LocalDate.ofEpochDay(epochDay);
+ switch (format.toString().toUpperCase(Locale.ROOT)) {
+ case "WEEK":
+ date = date.with(TemporalAdjusters.previousOrSame(DayOfWeek.MONDAY));
+ break;
+ case "MON":
+ case "MONTH":
+ case "MM":
+ date = date.withDayOfMonth(1);
+ break;
+ case "QUARTER":
+ date = date.withMonth(((date.getMonthValue() - 1) / 3) * 3 + 1).withDayOfMonth(1);
+ break;
+ case "YEAR":
+ case "YYYY":
+ case "YY":
+ date = date.withDayOfYear(1);
+ break;
+ default:
+ return null;
+ }
+ return Math.toIntExact(date.toEpochDay());
+ }
+
+ @Override
+ public Transform copyWithNewInputs(List<Object> inputs) {
+ return new DateTruncTransform(inputs);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ return o != null
+ && getClass() == o.getClass()
+ && Objects.equals(inputs, ((DateTruncTransform) o).inputs);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(inputs);
+ }
+
+ @Override
+ public String toString() {
+ return StringTransform.formatCall(name(), inputs);
+ }
+}
diff --git a/paimon-common/src/main/java/org/apache/paimon/predicate/DayOfWeekTransform.java b/paimon-common/src/main/java/org/apache/paimon/predicate/DayOfWeekTransform.java
new file mode 100644
index 0000000000..41d4f2032c
--- /dev/null
+++ b/paimon-common/src/main/java/org/apache/paimon/predicate/DayOfWeekTransform.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.predicate;
+
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator;
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty;
+
+import java.time.LocalDateTime;
+import java.util.Optional;
+
+/** Extracts the day of week, Sunday = 1 through Saturday = 7. */
+public class DayOfWeekTransform extends DateExtractTransform {
+
+ private static final long serialVersionUID = 1L;
+
+ public static final String NAME = "DAY_OF_WEEK";
+
+ @JsonCreator
+ public DayOfWeekTransform(
+ @JsonProperty(DateExtractTransform.FIELD_FIELD_REF) FieldRef fieldRef) {
+ super(fieldRef);
+ }
+
+ public static Optional<Transform> tryCreate(FieldRef fieldRef) {
+ return DateExtractTransform.tryCreate(fieldRef, DayOfWeekTransform::new);
+ }
+
+ @Override
+ public String name() {
+ return NAME;
+ }
+
+ @Override
+ protected Integer extract(LocalDateTime dateTime) {
+ return dateTime.getDayOfWeek().getValue() % 7 + 1;
+ }
+
+ @Override
+ protected Transform copy(FieldRef fieldRef) {
+ return new DayOfWeekTransform(fieldRef);
+ }
+}
diff --git a/paimon-common/src/main/java/org/apache/paimon/predicate/DayOfYearTransform.java b/paimon-common/src/main/java/org/apache/paimon/predicate/DayOfYearTransform.java
new file mode 100644
index 0000000000..846915ada5
--- /dev/null
+++ b/paimon-common/src/main/java/org/apache/paimon/predicate/DayOfYearTransform.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.predicate;
+
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator;
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty;
+
+import java.time.LocalDateTime;
+import java.util.Optional;
+
+/** Extracts the day of year, like SQL {@code EXTRACT(DAY_OF_YEAR FROM d)}. */
+public class DayOfYearTransform extends DateExtractTransform {
+
+ private static final long serialVersionUID = 1L;
+
+ public static final String NAME = "DAY_OF_YEAR";
+
+ @JsonCreator
+ public DayOfYearTransform(
+ @JsonProperty(DateExtractTransform.FIELD_FIELD_REF) FieldRef fieldRef) {
+ super(fieldRef);
+ }
+
+ public static Optional<Transform> tryCreate(FieldRef fieldRef) {
+ return DateExtractTransform.tryCreate(fieldRef, DayOfYearTransform::new);
+ }
+
+ @Override
+ public String name() {
+ return NAME;
+ }
+
+ @Override
+ protected Integer extract(LocalDateTime dateTime) {
+ return dateTime.getDayOfYear();
+ }
+
+ @Override
+ protected Transform copy(FieldRef fieldRef) {
+ return new DayOfYearTransform(fieldRef);
+ }
+}
diff --git a/paimon-common/src/main/java/org/apache/paimon/predicate/IsoDayOfWeekTransform.java b/paimon-common/src/main/java/org/apache/paimon/predicate/IsoDayOfWeekTransform.java
new file mode 100644
index 0000000000..afb53f242c
--- /dev/null
+++ b/paimon-common/src/main/java/org/apache/paimon/predicate/IsoDayOfWeekTransform.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.predicate;
+
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator;
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty;
+
+import java.time.LocalDateTime;
+import java.util.Optional;
+
+/** Extracts the ISO day of week, Monday = 1 through Sunday = 7. */
+public class IsoDayOfWeekTransform extends DateExtractTransform {
+
+ private static final long serialVersionUID = 1L;
+
+ public static final String NAME = "ISO_DAY_OF_WEEK";
+
+ @JsonCreator
+ public IsoDayOfWeekTransform(
+ @JsonProperty(DateExtractTransform.FIELD_FIELD_REF) FieldRef fieldRef) {
+ super(fieldRef);
+ }
+
+ public static Optional<Transform> tryCreate(FieldRef fieldRef) {
+ return DateExtractTransform.tryCreate(fieldRef, IsoDayOfWeekTransform::new);
+ }
+
+ @Override
+ public String name() {
+ return NAME;
+ }
+
+ @Override
+ protected Integer extract(LocalDateTime dateTime) {
+ return dateTime.getDayOfWeek().getValue();
+ }
+
+ @Override
+ protected Transform copy(FieldRef fieldRef) {
+ return new IsoDayOfWeekTransform(fieldRef);
+ }
+}
diff --git a/paimon-common/src/main/java/org/apache/paimon/predicate/OverlayTransform.java b/paimon-common/src/main/java/org/apache/paimon/predicate/OverlayTransform.java
new file mode 100644
index 0000000000..1122f1da20
--- /dev/null
+++ b/paimon-common/src/main/java/org/apache/paimon/predicate/OverlayTransform.java
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.predicate;
+
+import org.apache.paimon.data.BinaryString;
+import org.apache.paimon.data.InternalRow;
+import org.apache.paimon.types.DataType;
+import org.apache.paimon.types.DataTypes;
+
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator;
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter;
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnore;
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty;
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+
+import java.util.List;
+import java.util.Objects;
+
+import static org.apache.paimon.utils.Preconditions.checkArgument;
+
+/** SQL {@code OVERLAY(input PLACING replacement FROM position [FOR length])} transform. */
+public class OverlayTransform implements Transform {
+
+ private static final long serialVersionUID = 1L;
+
+ public static final String NAME = "OVERLAY";
+
+ private final List<Object> inputs;
+
+ @JsonCreator
+ public OverlayTransform(
+ @JsonProperty(StringTransform.FIELD_INPUTS)
+ @JsonDeserialize(contentUsing = SubstringTransform.InputDeserializer.class)
+ List<Object> inputs) {
+ checkArgument(
+ inputs.size() == 3 || inputs.size() == 4, "OVERLAY requires three or four inputs");
+ TransformInputUtils.checkString(inputs.get(0), "OVERLAY input must be a string");
+ TransformInputUtils.checkString(inputs.get(1), "OVERLAY replacement must be a string");
+ TransformInputUtils.checkInteger(inputs.get(2), "OVERLAY position must be an integer");
+ if (inputs.size() == 4) {
+ TransformInputUtils.checkInteger(inputs.get(3), "OVERLAY length must be an integer");
+ }
+ this.inputs = inputs;
+ }
+
+ @Override
+ public String name() {
+ return NAME;
+ }
+
+ @Override
+ @JsonIgnore
+ public List<Object> inputs() {
+ return inputs;
+ }
+
+ @JsonGetter(StringTransform.FIELD_INPUTS)
+ public List<Object> inputsForJson() {
+ return StringTransform.inputsForJson(inputs);
+ }
+
+ @Override
+ public DataType outputType() {
+ return DataTypes.STRING();
+ }
+
+ @Override
+ public Object transform(InternalRow row) {
+ BinaryString source = TransformInputUtils.string(inputs.get(0), row);
+ BinaryString replacement = TransformInputUtils.string(inputs.get(1), row);
+ Integer position = TransformInputUtils.integer(inputs.get(2), row);
+ Integer length =
+ inputs.size() == 4 ? TransformInputUtils.integer(inputs.get(3), row) : null;
+ if (source == null || replacement == null || position == null) {
+ return null;
+ }
+ if (inputs.size() == 4 && length == null) {
+ return null;
+ }
+ int replacedLength = length == null || length < 0 ? replacement.numChars() : length;
+ return BinaryString.concat(
+ source.substringSQL(1, position - 1),
+ replacement,
+ source.substringSQL(position + replacedLength, Integer.MAX_VALUE));
+ }
+
+ @Override
+ public Transform copyWithNewInputs(List<Object> inputs) {
+ return new OverlayTransform(inputs);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ return o != null
+ && getClass() == o.getClass()
+ && Objects.equals(inputs, ((OverlayTransform) o).inputs);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(inputs);
+ }
+
+ @Override
+ public String toString() {
+ return StringTransform.formatCall(name(), inputs);
+ }
+}
diff --git a/paimon-common/src/main/java/org/apache/paimon/predicate/PadTransform.java b/paimon-common/src/main/java/org/apache/paimon/predicate/PadTransform.java
new file mode 100644
index 0000000000..6b7478c4a3
--- /dev/null
+++ b/paimon-common/src/main/java/org/apache/paimon/predicate/PadTransform.java
@@ -0,0 +1,145 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.predicate;
+
+import org.apache.paimon.data.BinaryString;
+import org.apache.paimon.data.InternalRow;
+import org.apache.paimon.types.DataType;
+import org.apache.paimon.types.DataTypes;
+
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator;
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter;
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnore;
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty;
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+
+import java.util.List;
+import java.util.Objects;
+
+import static org.apache.paimon.utils.Preconditions.checkArgument;
+import static org.apache.paimon.utils.Preconditions.checkNotNull;
+
+/** SQL {@code LPAD} and {@code RPAD} transform. */
+public class PadTransform implements Transform {
+
+ private static final long serialVersionUID = 1L;
+
+ public static final String NAME = "PAD";
+ public static final String FIELD_DIRECTION = "direction";
+
+ private final List<Object> inputs;
+ private final Direction direction;
+
+ @JsonCreator
+ public PadTransform(
+ @JsonProperty(StringTransform.FIELD_INPUTS)
+ @JsonDeserialize(contentUsing = SubstringTransform.InputDeserializer.class)
+ List<Object> inputs,
+ @JsonProperty(FIELD_DIRECTION) Direction direction) {
+ checkArgument(inputs.size() == 3, "PAD requires exactly three inputs");
+ TransformInputUtils.checkString(inputs.get(0), "PAD input must be a string");
+ TransformInputUtils.checkInteger(inputs.get(1), "PAD length must be an integer");
+ TransformInputUtils.checkString(inputs.get(2), "PAD string must be a string");
+ this.inputs = inputs;
+ this.direction = checkNotNull(direction, "direction must not be null");
+ }
+
+ @Override
+ public String name() {
+ return NAME;
+ }
+
+ @Override
+ @JsonIgnore
+ public List<Object> inputs() {
+ return inputs;
+ }
+
+ @JsonGetter(StringTransform.FIELD_INPUTS)
+ public List<Object> inputsForJson() {
+ return StringTransform.inputsForJson(inputs);
+ }
+
+ @JsonGetter(FIELD_DIRECTION)
+ public Direction direction() {
+ return direction;
+ }
+
+ @Override
+ public DataType outputType() {
+ return DataTypes.STRING();
+ }
+
+ @Override
+ public Object transform(InternalRow row) {
+ BinaryString source = TransformInputUtils.string(inputs.get(0), row);
+ Integer length = TransformInputUtils.integer(inputs.get(1), row);
+ BinaryString pad = TransformInputUtils.string(inputs.get(2), row);
+ if (source == null || length == null || pad == null) {
+ return null;
+ }
+
+ int needed = length - source.numChars();
+ if (needed <= 0 || pad.getSizeInBytes() == 0) {
+ return source.substring(0, length);
+ }
+
+ int padChars = pad.numChars();
+ StringBuilder padding = new StringBuilder();
+ for (int i = 0; i < needed / padChars; i++) {
+ padding.append(pad);
+ }
+ padding.append(pad.substring(0, needed % padChars));
+ BinaryString padValue = BinaryString.fromString(padding.toString());
+ return direction == Direction.LEFT
+ ? BinaryString.concat(padValue, source)
+ : BinaryString.concat(source, padValue);
+ }
+
+ @Override
+ public Transform copyWithNewInputs(List<Object> inputs) {
+ return new PadTransform(inputs, direction);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ PadTransform that = (PadTransform) o;
+ return Objects.equals(inputs, that.inputs) && direction == that.direction;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(inputs, direction);
+ }
+
+ @Override
+ public String toString() {
+ return (direction == Direction.LEFT ? "LPAD" : "RPAD")
+ + StringTransform.formatCall("", inputs);
+ }
+
+ /** Pad side. */
+ public enum Direction {
+ LEFT,
+ RIGHT
+ }
+}
diff --git a/paimon-common/src/main/java/org/apache/paimon/predicate/QuarterTransform.java b/paimon-common/src/main/java/org/apache/paimon/predicate/QuarterTransform.java
new file mode 100644
index 0000000000..199f552cde
--- /dev/null
+++ b/paimon-common/src/main/java/org/apache/paimon/predicate/QuarterTransform.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.predicate;
+
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator;
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty;
+
+import java.time.LocalDateTime;
+import java.util.Optional;
+
+/** Extracts the quarter of year, like SQL {@code EXTRACT(QUARTER FROM d)}. */
+public class QuarterTransform extends DateExtractTransform {
+
+ private static final long serialVersionUID = 1L;
+
+ public static final String NAME = "QUARTER";
+
+ @JsonCreator
+ public QuarterTransform(@JsonProperty(DateExtractTransform.FIELD_FIELD_REF) FieldRef fieldRef) {
+ super(fieldRef);
+ }
+
+ public static Optional<Transform> tryCreate(FieldRef fieldRef) {
+ return DateExtractTransform.tryCreate(fieldRef, QuarterTransform::new);
+ }
+
+ @Override
+ public String name() {
+ return NAME;
+ }
+
+ @Override
+ protected Integer extract(LocalDateTime dateTime) {
+ return (dateTime.getMonthValue() - 1) / 3 + 1;
+ }
+
+ @Override
+ protected Transform copy(FieldRef fieldRef) {
+ return new QuarterTransform(fieldRef);
+ }
+}
diff --git a/paimon-common/src/main/java/org/apache/paimon/predicate/Transform.java b/paimon-common/src/main/java/org/apache/paimon/predicate/Transform.java
index dffa541397..f0aefe1b2d 100644
--- a/paimon-common/src/main/java/org/apache/paimon/predicate/Transform.java
+++ b/paimon-common/src/main/java/org/apache/paimon/predicate/Transform.java
@@ -43,11 +43,25 @@ import java.util.List;
@JsonSubTypes.Type(value = HourTransform.class, name = HourTransform.NAME),
@JsonSubTypes.Type(value = MinuteTransform.class, name = MinuteTransform.NAME),
@JsonSubTypes.Type(value = SecondTransform.class, name = SecondTransform.NAME),
+ @JsonSubTypes.Type(value = QuarterTransform.class, name = QuarterTransform.NAME),
+ @JsonSubTypes.Type(value = IsoDayOfWeekTransform.class, name = IsoDayOfWeekTransform.NAME),
+ @JsonSubTypes.Type(value = DayOfWeekTransform.class, name = DayOfWeekTransform.NAME),
+ @JsonSubTypes.Type(value = WeekdayTransform.class, name = WeekdayTransform.NAME),
+ @JsonSubTypes.Type(value = DayOfYearTransform.class, name = DayOfYearTransform.NAME),
+ @JsonSubTypes.Type(value = WeekTransform.class, name = WeekTransform.NAME),
+ @JsonSubTypes.Type(value = YearOfWeekTransform.class, name = YearOfWeekTransform.NAME),
@JsonSubTypes.Type(value = UpperTransform.class, name = UpperTransform.NAME),
@JsonSubTypes.Type(value = LowerTransform.class, name = LowerTransform.NAME),
@JsonSubTypes.Type(value = SubstringTransform.class, name = SubstringTransform.NAME),
@JsonSubTypes.Type(value = TrimTransform.class, name = TrimTransform.NAME),
@JsonSubTypes.Type(value = LengthTransform.class, name = LengthTransform.NAME),
+ @JsonSubTypes.Type(value = BitLengthTransform.class, name = BitLengthTransform.NAME),
+ @JsonSubTypes.Type(value = TranslateTransform.class, name = TranslateTransform.NAME),
+ @JsonSubTypes.Type(value = OverlayTransform.class, name = OverlayTransform.NAME),
+ @JsonSubTypes.Type(value = PadTransform.class, name = PadTransform.NAME),
+ @JsonSubTypes.Type(value = DateAddTransform.class, name = DateAddTransform.NAME),
+ @JsonSubTypes.Type(value = DateDiffTransform.class, name = DateDiffTransform.NAME),
+ @JsonSubTypes.Type(value = DateTruncTransform.class, name = DateTruncTransform.NAME),
@JsonSubTypes.Type(value = NullTransform.class, name = NullTransform.NAME)
})
public interface Transform extends Serializable {
diff --git a/paimon-common/src/main/java/org/apache/paimon/predicate/TransformInputUtils.java b/paimon-common/src/main/java/org/apache/paimon/predicate/TransformInputUtils.java
new file mode 100644
index 0000000000..11673a7c77
--- /dev/null
+++ b/paimon-common/src/main/java/org/apache/paimon/predicate/TransformInputUtils.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.predicate;
+
+import org.apache.paimon.data.BinaryString;
+import org.apache.paimon.data.InternalRow;
+
+import static org.apache.paimon.types.DataTypeFamily.CHARACTER_STRING;
+import static org.apache.paimon.types.DataTypeFamily.INTEGER_NUMERIC;
+import static org.apache.paimon.utils.InternalRowUtils.get;
+import static org.apache.paimon.utils.Preconditions.checkArgument;
+
+/** Utilities shared by transforms with mixed field-reference and literal inputs. */
+final class TransformInputUtils {
+
+ private TransformInputUtils() {}
+
+ static Object value(Object input, InternalRow row) {
+ if (input instanceof FieldRef) {
+ FieldRef ref = (FieldRef) input;
+ return get(row, ref.index(), ref.type());
+ }
+ return input;
+ }
+
+ static BinaryString string(Object input, InternalRow row) {
+ return (BinaryString) value(input, row);
+ }
+
+ static Integer integer(Object input, InternalRow row) {
+ Object value = value(input, row);
+ return value == null ? null : Math.toIntExact(((Number) value).longValue());
+ }
+
+ static void checkString(Object input, String message) {
+ if (input instanceof FieldRef) {
+ checkArgument(((FieldRef) input).type().is(CHARACTER_STRING), message);
+ } else {
+ checkArgument(input == null || input instanceof BinaryString, message);
+ }
+ }
+
+ static void checkInteger(Object input, String message) {
+ if (input instanceof FieldRef) {
+ checkArgument(((FieldRef) input).type().is(INTEGER_NUMERIC), message);
+ } else {
+ checkArgument(input == null || input instanceof Number, message);
+ }
+ }
+
+ static void checkDate(Object input, String message) {
+ if (input instanceof FieldRef) {
+ checkArgument(
+ ((FieldRef) input).type().getTypeRoot()
+ == org.apache.paimon.types.DataTypeRoot.DATE,
+ message);
+ } else {
+ checkArgument(input == null || input instanceof Integer, message);
+ }
+ }
+}
diff --git a/paimon-common/src/main/java/org/apache/paimon/predicate/TranslateTransform.java b/paimon-common/src/main/java/org/apache/paimon/predicate/TranslateTransform.java
new file mode 100644
index 0000000000..909f4f7134
--- /dev/null
+++ b/paimon-common/src/main/java/org/apache/paimon/predicate/TranslateTransform.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.predicate;
+
+import org.apache.paimon.data.BinaryString;
+
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator;
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty;
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.apache.paimon.utils.Preconditions.checkArgument;
+
+/** SQL {@code TRANSLATE(input, from, to)} transform. */
+public class TranslateTransform extends StringTransform {
+
+ private static final long serialVersionUID = 1L;
+
+ public static final String NAME = "TRANSLATE";
+
+ @JsonCreator
+ public TranslateTransform(
+ @JsonProperty(StringTransform.FIELD_INPUTS)
+ @JsonDeserialize(contentUsing = StringTransform.InputDeserializer.class)
+ List<Object> inputs) {
+ super(inputs);
+ checkArgument(inputs.size() == 3, "TRANSLATE requires exactly three inputs");
+ }
+
+ @Override
+ public String name() {
+ return NAME;
+ }
+
+ @Override
+ protected BinaryString transform(List<BinaryString> inputs) {
+ BinaryString source = inputs.get(0);
+ BinaryString matching = inputs.get(1);
+ BinaryString replacement = inputs.get(2);
+ if (source == null || matching == null || replacement == null) {
+ return null;
+ }
+
+ int[] matches = matching.toString().codePoints().toArray();
+ int[] replacements = replacement.toString().codePoints().toArray();
+ Map<Integer, Integer> dictionary = new HashMap<>();
+ for (int i = 0; i < matches.length; i++) {
+ // Keep the first mapping for duplicate characters, including a first mapping to
+ // deletion. Map.putIfAbsent cannot express that because it treats a null value as
+ // absent.
+ if (!dictionary.containsKey(matches[i])) {
+ dictionary.put(
+ matches[i],
+ i < replacements.length && replacements[i] != 0 ? replacements[i] : null);
+ }
+ }
+
+ StringBuilder result = new StringBuilder();
+ source.toString()
+ .codePoints()
+ .forEach(
+ codePoint -> {
+ if (!dictionary.containsKey(codePoint)) {
+ result.appendCodePoint(codePoint);
+ } else if (dictionary.get(codePoint) != null) {
+ result.appendCodePoint(dictionary.get(codePoint));
+ }
+ });
+ return BinaryString.fromString(result.toString());
+ }
+
+ @Override
+ public Transform copyWithNewInputs(List<Object> inputs) {
+ return new TranslateTransform(inputs);
+ }
+}
diff --git a/paimon-common/src/main/java/org/apache/paimon/predicate/WeekTransform.java b/paimon-common/src/main/java/org/apache/paimon/predicate/WeekTransform.java
new file mode 100644
index 0000000000..3f370ebdd7
--- /dev/null
+++ b/paimon-common/src/main/java/org/apache/paimon/predicate/WeekTransform.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.predicate;
+
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator;
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty;
+
+import java.time.LocalDateTime;
+import java.time.temporal.IsoFields;
+import java.util.Optional;
+
+/** Extracts the ISO week of the week-based year. */
+public class WeekTransform extends DateExtractTransform {
+
+ private static final long serialVersionUID = 1L;
+
+ public static final String NAME = "WEEK";
+
+ @JsonCreator
+ public WeekTransform(@JsonProperty(DateExtractTransform.FIELD_FIELD_REF) FieldRef fieldRef) {
+ super(fieldRef);
+ }
+
+ public static Optional<Transform> tryCreate(FieldRef fieldRef) {
+ return DateExtractTransform.tryCreate(fieldRef, WeekTransform::new);
+ }
+
+ @Override
+ public String name() {
+ return NAME;
+ }
+
+ @Override
+ protected Integer extract(LocalDateTime dateTime) {
+ return dateTime.get(IsoFields.WEEK_OF_WEEK_BASED_YEAR);
+ }
+
+ @Override
+ protected Transform copy(FieldRef fieldRef) {
+ return new WeekTransform(fieldRef);
+ }
+}
diff --git a/paimon-common/src/main/java/org/apache/paimon/predicate/WeekdayTransform.java b/paimon-common/src/main/java/org/apache/paimon/predicate/WeekdayTransform.java
new file mode 100644
index 0000000000..5e961887cd
--- /dev/null
+++ b/paimon-common/src/main/java/org/apache/paimon/predicate/WeekdayTransform.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.predicate;
+
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator;
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty;
+
+import java.time.LocalDateTime;
+import java.util.Optional;
+
+/** Extracts the weekday, Monday = 0 through Sunday = 6. */
+public class WeekdayTransform extends DateExtractTransform {
+
+ private static final long serialVersionUID = 1L;
+
+ public static final String NAME = "WEEKDAY";
+
+ @JsonCreator
+ public WeekdayTransform(@JsonProperty(DateExtractTransform.FIELD_FIELD_REF) FieldRef fieldRef) {
+ super(fieldRef);
+ }
+
+ public static Optional<Transform> tryCreate(FieldRef fieldRef) {
+ return DateExtractTransform.tryCreate(fieldRef, WeekdayTransform::new);
+ }
+
+ @Override
+ public String name() {
+ return NAME;
+ }
+
+ @Override
+ protected Integer extract(LocalDateTime dateTime) {
+ return dateTime.getDayOfWeek().getValue() - 1;
+ }
+
+ @Override
+ protected Transform copy(FieldRef fieldRef) {
+ return new WeekdayTransform(fieldRef);
+ }
+}
diff --git a/paimon-common/src/main/java/org/apache/paimon/predicate/YearOfWeekTransform.java b/paimon-common/src/main/java/org/apache/paimon/predicate/YearOfWeekTransform.java
new file mode 100644
index 0000000000..06b947bd5c
--- /dev/null
+++ b/paimon-common/src/main/java/org/apache/paimon/predicate/YearOfWeekTransform.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.predicate;
+
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator;
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty;
+
+import java.time.LocalDateTime;
+import java.time.temporal.IsoFields;
+import java.util.Optional;
+
+/** Extracts the ISO week-based year. */
+public class YearOfWeekTransform extends DateExtractTransform {
+
+ private static final long serialVersionUID = 1L;
+
+ public static final String NAME = "YEAR_OF_WEEK";
+
+ @JsonCreator
+ public YearOfWeekTransform(
+ @JsonProperty(DateExtractTransform.FIELD_FIELD_REF) FieldRef fieldRef) {
+ super(fieldRef);
+ }
+
+ public static Optional<Transform> tryCreate(FieldRef fieldRef) {
+ return DateExtractTransform.tryCreate(fieldRef, YearOfWeekTransform::new);
+ }
+
+ @Override
+ public String name() {
+ return NAME;
+ }
+
+ @Override
+ protected Integer extract(LocalDateTime dateTime) {
+ return dateTime.get(IsoFields.WEEK_BASED_YEAR);
+ }
+
+ @Override
+ protected Transform copy(FieldRef fieldRef) {
+ return new YearOfWeekTransform(fieldRef);
+ }
+}
diff --git a/paimon-common/src/test/java/org/apache/paimon/predicate/DateExtractTransformTest.java b/paimon-common/src/test/java/org/apache/paimon/predicate/DateExtractTransformTest.java
index 0a472c78e4..09baa8ad29 100644
--- a/paimon-common/src/test/java/org/apache/paimon/predicate/DateExtractTransformTest.java
+++ b/paimon-common/src/test/java/org/apache/paimon/predicate/DateExtractTransformTest.java
@@ -52,6 +52,23 @@ class DateExtractTransformTest {
assertThat(extract(DataTypes.DATE(), YearTransform::new, value)).isEqualTo(2023);
assertThat(extract(DataTypes.DATE(), MonthTransform::new, value)).isEqualTo(7);
assertThat(extract(DataTypes.DATE(), DayTransform::new, value)).isEqualTo(15);
+ assertThat(extract(DataTypes.DATE(), QuarterTransform::new, value)).isEqualTo(3);
+ assertThat(extract(DataTypes.DATE(), IsoDayOfWeekTransform::new, value)).isEqualTo(6);
+ assertThat(extract(DataTypes.DATE(), DayOfWeekTransform::new, value)).isEqualTo(7);
+ assertThat(extract(DataTypes.DATE(), WeekdayTransform::new, value)).isEqualTo(5);
+ assertThat(extract(DataTypes.DATE(), DayOfYearTransform::new, value)).isEqualTo(196);
+ assertThat(extract(DataTypes.DATE(), WeekTransform::new, value)).isEqualTo(28);
+ assertThat(extract(DataTypes.DATE(), YearOfWeekTransform::new, value)).isEqualTo(2023);
+ }
+
+ @Test
+ public void testIsoWeekCrossesCalendarYear() {
+ int value = epochDay(2024, 12, 30);
+ assertThat(extract(DataTypes.DATE(), WeekTransform::new, value)).isEqualTo(1);
+ assertThat(extract(DataTypes.DATE(), YearOfWeekTransform::new, value)).isEqualTo(2025);
+ assertThat(extract(DataTypes.DATE(), IsoDayOfWeekTransform::new, value)).isEqualTo(1);
+ assertThat(extract(DataTypes.DATE(), DayOfWeekTransform::new, value)).isEqualTo(2);
+ assertThat(extract(DataTypes.DATE(), WeekdayTransform::new, value)).isEqualTo(0);
}
@Test
diff --git a/paimon-common/src/test/java/org/apache/paimon/predicate/ScalarTransformTest.java b/paimon-common/src/test/java/org/apache/paimon/predicate/ScalarTransformTest.java
new file mode 100644
index 0000000000..918a4798f1
--- /dev/null
+++ b/paimon-common/src/test/java/org/apache/paimon/predicate/ScalarTransformTest.java
@@ -0,0 +1,171 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.predicate;
+
+import org.apache.paimon.data.BinaryString;
+import org.apache.paimon.data.GenericRow;
+import org.apache.paimon.types.DataTypes;
+
+import org.junit.jupiter.api.Test;
+
+import java.time.LocalDate;
+import java.util.Arrays;
+import java.util.Collections;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+/** Behavioral tests for scalar transforms. */
+class ScalarTransformTest {
+
+ @Test
+ void testBitLengthUsesUtf8BytesAndPropagatesNull() {
+ BitLengthTransform transform =
+ new BitLengthTransform(
+ Collections.singletonList(new FieldRef(0, "s", DataTypes.STRING())));
+
+ assertThat(transform.transform(GenericRow.of(string("é😀")))).isEqualTo(48);
+ assertThat(transform.transform(GenericRow.of((Object) null))).isNull();
+ }
+
+ @Test
+ void testTranslateCodePointAndDuplicateSemantics() {
+ assertThat(
+ new TranslateTransform(
+ Arrays.asList(string("A😀B"), string("😀"), string("界")))
+ .transform(GenericRow.of()))
+ .isEqualTo(string("A界B"));
+
+ // The first duplicate mapping wins even when it maps to deletion.
+ assertThat(
+ new TranslateTransform(
+ Arrays.asList(string("aba"), string("aa"), string("")))
+ .transform(GenericRow.of()))
+ .isEqualTo(string("b"));
+
+ // A NUL replacement code point acts as the deletion sentinel.
+ assertThat(
+ new TranslateTransform(
+ Arrays.asList(string("aba"), string("a"), string("\u0000")))
+ .transform(GenericRow.of()))
+ .isEqualTo(string("b"));
+
+ assertThat(
+ new TranslateTransform(Arrays.asList(null, string("a"), string("b")))
+ .transform(GenericRow.of()))
+ .isNull();
+ }
+
+ @Test
+ void testOverlayDefaultExplicitAndUnicodeSemantics() {
+ assertThat(
+ new OverlayTransform(Arrays.asList(string("Hello SQL"), string("_"), 6))
+ .transform(GenericRow.of()))
+ .isEqualTo(string("Hello_SQL"));
+ assertThat(
+ new OverlayTransform(
+ Arrays.asList(string("Hello SQL"), string("ANSI "), 7, 0))
+ .transform(GenericRow.of()))
+ .isEqualTo(string("Hello ANSI SQL"));
+ assertThat(
+ new OverlayTransform(Arrays.asList(string("a😀c"), string("界"), 2, 1))
+ .transform(GenericRow.of()))
+ .isEqualTo(string("a界c"));
+ assertThat(
+ new OverlayTransform(Arrays.asList(string("abc"), null, 1, 1))
+ .transform(GenericRow.of()))
+ .isNull();
+ }
+
+ @Test
+ void testPadForPaddingTruncationAndEmptyPad() {
+ assertThat(pad(PadTransform.Direction.LEFT, "hi", 5, "??")).isEqualTo(string("???hi"));
+ assertThat(pad(PadTransform.Direction.RIGHT, "hi", 5, "??")).isEqualTo(string("hi???"));
+ assertThat(pad(PadTransform.Direction.LEFT, "a😀c", 2, "x")).isEqualTo(string("a😀"));
+ assertThat(pad(PadTransform.Direction.RIGHT, "hi", 5, "")).isEqualTo(string("hi"));
+ assertThat(pad(PadTransform.Direction.LEFT, "hi", -1, "?")).isEqualTo(string(""));
+ }
+
+ @Test
+ void testDateAddAndDiffWithLiteralAndFieldInputs() {
+ FieldRef date = new FieldRef(0, "date", DataTypes.DATE());
+ FieldRef days = new FieldRef(1, "days", DataTypes.SMALLINT());
+ int january15 = epochDay("2025-01-15");
+
+ DateAddTransform add = new DateAddTransform(Arrays.asList(date, days));
+ assertThat(add.transform(GenericRow.of(january15, (short) 2)))
+ .isEqualTo(epochDay("2025-01-17"));
+ assertThat(add.transform(GenericRow.of(january15, null))).isNull();
+
+ DateDiffTransform diff = new DateDiffTransform(Arrays.asList(date, epochDay("2025-01-01")));
+ assertThat(diff.transform(GenericRow.of(january15))).isEqualTo(14);
+ assertThat(diff.transform(GenericRow.of((Object) null))).isNull();
+ }
+
+ @Test
+ void testDateTruncSupportsEveryDateLevel() {
+ FieldRef date = new FieldRef(0, "date", DataTypes.DATE());
+ GenericRow row = GenericRow.of(epochDay("2025-05-18"));
+
+ assertThat(trunc(date, "week").transform(row)).isEqualTo(epochDay("2025-05-12"));
+ assertThat(trunc(date, "mon").transform(row)).isEqualTo(epochDay("2025-05-01"));
+ assertThat(trunc(date, "quarter").transform(row)).isEqualTo(epochDay("2025-04-01"));
+ assertThat(trunc(date, "yyyy").transform(row)).isEqualTo(epochDay("2025-01-01"));
+ assertThat(trunc(date, "day").transform(row)).isNull();
+ assertThat(
+ new DateTruncTransform(Arrays.asList(date, null))
+ .transform(GenericRow.of(epochDay("2025-05-18"))))
+ .isNull();
+ }
+
+ @Test
+ void testInputTypesAreValidated() {
+ assertThatThrownBy(
+ () ->
+ new DateAddTransform(
+ Arrays.asList(new FieldRef(0, "s", DataTypes.STRING()), 1)))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("first input must be a date");
+ assertThatThrownBy(
+ () ->
+ new PadTransform(
+ Arrays.asList(string("x"), string("1"), string("_")),
+ PadTransform.Direction.LEFT))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("length must be an integer");
+ }
+
+ private static Object pad(
+ PadTransform.Direction direction, String value, int length, String padding) {
+ return new PadTransform(Arrays.asList(string(value), length, string(padding)), direction)
+ .transform(GenericRow.of());
+ }
+
+ private static DateTruncTransform trunc(FieldRef date, String format) {
+ return new DateTruncTransform(Arrays.asList(date, string(format)));
+ }
+
+ private static BinaryString string(String value) {
+ return BinaryString.fromString(value);
+ }
+
+ private static int epochDay(String value) {
+ return Math.toIntExact(LocalDate.parse(value).toEpochDay());
+ }
+}
diff --git a/paimon-common/src/test/java/org/apache/paimon/predicate/TransformJsonSerdeTest.java b/paimon-common/src/test/java/org/apache/paimon/predicate/TransformJsonSerdeTest.java
index f2a711e01e..47fac07736 100644
--- a/paimon-common/src/test/java/org/apache/paimon/predicate/TransformJsonSerdeTest.java
+++ b/paimon-common/src/test/java/org/apache/paimon/predicate/TransformJsonSerdeTest.java
@@ -131,6 +131,31 @@ class TransformJsonSerdeTest {
new MinuteTransform(new FieldRef(1, "t1", DataTypes.TIMESTAMP(3))))
.expectJson(
"{\"name\":\"MINUTE\",\"fieldRef\":{\"index\":1,\"name\":\"t1\",\"type\":\"TIMESTAMP(3)\"}}"),
+ TestSpec.forTransform(new QuarterTransform(new FieldRef(0, "d0", DataTypes.DATE())))
+ .expectJson(
+ "{\"name\":\"QUARTER\",\"fieldRef\":{\"index\":0,\"name\":\"d0\",\"type\":\"DATE\"}}"),
+ TestSpec.forTransform(
+ new IsoDayOfWeekTransform(new FieldRef(0, "d0", DataTypes.DATE())))
+ .expectJson(
+ "{\"name\":\"ISO_DAY_OF_WEEK\",\"fieldRef\":{\"index\":0,\"name\":\"d0\",\"type\":\"DATE\"}}"),
+ TestSpec.forTransform(
+ new DayOfWeekTransform(new FieldRef(0, "d0", DataTypes.DATE())))
+ .expectJson(
+ "{\"name\":\"DAY_OF_WEEK\",\"fieldRef\":{\"index\":0,\"name\":\"d0\",\"type\":\"DATE\"}}"),
+ TestSpec.forTransform(new WeekdayTransform(new FieldRef(0, "d0", DataTypes.DATE())))
+ .expectJson(
+ "{\"name\":\"WEEKDAY\",\"fieldRef\":{\"index\":0,\"name\":\"d0\",\"type\":\"DATE\"}}"),
+ TestSpec.forTransform(
+ new DayOfYearTransform(new FieldRef(0, "d0", DataTypes.DATE())))
+ .expectJson(
+ "{\"name\":\"DAY_OF_YEAR\",\"fieldRef\":{\"index\":0,\"name\":\"d0\",\"type\":\"DATE\"}}"),
+ TestSpec.forTransform(new WeekTransform(new FieldRef(0, "d0", DataTypes.DATE())))
+ .expectJson(
+ "{\"name\":\"WEEK\",\"fieldRef\":{\"index\":0,\"name\":\"d0\",\"type\":\"DATE\"}}"),
+ TestSpec.forTransform(
+ new YearOfWeekTransform(new FieldRef(0, "d0", DataTypes.DATE())))
+ .expectJson(
+ "{\"name\":\"YEAR_OF_WEEK\",\"fieldRef\":{\"index\":0,\"name\":\"d0\",\"type\":\"DATE\"}}"),
TestSpec.forTransform(
new SubstringTransform(
Arrays.asList(
@@ -195,6 +220,59 @@ class TransformJsonSerdeTest {
TestSpec.forTransform(new LengthTransform(Collections.singletonList(null)))
.expectJson("{\"name\":\"LENGTH\",\"inputs\":[null]}"),
+ // Additional scalar transforms
+ TestSpec.forTransform(
+ new BitLengthTransform(
+ Collections.singletonList(
+ new FieldRef(1, "f1", DataTypes.STRING()))))
+ .expectJson(
+ "{\"name\":\"BIT_LENGTH\",\"inputs\":[{\"index\":1,\"name\":\"f1\",\"type\":\"STRING\"}]}"),
+ TestSpec.forTransform(
+ new TranslateTransform(
+ Arrays.asList(
+ new FieldRef(1, "f1", DataTypes.STRING()),
+ BinaryString.fromString("ab"),
+ BinaryString.fromString("xy"))))
+ .expectJson(
+ "{\"name\":\"TRANSLATE\",\"inputs\":[{\"index\":1,\"name\":\"f1\",\"type\":\"STRING\"},\"ab\",\"xy\"]}"),
+ TestSpec.forTransform(
+ new OverlayTransform(
+ Arrays.asList(
+ new FieldRef(1, "f1", DataTypes.STRING()),
+ BinaryString.fromString("x"),
+ 2,
+ 1)))
+ .expectJson(
+ "{\"name\":\"OVERLAY\",\"inputs\":[{\"index\":1,\"name\":\"f1\",\"type\":\"STRING\"},\"x\",2,1]}"),
+ TestSpec.forTransform(
+ new PadTransform(
+ Arrays.asList(
+ new FieldRef(1, "f1", DataTypes.STRING()),
+ 5,
+ BinaryString.fromString("_")),
+ PadTransform.Direction.LEFT))
+ .expectJson(
+ "{\"name\":\"PAD\",\"inputs\":[{\"index\":1,\"name\":\"f1\",\"type\":\"STRING\"},5,\"_\"],\"direction\":\"LEFT\"}"),
+ TestSpec.forTransform(
+ new DateAddTransform(
+ Arrays.asList(new FieldRef(0, "d0", DataTypes.DATE()), 1)))
+ .expectJson(
+ "{\"name\":\"DATE_ADD\",\"inputs\":[{\"index\":0,\"name\":\"d0\",\"type\":\"DATE\"},1]}"),
+ TestSpec.forTransform(
+ new DateDiffTransform(
+ Arrays.asList(
+ new FieldRef(0, "d0", DataTypes.DATE()),
+ new FieldRef(1, "d1", DataTypes.DATE()))))
+ .expectJson(
+ "{\"name\":\"DATE_DIFF\",\"inputs\":[{\"index\":0,\"name\":\"d0\",\"type\":\"DATE\"},{\"index\":1,\"name\":\"d1\",\"type\":\"DATE\"}]}"),
+ TestSpec.forTransform(
+ new DateTruncTransform(
+ Arrays.asList(
+ new FieldRef(0, "d0", DataTypes.DATE()),
+ BinaryString.fromString("MONTH"))))
+ .expectJson(
+ "{\"name\":\"DATE_TRUNC\",\"inputs\":[{\"index\":0,\"name\":\"d0\",\"type\":\"DATE\"},\"MONTH\"]}"),
+
// error message testing
TestSpec.forJson("{\"name\":\"invalid\"}")
.expectErrorMessage("Could not resolve type id 'invalid'"),
diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/util/SparkExpressionConverter.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/util/SparkExpressionConverter.scala
index b90f4902c4..0e54b10075 100644
--- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/util/SparkExpressionConverter.scala
+++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/util/SparkExpressionConverter.scala
@@ -41,9 +41,22 @@ object SparkExpressionConverter {
private val LOWER = "LOWER"
private val SUBSTRING = "SUBSTRING"
private val CHAR_LENGTH = "CHAR_LENGTH"
+ private val BIT_LENGTH = "BIT_LENGTH"
+ private val TRANSLATE = "TRANSLATE"
+ private val OVERLAY = "OVERLAY"
+ private val LPAD = "LPAD"
+ private val RPAD = "RPAD"
private val TRIM = "TRIM"
private val LTRIM = "LTRIM"
private val RTRIM = "RTRIM"
+ private val DATE_ADD = "DATE_ADD"
+ private val DATE_DIFF = "DATE_DIFF"
+ private val TRUNC = "TRUNC"
+
+ // Spark encodes dayofweek and weekday as arithmetic over an ISO EXTRACT(DAY_OF_WEEK).
+ private val ADD = "+"
+ private val SUBTRACT = "-"
+ private val REMAINDER = "%"
// Supported fields of the EXTRACT expression
private val EXTRACT_YEAR = "YEAR"
@@ -52,6 +65,11 @@ object SparkExpressionConverter {
private val EXTRACT_HOUR = "HOUR"
private val EXTRACT_MINUTE = "MINUTE"
private val EXTRACT_SECOND = "SECOND"
+ private val EXTRACT_QUARTER = "QUARTER"
+ private val EXTRACT_DAY_OF_WEEK = "DAY_OF_WEEK"
+ private val EXTRACT_DAY_OF_YEAR = "DAY_OF_YEAR"
+ private val EXTRACT_WEEK = "WEEK"
+ private val EXTRACT_YEAR_OF_WEEK = "YEAR_OF_WEEK"
/** Convert Spark [[Expression]] to Paimon [[Transform]], return None if not supported. */
def toPaimonTransform(exp: Expression, rowType: RowType): Option[Transform] = {
@@ -69,6 +87,48 @@ object SparkExpressionConverter {
}
}
+ def literalEquals(exp: Expression, value: Int): Boolean = exp match {
+ case l: Literal[_] =>
+ l.value() match {
+ case i: Int => i == value
+ case _ => false
+ }
+ case _ => false
+ }
+
+ def dayOfWeekField(exp: Expression): Option[FieldRef] = {
+ if (org.apache.spark.SPARK_VERSION < "3.4") {
+ None
+ } else {
+ exp match {
+ case extract: Extract if extract.field() == EXTRACT_DAY_OF_WEEK =>
+ extract.source() match {
+ case n: NamedReference => Some(toPaimonFieldRef(n, rowType))
+ case _ => None
+ }
+ case _ => None
+ }
+ }
+ }
+
+ def sparkDayOfWeek(children: Seq[Expression]): Option[Transform] = children match {
+ case Seq(remainder: GeneralScalarExpression, one)
+ if literalEquals(one, 1) &&
+ remainder.name() == REMAINDER =>
+ remainder.children().toSeq match {
+ case Seq(extract, seven) if literalEquals(seven, 7) =>
+ dayOfWeekField(extract).flatMap(DayOfWeekTransform.tryCreate)
+ case _ => None
+ }
+ case _ => None
+ }
+
+ def sparkWeekday(children: Seq[Expression]): Option[Transform] = children match {
+ case Seq(extract, one) if literalEquals(one, 1) =>
+ dayOfWeekField(extract).flatMap(WeekdayTransform.tryCreate)
+ case _ => None
+ }
+
exp match {
case n: NamedReference => Some(new FieldTransform(toPaimonFieldRef(n, rowType)))
case s: GeneralScalarExpression =>
@@ -78,6 +138,14 @@ object SparkExpressionConverter {
case LOWER => convertChildren(s.children()).map(i => new LowerTransform(i))
case SUBSTRING => convertChildren(s.children()).map(i => new SubstringTransform(i))
case CHAR_LENGTH => convertChildren(s.children()).map(i => new LengthTransform(i))
+ case BIT_LENGTH => convertChildren(s.children()).map(i => new BitLengthTransform(i))
+ case TRANSLATE => convertChildren(s.children()).map(i => new TranslateTransform(i))
+ case OVERLAY => convertChildren(s.children()).map(i => new OverlayTransform(i))
+ case LPAD =>
+ convertChildren(s.children()).map(i => new PadTransform(i, PadTransform.Direction.LEFT))
+ case RPAD =>
+ convertChildren(s.children()).map(
+ i => new PadTransform(i, PadTransform.Direction.RIGHT))
case TRIM =>
convertChildren(s.children()).map(i => new TrimTransform(i, TrimTransform.Flag.BOTH))
case LTRIM =>
@@ -85,6 +153,11 @@ object SparkExpressionConverter {
case RTRIM =>
convertChildren(s.children()).map(
i => new TrimTransform(i, TrimTransform.Flag.TRAILING))
+ case DATE_ADD => convertChildren(s.children()).map(i => new DateAddTransform(i))
+ case DATE_DIFF => convertChildren(s.children()).map(i => new DateDiffTransform(i))
+ case TRUNC => convertChildren(s.children()).map(i => new DateTruncTransform(i))
+ case ADD => sparkDayOfWeek(s.children())
+ case SUBTRACT => sparkWeekday(s.children())
case _ => None
}
case c: Cast =>
@@ -119,6 +192,11 @@ object SparkExpressionConverter {
case EXTRACT_HOUR => HourTransform.tryCreate(fieldRef)
case EXTRACT_MINUTE => MinuteTransform.tryCreate(fieldRef)
case EXTRACT_SECOND => SecondTransform.tryCreate(fieldRef)
+ case EXTRACT_QUARTER => QuarterTransform.tryCreate(fieldRef)
+ case EXTRACT_DAY_OF_WEEK => IsoDayOfWeekTransform.tryCreate(fieldRef)
+ case EXTRACT_DAY_OF_YEAR => DayOfYearTransform.tryCreate(fieldRef)
+ case EXTRACT_WEEK => WeekTransform.tryCreate(fieldRef)
+ case EXTRACT_YEAR_OF_WEEK => YearOfWeekTransform.tryCreate(fieldRef)
case _ => None
}
}
diff --git a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/SparkV2FilterConverterTestBase.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/SparkV2FilterConverterTestBase.scala
index 7a87bd0666..6321a9d5d7 100644
--- a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/SparkV2FilterConverterTestBase.scala
+++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/SparkV2FilterConverterTestBase.scala
@@ -19,7 +19,7 @@
package org.apache.paimon.spark.sql
import org.apache.paimon.data.{BinaryString, Decimal, Timestamp}
-import org.apache.paimon.predicate.{DayTransform, FieldRef, HourTransform, LengthTransform, MinuteTransform, MonthTransform, PredicateBuilder, SecondTransform, YearTransform}
+import org.apache.paimon.predicate.{BitLengthTransform, DateAddTransform, DateDiffTransform, DateTruncTransform, DayOfWeekTransform, DayOfYearTransform, DayTransform, FieldRef, HourTransform, LengthTransform, MinuteTransform, MonthTransform, OverlayTransform, PadTransform, PredicateBuilder, QuarterTransform, SecondTransform, Transform, TranslateTransform, WeekdayTransform, WeekTransform, YearOfWeekTransform, YearTransform}
import org.apache.paimon.spark.{PaimonSparkTestBase, SparkV2FilterConverter}
import org.apache.paimon.spark.util.shim.TypeUtils.treatPaimonTimestampTypeAsSparkTimestampType
import org.apache.paimon.table.source.DataSplit
@@ -431,6 +431,76 @@ abstract class SparkV2FilterConverterTestBase extends PaimonSparkTestBase {
}
}
+ test("V2Filter: remaining Spark string scalar expressions") {
+ if (gteqSpark3_4) {
+ val stringFieldRef = new FieldRef(0, "string_col", rowType.getTypeAt(0))
+ val specs: Seq[(String, Transform, Object, Seq[Row])] = Seq(
+ (
+ "bit_length(string_col) = 16",
+ new BitLengthTransform(List[Object](stringFieldRef).asJava),
+ Int.box(16),
+ Seq(Row("hi"))),
+ (
+ "translate(string_col, 'hel', 'xyz') = 'xyzzo'",
+ new TranslateTransform(
+ List[Object](
+ stringFieldRef,
+ BinaryString.fromString("hel"),
+ BinaryString.fromString("xyz")).asJava),
+ BinaryString.fromString("xyzzo"),
+ Seq(Row("hello"))),
+ (
+ "overlay(string_col, 'XX', 2, 2) = 'hXXlo'",
+ new OverlayTransform(
+ List[Object](
+ stringFieldRef,
+ BinaryString.fromString("XX"),
+ Int.box(2),
+ Int.box(2)).asJava),
+ BinaryString.fromString("hXXlo"),
+ Seq(Row("hello")))
+ )
+ specs.foreach {
+ case (filter, transform, expectedValue, expectedRows) =>
+ assert(
+ converter.convert(v2Filter(filter)).contains(builder.equal(transform, expectedValue)),
+ filter)
+ checkAnswer(
+ sql(s"SELECT string_col FROM test_tbl WHERE $filter ORDER BY string_col"),
+ expectedRows)
+ }
+ }
+ }
+
+ test("V2Filter: Spark 4.1 pad expressions") {
+ if (gteqSpark4_1) {
+ val stringFieldRef = new FieldRef(0, "string_col", rowType.getTypeAt(0))
+ Seq(
+ (
+ "lpad(string_col, 7, '_') = '__hello'",
+ PadTransform.Direction.LEFT,
+ BinaryString.fromString("__hello"),
+ Seq(Row("hello"))),
+ (
+ "rpad(string_col, 7, '_') = 'hello__'",
+ PadTransform.Direction.RIGHT,
+ BinaryString.fromString("hello__"),
+ Seq(Row("hello")))
+ ).foreach {
+ case (filter, direction, expectedValue, expectedRows) =>
+ val transform = new PadTransform(
+ List[Object](stringFieldRef, Int.box(7), BinaryString.fromString("_")).asJava,
+ direction)
+ assert(
+ converter.convert(v2Filter(filter)).contains(builder.equal(transform, expectedValue)),
+ filter)
+ checkAnswer(
+ sql(s"SELECT string_col FROM test_tbl WHERE $filter ORDER BY string_col"),
+ expectedRows)
+ }
+ }
+ }
+
test("V2Filter: Year, Month and Day") {
if (gteqSpark3_4) {
val dateFieldRef = new FieldRef(9, "date_col", rowType.getTypeAt(9))
@@ -508,6 +578,67 @@ abstract class SparkV2FilterConverterTestBase extends PaimonSparkTestBase {
}
}
+ test("V2Filter: remaining date fields") {
+ if (gteqSpark3_4) {
+ val dateFieldRef = new FieldRef(9, "date_col", rowType.getTypeAt(9))
+ val specs: Seq[(String, Transform, Object, Int)] = Seq(
+ ("quarter(date_col) = 1", new QuarterTransform(dateFieldRef), Int.box(1), 4),
+ ("dayofweek(date_col) = 4", new DayOfWeekTransform(dateFieldRef), Int.box(4), 1),
+ ("weekday(date_col) = 2", new WeekdayTransform(dateFieldRef), Int.box(2), 1),
+ ("dayofyear(date_col) = 15", new DayOfYearTransform(dateFieldRef), Int.box(15), 1),
+ ("weekofyear(date_col) = 3", new WeekTransform(dateFieldRef), Int.box(3), 4),
+ (
+ "extract(YEAROFWEEK FROM date_col) = 2025",
+ new YearOfWeekTransform(dateFieldRef),
+ Int.box(2025),
+ 4)
+ )
+ specs.foreach {
+ case (filter, transform, expectedValue, expectedCount) =>
+ assert(
+ converter.convert(v2Filter(filter)).contains(builder.equal(transform, expectedValue)),
+ filter)
+ assert(sql(s"SELECT date_col FROM test_tbl WHERE $filter").count() == expectedCount)
+ }
+ }
+ }
+
+ test("V2Filter: date scalar expressions") {
+ if (gteqSpark3_4) {
+ val dateFieldRef = new FieldRef(9, "date_col", rowType.getTypeAt(9))
+ val specs: Seq[(String, Transform, Object, String)] = Seq(
+ (
+ "date_add(date_col, 1) = date('2025-01-16')",
+ new DateAddTransform(List[Object](dateFieldRef, Int.box(1)).asJava),
+ Int.box(LocalDate.parse("2025-01-16").toEpochDay.toInt),
+ "SELECT date('2025-01-15')"),
+ (
+ "datediff(date_col, date('2025-01-01')) = 14",
+ new DateDiffTransform(
+ List[Object](
+ dateFieldRef,
+ Int.box(LocalDate.parse("2025-01-01").toEpochDay.toInt)).asJava),
+ Int.box(14),
+ "SELECT date('2025-01-15')"),
+ (
+ "trunc(date_col, 'MONTH') = date('2025-01-01')",
+ new DateTruncTransform(
+ List[Object](dateFieldRef, BinaryString.fromString("MONTH")).asJava),
+ Int.box(LocalDate.parse("2025-01-01").toEpochDay.toInt),
+ "SELECT date_col FROM test_tbl ORDER BY date_col")
+ )
+ specs.foreach {
+ case (filter, transform, expectedValue, expectedSql) =>
+ assert(
+ converter.convert(v2Filter(filter)).contains(builder.equal(transform, expectedValue)),
+ filter)
+ checkAnswer(
+ sql(s"SELECT date_col FROM test_tbl WHERE $filter ORDER BY date_col"),
+ sql(expectedSql))
+ }
+ }
+ }
+
test("V2Filter: StartWith") {
val filter = "string_col LIKE 'h%'"
val actual = converter.convert(v2Filter(filter)).get
(paimon) 01/20: [python] Cover native dynamic PK events and compacted first-row scans (#9854)
Posted by ho...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
XiaoHongbo-Hope pushed a commit to branch release-2.1
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit f3862f6b4e5774a244b99104122d6767c144792b
Author: Jingsong Lee <ji...@gmail.com>
AuthorDate: Wed Sep 16 09:08:54 2026 +0800
[python] Cover native dynamic PK events and compacted first-row scans (#9854)
---
paimon-python/pypaimon/read/table_scan.py | 5 +--
.../tests/native_plan_capabilities_test.py | 36 ++++++++++++++++++++++
.../tests/native_plan_dynamic_bucket_test.py | 33 ++++++++++++++++++++
3 files changed, 72 insertions(+), 2 deletions(-)
diff --git a/paimon-python/pypaimon/read/table_scan.py b/paimon-python/pypaimon/read/table_scan.py
index 9a07b5ab9d..7e4889da51 100755
--- a/paimon-python/pypaimon/read/table_scan.py
+++ b/paimon-python/pypaimon/read/table_scan.py
@@ -180,8 +180,9 @@ class TableScan:
return False
if self.table.options.query_auth_enabled:
return False
- # Java batch first-row reads skip L0. Scans including L0 still need
- # first-row overlap packing that Rust does not currently provide.
+ # Ordinary Java first-row batch scans skip L0. The clustering override
+ # can combine first-row with DV merge-on-read, but its files need a
+ # separate audit because they may be sorted by non-primary-key columns.
if (self.table.options.merge_engine() == 'first-row'
and not fs.skip_level0 and not fs.is_streaming):
return False
diff --git a/paimon-python/pypaimon/tests/native_plan_capabilities_test.py b/paimon-python/pypaimon/tests/native_plan_capabilities_test.py
index 2a35b75b7c..867b3261cd 100644
--- a/paimon-python/pypaimon/tests/native_plan_capabilities_test.py
+++ b/paimon-python/pypaimon/tests/native_plan_capabilities_test.py
@@ -314,6 +314,42 @@ class NativePlanCapabilitiesTest(unittest.TestCase):
self.assertEqual(plan.splits()[0].data_deletion_files[0].cardinality, 2)
self._assert_parity(table.copy({'scan.snapshot-id': '1'}), rows, 1)
+ def test_first_row_compacted_runs_with_overlapping_key_ranges(self):
+ for bucket in ('1', '-1'):
+ with self.subTest(bucket=bucket):
+ table = self._create('first_row_' + bucket, {
+ 'bucket': bucket, 'merge-engine': 'first-row',
+ 'source.split.target-size': '1b',
+ 'source.split.open-file-cost': '1b',
+ }, primary_keys=['k'])
+ expected = [{'k': k, 'v': 'v%d' % k} for k in range(1, 5)]
+ for level, keys in ((1, (1, 3)), (2, (2, 4))):
+ builder = table.new_batch_write_builder()
+ writer, commit = builder.new_write(), builder.new_commit()
+ try:
+ writer.write_arrow(pa.Table.from_pylist(
+ [expected[key - 1] for key in keys], schema=self.schema))
+ messages = writer.prepare_commit()
+ # Each run already contains unique first rows. Their
+ # ranges overlap, but their actual keys are disjoint.
+ for message in messages:
+ message.new_files = [replace(file, level=level)
+ for file in message.new_files]
+ commit.commit(messages)
+ finally:
+ writer.close()
+ commit.close()
+ self._write(table, [{'k': 1, 'v': 'later'}, {'k': 5, 'v': 'pending'}])
+ plan = self._assert_parity(table, expected, 3)
+ self.assertTrue(all(split.raw_convertible for split in plan.splits()))
+ self.assertEqual({file.level for split in plan.splits()
+ for file in split.files}, {1, 2})
+ pb = table.new_read_builder().new_predicate_builder()
+ self._assert_parity(table, [expected[1]], 3, predicate=pb.equal('v', 'v2'))
+ self._assert_parity(table, [], 3, predicate=pb.equal('v', 'later'))
+ self._assert_parity(table.copy({'scan.snapshot-id': '1'}),
+ [expected[0], expected[2]], 1)
+
@unittest.skipUnless(native_version_at_least(0, 4, 0),
'pypaimon-rust>=0.4.0 required for native DV scans')
def test_external_deletion_vector_path_is_preserved(self):
diff --git a/paimon-python/pypaimon/tests/native_plan_dynamic_bucket_test.py b/paimon-python/pypaimon/tests/native_plan_dynamic_bucket_test.py
index 1fe383e773..27ca00ef6f 100644
--- a/paimon-python/pypaimon/tests/native_plan_dynamic_bucket_test.py
+++ b/paimon-python/pypaimon/tests/native_plan_dynamic_bucket_test.py
@@ -17,6 +17,7 @@
"""Real bucket growth and cross-partition updates through native planning."""
+import json
from unittest.mock import patch
import pyarrow as pa
@@ -75,6 +76,11 @@ def test_native_pk_bucket_growth_and_partition_migration(tmp_path, cross_partiti
writer.write_arrow(pa.RecordBatch.from_pylist(rows, schema=schema))
builder.new_commit().commit(writer.prepare_commit())
table = catalog.get_table('default.t')
+ # Give incremental windows deterministic boundaries.
+ path = table.snapshot_manager().get_snapshot_path(snapshot_id)
+ snapshot = json.loads(table.file_io.read_file_utf8(path))
+ snapshot['timeMillis'] = snapshot_id * 100
+ table.file_io.write_file(path, json.dumps(snapshot), overwrite=True)
pb = table.new_read_builder().new_predicate_builder()
for predicate in (None, pb.equal('id', 1), pb.equal('p', 'a'), pb.equal('v', 'updated')):
for shard in (None, (0, 2), (1, 2)):
@@ -99,3 +105,30 @@ def test_native_pk_bucket_growth_and_partition_migration(tmp_path, cross_partiti
old_plan, old_rows = read(table.copy({'scan.snapshot-id': '1'}), native)
assert old_plan.snapshot_id == 1
assert old_rows == initial
+
+ # Java DeleteExistingProcessor emits DELETE in the old partition with
+ # the incoming non-partition values. Batch merging hides this distinction;
+ # incremental readers must preserve both the partition and the row kind.
+ events = [(row['id'], row['p'], row['v'], 0) for row in initial + updates]
+ if cross_partition:
+ events.append((1, 'a', 'moved', 3))
+ for native in (False, True):
+ for partition in (None, 'a', 'b'):
+ builder = table.copy({
+ 'scan.native-plan.enabled': str(native).lower(),
+ 'incremental-between-timestamp': '0,200',
+ }).new_read_builder()
+ if partition is not None:
+ builder.with_filter(pb.equal('p', partition))
+ scan = builder.new_scan()
+ if native:
+ with patch.object(scan.file_scanner, 'scan', side_effect=AssertionError('native fallback')):
+ plan = scan.plan()
+ else:
+ plan = scan.plan()
+ assert all(split.is_streaming for split in plan.splits())
+ actual = [(row.get_field(0), row.get_field(1), row.get_field(2),
+ row.get_row_kind().value)
+ for row in builder.new_read().to_iterator(plan.splits())]
+ assert sorted(actual) == sorted(
+ event for event in events if partition is None or event[1] == partition)
(paimon) 05/20: [python] Speed up LeRobot video window reads (#9852)
Posted by ho...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
XiaoHongbo-Hope pushed a commit to branch release-2.1
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit 03d68bfec72e004731fbbc2956eadbadd566c8db
Author: XiaoHongbo <xi...@alibaba-inc.com>
AuthorDate: Wed Sep 16 15:21:41 2026 +0800
[python] Speed up LeRobot video window reads (#9852)
---
.../pypaimon/multimodal/lerobot/dataset.py | 118 +++++++++++-
.../pypaimon/tests/multimodal_lerobot_test.py | 203 +++++++++++++++++++++
2 files changed, 315 insertions(+), 6 deletions(-)
diff --git a/paimon-python/pypaimon/multimodal/lerobot/dataset.py b/paimon-python/pypaimon/multimodal/lerobot/dataset.py
index 1332b870cf..eca82cbfd6 100644
--- a/paimon-python/pypaimon/multimodal/lerobot/dataset.py
+++ b/paimon-python/pypaimon/multimodal/lerobot/dataset.py
@@ -65,7 +65,7 @@ _TORCH_DTYPE_NAMES = {
}
_IMAGE_READ_ATTEMPTS = 3
-_MAX_VIDEO_DECODE_WORKERS = 8
+_MAX_VISUAL_WORKERS = 8
_CONTROL_FEATURES = frozenset({
"index",
@@ -424,6 +424,14 @@ class PaimonDatasetReader(ABC):
self.return_uint8,
)
+ # Transforms may shrink frames; assemble those windows one sample at a time.
+ video_windows = {} if self.image_transforms is not None else _decode_video_windows(
+ plans, rows, getattr(self, "_video_collators", ()),
+ self._features, self.return_uint8)
+ for group in row_groups:
+ for row in group.values():
+ for key in video_windows:
+ row.pop(key, None)
_decode_video_rows(
row_groups, getattr(self, "_video_collators", ()))
converted = {
@@ -438,9 +446,14 @@ class PaimonDatasetReader(ABC):
})
import torch
+ visual_windows = _stack_visual_windows(
+ plans, converted, [key for key in self._visual_keys
+ if key not in video_windows]
+ ) if self.image_transforms is None and plans[0]["windows"] else {}
+ visual_windows.update(video_windows)
duplicates = _duplicate_indices(plans)
result = []
- for plan in plans:
+ for offset, plan in enumerate(plans):
item = dict(converted[plan["index"]])
if plan["index"] in duplicates:
item = {
@@ -448,9 +461,12 @@ class PaimonDatasetReader(ABC):
for key, value in item.items()
}
for key, positions in plan["windows"].items():
- item[key] = torch.stack([
- converted[position][key] for position in positions
- ])
+ if key in visual_windows:
+ item[key] = visual_windows[key][offset]
+ else:
+ item[key] = torch.stack([
+ converted[position][key] for position in positions
+ ])
item.update(plan["padding"])
if self.image_transforms is not None:
for key in self._visual_keys:
@@ -1309,6 +1325,96 @@ def _attach_task_labels(rows, task_names, subtask_names):
row["subtask"] = subtask_names[subtask_index]
+def _decode_video_windows(plans, rows, collators, features, return_uint8):
+ import torch
+
+ tasks = [c for c in collators if c.video_column in plans[0]["windows"]]
+ if not tasks or torch.get_num_threads() > 1:
+ return {}
+ grad_enabled = torch.is_grad_enabled()
+ inference_enabled = torch.is_inference_mode_enabled()
+
+ def decode(collator):
+ collator._ensure_process_local_cache()
+ key = collator.video_column
+ descriptors = {
+ position: collator._prepare_row(row)[1]
+ for position, row in rows.items() if key in row
+ }
+ requests = {}
+ for offset, plan in enumerate(plans):
+ window = [descriptors[p] for p in plan["windows"][key]]
+ if not window or any(d is None for d in window):
+ return key, None
+ payload = window[0].payload_descriptor
+ # Cross-file windows use the regular frame assembly path.
+ if any(d.payload_descriptor != payload for d in window):
+ return key, None
+ requests.setdefault(payload, []).append((
+ offset, [d.frame_index for d in window]))
+
+ output = [None] * len(plans)
+ with torch.inference_mode(inference_enabled), \
+ torch.set_grad_enabled(grad_enabled):
+ for payload, windows in requests.items():
+ decoder = collator._decoder(payload)
+ get_frames = getattr(decoder, "get_frames_at", None)
+ if not callable(get_frames):
+ return key, None
+ indices = sorted({i for _, window in windows for i in window})
+ frames = get_frames(indices=indices).data
+ if (not torch.is_tensor(frames) or frames.ndim != 4
+ or len(frames) != len(indices)):
+ raise ValueError("Video decoder must return one frame per index.")
+ _video_tensor(frames[0], features[key], return_uint8=True)
+ # Reorder reused frames once instead of once per window.
+ if sum(len(window) for _, window in windows) >= 2 * len(indices):
+ frames = frames.contiguous()
+ positions = {index: pos for pos, index in enumerate(indices)}
+ for offset, window in windows:
+ selection = [positions[index] for index in window]
+ start = selection[0]
+ if selection == list(range(start, start + len(selection))):
+ output[offset] = frames[start:start + len(selection)].clone(
+ memory_format=torch.contiguous_format)
+ else:
+ output[offset] = frames.index_select(0, torch.tensor(
+ selection, dtype=torch.long, device=frames.device))
+ if frames.dtype == torch.uint8 and not return_uint8:
+ output[offset] = output[offset].float().div_(255)
+ return key, output
+
+ if len(tasks) == 1:
+ decoded = [decode(tasks[0])]
+ else:
+ with ThreadPoolExecutor(
+ max_workers=min(len(tasks), _MAX_VISUAL_WORKERS)) as executor:
+ decoded = list(executor.map(decode, tasks))
+ return {key: windows for key, windows in decoded if windows is not None}
+
+
+def _stack_visual_windows(plans, rows, visual_keys):
+ import torch
+
+ keys = [key for key in plans[0]["windows"] if key in visual_keys]
+ # Let PyTorch handle parallelism when its own thread pool is enabled.
+ if len(keys) < 2 or torch.get_num_threads() > 1:
+ return {}
+ grad_enabled = torch.is_grad_enabled()
+ inference_enabled = torch.is_inference_mode_enabled()
+
+ def stack(key):
+ with torch.inference_mode(inference_enabled), \
+ torch.set_grad_enabled(grad_enabled):
+ return [torch.stack([
+ rows[position][key] for position in plan["windows"][key]
+ ]) for plan in plans]
+
+ with ThreadPoolExecutor(
+ max_workers=min(len(keys), _MAX_VISUAL_WORKERS)) as executor:
+ return dict(zip(keys, executor.map(stack, keys)))
+
+
def _torch_row(row, features, return_uint8=False):
import torch
@@ -1568,7 +1674,7 @@ def _decode_video_rows(row_groups, collators):
else:
with ThreadPoolExecutor(
max_workers=min(
- len(tasks), _MAX_VIDEO_DECODE_WORKERS)) as executor:
+ len(tasks), _MAX_VISUAL_WORKERS)) as executor:
decoded_groups = list(executor.map(
lambda task: task[0](task[2]), tasks))
diff --git a/paimon-python/pypaimon/tests/multimodal_lerobot_test.py b/paimon-python/pypaimon/tests/multimodal_lerobot_test.py
index a9d0ff6533..7c07c0718a 100644
--- a/paimon-python/pypaimon/tests/multimodal_lerobot_test.py
+++ b/paimon-python/pypaimon/tests/multimodal_lerobot_test.py
@@ -26,6 +26,7 @@ import sys
import tempfile
import threading
import unittest
+import weakref
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
from types import SimpleNamespace
@@ -48,10 +49,12 @@ from pypaimon.multimodal.lerobot.dataset import (
_PyAVVideoDecoder,
_arrow_rows,
_decode_video_rows,
+ _decode_video_windows,
_image_tensor,
_index_names,
_open_video_decoder,
_selected_episodes,
+ _stack_visual_windows,
_torch_row,
)
from pypaimon.multimodal.lerobot.api import _create_target_table
@@ -949,6 +952,206 @@ class LeRobotValidationTest(unittest.TestCase):
self.assertEqual([3, 4, 5], list(
sample["observation.image"].shape))
+ def test_transformed_windows_are_assembled_per_sample(self):
+ try:
+ import torch
+ except ImportError as error:
+ self.skipTest(str(error))
+
+ keys = ["left", "right"]
+ original_stack = torch.stack
+ for kind in ("image", "video"):
+ with self.subTest(kind=kind):
+ reader = object.__new__(_ManualDatasetReader)
+ reader._total_frames = 33
+ reader.episodes = None
+ reader._selected_ranges = None
+ reader._features = {
+ key: {"dtype": kind, "shape": [4, 5, 3]}
+ for key in keys
+ }
+ reader._projection = keys
+ reader._delta_projection = keys
+ reader._image_keys = []
+ reader._visual_keys = keys
+ reader._video_collators = []
+ reader._file_io = None
+ reader._task_names = ["task"]
+ reader._subtask_names = None
+ reader.return_uint8 = True
+ reader._plan = lambda i: {
+ "index": i, "windows": {key: [i, i, i + 1] for key in keys},
+ "padding": {},
+ }
+ reader._read_rows = lambda indices, projection: {
+ i: dict(task_index=0, **{
+ key: torch.full((3, 4, 5), i, dtype=torch.uint8)
+ for key in keys
+ }) for i in indices
+ }
+ windows = []
+ peak = []
+
+ def stack(*args, **kwargs):
+ value = original_stack(*args, **kwargs)
+ windows.append(weakref.ref(value))
+ peak.append(sum(ref() is not None for ref in windows))
+ return value
+
+ def resize(value):
+ return torch.nn.functional.interpolate(
+ value.float(), size=(2, 2), mode="nearest")
+
+ reader.image_transforms = resize
+ module = "pypaimon.multimodal.lerobot.dataset."
+ with patch("torch.get_num_threads", return_value=1), \
+ patch("torch.stack", side_effect=stack), \
+ patch(module + "_decode_video_windows",
+ wraps=_decode_video_windows) as decode, \
+ patch(module + "_stack_visual_windows",
+ wraps=_stack_visual_windows) as assemble:
+ result = reader.get_items(list(range(32)))
+ self.assertEqual(64, len(windows))
+ self.assertLessEqual(max(peak), len(keys))
+ decode.assert_not_called()
+ assemble.assert_not_called()
+ self.assertTrue(all(ref() is None for ref in windows))
+ for i, item in enumerate(result):
+ for key in keys:
+ expected = torch.tensor([i, i, i + 1]).reshape(3, 1, 1, 1)
+ self.assertTrue(torch.equal(
+ item[key], expected.expand(3, 3, 2, 2).float()))
+ result[0][keys[0]].zero_()
+ self.assertEqual(1, result[0][keys[1]][-1, 0, 0, 0].item())
+
+ def test_video_windows_decode_directly_and_fall_back(self):
+ try:
+ import torch
+ except ImportError as error:
+ self.skipTest(str(error))
+ from pypaimon.multimodal.video import VideoFrameCollator
+ from pypaimon.table.row.blob import VideoFrameDescriptor
+
+ key = "camera"
+ feature = {key: {"dtype": "video", "shape": [4, 5, 3]}}
+ rows = {i: {key: VideoFrameDescriptor("a.video", 0, 1, i).serialize()}
+ for i in range(2)}
+ plans = [{"windows": {key: [1, 0, 0]}}] * 2
+ pixels = torch.arange(120).reshape(2, 4, 5, 3).to(torch.uint8)
+ frames = pixels.permute(0, 3, 1, 2)
+ batch = frames.contiguous()
+ decoder = SimpleNamespace(get_frames_at=Mock(
+ return_value=SimpleNamespace(data=batch)))
+ collator = VideoFrameCollator(
+ SimpleNamespace(file_io=Mock()), video_column=key,
+ decoder_factory=Mock(), decode_fn=Mock())
+ with patch.object(collator, "_decoder", return_value=decoder) as open_decoder, \
+ patch("torch.get_num_threads", return_value=1):
+ for uint8, batch in ((True, frames), (False, frames),
+ (True, frames.contiguous()),
+ (False, frames.contiguous())):
+ decoder.get_frames_at.reset_mock()
+ decoder.get_frames_at.return_value = SimpleNamespace(data=batch)
+ result = _decode_video_windows(
+ plans, rows, [collator], feature, uint8)[key]
+ expected = batch[[1, 0, 0]]
+ if not uint8:
+ expected = expected.float().div(255)
+ self.assertTrue(torch.equal(result[0], expected))
+ self.assertTrue(result[0].is_contiguous())
+ result[0][1].zero_()
+ self.assertTrue(torch.equal(result[0][2], expected[2]))
+ self.assertTrue(torch.equal(result[1], expected))
+ self.assertTrue(torch.equal(batch, frames))
+ decoder.get_frames_at.assert_called_once_with(indices=[0, 1])
+
+ for indices in ([0, 1], [1, 0, 0]):
+ decoder.get_frames_at.reset_mock()
+ decoder.get_frames_at.return_value = SimpleNamespace(data=frames)
+ result = _decode_video_windows(
+ [{"windows": {key: indices}}], rows, [collator], feature, True)[key][0]
+ decoder.get_frames_at.assert_called_once_with(indices=[0, 1])
+ self.assertTrue(torch.equal(result, batch[indices]))
+ self.assertTrue(result.is_contiguous())
+ result.zero_()
+ self.assertTrue(torch.equal(batch, frames))
+
+ decoder.get_frames_at.reset_mock()
+ separate = dict(rows)
+ separate[2] = {key: VideoFrameDescriptor("b.video", 0, 1, 0).serialize()}
+ separate[3] = {key: VideoFrameDescriptor("b.video", 0, 1, 1).serialize()}
+ interleaved = [{"windows": {key: window}}
+ for window in ([1, 0, 0], [2, 3], [0, 1], [3, 2, 2])]
+ result = _decode_video_windows(
+ interleaved, separate, [collator], feature, True)[key]
+ self.assertEqual(2, decoder.get_frames_at.call_count)
+ self.assertEqual(
+ [[0, 1], [0, 1]],
+ [c.kwargs["indices"] for c in decoder.get_frames_at.call_args_list])
+ for actual, indices in zip(result, ([1, 0, 0], [0, 1], [0, 1], [1, 0, 0])):
+ self.assertTrue(torch.equal(actual, batch[indices]))
+ self.assertTrue(actual.is_contiguous())
+ result[2].zero_()
+ self.assertTrue(torch.equal(result[1], batch))
+ self.assertTrue(torch.equal(batch, frames))
+
+ open_decoder.reset_mock()
+ mixed = dict(rows)
+ mixed[1] = {key: VideoFrameDescriptor("b.video", 0, 1, 0).serialize()}
+ self.assertEqual({}, _decode_video_windows(
+ plans, mixed, [collator], feature, True))
+ open_decoder.assert_not_called()
+ self.assertEqual({}, _decode_video_windows(
+ plans, {0: {key: None}, 1: rows[1]}, [collator], feature, True))
+ open_decoder.return_value = SimpleNamespace()
+ self.assertEqual({}, _decode_video_windows(
+ plans, rows, [collator], feature, True))
+ open_decoder.return_value = decoder
+ decoder.get_frames_at.return_value = SimpleNamespace(data=batch[:1])
+ with self.assertRaisesRegex(ValueError, "one frame per index"):
+ _decode_video_windows(plans, rows, [collator], feature, True)
+
+ def test_visual_windows_preserve_order_padding_and_isolation(self):
+ try:
+ import torch
+ from torch.utils.data import default_collate
+ except ImportError as error:
+ self.skipTest(str(error))
+
+ keys = ["left", "right"]
+ plans = [{"windows": {key: indices for key in keys}}
+ for indices in ([1, 0, 0], [0, 1, 1], [1, 0, 0])]
+ for dtype in (torch.uint8, torch.float32):
+ frames = torch.arange(120).reshape(2, 4, 5, 3).to(dtype)
+ frames = frames.permute(0, 3, 1, 2)
+ rows = {i: {key: frames[i] for key in keys} for i in range(2)}
+ with patch("torch.get_num_threads", return_value=1):
+ actual = _stack_visual_windows(plans, rows, keys)
+ for key in keys:
+ expected = [torch.stack([rows[i][key] for i in
+ plan["windows"][key]]) for plan in plans]
+ self.assertTrue(torch.equal(
+ default_collate(actual[key]), default_collate(expected)))
+ self.assertTrue(all(value.is_contiguous()
+ for value in actual[key]))
+ actual[key][0][1].zero_()
+ self.assertTrue(torch.equal(actual[key][0][2], expected[0][2]))
+ self.assertTrue(torch.equal(actual[key][2], expected[2]))
+ self.assertTrue(torch.equal(frames[0], expected[0][1]))
+ with patch("torch.get_num_threads", return_value=2):
+ self.assertEqual({}, _stack_visual_windows(plans, rows, keys))
+ self.assertEqual({}, _stack_visual_windows(plans, rows, ["left"]))
+
+ rows = {i: {key: torch.ones(2, requires_grad=True) for key in keys}
+ for i in range(2)}
+ with patch("torch.get_num_threads", return_value=1), torch.no_grad():
+ result = _stack_visual_windows(plans, rows, keys)
+ self.assertFalse(result["left"][0].requires_grad)
+ with patch("torch.get_num_threads", return_value=1), \
+ torch.inference_mode():
+ result = _stack_visual_windows(plans, rows, keys)
+ self.assertTrue(result["left"][0].is_inference())
+
def test_dataset_return_uint8_requires_bool(self):
loaded = (
Mock(),
(paimon) 03/20: [python] Expand native planning across resolved tables and scan modes (#9873)
Posted by ho...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
XiaoHongbo-Hope pushed a commit to branch release-2.1
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit 8a429cfc363149b7d7078603c90fa8e730b664db
Author: Jingsong Lee <ji...@gmail.com>
AuthorDate: Wed Sep 16 13:07:08 2026 +0800
[python] Expand native planning across resolved tables and scan modes (#9873)
---
paimon-python/README.md | 35 +++-
paimon-python/pypaimon/common/json_util.py | 2 +
.../apply_deletion_vector_reader.py | 13 +-
paimon-python/pypaimon/read/native_plan.py | 84 +++++++++-
.../pypaimon/read/reader/format_pyarrow_reader.py | 3 +
.../read/scanner/chunk_shuffle_split_generator.py | 13 +-
paimon-python/pypaimon/read/split_read.py | 18 +-
paimon-python/pypaimon/read/table_scan.py | 151 +++++++++++------
paimon-python/pypaimon/table/file_store_table.py | 5 +-
.../tests/native_plan_chunk_shuffle_test.py | 172 +++++++++++++++++++
.../pypaimon/tests/native_plan_integration_test.py | 13 +-
.../tests/native_plan_materialized_pk_test.py | 186 +++++++++++++++++++++
.../tests/native_plan_resolved_schema_test.py | 184 ++++++++++++++++++++
.../pypaimon/tests/native_plan_rest_test.py | 151 +++++++++++++++++
paimon-python/pypaimon/tests/native_plan_test.py | 57 ++++++-
.../pypaimon/tests/reader_predicate_test.py | 78 +++++++++
.../pypaimon/tests/resolving_file_io_test.py | 21 +++
.../pypaimon/tests/schema_evolution_read_test.py | 53 +++++-
.../pypaimon/tests/snapshot_manager_test.py | 24 +++
19 files changed, 1177 insertions(+), 86 deletions(-)
diff --git a/paimon-python/README.md b/paimon-python/README.md
index 98dc4fc082..2af01d988c 100644
--- a/paimon-python/README.md
+++ b/paimon-python/README.md
@@ -50,6 +50,23 @@ Python planner for unsupported scans. New bindings preserve `plan.snapshot_id`
even when pruning removes every split. Native explain output includes snapshot
and split metadata; native pruning counters are not exposed.
+With Rust main's `Table.from_resolved_schema()` binding, filesystem and JDBC catalog
+tables preserve the Python table's resolved schema and complete effective
+options. Stale table objects, historical schemas, and `copy()` overrides or
+option removals no longer require catalog reloading or Python planning.
+Tables opened with `FileStoreTable.from_path(path, file_io_options=None)` use
+the same path with standard local, PyArrow or resolving FileIO. Storage options
+configure FileIO; use `copy()` for table read options.
+JDBC planning uses the resolved table location and storage properties without
+opening another database connection.
+REST tables use `Table.copy_with_resolved_schema()` to preserve the same schema
+and option semantics, including branches whose schemas are catalog-managed.
+The native table retains REST credentials, token refresh and catalog snapshot
+resolution. Database and table names containing dots are passed as separate
+identifier components. REST snapshot results (including empty results) take precedence over
+filesystem snapshots. REST errors, including HTTP 501, are propagated as in Java.
+Custom catalog/FileIO contexts still fall back when they cannot be reproduced.
+
Explicit row ranges on data-evolution tables require `ReadBuilder.with_row_ranges()`.
Watermark time travel requires Rust 0.4 or newer. Branch reads require the
branch-aware binding exposing `Table.branch()`, and the resolved branch is
@@ -90,14 +107,28 @@ bucket sharding. Cross-partition key migration is maintained by the writer's ind
Batch first-row scans follow Java and exclude un-compacted level-0 files; they can
use native planning. With deletion vectors, batch scans exclude level 0 unless
`deletion-vectors.merge-on-read=true`, in which case overlapping key ranges stay
-together for reader-side merging. Write scans and incremental scans retain level 0.
+together when they include L0 and require reader-side merging. Fully materialized
+DV files across levels use raw splits, including first-row clustering tables.
+First-row L0 runs can use native planning, including plans with materialized files
+in separate raw splits. Plans that require merging clustered materialized files
+still fall back to Python. Readers preserve physical row positions until deletion
+vectors are applied, then evaluate residual predicates after merging.
+Write scans and incremental scans retain level 0.
+
+Append and data-evolution chunk shuffle use Rust file and deletion-vector planning.
+Python retains live-row chunk sizing, seeded shuffle order and balanced worker
+assignment, so the same seed selects the same chunks with either planner.
+Projection does not remove aligned column files before chunk construction.
+Chunk shuffle supports partition predicates, deletion vectors and timestamp
+incremental scans; its existing restrictions on limits, slices, row ranges and
+global-index results still apply.
Scored global-index results on data-evolution append tables use native row-range
planning; Python attaches scores to the selected ranges and reads the data.
Primary-key sorted indexes refine native batch splits through Python's existing
index reader, preserving merge-required splits and the selected snapshot.
-Chunk shuffle, query authorization, batch first-row scans explicitly including L0,
+Query authorization, first-row plans mixing L0 with merge-required materialized files,
and precomputed primary-key global-index results still use the Python planner.
Continuous streaming and write planning also retain their Python entrypoints.
Native planning remains optional and is disabled by default.
diff --git a/paimon-python/pypaimon/common/json_util.py b/paimon-python/pypaimon/common/json_util.py
index 6effec7d1e..0ae8c232f9 100644
--- a/paimon-python/pypaimon/common/json_util.py
+++ b/paimon-python/pypaimon/common/json_util.py
@@ -148,6 +148,8 @@ class JSON:
kwargs[field_name] = (
None if value is None else decoder_mapping[json_name](value)
)
+ elif value is None:
+ kwargs[field_name] = None
elif json_name in type_mapping:
tp = getattr(type_mapping[json_name], '__origin__', None)
if tp in (list, List):
diff --git a/paimon-python/pypaimon/deletionvectors/apply_deletion_vector_reader.py b/paimon-python/pypaimon/deletionvectors/apply_deletion_vector_reader.py
index f491ecfa3f..237dae3775 100644
--- a/paimon-python/pypaimon/deletionvectors/apply_deletion_vector_reader.py
+++ b/paimon-python/pypaimon/deletionvectors/apply_deletion_vector_reader.py
@@ -75,6 +75,7 @@ class ApplyDeletionVectorReader(RecordBatchReader):
self._reader = reader
self._deletion_vector = deletion_vector
self._returned_position = 0
+ self._record_iterator = None
def reader(self) -> RecordReader:
return self._reader
@@ -108,12 +109,18 @@ class ApplyDeletionVectorReader(RecordBatchReader):
Returns:
A RecordIterator with deletion filtering, or None if no more data.
"""
+ if self._record_iterator is not None:
+ self._returned_position = self._record_iterator.returned_position() + 1
batch = self._reader.read_batch()
if batch is None:
return None
- return ApplyDeletionRecordIterator(batch, self._deletion_vector)
+ # Positions address the whole file, including rows deleted in earlier
+ # batches. Starting each iterator at zero would apply the DV repeatedly.
+ self._record_iterator = ApplyDeletionRecordIterator(
+ batch, self._deletion_vector, self._returned_position)
+ return self._record_iterator
def close(self):
self._reader.close()
@@ -129,6 +136,7 @@ class ApplyDeletionRecordIterator(RecordIterator):
self,
iterator: RecordIterator,
deletion_vector,
+ start_position: int = 0,
):
"""
Initialize an ApplyDeletionRecordIterator.
@@ -136,10 +144,11 @@ class ApplyDeletionRecordIterator(RecordIterator):
Args:
iterator: The underlying record iterator.
deletion_vector: The deletion vector to apply for filtering.
+ start_position: Position of the first record in this batch.
"""
self._iterator = iterator
self._deletion_vector = deletion_vector
- self._returned_position = -1
+ self._returned_position = start_position - 1
def iterator(self) -> RecordIterator:
return self._iterator
diff --git a/paimon-python/pypaimon/read/native_plan.py b/paimon-python/pypaimon/read/native_plan.py
index 253eeabb8f..ed12a98948 100644
--- a/paimon-python/pypaimon/read/native_plan.py
+++ b/paimon-python/pypaimon/read/native_plan.py
@@ -114,16 +114,23 @@ def _catalog_options(table) -> dict:
loader = getattr(getattr(table, 'catalog_environment', None), 'catalog_loader', None)
if loader is None:
raise ValueError("native_plan requires a catalog-backed table (no catalog loader)")
+ metastore = _catalog_metastore(loader)
+ if metastore is None:
+ raise ValueError("native_plan requires an exact built-in catalog loader")
+ normalized = _catalog_context_options(table)
+ normalized[CatalogOptions.METASTORE.key()] = metastore
+ return normalized
+
+
+def _catalog_context_options(table) -> dict:
+ """Normalize catalog storage properties without rebuilding the metastore."""
+ loader = table.catalog_environment.catalog_loader
options = loader.context().options.to_map()
normalized = {
str(key): _option_value_to_string(value)
for key, value in options.items()
if value is not None
}
- metastore = _catalog_metastore(loader)
- if metastore is None:
- raise ValueError("native_plan requires an exact built-in catalog loader")
- normalized[CatalogOptions.METASTORE.key()] = metastore
if str(getattr(table, 'table_path', '')).startswith('oss://'):
from pypaimon.filesystem.jindo_file_system_handler import (
JINDO_AVAILABLE,
@@ -169,6 +176,47 @@ def _read_options(table) -> dict:
return options
+def _resolved_schema_file_io_options(table) -> Optional[dict]:
+ """FileIO properties for tables whose metadata needs no catalog resolution."""
+ if not native_method_available('Table', 'from_resolved_schema'):
+ return None
+ environment = table.catalog_environment
+ loader = environment.catalog_loader
+ if loader is None:
+ from pypaimon.catalog.catalog_environment import CatalogEnvironment
+ from pypaimon.filesystem.local_file_io import LocalFileIO
+ from pypaimon.filesystem.pyarrow_file_io import PyArrowFileIO
+ from pypaimon.filesystem.resolving_file_io import ResolvingFileIO
+ # A custom environment or FileIO can supply metadata outside the path.
+ if (type(environment) is not CatalogEnvironment
+ or type(table.file_io) not in (LocalFileIO, PyArrowFileIO, ResolvingFileIO)):
+ return None
+ return {str(key): _option_value_to_string(value)
+ for key, value in table.file_io.properties.to_map().items()
+ if value is not None}
+ from pypaimon.catalog.jdbc_catalog_loader import JdbcCatalogLoader
+ if _catalog_metastore(loader) != 'filesystem' and type(loader) is not JdbcCatalogLoader:
+ # REST tables must retain catalog snapshot loading and token refresh.
+ return None
+ context = loader.context()
+ if context.options is None or any(getattr(context, attr, None) is not None for attr in (
+ 'hadoop_conf', 'prefer_io_loader', 'fallback_io_loader')):
+ return None
+ # JDBC, like filesystem catalogs, uses on-disk snapshots. Its already
+ # resolved table does not need another database connection during planning.
+ return _catalog_context_options(table)
+
+
+def _resolved_schema_json(table) -> str:
+ from pypaimon.common.json_util import JSON
+ options = {str(key): _option_value_to_string(value)
+ for key, value in table.table_schema.options.items() if value is not None}
+ options.update(_read_options(table))
+ # The timestamp string has already been converted to epoch millis.
+ options.pop(CoreOptions.SCAN_TIMESTAMP.key(), None)
+ return JSON.to_json(table.table_schema.copy(new_options=options))
+
+
def _predicate_to_native(predicate: Predicate) -> dict:
"""Convert PyPaimon's predicate tree to pypaimon-rust's dict API."""
if predicate.method in ('and', 'or'):
@@ -231,14 +279,34 @@ def native_plan(
if not native_runtime_available():
raise RuntimeError(
"scan.native-plan.enabled needs pypaimon-rust>=0.3.0 (split planning API)")
- from pypaimon_rust.datafusion import PaimonCatalog
-
- rt = PaimonCatalog(_catalog_options(table)).get_table(table.identifier.get_full_name())
+ file_io_options = _resolved_schema_file_io_options(table)
+ if file_io_options is not None:
+ from pypaimon_rust.datafusion import Table
+ rt = Table.from_resolved_schema(
+ table.table_path, _resolved_schema_json(table),
+ database=table.identifier.get_database_name(),
+ table=table.identifier.get_table_name(),
+ branch=table.current_branch(), options=file_io_options)
+ builder = rt.new_read_builder()
+ else:
+ from pypaimon_rust.datafusion import PaimonCatalog
+ catalog = PaimonCatalog(_catalog_options(table))
+ if native_method_available('Table', 'copy_with_resolved_schema'):
+ # REST may keep branch schemas in the catalog only. Load the base
+ # environment, then attach the schema/branch already resolved here.
+ rt = catalog.get_table((
+ table.identifier.get_database_name(), table.identifier.get_table_name()))
+ if rt.location() != table.table_path:
+ raise RuntimeError('Native catalog resolved a different table location')
+ rt = rt.copy_with_resolved_schema(_resolved_schema_json(table), branch=table.current_branch())
+ builder = rt.new_read_builder()
+ else:
+ rt = catalog.get_table(table.identifier.get_full_name())
+ builder = rt.new_read_builder(_read_options(table))
if table.current_branch() != 'main':
branch = getattr(rt, 'branch', None)
if not callable(branch) or branch() != table.current_branch():
raise RuntimeError("Native table did not resolve the requested branch")
- builder = rt.new_read_builder(_read_options(table))
if projection is not None:
builder = builder.with_projection(projection)
if predicate is not None:
diff --git a/paimon-python/pypaimon/read/reader/format_pyarrow_reader.py b/paimon-python/pypaimon/read/reader/format_pyarrow_reader.py
index 7ce7ec1d7a..33999fec75 100644
--- a/paimon-python/pypaimon/read/reader/format_pyarrow_reader.py
+++ b/paimon-python/pypaimon/read/reader/format_pyarrow_reader.py
@@ -341,6 +341,9 @@ class FormatPyArrowReader(RecordBatchReader):
row_indices: Optional[List[int]] = None,
row_ranges: Optional[List[Tuple[int, int]]] = None,
row_group_cache: Optional[_DecodedRowGroupCache] = None):
+ from pypaimon.filesystem.resolving_file_io import ResolvingFileIO
+ if isinstance(file_io, ResolvingFileIO):
+ file_io = file_io._get_fileio(file_path)
self._predicate_field_names = predicate_field_names or set()
file_path_for_pyarrow = file_io.to_filesystem_path(file_path)
self._row_group_cache = row_group_cache
diff --git a/paimon-python/pypaimon/read/scanner/chunk_shuffle_split_generator.py b/paimon-python/pypaimon/read/scanner/chunk_shuffle_split_generator.py
index 89216322cf..33881bafcb 100644
--- a/paimon-python/pypaimon/read/scanner/chunk_shuffle_split_generator.py
+++ b/paimon-python/pypaimon/read/scanner/chunk_shuffle_split_generator.py
@@ -230,12 +230,13 @@ class ChunkShuffleSplitGeneratorBase(AbstractSplitGenerator):
if f.file_name in seen_paths:
continue
seen_paths.add(f.file_name)
- f.set_file_path(
- self.table.table_path,
- partition_row,
- bucket,
- self.default_part_value,
- )
+ if not f.file_path:
+ f.set_file_path(
+ self.table.table_path,
+ partition_row,
+ bucket,
+ self.default_part_value,
+ )
for segments in self._slice_group_into_chunks(entries_in_group):
all_chunks.append(_Chunk(partition_row, bucket, segments))
diff --git a/paimon-python/pypaimon/read/split_read.py b/paimon-python/pypaimon/read/split_read.py
index ec1ee56689..4f1fbd64bf 100644
--- a/paimon-python/pypaimon/read/split_read.py
+++ b/paimon-python/pypaimon/read/split_read.py
@@ -240,6 +240,13 @@ class SplitRead(ABC):
read_arrow_predicate,
read_paimon_predicate,
) = self._get_fields_and_predicate(file.schema_id, read_fields)
+ if (file.file_name in self.deletion_file_readers
+ or (for_merge_read and self.row_ranges is not None)):
+ # DVs and indexed PK ranges refer to physical file positions.
+ # Filtering or skipping row groups here would renumber those rows.
+ # Apply the residual predicate after position selection and merging.
+ read_arrow_predicate = None
+ read_paimon_predicate = None
# Use external_path if available, otherwise use file_path
file_path = file.external_path if file.external_path else file.file_path
@@ -583,7 +590,11 @@ class SplitRead(ABC):
read_field for read_field in read_fields
if _is_reachable(read_field)
]
- read_predicate = trim_predicate_by_fields(self.push_down_predicate, read_file_fields)
+ # File readers filter physical column names before field-id schema
+ # normalization. A renamed or re-added name can identify a different
+ # column, so cross-schema filtering must run after normalization.
+ read_predicate = (trim_predicate_by_fields(self.push_down_predicate, read_file_fields)
+ if schema_id == self.table.table_schema.id else None)
read_arrow_predicate = (
read_predicate.to_arrow()
if read_predicate and self._arrow_filter_pushdown_enabled
@@ -899,7 +910,10 @@ class RawFileSplitRead(SplitRead):
reader = concat_reader
if (self.predicate_for_reader
and (self.table.is_primary_key_table
- or not self._arrow_filter_pushdown_enabled)):
+ or not self._arrow_filter_pushdown_enabled
+ or self.deletion_file_readers
+ or any(file.schema_id != self.table.table_schema.id
+ for file in self.split.files))):
reader = FilterRecordBatchReader(
reader,
self.predicate_for_reader,
diff --git a/paimon-python/pypaimon/read/table_scan.py b/paimon-python/pypaimon/read/table_scan.py
index 7e4889da51..74a5be1e6b 100755
--- a/paimon-python/pypaimon/read/table_scan.py
+++ b/paimon-python/pypaimon/read/table_scan.py
@@ -115,25 +115,26 @@ class TableScan:
def _native_plan_supported_impl(self) -> bool:
"""Fall back to the Python scanner for scans native can't carry:
- chunk-shuffle, primary-key global-index results, first-row scans which include L0,
+ primary-key global-index results,
a primary-key table whose trimmed PK is empty (PK equals the partition
- key; Rust rejects this schema), a stale
- schema without time travel, removed copy() options which Rust cannot
- represent, unsupported time travel selectors,
+ key; Rust rejects this schema), unsupported time travel selectors,
+ and catalog-loaded tables with schema/option overrides Rust cannot carry,
query auth, a missing/old
pypaimon-rust, or a catalog / identifier Rust cannot reconstruct. Keep
this capability gate in sync when adding scan features."""
from pypaimon.read.native_plan import (
- native_method_available, native_runtime_available, native_version_at_least,
+ _resolved_schema_file_io_options, native_method_available,
+ native_runtime_available, native_version_at_least,
)
if not native_runtime_available():
return False
fs = self.file_scanner
if fs.is_streaming and not native_method_available('Split', 'is_streaming'):
return False
- if (getattr(fs, 'chunk_shuffle', None) is not None
- or not self._native_global_index_result_supported()):
+ if not self._native_global_index_result_supported():
return False
+ if getattr(fs, 'chunk_shuffle', None) is not None:
+ fs._validate_chunk_shuffle_compat()
# Positional append distribution needs the stable partition/file order
# introduced in 0.4. Older bindings can assign different rows per call.
if (not self.table.is_primary_key_table and not fs.data_evolution
@@ -157,35 +158,34 @@ class TableScan:
if (self.table.current_branch() != 'main'
and not native_method_available('Table', 'branch')):
return False
- loader = getattr(
- getattr(self.table, 'catalog_environment', None),
- 'catalog_loader',
- None,
- )
- context_fn = getattr(loader, 'context', None)
- if not callable(context_fn):
- return False
- from pypaimon.read.native_plan import _catalog_metastore
- if _catalog_metastore(loader) is None:
- return False
- context = context_fn()
- catalog_options = getattr(context, 'options', None)
- if catalog_options is None:
- return False
- if any(getattr(context, attr, None) is not None for attr in (
- 'hadoop_conf', 'prefer_io_loader', 'fallback_io_loader')):
- return False
- database_name = self.table.identifier.get_database_name()
- if not database_name or database_name == UNKNOWN_DATABASE or '.' in database_name:
- return False
+ resolved_schema = _resolved_schema_file_io_options(self.table) is not None
+ if not resolved_schema:
+ loader = getattr(
+ getattr(self.table, 'catalog_environment', None),
+ 'catalog_loader',
+ None,
+ )
+ context_fn = getattr(loader, 'context', None)
+ if not callable(context_fn):
+ return False
+ from pypaimon.read.native_plan import _catalog_metastore
+ if _catalog_metastore(loader) is None:
+ return False
+ context = context_fn()
+ catalog_options = getattr(context, 'options', None)
+ if catalog_options is None:
+ return False
+ if any(getattr(context, attr, None) is not None for attr in (
+ 'hadoop_conf', 'prefer_io_loader', 'fallback_io_loader')):
+ return False
+ database_name = self.table.identifier.get_database_name()
+ if not database_name or database_name == UNKNOWN_DATABASE:
+ return False
+ if ('.' in database_name
+ and not native_method_available('Table', 'copy_with_resolved_schema')):
+ return False
if self.table.options.query_auth_enabled:
return False
- # Ordinary Java first-row batch scans skip L0. The clustering override
- # can combine first-row with DV merge-on-read, but its files need a
- # separate audit because they may be sorted by non-primary-key columns.
- if (self.table.options.merge_engine() == 'first-row'
- and not fs.skip_level0 and not fs.is_streaming):
- return False
# Rust rejects schemas whose primary keys are all partition keys.
if getattr(self.table, 'is_primary_key_table', False) \
and not self.table.trimmed_primary_keys:
@@ -199,23 +199,24 @@ class TableScan:
from pypaimon.read.native_plan import native_family_search_modes_available
if not native_family_search_modes_available():
return False
- supported_time_travel = any(
- options.contains_key(key) for key in _NATIVE_TIME_TRAVEL_OPTIONS)
- # Time travel intentionally carries a historical schema; other stale
- # table objects must still fall back because Rust reloads the latest.
- latest_schema = self.table.schema_manager.latest()
- if (not supported_time_travel and latest_schema is not None
- and latest_schema.id != self.table.table_schema.id):
- return False
- # Rust cannot remove an option persisted in the catalog-loaded schema.
- applied_options = getattr(self.table, '_applied_dynamic_options', {}) or {}
- allowed_options = (
- _NATIVE_FORWARDED_OPTIONS | _NATIVE_PLAN_INDEPENDENT_OPTIONS)
- if (set(applied_options) - allowed_options
- or any(key in (_NATIVE_TIME_TRAVEL_OPTIONS
- | _NATIVE_SEARCH_MODE_OPTIONS) and value is None
- for key, value in applied_options.items())):
- return False
+ if not resolved_schema and not native_method_available('Table', 'copy_with_resolved_schema'):
+ supported_time_travel = any(
+ options.contains_key(key) for key in _NATIVE_TIME_TRAVEL_OPTIONS)
+ # Time travel intentionally carries a historical schema; other stale
+ # table objects must still fall back because Rust reloads the latest.
+ latest_schema = self.table.schema_manager.latest()
+ if (not supported_time_travel and latest_schema is not None
+ and latest_schema.id != self.table.table_schema.id):
+ return False
+ # Rust cannot remove an option persisted in the catalog-loaded schema.
+ applied_options = getattr(self.table, '_applied_dynamic_options', {}) or {}
+ allowed_options = (
+ _NATIVE_FORWARDED_OPTIONS | _NATIVE_PLAN_INDEPENDENT_OPTIONS)
+ if (set(applied_options) - allowed_options
+ or any(key in (_NATIVE_TIME_TRAVEL_OPTIONS
+ | _NATIVE_SEARCH_MODE_OPTIONS) and value is None
+ for key, value in applied_options.items())):
+ return False
from pypaimon.snapshot.time_travel_util import SCAN_KEYS
unsupported_scan_keys = set(SCAN_KEYS) - _NATIVE_TIME_TRAVEL_OPTIONS
if any(options.contains_key(k) for k in unsupported_scan_keys):
@@ -261,7 +262,8 @@ class TableScan:
if self._incremental_snapshot_range is None:
return Plan([])
extra_options['incremental_range'] = self._incremental_snapshot_range
- if has_distribution and fs.data_evolution:
+ chunk_shuffle = fs.chunk_shuffle
+ if has_distribution and fs.data_evolution and chunk_shuffle is None:
if fs.idx_of_this_subtask is not None:
extra_options['row_position_shard'] = (
fs.idx_of_this_subtask, fs.number_of_para_subtasks)
@@ -283,11 +285,20 @@ class TableScan:
limit=None if has_distribution else self.limit,
projection=(
[field.name for field in self._read_type]
- if self._read_type is not None else None),
+ if self._read_type is not None and chunk_shuffle is None else None),
row_ranges=row_ranges,
**extra_options,
)
splits = plan.splits()
+ if (self.table.options.merge_engine() == 'first-row'
+ and not fs.skip_level0 and not fs.is_streaming
+ and any(file.level == 0 for split in splits for file in split.files)
+ and any(not split.raw_convertible and any(file.level > 0 for file in split.files)
+ for split in splits)):
+ # L0 runs are sorted by PK and can merge using first-row.
+ # Materialized clustered files must stay in raw splits: their
+ # physical order need not match the PK merge comparator.
+ return None
partition_predicate = self.file_scanner.partition_key_predicate
if partition_predicate is not None:
splits = [s for s in splits
@@ -298,7 +309,9 @@ class TableScan:
and plan.snapshot_id is not None):
snapshot = self.table.snapshot_manager().get_snapshot_by_id(plan.snapshot_id)
splits = fs._apply_primary_key_sorted_indexes(splits, snapshot)
- if has_distribution:
+ if chunk_shuffle is not None:
+ splits = self._chunk_shuffle_splits(splits, plan.snapshot_id)
+ elif has_distribution:
if self.table.is_primary_key_table:
splits = [s for s in splits
if s.bucket % fs.number_of_para_subtasks == fs.idx_of_this_subtask]
@@ -328,6 +341,36 @@ class TableScan:
"Native plan failed, falling back to the Python scanner: %s", e)
return None
+ def _chunk_shuffle_splits(self, splits, snapshot_id):
+ """Reuse native file/DV planning with Python's stable chunk assignment."""
+ from pypaimon.manifest.schema.manifest_entry import ManifestEntry
+ from pypaimon.read.scanner.chunk_shuffle_split_generator import (
+ AppendChunkShuffleSplitGenerator, DataEvolutionChunkShuffleSplitGenerator,
+ )
+ fs = self.file_scanner
+ entries, deletions = [], {}
+ for split in splits:
+ key = (tuple(split.partition.values), split.bucket)
+ for index, file in enumerate(split.files):
+ entries.append(ManifestEntry(
+ 0, split.partition, split.bucket, self.table.total_buckets, file))
+ if split.data_deletion_files and split.data_deletion_files[index] is not None:
+ deletions.setdefault(key, {})[file.file_name] = split.data_deletion_files[index]
+ generator_type = (DataEvolutionChunkShuffleSplitGenerator if fs.data_evolution
+ else AppendChunkShuffleSplitGenerator)
+ seed, chunk_size = fs.chunk_shuffle
+ generator = generator_type(self.table, fs.target_split_size, fs.open_file_cost,
+ deletions, seed=seed, chunk_size=chunk_size)
+ if fs.idx_of_this_subtask is not None:
+ generator.with_shard(fs.idx_of_this_subtask, fs.number_of_para_subtasks)
+ chunks = generator.create_splits(entries)
+ for split in chunks:
+ while callable(getattr(split, 'data_split', None)):
+ split = split.data_split()
+ split.snapshot_id = snapshot_id
+ split.is_streaming = fs.is_streaming
+ return chunks
+
def plan_for_write(self) -> Plan:
if self.__auth_query() is not None:
raise TableNoPermissionException(self.table.identifier)
diff --git a/paimon-python/pypaimon/table/file_store_table.py b/paimon-python/pypaimon/table/file_store_table.py
index 5519ee087d..c2962236b9 100644
--- a/paimon-python/pypaimon/table/file_store_table.py
+++ b/paimon-python/pypaimon/table/file_store_table.py
@@ -64,12 +64,13 @@ class FileStoreTable(Table):
file_io, table_path, branch=self.current_branch())
@classmethod
- def from_path(cls, table_path: str) -> 'FileStoreTable':
+ def from_path(cls, table_path: str, file_io_options: Optional[dict] = None) -> 'FileStoreTable':
"""
Create a FileStoreTable from a table path.
This is useful for reading tables created by Java without going through a catalog.
+ ``file_io_options`` configures storage access; use ``copy`` for table read options.
"""
- file_io = FileIO(table_path, Options({}))
+ file_io = FileIO.get(table_path, Options(file_io_options or {}))
schema_manager = SchemaManager(file_io, table_path)
table_schema = schema_manager.latest()
diff --git a/paimon-python/pypaimon/tests/native_plan_chunk_shuffle_test.py b/paimon-python/pypaimon/tests/native_plan_chunk_shuffle_test.py
new file mode 100644
index 0000000000..c2755aca17
--- /dev/null
+++ b/paimon-python/pypaimon/tests/native_plan_chunk_shuffle_test.py
@@ -0,0 +1,172 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import json
+from collections import Counter
+from unittest.mock import patch
+
+import pyarrow as pa
+import pytest
+
+from pypaimon import CatalogFactory, Schema
+from pypaimon.deletionvectors.bitmap_deletion_vector import BitmapDeletionVector
+from pypaimon.read.native_plan import native_runtime_available
+from pypaimon.write.commit_message import CommitMessage
+from pypaimon.write.table_delete import TableDeleteByRowId
+
+
+pytestmark = [pytest.mark.native_plan, pytest.mark.skipif(
+ not native_runtime_available(), reason='Rust planner required')]
+
+
+@pytest.fixture(params=['append', 'append-dv', 'de', 'de-dv'])
+def chunk_table(request, tmp_path):
+ de = request.param.startswith('de')
+ dv = request.param.endswith('dv')
+ fields = [('id', pa.int64()), ('p', pa.string())]
+ options = {'file.format': 'parquet', 'source.split.target-size': '1b',
+ 'source.split.open-file-cost': '1b'}
+ if de:
+ fields.append(('payload', pa.large_binary()))
+ options.update({'data-evolution.enabled': 'true', 'row-tracking.enabled': 'true',
+ 'blob.target-file-size': '1b'})
+ if dv:
+ options['deletion-vectors.enabled'] = 'true'
+ schema = pa.schema(fields)
+ catalog = CatalogFactory.create({'warehouse': str(tmp_path)})
+ catalog.create_database('default', True)
+ catalog.create_table('default.t', Schema.from_pyarrow_schema(
+ schema, partition_keys=['p'], options=options), False)
+ table = catalog.get_table('default.t')
+ expected = []
+ for start in (0, 6):
+ rows = [{'id': i, 'p': (None, 'a', 'b')[i % 3]} for i in range(start, start + 6)]
+ if de:
+ for row in rows:
+ row['payload'] = ('payload-%d' % row['id']).encode()
+ expected.extend(rows)
+ builder = table.new_batch_write_builder()
+ writer, commit = builder.new_write(), builder.new_commit()
+ try:
+ writer.write_arrow(pa.Table.from_pylist(rows, schema=schema))
+ commit.commit(writer.prepare_commit())
+ finally:
+ writer.close()
+ commit.close()
+ for snapshot_id in (1, 2):
+ path = table.snapshot_manager().get_snapshot_path(snapshot_id)
+ snapshot = json.loads(table.file_io.read_file_utf8(path))
+ snapshot['timeMillis'] = snapshot_id * 100
+ table.file_io.write_file(path, json.dumps(snapshot), overwrite=True)
+ before_deletes = list(expected)
+ if dv:
+ builder = table.copy({'scan.native-plan.enabled': 'false'}).new_read_builder()
+ plan = builder.new_scan().plan()
+ index_adds = []
+ for split in plan.splits():
+ # Delete the first physical row in every main file. Sidecar files
+ # inherit the anchor's DV through their aligned row-id group.
+ for file in split.files:
+ if not file.file_name.endswith('.parquet'):
+ continue
+ from pypaimon.read.split import DataSplit
+ one = DataSplit(files=[file], partition=split.partition, bucket=split.bucket,
+ raw_convertible=not de)
+ row = builder.with_projection(['id']).new_read().to_arrow([one]).to_pylist()[0]
+ expected = [candidate for candidate in expected if candidate['id'] != row['id']]
+ vector = BitmapDeletionVector()
+ vector.delete(0)
+ index_adds.append((tuple(split.partition.values), split.bucket,
+ TableDeleteByRowId(table)._write_deletion_vector_index(
+ split.partition, split.bucket, {file.file_name: vector})))
+ commit = table.new_batch_write_builder().new_commit()
+ try:
+ commit.commit([CommitMessage(partition=partition, bucket=bucket, new_files=[],
+ index_adds=[entry]) for partition, bucket, entry in index_adds])
+ finally:
+ commit.close()
+ return table, expected, before_deletes
+
+
+def _chunks(table, seed, chunk_size=3, shard=None, predicate=None, projection=None):
+ results = []
+ for native in (False, True):
+ builder = table.copy({'scan.native-plan.enabled': str(native).lower()}).new_read_builder()
+ if predicate is not None:
+ builder.with_filter(predicate)
+ if projection is not None:
+ builder.with_projection(projection)
+ scan = builder.new_scan().with_chunk_shuffle(seed, chunk_size)
+ if shard is not None:
+ scan.with_shard(*shard)
+ if native:
+ with patch.object(scan.file_scanner, 'scan', side_effect=AssertionError('native fallback')), \
+ patch.object(scan.file_scanner, 'plan_files', side_effect=AssertionError('Python manifest scan')):
+ plan = scan.plan()
+ else:
+ plan = scan.plan()
+ chunks = []
+ for split in plan.splits():
+ if table.options.options.contains_key('incremental-between-timestamp'):
+ assert split.is_streaming
+ rows = builder.new_read().to_arrow([split]).to_pylist()
+ assert 0 < len(rows) <= chunk_size
+ assert split.merged_row_count() == len(rows)
+ if rows and 'p' in rows[0]:
+ assert len({row['p'] for row in rows}) == 1
+ chunks.append(sorted(rows, key=lambda row: row['id']))
+ results.append((plan.snapshot_id, chunks))
+ assert results[0] == results[1]
+ return results[1]
+
+
+@pytest.mark.parametrize('seed', [-11, 42, 2 ** 70])
+def test_native_chunks_preserve_order_live_counts_and_worker_assignment(chunk_table, seed):
+ table, expected, _ = chunk_table
+ snapshot_id, chunks = _chunks(table, seed)
+ actual = [row for chunk in chunks for row in chunk]
+ assert sorted(actual, key=lambda row: row['id']) == expected
+ workers = []
+ for worker in range(5):
+ sid, assigned = _chunks(table, seed, shard=(worker, 5))
+ assert sid == snapshot_id
+ workers.extend(assigned)
+ assert workers == chunks
+ assert Counter(row['id'] for chunk in workers for row in chunk) == Counter(row['id'] for row in expected)
+
+
+def test_native_chunks_keep_partition_filter_projection_and_time_travel(chunk_table):
+ table, expected, before_deletes = chunk_table
+ pb = table.new_read_builder().new_predicate_builder()
+ for predicate, partition in ((pb.equal('p', 'a'), 'a'), (pb.is_null('p'), None)):
+ _, chunks = _chunks(table, 7, predicate=predicate, projection=['id'])
+ assert sorted(row['id'] for chunk in chunks for row in chunk) == [
+ row['id'] for row in expected if row['p'] == partition]
+ sid, chunks = _chunks(table.copy({'scan.snapshot-id': '2'}), 7)
+ assert sid == 2
+ assert sorted([row for chunk in chunks for row in chunk], key=lambda row: row['id']) == before_deletes
+ sid, chunks = _chunks(table, 7, predicate=pb.equal('p', 'missing'))
+ assert sid == table.snapshot_manager().get_latest_snapshot().id
+ assert chunks == []
+
+
+def test_native_incremental_chunks_keep_events_and_ignore_later_deletions(chunk_table):
+ table, _, before_deletes = chunk_table
+ incremental = table.copy({'incremental-between-timestamp': '0,200'})
+ sid, chunks = _chunks(incremental, 42)
+ assert sid == 2
+ assert sorted([row for chunk in chunks for row in chunk], key=lambda row: row['id']) == before_deletes
diff --git a/paimon-python/pypaimon/tests/native_plan_integration_test.py b/paimon-python/pypaimon/tests/native_plan_integration_test.py
index 8f1dcd257c..10cc5d3471 100644
--- a/paimon-python/pypaimon/tests/native_plan_integration_test.py
+++ b/paimon-python/pypaimon/tests/native_plan_integration_test.py
@@ -112,15 +112,22 @@ class NativePlanIntegrationTest(unittest.TestCase):
rb = native_table.new_read_builder()
rb.new_read().to_arrow(rb.new_scan().plan().splits())
- def test_copy_removed_persisted_scan_option_falls_back(self):
- # copy() removes a persisted scan.snapshot-id that Rust would still reload -> fall back.
+ def test_copy_removed_persisted_scan_option_uses_native(self):
+ # The resolved schema must replace, rather than merge, persisted options.
self.cat.create_table('default.snapopt_t', Schema.from_pyarrow_schema(
self.schema, options={'scan.snapshot-id': '1'}), False)
self._write('snapopt_t', [{'k': 1, 'v': 'a'}]) # snapshot 1
self._write('snapopt_t', [{'k': 2, 'v': 'b'}]) # snapshot 2
native = self.cat.get_table('default.snapopt_t').copy(
{'scan.snapshot-id': None, 'scan.native-plan.enabled': 'true'})
- self.assertFalse(native.new_read_builder().explain().native_planned)
+ builder = native.new_read_builder()
+ scan = builder.new_scan()
+ with patch.object(scan.file_scanner, 'scan', side_effect=AssertionError('native fallback')):
+ plan = scan.plan()
+ self.assertEqual(plan.snapshot_id, 2)
+ self.assertEqual(sorted(builder.new_read().to_arrow(plan.splits()).to_pylist(),
+ key=lambda row: row['k']),
+ [{'k': 1, 'v': 'a'}, {'k': 2, 'v': 'b'}])
def test_first_row_batch_scan_uses_native_plan(self):
self.cat.create_table('default.fr_t', Schema.from_pyarrow_schema(
diff --git a/paimon-python/pypaimon/tests/native_plan_materialized_pk_test.py b/paimon-python/pypaimon/tests/native_plan_materialized_pk_test.py
new file mode 100644
index 0000000000..6e9471088f
--- /dev/null
+++ b/paimon-python/pypaimon/tests/native_plan_materialized_pk_test.py
@@ -0,0 +1,186 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from dataclasses import replace
+from pathlib import Path
+from unittest.mock import patch
+
+import pyarrow as pa
+import pyarrow.parquet as pq
+import pytest
+
+from pypaimon import CatalogFactory, Schema
+from pypaimon.deletionvectors.bitmap_deletion_vector import BitmapDeletionVector
+from pypaimon.read.native_plan import native_runtime_available
+from pypaimon.table.row.generic_row import GenericRow
+from pypaimon.write.commit_message import CommitMessage
+from pypaimon.write.table_delete import TableDeleteByRowId
+
+
+pytestmark = [pytest.mark.native_plan, pytest.mark.skipif(
+ not native_runtime_available(), reason='Rust planner required')]
+
+
+@pytest.mark.parametrize('engine', ['deduplicate', 'first-row'])
+@pytest.mark.parametrize('merge_on_read', ['false', 'true'])
+@pytest.mark.parametrize('target_size', ['1b', '1mb'])
+def test_clustered_materialized_dv_files_use_native_raw_splits(tmp_path, engine, merge_on_read, target_size):
+ schema = pa.schema([('id', pa.int64()), ('value', pa.string())])
+ catalog = CatalogFactory.create({'warehouse': str(tmp_path)})
+ catalog.create_database('default', True)
+ catalog.create_table('default.t', Schema.from_pyarrow_schema(
+ schema, primary_keys=['id'], options={
+ 'bucket': '1', 'merge-engine': engine, 'file.format': 'parquet',
+ 'deletion-vectors.enabled': 'true', 'deletion-vectors.merge-on-read': merge_on_read,
+ 'pk-clustering-override': 'true', 'clustering.columns': 'value',
+ 'source.split.target-size': target_size, 'source.split.open-file-cost': '1b',
+ }), False)
+ table = catalog.get_table('default.t')
+ expected = [{'id': i, 'value': 'v%d' % (10 - i)} for i in range(1, 5)]
+ files = []
+ for level, keys in ((1, (1, 3)), (2, (2, 4))):
+ builder = table.new_batch_write_builder()
+ writer, commit = builder.new_write(), builder.new_commit()
+ try:
+ writer.write_arrow(pa.Table.from_pylist([expected[key - 1] for key in keys], schema=schema))
+ messages = writer.prepare_commit()
+ for message in messages:
+ rewritten = []
+ for file in message.new_files:
+ path = table.path_factory().bucket_path((), 0) + '/' + file.file_name
+ # Simulate Java clustering compaction: physical order is by
+ # value, opposite to PK order, while min/max PK ranges overlap.
+ data = pq.read_table(path).sort_by([('value', 'ascending')])
+ pq.write_table(data, path)
+ rewritten.append(replace(file, level=level,
+ file_size=Path(path).stat().st_size))
+ message.new_files = rewritten
+ files.extend(rewritten)
+ commit.commit(messages)
+ finally:
+ writer.close()
+ commit.close()
+
+ vector = BitmapDeletionVector()
+ vector.delete(0) # clustered first file starts with id=3
+ entry = TableDeleteByRowId(table)._write_deletion_vector_index(
+ GenericRow([], []), 0, {files[0].file_name: vector})
+ commit = table.new_batch_write_builder().new_commit()
+ try:
+ commit.commit([CommitMessage(partition=(), bucket=0, new_files=[], index_adds=[entry])])
+ finally:
+ commit.close()
+ expected = [row for row in expected if row['id'] != 3]
+ pb = table.new_read_builder().new_predicate_builder()
+ for predicate in (None, pb.equal('id', 3), pb.equal('value', 'v8')):
+ for native in (False, True):
+ builder = table.copy({'scan.native-plan.enabled': str(native).lower()}).new_read_builder()
+ if predicate is not None:
+ builder.with_filter(predicate)
+ scan = builder.new_scan()
+ if native:
+ with patch.object(scan.file_scanner, 'scan', side_effect=AssertionError('native fallback')):
+ plan = scan.plan()
+ else:
+ plan = scan.plan()
+ assert all(split.raw_convertible for split in plan.splits())
+ rows = builder.new_read().to_arrow(plan.splits()).to_pylist()
+ wanted = expected if predicate is None else ([] if predicate.field == 'id' else [expected[1]])
+ assert sorted(rows, key=lambda row: row['id']) == wanted
+ assert plan.snapshot_id == 3
+ if predicate is None:
+ assert len(plan.splits()) == (2 if target_size == '1b' else 1)
+
+ if engine == 'first-row' and merge_on_read == 'true':
+ builder = table.new_batch_write_builder()
+ writer, commit = builder.new_write(), builder.new_commit()
+ try:
+ writer.write_arrow(pa.Table.from_pylist([{'id': 5, 'value': 'pending'}], schema=schema))
+ commit.commit(writer.prepare_commit())
+ finally:
+ writer.close()
+ commit.close()
+ scan = table.copy({'scan.native-plan.enabled': 'true'}).new_read_builder().new_scan()
+ with patch.object(scan.file_scanner, 'scan', wraps=scan.file_scanner.scan) as fallback:
+ plan = scan.plan()
+ fallback.assert_called_once_with()
+ assert any(file.level == 0 for split in plan.splits() for file in split.files)
+
+
+@pytest.mark.parametrize('target_size', ['1b', '1mb'])
+@pytest.mark.parametrize('compacted_partition', [False, True])
+@pytest.mark.parametrize('batch_size', [1, 1024])
+def test_first_row_level_zero_merges_before_filtering(tmp_path, target_size, compacted_partition, batch_size):
+ catalog = CatalogFactory.create({'warehouse': str(tmp_path)})
+ catalog.create_database('default', True)
+ catalog.create_table('default.t', Schema.from_pyarrow_schema(
+ pa.schema([('id', pa.int64()), ('value', pa.string()), ('dt', pa.string())]),
+ primary_keys=['id', 'dt'], partition_keys=['dt'], options={
+ 'bucket': '1', 'merge-engine': 'first-row', 'file.format': 'parquet',
+ 'deletion-vectors.enabled': 'true', 'deletion-vectors.merge-on-read': 'true',
+ 'pk-clustering-override': 'true', 'clustering.columns': 'value',
+ 'source.split.target-size': target_size, 'source.split.open-file-cost': '1b',
+ 'read.batch-size': str(batch_size),
+ }), False)
+ table = catalog.get_table('default.t')
+ batches = [(0, [{'id': 1, 'value': 'first', 'dt': 'pending'},
+ {'id': 2, 'value': 'deleted', 'dt': 'pending'}]),
+ (0, [{'id': 1, 'value': 'later', 'dt': 'pending'},
+ {'id': 3, 'value': 'third', 'dt': 'pending'}])]
+ if compacted_partition:
+ batches.append((1, [{'id': 4, 'value': 'compacted', 'dt': 'ready'}]))
+ first_file = None
+ for level, rows in batches:
+ builder = table.new_batch_write_builder()
+ writer, commit = builder.new_write(), builder.new_commit()
+ try:
+ writer.write_arrow(pa.Table.from_pylist(rows))
+ messages = writer.prepare_commit()
+ for message in messages:
+ message.new_files = [replace(file, level=level) for file in message.new_files]
+ if first_file is None:
+ first_file = message.new_files[0]
+ commit.commit(messages)
+ finally:
+ writer.close()
+ commit.close()
+ vector = BitmapDeletionVector()
+ vector.delete(1) # Remove id=2 from the first sorted L0 run.
+ entry = TableDeleteByRowId(table)._write_deletion_vector_index(
+ GenericRow(['pending'], table.partition_keys_fields), 0, {first_file.file_name: vector})
+ commit = table.new_batch_write_builder().new_commit()
+ try:
+ commit.commit([CommitMessage(partition=('pending',), bucket=0, new_files=[], index_adds=[entry])])
+ finally:
+ commit.close()
+ pb = table.new_read_builder().new_predicate_builder()
+ all_ids = [1, 3, 4] if compacted_partition else [1, 3]
+ for predicate, expected in [(None, all_ids), (pb.equal('value', 'later'), []),
+ (pb.equal('value', 'first'), [1]), (pb.equal('id', 2), [])]:
+ for native in (False, True):
+ builder = table.copy({'scan.native-plan.enabled': str(native).lower()}).new_read_builder()
+ builder.with_projection(['id'])
+ if predicate is not None:
+ builder.with_filter(predicate)
+ scan = builder.new_scan()
+ if native:
+ with patch.object(scan.file_scanner, 'scan', side_effect=AssertionError('native fallback')):
+ plan = scan.plan()
+ else:
+ plan = scan.plan()
+ assert plan.snapshot_id == len(batches) + 1
+ assert sorted(builder.new_read().to_arrow(plan.splits()).column('id').to_pylist()) == expected
diff --git a/paimon-python/pypaimon/tests/native_plan_resolved_schema_test.py b/paimon-python/pypaimon/tests/native_plan_resolved_schema_test.py
new file mode 100644
index 0000000000..2d837694c8
--- /dev/null
+++ b/paimon-python/pypaimon/tests/native_plan_resolved_schema_test.py
@@ -0,0 +1,184 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from contextlib import ExitStack
+from unittest.mock import patch
+
+import pyarrow as pa
+import pytest
+
+from pypaimon import CatalogFactory, Schema
+from pypaimon.common.identifier import Identifier
+from pypaimon.read.native_plan import native_runtime_available
+from pypaimon.schema.data_types import AtomicType
+from pypaimon.schema.schema_change import SchemaChange
+from pypaimon.table.file_store_table import FileStoreTable
+from pypaimon.tests.native_plan_rest_test import rest_catalog # noqa: F401
+
+
+pytestmark = [pytest.mark.native_plan, pytest.mark.skipif(
+ not native_runtime_available(), reason='Rust planner required')]
+
+
+@pytest.fixture(params=[(mode, catalog) for mode in ('append', 'pk', 'de')
+ for catalog in ('filesystem', 'rest', 'jdbc')], ids=lambda case: '-'.join(case))
+def source(request, tmp_path):
+ mode, backend = request.param
+ if backend == 'rest':
+ catalog = request.getfixturevalue('rest_catalog')[0]
+ else:
+ options = {'warehouse': str(tmp_path / 'warehouse')}
+ if backend == 'jdbc':
+ options.update({'metastore': 'jdbc', 'uri': 'jdbc:sqlite:' + str(tmp_path / 'catalog.db')})
+ catalog = CatalogFactory.create(options)
+ catalog.create_database('default', True)
+ schema = pa.schema([('id', pa.int64()), ('value', pa.string())])
+ options = {'file.format': 'parquet'}
+ if mode == 'de':
+ options.update({'data-evolution.enabled': 'true', 'row-tracking.enabled': 'true'})
+ if mode == 'pk':
+ options['bucket'] = '1'
+ catalog.create_table('default.t', Schema.from_pyarrow_schema(
+ schema, options=options, primary_keys=['id'] if mode == 'pk' else []), False)
+ table = catalog.get_table('default.t')
+ _write(table, [{'id': 1, 'value': 'old'}])
+ yield catalog, table
+ if backend == 'jdbc':
+ catalog.close()
+
+
+def _write(table, rows):
+ builder = table.new_batch_write_builder()
+ writer, commit = builder.new_write(), builder.new_commit()
+ try:
+ writer.write_arrow(pa.Table.from_pylist(rows))
+ commit.commit(writer.prepare_commit())
+ finally:
+ writer.close()
+ commit.close()
+
+
+def _read(table, native, predicate=None, projection=None):
+ table = table.copy_without_time_travel({'scan.native-plan.enabled': str(native).lower()})
+ builder = table.new_read_builder()
+ if predicate is not None:
+ builder.with_filter(predicate)
+ if projection is not None:
+ builder.with_projection(projection)
+ scan = builder.new_scan()
+ if native:
+ from pypaimon.catalog.jdbc_catalog_loader import JdbcCatalogLoader
+ from pypaimon.catalog.rest.rest_catalog_loader import RESTCatalogLoader
+ with ExitStack() as stack:
+ stack.enter_context(patch.object(scan.file_scanner, 'scan',
+ side_effect=AssertionError('native fallback')))
+ stack.enter_context(patch.object(table.schema_manager, 'latest',
+ side_effect=AssertionError('schema reload')))
+ if type(table.catalog_environment.catalog_loader) is not RESTCatalogLoader:
+ stack.enter_context(patch('pypaimon_rust.datafusion.PaimonCatalog',
+ side_effect=AssertionError('catalog reload')))
+ if type(table.catalog_environment.catalog_loader) is JdbcCatalogLoader:
+ stack.enter_context(patch.object(JdbcCatalogLoader, 'load',
+ side_effect=AssertionError('JDBC connection during planning')))
+ plan = scan.plan()
+ else:
+ plan = scan.plan()
+ rows = builder.new_read().to_arrow(plan.splits()).to_pylist()
+ return plan.snapshot_id, sorted(rows, key=lambda row: row['id'])
+
+
+def _assert_parity(table, expected, snapshot_id, **kwargs):
+ for native in (False, True):
+ assert _read(table, native, **kwargs) == (snapshot_id, expected)
+
+
+def test_stale_schema_after_column_rename(source):
+ catalog, stale = source
+ catalog.alter_table('default.t', [SchemaChange.rename_column('value', 'renamed')], False)
+ latest = catalog.get_table('default.t')
+ _write(latest, [{'id': 2, 'renamed': 'new'}])
+ assert stale.table_schema.id != latest.table_schema.id
+ for table, name in ((stale, 'value'), (latest, 'renamed')):
+ _assert_parity(table, [{'id': 1, name: 'old'}, {'id': 2, name: 'new'}], 2)
+ predicate = table.new_read_builder().new_predicate_builder().equal(name, 'new')
+ _assert_parity(table, [{'id': 2}], 2, predicate=predicate, projection=['id'])
+
+
+def test_stale_schema_does_not_confuse_readded_column(source):
+ catalog, stale = source
+ catalog.alter_table('default.t', [SchemaChange.drop_column('value')], False)
+ catalog.alter_table('default.t', [SchemaChange.add_column('value', AtomicType('STRING'))], False)
+ latest = catalog.get_table('default.t')
+ _write(latest, [{'id': 2, 'value': 'new'}])
+ assert stale.fields[1].id != latest.fields[1].id
+ _assert_parity(stale, [{'id': 1, 'value': 'old'}, {'id': 2, 'value': None}], 2)
+ _assert_parity(latest, [{'id': 1, 'value': None}, {'id': 2, 'value': 'new'}], 2)
+ for table, value, id_ in ((stale, 'old', 1), (latest, 'new', 2)):
+ predicate = table.new_read_builder().new_predicate_builder().equal('value', value)
+ _assert_parity(table, [{'id': id_}], 2, predicate=predicate, projection=['id'])
+
+
+@pytest.mark.parametrize('uri', [False, True], ids=['path', 'file-uri'])
+@pytest.mark.parametrize('resolving', [False, True], ids=['local-io', 'resolving-io'])
+def test_catalogless_table_uses_resolved_schema(source, uri, resolving):
+ from pathlib import Path
+ from urllib.parse import unquote, urlparse
+ _, table = source
+ path = unquote(urlparse(table.table_path).path) if table.table_path.startswith('file:') else table.table_path
+ location = Path(path).as_uri() if uri else path
+ direct = FileStoreTable.from_path(location, {'resolving-file-io.enabled': str(resolving).lower()})
+ assert direct.catalog_environment.catalog_loader is None
+ _assert_parity(direct, [{'id': 1, 'value': 'old'}], 1)
+
+
+def test_dotted_database_does_not_require_catalog_parsing(source):
+ _, table = source
+ resolved = FileStoreTable(table.file_io, Identifier('namespace.database', 't'),
+ table.table_path, table.table_schema)
+ _assert_parity(resolved, [{'id': 1, 'value': 'old'}], 1)
+
+
+def test_copy_removes_search_options_and_normalizes_values(source):
+ catalog, table = source
+ catalog.alter_table('default.t', [SchemaChange.set_option('scalar-index.search-mode', 'full')], False)
+ table = catalog.get_table('default.t').copy({
+ 'scalar-index.search-mode': None, 'read.batch-size': 1, 'metadata.stats-mode': 'none'})
+ _assert_parity(table, [{'id': 1, 'value': 'old'}], 1)
+
+
+def test_historical_schema_remains_resolved_after_selector_removal(source):
+ catalog, old = source
+ catalog.alter_table('default.t', [SchemaChange.rename_column('value', 'renamed')], False)
+ latest = catalog.get_table('default.t')
+ _write(latest, [{'id': 2, 'renamed': 'new'}])
+ historical = latest.copy({'scan.snapshot-id': '1'})
+ assert historical.field_names == old.field_names
+ resumed = historical.copy_without_time_travel({'scan.snapshot-id': None})
+ _assert_parity(resumed, [{'id': 1, 'value': 'old'}, {'id': 2, 'value': 'new'}], 2)
+
+
+def test_copy_merge_engine_changes_level_zero_visibility(tmp_path):
+ catalog = CatalogFactory.create({'warehouse': str(tmp_path)})
+ catalog.create_database('default', True)
+ catalog.create_table('default.t', Schema.from_pyarrow_schema(
+ pa.schema([('id', pa.int64()), ('value', pa.string())]), primary_keys=['id'],
+ options={'bucket': '1', 'file.format': 'parquet'}), False)
+ table = catalog.get_table('default.t')
+ _write(table, [{'id': 1, 'value': 'old'}])
+ first_row = table.copy({'merge-engine': 'first-row'})
+ _assert_parity(first_row, [], 1)
+ _assert_parity(first_row.copy({'merge-engine': None}), [{'id': 1, 'value': 'old'}], 1)
diff --git a/paimon-python/pypaimon/tests/native_plan_rest_test.py b/paimon-python/pypaimon/tests/native_plan_rest_test.py
new file mode 100644
index 0000000000..2a34ce4a29
--- /dev/null
+++ b/paimon-python/pypaimon/tests/native_plan_rest_test.py
@@ -0,0 +1,151 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from unittest.mock import patch
+
+import pyarrow as pa
+import pytest
+
+from pypaimon import CatalogFactory, Schema
+from pypaimon.api.api_response import ConfigResponse, ErrorResponse, GetTableSnapshotResponse
+from pypaimon.api.auth import BearTokenAuthProvider
+from pypaimon.read.native_plan import native_runtime_available
+from pypaimon.snapshot.table_snapshot import TableSnapshot
+from pypaimon.tests.rest.rest_server import RESTCatalogServer
+
+
+pytestmark = [pytest.mark.native_plan, pytest.mark.skipif(
+ not native_runtime_available(), reason='Rust planner required')]
+
+
+@pytest.fixture
+def rest_catalog(tmp_path):
+ server = RESTCatalogServer(str(tmp_path), BearTokenAuthProvider('test-token'),
+ ConfigResponse(defaults={'prefix': 'native-test'}), 'warehouse')
+ server.start()
+ try:
+ catalog = CatalogFactory.create({
+ 'metastore': 'rest', 'uri': server.get_url(), 'warehouse': 'warehouse',
+ 'token.provider': 'bear', 'token': 'test-token', 'data-token.enabled': 'false'})
+ catalog.create_database('default', True)
+ yield catalog, server
+ finally:
+ server.shutdown()
+
+
+@pytest.fixture
+def rest_source(rest_catalog):
+ from pypaimon.tests.native_plan_resolved_schema_test import _write
+ catalog, server = rest_catalog
+ catalog.create_table('default.t', Schema.from_pyarrow_schema(
+ pa.schema([('id', pa.int64()), ('value', pa.string())])), False)
+ table = catalog.get_table('default.t')
+ _write(table, [{'id': 1, 'value': 'old'}])
+ first = table.snapshot_manager().get_latest_snapshot()
+ _write(table, [{'id': 2, 'value': 'new'}])
+ return table, server, first
+
+
+@pytest.mark.parametrize('response', ['first', 'empty', 'missing'])
+def test_catalog_snapshot_controls_native_plan(rest_source, response):
+ from pypaimon.tests.native_plan_resolved_schema_test import _assert_parity
+ table, server, first = rest_source
+ if response == 'first':
+ reply, code = GetTableSnapshotResponse(TableSnapshot(first, 1, 0, 1, first.time_millis)), 200
+ elif response == 'empty':
+ reply, code = GetTableSnapshotResponse(), 200
+ else:
+ code = 404
+ reply = ErrorResponse('SNAPSHOT', 't', response, code)
+ with patch.object(server, '_table_snapshot_handle', return_value=server._mock_response(reply, code)):
+ expected = [{'id': 1, 'value': 'old'}]
+ snapshot_id = 1
+ if response in ('empty', 'missing'):
+ expected, snapshot_id = [], None
+ _assert_parity(table, expected, snapshot_id)
+
+
+@pytest.mark.parametrize('code', [403, 404, 500, 501, 503])
+def test_catalog_snapshot_failure_never_reads_disk(rest_source, code):
+ from pypaimon.read.native_plan import native_plan
+ table, server, _ = rest_source
+ reply = ErrorResponse('TABLE', 't', 'snapshot unavailable', code)
+ with patch.object(server, '_table_snapshot_handle', return_value=server._mock_response(reply, code)):
+ # Test the binding directly too: a native error must not become a stale
+ # filesystem plan before TableScan gets a chance to fall back.
+ with pytest.raises(Exception, match='snapshot unavailable|does not exist|permission'):
+ native_plan(table)
+ with pytest.raises(Exception):
+ table.copy({'scan.native-plan.enabled': 'false'}).new_read_builder().new_scan().plan()
+
+
+@pytest.mark.parametrize('from_tag', [False, True], ids=['empty-branch', 'tagged-branch'])
+def test_rest_branch_keeps_catalog_snapshot_and_schema(rest_source, rest_catalog, from_tag):
+ from pypaimon.common.identifier import Identifier
+ from pypaimon.tests.native_plan_resolved_schema_test import _assert_parity
+ table, server, _ = rest_source
+ catalog, _ = rest_catalog
+ if from_tag:
+ catalog.create_tag(table.identifier, 'first', 1)
+ catalog.create_branch(table.identifier, 'dev', tag_name='first' if from_tag else None)
+ branch = catalog.get_table(Identifier('default', 't', branch='dev')).copy({'read.batch-size': '1'})
+ with patch.object(server, '_table_snapshot_handle', wraps=server._table_snapshot_handle) as load:
+ _assert_parity(branch, [{'id': 1, 'value': 'old'}] if from_tag else [], 1 if from_tag else None)
+ assert load.call_count >= 2
+ assert all(call.args[2] == 'dev' for call in load.call_args_list)
+
+
+def test_resolved_rest_table_keeps_refreshable_file_io(rest_source, rest_catalog):
+ from pypaimon.api.api_response import GetTableTokenResponse
+ from pypaimon.read.native_plan import _resolved_schema_json
+ from pypaimon_rust.datafusion import PaimonCatalog
+ table, server, _ = rest_source
+ catalog, _ = rest_catalog
+ options = dict(catalog.context.options.to_map(), **{'data-token.enabled': 'true'})
+ # Expiry is in the past, so a subsequent data access must refresh; no sleep.
+ expired = server._mock_response(GetTableTokenResponse(token={}, expires_at_millis=0), 200)
+ with patch.object(server, '_table_token_handle', return_value=expired):
+ rt = PaimonCatalog(options).get_table('default.t')
+ resolved = rt.copy_with_resolved_schema(_resolved_schema_json(table))
+ assert resolved.latest_snapshot().id() == 2
+ denied = server._mock_response(ErrorResponse('TABLE', 't', 'token denied', 403), 403)
+ with patch.object(server, '_table_token_handle', return_value=denied) as refresh:
+ with pytest.raises(Exception, match='token denied'):
+ resolved.new_read_builder().new_scan().plan()
+ refresh.assert_called()
+
+
+@pytest.mark.parametrize('branch', [None, 'dev'])
+def test_rest_dotted_database_and_table_keep_identity(rest_catalog, branch):
+ from pypaimon.common.identifier import Identifier
+ from pypaimon.tests.native_plan_resolved_schema_test import _assert_parity, _write
+ catalog, server = rest_catalog
+ identifier = Identifier('namespace.database', 'table.with.dots')
+ catalog.create_database(identifier.get_database_name(), False)
+ catalog.create_table(identifier, Schema.from_pyarrow_schema(
+ pa.schema([('id', pa.int64()), ('value', pa.string())])), False)
+ table = catalog.get_table(identifier)
+ _write(table, [{'id': 1, 'value': 'old'}])
+ if branch:
+ catalog.create_tag(identifier, 'first', 1)
+ catalog.create_branch(identifier, branch, tag_name='first')
+ _write(table, [{'id': 2, 'value': 'main'}])
+ table = catalog.get_table(Identifier('namespace.database', 'table.with.dots', branch=branch))
+ with patch.object(server, '_table_snapshot_handle', wraps=server._table_snapshot_handle) as load:
+ _assert_parity(table, [{'id': 1, 'value': 'old'}], 1)
+ assert all((call.args[1].get_database_name(), call.args[1].get_table_name())
+ == ('namespace.database', 'table.with.dots') for call in load.call_args_list)
diff --git a/paimon-python/pypaimon/tests/native_plan_test.py b/paimon-python/pypaimon/tests/native_plan_test.py
index 315e15b91e..b7892c1185 100644
--- a/paimon-python/pypaimon/tests/native_plan_test.py
+++ b/paimon-python/pypaimon/tests/native_plan_test.py
@@ -106,11 +106,65 @@ class NativePlanTest(unittest.TestCase):
version_patcher.start()
self.addCleanup(version_patcher.stop)
+ def test_resolved_schema_keeps_custom_io_and_rest_on_catalog_path(self):
+ from pypaimon.catalog.catalog_environment import CatalogEnvironment
+ from pypaimon.catalog.jdbc_catalog_loader import JdbcCatalogLoader
+ from pypaimon.filesystem.local_file_io import LocalFileIO
+ from pypaimon.read.native_plan import _resolved_schema_file_io_options
+
+ class CustomIO(LocalFileIO):
+ pass
+
+ class CustomEnvironment(CatalogEnvironment):
+ pass
+
+ class CustomLoader(FileSystemCatalogLoader):
+ pass
+
+ class CustomJdbcLoader(JdbcCatalogLoader):
+ pass
+
+ table = Mock(file_io=LocalFileIO(), catalog_environment=CatalogEnvironment.empty())
+ with patch('pypaimon.read.native_plan.native_method_available', return_value=True):
+ self.assertEqual(_resolved_schema_file_io_options(table), {})
+ table.file_io = CustomIO()
+ self.assertIsNone(_resolved_schema_file_io_options(table))
+ table.file_io = LocalFileIO()
+ table.catalog_environment = CustomEnvironment()
+ self.assertIsNone(_resolved_schema_file_io_options(table))
+ table.catalog_environment = CatalogEnvironment.empty()
+ for loader_type in (RESTCatalogLoader, CustomLoader, CustomJdbcLoader):
+ table.catalog_environment.catalog_loader = loader_type(
+ CatalogContext.create_from_options(Options({})))
+ self.assertIsNone(_resolved_schema_file_io_options(table))
+ for loader_type in (FileSystemCatalogLoader, JdbcCatalogLoader):
+ for attr in ('hadoop_conf', 'prefer_io_loader', 'fallback_io_loader'):
+ context = CatalogContext.create_from_options(Options({}))
+ setattr(context, attr, object())
+ table.catalog_environment.catalog_loader = loader_type(context)
+ self.assertIsNone(_resolved_schema_file_io_options(table))
+
def test_switch_defaults_off(self):
self.assertFalse(CoreOptions(Options({})).native_plan_enabled())
self.assertTrue(
CoreOptions(Options({"scan.native-plan.enabled": "true"})).native_plan_enabled())
+ def test_catalogless_standard_file_io_options_are_preserved(self):
+ from pypaimon.catalog.catalog_environment import CatalogEnvironment
+ from pypaimon.filesystem.pyarrow_file_io import PyArrowFileIO
+ from pypaimon.filesystem.resolving_file_io import ResolvingFileIO
+ from pypaimon.read.native_plan import _resolved_schema_file_io_options
+
+ properties = Options({'s3.path-style-access': True, 's3.endpoint': 'http://localhost:9000'})
+ # No storage connection is needed to check the resolved context transfer.
+ arrow = PyArrowFileIO.__new__(PyArrowFileIO)
+ arrow.properties = properties
+ for file_io in (arrow, ResolvingFileIO(properties)):
+ table = Mock(file_io=file_io, catalog_environment=CatalogEnvironment.empty())
+ with patch('pypaimon.read.native_plan.native_method_available', return_value=True):
+ self.assertEqual(_resolved_schema_file_io_options(table), {
+ 's3.path-style-access': 'true', 's3.endpoint': 'http://localhost:9000'})
+
def test_plan_uses_file_scanner_when_switch_off(self):
fs = Mock()
sentinel = object()
@@ -281,7 +335,6 @@ class NativePlanTest(unittest.TestCase):
setattr(fs, 'idx_of_this_subtask', 0)))
check(lambda s, fs: (setattr(fs, 'data_evolution', True),
setattr(fs, 'start_pos_of_this_subtask', 0)))
- check(lambda s, fs: setattr(fs, 'chunk_shuffle', (1, 100)))
check(lambda s, fs: setattr(fs, '_global_index_result', object()))
check(lambda s, fs: setattr(fs, '_row_ranges', [object()]))
check(lambda s, fs: setattr(fs, 'deletion_vectors_enabled', True))
@@ -293,8 +346,6 @@ class NativePlanTest(unittest.TestCase):
check(lambda s, fs: setattr(s.table.schema_manager.latest.return_value, 'id', 2))
check(lambda s, fs: s.table.schema_manager.latest.__setattr__(
'side_effect', RuntimeError('metadata read failed')))
- check(lambda s, fs: s.table.options.merge_engine.__setattr__(
- 'return_value', 'first-row'))
check(lambda s, fs: setattr(s.table.options, 'query_auth_enabled', True))
check(lambda s, fs: s.table.current_branch.__setattr__('return_value', 'b1'))
check(lambda s, fs: s.table.identifier.get_database_name.__setattr__(
diff --git a/paimon-python/pypaimon/tests/reader_predicate_test.py b/paimon-python/pypaimon/tests/reader_predicate_test.py
index 89e4efef6a..accce5555d 100644
--- a/paimon-python/pypaimon/tests/reader_predicate_test.py
+++ b/paimon-python/pypaimon/tests/reader_predicate_test.py
@@ -21,6 +21,7 @@ import tempfile
import unittest
import pyarrow as pa
+import pytest
from pypaimon import CatalogFactory
from pypaimon import Schema
@@ -30,6 +31,83 @@ from pypaimon.read.split import Split
from pypaimon.schema.data_types import AtomicType, DataField
+@pytest.mark.parametrize('mode', ['append', 'pk-raw', 'pk-merge', 'first-row'])
+@pytest.mark.parametrize('batch_size', [1, 1024])
+def test_predicate_preserves_deletion_vector_positions(tmp_path, mode, batch_size):
+ from dataclasses import replace
+ from pypaimon.deletionvectors.bitmap_deletion_vector import BitmapDeletionVector
+ from pypaimon.table.row.generic_row import GenericRow
+ from pypaimon.write.commit_message import CommitMessage
+ from pypaimon.write.table_delete import TableDeleteByRowId
+
+ catalog = CatalogFactory.create({'warehouse': str(tmp_path)})
+ catalog.create_database('default', False)
+ options = {'file.format': 'parquet', 'deletion-vectors.enabled': 'true',
+ 'deletion-vectors.merge-on-read': 'true', 'scan.native-plan.enabled': 'false',
+ 'read.batch-size': str(batch_size)}
+ if mode != 'append':
+ options['bucket'] = '1'
+ if mode == 'first-row':
+ options.update({'merge-engine': 'first-row', 'pk-clustering-override': 'true',
+ 'clustering.columns': 'id'})
+ catalog.create_table('default.t', Schema.from_pyarrow_schema(
+ pa.schema([('id', pa.int64())]), options=options,
+ primary_keys=[] if mode == 'append' else ['id']), False)
+ table = catalog.get_table('default.t')
+ builder = table.new_batch_write_builder()
+ writer, commit = builder.new_write(), builder.new_commit()
+ try:
+ writer.write_arrow(pa.Table.from_pydict({'id': [0, 1, 2]}))
+ messages = writer.prepare_commit()
+ if mode == 'pk-raw':
+ for message in messages:
+ message.new_files = [replace(file, level=1) for file in message.new_files]
+ commit.commit(messages)
+ finally:
+ writer.close()
+ commit.close()
+ vector = BitmapDeletionVector()
+ vector.delete(1)
+ entry = TableDeleteByRowId(table)._write_deletion_vector_index(
+ GenericRow([], []), messages[0].bucket, {messages[0].new_files[0].file_name: vector})
+ commit = builder.new_commit()
+ try:
+ commit.commit([CommitMessage(partition=(), bucket=messages[0].bucket,
+ new_files=[], index_adds=[entry])])
+ finally:
+ commit.close()
+ for id_, expected in [(0, [0]), (1, []), (2, [2])]:
+ read = table.new_read_builder()
+ read.with_filter(read.new_predicate_builder().equal('id', id_))
+ plan = read.new_scan().plan()
+ if mode in ('pk-merge', 'first-row'):
+ # A single unique L0 file can be planned raw; exercise the merge
+ # reader explicitly as overlapping runs and PK indexes do.
+ for split in plan.splits():
+ split.raw_convertible = False
+ assert read.new_read().to_arrow(plan.splits()).column('id').to_pylist() == expected
+ if mode != 'append':
+ from copy import copy
+ from pypaimon.globalindex.indexed_split import IndexedSplit
+ from pypaimon.utils.range import Range
+
+ # PK indexes select physical positions in one source file. Exercise
+ # range filtering both with and without a DV, including a deleted hit.
+ original = table.new_read_builder().new_scan().plan().splits()[0]
+ for with_dv in (False, True):
+ split = copy(original)
+ split.raw_convertible = False
+ if not with_dv:
+ split.data_deletion_files = None
+ for start, end in ((1, 1), (2, 2), (1, 2)):
+ indexed = IndexedSplit(split, [Range(start, end)])
+ for id_ in (1, 2):
+ read = table.new_read_builder()
+ read.with_filter(read.new_predicate_builder().equal('id', id_))
+ expected = [id_] if start <= id_ <= end and (not with_dv or id_ != 1) else []
+ assert read.new_read().to_arrow([indexed]).column('id').to_pylist() == expected
+
+
class ReaderPredicateTest(unittest.TestCase):
@classmethod
def setUpClass(cls):
diff --git a/paimon-python/pypaimon/tests/resolving_file_io_test.py b/paimon-python/pypaimon/tests/resolving_file_io_test.py
index c3dec4ebe5..f73948a0f9 100644
--- a/paimon-python/pypaimon/tests/resolving_file_io_test.py
+++ b/paimon-python/pypaimon/tests/resolving_file_io_test.py
@@ -65,6 +65,27 @@ class ResolvingFileIOTest(unittest.TestCase):
self.assertIsInstance(fio_local, LocalFileIO)
self.assertIsInstance(fio_file, LocalFileIO)
+ def test_pyarrow_reader_resolves_each_file_path(self):
+ import pyarrow as pa
+ import pyarrow.parquet as pq
+ from pathlib import Path
+ from pypaimon.read.reader.format_pyarrow_reader import FormatPyArrowReader
+ from pypaimon.schema.data_types import AtomicType, DataField
+
+ with tempfile.TemporaryDirectory() as directory:
+ resolving = ResolvingFileIO(Options({}))
+ for value, uri in ((1, False), (2, True)):
+ path = Path(directory) / ('data-%s.parquet' % value)
+ pq.write_table(pa.table({'id': [value]}), str(path))
+ reader = FormatPyArrowReader(
+ resolving, 'parquet', path.as_uri() if uri else str(path),
+ [DataField(0, 'id', AtomicType('BIGINT'))], None)
+ try:
+ self.assertEqual(reader.read_arrow_batch().to_pylist(), [{'id': value}])
+ self.assertIsNone(reader.read_arrow_batch())
+ finally:
+ reader.close()
+
def test_is_object_store_with_oss_warehouse(self):
opts = Options({CatalogOptions.WAREHOUSE.key(): 'oss://bucket/warehouse'})
resolving = ResolvingFileIO(opts)
diff --git a/paimon-python/pypaimon/tests/schema_evolution_read_test.py b/paimon-python/pypaimon/tests/schema_evolution_read_test.py
index 7bbee3d84a..916058812c 100644
--- a/paimon-python/pypaimon/tests/schema_evolution_read_test.py
+++ b/paimon-python/pypaimon/tests/schema_evolution_read_test.py
@@ -519,11 +519,9 @@ class SchemaEvolutionReadTest(unittest.TestCase):
table_read = read_builder.new_read()
actual = table_read.to_arrow(splits)
+ # Old files supply NULL for behavior; NULL = 'g' cannot pass AND.
expected = pa.Table.from_pydict({
- 'user_id': [1, 2, 4, 3, 7],
- 'item_id': [1001, 1002, 1004, 1003, 1007],
- 'dt': ["p1", "p1", "p1", "p2", "p2"],
- 'behavior': [None, None, None, None, "g"],
+ 'user_id': [7], 'item_id': [1007], 'dt': ["p2"], 'behavior': ["g"],
}, schema=pa_schema)
self.assertEqual(expected, actual)
@@ -873,3 +871,50 @@ class SchemaEvolutionReadTest(unittest.TestCase):
def _scan_table(self, read_builder):
splits = read_builder.new_scan().plan().splits()
return splits
+
+
+@pytest.mark.parametrize('file_format', ['parquet', 'avro'])
+@pytest.mark.parametrize('evolution', ['rename', 'readd'])
+def test_cross_schema_filter_runs_after_field_id_mapping(tmp_path, file_format, evolution):
+ catalog = CatalogFactory.create({'warehouse': str(tmp_path)})
+ catalog.create_database('default', True)
+ schema = pa.schema([('k', pa.int64()), ('v', pa.string())])
+ catalog.create_table('default.t', Schema.from_pyarrow_schema(
+ schema, options={'file.format': file_format}), False)
+
+ def write(table, values):
+ builder = table.new_batch_write_builder()
+ writer, commit = builder.new_write(), builder.new_commit()
+ try:
+ writer.write_arrow(pa.Table.from_pylist(values))
+ commit.commit(writer.prepare_commit())
+ finally:
+ writer.close()
+ commit.close()
+
+ write(catalog.get_table('default.t'), [{'k': 1, 'v': 'a'}, {'k': 2, 'v': 'b'}])
+ if evolution == 'rename':
+ catalog.alter_table('default.t', [SchemaChange.rename_column('v', 'renamed')], False)
+ latest_rows = [{'k': 3, 'renamed': 'c'}, {'k': 4, 'renamed': 'b'}]
+ else:
+ catalog.alter_table('default.t', [SchemaChange.drop_column('v')], False)
+ catalog.alter_table('default.t', [SchemaChange.add_column('v', AtomicType('STRING'))], False)
+ latest_rows = [{'k': 3, 'v': 'b'}, {'k': 4, 'v': None}]
+ table = catalog.get_table('default.t').copy({'scan.native-plan.enabled': 'false'})
+ write(table, latest_rows)
+ # Read an unfiltered plan so manifest statistics cannot hide a reader bug.
+ splits = table.new_read_builder().new_scan().plan().splits()
+ pb = table.new_read_builder().new_predicate_builder()
+ cases = ([(pb.equal('renamed', 'b'), [2, 4]), (pb.equal('renamed', 'missing'), [])]
+ if evolution == 'rename' else [(pb.is_null('v'), [1, 2, 4]), (pb.equal('v', 'b'), [3])])
+ for predicate, expected in cases:
+ for limit in (None, 1):
+ builder = table.new_read_builder().with_filter(predicate).with_projection(['k'])
+ if limit is not None:
+ builder.with_limit(limit)
+ actual = builder.new_read().to_arrow(splits, parallelism=1).column('k').to_pylist()
+ if limit is None:
+ assert sorted(actual) == expected
+ else:
+ assert len(actual) == min(1, len(expected))
+ assert all(key in expected for key in actual)
diff --git a/paimon-python/pypaimon/tests/snapshot_manager_test.py b/paimon-python/pypaimon/tests/snapshot_manager_test.py
index f09f61fc3a..21dbaa7860 100644
--- a/paimon-python/pypaimon/tests/snapshot_manager_test.py
+++ b/paimon-python/pypaimon/tests/snapshot_manager_test.py
@@ -47,6 +47,30 @@ def _build_manager(file_io):
class SnapshotManagerTest(unittest.TestCase):
"""Tests for SnapshotManager batch lookahead methods."""
+ def test_rest_snapshot_response_can_be_empty(self):
+ from pypaimon.api.api_response import GetTableSnapshotResponse
+ from pypaimon.common.json_util import JSON
+ for payload in ('{}', '{"snapshot": null}'):
+ self.assertIsNone(JSON.from_json(payload, GetTableSnapshotResponse).get_snapshot())
+
+ def test_only_unsupported_rest_snapshot_loader_falls_back(self):
+ from pypaimon.api.rest_exception import NotImplementedException, RESTException
+ from pypaimon.snapshot.snapshot_loader import SnapshotLoader
+ from pypaimon.snapshot.snapshot_manager import SnapshotManager
+ catalog_loader = Mock()
+ catalog_loader.load.return_value.load_snapshot.side_effect = NotImplementedError('unsupported')
+ manager = SnapshotManager(Mock(), '/table', snapshot_loader=SnapshotLoader(catalog_loader, Mock()))
+ snapshot = _create_mock_snapshot(2)
+ manager._get_latest_snapshot_from_filesystem = Mock(return_value=snapshot)
+ self.assertIs(manager.get_latest_snapshot(), snapshot)
+ manager._get_latest_snapshot_from_filesystem.assert_called_once()
+ manager._get_latest_snapshot_from_filesystem.reset_mock()
+ for error in (RESTException('unavailable'), NotImplementedException('unavailable')):
+ catalog_loader.load.return_value.load_snapshot.side_effect = error
+ with self.assertRaisesRegex(RuntimeError, 'unavailable'):
+ manager.get_latest_snapshot()
+ manager._get_latest_snapshot_from_filesystem.assert_not_called()
+
def test_find_next_scannable_returns_first_matching(self):
"""find_next_scannable should return the first snapshot that passes should_scan."""
file_io = Mock()
(paimon) 09/20: [python] Use conditional OSS puts for atomic metadata writes (#9715)
Posted by ho...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
XiaoHongbo-Hope pushed a commit to branch release-2.1
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit 32f0a9261e88fac86d039741452f055289b52315
Author: zhigang <wa...@apache.org>
AuthorDate: Wed Sep 16 19:21:14 2026 +0800
[python] Use conditional OSS puts for atomic metadata writes (#9715)
---
.github/workflows/ci-python.yml | 1 +
paimon-python/README.md | 74 +++++
paimon-python/dev/requirements-dev.txt | 1 +
.../pypaimon/catalog/rest/rest_token_file_io.py | 47 +---
paimon-python/pypaimon/common/file_io.py | 8 +-
paimon-python/pypaimon/common/options/config.py | 12 +
paimon-python/pypaimon/filesystem/oss_file_io.py | 157 +++++++++++
.../pypaimon/filesystem/resolving_file_io.py | 3 +
.../rest_catalog_blob_as_descriptor_sample.py | 23 +-
paimon-python/pypaimon/tests/file_io_test.py | 7 +-
paimon-python/pypaimon/tests/lance_utils_test.py | 6 +-
.../pypaimon/tests/oss_atomic_write_test.py | 308 +++++++++++++++++++++
paimon-python/pypaimon/tests/oss_file_io_test.py | 12 +-
.../pypaimon/tests/oss_legacy_mode_test.py | 19 +-
.../pypaimon/tests/py36/ao_simple_test.py | 8 +-
.../pypaimon/tests/rest/rest_token_file_io_test.py | 3 +
paimon-python/setup.py | 3 +
17 files changed, 619 insertions(+), 73 deletions(-)
diff --git a/.github/workflows/ci-python.yml b/.github/workflows/ci-python.yml
index 637bcad9dc..f7e127f074 100644
--- a/.github/workflows/ci-python.yml
+++ b/.github/workflows/ci-python.yml
@@ -167,6 +167,7 @@ jobs:
python -m pip check
fi
fi
+ python -m pip install 'oss2>=2.18,<3'
df -h
- name: Run lint-python.sh
diff --git a/paimon-python/README.md b/paimon-python/README.md
index 2af01d988c..c745cdc261 100644
--- a/paimon-python/README.md
+++ b/paimon-python/README.md
@@ -15,6 +15,80 @@ Pypaimon requires Python 3.6+.
The core dependencies are listed in `dev/requirements.txt`.
The development dependencies are listed in `dev/requirements-dev.txt`.
+# OSS metadata commits
+
+Install `pypaimon[oss]` (legacy PyArrow data access) or `pypaimon[jindo]`
+(Jindo data access). Both include `oss2` for atomic metadata writes.
+Configure `fs.oss.accessKeyId`, `fs.oss.accessKeySecret` and `fs.oss.endpoint`,
+plus `fs.oss.securityToken` when using STS. An endpoint without a scheme uses
+HTTPS for metadata writes. Credentials supplied only through an underlying
+filesystem's credential provider must also be supplied through these options.
+
+Metadata requests always use OSS Signature V4 for both AK and STS credentials,
+independently of the Jindo data-access signer setting. Set `fs.oss.region` to the
+bucket's region ID, such as `cn-hangzhou`. If unset, it is inferred from standard
+`oss-<region>.aliyuncs.com` or `oss-<region>-internal.aliyuncs.com` endpoints.
+Other endpoints, including acceleration endpoints, require an explicit region.
+
+Atomic metadata PUTs also forward the OSS server-side encryption options, using
+the same resolution as Java `OSSFileIO`:
+
+| Option | Behavior |
+| --- | --- |
+| `fs.oss.server-side-encryption` | `AES256`, `KMS` or `SM4` (case-insensitive) |
+| `fs.oss.server-side-encryption-key-id` | KMS key ID; implies `KMS` if the method is unset |
+| `fs.oss.server-side-data-encryption` | `SM4` with `KMS`; implies `KMS` if the method is unset |
+| `fs.oss.server-side-encryption-algorithm` | Legacy method fallback, used only if all three options above are unset |
+
+The first three options reject blank values and invalid combinations before any
+request is sent. If no encryption options are set, no encryption headers are
+added and OSS applies the bucket's default policy. These settings cover the
+atomic metadata PUT path; ordinary data writes still use the configured
+PyArrow/Jindo filesystem and its encryption capabilities.
+
+For `oss://` paths, `FileIO.get` selects `OssFileIO`, a thin `PyArrowFileIO`
+subclass that overrides atomic creation. Filesystem initialization, path handling,
+and ordinary PyArrow/Jindo file operations are inherited unchanged.
+Use `FileIO.get(path, options)` or construct `OssFileIO` explicitly for OSS atomic
+writes. REST token refresh and `ResolvingFileIO` also route atomic writes through
+this implementation.
+
+When the bucket is confirmed unversioned, `OssFileIO.try_to_write_atomic` uses a single OSS PUT
+with `x-oss-forbid-overwrite=true`. Exactly one writer can create a given object;
+`FileAlreadyExists` returns `False` so snapshot commits can retry. Other SDK errors
+are raised as `OSError`, retaining their cause for diagnostics. A lost PUT response
+is not retried by the SDK; the snapshot
+commit loop checks the commit user and identifier before retrying.
+
+The vendor SDK is an optional backend dependency, imported only for OSS atomic
+writes. The common FileIO API and Paimon table format remain independent of it.
+This implementation uses OSS-specific conditional creation; it does not provide
+the same atomic-write capability for every object store.
+
+Conditional creation requires a bucket that has **never enabled versioning**.
+The first atomic write on each `OssFileIO` instance checks `GetBucketVersioning`
+and caches the result, including the query-denied fallback. Concurrent first
+writes may repeat the check and warning. Query errors other than `403 AccessDenied`
+are not cached.
+If versioning is Enabled/Suspended,
+the state is unrecognized, or the query returns `403 AccessDenied`, the operation
+logs a warning when caching the fallback and uses the inherited PyArrow/Jindo
+temporary-file-and-rename path.
+This preserves legacy writes without making version-query permission mandatory,
+but the fallback does **not** guarantee safe concurrent commits. It also retains
+the existing backend's encryption behavior rather than applying the conditional
+PUT's OSS SSE headers. Invalid credentials, expired tokens, missing buckets, and
+other query failures still propagate as errors.
+
+Grant `oss:GetBucketVersioning` and keep versioning disabled to use conditional
+creation. Keep bucket versioning and version-query permissions unchanged for the
+instance's lifetime; recreate the FileIO after changing them. A configuration
+change is not guaranteed to produce an error and can invalidate the conditional-write guarantee.
+
+All concurrent writers must use conditional creation. Older Python clients or
+other clients that overwrite snapshot objects can still overwrite a successful
+commit. This change does not add conditional writes for other object stores.
+
# Build
You can build the source package by executing the following command:
diff --git a/paimon-python/dev/requirements-dev.txt b/paimon-python/dev/requirements-dev.txt
index a1b143383c..a122dd0bbe 100644
--- a/paimon-python/dev/requirements-dev.txt
+++ b/paimon-python/dev/requirements-dev.txt
@@ -20,6 +20,7 @@
# Test dependencies for pypaimon are as follows
duckdb==1.3.2
flake8==4.0.1
+oss2>=2.18,<3
pytest~=7.0
# HDF5 ingestion tests run in the supported h5py wheel lanes (Python 3.8+).
h5py>=3,<4; python_version >= "3.8"
diff --git a/paimon-python/pypaimon/catalog/rest/rest_token_file_io.py b/paimon-python/pypaimon/catalog/rest/rest_token_file_io.py
index 42dabb268c..6a87d004eb 100644
--- a/paimon-python/pypaimon/catalog/rest/rest_token_file_io.py
+++ b/paimon-python/pypaimon/catalog/rest/rest_token_file_io.py
@@ -22,13 +22,12 @@ from typing import Optional, Union
from cachetools import TTLCache
+from pypaimon.api.auth.bearer import BearTokenAuthProvider
+from pypaimon.api.auth.dlf_provider import DLFAuthProvider
from pypaimon.api.rest_api import RESTApi
from pypaimon.api.rest_util import RESTUtil
from pypaimon.catalog.rest.rest_token import RESTToken
from pypaimon.common.file_io import FileIO
-from pypaimon.filesystem.pyarrow_file_io import PyArrowFileIO
-from pypaimon.api.auth.bearer import BearTokenAuthProvider
-from pypaimon.api.auth.dlf_provider import DLFAuthProvider
from pypaimon.common.identifier import Identifier
from pypaimon.common.options import Options
from pypaimon.common.options.config import CatalogOptions, OssOptions
@@ -40,27 +39,12 @@ class RESTTokenFileIO(FileIO):
A FileIO to support getting token from REST Server.
"""
- _FILE_IO_CACHE_MAXSIZE = 1000
_FILE_IO_CACHE_TTL = 36000 # 10 hours in seconds
- _FILE_IO_CACHE: TTLCache = None
- _FILE_IO_CACHE_LOCK = threading.Lock()
-
_TOKEN_CACHE: dict = {}
_TOKEN_LOCKS: dict = {}
_TOKEN_LOCKS_LOCK = threading.Lock()
- @classmethod
- def _get_file_io_cache(cls) -> TTLCache:
- if cls._FILE_IO_CACHE is None:
- with cls._FILE_IO_CACHE_LOCK:
- if cls._FILE_IO_CACHE is None:
- cls._FILE_IO_CACHE = TTLCache(
- maxsize=cls._FILE_IO_CACHE_MAXSIZE,
- ttl=cls._FILE_IO_CACHE_TTL
- )
- return cls._FILE_IO_CACHE
-
def __init__(self, identifier: Identifier, path: str,
catalog_options: Optional[Union[dict, Options]] = None):
self.identifier = identifier
@@ -77,12 +61,20 @@ class RESTTokenFileIO(FileIO):
self.api_instance: Optional[RESTApi] = None
self.log = logging.getLogger(__name__)
self._uri_reader_factory_cache: Optional[UriReaderFactory] = None
+ self._init_file_io_cache()
+
+ def _init_file_io_cache(self):
+ # FileIO is bound to this instance's path and catalog options.
+ self._file_io_cache = TTLCache(maxsize=1, ttl=self._FILE_IO_CACHE_TTL)
+ self._file_io_cache_lock = threading.Lock()
def __getstate__(self):
state = self.__dict__.copy()
# Remove non-serializable objects
state.pop('api_instance', None)
state.pop('_uri_reader_factory_cache', None)
+ state.pop('_file_io_cache', None)
+ state.pop('_file_io_cache_lock', None)
# token can be serialized, but we'll refresh it on deserialization
return state
@@ -91,28 +83,17 @@ class RESTTokenFileIO(FileIO):
self._uri_reader_factory_cache = None
# api_instance will be recreated when needed
self.api_instance = None
+ self._init_file_io_cache()
def file_io(self) -> FileIO:
- self.try_to_refresh_token()
-
- if self.token is None:
- return FileIO.get(self.path, self.catalog_options or Options({}))
-
- cache_key = self.token
- cache = self._get_file_io_cache()
-
- file_io = cache.get(cache_key)
- if file_io is not None:
- return file_io
-
- with self._FILE_IO_CACHE_LOCK:
+ with self._file_io_cache_lock:
self.try_to_refresh_token()
if self.token is None:
return FileIO.get(self.path, self.catalog_options or Options({}))
cache_key = self.token
- cache = self._get_file_io_cache()
+ cache = self._file_io_cache
file_io = cache.get(cache_key)
if file_io is not None:
return file_io
@@ -127,7 +108,7 @@ class RESTTokenFileIO(FileIO):
merged_properties[OssOptions.OSS_ENDPOINT.key()] = dlf_oss_endpoint
merged_options = Options(merged_properties)
- file_io = PyArrowFileIO(self.path, merged_options)
+ file_io = FileIO.get(self.path, merged_options)
cache[cache_key] = file_io
return file_io
diff --git a/paimon-python/pypaimon/common/file_io.py b/paimon-python/pypaimon/common/file_io.py
index 10406849b3..720f9ae510 100644
--- a/paimon-python/pypaimon/common/file_io.py
+++ b/paimon-python/pypaimon/common/file_io.py
@@ -223,6 +223,7 @@ class FileIO(ABC):
def _read_ranges_coalesced(self, ranges, parallelism, max_gap, max_span,
max_retained_amplification, return_views):
from concurrent.futures import ThreadPoolExecutor
+
# Threads write disjoint results[idx]; safe under the GIL (no list resize).
results = [None] * len(ranges)
coalescible, singletons = [], []
@@ -598,7 +599,8 @@ class FileIO(ABC):
Returns a FileIO instance for accessing the file system identified by the given path.
- LocalFileIO for local file system (file:// or no scheme)
- HdfsNativeFileIO for HDFS/ViewFS (default; pure protocol client, no Hadoop install)
- - PyArrowFileIO for other remote file systems (oss://, s3://, gs://, ...),
+ - OssFileIO for OSS (oss://)
+ - PyArrowFileIO for other remote file systems (s3://, gs://, ...),
and for HDFS when explicitly requested via hdfs.client.impl=pyarrow
"""
import os as _os
@@ -651,5 +653,9 @@ class FileIO(ABC):
f"(from {impl_source}). Supported: 'native', 'pyarrow'."
)
+ if scheme == "oss":
+ from pypaimon.filesystem.oss_file_io import OssFileIO
+ return OssFileIO(path, opts)
+
from pypaimon.filesystem.pyarrow_file_io import PyArrowFileIO
return PyArrowFileIO(path, opts)
diff --git a/paimon-python/pypaimon/common/options/config.py b/paimon-python/pypaimon/common/options/config.py
index 5684ab62c3..98a466496b 100644
--- a/paimon-python/pypaimon/common/options/config.py
+++ b/paimon-python/pypaimon/common/options/config.py
@@ -31,6 +31,18 @@ class OssOptions:
OSS_ENDPOINT = ConfigOptions.key("fs.oss.endpoint").string_type().no_default_value().with_description(
"OSS endpoint")
OSS_REGION = ConfigOptions.key("fs.oss.region").string_type().no_default_value().with_description("OSS region")
+ OSS_SSE_METHOD = ConfigOptions.key(
+ "fs.oss.server-side-encryption").string_type().no_default_value().with_description(
+ "OSS atomic metadata encryption method: AES256, KMS or SM4")
+ OSS_SSE_KMS_KEY_ID = ConfigOptions.key(
+ "fs.oss.server-side-encryption-key-id").string_type().no_default_value().with_description(
+ "KMS key ID for OSS atomic metadata encryption")
+ OSS_SSE_DATA_ENCRYPTION = ConfigOptions.key(
+ "fs.oss.server-side-data-encryption").string_type().no_default_value().with_description(
+ "Data encryption algorithm for OSS atomic metadata encryption: SM4, with KMS only")
+ OSS_SSE_ALGORITHM = ConfigOptions.key(
+ "fs.oss.server-side-encryption-algorithm").string_type().no_default_value().with_description(
+ "Legacy OSS atomic metadata encryption method; used when the other SSE options are unset")
class S3Options:
diff --git a/paimon-python/pypaimon/filesystem/oss_file_io.py b/paimon-python/pypaimon/filesystem/oss_file_io.py
new file mode 100644
index 0000000000..23b5859738
--- /dev/null
+++ b/paimon-python/pypaimon/filesystem/oss_file_io.py
@@ -0,0 +1,157 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""OSS conditional metadata creation, reusing the existing Arrow/Jindo FileIO."""
+
+import re
+from urllib.parse import urlparse
+
+from pypaimon.common.options.config import OssOptions
+from pypaimon.filesystem.pyarrow_file_io import PyArrowFileIO
+
+
+class OssFileIO(PyArrowFileIO):
+ """Override atomic metadata creation; inherit all ordinary file operations."""
+
+ # Intentionally no lock: stable bucket configuration makes duplicate first
+ # queries acceptable. Without a lock, no additional __getstate__/__setstate__
+ # hooks are needed; the parent's pickle support already handles this None/bool
+ # cache, preserving its value under the same stability assumption.
+ _atomic_write_supported = None
+
+ def try_to_write_atomic(self, path: str, content: str) -> bool:
+ uri = urlparse(path)
+ if uri.scheme:
+ if uri.scheme != 'oss' or self._extract_oss_bucket(path) != self._oss_bucket:
+ raise ValueError("Atomic write must target the configured OSS bucket")
+ key = re.sub(r'/+', '/', uri.path).lstrip('/')
+ if '@' in uri.netloc:
+ key = key.partition('/')[2]
+ path = 'oss://{}/{}'.format(self._oss_bucket, key)
+ else:
+ key = path
+ if not self._use_jindo and not self._oss_bucket_in_endpoint:
+ bucket, _, key = path.partition('/')
+ if bucket != self._oss_bucket:
+ raise ValueError("Atomic write must target the configured OSS bucket")
+ if not key or key == '.' or key.endswith('/'):
+ return False
+
+ try:
+ import oss2
+ except ImportError as error:
+ raise ImportError(
+ "OSS atomic writes require oss2. Install pypaimon[oss] or pypaimon[jindo]."
+ ) from error
+
+ session = oss2.Session()
+ try:
+ bucket = self._create_oss_bucket(session)
+ headers = self._sse_headers()
+ headers['x-oss-forbid-overwrite'] = 'true'
+ if not self._supports_atomic_write(bucket):
+ return super().try_to_write_atomic(path, content)
+ try:
+ bucket.put_object(key, content.encode('utf-8'), headers=headers)
+ return True
+ except oss2.exceptions.ServerError as error:
+ if error.code == 'FileAlreadyExists':
+ return False
+ raise
+ except oss2.exceptions.OssError as error:
+ raise OSError("Failed to atomically write oss://{}/{}".format(self._oss_bucket, key)) from error
+ finally:
+ session.session.close()
+
+ def _supports_atomic_write(self, bucket):
+ """Cache the publication mode for this instance; failed queries remain retryable."""
+ import oss2
+
+ # Intentionally lock-free: duplicate first queries are acceptable with stable bucket configuration.
+ if self._atomic_write_supported is None:
+ try:
+ versioning = bucket.get_bucket_versioning().status
+ except oss2.exceptions.ServerError as error:
+ if error.status != 403 or error.code != 'AccessDenied':
+ raise
+ versioning = 'unknown (GetBucketVersioning denied)'
+ self._atomic_write_supported = versioning is None
+ if not self._atomic_write_supported:
+ self.logger.warning(
+ "Using legacy temporary-file-and-rename writes for OSS bucket %s "
+ "(versioning: %s). Concurrent commits are not protected against overwrites.",
+ self._oss_bucket, versioning)
+ return self._atomic_write_supported
+
+ def _create_oss_bucket(self, session):
+ """Build the metadata client with one V4 credential path for both AK and STS."""
+ import oss2
+ from oss2.credentials import StaticCredentialsProvider
+
+ access_key = self.properties.get(OssOptions.OSS_ACCESS_KEY_ID)
+ secret_key = self.properties.get(OssOptions.OSS_ACCESS_KEY_SECRET)
+ token = self.properties.get(OssOptions.OSS_SECURITY_TOKEN)
+ endpoint = self.properties.get(OssOptions.OSS_ENDPOINT)
+ if not access_key or not secret_key or not endpoint:
+ raise ValueError(
+ "OSS atomic writes require fs.oss.accessKeyId, fs.oss.accessKeySecret "
+ "and fs.oss.endpoint; pass fs.oss.securityToken for STS credentials.")
+ if '://' not in endpoint:
+ endpoint = 'https://' + endpoint
+ region = (self.properties.get(OssOptions.OSS_REGION) or '').strip()
+ if not region:
+ match = re.fullmatch(
+ r'oss-(?!accelerate(?:[.-]))([a-z0-9-]+?)(?:-internal)?\.aliyuncs\.com',
+ urlparse(endpoint).hostname or '')
+ region = match.group(1) if match else None
+ if not region:
+ raise ValueError("Set fs.oss.region for OSS V4 signing when the endpoint is not regional")
+ provider = StaticCredentialsProvider(access_key, secret_key, token)
+ auth = oss2.ProviderAuthV4(provider)
+ return oss2.Bucket(auth, endpoint, self._oss_bucket, session=session, region=region)
+
+ def _sse_headers(self):
+ """Match Java OSSFileIO's SSE resolution, including the native option fallback."""
+ settings = (OssOptions.OSS_SSE_METHOD, OssOptions.OSS_SSE_KMS_KEY_ID,
+ OssOptions.OSS_SSE_DATA_ENCRYPTION)
+ values = [self.properties.get(setting) for setting in settings]
+ for setting, value in zip(settings, values):
+ if value is not None and not value.strip():
+ raise ValueError("'{}' is set but blank".format(setting.key()))
+ method, key_id, data_encryption = [value.strip() if value is not None else None for value in values]
+ if method is None and key_id is None and data_encryption is None:
+ # Java forwards a nonempty native value as-is; the OSS service validates it.
+ algorithm = self.properties.get(OssOptions.OSS_SSE_ALGORITHM)
+ return {'x-oss-server-side-encryption': algorithm} if algorithm else {}
+
+ method = (method or 'KMS').upper()
+ if method not in ('AES256', 'KMS', 'SM4'):
+ raise ValueError("'{}' must be one of AES256/KMS/SM4".format(OssOptions.OSS_SSE_METHOD.key()))
+ headers = {'x-oss-server-side-encryption': method}
+ if key_id is not None:
+ if any(character.isspace() for character in key_id):
+ raise ValueError("'{}' must not contain whitespace".format(OssOptions.OSS_SSE_KMS_KEY_ID.key()))
+ if method != 'KMS':
+ raise ValueError("'{}' requires KMS".format(OssOptions.OSS_SSE_KMS_KEY_ID.key()))
+ headers['x-oss-server-side-encryption-key-id'] = key_id
+ if data_encryption is not None:
+ if method != 'KMS':
+ raise ValueError("'{}' requires KMS".format(OssOptions.OSS_SSE_DATA_ENCRYPTION.key()))
+ if data_encryption.upper() != 'SM4':
+ raise ValueError("'{}' only supports SM4".format(OssOptions.OSS_SSE_DATA_ENCRYPTION.key()))
+ headers['x-oss-server-side-data-encryption'] = 'SM4'
+ return headers
diff --git a/paimon-python/pypaimon/filesystem/resolving_file_io.py b/paimon-python/pypaimon/filesystem/resolving_file_io.py
index 576d160c81..e8f1c79d87 100644
--- a/paimon-python/pypaimon/filesystem/resolving_file_io.py
+++ b/paimon-python/pypaimon/filesystem/resolving_file_io.py
@@ -97,6 +97,9 @@ class ResolvingFileIO(FileIO):
def rename(self, src: str, dst: str) -> bool:
return self._get_fileio(src).rename(src, dst)
+ def try_to_write_atomic(self, path: str, content: str) -> bool:
+ return self._get_fileio(path).try_to_write_atomic(path, content)
+
def get_file_size(self, path: str) -> int:
return self._get_fileio(path).get_file_size(path)
diff --git a/paimon-python/pypaimon/sample/rest_catalog_blob_as_descriptor_sample.py b/paimon-python/pypaimon/sample/rest_catalog_blob_as_descriptor_sample.py
index 639b139d9f..1e90fa1bdf 100644
--- a/paimon-python/pypaimon/sample/rest_catalog_blob_as_descriptor_sample.py
+++ b/paimon-python/pypaimon/sample/rest_catalog_blob_as_descriptor_sample.py
@@ -18,13 +18,12 @@
"""
Sample demonstrating descriptor-stored blob fields with REST catalog.
"""
-from pypaimon import CatalogFactory
import pyarrow as pa
-from pypaimon import Schema
-from pypaimon.table.row.blob import BlobDescriptor, Blob
+from pypaimon import CatalogFactory, Schema
+from pypaimon.common.file_io import FileIO
from pypaimon.common.options import Options
-from pypaimon.filesystem.pyarrow_file_io import PyArrowFileIO
+from pypaimon.table.row.blob import Blob, BlobDescriptor
def write_table_with_blob(catalog, video_file_path: str, external_oss_options: dict):
@@ -66,7 +65,7 @@ def write_table_with_blob(catalog, video_file_path: str, external_oss_options: d
# Access external OSS file to get file size
try:
- external_file_io = PyArrowFileIO(video_file_path, Options(external_oss_options))
+ external_file_io = FileIO.get(video_file_path, Options(external_oss_options))
video_file_size = external_file_io.get_file_size(video_file_path)
except Exception as e:
raise FileNotFoundError(
@@ -92,7 +91,7 @@ def write_table_with_blob(catalog, video_file_path: str, external_oss_options: d
print("✓ Data committed successfully")
table_write.close()
table_commit.close()
-
+
return f'{database_name}.{table_name}'
@@ -103,10 +102,10 @@ def read_table_with_blob(catalog, table_name: str):
table_scan = read_builder.new_scan()
splits = table_scan.plan().splits()
table_read = read_builder.new_read()
-
+
result = table_read.to_arrow(splits)
print(f"✓ Read {result.num_rows} rows")
-
+
video_bytes_list = result.column('video').to_pylist()
for video_bytes in video_bytes_list:
if video_bytes is None:
@@ -118,7 +117,7 @@ def read_table_with_blob(catalog, table_name: str):
blob_data = blob.to_data()
print(f"✓ Blob data verified: {len(blob_data) / 1024 / 1024:.2f} MB")
break
-
+
return result
@@ -129,9 +128,9 @@ if __name__ == '__main__':
'fs.oss.endpoint': "oss-cn-hangzhou.aliyuncs.com",
'fs.oss.region': "cn-hangzhou",
}
-
+
video_file_path = "oss://your-bucket/blob_test/video.mov"
-
+
catalog_options = {
'metastore': 'rest',
'uri': "http://your-rest-catalog-uri",
@@ -145,7 +144,7 @@ if __name__ == '__main__':
}
catalog = CatalogFactory.create(catalog_options)
-
+
try:
table_name = write_table_with_blob(catalog, video_file_path, external_oss_options)
result = read_table_with_blob(catalog, table_name)
diff --git a/paimon-python/pypaimon/tests/file_io_test.py b/paimon-python/pypaimon/tests/file_io_test.py
index 9803a53790..53c8e8d296 100644
--- a/paimon-python/pypaimon/tests/file_io_test.py
+++ b/paimon-python/pypaimon/tests/file_io_test.py
@@ -29,6 +29,7 @@ from pypaimon.common.file_io import create_temp_path
from pypaimon.common.options import Options
from pypaimon.common.options.config import OssOptions
from pypaimon.filesystem.local_file_io import LocalFileIO, _file_uri_path
+from pypaimon.filesystem.oss_file_io import OssFileIO
from pypaimon.filesystem.pyarrow_file_io import PyArrowFileIO, _pyarrow_lt_7
@@ -95,7 +96,7 @@ class FileIOTest(unittest.TestCase):
parent_str = str(Path(converted_path).parent)
self.assertEqual(file_io.to_filesystem_path(parent_str), parent_str)
- oss_io = PyArrowFileIO("oss://test-bucket/warehouse", Options({
+ oss_io = OssFileIO("oss://test-bucket/warehouse", Options({
OssOptions.OSS_ENDPOINT.key(): 'oss-cn-hangzhou.aliyuncs.com',
OssOptions.OSS_ACCESS_KEY_ID.key(): 'test-key',
OssOptions.OSS_ACCESS_KEY_SECRET.key(): 'test-secret',
@@ -348,7 +349,7 @@ class FileIOTest(unittest.TestCase):
file_io.delete_quietly("file:///some/path")
file_io.delete_directory_quietly("file:///some/path")
- oss_io = PyArrowFileIO("oss://test-bucket/warehouse", Options({
+ oss_io = OssFileIO("oss://test-bucket/warehouse", Options({
OssOptions.OSS_ENDPOINT.key(): 'oss-cn-hangzhou.aliyuncs.com',
OssOptions.OSS_ACCESS_KEY_ID.key(): 'test-key',
OssOptions.OSS_ACCESS_KEY_SECRET.key(): 'test-secret',
@@ -525,7 +526,7 @@ class FileIOTest(unittest.TestCase):
shutil.rmtree(temp_dir, ignore_errors=True)
def test_path_on_windows(self):
- oss_io = PyArrowFileIO("oss://test-bucket/warehouse", Options({
+ oss_io = OssFileIO("oss://test-bucket/warehouse", Options({
OssOptions.OSS_ENDPOINT.key(): 'oss-cn-hangzhou.aliyuncs.com',
OssOptions.OSS_ACCESS_KEY_ID.key(): 'test-key',
OssOptions.OSS_ACCESS_KEY_SECRET.key(): 'test-secret',
diff --git a/paimon-python/pypaimon/tests/lance_utils_test.py b/paimon-python/pypaimon/tests/lance_utils_test.py
index 9eb12566b5..2ffb220efa 100644
--- a/paimon-python/pypaimon/tests/lance_utils_test.py
+++ b/paimon-python/pypaimon/tests/lance_utils_test.py
@@ -17,9 +17,9 @@
import unittest
+from pypaimon.common.file_io import FileIO
from pypaimon.common.options import Options
from pypaimon.common.options.config import OssOptions
-from pypaimon.filesystem.pyarrow_file_io import PyArrowFileIO
from pypaimon.read.reader.lance_utils import to_lance_specified
@@ -34,7 +34,7 @@ class LanceUtilsTest(unittest.TestCase):
OssOptions.OSS_ACCESS_KEY_SECRET.key(): "test-secret",
})
- file_io = PyArrowFileIO(file_path, properties)
+ file_io = FileIO.get(file_path, properties)
file_path_for_lance, storage_options = to_lance_specified(file_io, file_path)
self.assertEqual(
@@ -60,7 +60,7 @@ class LanceUtilsTest(unittest.TestCase):
OssOptions.OSS_SECURITY_TOKEN.key(): "test-token",
})
- file_io = PyArrowFileIO(file_path, properties)
+ file_io = FileIO.get(file_path, properties)
file_path_for_lance, storage_options = to_lance_specified(file_io, file_path)
self.assertEqual(file_path_for_lance, "oss://my-bucket/path/to/file.lance")
diff --git a/paimon-python/pypaimon/tests/oss_atomic_write_test.py b/paimon-python/pypaimon/tests/oss_atomic_write_test.py
new file mode 100644
index 0000000000..e909957007
--- /dev/null
+++ b/paimon-python/pypaimon/tests/oss_atomic_write_test.py
@@ -0,0 +1,308 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Exercise conditional OSS writes through the real SDK against a local HTTP server."""
+
+import socket
+import threading
+from concurrent.futures import ThreadPoolExecutor
+from http.server import BaseHTTPRequestHandler, HTTPServer
+from socketserver import ThreadingMixIn
+from unittest import mock
+from urllib.parse import unquote, urlsplit
+
+import pyarrow.fs as pafs
+import pytest
+
+from pypaimon.catalog.rest.rest_token import RESTToken
+from pypaimon.catalog.rest.rest_token_file_io import RESTTokenFileIO
+from pypaimon.common.file_io import FileIO
+from pypaimon.common.identifier import Identifier
+from pypaimon.common.options import Options
+from pypaimon.common.options.config import CatalogOptions
+from pypaimon.filesystem.oss_file_io import OssFileIO
+
+oss2 = pytest.importorskip("oss2")
+
+
+@pytest.fixture
+def oss_server():
+ class Handler(BaseHTTPRequestHandler):
+ def log_message(self, *args):
+ pass
+
+ def respond(self, status, body):
+ self.send_response(status)
+ self.send_header('Content-Length', str(len(body)))
+ self.end_headers()
+ self.wfile.write(body)
+
+ def error(self, status, code):
+ self.respond(status, ('<Error><Code>' + code + '</Code></Error>').encode())
+
+ def authenticate(self):
+ authorization = self.headers.get('Authorization', '')
+ server.auth_headers.append((self.command, authorization, self.headers.get('x-oss-security-token')))
+ if not (authorization.startswith('OSS4-HMAC-SHA256 ') and
+ '/cn-hangzhou/oss/aliyun_v4_request' in authorization):
+ self.error(403, 'AccessDenied')
+ return False
+ return True
+
+ def do_GET(self):
+ server.gets += 1
+ if not self.authenticate():
+ return
+ assert urlsplit(self.path).query in ('versioning', 'versioning=')
+ if server.fail_method == 'GET':
+ self.error(*server.failure)
+ return
+ status = '' if server.versioning is None else '<Status>' + server.versioning + '</Status>'
+ self.respond(200, ('<VersioningConfiguration>' + status +
+ '</VersioningConfiguration>').encode())
+
+ def do_PUT(self):
+ if not self.authenticate():
+ return
+ key = unquote(urlsplit(self.path).path)
+ data = self.rfile.read(int(self.headers['Content-Length']))
+ with server.lock:
+ server.puts += 1
+ server.token = self.headers.get('x-oss-security-token')
+ server.sse_headers = {key.lower()[len('x-oss-'):]: value
+ for key, value in self.headers.items()
+ if key.lower().startswith('x-oss-server-side-')}
+ if server.fail_method == 'PUT':
+ self.error(*server.failure)
+ return
+ if key in server.objects and self.headers.get('x-oss-forbid-overwrite') == 'true':
+ self.error(409, 'FileAlreadyExists')
+ return
+ server.objects[key] = data
+ if server.lose_response:
+ self.connection.shutdown(socket.SHUT_RDWR)
+ self.connection.close()
+ return
+ self.respond(200, b'')
+
+ class ThreadingHTTPServer(ThreadingMixIn, HTTPServer):
+ daemon_threads = True
+
+ server = ThreadingHTTPServer(('127.0.0.1', 0), Handler)
+ server.objects = {}
+ server.lock = threading.Lock()
+ server.versioning = None
+ server.fail_method = None
+ server.failure = (403, 'AccessDenied')
+ server.lose_response = False
+ server.puts = 0
+ server.gets = 0
+ server.token = None
+ server.sse_headers = {}
+ server.auth_headers = []
+ thread = threading.Thread(target=lambda: server.serve_forever(poll_interval=0.01), daemon=True)
+ thread.start()
+ yield server
+ server.shutdown()
+ server.server_close()
+ thread.join()
+
+
+def options_for(server, token=None):
+ return Options({
+ 'fs.oss.impl': 'legacy',
+ 'fs.oss.endpoint': 'http://127.0.0.1:{}'.format(server.server_port),
+ 'fs.oss.region': 'cn-hangzhou',
+ 'fs.oss.accessKeyId': 'test-ak',
+ 'fs.oss.accessKeySecret': 'test-sk',
+ 'fs.oss.securityToken': token,
+ })
+
+
+def file_io(server, resolving=False):
+ options = options_for(server)
+ if resolving:
+ options = Options(dict(options.to_map(), **{
+ CatalogOptions.RESOLVING_FILE_IO_ENABLED.key(): 'true'}))
+ with mock.patch.object(OssFileIO, '_initialize_oss_fs'):
+ return FileIO.get('oss://test-bucket/', options)
+
+
+@pytest.mark.parametrize('token,endpoint,region', [
+ (None, 'https://oss-cn-beijing.aliyuncs.com', 'cn-hangzhou'),
+ ('sts-token', None, 'cn-hangzhou'),
+ (None, 'https://oss-cn-hangzhou.aliyuncs.com', None),
+ ('sts-token', 'https://oss-cn-hangzhou-internal.aliyuncs.com', None),
+])
+def test_v4_authentication_reaches_conditional_put(oss_server, monkeypatch, tmp_path, token, endpoint, region):
+ io = file_io(oss_server)
+ io._use_jindo = True
+ io.filesystem = pafs.SubTreeFileSystem(str(tmp_path), pafs.LocalFileSystem())
+ local_endpoint = io.properties.to_map()['fs.oss.endpoint']
+ io.properties = Options(dict(options_for(oss_server, token).to_map(), **{
+ 'fs.oss.endpoint': endpoint or local_endpoint,
+ 'fs.oss.region': region,
+ 'fs.oss.signer.version': '4',
+ }))
+ send = oss2.Session.do_request
+
+ def redirect(session, request, timeout):
+ # Sign the original endpoint, then send the real SDK request to the local server.
+ request.url = local_endpoint + urlsplit(request.url).path
+ return send(session, request, timeout)
+
+ monkeypatch.setattr(oss2.Session, 'do_request', redirect)
+ path = 'oss://test-bucket/snapshot-1'
+ assert io.try_to_write_atomic(path, 'data') is True
+ assert io.try_to_write_atomic(path, 'overwrite') is False
+ assert list(oss_server.objects.values()) == [b'data']
+ assert [method for method, _, _ in oss_server.auth_headers] == ['GET', 'PUT', 'PUT']
+ assert all(header_token == token for _, _, header_token in oss_server.auth_headers)
+
+
+@pytest.mark.parametrize('endpoint', ['http://127.0.0.1', 'https://oss-accelerate.aliyuncs.com'])
+def test_v4_requires_region_for_non_regional_endpoints(oss_server, endpoint):
+ io = file_io(oss_server)
+ io.properties = Options(dict(io.properties.to_map(), **{'fs.oss.endpoint': endpoint, 'fs.oss.region': None}))
+ with pytest.raises(ValueError, match='fs.oss.region'):
+ io.try_to_write_atomic('oss://test-bucket/snapshot-1', 'data')
+ assert oss_server.gets == oss_server.puts == 0
+
+
+@pytest.mark.parametrize('resolving', [False, True])
+def test_atomic_competition_and_existing_content(oss_server, resolving):
+ io = file_io(oss_server, resolving)
+ path = 'oss://test-bucket/table/p=a%2Fb/snapshot-1'
+ barrier = threading.Barrier(2)
+
+ def write(index):
+ barrier.wait(timeout=10)
+ target = 'oss://AK:SK@endpoint/test-bucket/table/p=a%2Fb/snapshot-1' if index == 0 else path
+ return io.try_to_write_atomic(target, contents[index])
+
+ contents = ['提交者一', '提交者二']
+ with mock.patch.object(OssFileIO, '_initialize_oss_fs'), ThreadPoolExecutor(max_workers=2) as pool:
+ results = list(pool.map(write, range(2)))
+ assert sorted(results) == [False, True]
+ assert oss_server.objects == {'/test-bucket/table/p=a%2Fb/snapshot-1': contents[results.index(True)].encode()}
+ versioning_queries = oss_server.gets
+ assert io.try_to_write_atomic(path, 'overwrite') is False
+ assert list(oss_server.objects.values()) == [contents[results.index(True)].encode()]
+ assert oss_server.gets == versioning_queries
+
+
+@pytest.mark.parametrize('second_path,method', [
+ ('oss://other-bucket/table', 'AES256'),
+ ('oss://test-bucket/table', 'KMS'),
+])
+def test_rest_file_io_isolates_bucket_and_encryption(oss_server, second_path, method):
+ with mock.patch.object(RESTTokenFileIO, 'try_to_refresh_token'), \
+ mock.patch.object(OssFileIO, '_initialize_oss_fs'):
+ for index, (path, encryption) in enumerate([
+ ('oss://test-bucket/table', 'AES256'), (second_path, method)]):
+ options = dict(options_for(oss_server).to_map())
+ options['fs.oss.server-side-encryption'] = encryption
+ io = RESTTokenFileIO(Identifier.from_string('default.table'), path, Options(options))
+ io.token = RESTToken({'fs.oss.securityToken': 'shared-token'}, oss_server.server_port)
+ target = path + '/snapshot-' + str(index)
+ assert io.try_to_write_atomic(target, str(index))
+ assert oss_server.sse_headers == {'server-side-encryption': encryption}
+ assert oss_server.objects['/' + target[len('oss://'):]] == str(index).encode()
+ assert io.try_to_write_atomic(target, 'overwrite') is False
+ io.token = RESTToken({'fs.oss.securityToken': 'refreshed-token'}, oss_server.server_port + 1)
+ assert io.try_to_write_atomic(target + '-next', 'next')
+ assert oss_server.token == 'refreshed-token'
+ assert oss_server.gets == 2 * (index + 1)
+
+
+@pytest.mark.parametrize('versioning', ['Enabled', 'Suspended', None])
+def test_versioning_fallback_preserves_legacy_writes(oss_server, versioning, tmp_path, caplog):
+ oss_server.versioning = versioning
+ if versioning is None:
+ oss_server.fail_method = 'GET'
+ oss_server.failure = (403, 'AccessDenied')
+ io = file_io(oss_server)
+ # Exercise inherited stream/rename operations using Jindo's key-only path convention.
+ io._use_jindo = True
+ io.filesystem = pafs.SubTreeFileSystem(str(tmp_path), pafs.LocalFileSystem())
+ path = 'oss://AK:SK@endpoint/test-bucket/snapshot-1'
+ assert io.try_to_write_atomic(path, '兼容写入') is True
+ assert io.try_to_write_atomic(path, 'overwrite') is False
+ assert (tmp_path / 'snapshot-1').read_text() == '兼容写入'
+ assert sorted(p.name for p in tmp_path.iterdir()) == ['snapshot-1']
+ assert oss_server.puts == 0
+ assert oss_server.gets == 1
+ assert caplog.text.count('Concurrent commits are not protected') == 1
+
+
+@pytest.mark.parametrize('method,status,code', [
+ ('GET', 403, 'SecurityTokenExpired'),
+ ('PUT', 403, 'AccessDenied'),
+ ('PUT', 409, 'OtherConflict'),
+])
+def test_errors_are_not_competition(oss_server, method, status, code):
+ oss_server.fail_method, oss_server.failure = method, (status, code)
+ io = file_io(oss_server)
+ with pytest.raises(OSError) as caught:
+ io.try_to_write_atomic('oss://test-bucket/snapshot-1', 'data')
+ assert isinstance(caught.value.__cause__, oss2.exceptions.ServerError)
+ assert caught.value.__cause__.code == code
+ assert oss_server.objects == {}
+ oss_server.fail_method = None
+ assert io.try_to_write_atomic('oss://test-bucket/snapshot-1', 'data')
+ assert oss_server.gets == (2 if method == 'GET' else 1)
+
+
+def test_lost_response_is_not_replayed_or_reported_as_conflict(oss_server):
+ oss_server.lose_response = True
+ with pytest.raises(OSError) as caught:
+ file_io(oss_server).try_to_write_atomic('oss://test-bucket/snapshot-1', 'data')
+ assert isinstance(caught.value.__cause__, oss2.exceptions.RequestError)
+ assert oss_server.puts == 1
+ assert list(oss_server.objects.values()) == [b'data']
+
+
+@pytest.mark.parametrize('settings,expected', [
+ ({'server-side-encryption-key-id': ' my-cmk ',
+ 'server-side-data-encryption': 'sm4', 'server-side-encryption-algorithm': 'AES256'},
+ {'server-side-encryption': 'KMS', 'server-side-encryption-key-id': 'my-cmk',
+ 'server-side-data-encryption': 'SM4'}),
+ ({'server-side-encryption-algorithm': 'AES256'}, {'server-side-encryption': 'AES256'}),
+])
+def test_sse_headers_and_conditional_creation(oss_server, settings, expected):
+ io = file_io(oss_server)
+ io.properties = Options(dict(io.properties.to_map(), **{
+ 'fs.oss.' + key: value for key, value in settings.items()}))
+ path = 'oss://test-bucket/snapshot-1'
+ assert io.try_to_write_atomic(path, 'encrypted metadata')
+ assert oss_server.sse_headers == expected
+ assert io.try_to_write_atomic(path, 'overwrite') is False
+ assert list(oss_server.objects.values()) == [b'encrypted metadata']
+
+
+@pytest.mark.parametrize('settings', [
+ {'server-side-encryption': 'AES256', 'server-side-encryption-key-id': 'my-cmk'},
+ {'server-side-encryption': '', 'server-side-encryption-algorithm': 'AES256'},
+])
+def test_invalid_sse_is_rejected_before_io(oss_server, settings):
+ io = file_io(oss_server)
+ io.properties = Options(dict(io.properties.to_map(), **{
+ 'fs.oss.' + key: value for key, value in settings.items()}))
+ with pytest.raises(ValueError, match='fs.oss.server-side-'):
+ io.try_to_write_atomic('oss://test-bucket/snapshot-1', 'data')
+ assert oss_server.gets == 0
+ assert oss_server.puts == 0
diff --git a/paimon-python/pypaimon/tests/oss_file_io_test.py b/paimon-python/pypaimon/tests/oss_file_io_test.py
index d153760785..bb8506d0e7 100644
--- a/paimon-python/pypaimon/tests/oss_file_io_test.py
+++ b/paimon-python/pypaimon/tests/oss_file_io_test.py
@@ -23,11 +23,11 @@ import pyarrow.fs as pafs
from pypaimon.common.options import Options
from pypaimon.common.options.config import OssOptions
-from pypaimon.filesystem.pyarrow_file_io import PyArrowFileIO
+from pypaimon.filesystem.oss_file_io import OssFileIO
class OSSFileIOTest(unittest.TestCase):
- """Test cases for PyArrowFileIO with OSS."""
+ """Test cases for OssFileIO with OSS."""
def setUp(self):
"""Set up test fixtures."""
@@ -60,8 +60,8 @@ class OSSFileIOTest(unittest.TestCase):
OssOptions.OSS_IMPL.key(): oss_impl,
})
- # Create PyArrowFileIO instance
- self.file_io = PyArrowFileIO(self.root_path, self.catalog_options)
+ # Create OssFileIO instance
+ self.file_io = OssFileIO(self.root_path, self.catalog_options)
# Create unique test prefix to avoid conflicts
self.test_prefix = f"test-{uuid.uuid4().hex[:8]}/"
@@ -332,7 +332,7 @@ class OSSFileIOTest(unittest.TestCase):
self.file_io.mkdirs(target_dir)
self.assertFalse(
self.file_io.try_to_write_atomic(target_dir, "test content"),
- "PyArrowFileIO should return False when target is a directory")
+ "OssFileIO should return False when target is a directory")
# Verify no file was created inside the directory
# List directory contents to verify it's empty
@@ -348,7 +348,7 @@ class OSSFileIOTest(unittest.TestCase):
self.file_io.delete(normal_file)
self.assertFalse(
self.file_io.try_to_write_atomic(target_dir, "test content"),
- "PyArrowFileIO should return False when target is a directory")
+ "OssFileIO should return False when target is a directory")
# Verify no file was created inside the directory
dir_contents = self.file_io.filesystem.get_file_info(selector)
diff --git a/paimon-python/pypaimon/tests/oss_legacy_mode_test.py b/paimon-python/pypaimon/tests/oss_legacy_mode_test.py
index 43c4972f0c..dc300cb391 100644
--- a/paimon-python/pypaimon/tests/oss_legacy_mode_test.py
+++ b/paimon-python/pypaimon/tests/oss_legacy_mode_test.py
@@ -15,9 +15,9 @@
# specific language governing permissions and limitations
# under the License.
-"""Unit tests for the OSS bucket-in-endpoint mode (PyArrow < 16) of PyArrowFileIO.
+"""Unit tests for the OSS bucket-in-endpoint mode (PyArrow < 16) of OssFileIO.
-See ``PyArrowFileIO._legacy_oss_mode`` for why bucket-level operations
+See ``OssFileIO._legacy_oss_mode`` for why bucket-level operations
must be guarded in this mode. No real OSS access is required.
"""
@@ -28,11 +28,8 @@ import pyarrow.fs as pafs
from pypaimon.common.options import Options
from pypaimon.common.options.config import OssOptions
-from pypaimon.filesystem.pyarrow_file_io import (
- LegacyOssDirectoryListingError,
- PyArrowFileIO,
-)
-
+from pypaimon.filesystem.oss_file_io import OssFileIO
+from pypaimon.filesystem.pyarrow_file_io import LegacyOssDirectoryListingError
TABLE_PATH = "oss://test-bucket/db-uuid.db/tbl-uuid"
@@ -48,7 +45,7 @@ def _probe_response(status_code, body):
class OssLegacyModeTest(unittest.TestCase):
- """Behavior of PyArrowFileIO when OSS runs on PyArrow < 16."""
+ """Behavior of OssFileIO when OSS runs on PyArrow < 16."""
def _new_file_io(self, legacy):
options = Options({
@@ -59,8 +56,8 @@ class OssLegacyModeTest(unittest.TestCase):
OssOptions.OSS_IMPL.key(): "legacy",
})
with mock.patch.object(
- PyArrowFileIO, "_initialize_oss_fs", return_value=mock.Mock()):
- file_io = PyArrowFileIO("oss://test-bucket/", options)
+ OssFileIO, "_initialize_oss_fs", return_value=mock.Mock()):
+ file_io = OssFileIO("oss://test-bucket/", options)
# _legacy_oss_mode() keys off the bucket-in-endpoint flag (PyArrow < 16).
file_io._oss_bucket_in_endpoint = legacy
file_io.filesystem = mock.Mock()
@@ -214,7 +211,7 @@ class OssLegacyModeTest(unittest.TestCase):
OssOptions.OSS_REGION.key(): "cn-test",
OssOptions.OSS_IMPL.key(): "legacy",
})
- file_io = PyArrowFileIO("oss://test-bucket/wh", options)
+ file_io = OssFileIO("oss://test-bucket/wh", options)
file_io._legacy_bucket_checked = True
file_io._legacy_bucket_error = "OSS bucket 'test-bucket' does not exist"
diff --git a/paimon-python/pypaimon/tests/py36/ao_simple_test.py b/paimon-python/pypaimon/tests/py36/ao_simple_test.py
index 792387179d..662d0b720e 100644
--- a/paimon-python/pypaimon/tests/py36/ao_simple_test.py
+++ b/paimon-python/pypaimon/tests/py36/ao_simple_test.py
@@ -26,7 +26,7 @@ from pypaimon.catalog.catalog_exception import (DatabaseAlreadyExistException,
TableNotExistException)
from pypaimon.common.options import Options
from pypaimon.common.options.config import OssOptions
-from pypaimon.filesystem.pyarrow_file_io import PyArrowFileIO
+from pypaimon.filesystem.oss_file_io import OssFileIO
from pypaimon.tests.py36.pyarrow_compat import table_sort_by
from pypaimon.tests.rest.rest_base_test import RESTBaseTest
@@ -404,19 +404,19 @@ class AOSimpleTest(RESTBaseTest):
with patch("pypaimon.common.file_io.pyarrow.__version__", "6.0.0"), \
patch("pyarrow.fs.S3FileSystem") as mock_s3fs:
- PyArrowFileIO("oss://oss-bucket/paimon-database/paimon-table", Options(props))
+ OssFileIO("oss://oss-bucket/paimon-database/paimon-table", Options(props))
mock_s3fs.assert_called_once_with(access_key="AKID",
secret_key="SECRET",
session_token="TOKEN",
region="cn-hangzhou",
endpoint_override="oss-bucket." + props[OssOptions.OSS_ENDPOINT.key()])
- PyArrowFileIO("oss://oss-bucket.endpoint/paimon-database/paimon-table", Options(props))
+ OssFileIO("oss://oss-bucket.endpoint/paimon-database/paimon-table", Options(props))
mock_s3fs.assert_called_with(access_key="AKID",
secret_key="SECRET",
session_token="TOKEN",
region="cn-hangzhou",
endpoint_override="oss-bucket." + props[OssOptions.OSS_ENDPOINT.key()])
- PyArrowFileIO("oss://access_id:secret_key@Endpoint/oss-bucket/paimon-database/paimon-table", Options(props))
+ OssFileIO("oss://access_id:secret_key@Endpoint/oss-bucket/paimon-database/paimon-table", Options(props))
mock_s3fs.assert_called_with(access_key="AKID",
secret_key="SECRET",
session_token="TOKEN",
diff --git a/paimon-python/pypaimon/tests/rest/rest_token_file_io_test.py b/paimon-python/pypaimon/tests/rest/rest_token_file_io_test.py
index dbb919eed2..cf80fc95d1 100644
--- a/paimon-python/pypaimon/tests/rest/rest_token_file_io_test.py
+++ b/paimon-python/pypaimon/tests/rest/rest_token_file_io_test.py
@@ -147,6 +147,8 @@ class RESTTokenFileIOTest(unittest.TestCase):
self.warehouse_path,
self.catalog_options
)
+ original_file_io.token = RESTToken({}, 1)
+ original_backend = original_file_io.file_io()
pickled = pickle.dumps(original_file_io)
@@ -157,6 +159,7 @@ class RESTTokenFileIOTest(unittest.TestCase):
self.assertEqual(deserialized_file_io.properties.data, original_file_io.properties.data)
self.assertIsNone(deserialized_file_io.api_instance)
+ self.assertIsNot(deserialized_file_io.file_io(), original_backend)
test_file_path = f"file://{self.temp_dir}/pickle_test.txt"
test_content = b"pickle test content"
diff --git a/paimon-python/setup.py b/paimon-python/setup.py
index 0a08d7acca..60cfe979d3 100644
--- a/paimon-python/setup.py
+++ b/paimon-python/setup.py
@@ -22,6 +22,7 @@ import subprocess
import sys
import tarfile
import tempfile
+
from setuptools import find_packages, setup
from setuptools.command.build_py import build_py
from setuptools.command.sdist import sdist
@@ -270,10 +271,12 @@ setup(
'daft>=0.7.6; python_version>="3.10"',
],
'oss': [
+ 'oss2>=2.18,<3',
'ossfs>=2021.8; python_version<"3.8"',
'ossfs>=2023; python_version>="3.8"'
],
'jindo': [
+ 'oss2>=2.18,<3',
'pyjindosdk>=6.10.4',
],
'lance': [
(paimon) 04/20: [core] Execute complete filters with unprojected fields (#9872)
Posted by ho...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
XiaoHongbo-Hope pushed a commit to branch release-2.1
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit 72e4a32b56e00f411dac32452ee5dd485123ecf0
Author: Jingsong Lee <ji...@gmail.com>
AuthorDate: Wed Sep 16 13:07:25 2026 +0800
[core] Execute complete filters with unprojected fields (#9872)
---
.../apache/paimon/predicate/PredicateRemapper.java | 87 +++++
.../java/org/apache/paimon/utils/TypeUtils.java | 19 ++
.../paimon/predicate/PredicateRemapperTest.java | 144 ++++++++
.../paimon/catalog/TableQueryAuthResult.java | 133 +++-----
.../operation/DataEvolutionFileStoreScan.java | 17 +-
.../paimon/table/format/FormatReadBuilder.java | 18 +-
.../paimon/table/format/FormatTableRead.java | 40 +--
.../paimon/table/source/AbstractDataTableRead.java | 199 ++---------
.../paimon/table/source/AbstractDataTableScan.java | 56 ++--
.../apache/paimon/table/source/ReadTransform.java | 178 ++++++++++
.../org/apache/paimon/table/source/TableRead.java | 12 +
.../org/apache/paimon/rest/RESTCatalogTest.java | 87 +++++
.../paimon/table/AppendOnlySimpleTableTest.java | 10 +-
.../paimon/table/DataEvolutionFileIndexTest.java | 44 ++-
.../paimon/table/PrimaryKeySimpleTableTest.java | 26 ++
.../paimon/table/format/FormatTableReadTest.java | 109 ++++++
.../table/source/AbstractDataTableReadTest.java | 369 ++++++++++++++++++++-
.../spark/procedure/PermissionProcedureTest.scala | 4 +-
18 files changed, 1197 insertions(+), 355 deletions(-)
diff --git a/paimon-common/src/main/java/org/apache/paimon/predicate/PredicateRemapper.java b/paimon-common/src/main/java/org/apache/paimon/predicate/PredicateRemapper.java
new file mode 100644
index 0000000000..8dc9d8b0f1
--- /dev/null
+++ b/paimon-common/src/main/java/org/apache/paimon/predicate/PredicateRemapper.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.predicate;
+
+import org.apache.paimon.types.RowType;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.apache.paimon.utils.Preconditions.checkArgument;
+
+/**
+ * Resolves field references by name to positional indices and types in a read schema. Every
+ * referenced field must exist; unresolved fields and malformed compound predicates are rejected.
+ */
+public final class PredicateRemapper implements PredicateVisitor<Predicate> {
+
+ private final RowType rowType;
+
+ private PredicateRemapper(RowType rowType) {
+ this.rowType = rowType;
+ }
+
+ public static Predicate remap(Predicate predicate, RowType rowType) {
+ return predicate.visit(new PredicateRemapper(rowType));
+ }
+
+ public static Transform remap(Transform transform, RowType rowType) {
+ return transform.copyWithNewInputs(new PredicateRemapper(rowType).remapInputs(transform));
+ }
+
+ private List<Object> remapInputs(Transform transform) {
+ List<Object> inputs = new ArrayList<>();
+ for (Object input : transform.inputs()) {
+ if (input instanceof FieldRef) {
+ FieldRef field = (FieldRef) input;
+ int index = rowType.getFieldIndex(field.name());
+ checkArgument(
+ index >= 0,
+ "Cannot resolve field '%s' in read schema %s.",
+ field.name(),
+ rowType);
+ inputs.add(new FieldRef(index, field.name(), rowType.getTypeAt(index)));
+ } else {
+ inputs.add(input);
+ }
+ }
+ return inputs;
+ }
+
+ @Override
+ public Predicate visit(LeafPredicate predicate) {
+ return predicate.copyWithNewInputs(remapInputs(predicate.transform()));
+ }
+
+ @Override
+ public Predicate visit(CompoundPredicate predicate) {
+ checkArgument(predicate.function() != null, "Compound predicate function cannot be null.");
+ checkArgument(predicate.children() != null, "Compound predicate children cannot be null.");
+ checkArgument(
+ !predicate.children().isEmpty(), "Compound predicate must contain a predicate.");
+ List<Predicate> children = new ArrayList<>();
+ for (Predicate child : predicate.children()) {
+ checkArgument(child != null, "Compound predicate child cannot be null.");
+ children.add(child.visit(this));
+ }
+ return children.size() == 1
+ ? children.get(0)
+ : new CompoundPredicate(predicate.function(), children);
+ }
+}
diff --git a/paimon-common/src/main/java/org/apache/paimon/utils/TypeUtils.java b/paimon-common/src/main/java/org/apache/paimon/utils/TypeUtils.java
index cfe0f95afb..6fdc97ecde 100644
--- a/paimon-common/src/main/java/org/apache/paimon/utils/TypeUtils.java
+++ b/paimon-common/src/main/java/org/apache/paimon/utils/TypeUtils.java
@@ -54,6 +54,7 @@ import java.util.Base64;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import java.util.TimeZone;
import java.util.stream.Collectors;
@@ -91,6 +92,24 @@ public class TypeUtils {
.collect(Collectors.toList()));
}
+ /**
+ * Append required fields available in the table schema to a read type. Existing fields,
+ * including nested projections, are preserved. Returns the original read type if unchanged.
+ */
+ public static RowType withMissingFields(
+ RowType tableType, RowType readType, Set<String> requiredFields) {
+ List<DataField> fields = null;
+ for (DataField field : tableType.getFields()) {
+ if (requiredFields.contains(field.name()) && !readType.containsField(field.name())) {
+ if (fields == null) {
+ fields = new ArrayList<>(readType.getFields());
+ }
+ fields.add(field);
+ }
+ }
+ return fields == null ? readType : readType.copy(fields);
+ }
+
public static Object castFromString(String s, DataType type) {
return castFromStringInternal(s, type, false);
}
diff --git a/paimon-common/src/test/java/org/apache/paimon/predicate/PredicateRemapperTest.java b/paimon-common/src/test/java/org/apache/paimon/predicate/PredicateRemapperTest.java
new file mode 100644
index 0000000000..e9f338427c
--- /dev/null
+++ b/paimon-common/src/test/java/org/apache/paimon/predicate/PredicateRemapperTest.java
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.predicate;
+
+import org.apache.paimon.data.BinaryString;
+import org.apache.paimon.data.GenericRow;
+import org.apache.paimon.types.DataField;
+import org.apache.paimon.types.DataTypes;
+import org.apache.paimon.types.RowType;
+import org.apache.paimon.utils.JsonSerdeUtil;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import java.util.Arrays;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+/** Tests exact predicate binding by name, including transforms and missing operands. */
+class PredicateRemapperTest {
+
+ private static final RowType TYPE =
+ RowType.of(
+ new DataField(17, "a", DataTypes.STRING()),
+ new DataField(41, "b", DataTypes.STRING()),
+ new DataField(71, "c", DataTypes.INT()));
+
+ @Test
+ void testCompoundPredicateOnReorderedFields() {
+ PredicateBuilder builder = new PredicateBuilder(TYPE);
+ Predicate predicate =
+ PredicateBuilder.and(
+ builder.equal(0, BinaryString.fromString("left")),
+ PredicateBuilder.or(
+ builder.equal(1, BinaryString.fromString("right")),
+ builder.greaterThan(2, 1)));
+ Predicate remapped = PredicateRemapper.remap(predicate, TYPE.project("b", "c", "a"));
+ assertThat(
+ remapped.test(
+ GenericRow.of(
+ BinaryString.fromString("right"),
+ 0,
+ BinaryString.fromString("left"))))
+ .isTrue();
+ assertThat(
+ remapped.test(
+ GenericRow.of(
+ BinaryString.fromString("other"),
+ 2,
+ BinaryString.fromString("left"))))
+ .isTrue();
+ assertThat(
+ remapped.test(
+ GenericRow.of(
+ BinaryString.fromString("other"),
+ 0,
+ BinaryString.fromString("left"))))
+ .isFalse();
+ assertThat(
+ remapped.test(
+ GenericRow.of(
+ BinaryString.fromString("right"),
+ 2,
+ BinaryString.fromString("wrong"))))
+ .isFalse();
+ // Binding must not mutate the original predicate or its positional references.
+ assertThat(
+ predicate.test(
+ GenericRow.of(
+ BinaryString.fromString("left"),
+ BinaryString.fromString("right"),
+ 0)))
+ .isTrue();
+ }
+
+ @Test
+ void testFieldIdsAndLiteralTransformInputs() {
+ Transform transform =
+ new ConcatWsTransform(
+ Arrays.asList(
+ BinaryString.fromString("-"),
+ new FieldRef(17, "a", DataTypes.STRING()),
+ new FieldRef(41, "b", DataTypes.STRING())));
+ RowType readType = TYPE.project("b", "a");
+ GenericRow row =
+ GenericRow.of(BinaryString.fromString("right"), BinaryString.fromString("left"));
+ assertThat(PredicateRemapper.remap(transform, readType).transform(row))
+ .isEqualTo(BinaryString.fromString("left-right"));
+ Predicate predicate =
+ new PredicateBuilder(TYPE).equal(transform, BinaryString.fromString("left-right"));
+ assertThat(PredicateRemapper.remap(predicate, readType).test(row)).isTrue();
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {false, true})
+ void testMissingConjunctOrDisjunctIsRejected(boolean and) {
+ PredicateBuilder builder = new PredicateBuilder(TYPE);
+ Predicate a = builder.isNotNull(0);
+ Predicate b = builder.isNotNull(1);
+ Predicate predicate = and ? PredicateBuilder.and(a, b) : PredicateBuilder.or(a, b);
+ assertThatThrownBy(() -> PredicateRemapper.remap(predicate, TYPE.project("a")))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("Cannot resolve field 'b'");
+ }
+
+ @Test
+ void testNullPredicateOnReorderedFields() {
+ Predicate predicate = new PredicateBuilder(TYPE).isNull(0);
+ Predicate remapped = PredicateRemapper.remap(predicate, TYPE.project("b", "a"));
+ assertThat(remapped.test(GenericRow.of(BinaryString.fromString("b"), null))).isTrue();
+ assertThat(remapped.test(GenericRow.of(null, BinaryString.fromString("a")))).isFalse();
+ }
+
+ @Test
+ void testMalformedCompoundIsRejected() {
+ for (String json :
+ Arrays.asList(
+ "{\"kind\":\"COMPOUND\",\"function\":\"AND\",\"children\":[]}",
+ "{\"kind\":\"COMPOUND\",\"function\":null,\"children\":[]}",
+ "{\"kind\":\"COMPOUND\",\"function\":\"AND\",\"children\":[null]}")) {
+ Predicate predicate = JsonSerdeUtil.fromJson(json, Predicate.class);
+ assertThatThrownBy(() -> PredicateRemapper.remap(predicate, TYPE))
+ .isInstanceOf(IllegalArgumentException.class);
+ }
+ }
+}
diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/TableQueryAuthResult.java b/paimon-core/src/main/java/org/apache/paimon/catalog/TableQueryAuthResult.java
index e3fd81442b..6ab47711dd 100644
--- a/paimon-core/src/main/java/org/apache/paimon/catalog/TableQueryAuthResult.java
+++ b/paimon-core/src/main/java/org/apache/paimon/catalog/TableQueryAuthResult.java
@@ -22,10 +22,9 @@ import org.apache.paimon.data.GenericRow;
import org.apache.paimon.data.InternalRow;
import org.apache.paimon.predicate.And;
import org.apache.paimon.predicate.CompoundPredicate;
-import org.apache.paimon.predicate.FieldRef;
-import org.apache.paimon.predicate.LeafPredicate;
import org.apache.paimon.predicate.Predicate;
import org.apache.paimon.predicate.PredicateBuilder;
+import org.apache.paimon.predicate.PredicateRemapper;
import org.apache.paimon.predicate.PredicateVisitor;
import org.apache.paimon.predicate.Transform;
import org.apache.paimon.reader.RecordReader;
@@ -43,6 +42,7 @@ import org.apache.paimon.utils.InternalRowUtils;
import org.apache.paimon.utils.JsonSerdeUtil;
import org.apache.paimon.utils.ListUtils;
import org.apache.paimon.utils.StringUtils;
+import org.apache.paimon.utils.TypeUtils;
import javax.annotation.Nullable;
@@ -186,16 +186,8 @@ public class TableQueryAuthResult implements Serializable {
@Nullable
public static RowType appendMissingFields(
RowType tableType, RowType readType, Set<String> ruleFields) {
- List<DataField> widenedFields = null;
- for (DataField field : tableType.getFields()) {
- if (ruleFields.contains(field.name()) && !readType.containsField(field.name())) {
- if (widenedFields == null) {
- widenedFields = new ArrayList<>(readType.getFields());
- }
- widenedFields.add(field);
- }
- }
- return widenedFields == null ? null : readType.copy(widenedFields);
+ RowType widened = TypeUtils.withMissingFields(tableType, readType, ruleFields);
+ return widened == readType ? null : widened;
}
public TableScan.Plan convertPlan(TableScan.Plan plan) {
@@ -246,7 +238,7 @@ public class TableQueryAuthResult implements Serializable {
*/
@Nullable
public static Predicate remapPredicate(Predicate predicate, RowType rowType) {
- return predicate.visit(new PredicateRemapper(rowType));
+ return PredicateRemapper.remap(predicate, rowType);
}
public Map<String, Transform> extractColumnMasking() {
@@ -415,6 +407,45 @@ public class TableQueryAuthResult implements Serializable {
tableType.getFieldNames());
}
+ /** Validate a physical projection before adding columns required by authorization rules. */
+ public void validateReadType(
+ RowType tableType,
+ RowType readType,
+ Set<String> ruleFields,
+ Set<String> resolvedBlobViewFields) {
+ Set<String> maskTargets = extractColumnMasking().keySet();
+ for (String name : readType.getFieldNames()) {
+ if (!ruleFields.contains(name) && !maskTargets.contains(name)) {
+ continue;
+ }
+ if (!tableType.containsField(name)) {
+ continue;
+ }
+ // rules must not touch a nested-pruned column (partial value)
+ DataField tableField = tableType.getField(name);
+ if (!readType.getField(name).type().equals(tableField.type())) {
+ throw new IllegalStateException(
+ String.format(
+ "Query auth rules involve column '%s', which the query "
+ + "projects with a pruned type %s instead of its "
+ + "table type %s; cannot apply the rules to a "
+ + "partial column.",
+ name, readType.getField(name).type(), tableField.type()));
+ }
+ }
+ for (String name : ruleFields) {
+ if (resolvedBlobViewFields.contains(name) && !readType.containsField(name)) {
+ // auth-added columns bypass blob-view resolution
+ throw new IllegalStateException(
+ String.format(
+ "Query auth rules read blob-view column '%s', which the query "
+ + "does not project; such columns cannot be resolved. "
+ + "Project the column or adjust the rule.",
+ name));
+ }
+ }
+ }
+
/**
* Applies the row filter and column masking to {@code reader}. Rules are remapped by name;
* masks apply only to targets in {@code activeFields}, the columns readable from the query.
@@ -481,82 +512,8 @@ public class TableQueryAuthResult implements Serializable {
targetColumn,
outputRowType);
- List<Object> newInputs = new ArrayList<>();
- for (Object input : transform.inputs()) {
- if (input instanceof FieldRef) {
- FieldRef ref = (FieldRef) input;
- int newIndex = outputRowType.getFieldIndex(ref.name());
- if (newIndex < 0) {
- throw new IllegalArgumentException(
- "Column masking refers to field '"
- + ref.name()
- + "' which is not present in output row type "
- + outputRowType);
- }
- DataType type = outputRowType.getTypeAt(newIndex);
- newInputs.add(new FieldRef(newIndex, ref.name(), type));
- } else {
- newInputs.add(input);
- }
- }
- out.put(targetIndex, transform.copyWithNewInputs(newInputs));
+ out.put(targetIndex, PredicateRemapper.remap(transform, outputRowType));
}
return out;
}
-
- private static class PredicateRemapper implements PredicateVisitor<Predicate> {
-
- private final RowType outputRowType;
-
- private PredicateRemapper(RowType outputRowType) {
- this.outputRowType = outputRowType;
- }
-
- @Override
- public Predicate visit(LeafPredicate predicate) {
- Transform transform = predicate.transform();
- List<Object> newInputs = new ArrayList<>();
- for (Object input : transform.inputs()) {
- if (input instanceof FieldRef) {
- FieldRef ref = (FieldRef) input;
- String fieldName = ref.name();
- int newIndex = outputRowType.getFieldIndex(fieldName);
- if (newIndex < 0) {
- throw new IllegalArgumentException(
- String.format(
- "Unable to read data without column %s when row filter enabled.",
- fieldName));
- }
- DataType type = outputRowType.getTypeAt(newIndex);
- newInputs.add(new FieldRef(newIndex, fieldName, type));
- } else {
- newInputs.add(input);
- }
- }
- return predicate.copyWithNewInputs(newInputs);
- }
-
- @Override
- public Predicate visit(CompoundPredicate predicate) {
- checkArgument(
- predicate.function() != null, "Compound row filter function cannot be null.");
- checkArgument(
- predicate.children() != null, "Compound row filter children cannot be null.");
- List<Predicate> remappedChildren = new ArrayList<>();
- for (Predicate child : predicate.children()) {
- checkArgument(child != null, "Compound row filter child cannot be null.");
- Predicate remapped = child.visit(this);
- if (remapped != null) {
- remappedChildren.add(remapped);
- }
- }
- if (remappedChildren.isEmpty()) {
- throw new IllegalArgumentException("Compound row filter must contain a predicate.");
- }
- if (remappedChildren.size() == 1) {
- return remappedChildren.get(0);
- }
- return new CompoundPredicate(predicate.function(), remappedChildren);
- }
- }
}
diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/DataEvolutionFileStoreScan.java b/paimon-core/src/main/java/org/apache/paimon/operation/DataEvolutionFileStoreScan.java
index ee054d445c..3523b1a62e 100644
--- a/paimon-core/src/main/java/org/apache/paimon/operation/DataEvolutionFileStoreScan.java
+++ b/paimon-core/src/main/java/org/apache/paimon/operation/DataEvolutionFileStoreScan.java
@@ -27,6 +27,7 @@ import org.apache.paimon.manifest.ManifestEntry;
import org.apache.paimon.manifest.ManifestFile;
import org.apache.paimon.manifest.ManifestFileMeta;
import org.apache.paimon.predicate.Predicate;
+import org.apache.paimon.predicate.PredicateVisitor;
import org.apache.paimon.reader.DataEvolutionArray;
import org.apache.paimon.reader.DataEvolutionRow;
import org.apache.paimon.schema.SchemaManager;
@@ -215,9 +216,9 @@ public class DataEvolutionFileStoreScan extends AppendOnlyFileStoreScan {
/**
* Per-file column pruning within a row-id-range group: drop files whose physical columns have
- * no overlap with the query's {@code readType}. Necessary for columnar-split DE scenarios where
- * a logical row is reconstructed from multiple files in the same row id range — a query that
- * does not reference a file's columns has no reason to read it.
+ * no overlap with the query's {@code readType} or filter. Necessary for columnar-split DE
+ * scenarios where a logical row is reconstructed from multiple files in the same row id range —
+ * a query that does not reference a file's columns has no reason to read it.
*
* <p>When every file in the group lacks a requested column (e.g. an ADD COLUMN projection over
* a row-disjoint pre-ALTER group), one file is kept as a row-count representative so the reader
@@ -236,6 +237,16 @@ public class DataEvolutionFileStoreScan extends AppendOnlyFileStoreScan {
for (DataField f : readType.getFields()) {
readFieldIds.add(f.id());
}
+ if (inputFilter != null) {
+ // executeFilter may need columns absent from the output projection. Keep their latest
+ // files too, otherwise widening the reader could see an older value or a null.
+ Set<String> filterFields = PredicateVisitor.collectFieldNames(inputFilter);
+ for (DataField field : schema.fields()) {
+ if (filterFields.contains(field.name())) {
+ readFieldIds.add(field.id());
+ }
+ }
+ }
List<ManifestEntry> kept = new ArrayList<>(group.size());
for (ManifestEntry entry : group) {
Set<Integer> fileIds = fileFieldIdsForEntry(entry);
diff --git a/paimon-core/src/main/java/org/apache/paimon/table/format/FormatReadBuilder.java b/paimon-core/src/main/java/org/apache/paimon/table/format/FormatReadBuilder.java
index 924bb3c9dc..97762e225c 100644
--- a/paimon-core/src/main/java/org/apache/paimon/table/format/FormatReadBuilder.java
+++ b/paimon-core/src/main/java/org/apache/paimon/table/format/FormatReadBuilder.java
@@ -190,12 +190,18 @@ public class FormatReadBuilder implements ReadBuilder {
protected RecordReader<InternalRow> createReader(
FormatDataSplit dataSplit, @Nullable ReadBatchSizer readBatchSizer) throws IOException {
+ return createReader(dataSplit, readBatchSizer, readType());
+ }
+
+ protected RecordReader<InternalRow> createReader(
+ FormatDataSplit dataSplit, @Nullable ReadBatchSizer readBatchSizer, RowType readType)
+ throws IOException {
// Skip pushing down partition filters to reader.
List<Predicate> readFilters =
excludePredicateWithFields(
PredicateBuilder.splitAnd(filter), new HashSet<>(table.partitionKeys()));
RowType dataRowType = getRowTypeWithoutPartition(table.rowType(), table.partitionKeys());
- RowType readRowType = getRowTypeWithoutPartition(readType(), table.partitionKeys());
+ RowType readRowType = getRowTypeWithoutPartition(readType, table.partitionKeys());
FormatReaderFactory readerFactory =
FileFormatDiscover.of(options)
.discover(options.formatType())
@@ -203,7 +209,7 @@ public class FormatReadBuilder implements ReadBuilder {
Pair<int[], RowType> partitionMapping =
PartitionUtils.getPartitionMapping(
- table.partitionKeys(), readType().getFields(), table.partitionType());
+ table.partitionKeys(), readType.getFields(), table.partitionType());
BinaryRow partition = dataSplit.partition();
FileIO fileIO = fileIOResolver().fileIO(dataSplit.useCatalogContextFileIO());
@@ -217,7 +223,8 @@ public class FormatReadBuilder implements ReadBuilder {
partition,
readerFactory,
partitionMapping,
- readBatchSizer));
+ readBatchSizer,
+ readType));
}
return ConcatRecordReader.create(suppliers);
}
@@ -228,7 +235,8 @@ public class FormatReadBuilder implements ReadBuilder {
@Nullable BinaryRow partition,
FormatReaderFactory readerFactory,
Pair<int[], RowType> partitionMapping,
- @Nullable ReadBatchSizer readBatchSizer)
+ @Nullable ReadBatchSizer readBatchSizer,
+ RowType readType)
throws IOException {
FormatReaderContext formatReaderContext =
new FormatReaderContext(
@@ -243,7 +251,7 @@ public class FormatReadBuilder implements ReadBuilder {
reader = readerFactory.createReader(formatReaderContext);
}
return new DataFileRecordReader(
- readType(),
+ readType,
reader,
options.scanIgnoreCorruptFile(),
options.scanIgnoreLostFile(),
diff --git a/paimon-core/src/main/java/org/apache/paimon/table/format/FormatTableRead.java b/paimon-core/src/main/java/org/apache/paimon/table/format/FormatTableRead.java
index 18e3ce3505..51f3d2b1e4 100644
--- a/paimon-core/src/main/java/org/apache/paimon/table/format/FormatTableRead.java
+++ b/paimon-core/src/main/java/org/apache/paimon/table/format/FormatTableRead.java
@@ -22,11 +22,11 @@ import org.apache.paimon.data.InternalRow;
import org.apache.paimon.disk.IOManager;
import org.apache.paimon.metrics.MetricRegistry;
import org.apache.paimon.predicate.Predicate;
-import org.apache.paimon.predicate.PredicateProjectionConverter;
import org.apache.paimon.reader.LimitRecordReader;
import org.apache.paimon.reader.ReadBatchSizer;
import org.apache.paimon.reader.RecordReader;
import org.apache.paimon.table.FormatTable;
+import org.apache.paimon.table.source.ReadTransform;
import org.apache.paimon.table.source.Split;
import org.apache.paimon.table.source.TableRead;
import org.apache.paimon.types.RowType;
@@ -34,7 +34,7 @@ import org.apache.paimon.types.RowType;
import javax.annotation.Nullable;
import java.io.IOException;
-import java.util.Optional;
+import java.util.Collections;
/** A {@link TableRead} implementation for {@link FormatTable}. */
public class FormatTableRead implements TableRead {
@@ -55,7 +55,7 @@ public class FormatTableRead implements TableRead {
Predicate predicate,
Integer limit) {
this.tableRowType = tableRowType;
- this.readType = readType;
+ this.readType = readType == null ? tableRowType : readType;
this.read = read;
this.predicate = predicate;
this.limit = limit;
@@ -89,30 +89,16 @@ public class FormatTableRead implements TableRead {
// Capture the binding per TableRead so lazy file suppliers cannot observe another read's
// sizer.
ReadBatchSizer sizer = this.readBatchSizer;
- RecordReader<InternalRow> reader = read.createReader(dataSplit, sizer);
- if (executeFilter) {
- reader = executeFilter(reader);
- }
+ ReadTransform transform =
+ ReadTransform.create(
+ tableRowType,
+ readType,
+ predicate,
+ executeFilter,
+ null,
+ Collections.emptySet());
+ RecordReader<InternalRow> reader =
+ transform.apply(read.createReader(dataSplit, sizer, transform.readType()));
return LimitRecordReader.limit(reader, limit);
}
-
- private RecordReader<InternalRow> executeFilter(RecordReader<InternalRow> reader) {
- if (predicate == null) {
- return reader;
- }
-
- Predicate predicate = this.predicate;
- if (readType != null) {
- int[] projection = tableRowType.getFieldIndices(readType.getFieldNames());
- Optional<Predicate> optional =
- predicate.visit(PredicateProjectionConverter.fromProjection(projection));
- if (!optional.isPresent()) {
- return reader;
- }
- predicate = optional.get();
- }
-
- Predicate finalFilter = predicate;
- return reader.filter(finalFilter::test);
- }
}
diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataTableRead.java b/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataTableRead.java
index 7b14dd1cac..4c46ddd4cb 100644
--- a/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataTableRead.java
+++ b/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataTableRead.java
@@ -23,24 +23,14 @@ import org.apache.paimon.catalog.TableQueryAuthResult;
import org.apache.paimon.data.InternalRow;
import org.apache.paimon.disk.IOManager;
import org.apache.paimon.predicate.Predicate;
-import org.apache.paimon.predicate.PredicateProjectionConverter;
-import org.apache.paimon.predicate.PredicateVisitor;
-import org.apache.paimon.predicate.Transform;
import org.apache.paimon.reader.RecordReader;
import org.apache.paimon.schema.TableSchema;
-import org.apache.paimon.types.DataField;
import org.apache.paimon.types.RowType;
-import org.apache.paimon.utils.ProjectedRow;
import javax.annotation.Nullable;
import java.io.IOException;
import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
import java.util.Set;
/** A {@link InnerTableRead} for data table. */
@@ -55,9 +45,6 @@ public abstract class AbstractDataTableRead implements InnerTableRead {
// as read-level TopN already does (see ReadBuilderImpl)
private final boolean queryAuthEnabled;
- // the auth-widened read type currently applied, or null when the plain read type is
- @Nullable private RowType appliedReadType;
-
// blob-view columns that only resolve through the dedicated blob-view read path
private final Set<String> resolvedBlobViewFields;
@@ -113,7 +100,6 @@ public abstract class AbstractDataTableRead implements InnerTableRead {
@Override
public final InnerTableRead withReadType(RowType readType) {
this.readType = readType;
- this.appliedReadType = null;
applyReadType(readType);
return this;
}
@@ -147,173 +133,26 @@ public abstract class AbstractDataTableRead implements InnerTableRead {
protected final RecordReader<InternalRow> createDataReader(
Split split, @Nullable TableQueryAuthResult authResult) throws IOException {
- // A TableRead can be reused for multiple splits. Authentication may have expanded an
- // explicitly configured physical projection for the previous split, so restore it before
- // applying the current split's authorization dependencies. Without an explicit projection,
- // the underlying reader must retain its own default read type.
- if (readType != null) {
- applyReadType(readType);
- appliedReadType = null;
- }
- RecordReader<InternalRow> reader;
- if (authResult == null) {
- reader = backProject(reader(split));
- } else {
- reader = authedReader(split, authResult);
- }
- if (executeFilter) {
- reader = executeFilter(reader);
- }
-
- return reader;
- }
-
- private RecordReader<InternalRow> authedReader(Split split, TableQueryAuthResult authResult)
- throws IOException {
- List<String> readFields = currentReadType().getFieldNames();
- // masked filter columns are read and masked like rule fields, then evaluated post-mask
- Set<String> maskedFilterFields =
- maskedFilterFields(authResult.extractColumnMasking().keySet());
- Set<String> ruleFields = authResult.authFields(readFields, predicate);
- RowType widened = widenedReadType(authResult, ruleFields);
- if (widened != null && !widened.equals(appliedReadType)) {
- applyReadType(widened);
- appliedReadType = widened;
- }
- // the split read emits appliedReadType; rules are remapped against it by name
- RowType outputType = appliedReadType != null ? appliedReadType : currentReadType();
- // masks apply only to columns readable from the query: the ones it projects plus the
- // ones the rules pulled in; a mask on anything else is inert
- Map<String, Transform> masking = authResult.extractColumnMasking();
- Map<String, Transform> selectedColumnMasking = Collections.emptyMap();
- if (!masking.isEmpty()) {
- Set<String> activeFields = new HashSet<>(readFields);
- activeFields.addAll(ruleFields);
- selectedColumnMasking = new HashMap<>();
- for (Map.Entry<String, Transform> mask : masking.entrySet()) {
- if (activeFields.contains(mask.getKey())) {
- selectedColumnMasking.put(mask.getKey(), mask.getValue());
- }
- }
- }
- RecordReader<InternalRow> reader =
- authResult.doAuth(
- reader(split),
- outputType,
- authResult.extractPredicate(),
- selectedColumnMasking);
- reader = filterMaskedConjuncts(reader, outputType, maskedFilterFields);
- return backProject(reader);
- }
-
- private Set<String> maskedFilterFields(Set<String> maskTargets) {
- if (predicate == null || maskTargets.isEmpty()) {
- return Collections.emptySet();
- }
- Set<String> fields = new HashSet<>(PredicateVisitor.collectFieldNames(predicate));
- fields.retainAll(maskTargets);
- return fields;
- }
-
- /**
- * Evaluates the filter conjuncts on masked columns, on the masked output: they are never pushed
- * down, and engines do not re-evaluate the conjuncts they consumed.
- */
- private RecordReader<InternalRow> filterMaskedConjuncts(
- RecordReader<InternalRow> reader, RowType outputType, Set<String> maskedFilterFields) {
- if (maskedFilterFields.isEmpty()) {
- return reader;
- }
- Predicate maskedPart = TableQueryAuthResult.retainFields(predicate, maskedFilterFields);
- if (maskedPart == null) {
- return reader;
- }
- // by name against the emitted schema: it may carry system fields the table schema lacks
- Predicate filter;
- try {
- filter = TableQueryAuthResult.remapPredicate(maskedPart, outputType);
- } catch (RuntimeException e) {
- throw new IllegalStateException(
- "Filter on masked columns "
- + maskedFilterFields
- + " cannot be evaluated on read schema "
- + outputType.getFieldNames(),
- e);
- }
- return reader.filter(filter::test);
- }
-
- /** Project auth-widened rows back to the read type the query asked for. */
- private RecordReader<InternalRow> backProject(RecordReader<InternalRow> reader) {
- if (appliedReadType == null) {
- return reader;
- }
- ProjectedRow backRow =
- ProjectedRow.from(
- appliedReadType.projectIndexes(currentReadType().getFieldNames()));
- return reader.transform(backRow::replaceRow);
- }
-
- /**
- * The read type widened with the unprojected columns the auth rules read, or null when the
- * projection already covers them. Projected fields are kept as-is to preserve nested pruning.
- */
- @Nullable
- private RowType widenedReadType(TableQueryAuthResult authResult, Set<String> ruleFields) {
- RowType tableType = schema.logicalRowType();
- RowType readType = currentReadType();
- Set<String> maskTargets = authResult.extractColumnMasking().keySet();
- for (String name : readType.getFieldNames()) {
- if (!ruleFields.contains(name) && !maskTargets.contains(name)) {
- continue;
- }
- if (!tableType.containsField(name)) {
- continue;
- }
- // rules must not touch a nested-pruned column (partial value)
- DataField tableField = tableType.getField(name);
- if (!readType.getField(name).type().equals(tableField.type())) {
- throw new IllegalStateException(
- String.format(
- "Query auth rules involve column '%s', which the query "
- + "projects with a pruned type %s instead of its "
- + "table type %s; cannot apply the rules to a "
- + "partial column.",
- name, readType.getField(name).type(), tableField.type()));
- }
- }
- for (String name : ruleFields) {
- if (resolvedBlobViewFields.contains(name) && !readType.containsField(name)) {
- // auth-added columns bypass blob-view resolution
- throw new IllegalStateException(
- String.format(
- "Query auth rules read blob-view column '%s', which the query "
- + "does not project; such columns cannot be resolved. "
- + "Project the column or adjust the rule.",
- name));
+ if (authResult == null && !(executeFilter && predicate != null)) {
+ // Restore an explicit projection after a previous split needed authorization columns.
+ // Without a projection, preserve the underlying reader's default read type.
+ if (readType != null) {
+ applyReadType(readType);
}
- }
- return TableQueryAuthResult.appendMissingFields(tableType, readType, ruleFields);
- }
-
- private RecordReader<InternalRow> executeFilter(RecordReader<InternalRow> reader) {
- if (predicate == null) {
- return reader;
- }
-
- Predicate predicate = this.predicate;
- if (readType != null) {
- int[] projection = schema.logicalRowType().getFieldIndices(readType.getFieldNames());
- Optional<Predicate> optional =
- predicate.visit(PredicateProjectionConverter.fromProjection(projection));
- if (!optional.isPresent()) {
- return reader;
- }
- predicate = optional.get();
- }
-
- Predicate finalFilter = predicate;
- return reader.filter(finalFilter::test);
+ return reader(split);
+ }
+ ReadTransform transform =
+ ReadTransform.create(
+ schema.logicalRowType(),
+ currentReadType(),
+ predicate,
+ executeFilter,
+ authResult,
+ resolvedBlobViewFields);
+ if (readType != null || !transform.readType().equals(currentReadType())) {
+ applyReadType(transform.readType());
+ }
+ return transform.apply(reader(split));
}
/** Split with auth context. */
diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataTableScan.java b/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataTableScan.java
index 0cdef986fe..8be73caff6 100644
--- a/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataTableScan.java
+++ b/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataTableScan.java
@@ -30,6 +30,7 @@ import org.apache.paimon.operation.FileStoreScan;
import org.apache.paimon.options.Options;
import org.apache.paimon.partition.PartitionPredicate;
import org.apache.paimon.predicate.Predicate;
+import org.apache.paimon.predicate.PredicateRemapper;
import org.apache.paimon.predicate.PredicateVisitor;
import org.apache.paimon.schema.SchemaManager;
import org.apache.paimon.schema.TableSchema;
@@ -62,14 +63,17 @@ import org.apache.paimon.utils.Range;
import org.apache.paimon.utils.RowRangeIndex;
import org.apache.paimon.utils.SnapshotManager;
import org.apache.paimon.utils.TagManager;
+import org.apache.paimon.utils.TypeUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.annotation.Nullable;
+import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
+import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
@@ -167,7 +171,7 @@ abstract class AbstractDataTableScan implements DataTableScan {
// Remap field-id FieldRefs to positional indices by name (as doAuth does on read), so
// pruning stays correct across schema evolution.
Predicate remappedAuth =
- TableQueryAuthResult.remapPredicate(authPredicate, schema.logicalRowType());
+ PredicateRemapper.remap(authPredicate, schema.logicalRowType());
if (remappedAuth != null) {
Pair<Optional<PartitionPredicate>, List<Predicate>> split =
PartitionPredicate.splitPartitionPredicatesAndDataPredicates(
@@ -279,6 +283,14 @@ abstract class AbstractDataTableScan implements DataTableScan {
return null;
}
List<String> select = readType == null ? null : readType.getFieldNames();
+ if (select != null && (userFilter != null || !partitionFilterFields.isEmpty())) {
+ // Authorize query operands before pruning or reading their values. Dependencies of
+ // trusted row filters and masks are supplied by the catalog and are not user selects.
+ Set<String> fields = new LinkedHashSet<>(select);
+ fields.addAll(PredicateVisitor.collectFieldNames(userFilter));
+ fields.addAll(partitionFilterFields);
+ select = new ArrayList<>(fields);
+ }
TableQueryAuthResult result = queryAuth.auth(select);
if (result != null && result.hasRules()) {
// re-validated every plan, so a schema change under a live scan fails closed
@@ -320,11 +332,23 @@ abstract class AbstractDataTableScan implements DataTableScan {
/** The partition columns a predicate references, or all of them when it cannot be read. */
private Set<String> partitionPredicateFields(PartitionPredicate partitionPredicate) {
+ if (partitionPredicate == PartitionPredicate.ALWAYS_TRUE
+ || partitionPredicate == PartitionPredicate.ALWAYS_FALSE) {
+ return Collections.emptySet();
+ }
if (partitionPredicate instanceof PartitionPredicate.DefaultPartitionPredicate) {
return PredicateVisitor.collectFieldNames(
((PartitionPredicate.DefaultPartitionPredicate) partitionPredicate)
.predicate());
}
+ if (partitionPredicate instanceof PartitionPredicate.AndPartitionPredicate) {
+ Set<String> fields = new HashSet<>();
+ for (PartitionPredicate child :
+ ((PartitionPredicate.AndPartitionPredicate) partitionPredicate).predicates()) {
+ fields.addAll(partitionPredicateFields(child));
+ }
+ return fields;
+ }
return new HashSet<>(schema.partitionKeys());
}
@@ -383,34 +407,24 @@ abstract class AbstractDataTableScan implements DataTableScan {
}
/**
- * Push the auth-widened read type to the snapshot reader before planning, so file-level column
- * pruning keeps the files of the columns the rules read.
+ * Push the read type expanded for filters and auth rules to the snapshot reader before
+ * planning, so file-level column pruning keeps their dependencies.
*/
private void applyAuthReadType(@Nullable TableQueryAuthResult queryAuthResult) {
if (readType == null) {
return;
}
- RowType desired = readType;
- if (queryAuthResult != null && queryAuthResult.hasRules()) {
- // post-mask conjuncts are evaluated at read time; their columns must survive planning
- RowType widened =
- TableQueryAuthResult.appendMissingFields(
- schema.logicalRowType(),
- readType,
- queryAuthResult.authFields(readType.getFieldNames(), userFilter));
- if (widened != null) {
- desired = widened;
- }
- }
- // never narrow within this scan's lifetime: readers fix their schema on first use
- RowType widenedToApplied =
- TableQueryAuthResult.appendMissingFields(
+ RowType desired =
+ TypeUtils.withMissingFields(
+ schema.logicalRowType(),
+ readType,
+ ReadTransform.requiredFields(readType, userFilter, queryAuthResult));
+ // Never narrow within this scan's lifetime: readers may retain their physical schema.
+ desired =
+ TypeUtils.withMissingFields(
appliedScanReadType,
desired,
new HashSet<>(appliedScanReadType.getFieldNames()));
- if (widenedToApplied != null) {
- desired = widenedToApplied;
- }
if (!desired.equals(appliedScanReadType)) {
snapshotReader.withReadType(desired);
appliedScanReadType = desired;
diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/ReadTransform.java b/paimon-core/src/main/java/org/apache/paimon/table/source/ReadTransform.java
new file mode 100644
index 0000000000..33f2722cc9
--- /dev/null
+++ b/paimon-core/src/main/java/org/apache/paimon/table/source/ReadTransform.java
@@ -0,0 +1,178 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.table.source;
+
+import org.apache.paimon.catalog.TableQueryAuthResult;
+import org.apache.paimon.data.InternalRow;
+import org.apache.paimon.predicate.Predicate;
+import org.apache.paimon.predicate.PredicateRemapper;
+import org.apache.paimon.predicate.PredicateVisitor;
+import org.apache.paimon.predicate.Transform;
+import org.apache.paimon.reader.RecordReader;
+import org.apache.paimon.types.DataField;
+import org.apache.paimon.types.RowType;
+import org.apache.paimon.utils.NestedProjectedRow;
+import org.apache.paimon.utils.TypeUtils;
+
+import javax.annotation.Nullable;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static org.apache.paimon.utils.Preconditions.checkArgument;
+
+/**
+ * The physical read type and ordered transformations for one reader. Create a new instance for each
+ * split so authorization rules and query settings cannot leak between readers.
+ */
+public final class ReadTransform {
+
+ private final RowType readType;
+ private final RowType outputType;
+ @Nullable private final TableQueryAuthResult authResult;
+ private final Map<String, Transform> masking;
+ @Nullable private final Predicate filter;
+
+ private ReadTransform(
+ RowType readType,
+ RowType outputType,
+ @Nullable TableQueryAuthResult authResult,
+ Map<String, Transform> masking,
+ @Nullable Predicate filter) {
+ this.readType = readType;
+ this.outputType = outputType;
+ this.authResult = authResult;
+ this.masking = masking;
+ this.filter = filter;
+ }
+
+ public static ReadTransform create(
+ RowType tableType,
+ RowType outputType,
+ @Nullable Predicate queryFilter,
+ boolean executeFilter,
+ @Nullable TableQueryAuthResult authResult,
+ Set<String> resolvedBlobViewFields) {
+ Map<String, Transform> masks =
+ authResult == null ? Collections.emptyMap() : authResult.extractColumnMasking();
+ // Without executeFilter, engines still rely on us to evaluate conjuncts on masked columns.
+ Predicate filter =
+ executeFilter ? queryFilter : maskedQueryFilter(queryFilter, masks.keySet());
+ RowType filterType =
+ executeFilter && queryFilter != null
+ ? withFilterFields(tableType, outputType, queryFilter)
+ : outputType;
+ Set<String> required = requiredFields(outputType, filter, authResult);
+ if (authResult != null) {
+ authResult.validateReadType(tableType, filterType, required, resolvedBlobViewFields);
+ }
+ RowType readType = TypeUtils.withMissingFields(tableType, filterType, required);
+
+ Map<String, Transform> selectedMasks = new HashMap<>();
+ Set<String> active = new HashSet<>(filterType.getFieldNames());
+ active.addAll(required);
+ for (Map.Entry<String, Transform> mask : masks.entrySet()) {
+ if (active.contains(mask.getKey())) {
+ selectedMasks.put(mask.getKey(), mask.getValue());
+ }
+ }
+ if (filter != null) {
+ try {
+ filter = PredicateRemapper.remap(filter, readType);
+ } catch (RuntimeException e) {
+ if (executeFilter) {
+ throw e;
+ }
+ throw new IllegalStateException(
+ "Filter on masked columns "
+ + masks.keySet()
+ + " cannot be evaluated on read schema "
+ + readType.getFieldNames(),
+ e);
+ }
+ }
+ return new ReadTransform(readType, outputType, authResult, selectedMasks, filter);
+ }
+
+ /**
+ * Columns needed in addition to the output projection. Planning passes the full query filter so
+ * its files survive pruning; readers pass the query predicate they will actually execute.
+ */
+ public static Set<String> requiredFields(
+ RowType outputType,
+ @Nullable Predicate queryFilter,
+ @Nullable TableQueryAuthResult authResult) {
+ Set<String> fields = new HashSet<>(PredicateVisitor.collectFieldNames(queryFilter));
+ if (authResult != null) {
+ Set<String> visible = new HashSet<>(outputType.getFieldNames());
+ visible.addAll(fields);
+ fields.addAll(authResult.authFields(new ArrayList<>(visible), queryFilter));
+ }
+ return fields;
+ }
+
+ public RowType readType() {
+ return readType;
+ }
+
+ /** Apply authorization, masking, the query filter, and the output projection, in that order. */
+ public RecordReader<InternalRow> apply(RecordReader<InternalRow> reader) {
+ if (authResult != null) {
+ reader = authResult.doAuth(reader, readType, authResult.extractPredicate(), masking);
+ }
+ if (filter != null) {
+ reader = reader.filter(filter::test);
+ }
+ NestedProjectedRow projection = NestedProjectedRow.create(readType, outputType);
+ return projection == null ? reader : reader.transform(projection::replaceRow);
+ }
+
+ @Nullable
+ private static Predicate maskedQueryFilter(
+ @Nullable Predicate filter, Set<String> maskTargets) {
+ return filter == null || maskTargets.isEmpty()
+ ? null
+ : TableQueryAuthResult.retainFields(filter, maskTargets);
+ }
+
+ private static RowType withFilterFields(
+ RowType tableType, RowType readType, Predicate predicate) {
+ Set<String> fields = PredicateVisitor.collectFieldNames(predicate);
+ RowType widened = TypeUtils.withMissingFields(tableType, readType, fields);
+ List<DataField> fullFields = new ArrayList<>(widened.getFields());
+ for (int i = 0; i < fullFields.size(); i++) {
+ DataField field = fullFields.get(i);
+ if (fields.contains(field.name()) && tableType.containsField(field.name())) {
+ // Filter operands use their full type; nested pruning is restored after filtering.
+ fullFields.set(i, tableType.getField(field.name()));
+ }
+ }
+ checkArgument(
+ widened.getFieldNames().containsAll(fields),
+ "Cannot execute filter on fields %s with read type %s.",
+ fields,
+ widened);
+ return widened.copy(fullFields);
+ }
+}
diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/TableRead.java b/paimon-core/src/main/java/org/apache/paimon/table/source/TableRead.java
index 930a924ae6..91e0b7d014 100644
--- a/paimon-core/src/main/java/org/apache/paimon/table/source/TableRead.java
+++ b/paimon-core/src/main/java/org/apache/paimon/table/source/TableRead.java
@@ -43,6 +43,18 @@ public interface TableRead {
/** Set {@link MetricRegistry} to table read. */
TableRead withMetricRegistry(MetricRegistry registry);
+ /**
+ * Enable row-level evaluation of the complete configured filter for subsequently created
+ * readers. Without this call, filter pushdown may only prune files or row groups and can return
+ * rows that do not satisfy the filter.
+ *
+ * <p>Fields referenced by the filter are read even if they are absent from the requested read
+ * type. The filter is evaluated before the final output projection, so the returned rows retain
+ * the requested fields and their order. For tables with query authorization, the filter is
+ * evaluated after authorization and column masking.
+ *
+ * @return this read
+ */
TableRead executeFilter();
TableRead withIOManager(IOManager ioManager);
diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java
index 1cf891dbd3..05f62688eb 100644
--- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java
@@ -51,6 +51,7 @@ import org.apache.paimon.operation.BaseAppendFileStoreWrite;
import org.apache.paimon.operation.FileStoreWrite;
import org.apache.paimon.options.Options;
import org.apache.paimon.partition.Partition;
+import org.apache.paimon.partition.PartitionPredicate;
import org.apache.paimon.partition.PartitionStatistics;
import org.apache.paimon.predicate.CastTransform;
import org.apache.paimon.predicate.ConcatTransform;
@@ -126,6 +127,8 @@ import org.apache.commons.io.FileUtils;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
import org.mockito.Mockito;
import java.io.File;
@@ -3097,6 +3100,84 @@ public abstract class RESTCatalogTest extends CatalogTestBase {
table.newReadBuilder().withProjection(new int[] {1}).newScan().plan();
}
+ @ParameterizedTest
+ @CsvSource({"false,false", "true,false", "false,true", "true,true"})
+ void testTableAuthIncludesUnprojectedFilterFields(boolean streaming, boolean partitionFilter)
+ throws Exception {
+ Identifier identifier = Identifier.create("test_table_db", "auth_filter_columns");
+ Table table =
+ createMaskingAuthTable(
+ identifier,
+ Arrays.asList(
+ new DataField(0, "public_id", DataTypes.INT()),
+ new DataField(1, "secret_score", DataTypes.INT())),
+ partitionFilter ? singletonList("secret_score") : Collections.emptyList(),
+ Collections.emptyList(),
+ Collections.emptyMap());
+ commitRows(table, GenericRow.of(1, 10), GenericRow.of(2, 20), GenericRow.of(3, 30));
+ authTableColumns(identifier, singletonList("public_id"));
+
+ assertThatThrownBy(
+ () -> table.newReadBuilder().withProjection(new int[] {1}).newScan().plan())
+ .hasMessageContaining("has no permission");
+
+ ReadBuilder readBuilder = table.newReadBuilder().withProjection(new int[] {0});
+ if (partitionFilter) {
+ readBuilder.withPartitionFilter(singletonMap("secret_score", "20"));
+ } else {
+ readBuilder.withFilter(new PredicateBuilder(table.rowType()).equal(1, 20));
+ }
+ TableScan scan = streaming ? readBuilder.newStreamScan() : readBuilder.newScan();
+ TableRead read = readBuilder.newRead().executeFilter();
+ assertThatThrownBy(
+ () -> {
+ try (RecordReader<InternalRow> reader =
+ read.createReader(scan.plan().splits())) {
+ reader.forEachRemaining(row -> {});
+ }
+ })
+ .hasMessageContaining("has no permission");
+
+ // Granting the filter operand allows the same query without exposing the extra column.
+ authTableColumns(identifier, Arrays.asList("public_id", "secret_score"));
+ List<InternalRow> rows =
+ collectRows(
+ read.createReader(scan.plan().splits()),
+ table.rowType().project("public_id"));
+ assertThat(rows).hasSize(1);
+ assertThat(rows.get(0).getFieldCount()).isEqualTo(1);
+ assertThat(rows.get(0).getInt(0)).isEqualTo(2);
+ }
+
+ @Test
+ void testTableAuthWithCombinedPartitionFilter() throws Exception {
+ Identifier identifier =
+ Identifier.create("test_table_db", "auth_combined_partition_filter");
+ Table table =
+ createMaskingAuthTable(
+ identifier,
+ stringFields("p1", "p2", "v"),
+ Arrays.asList("p1", "p2"),
+ Collections.emptyList(),
+ Collections.emptyMap());
+ writeStringRows(table, new String[] {"x", "a", "v1"}, new String[] {"y", "b", "v2"});
+ authTableColumns(identifier, Arrays.asList("p1", "v"));
+
+ RowType partitionType = table.rowType().project(table.partitionKeys());
+ Predicate onP1 = new PredicateBuilder(partitionType).equal(0, BinaryString.fromString("x"));
+ ReadBuilder readBuilder =
+ table.newReadBuilder()
+ .withProjection(new int[] {2})
+ .withPartitionFilter(
+ PartitionPredicate.and(
+ Arrays.asList(
+ PartitionPredicate.fromPredicate(
+ partitionType, onP1),
+ PartitionPredicate.ALWAYS_TRUE)));
+ assertThat(batchRead(table, readBuilder.newScan().plan().splits(), readBuilder))
+ .containsExactly("+I[v1]");
+ }
+
@Test
void testSnapshotMethods() throws Exception {
Identifier identifier = Identifier.create("test_table_db", "snapshots_table");
@@ -4076,6 +4157,9 @@ public abstract class RESTCatalogTest extends CatalogTestBase {
new String[] {"jane", "roe", "ignored", "o2"});
maskDisplayWithFullName(identifier);
+ // Trusted masking inputs do not require the caller to have column access.
+ authTableColumns(identifier, singletonList("display"));
+
ReadBuilder readBuilder = table.newReadBuilder().withProjection(new int[] {2});
List<Split> splits = readBuilder.newScan().plan().splits();
List<InternalRow> rows =
@@ -4145,6 +4229,9 @@ public abstract class RESTCatalogTest extends CatalogTestBase {
setRowFilter(identifier, Collections.singletonList(displayFilter));
maskDisplayWithFullName(identifier);
+ // Trusted row-filter inputs do not require the caller to have column access either.
+ authTableColumns(identifier, singletonList("other"));
+
ReadBuilder readBuilder = table.newReadBuilder().withProjection(new int[] {3});
List<Split> splits = readBuilder.newScan().plan().splits();
List<InternalRow> rows =
diff --git a/paimon-core/src/test/java/org/apache/paimon/table/AppendOnlySimpleTableTest.java b/paimon-core/src/test/java/org/apache/paimon/table/AppendOnlySimpleTableTest.java
index 0ee799c032..dd6f531f8c 100644
--- a/paimon-core/src/test/java/org/apache/paimon/table/AppendOnlySimpleTableTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/table/AppendOnlySimpleTableTest.java
@@ -614,7 +614,7 @@ public class AppendOnlySimpleTableTest extends SimpleTableTestBase {
assertThat(getResult(read, splits, binaryRow(2), 0, toString))
.hasSameElementsAs(Arrays.asList("201|binary", "201|binary"));
- // projection contains unknown index or
+ // OR includes a field outside the output projection.
read =
table.newRead()
.withFilter(
@@ -622,10 +622,9 @@ public class AppendOnlySimpleTableTest extends SimpleTableTestBase {
.withProjection(new int[] {3, 2})
.executeFilter();
assertThat(getResult(read, splits, binaryRow(2), 0, toString))
- .hasSameElementsAs(
- Arrays.asList("200|binary", "201|binary", "202|binary", "201|binary"));
+ .hasSameElementsAs(Arrays.asList("201|binary", "201|binary"));
- // projection contains unknown index and
+ // AND must evaluate the unprojected partition field too.
read =
table.newRead()
.withFilter(
@@ -633,8 +632,7 @@ public class AppendOnlySimpleTableTest extends SimpleTableTestBase {
.withProjection(new int[] {3, 2})
.executeFilter();
assertThat(getResult(read, splits, binaryRow(1), 0, toString)).isEmpty();
- assertThat(getResult(read, splits, binaryRow(2), 0, toString))
- .hasSameElementsAs(Arrays.asList("201|binary", "201|binary"));
+ assertThat(getResult(read, splits, binaryRow(2), 0, toString)).isEmpty();
}
@Test
diff --git a/paimon-core/src/test/java/org/apache/paimon/table/DataEvolutionFileIndexTest.java b/paimon-core/src/test/java/org/apache/paimon/table/DataEvolutionFileIndexTest.java
index a45a508057..b3cfa4965d 100644
--- a/paimon-core/src/test/java/org/apache/paimon/table/DataEvolutionFileIndexTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/table/DataEvolutionFileIndexTest.java
@@ -89,9 +89,8 @@ import static org.assertj.core.api.Assertions.assertThat;
* down is an optimization and must never change the result. A row count alone can not say that, it
* passes just as well when nothing is pushed down at all.
*
- * <p>A filter on a column that is not part of the read type has no {@link #query} counterpart:
- * {@link TableRead#executeFilter()} can not project such a predicate onto the read row and silently
- * keeps every row, so only the split level assertion says anything.
+ * <p>Filters on unprojected columns are also checked through {@link TableRead#executeFilter()},
+ * which reads the filter operands before restoring the requested output projection.
*
* <p>Filter values are always picked inside the min/max range of the column, otherwise the group
* level stats pruning in {@link org.apache.paimon.operation.DataEvolutionFileStoreScan} would drop
@@ -324,15 +323,38 @@ public class DataEvolutionFileIndexTest extends DataEvolutionTestBase {
assertThat(rows).anyMatch(row -> row.getInt(0) == 50);
}
+ @Test
+ public void testExecuteFilterWithUnprojectedOverwrittenColumn() throws Exception {
+ FileStoreTable table = createTable("execute_filter_projection", Collections.emptyMap());
+ writeThenOverwriteF1(table, ROW_COUNT);
+ FileStoreTable latest = getTable(identifier(table.name()));
+ RowType outputType = rowType().project("f0");
+ ReadBuilder readBuilder =
+ latest.newReadBuilder().withReadType(outputType).withFilter(equalF1(c1(50)));
+ List<InternalRow> rows =
+ collect(
+ readBuilder.newRead().executeFilter(),
+ readBuilder.newScan().plan(),
+ outputType);
+ assertThat(rows).hasSize(1);
+ assertThat(rows.get(0).getInt(0)).isEqualTo(50);
+
+ readBuilder = latest.newReadBuilder().withReadType(outputType).withFilter(equalF1(f1(50)));
+ assertThat(
+ collect(
+ readBuilder.newRead().executeFilter(),
+ readBuilder.newScan().plan(),
+ outputType))
+ .isEmpty();
+ }
+
@Test
public void testFileIndexIsGivenUpForAColumnOutsideTheReadType() throws Exception {
FileStoreTable table = createTable("projection_index", bloomOptions("f1", "1 B"));
writeAllColumns(table, ROW_COUNT);
- // the index of this file does prove that no row of it matches, but a file only owns the
- // columns of the read type: the file that owns f1 can be pruned out of the split, see
- // testProjectionPruningAwayTheWinnerOfTheFilterColumn, so the push down gives the column
- // up rather than trust whichever file is left holding it
+ // The split reader only evaluates filters over its read type. Without executeFilter,
+ // f1 remains outside that type even though planning retains its files.
assertThat(readWithFilter(table, equalF1(MISSING_F1), rowType().project("f0")))
.hasSize(ROW_COUNT);
@@ -356,14 +378,12 @@ public class DataEvolutionFileIndexTest extends DataEvolutionTestBase {
}
@Test
- public void testProjectionPruningAwayTheWinnerOfTheFilterColumn() throws Exception {
+ public void testUnprojectedOverwrittenFilterColumnWithoutExecution() throws Exception {
FileStoreTable table = createTable("pruned_winner", Collections.emptyMap());
writeThenOverwriteF1(table, ROW_COUNT);
- // f1 was rewritten as c* by a second file, and projecting f0 prunes that file out of the
- // split, see DataEvolutionFileStoreScan#pruneByReadType. The old file is left holding a*
- // and a bloom index that knows nothing about c*, so nothing about it may be used to prove
- // that a row does not match: the row does match, through the file that is not there.
+ // f1 was rewritten as c* by a second file. Without executeFilter the reader still only
+ // reads f0, so an unprojected filter must not reject the matching row.
List<InternalRow> rows = readWithFilter(table, equalF1(c1(50)), rowType().project("f0"));
assertThat(rows).anyMatch(row -> row.getInt(0) == 50);
}
diff --git a/paimon-core/src/test/java/org/apache/paimon/table/PrimaryKeySimpleTableTest.java b/paimon-core/src/test/java/org/apache/paimon/table/PrimaryKeySimpleTableTest.java
index 6a7a722e85..12339f35b9 100644
--- a/paimon-core/src/test/java/org/apache/paimon/table/PrimaryKeySimpleTableTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/table/PrimaryKeySimpleTableTest.java
@@ -585,6 +585,32 @@ public class PrimaryKeySimpleTableTest extends SimpleTableTestBase {
"2|22|202|binary|varbinary|mapKey:mapVal|multiset"));
}
+ @Test
+ public void testExecuteFilterWithUnprojectedValue() throws Exception {
+ writeData();
+ FileStoreTable table = createFileStoreTable();
+ List<Split> splits = toSplits(table.newSnapshotReader().read().dataSplits());
+ PredicateBuilder builder = new PredicateBuilder(table.rowType());
+ TableRead read =
+ table.newRead()
+ .withReadType(table.rowType().project(new int[] {1, 0}))
+ .withFilter(builder.equal(2, 20001L))
+ .executeFilter();
+ Function<InternalRow, String> toString =
+ row -> {
+ assertThat(row.getFieldCount()).isEqualTo(2);
+ return row.getInt(0) + "|" + row.getInt(1);
+ };
+ assertThat(getResult(read, splits, toString)).containsExactly("21|2");
+ // The old value of the same primary key must not survive merge and filtering.
+ read =
+ table.newRead()
+ .withReadType(table.rowType().project(new int[] {1, 0}))
+ .withFilter(builder.equal(2, 201L))
+ .executeFilter();
+ assertThat(getResult(read, splits, toString)).isEmpty();
+ }
+
@Test
public void testBranchBatchReadWrite() throws Exception {
FileStoreTable table = createFileStoreTable();
diff --git a/paimon-core/src/test/java/org/apache/paimon/table/format/FormatTableReadTest.java b/paimon-core/src/test/java/org/apache/paimon/table/format/FormatTableReadTest.java
new file mode 100644
index 0000000000..dccbdc0ac1
--- /dev/null
+++ b/paimon-core/src/test/java/org/apache/paimon/table/format/FormatTableReadTest.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.table.format;
+
+import org.apache.paimon.catalog.Identifier;
+import org.apache.paimon.data.InternalRow;
+import org.apache.paimon.fs.local.LocalFileIO;
+import org.apache.paimon.predicate.Predicate;
+import org.apache.paimon.predicate.PredicateBuilder;
+import org.apache.paimon.reader.RecordReader;
+import org.apache.paimon.table.FormatTable;
+import org.apache.paimon.table.source.ReadBuilder;
+import org.apache.paimon.table.source.TableRead;
+import org.apache.paimon.table.source.TableScan;
+import org.apache.paimon.types.DataTypes;
+import org.apache.paimon.types.RowType;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/** Tests exact filtering with projected columns in {@link FormatTableRead}. */
+class FormatTableReadTest {
+
+ @TempDir Path tempDir;
+
+ @Test
+ void testExecuteFilterWithUnprojectedFields() throws Exception {
+ RowType type = RowType.of(DataTypes.INT(), DataTypes.INT(), DataTypes.STRING());
+ Files.write(
+ tempDir.resolve("data.csv"),
+ Arrays.asList("1,1,first", "1,2,second", "2,2,third", "2,3,last"),
+ StandardCharsets.UTF_8);
+ FormatTable table =
+ FormatTable.builder()
+ .fileIO(LocalFileIO.create())
+ .identifier(Identifier.create("test_db", "test_table"))
+ .rowType(type)
+ .partitionKeys(Collections.emptyList())
+ .location(tempDir.toString())
+ .format(FormatTable.Format.CSV)
+ .options(Collections.singletonMap("file.format", "csv"))
+ .build();
+ PredicateBuilder predicateBuilder = new PredicateBuilder(type);
+ List<Predicate> filters =
+ Arrays.asList(
+ predicateBuilder.equal(1, 2),
+ PredicateBuilder.and(
+ predicateBuilder.equal(0, 1), predicateBuilder.equal(1, 2)),
+ PredicateBuilder.or(
+ predicateBuilder.equal(0, 1), predicateBuilder.equal(1, 2)));
+ List<List<String>> expected =
+ Arrays.asList(
+ Arrays.asList("second:1", "third:2"),
+ Collections.singletonList("second:1"),
+ Arrays.asList("first:1", "second:1", "third:2"));
+ TableScan.Plan plan = table.newReadBuilder().newScan().plan();
+ for (int i = 0; i < filters.size(); i++) {
+ ReadBuilder builder =
+ table.newReadBuilder()
+ .withReadType(type.project(new int[] {2, 0}))
+ .withFilter(filters.get(i));
+ TableRead unfiltered = builder.newRead();
+ TableRead filtered = builder.newRead().executeFilter();
+ assertThat(readRows(filtered, plan)).containsExactlyElementsOf(expected.get(i));
+ assertThat(readRows(filtered, plan)).containsExactlyElementsOf(expected.get(i));
+ // Expanding one TableRead must not change another read sharing the builder.
+ assertThat(readRows(unfiltered, plan))
+ .containsExactly("first:1", "second:1", "third:2", "last:2");
+ }
+ }
+
+ private static List<String> readRows(TableRead read, TableScan.Plan plan) throws Exception {
+ List<String> result = new ArrayList<>();
+ try (RecordReader<InternalRow> reader = read.createReader(plan)) {
+ reader.forEachRemaining(
+ row -> {
+ assertThat(row.getFieldCount()).isEqualTo(2);
+ result.add(row.getString(0) + ":" + row.getInt(1));
+ });
+ }
+ return result;
+ }
+}
diff --git a/paimon-core/src/test/java/org/apache/paimon/table/source/AbstractDataTableReadTest.java b/paimon-core/src/test/java/org/apache/paimon/table/source/AbstractDataTableReadTest.java
index fa1934c4fe..44417bebd4 100644
--- a/paimon-core/src/test/java/org/apache/paimon/table/source/AbstractDataTableReadTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/table/source/AbstractDataTableReadTest.java
@@ -19,31 +19,375 @@
package org.apache.paimon.table.source;
import org.apache.paimon.catalog.TableQueryAuthResult;
+import org.apache.paimon.data.BinaryString;
+import org.apache.paimon.data.GenericRow;
import org.apache.paimon.data.InternalRow;
+import org.apache.paimon.data.serializer.InternalRowSerializer;
import org.apache.paimon.predicate.FieldRef;
+import org.apache.paimon.predicate.FieldTransform;
import org.apache.paimon.predicate.Predicate;
+import org.apache.paimon.predicate.PredicateBuilder;
+import org.apache.paimon.predicate.Transform;
import org.apache.paimon.predicate.UpperTransform;
import org.apache.paimon.reader.RecordReader;
import org.apache.paimon.schema.TableSchema;
import org.apache.paimon.types.DataField;
import org.apache.paimon.types.DataTypes;
import org.apache.paimon.types.RowType;
+import org.apache.paimon.utils.IteratorRecordReader;
import org.apache.paimon.utils.JsonSerdeUtil;
+import org.apache.paimon.utils.NestedProjectedRow;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.LinkedHashMap;
+import java.util.List;
import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.mock;
-/** Tests query-authorization projection expansion in {@link AbstractDataTableRead}. */
+/**
+ * Tests filtering and query-authorization projection expansion in {@link AbstractDataTableRead}.
+ */
class AbstractDataTableReadTest {
+ @Test
+ void testExecuteFilterWithUnprojectedFields() throws IOException {
+ RowType type = RowType.of(DataTypes.INT(), DataTypes.INT(), DataTypes.STRING());
+ TestingDataTableRead read =
+ new TestingDataTableRead(
+ schema(type),
+ GenericRow.of(1, 1, BinaryString.fromString("first")),
+ GenericRow.of(1, 2, BinaryString.fromString("second")),
+ GenericRow.of(2, 2, BinaryString.fromString("third")),
+ GenericRow.of(2, null, BinaryString.fromString("null")),
+ GenericRow.of(2, 3, BinaryString.fromString("last")));
+ PredicateBuilder builder = new PredicateBuilder(type);
+ RowType outputType = type.project(new int[] {2, 0});
+ read.withReadType(outputType);
+ read.executeFilter();
+
+ read.withFilter(builder.equal(1, 2));
+ assertThat(readRows(read, outputType)).containsExactly("second:1", "third:2");
+ // Reuse the same read across splits, then change its filter.
+ assertThat(readRows(read, outputType)).containsExactly("second:1", "third:2");
+ read.withFilter(PredicateBuilder.and(builder.equal(0, 1), builder.equal(1, 2)));
+ assertThat(readRows(read, outputType)).containsExactly("second:1");
+ read.withFilter(PredicateBuilder.or(builder.equal(0, 1), builder.equal(1, 2)));
+ assertThat(readRows(read, outputType)).containsExactly("first:1", "second:1", "third:2");
+ read.withFilter(builder.isNull(1));
+ assertThat(readRows(read, outputType)).containsExactly("null:2");
+
+ read.withReadType(RowType.of());
+ read.withFilter(builder.equal(1, 2));
+ List<Integer> arities = new ArrayList<>();
+ try (RecordReader<InternalRow> reader = read.createReader(mock(Split.class))) {
+ reader.forEachRemaining(row -> arities.add(row.getFieldCount()));
+ }
+ assertThat(arities).containsExactly(0, 0);
+ }
+
+ @Test
+ void testExecuteFilterOnUnprojectedMaskedField() throws IOException {
+ RowType type = RowType.of(DataTypes.INT(), DataTypes.STRING(), DataTypes.STRING());
+ TestingDataTableRead read =
+ new TestingDataTableRead(
+ schema(type),
+ GenericRow.of(
+ 1,
+ BinaryString.fromString("wrong"),
+ BinaryString.fromString("match")),
+ GenericRow.of(
+ 2,
+ BinaryString.fromString("MATCH"),
+ BinaryString.fromString("no")));
+ read.withReadType(type.project(new int[] {0}));
+ read.withFilter(new PredicateBuilder(type).equal(1, BinaryString.fromString("MATCH")));
+ read.executeFilter();
+ TableQueryAuthResult auth =
+ new TableQueryAuthResult(
+ null,
+ Collections.singletonMap(
+ type.getFieldNames().get(1),
+ JsonSerdeUtil.toFlatJson(
+ new UpperTransform(
+ Collections.singletonList(
+ new FieldRef(
+ 2,
+ type.getFieldNames().get(2),
+ DataTypes.STRING()))))));
+ List<Integer> result = new ArrayList<>();
+ try (RecordReader<InternalRow> reader = read.createDataReader(mock(Split.class), auth)) {
+ reader.forEachRemaining(
+ row -> {
+ assertThat(row.getFieldCount()).isEqualTo(1);
+ result.add(row.getInt(0));
+ });
+ }
+ assertThat(result).containsExactly(1);
+ }
+
+ @Test
+ void testMaskedQueryFilterIsEvaluatedOnceAfterAuthorization() throws IOException {
+ RowType type = RowType.of(DataTypes.INT(), DataTypes.STRING(), DataTypes.STRING());
+ TestingDataTableRead read =
+ new TestingDataTableRead(
+ schema(type),
+ GenericRow.of(
+ 0,
+ BinaryString.fromString("wrong"),
+ BinaryString.fromString("match")),
+ GenericRow.of(
+ 1,
+ BinaryString.fromString("wrong"),
+ BinaryString.fromString("match")),
+ GenericRow.of(
+ 2,
+ BinaryString.fromString("MATCH"),
+ BinaryString.fromString("no")));
+ read.withReadType(type.project(new int[] {0}));
+ AtomicInteger evaluations = new AtomicInteger();
+ PredicateBuilder builder = new PredicateBuilder(type);
+ read.withFilter(
+ builder.equal(
+ new CountingFieldTransform(
+ new FieldRef(1, type.getFieldNames().get(1), DataTypes.STRING()),
+ evaluations),
+ BinaryString.fromString("MATCH")));
+ read.executeFilter();
+ TableQueryAuthResult auth =
+ new TableQueryAuthResult(
+ Collections.singletonList(
+ JsonSerdeUtil.toFlatJson(builder.greaterThan(0, 0))),
+ Collections.singletonMap(
+ type.getFieldNames().get(1),
+ JsonSerdeUtil.toFlatJson(
+ new UpperTransform(
+ Collections.singletonList(
+ new FieldRef(
+ 2,
+ type.getFieldNames().get(2),
+ DataTypes.STRING()))))));
+ List<Integer> result = new ArrayList<>();
+ try (RecordReader<InternalRow> reader = read.createDataReader(mock(Split.class), auth)) {
+ reader.forEachRemaining(row -> result.add(row.getInt(0)));
+ }
+ assertThat(result).containsExactly(1);
+ // Only the two authorized rows reach the query expression, once each.
+ assertThat(evaluations.get()).isEqualTo(2);
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {false, true})
+ void testMaskedDisjunctionPreservesUnmaskedOperand(boolean executeFilter) throws IOException {
+ RowType type =
+ RowType.of(
+ DataTypes.INT(), DataTypes.INT(), DataTypes.STRING(), DataTypes.STRING());
+ TestingDataTableRead read =
+ new TestingDataTableRead(
+ schema(type),
+ GenericRow.of(
+ 1,
+ 0,
+ BinaryString.fromString("raw"),
+ BinaryString.fromString("match")),
+ GenericRow.of(
+ 11,
+ 0,
+ BinaryString.fromString("raw"),
+ BinaryString.fromString("match")),
+ GenericRow.of(
+ 12,
+ 1,
+ BinaryString.fromString("raw"),
+ BinaryString.fromString("no")),
+ GenericRow.of(
+ 13,
+ 0,
+ BinaryString.fromString("MATCH"),
+ BinaryString.fromString("no")));
+ read.withReadType(type.project(new int[] {0}));
+ PredicateBuilder builder = new PredicateBuilder(type);
+ read.withFilter(
+ PredicateBuilder.and(
+ builder.greaterThan(0, 10),
+ PredicateBuilder.or(
+ builder.equal(2, BinaryString.fromString("MATCH")),
+ builder.equal(1, 1))));
+ if (executeFilter) {
+ read.executeFilter();
+ }
+ TableQueryAuthResult auth =
+ new TableQueryAuthResult(
+ null,
+ Collections.singletonMap(
+ type.getFieldNames().get(2),
+ JsonSerdeUtil.toFlatJson(
+ new UpperTransform(
+ Collections.singletonList(
+ new FieldRef(
+ 3,
+ type.getFieldNames().get(3),
+ DataTypes.STRING()))))));
+ List<Integer> result = new ArrayList<>();
+ try (RecordReader<InternalRow> reader = read.createDataReader(mock(Split.class), auth)) {
+ reader.forEachRemaining(
+ row -> {
+ assertThat(row.getFieldCount()).isEqualTo(1);
+ result.add(row.getInt(0));
+ });
+ }
+ // The unmasked standalone conjunct is left to the engine unless full execution is enabled.
+ assertThat(result)
+ .containsExactlyElementsOf(
+ executeFilter ? Arrays.asList(11, 12) : Arrays.asList(1, 11, 12));
+ }
+
+ @Test
+ void testReadersKeepTheirOwnAuthorization() throws IOException {
+ RowType type = RowType.of(DataTypes.STRING(), DataTypes.STRING(), DataTypes.STRING());
+ TestingDataTableRead read =
+ new TestingDataTableRead(
+ schema(type),
+ GenericRow.of(
+ BinaryString.fromString("raw"),
+ BinaryString.fromString("alpha"),
+ BinaryString.fromString("beta")));
+ read.withReadType(type.project(new int[] {0}));
+ read.withFilter(new PredicateBuilder(type).equal(0, BinaryString.fromString("ALPHA")))
+ .executeFilter();
+ TableQueryAuthResult firstAuth =
+ new TableQueryAuthResult(
+ null,
+ Collections.singletonMap(
+ type.getFieldNames().get(0),
+ JsonSerdeUtil.toFlatJson(
+ new UpperTransform(
+ Collections.singletonList(
+ new FieldRef(
+ 1,
+ type.getFieldNames().get(1),
+ DataTypes.STRING()))))));
+ TableQueryAuthResult secondAuth =
+ new TableQueryAuthResult(
+ null,
+ Collections.singletonMap(
+ type.getFieldNames().get(0),
+ JsonSerdeUtil.toFlatJson(
+ new UpperTransform(
+ Collections.singletonList(
+ new FieldRef(
+ 2,
+ type.getFieldNames().get(2),
+ DataTypes.STRING()))))));
+ try (RecordReader<InternalRow> first = read.createDataReader(mock(Split.class), firstAuth);
+ RecordReader<InternalRow> second =
+ read.createDataReader(mock(Split.class), secondAuth)) {
+ List<String> firstRows = new ArrayList<>();
+ first.forEachRemaining(row -> firstRows.add(row.getString(0).toString()));
+ List<String> secondRows = new ArrayList<>();
+ second.forEachRemaining(row -> secondRows.add(row.getString(0).toString()));
+ assertThat(firstRows).containsExactly("ALPHA");
+ assertThat(secondRows).isEmpty();
+ }
+ read.withFilter((Predicate) null);
+ List<String> rawRows = new ArrayList<>();
+ try (RecordReader<InternalRow> reader = read.createReader(mock(Split.class))) {
+ reader.forEachRemaining(
+ row -> {
+ assertThat(row.getFieldCount()).isEqualTo(1);
+ rawRows.add(row.getString(0).toString());
+ });
+ }
+ assertThat(rawRows).containsExactly("raw");
+ }
+
+ private static class CountingFieldTransform extends FieldTransform {
+
+ private static final long serialVersionUID = 1L;
+ private final AtomicInteger evaluations;
+
+ private CountingFieldTransform(FieldRef field, AtomicInteger evaluations) {
+ super(field);
+ this.evaluations = evaluations;
+ }
+
+ @Override
+ public Object transform(InternalRow row) {
+ evaluations.incrementAndGet();
+ return super.transform(row);
+ }
+
+ @Override
+ public Transform copyWithNewInputs(List<Object> inputs) {
+ return new CountingFieldTransform((FieldRef) inputs.get(0), evaluations);
+ }
+ }
+
+ private static TableSchema schema(RowType type) {
+ return new TableSchema(
+ 0,
+ type.getFields(),
+ type.getFieldCount() - 1,
+ Collections.emptyList(),
+ Collections.emptyList(),
+ Collections.emptyMap(),
+ null);
+ }
+
+ @Test
+ void testExecuteFilterPreservesNestedOutputProjection() throws IOException {
+ RowType nested =
+ new RowType(
+ Arrays.asList(
+ new DataField(1, "a", DataTypes.INT()),
+ new DataField(2, "b", DataTypes.STRING())));
+ RowType type = new RowType(Collections.singletonList(new DataField(0, "profile", nested)));
+ RowType outputType =
+ new RowType(
+ Collections.singletonList(
+ type.getFields().get(0).newType(nested.project("b"))));
+ TestingDataTableRead read =
+ new TestingDataTableRead(
+ schema(type),
+ GenericRow.of(GenericRow.of(10, BinaryString.fromString("kept"))),
+ GenericRow.of((Object) null));
+ read.withReadType(outputType)
+ .withFilter(new PredicateBuilder(type).isNotNull(0))
+ .executeFilter();
+ List<String> result = new ArrayList<>();
+ try (RecordReader<InternalRow> reader = read.createReader(mock(Split.class))) {
+ reader.forEachRemaining(
+ row -> {
+ InternalRow profile = row.getRow(0, 1);
+ assertThat(profile.getFieldCount()).isEqualTo(1);
+ result.add(profile.getString(0).toString());
+ });
+ }
+ assertThat(result).containsExactly("kept");
+ }
+
+ private static List<String> readRows(TestingDataTableRead read, RowType outputType)
+ throws IOException {
+ List<String> result = new ArrayList<>();
+ try (RecordReader<InternalRow> reader = read.createReader(mock(Split.class))) {
+ reader.forEachRemaining(
+ row -> {
+ assertThat(row.getFieldCount()).isEqualTo(outputType.getFieldCount());
+ result.add(row.getString(0) + ":" + row.getInt(1));
+ });
+ }
+ return result;
+ }
+
@Test
void testNoProjectionResetWithoutExplicitReadType() throws IOException {
TableSchema schema =
@@ -119,9 +463,11 @@ class AbstractDataTableReadTest {
private static class TestingDataTableRead extends AbstractDataTableRead {
private RowType appliedReadType;
+ private final List<InternalRow> rows;
- private TestingDataTableRead(TableSchema schema) {
+ private TestingDataTableRead(TableSchema schema, InternalRow... rows) {
super(schema);
+ this.rows = Arrays.asList(rows);
}
@Override
@@ -131,15 +477,16 @@ class AbstractDataTableReadTest {
@Override
public RecordReader<InternalRow> reader(Split split) {
- return new RecordReader<InternalRow>() {
- @Override
- public RecordIterator<InternalRow> readBatch() {
- return null;
- }
-
- @Override
- public void close() {}
- };
+ RowType type = appliedReadType == null ? schema().logicalRowType() : appliedReadType;
+ NestedProjectedRow projection =
+ NestedProjectedRow.create(schema().logicalRowType(), type);
+ InternalRowSerializer serializer = new InternalRowSerializer(type);
+ List<InternalRow> projected = new ArrayList<>();
+ for (InternalRow row : rows) {
+ projected.add(
+ serializer.copy(projection == null ? row : projection.replaceRow(row)));
+ }
+ return new IteratorRecordReader<>(projected.iterator());
}
@Override
diff --git a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/PermissionProcedureTest.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/PermissionProcedureTest.scala
index 3e34fe56f2..53fcc31c17 100644
--- a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/PermissionProcedureTest.scala
+++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/PermissionProcedureTest.scala
@@ -542,7 +542,7 @@ class PermissionProcedureTest extends PaimonSparkTestWithRestCatalogBase {
|""".stripMargin)
.collect()
}
- assertThat(missingColumn.getMessage).contains("column unknown")
+ assertThat(missingColumn.getMessage).contains("Cannot resolve field 'unknown'")
val missingPrincipal = intercept[Exception] {
spark
@@ -664,7 +664,7 @@ class PermissionProcedureTest extends PaimonSparkTestWithRestCatalogBase {
.sql("ALTER TABLE paimon.sales.renamed_orders RENAME COLUMN region TO area")
.collect()
}
- assertThat(renameColumn.getMessage).contains("column region")
+ assertThat(renameColumn.getMessage).contains("Cannot resolve field 'region'")
spark.sql("DROP TABLE paimon.sales.renamed_orders")
spark.sql("""CREATE TABLE paimon.sales.renamed_orders (