You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@paimon.apache.org by ho...@apache.org on 2026/04/01 02:34:05 UTC
(paimon) branch release-1.4 updated (f83b39579c -> f509a8fe15)
This is an automated email from the ASF dual-hosted git repository.
hope pushed a change to branch release-1.4
in repository https://gitbox.apache.org/repos/asf/paimon.git
discard f83b39579c [spark] Fix view resolution for CTE and ORDER BY ordinal (#7552)
discard 458acfc894 [python] Add doc and refine config for using pyjindosdk in pypaimon (#7565)
discard 73825eb631 [python] Push down row ranges to vortex reader (#7558)
discard 7d19566251 [python] Support rename branch api (#7561)
discard 5c50f908eb [python] Fix token cache pollution in Python `RESTTokenFileIO` by aligning with the Java implementation. (#7562)
discard 749aa28f83 Introduce scan.primary-branch which is symmetrical to scan.fallback-branch (#7553)
discard 8aa7c6c5ee [python] Introduce file pruning for dv pk table (#7557)
discard db8d3d9a1f [doc] Bucket pruning also works for pk clustering override (#7554)
discard 699e453d67 [Python] Fix data evolution read IndexError when file has no write_cols (#7556)
discard 306d0b714e [flink] support vector search procedure for flink (#7550)
discard 3799094b00 [python] Add FUSE support for REST Catalog (#7483)
discard edc7b3fad8 [python] Support PyJindo in pyarrow_file_io (#7410)
discard 773c438296 [tantivy] Introduce paimon-tantivy for full text search index (#7551)
discard fb2484ed0f [core] Support rename branch API and procedures (#7549)
discard 68eb6b2e0b [core][spark] Introduce Full Text Search interfaces and Spark function (#7548)
discard 57fdb1719d [python] Introduce vortex file format integration (#7547)
discard e1b930c800 [vortex] Introduce vortex file format integration (#7543)
discard dfbac71c78 [core] Ensuring JDBC connection is not stale before using it (#7470)
discard 1e00fbbd5d [hotfix] Add vortex jni ignore to gitignore
discard 166a6f722e [python] Fix data evolution merge read with disordered projection (#7544)
discard 5274a4021f [flink] Sopport refreshing partition asynchronously for lookup join (#7402)
discard 12c9c9a441 [spark] Fix nested struct field mapping for V2 write merge schema (#7542)
discard 235e7e316e [core] Fix frequent compact triggers during streaming writes for pk clustering override (#7540)
discard 9898d917fd [spark] Support merge schema through catalog for REST catalog (#7539)
discard 8670853c75 [spark] Support reading ObjectTable via Spark SQL (#7535)
discard f137b6ac57 [core] Remove stale dv files in ClusteringCompactManager (#7538)
discard 8796587522 [doc] Modify scenario-guide to changelog-producer
discard ca2534fad1 [core] File paths in SimpleLsmKvDb should contain UUID (#7536)
discard 191fbb31cc [python] Include LICENSE and NOTICE in sdist for Apache release compliance (#7533)
discard 81faaf3c5e [doc] Introduce Scenario Guide in Learn Paimon
discard 60ca3b1e04 [core] Respect rowsPerShard as hard constraint in global index build (#7530)
discard aa17b3ecda [core][vector] add log for lumina index writer (#7512)
discard 350ed2c6eb [core] Chain table support group partition. (#7524)
discard fcf405e54d [doc] Update Documentations for Append table
discard 8904cc0409 [core] Add validation for 'pk-clustering-override' (#7528)
discard a85142ed84 [flink][spark] path_to_descriptor function supports http url (#7529)
discard 3c2a84146f [spark] Add OVERWRITE_BY_FILTER capabilities to PartitionedFormatTable for Spark 4 compatibility (#7517)
discard 5826b2ea91 [fix] Fix IndexFileHandler to support deleting all index types during snapshot expiration (#7523)
discard 78b370d014 [python] Add comment method for file_store_table (#7527)
discard 868b080060 [core] Support config compaction file num limit and report if reach the limit in batch job (#7520)
discard 6bab1c62f7 [core] Do not create GlobalIndexScanner when no index files (#7521)
discard 1713285c9e [python] Deduplicate input to keep last one in TableUpsertByKey (#7526)
new 7cdbd23aff [python] Deduplicate input to keep last one in TableUpsertByKey (#7526)
new 18003d5c15 [core] Do not create GlobalIndexScanner when no index files (#7521)
new 09e19b1331 [core] Support config compaction file num limit and report if reach the limit in batch job (#7520)
new 86e2364a64 [python] Add comment method for file_store_table (#7527)
new a53aa81c59 [fix] Fix IndexFileHandler to support deleting all index types during snapshot expiration (#7523)
new 0fec396774 [spark] Add OVERWRITE_BY_FILTER capabilities to PartitionedFormatTable for Spark 4 compatibility (#7517)
new d29f5a685d [flink][spark] path_to_descriptor function supports http url (#7529)
new bf95e9ff7b [core] Add validation for 'pk-clustering-override' (#7528)
new 0c2bae8114 [doc] Update Documentations for Append table
new 42812e5858 [core] Chain table support group partition. (#7524)
new efa88fd500 [core][vector] add log for lumina index writer (#7512)
new c1275261a2 [core] Respect rowsPerShard as hard constraint in global index build (#7530)
new 77e6437a63 [doc] Introduce Scenario Guide in Learn Paimon
new 78dcf0be56 [python] Include LICENSE and NOTICE in sdist for Apache release compliance (#7533)
new 89ab4f4762 [core] File paths in SimpleLsmKvDb should contain UUID (#7536)
new 94222cffaf [doc] Modify scenario-guide to changelog-producer
new 6cf1b374a9 [core] Remove stale dv files in ClusteringCompactManager (#7538)
new b6fc657b4e [spark] Support reading ObjectTable via Spark SQL (#7535)
new 3e0b96b6e9 [spark] Support merge schema through catalog for REST catalog (#7539)
new 96f476c160 [core] Fix frequent compact triggers during streaming writes for pk clustering override (#7540)
new a657f46d51 [spark] Fix nested struct field mapping for V2 write merge schema (#7542)
new ec02f8e682 [flink] Sopport refreshing partition asynchronously for lookup join (#7402)
new bfee09f7d2 [python] Fix data evolution merge read with disordered projection (#7544)
new 9eee82f6e1 [core] Ensuring JDBC connection is not stale before using it (#7470)
new e6231d01c6 [core][spark] Introduce Full Text Search interfaces and Spark function (#7548)
new 74c5b493ab [core] Support rename branch API and procedures (#7549)
new c918c47f38 [python] Support PyJindo in pyarrow_file_io (#7410)
new afb2effb1f [python] Add FUSE support for REST Catalog (#7483)
new 53e674ed48 [flink] support vector search procedure for flink (#7550)
new cb59e904ae [Python] Fix data evolution read IndexError when file has no write_cols (#7556)
new e5aa956656 [doc] Bucket pruning also works for pk clustering override (#7554)
new 2f62a4f998 [python] Introduce file pruning for dv pk table (#7557)
new 57f6db7960 Introduce scan.primary-branch which is symmetrical to scan.fallback-branch (#7553)
new a413c697c4 [python] Fix token cache pollution in Python `RESTTokenFileIO` by aligning with the Java implementation. (#7562)
new a6a47a89a5 [python] Support rename branch api (#7561)
new a1f7939901 [python] Add doc and refine config for using pyjindosdk in pypaimon (#7565)
new f509a8fe15 [spark] Fix view resolution for CTE and ORDER BY ordinal (#7552)
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (f83b39579c)
\
N -- N -- N refs/heads/release-1.4 (f509a8fe15)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 37 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/workflows/paimon-python-checks.yml | 5 +-
.github/workflows/utitcase-rust-native.yml | 109 ---
.gitignore | 7 -
docs/content/append-table/global-index.md | 55 +-
.../java/org/apache/paimon/arrow/ArrowUtils.java | 9 +-
.../pypaimon/benchmark/clickbench_format.py | 374 -----------
paimon-python/pypaimon/common/file_io.py | 3 -
.../pypaimon/common/options/core_options.py | 1 -
paimon-python/pypaimon/filesystem/local_file_io.py | 21 +-
.../pypaimon/filesystem/pyarrow_file_io.py | 18 -
.../pypaimon/read/reader/format_vortex_reader.py | 152 -----
paimon-python/pypaimon/read/reader/vortex_utils.py | 70 --
paimon-python/pypaimon/read/split_read.py | 54 +-
.../pypaimon/tests/data_evolution_test.py | 156 -----
.../pypaimon/tests/reader_append_only_test.py | 80 ---
paimon-python/pypaimon/write/writer/data_writer.py | 2 -
paimon-tantivy/paimon-tantivy-index/README.md | 148 -----
paimon-tantivy/paimon-tantivy-index/pom.xml | 91 ---
.../index/TantivyFullTextGlobalIndexReader.java | 311 ---------
.../index/TantivyFullTextGlobalIndexWriter.java | 217 ------
.../index/TantivyFullTextGlobalIndexer.java | 45 --
.../index/TantivyFullTextGlobalIndexerFactory.java | 40 --
.../index/TantivyScoredGlobalIndexResult.java | 57 --
....apache.paimon.globalindex.GlobalIndexerFactory | 16 -
.../index/TantivyFullTextGlobalIndexTest.java | 300 ---------
paimon-tantivy/paimon-tantivy-jni/README.md | 60 --
paimon-tantivy/paimon-tantivy-jni/pom.xml | 69 --
paimon-tantivy/paimon-tantivy-jni/rust/Cargo.toml | 13 -
.../paimon-tantivy-jni/rust/src/jni_directory.rs | 259 --------
paimon-tantivy/paimon-tantivy-jni/rust/src/lib.rs | 358 ----------
.../org/apache/paimon/tantivy/NativeLoader.java | 83 ---
.../org/apache/paimon/tantivy/SearchResult.java | 43 --
.../org/apache/paimon/tantivy/StreamFileInput.java | 32 -
.../apache/paimon/tantivy/TantivyIndexWriter.java | 70 --
.../org/apache/paimon/tantivy/TantivySearcher.java | 91 ---
.../src/main/resources/META-INF/NOTICE | 17 -
.../org/apache/paimon/tantivy/TantivyJniTest.java | 77 ---
.../paimon/tantivy/TantivyStreamSearchTest.java | 174 -----
paimon-tantivy/pom.xml | 39 --
paimon-vortex/paimon-vortex-format/pom.xml | 93 ---
.../org/apache/arrow/memory/BoundsChecking.java | 24 -
.../paimon/format/vortex/VortexFileFormat.java | 219 ------
.../format/vortex/VortexFileFormatFactory.java | 37 --
.../format/vortex/VortexPredicateConverter.java | 211 ------
.../paimon/format/vortex/VortexReaderFactory.java | 79 ---
.../paimon/format/vortex/VortexRecordsReader.java | 154 -----
.../paimon/format/vortex/VortexRecordsWriter.java | 118 ----
.../paimon/format/vortex/VortexTypeUtils.java | 198 ------
.../apache/paimon/format/vortex/VortexUtils.java | 112 ----
.../paimon/format/vortex/VortexWriterFactory.java | 70 --
.../org.apache.paimon.format.FileFormatFactory | 16 -
.../vortex/VortexFileFormatReadWriteTest.java | 152 -----
.../paimon/format/vortex/VortexFileFormatTest.java | 131 ----
.../vortex/VortexPredicateConverterTest.java | 244 -------
.../format/vortex/VortexReaderWriterTest.java | 366 ----------
.../paimon/format/vortex/VortexTypeUtilsTest.java | 132 ----
paimon-vortex/paimon-vortex-jni/README.md | 89 ---
paimon-vortex/paimon-vortex-jni/pom.xml | 157 -----
.../src/main/java/dev/vortex/api/Array.java | 68 --
.../main/java/dev/vortex/api/ArrayIterator.java | 29 -
.../src/main/java/dev/vortex/api/DType.java | 249 -------
.../src/main/java/dev/vortex/api/Expression.java | 56 --
.../src/main/java/dev/vortex/api/File.java | 31 -
.../src/main/java/dev/vortex/api/Files.java | 51 --
.../src/main/java/dev/vortex/api/ScanOptions.java | 44 --
.../src/main/java/dev/vortex/api/VortexWriter.java | 45 --
.../java/dev/vortex/api/expressions/Binary.java | 228 -------
.../java/dev/vortex/api/expressions/GetItem.java | 95 ---
.../java/dev/vortex/api/expressions/Literal.java | 733 ---------------------
.../main/java/dev/vortex/api/expressions/Not.java | 91 ---
.../main/java/dev/vortex/api/expressions/Root.java | 64 --
.../java/dev/vortex/api/expressions/Unknown.java | 52 --
.../src/main/java/dev/vortex/api/proto/DTypes.java | 224 -------
.../java/dev/vortex/api/proto/EndianUtils.java | 74 ---
.../java/dev/vortex/api/proto/Expressions.java | 81 ---
.../main/java/dev/vortex/api/proto/Scalars.java | 368 -----------
.../dev/vortex/api/proto/TemporalMetadatas.java | 93 ---
.../java/dev/vortex/arrow/ArrowAllocation.java | 33 -
.../src/main/java/dev/vortex/jni/JNIArray.java | 168 -----
.../main/java/dev/vortex/jni/JNIArrayIterator.java | 76 ---
.../src/main/java/dev/vortex/jni/JNIDType.java | 131 ----
.../src/main/java/dev/vortex/jni/JNIFile.java | 78 ---
.../src/main/java/dev/vortex/jni/JNIWriter.java | 63 --
.../dev/vortex/jni/NativeArrayIteratorMethods.java | 34 -
.../java/dev/vortex/jni/NativeArrayMethods.java | 75 ---
.../java/dev/vortex/jni/NativeDTypeMethods.java | 92 ---
.../java/dev/vortex/jni/NativeFileMethods.java | 50 --
.../src/main/java/dev/vortex/jni/NativeLoader.java | 83 ---
.../main/java/dev/vortex/jni/NativeLogging.java | 36 -
.../java/dev/vortex/jni/NativeWriterMethods.java | 37 --
.../paimon-vortex-jni/src/main/proto/dtype.proto | 101 ---
.../paimon-vortex-jni/src/main/proto/expr.proto | 84 ---
.../paimon-vortex-jni/src/main/proto/scalar.proto | 37 --
.../src/main/resources/META-INF/NOTICE | 9 -
paimon-vortex/pom.xml | 39 --
pom.xml | 4 -
96 files changed, 20 insertions(+), 10214 deletions(-)
delete mode 100644 .github/workflows/utitcase-rust-native.yml
delete mode 100644 paimon-python/pypaimon/benchmark/clickbench_format.py
delete mode 100644 paimon-python/pypaimon/read/reader/format_vortex_reader.py
delete mode 100644 paimon-python/pypaimon/read/reader/vortex_utils.py
delete mode 100644 paimon-tantivy/paimon-tantivy-index/README.md
delete mode 100644 paimon-tantivy/paimon-tantivy-index/pom.xml
delete mode 100644 paimon-tantivy/paimon-tantivy-index/src/main/java/org/apache/paimon/tantivy/index/TantivyFullTextGlobalIndexReader.java
delete mode 100644 paimon-tantivy/paimon-tantivy-index/src/main/java/org/apache/paimon/tantivy/index/TantivyFullTextGlobalIndexWriter.java
delete mode 100644 paimon-tantivy/paimon-tantivy-index/src/main/java/org/apache/paimon/tantivy/index/TantivyFullTextGlobalIndexer.java
delete mode 100644 paimon-tantivy/paimon-tantivy-index/src/main/java/org/apache/paimon/tantivy/index/TantivyFullTextGlobalIndexerFactory.java
delete mode 100644 paimon-tantivy/paimon-tantivy-index/src/main/java/org/apache/paimon/tantivy/index/TantivyScoredGlobalIndexResult.java
delete mode 100644 paimon-tantivy/paimon-tantivy-index/src/main/resources/META-INF/services/org.apache.paimon.globalindex.GlobalIndexerFactory
delete mode 100644 paimon-tantivy/paimon-tantivy-index/src/test/java/org/apache/paimon/tantivy/index/TantivyFullTextGlobalIndexTest.java
delete mode 100644 paimon-tantivy/paimon-tantivy-jni/README.md
delete mode 100644 paimon-tantivy/paimon-tantivy-jni/pom.xml
delete mode 100644 paimon-tantivy/paimon-tantivy-jni/rust/Cargo.toml
delete mode 100644 paimon-tantivy/paimon-tantivy-jni/rust/src/jni_directory.rs
delete mode 100644 paimon-tantivy/paimon-tantivy-jni/rust/src/lib.rs
delete mode 100644 paimon-tantivy/paimon-tantivy-jni/src/main/java/org/apache/paimon/tantivy/NativeLoader.java
delete mode 100644 paimon-tantivy/paimon-tantivy-jni/src/main/java/org/apache/paimon/tantivy/SearchResult.java
delete mode 100644 paimon-tantivy/paimon-tantivy-jni/src/main/java/org/apache/paimon/tantivy/StreamFileInput.java
delete mode 100644 paimon-tantivy/paimon-tantivy-jni/src/main/java/org/apache/paimon/tantivy/TantivyIndexWriter.java
delete mode 100644 paimon-tantivy/paimon-tantivy-jni/src/main/java/org/apache/paimon/tantivy/TantivySearcher.java
delete mode 100644 paimon-tantivy/paimon-tantivy-jni/src/main/resources/META-INF/NOTICE
delete mode 100644 paimon-tantivy/paimon-tantivy-jni/src/test/java/org/apache/paimon/tantivy/TantivyJniTest.java
delete mode 100644 paimon-tantivy/paimon-tantivy-jni/src/test/java/org/apache/paimon/tantivy/TantivyStreamSearchTest.java
delete mode 100644 paimon-tantivy/pom.xml
delete mode 100644 paimon-vortex/paimon-vortex-format/pom.xml
delete mode 100644 paimon-vortex/paimon-vortex-format/src/main/java/org/apache/arrow/memory/BoundsChecking.java
delete mode 100644 paimon-vortex/paimon-vortex-format/src/main/java/org/apache/paimon/format/vortex/VortexFileFormat.java
delete mode 100644 paimon-vortex/paimon-vortex-format/src/main/java/org/apache/paimon/format/vortex/VortexFileFormatFactory.java
delete mode 100644 paimon-vortex/paimon-vortex-format/src/main/java/org/apache/paimon/format/vortex/VortexPredicateConverter.java
delete mode 100644 paimon-vortex/paimon-vortex-format/src/main/java/org/apache/paimon/format/vortex/VortexReaderFactory.java
delete mode 100644 paimon-vortex/paimon-vortex-format/src/main/java/org/apache/paimon/format/vortex/VortexRecordsReader.java
delete mode 100644 paimon-vortex/paimon-vortex-format/src/main/java/org/apache/paimon/format/vortex/VortexRecordsWriter.java
delete mode 100644 paimon-vortex/paimon-vortex-format/src/main/java/org/apache/paimon/format/vortex/VortexTypeUtils.java
delete mode 100644 paimon-vortex/paimon-vortex-format/src/main/java/org/apache/paimon/format/vortex/VortexUtils.java
delete mode 100644 paimon-vortex/paimon-vortex-format/src/main/java/org/apache/paimon/format/vortex/VortexWriterFactory.java
delete mode 100644 paimon-vortex/paimon-vortex-format/src/main/resources/META-INF/services/org.apache.paimon.format.FileFormatFactory
delete mode 100644 paimon-vortex/paimon-vortex-format/src/test/java/org/apache/paimon/format/vortex/VortexFileFormatReadWriteTest.java
delete mode 100644 paimon-vortex/paimon-vortex-format/src/test/java/org/apache/paimon/format/vortex/VortexFileFormatTest.java
delete mode 100644 paimon-vortex/paimon-vortex-format/src/test/java/org/apache/paimon/format/vortex/VortexPredicateConverterTest.java
delete mode 100644 paimon-vortex/paimon-vortex-format/src/test/java/org/apache/paimon/format/vortex/VortexReaderWriterTest.java
delete mode 100644 paimon-vortex/paimon-vortex-format/src/test/java/org/apache/paimon/format/vortex/VortexTypeUtilsTest.java
delete mode 100644 paimon-vortex/paimon-vortex-jni/README.md
delete mode 100644 paimon-vortex/paimon-vortex-jni/pom.xml
delete mode 100644 paimon-vortex/paimon-vortex-jni/src/main/java/dev/vortex/api/Array.java
delete mode 100644 paimon-vortex/paimon-vortex-jni/src/main/java/dev/vortex/api/ArrayIterator.java
delete mode 100644 paimon-vortex/paimon-vortex-jni/src/main/java/dev/vortex/api/DType.java
delete mode 100644 paimon-vortex/paimon-vortex-jni/src/main/java/dev/vortex/api/Expression.java
delete mode 100644 paimon-vortex/paimon-vortex-jni/src/main/java/dev/vortex/api/File.java
delete mode 100644 paimon-vortex/paimon-vortex-jni/src/main/java/dev/vortex/api/Files.java
delete mode 100644 paimon-vortex/paimon-vortex-jni/src/main/java/dev/vortex/api/ScanOptions.java
delete mode 100644 paimon-vortex/paimon-vortex-jni/src/main/java/dev/vortex/api/VortexWriter.java
delete mode 100644 paimon-vortex/paimon-vortex-jni/src/main/java/dev/vortex/api/expressions/Binary.java
delete mode 100644 paimon-vortex/paimon-vortex-jni/src/main/java/dev/vortex/api/expressions/GetItem.java
delete mode 100644 paimon-vortex/paimon-vortex-jni/src/main/java/dev/vortex/api/expressions/Literal.java
delete mode 100644 paimon-vortex/paimon-vortex-jni/src/main/java/dev/vortex/api/expressions/Not.java
delete mode 100644 paimon-vortex/paimon-vortex-jni/src/main/java/dev/vortex/api/expressions/Root.java
delete mode 100644 paimon-vortex/paimon-vortex-jni/src/main/java/dev/vortex/api/expressions/Unknown.java
delete mode 100644 paimon-vortex/paimon-vortex-jni/src/main/java/dev/vortex/api/proto/DTypes.java
delete mode 100644 paimon-vortex/paimon-vortex-jni/src/main/java/dev/vortex/api/proto/EndianUtils.java
delete mode 100644 paimon-vortex/paimon-vortex-jni/src/main/java/dev/vortex/api/proto/Expressions.java
delete mode 100644 paimon-vortex/paimon-vortex-jni/src/main/java/dev/vortex/api/proto/Scalars.java
delete mode 100644 paimon-vortex/paimon-vortex-jni/src/main/java/dev/vortex/api/proto/TemporalMetadatas.java
delete mode 100644 paimon-vortex/paimon-vortex-jni/src/main/java/dev/vortex/arrow/ArrowAllocation.java
delete mode 100644 paimon-vortex/paimon-vortex-jni/src/main/java/dev/vortex/jni/JNIArray.java
delete mode 100644 paimon-vortex/paimon-vortex-jni/src/main/java/dev/vortex/jni/JNIArrayIterator.java
delete mode 100644 paimon-vortex/paimon-vortex-jni/src/main/java/dev/vortex/jni/JNIDType.java
delete mode 100644 paimon-vortex/paimon-vortex-jni/src/main/java/dev/vortex/jni/JNIFile.java
delete mode 100644 paimon-vortex/paimon-vortex-jni/src/main/java/dev/vortex/jni/JNIWriter.java
delete mode 100644 paimon-vortex/paimon-vortex-jni/src/main/java/dev/vortex/jni/NativeArrayIteratorMethods.java
delete mode 100644 paimon-vortex/paimon-vortex-jni/src/main/java/dev/vortex/jni/NativeArrayMethods.java
delete mode 100644 paimon-vortex/paimon-vortex-jni/src/main/java/dev/vortex/jni/NativeDTypeMethods.java
delete mode 100644 paimon-vortex/paimon-vortex-jni/src/main/java/dev/vortex/jni/NativeFileMethods.java
delete mode 100644 paimon-vortex/paimon-vortex-jni/src/main/java/dev/vortex/jni/NativeLoader.java
delete mode 100644 paimon-vortex/paimon-vortex-jni/src/main/java/dev/vortex/jni/NativeLogging.java
delete mode 100644 paimon-vortex/paimon-vortex-jni/src/main/java/dev/vortex/jni/NativeWriterMethods.java
delete mode 100644 paimon-vortex/paimon-vortex-jni/src/main/proto/dtype.proto
delete mode 100644 paimon-vortex/paimon-vortex-jni/src/main/proto/expr.proto
delete mode 100644 paimon-vortex/paimon-vortex-jni/src/main/proto/scalar.proto
delete mode 100644 paimon-vortex/paimon-vortex-jni/src/main/resources/META-INF/NOTICE
delete mode 100644 paimon-vortex/pom.xml
(paimon) 09/37: [doc] Update Documentations for Append table
Posted by ho...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hope pushed a commit to branch release-1.4
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit 0c2bae81142f387b5030698b53718471b517fe39
Author: JingsongLi <ji...@gmail.com>
AuthorDate: Wed Mar 25 23:10:11 2026 +0800
[doc] Update Documentations for Append table
---
docs/content/append-table/bucketed.md | 89 ++++++------
docs/content/append-table/data-evolution.md | 2 +-
.../content/append-table/incremental-clustering.md | 2 +-
docs/content/append-table/overview.md | 157 ++++++++++++++++++++-
docs/content/append-table/query-performance.md | 102 -------------
docs/content/append-table/streaming.md | 104 --------------
docs/content/append-table/update.md | 41 ------
docs/content/learn-paimon/understand-files.md | 2 +-
8 files changed, 198 insertions(+), 301 deletions(-)
diff --git a/docs/content/append-table/bucketed.md b/docs/content/append-table/bucketed.md
index 5643da0c05..04dc30699b 100644
--- a/docs/content/append-table/bucketed.md
+++ b/docs/content/append-table/bucketed.md
@@ -1,6 +1,6 @@
---
title: "Bucketed"
-weight: 5
+weight: 3
type: docs
aliases:
- /append-table/bucketed.html
@@ -46,7 +46,46 @@ CREATE TABLE my_table (
{{< /tab >}}
{{< /tabs >}}
-## Streaming
+## Data Skipping
+
+The primary and most significant advantage of a bucketed append table is **data skipping**. When queries contain
+equality (`=`) or `IN` filter conditions on the `bucket-key`, Paimon can efficiently push these predicates down to
+skip irrelevant bucket files entirely. This means a large number of files that do not match the filter are pruned
+before reading, drastically reducing I/O and accelerating queries.
+
+For example, if `bucket-key` is `product_id` and you query:
+
+```sql
+SELECT * FROM my_table WHERE product_id = 12345;
+
+SELECT * FROM my_table WHERE product_id IN (1, 2, 3);
+```
+
+Paimon will only read the bucket that contains the matching `product_id` values, filtering out all other bucket files.
+This is extremely effective when the table has many buckets and you are querying a small subset of bucket-key values.
+
+## Bucketed Join
+
+Bucketed table can also be used to accelerate join queries by avoiding costly shuffle operations in batch processing.
+For example, you can use the following Spark SQL to read a Paimon table:
+
+```sql
+SET spark.sql.sources.v2.bucketing.enabled = true;
+
+CREATE TABLE FACT_TABLE (order_id INT, f1 STRING) TBLPROPERTIES ('bucket'='10', 'bucket-key' = 'order_id');
+
+CREATE TABLE DIM_TABLE (order_id INT, f2 STRING) TBLPROPERTIES ('bucket'='10', 'primary-key' = 'order_id');
+
+SELECT * FROM FACT_TABLE JOIN DIM_TABLE on t1.order_id = t4.order_id;
+```
+
+The `spark.sql.sources.v2.bucketing.enabled` config is used to enable bucketing for V2 data sources. When turned on,
+Spark will recognize the specific distribution reported by a V2 data source through SupportsReportPartitioning, and
+will try to avoid shuffle if necessary.
+
+The costly join shuffle will be avoided if two tables have the same bucketing strategy and same number of buckets.
+
+## Bucketed Streaming
An ordinary Append table has no strict ordering guarantees for its streaming writes and reads, but there are some cases
where you need to define a key similar to Kafka's.
@@ -57,43 +96,7 @@ bucket as a queue.
{{< img src="/img/for-queue.png">}}
-### Compaction in Bucket
-
-By default, the sink node will automatically perform compaction to control the number of files. The following options
-control the strategy of compaction:
-
-<table class="configuration table table-bordered">
- <thead>
- <tr>
- <th class="text-left" style="width: 20%">Key</th>
- <th class="text-left" style="width: 15%">Default</th>
- <th class="text-left" style="width: 10%">Type</th>
- <th class="text-left" style="width: 55%">Description</th>
- </tr>
- </thead>
- <tbody>
- <tr>
- <td><h5>write-only</h5></td>
- <td style="word-wrap: break-word;">false</td>
- <td>Boolean</td>
- <td>If set to true, compactions and snapshot expiration will be skipped. This option is used along with dedicated compact jobs.</td>
- </tr>
- <tr>
- <td><h5>compaction.min.file-num</h5></td>
- <td style="word-wrap: break-word;">5</td>
- <td>Integer</td>
- <td>For file set [f_0,...,f_N], the minimum file number to trigger a compaction for append table.</td>
- </tr>
- <tr>
- <td><h5>full-compaction.delta-commits</h5></td>
- <td style="word-wrap: break-word;">(none)</td>
- <td>Integer</td>
- <td>Full compaction will be constantly triggered after delta commits.</td>
- </tr>
- </tbody>
-</table>
-
-### Streaming Read Order
+**Streaming Read Order**
For streaming reads, records are produced in the following order:
@@ -103,7 +106,7 @@ For streaming reads, records are produced in the following order:
* For any two records from the same partition and the same bucket, the first written record will be produced first.
* For any two records from the same partition but two different buckets, different buckets are processed by different tasks, there is no order guarantee between them.
-### Watermark Definition
+**Watermark Definition**
You can define watermark for reading Paimon tables:
@@ -148,7 +151,7 @@ which will make sure no sources/splits/shards/partitions increase their watermar
</tbody>
</table>
-### Bounded Stream
+**Bounded Stream**
Streaming Source can also be bounded, you can specify 'scan.bounded.watermark' to define the end condition for bounded streaming mode, stream reading will end until a larger watermark snapshot is encountered.
@@ -170,7 +173,3 @@ INSERT INTO paimon_table SELECT * FROM kakfa_table;
-- launch a bounded streaming job to read paimon_table
SELECT * FROM paimon_table /*+ OPTIONS('scan.bounded.watermark'='...') */;
```
-
-## Bucketed Join
-
-Bucketed table can be used to avoid shuffle if necessary in batch query, see [Bucketed Join]({{< ref "append-table/query-performance#bucketed-join" >}}).
diff --git a/docs/content/append-table/data-evolution.md b/docs/content/append-table/data-evolution.md
index fb3b91da5f..2356ef365d 100644
--- a/docs/content/append-table/data-evolution.md
+++ b/docs/content/append-table/data-evolution.md
@@ -84,7 +84,7 @@ This statement updates only the `b` column in the target table `target_table` ba
`source_table`. The `id` column and `c` column remain unchanged, and new records are inserted with the specified values. The difference between this and table those are not enabled with data evolution is that only the `b` column data is written to new files.
Note that:
-* Data Evolution Table does not support 'Delete', 'Update', or 'Compact' statement yet.
+* Data Evolution Table does not support 'Delete' and 'Update' statement yet.
* Merge Into for Data Evolution Table does not support 'WHEN NOT MATCHED BY SOURCE' clause.
### Flink
diff --git a/docs/content/append-table/incremental-clustering.md b/docs/content/append-table/incremental-clustering.md
index 5358c040d1..232ee13d7c 100644
--- a/docs/content/append-table/incremental-clustering.md
+++ b/docs/content/append-table/incremental-clustering.md
@@ -1,6 +1,6 @@
---
title: "Incremental Clustering"
-weight: 4
+weight: 2
type: docs
aliases:
- /append-table/incremental-clustering.html
diff --git a/docs/content/append-table/overview.md b/docs/content/append-table/overview.md
index 67d063c584..8644106e83 100644
--- a/docs/content/append-table/overview.md
+++ b/docs/content/append-table/overview.md
@@ -50,9 +50,154 @@ CREATE TABLE my_table (
Batch write and batch read in typical application scenarios, similar to a regular Hive partition table, but compared to
the Hive table, it can bring:
-1. Object storage (S3, OSS) friendly
-2. Time Travel and Rollback
-3. DELETE / UPDATE with low cost
-4. Automatic small file merging in streaming sink
-5. Streaming read & write like a queue
-6. High performance query with order and index
+1. Time travel enables reproducible queries that use exactly the same table snapshot, or lets users easily examine
+ changes. Version rollback allows users to quickly correct problems by resetting tables to a good state.
+2. Scan planning is fast — data files are pruned with partition and column-level stats, using table metadata. File
+ Index (BloomFilter, Bitmap, Range Bitmap) and aggregate push-down further accelerate queries.
+3. Schema evolution supports add, drop, update, or rename columns, and has no side-effects.
+4. Rich ecosystem — adds tables to compute engines including Flink, Spark, Hive, Trino, Presto, StarRocks, and Doris,
+ working just like a SQL table.
+5. Incremental Clustering with z-order/hilbert/order sorting to optimize data layout at low cost.
+6. Streaming read & write like a queue, DELETE / UPDATE / MERGE INTO support low-cost row-level operations.
+
+---
+title: "Streaming"
+weight: 2
+type: docs
+aliases:
+- /append-table/streaming.html
+---
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+## Append Streaming
+
+You can stream write to the Append table in a very flexible way through Flink, or read the Append table through
+Flink, using it like a queue. The only difference is that its latency is in minutes. Its advantages are very low cost
+and the ability to push down filters and projection.
+
+**Pre small files merging**
+
+"Pre" means that this compact occurs before committing files to the snapshot.
+
+If Flink's checkpoint interval is short (for example, 30 seconds), each snapshot may produce lots of small changelog
+files. Too many files may put a burden on the distributed storage cluster.
+
+In order to compact small changelog files into large ones, you can set the table option `precommit-compact = true`.
+Default value of this option is false, if true, it will add a compact coordinator and worker operator after the writer
+operator, which copies changelog files into large ones.
+
+**Post small files merging**
+
+"Post" means that this compact occurs after committing files to the snapshot.
+
+In streaming write job, without bucket definition, there is no compaction in writer, instead, will use
+`Compact Coordinator` to scan the small files and pass compaction task to `Compact Worker`. In streaming mode, if you
+run insert sql in flink, the topology will be like this:
+
+{{< img src="/img/unaware-bucket-topo.png">}}
+
+Do not worry about backpressure, compaction never backpressure.
+
+If you set `write-only` to true, the `Compact Coordinator` and `Compact Worker` will be removed in the topology.
+
+The auto compaction is only supported in Flink engine streaming mode. You can also start a compaction job in Flink by
+Flink action in Paimon and disable all the other compactions by setting `write-only`.
+
+**Streaming Query**
+
+You can stream the Append table and use it like a Message Queue. As with primary key tables, there are two options
+for streaming reads:
+1. By default, Streaming read produces the latest snapshot on the table upon first startup, and continue to read the
+ latest incremental records.
+2. You can specify `scan.mode`, `scan.snapshot-id`, `scan.timestamp-millis` and/or `scan.file-creation-time-millis` to
+ stream read incremental only.
+
+Similar to flink-kafka, order is not guaranteed by default, if your data has some sort of order requirement, you also
+need to consider defining a `bucket-key`, see [Bucketed Append]({{< ref "append-table/bucketed" >}})
+
+## Aggregate push down
+
+Append Table supports aggregate push down:
+
+```sql
+SELECT COUNT(*) FROM TABLE WHERE DT = '20230101';
+```
+
+This query can be accelerated during compilation and returns very quickly.
+
+For Spark SQL, table with default `metadata.stats-mode` can be accelerated:
+
+```sql
+SELECT MIN(a), MAX(b) FROM TABLE WHERE DT = '20230101';
+
+SELECT * FROM TABLE ORDER BY a LIMIT 1;
+```
+
+Min max topN query can be also accelerated during compilation and returns very quickly.
+
+## Data Skipping By Order
+
+Paimon by default records the maximum and minimum values of each field in the manifest file.
+
+In the query, according to the `WHERE` condition of the query, together with the statistics in the manifest we can
+perform file filtering. If the filtering effect is good, the query that would have cost minutes will be accelerated to
+milliseconds to complete the execution.
+
+Often the data distribution is not always ideal for filtering, so can we sort the data by the field in `WHERE` condition?
+You can take a look at [Flink COMPACT Action]({{< ref "maintenance/dedicated-compaction#sort-compact" >}}),
+[Flink COMPACT Procedure]({{< ref "flink/procedures" >}}) or [Spark COMPACT Procedure]({{< ref "spark/procedures" >}}).
+
+## Data Skipping By File Index
+
+You can use file index too, it filters files by indexing on the reading side.
+
+Define `file-index.bitmap.columns`, Data file index is an external index file and Paimon will create its
+corresponding index file for each file. If the index file is too small, it will be stored directly in the manifest,
+otherwise in the directory of the data file. Each data file corresponds to an index file, which has a separate file
+definition and can contain different types of indexes with multiple columns.
+
+Different file indexes may be efficient in different scenarios. For example bloom filter may speed up query in point lookup
+scenario. Using a bitmap may consume more space but can result in greater accuracy.
+
+* [BloomFilter]({{< ref "concepts/spec/fileindex#index-bloomfilter" >}}): `file-index.bloom-filter.columns`.
+* [Bitmap]({{< ref "concepts/spec/fileindex#index-bitmap" >}}): `file-index.bitmap.columns`.
+* [Range Bitmap]({{< ref "concepts/spec/fileindex#index-range-bitmap" >}}): `file-index.range-bitmap.columns`.
+
+If you want to add file index to existing table, without any rewrite, you can use `rewrite_file_index` procedure. Before
+we use the procedure, you should config appropriate configurations in target table. You can use ALTER clause to config
+`file-index.<filter-type>.columns` to the table.
+
+How to invoke: see [flink procedures]({{< ref "flink/procedures#procedures" >}})
+
+## Row Level Operations
+
+Now, only Spark SQL supports DELETE & UPDATE & MERGE INTO, you can take a look at [Spark Write]({{< ref "spark/sql-write" >}}).
+
+Example:
+```sql
+DELETE FROM my_table WHERE currency = 'UNKNOWN';
+```
+
+Update append table has two modes:
+
+1. COW (Copy on Write): search for the hit files and then rewrite each file to remove the data that needs to be deleted
+ from the files. This operation is costly.
+2. MOW (Merge on Write): By specifying `'deletion-vectors.enabled' = 'true'`, the Deletion Vectors mode can be enabled.
+ Only marks certain records of the corresponding file for deletion and writes the deletion file, without rewriting the entire file.
diff --git a/docs/content/append-table/query-performance.md b/docs/content/append-table/query-performance.md
deleted file mode 100644
index aad62636e5..0000000000
--- a/docs/content/append-table/query-performance.md
+++ /dev/null
@@ -1,102 +0,0 @@
----
-title: "Query Performance"
-weight: 3
-type: docs
-aliases:
-- /append-table/query-performance.html
----
-<!--
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
--->
-
-# Query Performance
-
-## Aggregate push down
-
-Append Table supports aggregate push down:
-
-```sql
-SELECT COUNT(*) FROM TABLE WHERE DT = '20230101';
-```
-
-This query can be accelerated during compilation and returns very quickly.
-
-For Spark SQL, table with default `metadata.stats-mode` can be accelerated:
-
-```sql
-SELECT MIN(a), MAX(b) FROM TABLE WHERE DT = '20230101';
-
-SELECT * FROM TABLE ORDER BY a LIMIT 1;
-```
-
-Min max topN query can be also accelerated during compilation and returns very quickly.
-
-## Data Skipping By Order
-
-Paimon by default records the maximum and minimum values of each field in the manifest file.
-
-In the query, according to the `WHERE` condition of the query, together with the statistics in the manifest we can
-perform file filtering. If the filtering effect is good, the query that would have cost minutes will be accelerated to
-milliseconds to complete the execution.
-
-Often the data distribution is not always ideal for filtering, so can we sort the data by the field in `WHERE` condition?
-You can take a look at [Flink COMPACT Action]({{< ref "maintenance/dedicated-compaction#sort-compact" >}}),
-[Flink COMPACT Procedure]({{< ref "flink/procedures" >}}) or [Spark COMPACT Procedure]({{< ref "spark/procedures" >}}).
-
-## Data Skipping By File Index
-
-You can use file index too, it filters files by indexing on the reading side.
-
-Define `file-index.bitmap.columns`, Data file index is an external index file and Paimon will create its
-corresponding index file for each file. If the index file is too small, it will be stored directly in the manifest,
-otherwise in the directory of the data file. Each data file corresponds to an index file, which has a separate file
-definition and can contain different types of indexes with multiple columns.
-
-Different file indexes may be efficient in different scenarios. For example bloom filter may speed up query in point lookup
-scenario. Using a bitmap may consume more space but can result in greater accuracy.
-
-* [BloomFilter]({{< ref "concepts/spec/fileindex#index-bloomfilter" >}}): `file-index.bloom-filter.columns`.
-* [Bitmap]({{< ref "concepts/spec/fileindex#index-bitmap" >}}): `file-index.bitmap.columns`.
-* [Range Bitmap]({{< ref "concepts/spec/fileindex#index-range-bitmap" >}}): `file-index.range-bitmap.columns`.
-
-If you want to add file index to existing table, without any rewrite, you can use `rewrite_file_index` procedure. Before
-we use the procedure, you should config appropriate configurations in target table. You can use ALTER clause to config
-`file-index.<filter-type>.columns` to the table.
-
-How to invoke: see [flink procedures]({{< ref "flink/procedures#procedures" >}})
-
-## Bucketed Join
-
-Bucketed table can be used to avoid shuffle if necessary in batch query, for example, you can use the following Spark
-SQL to read a Paimon table:
-
-```sql
-SET spark.sql.sources.v2.bucketing.enabled = true;
-
-CREATE TABLE FACT_TABLE (order_id INT, f1 STRING) TBLPROPERTIES ('bucket'='10', 'bucket-key' = 'order_id');
-
-CREATE TABLE DIM_TABLE (order_id INT, f2 STRING) TBLPROPERTIES ('bucket'='10', 'primary-key' = 'order_id');
-
-SELECT * FROM FACT_TABLE JOIN DIM_TABLE on t1.order_id = t4.order_id;
-```
-
-The `spark.sql.sources.v2.bucketing.enabled` config is used to enable bucketing for V2 data sources. When turned on,
-Spark will recognize the specific distribution reported by a V2 data source through SupportsReportPartitioning, and
-will try to avoid shuffle if necessary.
-
-The costly join shuffle will be avoided if two tables have the same bucketing strategy and same number of buckets.
diff --git a/docs/content/append-table/streaming.md b/docs/content/append-table/streaming.md
deleted file mode 100644
index 49f44a7f13..0000000000
--- a/docs/content/append-table/streaming.md
+++ /dev/null
@@ -1,104 +0,0 @@
----
-title: "Streaming"
-weight: 2
-type: docs
-aliases:
-- /append-table/streaming.html
----
-<!--
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
--->
-
-# Streaming
-
-You can stream write to the Append table in a very flexible way through Flink, or read the Append table through
-Flink, using it like a queue. The only difference is that its latency is in minutes. Its advantages are very low cost
-and the ability to push down filters and projection.
-
-## Pre small files merging
-
-"Pre" means that this compact occurs before committing files to the snapshot.
-
-If Flink's checkpoint interval is short (for example, 30 seconds), each snapshot may produce lots of small changelog
-files. Too many files may put a burden on the distributed storage cluster.
-
-In order to compact small changelog files into large ones, you can set the table option `precommit-compact = true`.
-Default value of this option is false, if true, it will add a compact coordinator and worker operator after the writer
-operator, which copies changelog files into large ones.
-
-## Post small files merging
-
-"Post" means that this compact occurs after committing files to the snapshot.
-
-In streaming write job, without bucket definition, there is no compaction in writer, instead, will use
-`Compact Coordinator` to scan the small files and pass compaction task to `Compact Worker`. In streaming mode, if you
-run insert sql in flink, the topology will be like this:
-
-{{< img src="/img/unaware-bucket-topo.png">}}
-
-Do not worry about backpressure, compaction never backpressure.
-
-If you set `write-only` to true, the `Compact Coordinator` and `Compact Worker` will be removed in the topology.
-
-The auto compaction is only supported in Flink engine streaming mode. You can also start a compaction job in Flink by
-Flink action in Paimon and disable all the other compactions by setting `write-only`.
-
-The following options control the strategy of compaction:
-
-<table class="configuration table table-bordered">
- <thead>
- <tr>
- <th class="text-left" style="width: 20%">Key</th>
- <th class="text-left" style="width: 15%">Default</th>
- <th class="text-left" style="width: 10%">Type</th>
- <th class="text-left" style="width: 55%">Description</th>
- </tr>
- </thead>
- <tbody>
- <tr>
- <td><h5>write-only</h5></td>
- <td style="word-wrap: break-word;">false</td>
- <td>Boolean</td>
- <td>If set to true, compactions and snapshot expiration will be skipped. This option is used along with dedicated compact jobs.</td>
- </tr>
- <tr>
- <td><h5>compaction.min.file-num</h5></td>
- <td style="word-wrap: break-word;">5</td>
- <td>Integer</td>
- <td>For file set [f_0,...,f_N], the minimum file number to trigger a compaction for append table.</td>
- </tr>
- <tr>
- <td><h5>compaction.delete-ratio-threshold</h5></td>
- <td style="word-wrap: break-word;">(none)</td>
- <td>Double</td>
- <td>Ratio of the deleted rows in a data file to be forced compacted.</td>
- </tr>
- </tbody>
-</table>
-
-## Streaming Query
-
-You can stream the Append table and use it like a Message Queue. As with primary key tables, there are two options
-for streaming reads:
-1. By default, Streaming read produces the latest snapshot on the table upon first startup, and continue to read the
- latest incremental records.
-2. You can specify `scan.mode`, `scan.snapshot-id`, `scan.timestamp-millis` and/or `scan.file-creation-time-millis` to
- stream read incremental only.
-
-Similar to flink-kafka, order is not guaranteed by default, if your data has some sort of order requirement, you also
-need to consider defining a `bucket-key`, see [Bucketed Append]({{< ref "append-table/bucketed" >}})
diff --git a/docs/content/append-table/update.md b/docs/content/append-table/update.md
deleted file mode 100644
index 5e373cbd69..0000000000
--- a/docs/content/append-table/update.md
+++ /dev/null
@@ -1,41 +0,0 @@
----
-title: "Update"
-weight: 4
-type: docs
-aliases:
-- /append-table/update.html
----
-<!--
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
--->
-
-# Update
-
-Now, only Spark SQL supports DELETE & UPDATE, you can take a look at [Spark Write]({{< ref "spark/sql-write" >}}).
-
-Example:
-```sql
-DELETE FROM my_table WHERE currency = 'UNKNOWN';
-```
-
-Update append table has two modes:
-
-1. COW (Copy on Write): search for the hit files and then rewrite each file to remove the data that needs to be deleted
- from the files. This operation is costly.
-2. MOW (Merge on Write): By specifying `'deletion-vectors.enabled' = 'true'`, the Deletion Vectors mode can be enabled.
- Only marks certain records of the corresponding file for deletion and writes the deletion file, without rewriting the entire file.
diff --git a/docs/content/learn-paimon/understand-files.md b/docs/content/learn-paimon/understand-files.md
index a8a9ac52ce..58fed51b84 100644
--- a/docs/content/learn-paimon/understand-files.md
+++ b/docs/content/learn-paimon/understand-files.md
@@ -488,7 +488,7 @@ this means that there are at least 5 files in a bucket. If you want to reduce th
By default, Append also does automatic compaction to reduce the number of small files.
However, for Bucketed Append table, it will only compact the files within the Bucket for sequential
-purposes, which may keep more small files. See [Bucketed Append]({{< ref "append-table/streaming#bucketed-append" >}}).
+purposes, which may keep more small files. See [Bucketed Append]({{< ref "append-table/bucketed" >}}).
### Understand Full-Compaction
(paimon) 02/37: [core] Do not create GlobalIndexScanner when no index files (#7521)
Posted by ho...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hope pushed a commit to branch release-1.4
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit 18003d5c158e79f596f6121e9cae6ff9c8d19dc6
Author: Jingsong Lee <ji...@gmail.com>
AuthorDate: Wed Mar 25 16:25:02 2026 +0800
[core] Do not create GlobalIndexScanner when no index files (#7521)
Bypass `GlobalIndexScanner` for better performance.
---
.../paimon/globalindex/DataEvolutionBatchScan.java | 9 +++++++--
.../paimon/globalindex/GlobalIndexScanner.java | 21 +++++++++++---------
.../apache/paimon/table/source/VectorReadImpl.java | 8 +++++---
.../paimon/table/BitmapGlobalIndexTableTest.java | 2 +-
.../paimon/table/BtreeGlobalIndexTableTest.java | 23 +---------------------
.../pypaimon/globalindex/global_index_scanner.py | 6 +++++-
.../pypaimon/read/scanner/file_scanner.py | 2 ++
7 files changed, 33 insertions(+), 38 deletions(-)
diff --git a/paimon-core/src/main/java/org/apache/paimon/globalindex/DataEvolutionBatchScan.java b/paimon-core/src/main/java/org/apache/paimon/globalindex/DataEvolutionBatchScan.java
index af169f4c6d..0542b7d95f 100644
--- a/paimon-core/src/main/java/org/apache/paimon/globalindex/DataEvolutionBatchScan.java
+++ b/paimon-core/src/main/java/org/apache/paimon/globalindex/DataEvolutionBatchScan.java
@@ -267,8 +267,13 @@ public class DataEvolutionBatchScan implements DataTableScan {
}
PartitionPredicate partitionFilter =
batchScan.snapshotReader().manifestsReader().partitionFilter();
- try (GlobalIndexScanner scanner =
- GlobalIndexScanner.create(table, partitionFilter, filter)) {
+ Optional<GlobalIndexScanner> optionalScanner =
+ GlobalIndexScanner.create(table, partitionFilter, filter);
+ if (!optionalScanner.isPresent()) {
+ return Optional.empty();
+ }
+
+ try (GlobalIndexScanner scanner = optionalScanner.get()) {
return scanner.scan(filter);
} catch (IOException e) {
throw new RuntimeException(e);
diff --git a/paimon-core/src/main/java/org/apache/paimon/globalindex/GlobalIndexScanner.java b/paimon-core/src/main/java/org/apache/paimon/globalindex/GlobalIndexScanner.java
index 1312f17b43..3092f1b3da 100644
--- a/paimon-core/src/main/java/org/apache/paimon/globalindex/GlobalIndexScanner.java
+++ b/paimon-core/src/main/java/org/apache/paimon/globalindex/GlobalIndexScanner.java
@@ -97,17 +97,21 @@ public class GlobalIndexScanner implements Closeable {
this.globalIndexEvaluator = new GlobalIndexEvaluator(rowType, readersFunction);
}
- public static GlobalIndexScanner create(
+ public static Optional<GlobalIndexScanner> create(
FileStoreTable table, Collection<IndexFileMeta> indexFiles) {
- return new GlobalIndexScanner(
- table.coreOptions().toConfiguration(),
- table.rowType(),
- table.fileIO(),
- table.store().pathFactory().globalIndexFileFactory(),
- indexFiles);
+ if (indexFiles.isEmpty()) {
+ return Optional.empty();
+ }
+ return Optional.of(
+ new GlobalIndexScanner(
+ table.coreOptions().toConfiguration(),
+ table.rowType(),
+ table.fileIO(),
+ table.store().pathFactory().globalIndexFileFactory(),
+ indexFiles));
}
- public static GlobalIndexScanner create(
+ public static Optional<GlobalIndexScanner> create(
FileStoreTable table, PartitionPredicate partitionFilter, Predicate filter) {
Set<Integer> filterFieldIds =
collectFieldNames(filter).stream()
@@ -131,7 +135,6 @@ public class GlobalIndexScanner implements Closeable {
.stream()
.map(IndexManifestEntry::indexFile)
.collect(Collectors.toList());
-
return create(table, indexFiles);
}
diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/VectorReadImpl.java b/paimon-core/src/main/java/org/apache/paimon/table/source/VectorReadImpl.java
index 49708a0a2a..6971bb9084 100644
--- a/paimon-core/src/main/java/org/apache/paimon/table/source/VectorReadImpl.java
+++ b/paimon-core/src/main/java/org/apache/paimon/table/source/VectorReadImpl.java
@@ -119,11 +119,13 @@ public class VectorReadImpl implements VectorRead {
for (VectorSearchSplit split : splits) {
scalarIndexFiles.addAll(split.scalarIndexFiles());
}
- if (scalarIndexFiles.isEmpty()) {
+
+ Optional<GlobalIndexScanner> optionalScanner =
+ GlobalIndexScanner.create(table, scalarIndexFiles);
+ if (!optionalScanner.isPresent()) {
return Optional.empty();
}
-
- try (GlobalIndexScanner scanner = GlobalIndexScanner.create(table, scalarIndexFiles)) {
+ try (GlobalIndexScanner scanner = optionalScanner.get()) {
return scanner.scan(filter).map(GlobalIndexResult::results);
} catch (IOException e) {
throw new RuntimeException(e);
diff --git a/paimon-core/src/test/java/org/apache/paimon/table/BitmapGlobalIndexTableTest.java b/paimon-core/src/test/java/org/apache/paimon/table/BitmapGlobalIndexTableTest.java
index cd888e6982..6443c6a5ec 100644
--- a/paimon-core/src/test/java/org/apache/paimon/table/BitmapGlobalIndexTableTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/table/BitmapGlobalIndexTableTest.java
@@ -248,7 +248,7 @@ public class BitmapGlobalIndexTableTest extends DataEvolutionTestBase {
private RoaringNavigableMap64 globalIndexScan(FileStoreTable table, Predicate predicate)
throws Exception {
try (GlobalIndexScanner scanner =
- GlobalIndexScanner.create(table, PartitionPredicate.ALWAYS_TRUE, predicate)) {
+ GlobalIndexScanner.create(table, PartitionPredicate.ALWAYS_TRUE, predicate).get()) {
return scanner.scan(predicate).get().results();
}
}
diff --git a/paimon-core/src/test/java/org/apache/paimon/table/BtreeGlobalIndexTableTest.java b/paimon-core/src/test/java/org/apache/paimon/table/BtreeGlobalIndexTableTest.java
index 98f730359d..4be621942b 100644
--- a/paimon-core/src/test/java/org/apache/paimon/table/BtreeGlobalIndexTableTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/table/BtreeGlobalIndexTableTest.java
@@ -19,7 +19,6 @@
package org.apache.paimon.table;
import org.apache.paimon.data.BinaryString;
-import org.apache.paimon.data.GenericRow;
import org.apache.paimon.globalindex.DataEvolutionBatchScan;
import org.apache.paimon.globalindex.GlobalIndexResult;
import org.apache.paimon.globalindex.GlobalIndexScanner;
@@ -29,13 +28,10 @@ import org.apache.paimon.partition.PartitionPredicate;
import org.apache.paimon.predicate.Predicate;
import org.apache.paimon.predicate.PredicateBuilder;
import org.apache.paimon.table.sink.BatchTableCommit;
-import org.apache.paimon.table.sink.BatchTableWrite;
-import org.apache.paimon.table.sink.BatchWriteBuilder;
import org.apache.paimon.table.sink.CommitMessage;
import org.apache.paimon.table.source.DataSplit;
import org.apache.paimon.table.source.ReadBuilder;
import org.apache.paimon.table.source.Split;
-import org.apache.paimon.types.RowType;
import org.apache.paimon.utils.Range;
import org.apache.paimon.utils.RoaringNavigableMap64;
@@ -206,27 +202,10 @@ public class BtreeGlobalIndexTableTest extends DataEvolutionTestBase {
.collect(Collectors.toList());
}
- private void append(int startInclusive, int endExclusive) throws Exception {
- BatchWriteBuilder builder = getTableDefault().newBatchWriteBuilder();
- RowType writeType = schemaDefault().rowType();
- try (BatchTableWrite write0 = builder.newWrite().withWriteType(writeType)) {
- for (int i = startInclusive; i < endExclusive; i++) {
- write0.write(
- GenericRow.of(
- i,
- BinaryString.fromString("a" + i),
- BinaryString.fromString("b" + i)));
- }
- try (BatchTableCommit commit = builder.newCommit()) {
- commit.commit(write0.prepareCommit());
- }
- }
- }
-
private RoaringNavigableMap64 globalIndexScan(FileStoreTable table, Predicate predicate)
throws Exception {
try (GlobalIndexScanner scanner =
- GlobalIndexScanner.create(table, PartitionPredicate.ALWAYS_TRUE, predicate)) {
+ GlobalIndexScanner.create(table, PartitionPredicate.ALWAYS_TRUE, predicate).get()) {
return scanner.scan(predicate).get().results();
}
}
diff --git a/paimon-python/pypaimon/globalindex/global_index_scanner.py b/paimon-python/pypaimon/globalindex/global_index_scanner.py
index c3d144811c..515600d77f 100644
--- a/paimon-python/pypaimon/globalindex/global_index_scanner.py
+++ b/paimon-python/pypaimon/globalindex/global_index_scanner.py
@@ -76,7 +76,7 @@ class GlobalIndexScanner:
return GlobalIndexEvaluator(fields, readers_function)
@staticmethod
- def create(table, index_files=None, partition_filter=None, predicate=None):
+ def create(table, index_files=None, partition_filter=None, predicate=None) -> Optional['GlobalIndexScanner']:
"""Create a GlobalIndexScanner.
Can be called in two ways:
@@ -86,6 +86,8 @@ class GlobalIndexScanner:
from pypaimon.index.index_file_handler import IndexFileHandler
if index_files is not None:
+ if len(index_files) == 0:
+ return None
return GlobalIndexScanner(
options=table.table_schema.options,
fields=table.fields,
@@ -117,6 +119,8 @@ class GlobalIndexScanner:
entries = index_file_handler.scan(snapshot, index_file_filter)
scanned_index_files = [entry.index_file for entry in entries]
+ if len(scanned_index_files) == 0:
+ return None
return GlobalIndexScanner(
options=table.table_schema.options,
fields=table.fields,
diff --git a/paimon-python/pypaimon/read/scanner/file_scanner.py b/paimon-python/pypaimon/read/scanner/file_scanner.py
index b770b15916..a5e8576802 100755
--- a/paimon-python/pypaimon/read/scanner/file_scanner.py
+++ b/paimon-python/pypaimon/read/scanner/file_scanner.py
@@ -313,6 +313,8 @@ class FileScanner:
partition_filter=self.partition_key_predicate,
predicate=self.predicate
)
+ if scanner is None:
+ return None
with scanner:
return scanner.scan(self.predicate)
except Exception:
(paimon) 15/37: [core] File paths in SimpleLsmKvDb should contain UUID (#7536)
Posted by ho...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hope pushed a commit to branch release-1.4
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit 89ab4f47627eda5caf46071a310c6e4a4ac9f188
Author: Jingsong Lee <ji...@gmail.com>
AuthorDate: Thu Mar 26 17:41:09 2026 +0800
[core] File paths in SimpleLsmKvDb should contain UUID (#7536)
During construction, the ClusteringCompactManager obtains a temporary
directory via ioManager.pickTempDir() and creates a SimpleLsmKvDb within
that directory. When bucket > 1, multiple ClusteringCompactManager
instances across different buckets may end up using the same temporary
directory. The original SimpleLsmKvDb.newSstFile() method used
sequentially numbered names like sst-000000.db, causing SST files from
different instances to overwrite each other. This leads to corrupted key
indexes and failed deduplication of primary keys (e.g., duplicate values
30 and 33 for key=3).
---
.../paimon/lookup/sort/db/SimpleLsmKvDb.java | 5 ++-
.../paimon/lookup/sort/db/SimpleLsmKvDbTest.java | 51 ++++++++++++++++++++++
.../paimon/crosspartition/GlobalIndexAssigner.java | 2 +-
.../java/org/apache/paimon/disk/IOManager.java | 2 +-
.../java/org/apache/paimon/disk/IOManagerImpl.java | 2 +-
.../clustering/ClusteringCompactManager.java | 2 +-
.../paimon/separated/ClusteringTableTest.java | 2 +-
7 files changed, 60 insertions(+), 6 deletions(-)
diff --git a/paimon-common/src/main/java/org/apache/paimon/lookup/sort/db/SimpleLsmKvDb.java b/paimon-common/src/main/java/org/apache/paimon/lookup/sort/db/SimpleLsmKvDb.java
index 8871fcab48..c5b8c53fd9 100644
--- a/paimon-common/src/main/java/org/apache/paimon/lookup/sort/db/SimpleLsmKvDb.java
+++ b/paimon-common/src/main/java/org/apache/paimon/lookup/sort/db/SimpleLsmKvDb.java
@@ -42,6 +42,7 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
+import java.util.UUID;
/**
* A simple LSM-Tree based KV database built on top of {@link SortLookupStoreFactory}.
@@ -92,6 +93,7 @@ public class SimpleLsmKvDb implements Closeable {
static final long PER_ENTRY_OVERHEAD = 160;
private final File dataDirectory;
+ private final String uuid;
private final SortLookupStoreFactory storeFactory;
private final Comparator<MemorySlice> keyComparator;
private final long memTableFlushThreshold;
@@ -126,6 +128,7 @@ public class SimpleLsmKvDb implements Closeable {
int level0FileNumCompactTrigger,
int sizeRatio) {
this.dataDirectory = dataDirectory;
+ this.uuid = UUID.randomUUID().toString();
this.storeFactory = storeFactory;
this.keyComparator = keyComparator;
this.memTableFlushThreshold = memTableFlushThreshold;
@@ -540,7 +543,7 @@ public class SimpleLsmKvDb implements Closeable {
private File newSstFile() {
long sequence = fileSequence++;
- return new File(dataDirectory, String.format("sst-%06d.db", sequence));
+ return new File(dataDirectory, String.format("sst-%s-%06d.db", uuid, sequence));
}
private void ensureOpen() {
diff --git a/paimon-common/src/test/java/org/apache/paimon/lookup/sort/db/SimpleLsmKvDbTest.java b/paimon-common/src/test/java/org/apache/paimon/lookup/sort/db/SimpleLsmKvDbTest.java
index 44d7d46ff5..92a5488591 100644
--- a/paimon-common/src/test/java/org/apache/paimon/lookup/sort/db/SimpleLsmKvDbTest.java
+++ b/paimon-common/src/test/java/org/apache/paimon/lookup/sort/db/SimpleLsmKvDbTest.java
@@ -1430,6 +1430,57 @@ public class SimpleLsmKvDbTest {
}
}
+ @Test
+ public void testTwoInstancesSameDirectoryNoFileCollision() throws IOException {
+ // Two SimpleLsmKvDb instances sharing the same dataDirectory should not interfere
+ // with each other because SST file names contain a unique UUID.
+ File sharedDir = new File(tempDir.toFile(), "shared-dir-db");
+
+ try (SimpleLsmKvDb db1 =
+ SimpleLsmKvDb.builder(sharedDir)
+ .memTableFlushThreshold(1024)
+ .blockSize(256)
+ .level0FileNumCompactTrigger(4)
+ .compressOptions(new CompressOptions("none", 1))
+ .build();
+ SimpleLsmKvDb db2 =
+ SimpleLsmKvDb.builder(sharedDir)
+ .memTableFlushThreshold(1024)
+ .blockSize(256)
+ .level0FileNumCompactTrigger(4)
+ .compressOptions(new CompressOptions("none", 1))
+ .build()) {
+
+ // Write different data to each instance
+ putString(db1, "key-a", "from-db1");
+ putString(db1, "key-b", "from-db1");
+ db1.flush();
+
+ putString(db2, "key-a", "from-db2");
+ putString(db2, "key-c", "from-db2");
+ db2.flush();
+
+ // Each instance should see only its own data
+ Assertions.assertEquals("from-db1", getString(db1, "key-a"));
+ Assertions.assertEquals("from-db1", getString(db1, "key-b"));
+ Assertions.assertNull(getString(db1, "key-c"));
+
+ Assertions.assertEquals("from-db2", getString(db2, "key-a"));
+ Assertions.assertNull(getString(db2, "key-b"));
+ Assertions.assertEquals("from-db2", getString(db2, "key-c"));
+
+ // Write more data and flush again to ensure no cross-contamination
+ putString(db1, "key-a", "updated-db1");
+ db1.flush();
+
+ putString(db2, "key-a", "updated-db2");
+ db2.flush();
+
+ Assertions.assertEquals("updated-db1", getString(db1, "key-a"));
+ Assertions.assertEquals("updated-db2", getString(db2, "key-a"));
+ }
+ }
+
@Test
public void testBulkLoadFailsOnNonEmptyDb() throws IOException {
try (SimpleLsmKvDb db = createDb()) {
diff --git a/paimon-core/src/main/java/org/apache/paimon/crosspartition/GlobalIndexAssigner.java b/paimon-core/src/main/java/org/apache/paimon/crosspartition/GlobalIndexAssigner.java
index c387f9abdf..9097c5c482 100644
--- a/paimon-core/src/main/java/org/apache/paimon/crosspartition/GlobalIndexAssigner.java
+++ b/paimon-core/src/main/java/org/apache/paimon/crosspartition/GlobalIndexAssigner.java
@@ -132,7 +132,7 @@ public class GlobalIndexAssigner implements Serializable, Closeable {
this.extractor = new RowPartitionAllPrimaryKeyExtractor(table.schema());
this.keyPartExtractor = new KeyPartPartitionKeyExtractor(table.schema());
- String tmpDir = ioManager.pickRandomTempDir();
+ String tmpDir = ioManager.pickTempDir();
this.path = new File(tmpDir, "rocksdb-" + UUID.randomUUID());
if (!this.path.mkdirs()) {
throw new RuntimeException(
diff --git a/paimon-core/src/main/java/org/apache/paimon/disk/IOManager.java b/paimon-core/src/main/java/org/apache/paimon/disk/IOManager.java
index 4a35167e1e..352f8c4a73 100644
--- a/paimon-core/src/main/java/org/apache/paimon/disk/IOManager.java
+++ b/paimon-core/src/main/java/org/apache/paimon/disk/IOManager.java
@@ -38,7 +38,7 @@ public interface IOManager extends AutoCloseable {
String[] tempDirs();
- String pickRandomTempDir();
+ String pickTempDir();
Enumerator createChannelEnumerator();
diff --git a/paimon-core/src/main/java/org/apache/paimon/disk/IOManagerImpl.java b/paimon-core/src/main/java/org/apache/paimon/disk/IOManagerImpl.java
index 57926d0154..c0028c6bc6 100644
--- a/paimon-core/src/main/java/org/apache/paimon/disk/IOManagerImpl.java
+++ b/paimon-core/src/main/java/org/apache/paimon/disk/IOManagerImpl.java
@@ -95,7 +95,7 @@ public class IOManagerImpl implements IOManager {
}
@Override
- public String pickRandomTempDir() {
+ public String pickTempDir() {
return pickRandomly(Arrays.asList(tempDirs()));
}
diff --git a/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/clustering/ClusteringCompactManager.java b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/clustering/ClusteringCompactManager.java
index 9e925d0764..12acea3be6 100644
--- a/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/clustering/ClusteringCompactManager.java
+++ b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/clustering/ClusteringCompactManager.java
@@ -119,7 +119,7 @@ public class ClusteringCompactManager extends CompactFutureManager {
valueType.getFieldTypes(), clusteringColumnIndexes, true);
SimpleLsmKvDb kvDb =
- SimpleLsmKvDb.builder(new File(ioManager.pickRandomTempDir()))
+ SimpleLsmKvDb.builder(new File(ioManager.pickTempDir()))
.cacheManager(cacheManager)
.keyComparator(new RowCompactedSerializer(keyType).createSliceComparator())
.build();
diff --git a/paimon-core/src/test/java/org/apache/paimon/separated/ClusteringTableTest.java b/paimon-core/src/test/java/org/apache/paimon/separated/ClusteringTableTest.java
index aaf661be2f..353dd6d704 100644
--- a/paimon-core/src/test/java/org/apache/paimon/separated/ClusteringTableTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/separated/ClusteringTableTest.java
@@ -74,7 +74,7 @@ class ClusteringTableTest {
.column("b", DataTypes.INT())
.primaryKey("a")
.option(DELETION_VECTORS_ENABLED.key(), "true")
- .option(BUCKET.key(), "1")
+ .option(BUCKET.key(), "2")
.option(CLUSTERING_COLUMNS.key(), "b")
.option(PK_CLUSTERING_OVERRIDE.key(), "true")
.build();
(paimon) 03/37: [core] Support config compaction file num limit and report if reach the limit in batch job (#7520)
Posted by ho...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hope pushed a commit to branch release-1.4
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit 09e19b1331c752d2966e32c683008e3eb7f22f2b
Author: yuzelin <33...@users.noreply.github.com>
AuthorDate: Wed Mar 25 16:25:47 2026 +0800
[core] Support config compaction file num limit and report if reach the limit in batch job (#7520)
1. In batch job, we should notice the outside system that whether
current compaction has handled all files, so they can determine if they
need to compact again.
2. Make the limit configurable.
---
.../shortcodes/generated/core_configuration.html | 6 ++
.../main/java/org/apache/paimon/CoreOptions.java | 12 ++++
.../paimon/append/AppendCompactCoordinator.java | 25 ++++++-
.../append/AppendOnlyTableCompactionTest.java | 78 ++++++++++++++++++++++
4 files changed, 118 insertions(+), 3 deletions(-)
diff --git a/docs/layouts/shortcodes/generated/core_configuration.html b/docs/layouts/shortcodes/generated/core_configuration.html
index 44aeee4980..67618e4e83 100644
--- a/docs/layouts/shortcodes/generated/core_configuration.html
+++ b/docs/layouts/shortcodes/generated/core_configuration.html
@@ -308,6 +308,12 @@ under the License.
<td>Double</td>
<td>Ratio of the deleted rows in a data file to be forced compacted for append-only table.</td>
</tr>
+ <tr>
+ <td><h5>compaction.file-num-limit</h5></td>
+ <td style="word-wrap: break-word;">100000</td>
+ <td>Integer</td>
+ <td>To avoid OOM caused by scanning compaction files, you can use this option to limit the for unaware-bucket append table compaction.</td>
+ </tr>
<tr>
<td><h5>compaction.force-rewrite-all-files</h5></td>
<td style="word-wrap: break-word;">false</td>
diff --git a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java
index 569615f8b0..e8ae69e9be 100644
--- a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java
+++ b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java
@@ -874,6 +874,14 @@ public class CoreOptions implements Serializable {
"For file set [f_0,...,f_N], the minimum file number to trigger a compaction for "
+ "append-only table.");
+ public static final ConfigOption<Integer> COMPACTION_FILE_NUM_LIMIT =
+ key("compaction.file-num-limit")
+ .intType()
+ .defaultValue(100_000)
+ .withDescription(
+ "To avoid OOM caused by scanning compaction files, you can use this option to limit the "
+ + "for unaware-bucket append table compaction.");
+
public static final ConfigOption<Double> COMPACTION_DELETE_RATIO_THRESHOLD =
key("compaction.delete-ratio-threshold")
.doubleType()
@@ -3026,6 +3034,10 @@ public class CoreOptions implements Serializable {
return options.get(COMPACTION_MIN_FILE_NUM);
}
+ public int compactionFileNumLimit() {
+ return options.get(COMPACTION_FILE_NUM_LIMIT);
+ }
+
public double compactionDeleteRatioThreshold() {
return options.get(COMPACTION_DELETE_RATIO_THRESHOLD);
}
diff --git a/paimon-core/src/main/java/org/apache/paimon/append/AppendCompactCoordinator.java b/paimon-core/src/main/java/org/apache/paimon/append/AppendCompactCoordinator.java
index 07374831e7..70e5040e3b 100644
--- a/paimon-core/src/main/java/org/apache/paimon/append/AppendCompactCoordinator.java
+++ b/paimon-core/src/main/java/org/apache/paimon/append/AppendCompactCoordinator.java
@@ -69,8 +69,6 @@ import static org.apache.paimon.utils.Preconditions.checkArgument;
*/
public class AppendCompactCoordinator {
- private static final int FILES_BATCH = 100_000;
-
protected static final int REMOVE_AGE = 10;
protected static final int COMPACT_AGE = 5;
@@ -80,11 +78,15 @@ public class AppendCompactCoordinator {
private final double deleteThreshold;
private final long openFileCost;
private final int minFileNum;
+ private final int fileNumLimit;
private final DvMaintainerCache dvMaintainerCache;
private final FilesIterator filesIterator;
final Map<BinaryRow, SubCoordinator> subCoordinators = new HashMap<>();
+ private final boolean isStreaming;
+ private boolean batchRemainFiles;
+
public AppendCompactCoordinator(FileStoreTable table, boolean isStreaming) {
this(table, isStreaming, null);
}
@@ -101,11 +103,13 @@ public class AppendCompactCoordinator {
this.deleteThreshold = options.compactionDeleteRatioThreshold();
this.openFileCost = options.splitOpenFileCost();
this.minFileNum = options.compactionMinFileNum();
+ this.fileNumLimit = options.compactionFileNumLimit();
this.dvMaintainerCache =
options.deletionVectorsEnabled()
? new DvMaintainerCache(table.store().newIndexFileHandler())
: null;
this.filesIterator = new FilesIterator(table, isStreaming, partitionPredicate);
+ this.isStreaming = isStreaming;
}
public List<AppendCompactTask> run() {
@@ -121,11 +125,12 @@ public class AppendCompactCoordinator {
@VisibleForTesting
boolean scan() {
Map<BinaryRow, List<DataFileMeta>> files = new HashMap<>();
- for (int i = 0; i < FILES_BATCH; i++) {
+ for (int i = 0; i < fileNumLimit; i++) {
ManifestEntry entry;
try {
entry = filesIterator.next();
} catch (EndOfScanException e) {
+ batchRemainFiles = false;
if (!files.isEmpty()) {
files.forEach(this::notifyNewFiles);
return true;
@@ -140,13 +145,27 @@ public class AppendCompactCoordinator {
}
if (files.isEmpty()) {
+ batchRemainFiles = false;
return false;
}
+ if (!isStreaming) {
+ try {
+ ManifestEntry entry = filesIterator.next();
+ batchRemainFiles = entry != null;
+ } catch (EndOfScanException e) {
+ batchRemainFiles = false;
+ }
+ }
+
files.forEach(this::notifyNewFiles);
return true;
}
+ public boolean batchRemainFiles() {
+ return batchRemainFiles;
+ }
+
@VisibleForTesting
FilesIterator filesIterator() {
return filesIterator;
diff --git a/paimon-core/src/test/java/org/apache/paimon/append/AppendOnlyTableCompactionTest.java b/paimon-core/src/test/java/org/apache/paimon/append/AppendOnlyTableCompactionTest.java
index 5af16aa316..e4d1c43ed2 100644
--- a/paimon-core/src/test/java/org/apache/paimon/append/AppendOnlyTableCompactionTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/append/AppendOnlyTableCompactionTest.java
@@ -34,6 +34,7 @@ import org.apache.paimon.table.sink.CommitMessage;
import org.apache.paimon.table.sink.CommitMessageImpl;
import org.apache.paimon.table.sink.StreamTableWrite;
import org.apache.paimon.table.sink.TableCommitImpl;
+import org.apache.paimon.table.source.EndOfScanException;
import org.apache.paimon.types.DataTypes;
import org.apache.paimon.utils.SnapshotManager;
@@ -43,13 +44,18 @@ import org.junit.jupiter.api.io.TempDir;
import java.nio.file.Path;
import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import java.util.Random;
import java.util.UUID;
import static java.util.Collections.singletonMap;
import static org.apache.paimon.SnapshotTest.newSnapshotManager;
+import static org.apache.paimon.testutils.assertj.PaimonAssertions.anyCauseMatches;
import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
/** Test for append table compaction. */
public class AppendOnlyTableCompactionTest {
@@ -208,6 +214,78 @@ public class AppendOnlyTableCompactionTest {
return result;
}
+ @Test
+ public void testBatchScanFileNumLimitWithRemainingFiles() throws Exception {
+ commit(writeCommit(10));
+
+ Map<String, String> options = new HashMap<>();
+ options.put("compaction.file-num-limit", "5");
+ AppendCompactCoordinator batchCoordinator = createBatchCoordinator(options);
+
+ assertThat(batchCoordinator.scan()).isTrue();
+ assertThat(batchCoordinator.batchRemainFiles()).isTrue();
+ assertThat(batchCoordinator.listRestoredFiles().size()).isEqualTo(5);
+ }
+
+ @Test
+ public void testBatchScanFileNumLimitNoRemainingFiles() throws Exception {
+ commit(writeCommit(10));
+
+ // default 100_000
+ AppendCompactCoordinator batchCoordinator = createBatchCoordinator(Collections.emptyMap());
+
+ assertThat(batchCoordinator.scan()).isTrue();
+ assertThat(batchCoordinator.batchRemainFiles()).isFalse();
+ assertThat(batchCoordinator.listRestoredFiles().size()).isEqualTo(10);
+ }
+
+ @Test
+ public void testBatchScanFileNumLimitExactMatch() throws Exception {
+ commit(writeCommit(5));
+
+ Map<String, String> options = new HashMap<>();
+ options.put("compaction.file-num-limit", "5");
+ AppendCompactCoordinator batchCoordinator = createBatchCoordinator(options);
+
+ assertThat(batchCoordinator.scan()).isTrue();
+ assertThat(batchCoordinator.batchRemainFiles()).isFalse();
+ assertThat(batchCoordinator.listRestoredFiles().size()).isEqualTo(5);
+ }
+
+ @Test
+ public void testBatchScanEmptyTableBatchRemainFiles() {
+ AppendCompactCoordinator batchCoordinator = createBatchCoordinator(Collections.emptyMap());
+
+ assertThatThrownBy(batchCoordinator::scan)
+ .satisfies(anyCauseMatches(EndOfScanException.class));
+ assertThat(batchCoordinator.batchRemainFiles()).isFalse();
+ }
+
+ @Test
+ public void testBatchScanAllFilesFilteredOut() throws Exception {
+ commit(writeCommit(10));
+
+ // Set target-file-size to 1 byte, making compactionFileSize = 0,
+ // so shouldCompact returns false for all files
+ Map<String, String> options = new HashMap<>();
+ options.put("target-file-size", "1 b");
+ AppendCompactCoordinator batchCoordinator = createBatchCoordinator(options);
+
+ assertThatThrownBy(batchCoordinator::scan)
+ .satisfies(anyCauseMatches(EndOfScanException.class));
+ assertThat(batchCoordinator.batchRemainFiles()).isFalse();
+ }
+
+ private AppendCompactCoordinator createBatchCoordinator(Map<String, String> extraOptions) {
+ TableSchema schema = tableSchema.copy(extraOptions);
+ FileStoreTable table =
+ FileStoreTableFactory.create(
+ LocalFileIO.create(),
+ new org.apache.paimon.fs.Path(tempDir.toString()),
+ schema);
+ return new AppendCompactCoordinator(table, false);
+ }
+
private InternalRow randomRow() {
return GenericRow.of(
random.nextInt(100),
(paimon) 10/37: [core] Chain table support group partition. (#7524)
Posted by ho...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hope pushed a commit to branch release-1.4
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit 42812e5858e5de3ccc9142b52991e16dd0fa2234
Author: Wenchao Wu <60...@users.noreply.github.com>
AuthorDate: Thu Mar 26 09:47:28 2026 +0800
[core] Chain table support group partition. (#7524)
[Feature] chain table support group partition. #7522
Previously, the chain table merge logic assumed all partition fields
participate in the chain dimension. For example, a table partitioned by
(region, date) was treated as a single flat sequence, meaning a
chain-table query for region=US could incorrectly pick up a snapshot
anchor from region=EU.
This change introduces the concept of a group partition: the non-chain
prefix fields of the partition key. Delta partitions are now grouped by
their group key before performing the chain-merge, so that the anchor
search for each group is scoped correctly.
---
docs/content/primary-key-table/chain-table.md | 59 +
.../shortcodes/generated/core_configuration.html | 6 +
.../main/java/org/apache/paimon/CoreOptions.java | 17 +
.../org/apache/paimon/schema/SchemaValidation.java | 23 +
.../apache/paimon/table/ChainGroupReadTable.java | 307 +++--
.../paimon/utils/ChainPartitionProjector.java | 160 +++
.../org/apache/paimon/utils/ChainTableUtils.java | 132 ++
.../paimon/utils/ChainPartitionProjectorTest.java | 432 ++++++
.../apache/paimon/utils/ChainTableUtilsTest.java | 532 +++++++
.../apache/paimon/spark/SparkChainTableITCase.java | 1456 ++++++++++++++++++++
10 files changed, 3008 insertions(+), 116 deletions(-)
diff --git a/docs/content/primary-key-table/chain-table.md b/docs/content/primary-key-table/chain-table.md
index d8d4120f91..839eb12fc9 100644
--- a/docs/content/primary-key-table/chain-table.md
+++ b/docs/content/primary-key-table/chain-table.md
@@ -147,3 +147,62 @@ you will get the following result:
| 2 | 1| 1 |
+---+----+-----+
```
+
+## Group Partition
+
+In real-world scenarios, a table often has multiple partition dimensions. For example, data may be
+partitioned by both `region` and `date`. In such cases, different regions are independent data silos —
+each should maintain its own chain independently rather than sharing one global chain across all regions.
+
+Paimon supports this pattern via **group partition**: partition keys are divided into two parts:
+- **Group partition keys** (prefix fields): Dimensions that identify independent data silos (e.g., `region`).
+ Each distinct combination of group partition values forms its own independent chain.
+- **Chain partition keys** (suffix fields): Dimensions that form the time-ordered chain within a group
+ (e.g., `date`).
+
+Use `chain-table.chain-partition-keys` to specify the chain dimension. This value must be a
+**contiguous suffix** of the table's partition keys. Partition fields before it automatically become the
+group dimension. If this option is not set, all partitions belong to a single implicit group (the
+default behavior for single-dimension partitioned tables).
+
+Consider an example where the table is partitioned by `region` and `date`, and you want each region to
+have its own chain:
+
+```sql
+CREATE TABLE default.t (
+ `t1` string ,
+ `t2` string ,
+ `t3` string
+) PARTITIONED BY (`region` string, `date` string)
+TBLPROPERTIES (
+ 'chain-table.enabled' = 'true',
+ 'primary-key' = 'region,date,t1',
+ 'sequence.field' = 't2',
+ 'bucket-key' = 't1',
+ 'bucket' = '2',
+ 'partition.timestamp-pattern' = '$date',
+ 'partition.timestamp-formatter' = 'yyyyMMdd',
+ -- specify that only `date` is the chain dimension; `region` becomes the group dimension
+ 'chain-table.chain-partition-keys' = 'date'
+);
+```
+
+With this configuration:
+- Partition keys: `[region, date]`
+- Group partition keys: `[region]` — CN and US each have their own independent chain
+- Chain partition keys: `[date]` — time-ordered chain within each region
+
+When reading a partition like `(region='CN', date='20250811')`, Paimon finds the nearest earlier
+snapshot partition **within the same region** (e.g., `(region='CN', date='20250810')`) as the chain
+anchor, and merges forward through the delta data for the CN group only. The US group is resolved
+independently using its own anchor.
+
+For hourly partitioned tables with a regional dimension, you can set both `dt` and `hour` as chain
+partition keys:
+
+```sql
+'chain-table.chain-partition-keys' = 'dt,hour'
+```
+
+This treats `(dt, hour)` as the composite chain dimension and everything before it (e.g., `region`) as
+the group dimension.
diff --git a/docs/layouts/shortcodes/generated/core_configuration.html b/docs/layouts/shortcodes/generated/core_configuration.html
index 67618e4e83..220ca4cdc4 100644
--- a/docs/layouts/shortcodes/generated/core_configuration.html
+++ b/docs/layouts/shortcodes/generated/core_configuration.html
@@ -1187,6 +1187,12 @@ This config option does not affect the default filesystem metastore.</td>
<td>String</td>
<td>When a batch job queries from a chain table, if a partition does not exist in the main branch, the reader will try to get this partition from chain snapshot branch.</td>
</tr>
+ <tr>
+ <td><h5>chain-table.chain-partition-keys</h5></td>
+ <td style="word-wrap: break-word;">(none)</td>
+ <td>String</td>
+ <td>Partition keys that participate in chain logic. Must be a contiguous suffix of the table's partition keys. Comma-separated. If not set, all partition keys participate in chain.</td>
+ </tr>
<tr>
<td><h5>scan.file-creation-time-millis</h5></td>
<td style="word-wrap: break-word;">(none)</td>
diff --git a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java
index e8ae69e9be..4dc495c12b 100644
--- a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java
+++ b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java
@@ -273,6 +273,15 @@ public class CoreOptions implements Serializable {
"When a batch job queries from a chain table, if a partition does not exist in either main or snapshot branch, "
+ "the reader will try to get this partition from chain snapshot and delta branch together.");
+ public static final ConfigOption<String> CHAIN_TABLE_CHAIN_PARTITION_KEYS =
+ key("chain-table.chain-partition-keys")
+ .stringType()
+ .noDefaultValue()
+ .withDescription(
+ "Partition keys that participate in chain logic. Must be a contiguous "
+ + "suffix of the table's partition keys. Comma-separated. "
+ + "If not set, all partition keys participate in chain.");
+
public static final String FILE_FORMAT_ORC = "orc";
public static final String FILE_FORMAT_AVRO = "avro";
public static final String FILE_FORMAT_PARQUET = "parquet";
@@ -3694,6 +3703,14 @@ public class CoreOptions implements Serializable {
return options.get(SCAN_FALLBACK_DELTA_BRANCH);
}
+ public List<String> chainTableChainPartitionKeys() {
+ String value = options.get(CHAIN_TABLE_CHAIN_PARTITION_KEYS);
+ if (value == null || value.trim().isEmpty()) {
+ return null;
+ }
+ return Arrays.stream(value.split(",")).map(String::trim).collect(Collectors.toList());
+ }
+
public boolean formatTableImplementationIsPaimon() {
return options.get(FORMAT_TABLE_IMPLEMENTATION) == FormatTableImplementation.PAIMON;
}
diff --git a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java
index 130f9b29eb..b9b68fc647 100644
--- a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java
+++ b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java
@@ -784,6 +784,29 @@ public class SchemaValidation {
Preconditions.checkArgument(
options.partitionTimestampFormatter() != null,
"Partition timestamp formatter is required for chain table.");
+
+ // validate chain-table.chain-partition-keys
+ List<String> chainPartKeys = options.chainTableChainPartitionKeys();
+ if (chainPartKeys != null) {
+ List<String> partitionKeys = schema.partitionKeys();
+
+ Preconditions.checkArgument(
+ !chainPartKeys.isEmpty(),
+ "chain-table.chain-partition-keys must not be empty.");
+
+ // chain partition keys should be in the tails of partition keys.
+ int chainStart = partitionKeys.size() - chainPartKeys.size();
+ Preconditions.checkArgument(
+ chainStart >= 0
+ && partitionKeys
+ .subList(chainStart, partitionKeys.size())
+ .equals(chainPartKeys),
+ "chain-table.chain-partition-keys must be a contiguous suffix "
+ + "of the table's partition keys. "
+ + "Partition keys: %s, chain partition keys: %s.",
+ partitionKeys,
+ chainPartKeys);
+ }
}
}
diff --git a/paimon-core/src/main/java/org/apache/paimon/table/ChainGroupReadTable.java b/paimon-core/src/main/java/org/apache/paimon/table/ChainGroupReadTable.java
index 2379882070..78752acfc6 100644
--- a/paimon-core/src/main/java/org/apache/paimon/table/ChainGroupReadTable.java
+++ b/paimon-core/src/main/java/org/apache/paimon/table/ChainGroupReadTable.java
@@ -39,9 +39,9 @@ import org.apache.paimon.table.source.InnerTableRead;
import org.apache.paimon.table.source.Split;
import org.apache.paimon.table.source.TableRead;
import org.apache.paimon.types.RowType;
+import org.apache.paimon.utils.ChainPartitionProjector;
import org.apache.paimon.utils.ChainTableUtils;
import org.apache.paimon.utils.Filter;
-import org.apache.paimon.utils.InternalRowPartitionComputer;
import org.apache.paimon.utils.Pair;
import org.apache.paimon.utils.RowDataToObjectArrayConverter;
@@ -129,10 +129,11 @@ public class ChainGroupReadTable extends FallbackReadFileStoreTable {
public static class ChainTableBatchScan extends FallbackReadScan {
private final RowDataToObjectArrayConverter partitionConverter;
- private final InternalRowPartitionComputer partitionComputer;
private final CoreOptions options;
private final RecordComparator partitionComparator;
private final ChainGroupReadTable chainGroupReadTable;
+ private final RecordComparator chainPartitionComparator;
+ private final ChainPartitionProjector partitionProjector;
private Predicate dataPredicate;
private Filter<Integer> bucketFilter;
@@ -151,21 +152,28 @@ public class ChainGroupReadTable extends FallbackReadFileStoreTable {
this.chainGroupReadTable = chainGroupReadTable;
this.partitionConverter =
new RowDataToObjectArrayConverter(tableSchema.logicalPartitionType());
- this.partitionComputer =
- new InternalRowPartitionComputer(
- options.partitionDefaultName(),
- tableSchema.logicalPartitionType(),
- tableSchema.partitionKeys().toArray(new String[0]),
- options.legacyPartitionName());
this.partitionComparator =
CodeGenUtils.newRecordComparator(
tableSchema.logicalPartitionType().getFieldTypes());
+ List<String> chainKeys =
+ ChainTableUtils.chainPartitionKeys(options, tableSchema.partitionKeys());
+ int chainFieldCount = chainKeys.size();
+ // full partition comparator
+ this.partitionProjector =
+ new ChainPartitionProjector(
+ tableSchema.logicalPartitionType(), chainFieldCount);
+ // chain dimension comparator (compares only the last chainFieldCount fields)
+ this.chainPartitionComparator =
+ CodeGenUtils.newRecordComparator(
+ partitionProjector.chainPartitionType().getFieldTypes());
}
@Override
public ChainTableBatchScan withFilter(Predicate predicate) {
super.withFilter(predicate);
- if (predicate != null) {
+ if (predicate == null) {
+ dataPredicate = null;
+ } else {
Pair<Optional<PartitionPredicate>, List<Predicate>> pair =
PartitionPredicate.splitPartitionPredicatesAndDataPredicates(
predicate,
@@ -217,11 +225,25 @@ public class ChainGroupReadTable extends FallbackReadFileStoreTable {
/**
* Builds a plan for chain tables.
*
- * <p>Partitions that exist in the snapshot branch (based on partition predicates only) are
- * treated as complete and are read directly from snapshot, subject to row-level predicates.
- * Partitions that exist only in the delta branch are planned as chain splits by pairing
- * each delta partition with the latest snapshot partition at or before it (if any), so the
- * reader sees a full partition view.
+ * <p>Partitions that exist in the snapshot branch (matched via the user's partition
+ * predicate) are treated as complete and are read directly from snapshot, subject to
+ * row-level predicates.
+ *
+ * <p>Partitions that exist only in the delta branch are planned group-by-group:
+ *
+ * <ol>
+ * <li>Delta partitions are grouped by their <em>group</em> key prefix (the non-chain
+ * partition fields). When {@code chain-table.chain-partition-keys} is not configured,
+ * all partitions fall into a single implicit group.
+ * <li>For each group the nearest earlier snapshot partition is used as an <em>anchor</em>
+ * for the chain merge. The anchor search uses a targeted predicate (group fields
+ * exact-match AND chain < maxChainInGroup) that intentionally bypasses the user's
+ * partition predicate, so the correct anchor is found even when the user queries a
+ * future or non-existent partition.
+ * <li>A {@link ChainSplit} is built that merges the anchor snapshot data with the delta
+ * data in the range {@code (anchorChain, queryChain]}, giving the reader a complete
+ * view of the queried partition.
+ * </ol>
*/
@Override
public Plan plan() {
@@ -258,110 +280,163 @@ public class ChainGroupReadTable extends FallbackReadFileStoreTable {
.collect(Collectors.toList());
if (!deltaPartitions.isEmpty()) {
- BinaryRow maxPartition = deltaPartitions.get(deltaPartitions.size() - 1);
- Predicate snapshotPredicate =
- ChainTableUtils.createTriangularPredicate(
- maxPartition,
- partitionConverter,
- builder::equal,
- builder::lessThan);
- PartitionPredicate snapshotPartitionPredicate =
- PartitionPredicate.fromPredicate(
- tableSchema.logicalPartitionType(), snapshotPredicate);
- DataTableScan snapshotPartitionsScan =
- newChainPartitionListingScan(true, snapshotPartitionPredicate);
- List<BinaryRow> candidateSnapshotPartitions =
- snapshotPartitionsScan.listPartitions();
- candidateSnapshotPartitions =
- candidateSnapshotPartitions.stream()
- .sorted(partitionComparator)
- .collect(Collectors.toList());
- Map<BinaryRow, BinaryRow> partitionMapping =
- ChainTableUtils.findFirstLatestPartitions(
- deltaPartitions, candidateSnapshotPartitions, partitionComparator);
- for (Map.Entry<BinaryRow, BinaryRow> partitionParis : partitionMapping.entrySet()) {
- DataTableScan snapshotScan = newFilteredScan(true);
- DataTableScan deltaScan = newFilteredScan(false);
- if (partitionParis.getValue() == null) {
- List<Predicate> predicates = new ArrayList<>();
- predicates.add(
- ChainTableUtils.createTriangularPredicate(
- partitionParis.getKey(),
- partitionConverter,
- builder::equal,
- builder::lessThan));
- predicates.add(
- ChainTableUtils.createLinearPredicate(
- partitionParis.getKey(),
- partitionConverter,
- builder::equal));
- deltaScan.withPartitionFilter(PredicateBuilder.or(predicates));
- } else {
- List<BinaryRow> selectedDeltaPartitions =
- ChainTableUtils.getDeltaPartitions(
- partitionParis.getValue(),
- partitionParis.getKey(),
- tableSchema.partitionKeys(),
- tableSchema.logicalPartitionType(),
- options,
- partitionComparator,
- partitionComputer);
- deltaScan.withPartitionFilter(selectedDeltaPartitions);
- }
- List<Split> subSplits = deltaScan.plan().splits();
- Set<String> snapshotFileNames = new HashSet<>();
- if (partitionParis.getValue() != null) {
- snapshotScan.withPartitionFilter(
- Collections.singletonList(partitionParis.getValue()));
- List<Split> mainSubSplits = snapshotScan.plan().splits();
- snapshotFileNames =
- mainSubSplits.stream()
- .flatMap(
- s ->
- ((DataSplit) s)
- .dataFiles().stream()
- .map(
- DataFileMeta
- ::fileName))
- .collect(Collectors.toSet());
- subSplits.addAll(mainSubSplits);
- }
- Map<Integer, List<DataSplit>> bucketSplits = new LinkedHashMap<>();
- for (Split split : subSplits) {
- DataSplit dataSplit = (DataSplit) split;
- Integer totalBuckets = dataSplit.totalBuckets();
- checkNotNull(totalBuckets);
- checkArgument(
- totalBuckets == options.bucket(),
- "Inconsistent bucket num " + dataSplit.bucket());
- bucketSplits
- .computeIfAbsent(dataSplit.bucket(), k -> new ArrayList<>())
- .add(dataSplit);
- }
- for (Map.Entry<Integer, List<DataSplit>> entry : bucketSplits.entrySet()) {
- HashMap<String, String> fileBucketPathMapping = new HashMap<>();
- HashMap<String, String> fileBranchMapping = new HashMap<>();
- List<DataSplit> splitList = entry.getValue();
- for (DataSplit dataSplit : splitList) {
- for (DataFileMeta file : dataSplit.dataFiles()) {
- fileBucketPathMapping.put(file.fileName(), dataSplit.bucketPath());
- String branch =
- snapshotFileNames.contains(file.fileName())
- ? options.scanFallbackSnapshotBranch()
- : options.scanFallbackDeltaBranch();
- fileBranchMapping.put(file.fileName(), branch);
+ // Group delta partitions by group key.
+ // When groupFieldCount == 0, extractGroupPartition returns a zero-field BinaryRow,
+ // so all partitions are placed in a single group.
+ Map<BinaryRow, List<BinaryRow>> groupedDeltaPartitions = new LinkedHashMap<>();
+ for (BinaryRow partition : deltaPartitions) {
+ BinaryRow groupKey = partitionProjector.extractGroupPartition(partition);
+ groupedDeltaPartitions
+ .computeIfAbsent(groupKey, k -> new ArrayList<>())
+ .add(partition);
+ }
+
+ for (List<BinaryRow> deltaPartitionsInGroup : groupedDeltaPartitions.values()) {
+
+ // Sort delta by chain dimension ascending.
+ // chainPartitionForCompare avoids copying BinaryRow in the comparator hot path.
+ deltaPartitionsInGroup.sort(
+ (a, b) ->
+ chainPartitionComparator.compare(
+ partitionProjector.chainPartitionForCompare(a),
+ partitionProjector.chainPartitionForCompare(b)));
+
+ // Build a targeted snapshot-anchor predicate:
+ // group fields exact-match AND chain < maxChainInGroup
+ // When groupFieldCount == 0 this degenerates to the plain triangular predicate
+ // (equivalent to the former planWithoutGroupPartition snapshot listing).
+ BinaryRow maxDeltaInGroup =
+ deltaPartitionsInGroup.get(deltaPartitionsInGroup.size() - 1);
+ Predicate snapshotSearchPred =
+ ChainTableUtils.createGroupChainPredicate(
+ maxDeltaInGroup,
+ partitionConverter,
+ partitionProjector.groupFieldCount(),
+ builder::equal,
+ builder::lessThan);
+ PartitionPredicate snapshotAnchorPredicate =
+ PartitionPredicate.fromPredicate(
+ tableSchema.logicalPartitionType(), snapshotSearchPred);
+
+ // List snapshot partitions for this group, sorted by chain dimension.
+ List<BinaryRow> snapshotPartitionsInGroup =
+ newChainPartitionListingScan(true, snapshotAnchorPredicate)
+ .listPartitions().stream()
+ .sorted(
+ (a, b) ->
+ chainPartitionComparator.compare(
+ partitionProjector
+ .chainPartitionForCompare(a),
+ partitionProjector
+ .chainPartitionForCompare(b)))
+ .collect(Collectors.toList());
+
+ // Find delta → snapshot mapping (for each delta partition, find the nearest
+ // earlier
+ // snapshot)
+ // Note: uses chainPartitionComparator, comparing only the chain dimension
+ Map<BinaryRow, BinaryRow> partitionMapping =
+ ChainTableUtils.findFirstLatestPartitionsWithProjector(
+ deltaPartitionsInGroup,
+ snapshotPartitionsInGroup,
+ chainPartitionComparator,
+ partitionProjector);
+
+ for (Map.Entry<BinaryRow, BinaryRow> partitionPairs :
+ partitionMapping.entrySet()) {
+ BinaryRow deltaPartition = partitionPairs.getKey();
+ BinaryRow snapshotPartition = partitionPairs.getValue();
+
+ DataTableScan snapshotScan = newFilteredScan(true);
+ DataTableScan deltaScan = newFilteredScan(false);
+
+ if (snapshotPartition == null) {
+ List<Predicate> predicates = new ArrayList<>();
+ predicates.add(
+ ChainTableUtils.createGroupChainPredicate(
+ deltaPartition,
+ partitionConverter,
+ partitionProjector.groupFieldCount(),
+ builder::equal,
+ builder::lessThan));
+ predicates.add(
+ ChainTableUtils.createLinearPredicate(
+ deltaPartition, partitionConverter, builder::equal));
+ deltaScan.withPartitionFilter(PredicateBuilder.or(predicates));
+ } else {
+ List<BinaryRow> selectedDeltaPartitions =
+ ChainTableUtils.getDeltaPartitionsWithProjector(
+ snapshotPartition,
+ deltaPartition,
+ options,
+ chainPartitionComparator,
+ partitionProjector);
+ deltaScan.withPartitionFilter(selectedDeltaPartitions);
+ }
+
+ List<Split> subSplits = deltaScan.plan().splits();
+ Set<String> snapshotFileNames = new HashSet<>();
+ if (partitionPairs.getValue() != null) {
+ snapshotScan.withPartitionFilter(
+ Collections.singletonList(partitionPairs.getValue()));
+ List<Split> mainSubSplits = snapshotScan.plan().splits();
+ snapshotFileNames =
+ mainSubSplits.stream()
+ .flatMap(
+ s ->
+ ((DataSplit) s)
+ .dataFiles().stream()
+ .map(
+ DataFileMeta
+ ::fileName))
+ .collect(Collectors.toSet());
+ subSplits.addAll(mainSubSplits);
+ }
+ Map<Integer, List<DataSplit>> bucketSplits = new LinkedHashMap<>();
+ Integer bucketInAll = null;
+ for (Split split : subSplits) {
+ DataSplit dataSplit = (DataSplit) split;
+ Integer totalBuckets = dataSplit.totalBuckets();
+ checkNotNull(totalBuckets);
+ if (bucketInAll == null) {
+ bucketInAll = totalBuckets;
+ } else {
+ checkArgument(
+ totalBuckets == bucketInAll,
+ "Inconsistent bucket num " + dataSplit.bucket());
+ }
+
+ bucketSplits
+ .computeIfAbsent(dataSplit.bucket(), k -> new ArrayList<>())
+ .add(dataSplit);
+ }
+ for (Map.Entry<Integer, List<DataSplit>> entry : bucketSplits.entrySet()) {
+ HashMap<String, String> fileBucketPathMapping = new HashMap<>();
+ HashMap<String, String> fileBranchMapping = new HashMap<>();
+ List<DataSplit> splitList = entry.getValue();
+ for (DataSplit dataSplit : splitList) {
+ for (DataFileMeta file : dataSplit.dataFiles()) {
+ fileBucketPathMapping.put(
+ file.fileName(), dataSplit.bucketPath());
+ String branch =
+ snapshotFileNames.contains(file.fileName())
+ ? options.scanFallbackSnapshotBranch()
+ : options.scanFallbackDeltaBranch();
+ fileBranchMapping.put(file.fileName(), branch);
+ }
}
+ ChainSplit split =
+ new ChainSplit(
+ partitionPairs.getKey(),
+ entry.getValue().stream()
+ .flatMap(
+ dataSplit ->
+ dataSplit.dataFiles().stream())
+ .collect(Collectors.toList()),
+ fileBranchMapping,
+ fileBucketPathMapping);
+ splits.add(split);
}
- ChainSplit split =
- new ChainSplit(
- partitionParis.getKey(),
- entry.getValue().stream()
- .flatMap(
- datsSplit -> datsSplit.dataFiles().stream())
- .collect(Collectors.toList()),
- fileBranchMapping,
- fileBucketPathMapping);
- splits.add(split);
}
}
}
diff --git a/paimon-core/src/main/java/org/apache/paimon/utils/ChainPartitionProjector.java b/paimon-core/src/main/java/org/apache/paimon/utils/ChainPartitionProjector.java
new file mode 100644
index 0000000000..58fd7eb684
--- /dev/null
+++ b/paimon-core/src/main/java/org/apache/paimon/utils/ChainPartitionProjector.java
@@ -0,0 +1,160 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.utils;
+
+import org.apache.paimon.codegen.CodeGenUtils;
+import org.apache.paimon.codegen.Projection;
+import org.apache.paimon.data.BinaryRow;
+import org.apache.paimon.data.GenericRow;
+import org.apache.paimon.data.InternalRow;
+import org.apache.paimon.data.serializer.InternalRowSerializer;
+import org.apache.paimon.types.RowType;
+
+import java.util.stream.IntStream;
+
+/**
+ * Utility to project full partition rows into group (non-chain) and chain parts, and to combine
+ * them back into full partition rows.
+ *
+ * <p>Given partition keys [region, date] with chain partition keys = [date]:
+ *
+ * <ul>
+ * <li>groupPartition = projection of [region]
+ * <li>chainPartition = projection of [date]
+ * </ul>
+ */
+public class ChainPartitionProjector {
+ private final Projection groupProjection;
+ private final Projection chainProjection;
+ private final InternalRowSerializer fullSerializer;
+ private final RowType fullPartitionType;
+ private final int groupFieldCount;
+ private final int chainFieldCount;
+
+ // Cached projected RowTypes to avoid repeated IntStream allocation on each call.
+ private final RowType groupType;
+ private final RowType chainType;
+
+ // Pre-built FieldGetter arrays for combinePartition
+ private final InternalRow.FieldGetter[] groupFieldGetters;
+ private final InternalRow.FieldGetter[] chainFieldGetters;
+
+ /**
+ * @param fullPartitionType the full partition RowType
+ * @param chainFieldCount number of chain partition fields (counted from the end)
+ */
+ public ChainPartitionProjector(RowType fullPartitionType, int chainFieldCount) {
+ this.fullPartitionType = fullPartitionType;
+ int totalFields = fullPartitionType.getFieldCount();
+ this.groupFieldCount = totalFields - chainFieldCount;
+ this.chainFieldCount = chainFieldCount;
+
+ // group: first groupFieldCount fields
+ int[] groupIndices = new int[groupFieldCount];
+ for (int i = 0; i < groupFieldCount; i++) {
+ groupIndices[i] = i;
+ }
+ this.groupProjection = CodeGenUtils.newProjection(fullPartitionType, groupIndices);
+
+ // chain: last chainFieldCount fields
+ int[] chainIndices = new int[chainFieldCount];
+ for (int i = 0; i < chainFieldCount; i++) {
+ chainIndices[i] = groupFieldCount + i;
+ }
+ this.chainProjection = CodeGenUtils.newProjection(fullPartitionType, chainIndices);
+
+ this.fullSerializer = new InternalRowSerializer(fullPartitionType);
+
+ // Cache projected RowTypes once; used by groupPartitionType()/chainPartitionType()
+ // and the FieldGetter initialization below.
+ this.groupType = fullPartitionType.project(IntStream.range(0, groupFieldCount).toArray());
+ this.chainType =
+ fullPartitionType.project(
+ IntStream.range(groupFieldCount, groupFieldCount + chainFieldCount)
+ .toArray());
+
+ // Pre-build FieldGetters for type-safe field access from group/chain BinaryRow
+ this.groupFieldGetters = new InternalRow.FieldGetter[groupFieldCount];
+ for (int i = 0; i < groupFieldCount; i++) {
+ groupFieldGetters[i] = InternalRow.createFieldGetter(groupType.getTypeAt(i), i);
+ }
+
+ this.chainFieldGetters = new InternalRow.FieldGetter[chainFieldCount];
+ for (int i = 0; i < chainFieldCount; i++) {
+ chainFieldGetters[i] = InternalRow.createFieldGetter(chainType.getTypeAt(i), i);
+ }
+ }
+
+ /** Extract group partitions from full partition. */
+ public BinaryRow extractGroupPartition(BinaryRow fullPartition) {
+ return groupProjection.apply(fullPartition).copy();
+ }
+
+ /** Extract chain partitions from full partition. */
+ public BinaryRow extractChainPartition(BinaryRow fullPartition) {
+ return chainProjection.apply(fullPartition).copy();
+ }
+
+ /**
+ * Combines the group part and chain part back into a full partition row. Uses
+ * InternalRow.createFieldGetter for type-safe field value extraction.
+ */
+ public BinaryRow combinePartition(BinaryRow groupPart, BinaryRow chainPart) {
+ GenericRow combined = new GenericRow(fullPartitionType.getFieldCount());
+ for (int i = 0; i < groupFieldCount; i++) {
+ combined.setField(i, groupFieldGetters[i].getFieldOrNull(groupPart));
+ }
+ for (int i = 0; i < chainFieldCount; i++) {
+ combined.setField(groupFieldCount + i, chainFieldGetters[i].getFieldOrNull(chainPart));
+ }
+ return fullSerializer.toBinaryRow(combined).copy();
+ }
+
+ public boolean hasGroupPartition() {
+ return groupFieldCount > 0;
+ }
+
+ public int groupFieldCount() {
+ return groupFieldCount;
+ }
+
+ public int chainFieldCount() {
+ return chainFieldCount;
+ }
+
+ public RowType groupPartitionType() {
+ return groupType;
+ }
+
+ public RowType chainPartitionType() {
+ return chainType;
+ }
+
+ /**
+ * Returns a projection of the chain fields from the given full-partition row <em>without
+ * copying</em> the underlying storage.
+ *
+ * <p>The returned {@link InternalRow} shares the backing bytes of {@code fullPartition} and
+ * must not be retained across mutations of that row. Use this method only for transient,
+ * read-only comparisons to avoid unnecessary {@link BinaryRow} allocations.
+ */
+ public InternalRow chainPartitionForCompare(BinaryRow fullPartition) {
+ return chainProjection.apply(fullPartition);
+ }
+}
diff --git a/paimon-core/src/main/java/org/apache/paimon/utils/ChainTableUtils.java b/paimon-core/src/main/java/org/apache/paimon/utils/ChainTableUtils.java
index f6c573440f..6ccbbbc8fc 100644
--- a/paimon-core/src/main/java/org/apache/paimon/utils/ChainTableUtils.java
+++ b/paimon-core/src/main/java/org/apache/paimon/utils/ChainTableUtils.java
@@ -225,4 +225,136 @@ public class ChainTableUtils {
}
return table;
}
+
+ public static List<String> chainPartitionKeys(
+ CoreOptions options, List<String> allPartitionKeys) {
+ List<String> chainPartitionKeys = options.chainTableChainPartitionKeys();
+ if (chainPartitionKeys == null) {
+ return allPartitionKeys;
+ }
+ return chainPartitionKeys;
+ }
+
+ /**
+ * Within the same group, find the nearest smaller partition in the target list for each source
+ * partition, comparing only on the chain dimension.
+ *
+ * @param sortedSourcePartitions full partitions (sorted by chain dimension)
+ * @param sortedTargetPartitions full partitions (sorted by chain dimension, same group)
+ * @param chainComparator compares chain dimension only
+ * @param projector partition projector
+ * @return source → target mapping, target may be null
+ */
+ public static Map<BinaryRow, BinaryRow> findFirstLatestPartitionsWithProjector(
+ List<BinaryRow> sortedSourcePartitions,
+ List<BinaryRow> sortedTargetPartitions,
+ RecordComparator chainComparator,
+ ChainPartitionProjector projector) {
+
+ Map<BinaryRow, BinaryRow> partitionMapping = new HashMap<>();
+ int targetIndex = 0;
+
+ for (BinaryRow sourceRow : sortedSourcePartitions) {
+ BinaryRow sourceChain = projector.extractChainPartition(sourceRow);
+ while (targetIndex < sortedTargetPartitions.size()) {
+ BinaryRow targetChain =
+ projector.extractChainPartition(sortedTargetPartitions.get(targetIndex));
+ if (chainComparator.compare(targetChain, sourceChain) < 0) {
+ targetIndex++;
+ } else {
+ break;
+ }
+ }
+ BinaryRow firstSmaller =
+ (targetIndex > 0) ? sortedTargetPartitions.get(targetIndex - 1) : null;
+ partitionMapping.put(sourceRow, firstSmaller);
+ }
+ return partitionMapping;
+ }
+
+ /**
+ * Generates the list of delta partitions in the range (beginPartition, endPartition].
+ * Enumerates time range only on the chain dimension; the group dimension stays unchanged.
+ */
+ public static List<BinaryRow> getDeltaPartitionsWithProjector(
+ BinaryRow beginPartition,
+ BinaryRow endPartition,
+ CoreOptions options,
+ RecordComparator chainPartitionComparator,
+ ChainPartitionProjector projector) {
+
+ // Extract the chain parts from begin/end
+ BinaryRow beginChain = projector.extractChainPartition(beginPartition);
+ BinaryRow endChain = projector.extractChainPartition(endPartition);
+
+ // Build chain-dimension RowType, column names, PartitionComputer
+ RowType chainPartType = projector.chainPartitionType();
+ List<String> chainPartitionColumns = chainPartType.getFieldNames();
+ InternalRowPartitionComputer chainPartitionComputer =
+ new InternalRowPartitionComputer(
+ options.partitionDefaultName(),
+ chainPartType,
+ chainPartitionColumns.toArray(new String[0]),
+ options.legacyPartitionName());
+
+ // Reuse existing getDeltaPartitions to enumerate on chain dimension
+ List<BinaryRow> chainOnlyDeltas =
+ getDeltaPartitions(
+ beginChain,
+ endChain,
+ chainPartitionColumns,
+ chainPartType,
+ options,
+ chainPartitionComparator,
+ chainPartitionComputer);
+
+ // Combine each chain-only BinaryRow with the group part into a full partition
+ BinaryRow groupPart = projector.extractGroupPartition(beginPartition);
+ List<BinaryRow> fullDeltas = new ArrayList<>(chainOnlyDeltas.size());
+ for (BinaryRow chainDelta : chainOnlyDeltas) {
+ fullDeltas.add(projector.combinePartition(groupPart, chainDelta));
+ }
+ return fullDeltas;
+ }
+
+ /**
+ * Builds a compound predicate: group fields exact match AND chain fields triangular range.
+ *
+ * @param fullPartition full partition row
+ * @param converter converter for the full partition
+ * @param groupFieldCount number of group fields
+ * @param innerFunc equality function (field_index, value) → Predicate
+ * @param outerFunc range function (field_index, value) → Predicate
+ */
+ public static Predicate createGroupChainPredicate(
+ BinaryRow fullPartition,
+ RowDataToObjectArrayConverter converter,
+ int groupFieldCount,
+ BiFunction<Integer, Object, Predicate> innerFunc,
+ BiFunction<Integer, Object, Predicate> outerFunc) {
+
+ Object[] allValues = converter.convert(fullPartition);
+ List<Predicate> conditions = new ArrayList<>();
+
+ for (int i = 0; i < groupFieldCount; i++) {
+ conditions.add(innerFunc.apply(i, allValues[i]));
+ }
+
+ List<Predicate> chainFieldPredicates = new ArrayList<>();
+ int totalFields = converter.getArity();
+ for (int i = groupFieldCount; i < totalFields; i++) {
+ List<Predicate> andConditions = new ArrayList<>();
+ for (int j = groupFieldCount; j < i; j++) {
+ andConditions.add(innerFunc.apply(j, allValues[j]));
+ }
+ andConditions.add(outerFunc.apply(i, allValues[i]));
+ chainFieldPredicates.add(PredicateBuilder.and(andConditions));
+ }
+
+ if (!chainFieldPredicates.isEmpty()) {
+ conditions.add(PredicateBuilder.or(chainFieldPredicates));
+ }
+
+ return PredicateBuilder.and(conditions);
+ }
}
diff --git a/paimon-core/src/test/java/org/apache/paimon/utils/ChainPartitionProjectorTest.java b/paimon-core/src/test/java/org/apache/paimon/utils/ChainPartitionProjectorTest.java
new file mode 100644
index 0000000000..8f7cd36393
--- /dev/null
+++ b/paimon-core/src/test/java/org/apache/paimon/utils/ChainPartitionProjectorTest.java
@@ -0,0 +1,432 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.utils;
+
+import org.apache.paimon.data.BinaryRow;
+import org.apache.paimon.data.BinaryRowWriter;
+import org.apache.paimon.data.BinaryString;
+import org.apache.paimon.data.serializer.InternalRowSerializer;
+import org.apache.paimon.types.DataTypes;
+import org.apache.paimon.types.RowType;
+
+import org.junit.jupiter.api.Test;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/** Test class for {@link ChainPartitionProjector}. */
+public class ChainPartitionProjectorTest {
+
+ // ========================== Helper Methods ==========================
+
+ /** Build a BinaryRow with given string values for a STRING-only RowType. */
+ private static BinaryRow buildStringRow(String... values) {
+ BinaryRow row = new BinaryRow(values.length);
+ BinaryRowWriter writer = new BinaryRowWriter(row);
+ for (int i = 0; i < values.length; i++) {
+ if (values[i] == null) {
+ writer.setNullAt(i);
+ } else {
+ writer.writeString(i, BinaryString.fromString(values[i]));
+ }
+ }
+ writer.complete();
+ return row;
+ }
+
+ /** Build a BinaryRow with mixed types: string at pos 0, int at pos 1. */
+ private static BinaryRow buildStringIntRow(String strVal, int intVal) {
+ BinaryRow row = new BinaryRow(2);
+ BinaryRowWriter writer = new BinaryRowWriter(row);
+ writer.writeString(0, BinaryString.fromString(strVal));
+ writer.writeInt(1, intVal);
+ writer.complete();
+ return row;
+ }
+
+ /** Extract a string value from a BinaryRow at the given position. */
+ private static String getString(BinaryRow row, int pos) {
+ return row.getString(pos).toString();
+ }
+
+ // ========================== Test: Two STRING fields (region, date) ==========================
+
+ @Test
+ public void testExtractGroupAndChainWithTwoStringFields() {
+ // partition keys: (region, date), chain partition keys: (date)
+ RowType fullType =
+ RowType.builder()
+ .field("region", DataTypes.STRING().notNull())
+ .field("date", DataTypes.STRING().notNull())
+ .build();
+
+ ChainPartitionProjector projector = new ChainPartitionProjector(fullType, 1);
+
+ BinaryRow fullPartition = buildStringRow("US", "20250726");
+
+ // extract group
+ BinaryRow groupPart = projector.extractGroupPartition(fullPartition);
+ assertThat(groupPart.getFieldCount()).isEqualTo(1);
+ assertThat(getString(groupPart, 0)).isEqualTo("US");
+
+ // extract chain
+ BinaryRow chainPart = projector.extractChainPartition(fullPartition);
+ assertThat(chainPart.getFieldCount()).isEqualTo(1);
+ assertThat(getString(chainPart, 0)).isEqualTo("20250726");
+ }
+
+ @Test
+ public void testCombinePartitionWithTwoStringFields() {
+ RowType fullType =
+ RowType.builder()
+ .field("region", DataTypes.STRING().notNull())
+ .field("date", DataTypes.STRING().notNull())
+ .build();
+
+ ChainPartitionProjector projector = new ChainPartitionProjector(fullType, 1);
+
+ BinaryRow groupPart = buildStringRow("EU");
+ BinaryRow chainPart = buildStringRow("20250801");
+
+ BinaryRow combined = projector.combinePartition(groupPart, chainPart);
+ assertThat(combined.getFieldCount()).isEqualTo(2);
+ assertThat(getString(combined, 0)).isEqualTo("EU");
+ assertThat(getString(combined, 1)).isEqualTo("20250801");
+ }
+
+ @Test
+ public void testRoundTripExtractThenCombineWithTwoStringFields() {
+ RowType fullType =
+ RowType.builder()
+ .field("region", DataTypes.STRING().notNull())
+ .field("date", DataTypes.STRING().notNull())
+ .build();
+
+ ChainPartitionProjector projector = new ChainPartitionProjector(fullType, 1);
+
+ BinaryRow original = buildStringRow("CN", "20250720");
+
+ BinaryRow groupPart = projector.extractGroupPartition(original);
+ BinaryRow chainPart = projector.extractChainPartition(original);
+ BinaryRow reconstructed = projector.combinePartition(groupPart, chainPart);
+
+ assertThat(getString(reconstructed, 0)).isEqualTo("CN");
+ assertThat(getString(reconstructed, 1)).isEqualTo("20250720");
+
+ // verify binary equality via serialization
+ InternalRowSerializer serializer = new InternalRowSerializer(fullType);
+ assertThat(serializer.toBinaryRow(reconstructed).copy())
+ .isEqualTo(serializer.toBinaryRow(original).copy());
+ }
+
+ // ========================== Test: Three STRING fields ==========================
+
+ @Test
+ public void testThreeStringFieldsWithTwoChain() {
+ // partition keys: (country, city, date), chain partition keys: (city, date)
+ RowType fullType =
+ RowType.builder()
+ .field("country", DataTypes.STRING().notNull())
+ .field("city", DataTypes.STRING().notNull())
+ .field("date", DataTypes.STRING().notNull())
+ .build();
+
+ ChainPartitionProjector projector = new ChainPartitionProjector(fullType, 2);
+
+ assertThat(projector.groupFieldCount()).isEqualTo(1);
+ assertThat(projector.chainFieldCount()).isEqualTo(2);
+ assertThat(projector.hasGroupPartition()).isTrue();
+
+ BinaryRow fullPartition = buildStringRow("China", "Beijing", "20250801");
+
+ // group = (country)
+ BinaryRow groupPart = projector.extractGroupPartition(fullPartition);
+ assertThat(groupPart.getFieldCount()).isEqualTo(1);
+ assertThat(getString(groupPart, 0)).isEqualTo("China");
+
+ // chain = (city, date)
+ BinaryRow chainPart = projector.extractChainPartition(fullPartition);
+ assertThat(chainPart.getFieldCount()).isEqualTo(2);
+ assertThat(getString(chainPart, 0)).isEqualTo("Beijing");
+ assertThat(getString(chainPart, 1)).isEqualTo("20250801");
+
+ // round-trip
+ BinaryRow combined = projector.combinePartition(groupPart, chainPart);
+ assertThat(getString(combined, 0)).isEqualTo("China");
+ assertThat(getString(combined, 1)).isEqualTo("Beijing");
+ assertThat(getString(combined, 2)).isEqualTo("20250801");
+ }
+
+ @Test
+ public void testThreeStringFieldsWithOneChain() {
+ // partition keys: (country, city, date), chain partition keys: (date)
+ RowType fullType =
+ RowType.builder()
+ .field("country", DataTypes.STRING().notNull())
+ .field("city", DataTypes.STRING().notNull())
+ .field("date", DataTypes.STRING().notNull())
+ .build();
+
+ ChainPartitionProjector projector = new ChainPartitionProjector(fullType, 1);
+
+ assertThat(projector.groupFieldCount()).isEqualTo(2);
+ assertThat(projector.chainFieldCount()).isEqualTo(1);
+
+ BinaryRow fullPartition = buildStringRow("US", "NYC", "20250901");
+
+ // group = (country, city)
+ BinaryRow groupPart = projector.extractGroupPartition(fullPartition);
+ assertThat(groupPart.getFieldCount()).isEqualTo(2);
+ assertThat(getString(groupPart, 0)).isEqualTo("US");
+ assertThat(getString(groupPart, 1)).isEqualTo("NYC");
+
+ // chain = (date)
+ BinaryRow chainPart = projector.extractChainPartition(fullPartition);
+ assertThat(chainPart.getFieldCount()).isEqualTo(1);
+ assertThat(getString(chainPart, 0)).isEqualTo("20250901");
+ }
+
+ // ========================== Test: All partitions are chain (no group)
+ // ==========================
+
+ @Test
+ public void testAllChainWithNoGroup() {
+ // partition keys: (date), chain partition keys: (date)
+ RowType fullType = RowType.builder().field("date", DataTypes.STRING().notNull()).build();
+
+ ChainPartitionProjector projector = new ChainPartitionProjector(fullType, 1);
+
+ assertThat(projector.groupFieldCount()).isEqualTo(0);
+ assertThat(projector.chainFieldCount()).isEqualTo(1);
+ assertThat(projector.hasGroupPartition()).isFalse();
+
+ BinaryRow fullPartition = buildStringRow("20250726");
+
+ // group is empty
+ BinaryRow groupPart = projector.extractGroupPartition(fullPartition);
+ assertThat(groupPart.getFieldCount()).isEqualTo(0);
+
+ // chain is the whole partition
+ BinaryRow chainPart = projector.extractChainPartition(fullPartition);
+ assertThat(chainPart.getFieldCount()).isEqualTo(1);
+ assertThat(getString(chainPart, 0)).isEqualTo("20250726");
+ }
+
+ @Test
+ public void testAllChainWithTwoFields() {
+ // partition keys: (dt, hour), chain partition keys: (dt, hour)
+ RowType fullType =
+ RowType.builder()
+ .field("dt", DataTypes.STRING().notNull())
+ .field("hour", DataTypes.STRING().notNull())
+ .build();
+
+ ChainPartitionProjector projector = new ChainPartitionProjector(fullType, 2);
+
+ assertThat(projector.hasGroupPartition()).isFalse();
+ assertThat(projector.groupFieldCount()).isEqualTo(0);
+ assertThat(projector.chainFieldCount()).isEqualTo(2);
+
+ BinaryRow fullPartition = buildStringRow("20250810", "14");
+
+ BinaryRow chainPart = projector.extractChainPartition(fullPartition);
+ assertThat(getString(chainPart, 0)).isEqualTo("20250810");
+ assertThat(getString(chainPart, 1)).isEqualTo("14");
+ }
+
+ // ========================== Test: Mixed types (STRING + INT) ==========================
+
+ @Test
+ public void testMixedTypesWithStringAndInt() {
+ // partition keys: (region STRING, bucket_id INT), chain partition keys: (bucket_id)
+ RowType fullType =
+ RowType.builder()
+ .field("region", DataTypes.STRING().notNull())
+ .field("bucket_id", DataTypes.INT().notNull())
+ .build();
+
+ ChainPartitionProjector projector = new ChainPartitionProjector(fullType, 1);
+
+ BinaryRow fullPartition = buildStringIntRow("US", 42);
+
+ // group = (region)
+ BinaryRow groupPart = projector.extractGroupPartition(fullPartition);
+ assertThat(groupPart.getFieldCount()).isEqualTo(1);
+ assertThat(getString(groupPart, 0)).isEqualTo("US");
+
+ // chain = (bucket_id)
+ BinaryRow chainPart = projector.extractChainPartition(fullPartition);
+ assertThat(chainPart.getFieldCount()).isEqualTo(1);
+ assertThat(chainPart.getInt(0)).isEqualTo(42);
+
+ // combine
+ BinaryRow combined = projector.combinePartition(groupPart, chainPart);
+ assertThat(getString(combined, 0)).isEqualTo("US");
+ assertThat(combined.getInt(1)).isEqualTo(42);
+ }
+
+ // ========================== Test: RowType accessors ==========================
+
+ @Test
+ public void testGroupPartitionType() {
+ RowType fullType =
+ RowType.builder()
+ .field("region", DataTypes.STRING().notNull())
+ .field("city", DataTypes.STRING().notNull())
+ .field("date", DataTypes.STRING().notNull())
+ .build();
+
+ ChainPartitionProjector projector = new ChainPartitionProjector(fullType, 1);
+
+ RowType groupType = projector.groupPartitionType();
+ assertThat(groupType.getFieldCount()).isEqualTo(2);
+ assertThat(groupType.getFieldNames().get(0)).isEqualTo("region");
+ assertThat(groupType.getFieldNames().get(1)).isEqualTo("city");
+
+ RowType chainType = projector.chainPartitionType();
+ assertThat(chainType.getFieldCount()).isEqualTo(1);
+ assertThat(chainType.getFieldNames().get(0)).isEqualTo("date");
+ }
+
+ @Test
+ public void testGroupPartitionTypeWithNoGroup() {
+ RowType fullType = RowType.builder().field("date", DataTypes.STRING().notNull()).build();
+
+ ChainPartitionProjector projector = new ChainPartitionProjector(fullType, 1);
+
+ RowType groupType = projector.groupPartitionType();
+ assertThat(groupType.getFieldCount()).isEqualTo(0);
+
+ RowType chainType = projector.chainPartitionType();
+ assertThat(chainType.getFieldCount()).isEqualTo(1);
+ assertThat(chainType.getFieldNames().get(0)).isEqualTo("date");
+ }
+
+ // ========================== Test: Multiple partitions grouping ==========================
+
+ @Test
+ public void testGroupPartitionConsistency() {
+ // Verify that partitions with same group prefix produce equal group projections
+ RowType fullType =
+ RowType.builder()
+ .field("region", DataTypes.STRING().notNull())
+ .field("date", DataTypes.STRING().notNull())
+ .build();
+
+ ChainPartitionProjector projector = new ChainPartitionProjector(fullType, 1);
+
+ BinaryRow partition1 = buildStringRow("US", "20250721");
+ BinaryRow partition2 = buildStringRow("US", "20250722");
+ BinaryRow partition3 = buildStringRow("EU", "20250721");
+
+ BinaryRow group1 = projector.extractGroupPartition(partition1).copy();
+ BinaryRow group2 = projector.extractGroupPartition(partition2).copy();
+ BinaryRow group3 = projector.extractGroupPartition(partition3).copy();
+
+ // Same region => equal group partitions
+ assertThat(group1).isEqualTo(group2);
+
+ // Different region => different group partitions
+ assertThat(group1).isNotEqualTo(group3);
+ }
+
+ @Test
+ public void testChainPartitionOrdering() {
+ // Verify that chain projections can be compared correctly
+ RowType fullType =
+ RowType.builder()
+ .field("region", DataTypes.STRING().notNull())
+ .field("date", DataTypes.STRING().notNull())
+ .build();
+
+ ChainPartitionProjector projector = new ChainPartitionProjector(fullType, 1);
+
+ BinaryRow partition1 = buildStringRow("US", "20250720");
+ BinaryRow partition2 = buildStringRow("US", "20250726");
+ BinaryRow partition3 = buildStringRow("EU", "20250720");
+
+ BinaryRow chain1 = projector.extractChainPartition(partition1).copy();
+ BinaryRow chain2 = projector.extractChainPartition(partition2).copy();
+ BinaryRow chain3 = projector.extractChainPartition(partition3).copy();
+
+ // Same date, different region => equal chain partitions
+ assertThat(chain1).isEqualTo(chain3);
+
+ // Different date => different chain partitions
+ assertThat(chain1).isNotEqualTo(chain2);
+
+ // Verify string ordering: "20250720" < "20250726"
+ assertThat(chain1.getString(0).compareTo(chain2.getString(0))).isLessThan(0);
+ }
+
+ // ========================== Test: Nullable fields ==========================
+
+ @Test
+ public void testNullableGroupField() {
+ RowType fullType =
+ RowType.builder()
+ .field("region", DataTypes.STRING())
+ .field("date", DataTypes.STRING().notNull())
+ .build();
+
+ ChainPartitionProjector projector = new ChainPartitionProjector(fullType, 1);
+
+ // build a row with null region
+ BinaryRow row = new BinaryRow(2);
+ BinaryRowWriter writer = new BinaryRowWriter(row);
+ writer.setNullAt(0);
+ writer.writeString(1, BinaryString.fromString("20250801"));
+ writer.complete();
+
+ BinaryRow groupPart = projector.extractGroupPartition(row);
+ assertThat(groupPart.isNullAt(0)).isTrue();
+
+ BinaryRow chainPart = projector.extractChainPartition(row);
+ assertThat(getString(chainPart, 0)).isEqualTo("20250801");
+
+ // combine should preserve null
+ BinaryRow combined = projector.combinePartition(groupPart, chainPart);
+ assertThat(combined.isNullAt(0)).isTrue();
+ assertThat(getString(combined, 1)).isEqualTo("20250801");
+ }
+
+ // ========================== Test: Combine with different group/chain sources
+
+ @Test
+ public void testCombineFromDifferentSources() {
+ // Simulate: take group from one partition, chain from another
+ RowType fullType =
+ RowType.builder()
+ .field("region", DataTypes.STRING().notNull())
+ .field("date", DataTypes.STRING().notNull())
+ .build();
+
+ ChainPartitionProjector projector = new ChainPartitionProjector(fullType, 1);
+
+ BinaryRow partitionA = buildStringRow("US", "20250720");
+ BinaryRow partitionB = buildStringRow("EU", "20250801");
+
+ // Take group from A (US), chain from B (20250801)
+ BinaryRow groupFromA = projector.extractGroupPartition(partitionA);
+ BinaryRow chainFromB = projector.extractChainPartition(partitionB);
+
+ BinaryRow mixed = projector.combinePartition(groupFromA, chainFromB);
+ assertThat(getString(mixed, 0)).isEqualTo("US");
+ assertThat(getString(mixed, 1)).isEqualTo("20250801");
+ }
+}
diff --git a/paimon-core/src/test/java/org/apache/paimon/utils/ChainTableUtilsTest.java b/paimon-core/src/test/java/org/apache/paimon/utils/ChainTableUtilsTest.java
index 99ba22c451..051323e180 100644
--- a/paimon-core/src/test/java/org/apache/paimon/utils/ChainTableUtilsTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/utils/ChainTableUtilsTest.java
@@ -18,16 +18,19 @@
package org.apache.paimon.utils;
+import org.apache.paimon.CoreOptions;
import org.apache.paimon.codegen.RecordComparator;
import org.apache.paimon.data.BinaryRow;
import org.apache.paimon.data.BinaryRowWriter;
import org.apache.paimon.data.BinaryString;
import org.apache.paimon.data.InternalRow;
+import org.apache.paimon.options.Options;
import org.apache.paimon.predicate.Predicate;
import org.apache.paimon.predicate.PredicateBuilder;
import org.apache.paimon.types.DataTypes;
import org.apache.paimon.types.RowType;
+import org.assertj.core.util.Lists;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
@@ -40,6 +43,7 @@ import java.util.Map;
import java.util.stream.Collectors;
import java.util.stream.Stream;
+import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
/** Test class for {@link org.apache.paimon.utils.ChainTableUtils}. */
@@ -56,6 +60,21 @@ public class ChainTableUtilsTest {
return builder.toString();
}
+ /** Extract a string value from a BinaryRow at the given position. */
+ private static String getString(BinaryRow row, int pos) {
+ return row.getString(pos).toString();
+ }
+
+ private static BinaryRow row(List<String> values) {
+ BinaryRow row = new BinaryRow(values.size());
+ BinaryRowWriter writer = new BinaryRowWriter(row);
+ for (int i = 0; i < values.size(); i++) {
+ writer.writeString(i, BinaryString.fromString(values.get(i)));
+ }
+ writer.complete();
+ return row;
+ }
+
@Test
public void testFindFirstLatestPartitions() {
String[] snapshotPartitions = {"20251105", "20251101"};
@@ -189,4 +208,517 @@ public class ChainTableUtilsTest {
},
partitionValues);
}
+
+ // ========================== Tests for findFirstLatestPartitionsWithProjector
+ // ==========================
+
+ @Test
+ public void testFindFirstLatestPartitionsWithProjectorWithBasicGrouping() {
+ // partition keys: (region, date), chain keys: (date)
+ // Snapshot: (US,20250720), (US,20250725)
+ // Delta: (US,20250722), (US,20250727)
+ // Expected: 20250722 → 20250720, 20250727 → 20250725
+ RowType fullType =
+ RowType.builder()
+ .field("region", DataTypes.STRING().notNull())
+ .field("date", DataTypes.STRING().notNull())
+ .build();
+
+ ChainPartitionProjector projector = new ChainPartitionProjector(fullType, 1);
+ RecordComparator chainComparator = (a, b) -> a.getString(0).compareTo(b.getString(0));
+
+ // Snapshot partitions, pre-sorted by chain field (date)
+ BinaryRow snap1 = row(Lists.newArrayList("US", "20250720"));
+ BinaryRow snap2 = row(Lists.newArrayList("US", "20250725"));
+ List<BinaryRow> snapshotPartitions = Arrays.asList(snap1, snap2);
+
+ // Delta partitions, pre-sorted by chain field (date)
+ BinaryRow delta1 = row(Lists.newArrayList("US", "20250722"));
+ BinaryRow delta2 = row(Lists.newArrayList("US", "20250727"));
+ List<BinaryRow> deltaPartitions = Arrays.asList(delta1, delta2);
+
+ Map<BinaryRow, BinaryRow> mapping =
+ ChainTableUtils.findFirstLatestPartitionsWithProjector(
+ deltaPartitions, snapshotPartitions, chainComparator, projector);
+
+ // delta (US,20250722) → snapshot (US,20250720)
+ BinaryRow matched1 = mapping.get(delta1);
+ assertThat(matched1).isNotNull();
+ assertThat(getString(matched1, 0)).isEqualTo("US");
+ assertThat(getString(matched1, 1)).isEqualTo("20250720");
+
+ // delta (US,20250727) → snapshot (US,20250725)
+ BinaryRow matched2 = mapping.get(delta2);
+ assertThat(matched2).isNotNull();
+ assertThat(getString(matched2, 0)).isEqualTo("US");
+ assertThat(getString(matched2, 1)).isEqualTo("20250725");
+ }
+
+ @Test
+ public void testFindFirstLatestPartitionsWithProjectorWithNoMatchingSnapshot() {
+ // Delta partition is earlier than all snapshots → null mapping
+ RowType fullType =
+ RowType.builder()
+ .field("region", DataTypes.STRING().notNull())
+ .field("date", DataTypes.STRING().notNull())
+ .build();
+
+ ChainPartitionProjector projector = new ChainPartitionProjector(fullType, 1);
+ RecordComparator chainComparator = (a, b) -> a.getString(0).compareTo(b.getString(0));
+
+ List<BinaryRow> snapshotPartitions =
+ Arrays.asList(row(Lists.newArrayList("US", "20250725")));
+ BinaryRow delta1 = row(Lists.newArrayList("US", "20250720"));
+ List<BinaryRow> deltaPartitions = Arrays.asList(delta1);
+
+ Map<BinaryRow, BinaryRow> mapping =
+ ChainTableUtils.findFirstLatestPartitionsWithProjector(
+ deltaPartitions, snapshotPartitions, chainComparator, projector);
+
+ // delta 20250720 < snapshot 20250725, no earlier snapshot → null
+ assertThat(mapping.get(delta1)).isNull();
+ }
+
+ @Test
+ public void testFindFirstLatestPartitionsWithProjectorWithEmptySnapshot() {
+ // No snapshot partitions at all → all mappings are null
+ RowType fullType =
+ RowType.builder()
+ .field("region", DataTypes.STRING().notNull())
+ .field("date", DataTypes.STRING().notNull())
+ .build();
+
+ ChainPartitionProjector projector = new ChainPartitionProjector(fullType, 1);
+ RecordComparator chainComparator = (a, b) -> a.getString(0).compareTo(b.getString(0));
+
+ BinaryRow delta1 = row(Lists.newArrayList("US", "20250720"));
+ BinaryRow delta2 = row(Lists.newArrayList("US", "20250722"));
+ List<BinaryRow> deltaPartitions = Arrays.asList(delta1, delta2);
+
+ Map<BinaryRow, BinaryRow> mapping =
+ ChainTableUtils.findFirstLatestPartitionsWithProjector(
+ deltaPartitions, new ArrayList<>(), chainComparator, projector);
+
+ for (BinaryRow delta : deltaPartitions) {
+ assertThat(mapping.get(delta)).isNull();
+ }
+ }
+
+ @Test
+ public void testFindFirstLatestPartitionsWithProjectorWithMultipleChainFields() {
+ // partition keys: (region, dt, hour), chain keys: (dt, hour)
+ RowType fullType =
+ RowType.builder()
+ .field("region", DataTypes.STRING().notNull())
+ .field("dt", DataTypes.STRING().notNull())
+ .field("hour", DataTypes.STRING().notNull())
+ .build();
+
+ ChainPartitionProjector projector = new ChainPartitionProjector(fullType, 2);
+
+ // Compare chain partition (dt, hour) lexicographically
+ RecordComparator chainComparator =
+ (a, b) -> {
+ int cmp = a.getString(0).compareTo(b.getString(0));
+ if (cmp != 0) {
+ return cmp;
+ }
+ return a.getString(1).compareTo(b.getString(1));
+ };
+
+ // snapshot: (US, 20250720, 12)
+ BinaryRow snap1 = row(Lists.newArrayList("US", "20250720", "12"));
+ List<BinaryRow> snapshotPartitions = Arrays.asList(snap1);
+
+ // delta: (US, 20250720, 10), (US, 20250720, 14) — pre-sorted by chain (dt, hour)
+ BinaryRow delta1 = row(Lists.newArrayList("US", "20250720", "10"));
+ BinaryRow delta2 = row(Lists.newArrayList("US", "20250720", "14"));
+ List<BinaryRow> deltaPartitions = Arrays.asList(delta1, delta2);
+
+ Map<BinaryRow, BinaryRow> mapping =
+ ChainTableUtils.findFirstLatestPartitionsWithProjector(
+ deltaPartitions, snapshotPartitions, chainComparator, projector);
+
+ // delta (US,20250720,10): chain (20250720,10) < snapshot chain (20250720,12) → null
+ assertThat(mapping.get(delta1)).isNull();
+
+ // delta (US,20250720,14): chain (20250720,14) > snapshot chain (20250720,12) → match
+ BinaryRow matched = mapping.get(delta2);
+ assertThat(matched).isNotNull();
+ assertThat(getString(matched, 1)).isEqualTo("20250720");
+ assertThat(getString(matched, 2)).isEqualTo("12");
+ }
+
+ // ========================== Tests for getDeltaPartitionsWithProjector
+ // ==========================
+
+ @Test
+ public void testGetDeltaPartitionsWithProjectorWithDailyPartition() {
+ // partition keys: (region, date), chain keys: (date)
+ // begin: (US, 20250720), end: (US, 20250724)
+ // Expected delta: (US,20250721), (US,20250722), (US,20250723), (US,20250724)
+ RowType fullType =
+ RowType.builder()
+ .field("region", DataTypes.STRING().notNull())
+ .field("date", DataTypes.STRING().notNull())
+ .build();
+
+ ChainPartitionProjector projector = new ChainPartitionProjector(fullType, 1);
+ RecordComparator chainComparator = (a, b) -> a.getString(0).compareTo(b.getString(0));
+
+ Options opts = new Options();
+ opts.set(CoreOptions.PARTITION_TIMESTAMP_PATTERN, "$date");
+ opts.set(CoreOptions.PARTITION_TIMESTAMP_FORMATTER, "yyyyMMdd");
+ CoreOptions options = new CoreOptions(opts);
+
+ BinaryRow begin = row(Lists.newArrayList("US", "20250720"));
+ BinaryRow end = row(Lists.newArrayList("US", "20250724"));
+
+ List<BinaryRow> deltas =
+ ChainTableUtils.getDeltaPartitionsWithProjector(
+ begin, end, options, chainComparator, projector);
+
+ // (20250720, 20250724]: 4 days
+ assertThat(deltas).hasSize(4);
+ assertThat(getString(deltas.get(0), 0)).isEqualTo("US");
+ assertThat(getString(deltas.get(0), 1)).isEqualTo("20250721");
+ assertThat(getString(deltas.get(1), 1)).isEqualTo("20250722");
+ assertThat(getString(deltas.get(2), 1)).isEqualTo("20250723");
+ assertThat(getString(deltas.get(3), 1)).isEqualTo("20250724");
+
+ // all should have group = US
+ for (BinaryRow delta : deltas) {
+ assertThat(getString(delta, 0)).isEqualTo("US");
+ }
+ }
+
+ @Test
+ public void testGetDeltaPartitionsWithProjectorWithSameBeginAndEnd() {
+ // begin == end → empty result (open-closed interval: (begin, end])
+ RowType fullType =
+ RowType.builder()
+ .field("region", DataTypes.STRING().notNull())
+ .field("date", DataTypes.STRING().notNull())
+ .build();
+
+ ChainPartitionProjector projector = new ChainPartitionProjector(fullType, 1);
+ RecordComparator chainComparator = (a, b) -> a.getString(0).compareTo(b.getString(0));
+
+ Options opts = new Options();
+ opts.set(CoreOptions.PARTITION_TIMESTAMP_PATTERN, "$date");
+ opts.set(CoreOptions.PARTITION_TIMESTAMP_FORMATTER, "yyyyMMdd");
+ CoreOptions options = new CoreOptions(opts);
+
+ BinaryRow partition = row(Lists.newArrayList("EU", "20250720"));
+
+ List<BinaryRow> deltas =
+ ChainTableUtils.getDeltaPartitionsWithProjector(
+ partition, partition, options, chainComparator, projector);
+
+ assertThat(deltas).isEmpty();
+ }
+
+ @Test
+ public void testGetDeltaPartitionsWithProjectorWithConsecutiveDays() {
+ // begin: (EU, 20250720), end: (EU, 20250721) → only (EU, 20250721)
+ RowType fullType =
+ RowType.builder()
+ .field("region", DataTypes.STRING().notNull())
+ .field("date", DataTypes.STRING().notNull())
+ .build();
+
+ ChainPartitionProjector projector = new ChainPartitionProjector(fullType, 1);
+ RecordComparator chainComparator = (a, b) -> a.getString(0).compareTo(b.getString(0));
+
+ Options opts = new Options();
+ opts.set(CoreOptions.PARTITION_TIMESTAMP_PATTERN, "$date");
+ opts.set(CoreOptions.PARTITION_TIMESTAMP_FORMATTER, "yyyyMMdd");
+ CoreOptions options = new CoreOptions(opts);
+
+ BinaryRow begin = row(Lists.newArrayList("EU", "20250720"));
+ BinaryRow end = row(Lists.newArrayList("EU", "20250721"));
+
+ List<BinaryRow> deltas =
+ ChainTableUtils.getDeltaPartitionsWithProjector(
+ begin, end, options, chainComparator, projector);
+
+ assertThat(deltas).hasSize(1);
+ assertThat(getString(deltas.get(0), 0)).isEqualTo("EU");
+ assertThat(getString(deltas.get(0), 1)).isEqualTo("20250721");
+ }
+
+ @Test
+ public void testGetDeltaPartitionsWithProjectorWithGroupPreserved() {
+ // Verify that group part from beginPartition is preserved for all results
+ RowType fullType =
+ RowType.builder()
+ .field("country", DataTypes.STRING().notNull())
+ .field("city", DataTypes.STRING().notNull())
+ .field("date", DataTypes.STRING().notNull())
+ .build();
+
+ // chain keys: (date), group keys: (country, city)
+ ChainPartitionProjector projector = new ChainPartitionProjector(fullType, 1);
+ RecordComparator chainComparator = (a, b) -> a.getString(0).compareTo(b.getString(0));
+
+ Options opts = new Options();
+ opts.set(CoreOptions.PARTITION_TIMESTAMP_PATTERN, "$date");
+ opts.set(CoreOptions.PARTITION_TIMESTAMP_FORMATTER, "yyyyMMdd");
+ CoreOptions options = new CoreOptions(opts);
+
+ BinaryRow begin = row(Lists.newArrayList("China", "Beijing", "20250720"));
+ BinaryRow end = row(Lists.newArrayList("China", "Beijing", "20250723"));
+
+ List<BinaryRow> deltas =
+ ChainTableUtils.getDeltaPartitionsWithProjector(
+ begin, end, options, chainComparator, projector);
+
+ assertThat(deltas).hasSize(3);
+ for (BinaryRow delta : deltas) {
+ assertThat(getString(delta, 0)).isEqualTo("China");
+ assertThat(getString(delta, 1)).isEqualTo("Beijing");
+ }
+ assertThat(getString(deltas.get(0), 2)).isEqualTo("20250721");
+ assertThat(getString(deltas.get(1), 2)).isEqualTo("20250722");
+ assertThat(getString(deltas.get(2), 2)).isEqualTo("20250723");
+ }
+
+ // ========================== Tests for createGroupChainPredicate ==========================
+
+ @Test
+ public void testCreateGroupChainPredicateWithOneGroupOneChainLessThan() {
+ // partition: (region, date), groupFieldCount=1
+ // Expected: region=equal AND date=lessThan
+ RowType rowType =
+ RowType.builder()
+ .field("region", DataTypes.STRING().notNull())
+ .field("date", DataTypes.STRING().notNull())
+ .build();
+
+ BinaryRow partition = row(Lists.newArrayList("US", "20250726"));
+ RowDataToObjectArrayConverter converter = new RowDataToObjectArrayConverter(rowType);
+ PredicateBuilder builder = new PredicateBuilder(rowType);
+
+ Predicate predicate =
+ ChainTableUtils.createGroupChainPredicate(
+ partition,
+ converter,
+ 1,
+ (Integer i, Object v) -> builder.equal(i, v),
+ (Integer i, Object v) -> builder.lessThan(i, v));
+
+ // Expected: region='US' AND date < '20250726'
+ List<Predicate> expected = new ArrayList<>();
+ expected.add(builder.equal(0, BinaryString.fromString("US")));
+ expected.add(builder.lessThan(1, BinaryString.fromString("20250726")));
+ Predicate expectedPredicate = PredicateBuilder.and(expected);
+
+ assertThat(predicate).isEqualTo(expectedPredicate);
+
+ // Verify: (US, 20250725) should match
+ assertThat(predicate.test(row(Lists.newArrayList("US", "20250725")))).isTrue();
+
+ // Verify: (US, 20250726) should NOT match (not less than)
+ assertThat(predicate.test(partition)).isFalse();
+
+ // Verify: (EU, 20250725) should NOT match (region mismatch)
+ assertThat(predicate.test(row(Lists.newArrayList("EU", "20250725")))).isFalse();
+ }
+
+ @Test
+ public void testCreateGroupChainPredicateWithCombinedWithLinearForLessOrEqual() {
+ // Correct pattern: OR(groupChainPredicate(lessThan), linearPredicate(equal))
+ // to achieve partition <= target with group exact match.
+ // This is the pattern used in planChainInGroup.
+ RowType rowType =
+ RowType.builder()
+ .field("region", DataTypes.STRING().notNull())
+ .field("date", DataTypes.STRING().notNull())
+ .build();
+
+ BinaryRow partition = row(Lists.newArrayList("EU", "20250801"));
+ RowDataToObjectArrayConverter converter = new RowDataToObjectArrayConverter(rowType);
+ PredicateBuilder builder = new PredicateBuilder(rowType);
+
+ List<Predicate> predicates = new ArrayList<>();
+ predicates.add(
+ ChainTableUtils.createGroupChainPredicate(
+ partition,
+ converter,
+ 1,
+ (Integer i, Object v) -> builder.equal(i, v),
+ (Integer i, Object v) -> builder.lessThan(i, v)));
+ predicates.add(
+ ChainTableUtils.createLinearPredicate(
+ partition, converter, (Integer i, Object v) -> builder.equal(i, v)));
+ Predicate combined = PredicateBuilder.or(predicates);
+
+ // Exact match → match (via linearPredicate)
+ assertThat(combined.test(partition)).isTrue();
+
+ // Earlier date, same region → match (via groupChainPredicate)
+ assertThat(combined.test(row(Lists.newArrayList("EU", "20250731")))).isTrue();
+
+ // Later date → no match
+ assertThat(combined.test(row(Lists.newArrayList("EU", "20250802")))).isFalse();
+
+ // Different region → no match
+ assertThat(combined.test(row(Lists.newArrayList("US", "20250801")))).isFalse();
+ }
+
+ @Test
+ public void testCreateGroupChainPredicateWithOneGroupTwoChainLessThan() {
+ // partition: (region, dt, hour), groupFieldCount=1
+ // Expected: region=equal AND triangular(dt, hour) with lessThan
+ // → region='US' AND (dt < '20250810' OR (dt = '20250810' AND hour < '23'))
+ RowType rowType =
+ RowType.builder()
+ .field("region", DataTypes.STRING().notNull())
+ .field("dt", DataTypes.STRING().notNull())
+ .field("hour", DataTypes.STRING().notNull())
+ .build();
+
+ BinaryRow partition = row(Lists.newArrayList("US", "20250810", "23"));
+ RowDataToObjectArrayConverter converter = new RowDataToObjectArrayConverter(rowType);
+ PredicateBuilder builder = new PredicateBuilder(rowType);
+
+ Predicate predicate =
+ ChainTableUtils.createGroupChainPredicate(
+ partition,
+ converter,
+ 1,
+ (Integer i, Object v) -> builder.equal(i, v),
+ (Integer i, Object v) -> builder.lessThan(i, v));
+
+ // (US, 20250809, 23) → dt < 20250810 → match
+ assertThat(predicate.test(row(Lists.newArrayList("US", "20250809", "23")))).isTrue();
+
+ // (US, 20250810, 22) → dt=20250810 AND hour<23 → match
+ assertThat(predicate.test(row(Lists.newArrayList("US", "20250810", "22")))).isTrue();
+
+ // (US, 20250810, 23) → not less than → no match
+ assertThat(predicate.test(partition)).isFalse();
+
+ // (EU, 20250809, 23) → region mismatch → no match
+ assertThat(predicate.test(row(Lists.newArrayList("EU", "20250809", "23")))).isFalse();
+ }
+
+ @Test
+ public void testCreateGroupChainPredicateWithOneGroupTwoChainLessOrEqual() {
+ // partition: (region, dt, hour), groupFieldCount=1
+ // Combine groupChainPredicate(lessThan) OR linearPredicate(equal)
+ // to achieve <= semantics with correct multi-chain-field handling.
+ RowType rowType =
+ RowType.builder()
+ .field("region", DataTypes.STRING().notNull())
+ .field("dt", DataTypes.STRING().notNull())
+ .field("hour", DataTypes.STRING().notNull())
+ .build();
+
+ BinaryRow partition = row(Lists.newArrayList("US", "20250810", "14"));
+ RowDataToObjectArrayConverter converter = new RowDataToObjectArrayConverter(rowType);
+ PredicateBuilder builder = new PredicateBuilder(rowType);
+
+ List<Predicate> predicates = new ArrayList<>();
+ predicates.add(
+ ChainTableUtils.createGroupChainPredicate(
+ partition,
+ converter,
+ 1,
+ (Integer i, Object v) -> builder.equal(i, v),
+ (Integer i, Object v) -> builder.lessThan(i, v)));
+ predicates.add(
+ ChainTableUtils.createLinearPredicate(
+ partition, converter, (Integer i, Object v) -> builder.equal(i, v)));
+ Predicate combined = PredicateBuilder.or(predicates);
+
+ // (US, 20250810, 14) → exact match → match (via linear)
+ assertThat(combined.test(partition)).isTrue();
+
+ // (US, 20250810, 13) → same dt, hour < 14 → match (via groupChain)
+ assertThat(combined.test(row(Lists.newArrayList("US", "20250810", "13")))).isTrue();
+
+ // (US, 20250809, 23) → dt < 20250810 → match (via groupChain)
+ assertThat(combined.test(row(Lists.newArrayList("US", "20250809", "23")))).isTrue();
+
+ // (US, 20250810, 15) → same dt, hour > 14, no exact match → no match
+ assertThat(combined.test(row(Lists.newArrayList("US", "20250810", "15")))).isFalse();
+
+ // (US, 20250811, 00) → dt > 20250810 → no match
+ assertThat(combined.test(row(Lists.newArrayList("US", "20250811", "00")))).isFalse();
+
+ // (EU, 20250810, 14) → region mismatch → no match
+ assertThat(combined.test(row(Lists.newArrayList("EU", "20250810", "14")))).isFalse();
+ }
+
+ @Test
+ public void testCreateGroupChainPredicateWithZeroGroup() {
+ // groupFieldCount=0 → same as createTriangularPredicate
+ RowType rowType =
+ RowType.builder()
+ .field("dt", DataTypes.STRING().notNull())
+ .field("hour", DataTypes.STRING().notNull())
+ .build();
+
+ BinaryRow partition = row(Lists.newArrayList("20250810", "23"));
+ RowDataToObjectArrayConverter converter = new RowDataToObjectArrayConverter(rowType);
+ PredicateBuilder builder = new PredicateBuilder(rowType);
+
+ Predicate groupChainPredicate =
+ ChainTableUtils.createGroupChainPredicate(
+ partition,
+ converter,
+ 0,
+ (Integer i, Object v) -> builder.equal(i, v),
+ (Integer i, Object v) -> builder.lessThan(i, v));
+
+ Predicate triangularPredicate =
+ ChainTableUtils.createTriangularPredicate(
+ partition,
+ converter,
+ (Integer i, Object v) -> builder.equal(i, v),
+ (Integer i, Object v) -> builder.lessThan(i, v));
+
+ // When groupFieldCount=0, createGroupChainPredicate should produce
+ // the same result as createTriangularPredicate
+ assertThat(groupChainPredicate).isEqualTo(triangularPredicate);
+ }
+
+ @Test
+ public void testCreateGroupChainPredicateWithTwoGroupOneChain() {
+ // partition: (country, city, date), groupFieldCount=2
+ // Expected: country=equal AND city=equal AND date=lessThan
+ RowType rowType =
+ RowType.builder()
+ .field("country", DataTypes.STRING().notNull())
+ .field("city", DataTypes.STRING().notNull())
+ .field("date", DataTypes.STRING().notNull())
+ .build();
+
+ BinaryRow partition = row(Lists.newArrayList("China", "Beijing", "20250801"));
+ RowDataToObjectArrayConverter converter = new RowDataToObjectArrayConverter(rowType);
+ PredicateBuilder builder = new PredicateBuilder(rowType);
+
+ Predicate predicate =
+ ChainTableUtils.createGroupChainPredicate(
+ partition,
+ converter,
+ 2,
+ (Integer i, Object v) -> builder.equal(i, v),
+ (Integer i, Object v) -> builder.lessThan(i, v));
+
+ // (China, Beijing, 20250731) → match
+ assertThat(predicate.test(row(Lists.newArrayList("China", "Beijing", "20250731"))))
+ .isTrue();
+
+ // (China, Beijing, 20250801) → not less than → no match
+ assertThat(predicate.test(partition)).isFalse();
+
+ // (China, Shanghai, 20250731) → city mismatch → no match
+ assertThat(predicate.test(row(Lists.newArrayList("China", "Shanghai", "20250731"))))
+ .isFalse();
+
+ // (US, Beijing, 20250731) → country mismatch → no match
+ assertThat(predicate.test(row(Lists.newArrayList("US", "Beijing", "20250731")))).isFalse();
+ }
}
diff --git a/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkChainTableITCase.java b/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkChainTableITCase.java
index 83a77f1232..1907c7fcf3 100644
--- a/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkChainTableITCase.java
+++ b/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkChainTableITCase.java
@@ -842,4 +842,1460 @@ public class SparkChainTableITCase {
spark.sql("DROP TABLE IF EXISTS `my_db1`.`chain_test_t`;");
spark.close();
}
+
+ @Test
+ public void testChainTableWithGroupPartition(@TempDir java.nio.file.Path tempDir)
+ throws IOException {
+ Path warehousePath = new Path("file:" + tempDir.toString());
+ SparkSession.Builder builder =
+ SparkSession.builder()
+ .config("spark.sql.warehouse.dir", warehousePath.toString())
+ // with hive metastore
+ .config("spark.sql.catalogImplementation", "hive")
+ .config("hive.metastore.uris", "thrift://localhost:" + PORT)
+ .config("spark.sql.catalog.spark_catalog", SparkCatalog.class.getName())
+ .config("spark.sql.catalog.spark_catalog.metastore", "hive")
+ .config(
+ "spark.sql.catalog.spark_catalog.hive.metastore.uris",
+ "thrift://localhost:" + PORT)
+ .config("spark.sql.catalog.spark_catalog.format-table.enabled", "true")
+ .config(
+ "spark.sql.catalog.spark_catalog.warehouse",
+ warehousePath.toString())
+ .config(
+ "spark.sql.extensions",
+ "org.apache.paimon.spark.extensions.PaimonSparkSessionExtensions")
+ .master("local[2]");
+ SparkSession spark = builder.getOrCreate();
+ spark.sql("CREATE DATABASE IF NOT EXISTS my_db1");
+ spark.sql("USE spark_catalog.my_db1");
+
+ /** Create table */
+ spark.sql(
+ "CREATE TABLE IF NOT EXISTS \n"
+ + " `my_db1`.`chain_test` (\n"
+ + " `t1` BIGINT COMMENT 't1',\n"
+ + " `t2` BIGINT COMMENT 't2',\n"
+ + " `t3` STRING COMMENT 't3'\n"
+ + " ) PARTITIONED BY (`region` STRING, `dt` STRING COMMENT 'dt') ROW FORMAT SERDE 'org.apache.paimon.hive.PaimonSerDe'\n"
+ + "WITH\n"
+ + " SERDEPROPERTIES ('serialization.format' = '1') STORED AS INPUTFORMAT 'org.apache.paimon.hive.mapred.PaimonInputFormat' OUTPUTFORMAT 'org.apache.paimon.hive.mapred.PaimonOutputFormat' TBLPROPERTIES (\n"
+ + " 'bucket-key' = 't1',\n"
+ + " 'primary-key' = 'region,dt,t1',\n"
+ + " 'partition.timestamp-pattern' = '$dt',\n"
+ + " 'partition.timestamp-formatter' = 'yyyyMMdd',\n"
+ + " 'chain-table.enabled' = 'true',\n"
+ + " 'bucket' = '2',\n"
+ + " 'merge-engine' = 'deduplicate', \n"
+ + " 'sequence.field' = 't2',\n"
+ + " 'chain-table.chain-partition-keys' = 'dt'\n"
+ + " )");
+
+ /** Create branch */
+ spark.sql("CALL sys.create_branch('my_db1.chain_test', 'snapshot');");
+ spark.sql("CALL sys.create_branch('my_db1.chain_test', 'delta')");
+
+ /** Set branch */
+ spark.sql(
+ "ALTER TABLE my_db1.chain_test SET tblproperties ("
+ + "'scan.fallback-snapshot-branch' = 'snapshot', "
+ + "'scan.fallback-delta-branch' = 'delta')");
+ spark.sql(
+ "ALTER TABLE `my_db1`.`chain_test$branch_snapshot` SET tblproperties ("
+ + "'scan.fallback-snapshot-branch' = 'snapshot',"
+ + "'scan.fallback-delta-branch' = 'delta')");
+ spark.sql(
+ "ALTER TABLE `my_db1`.`chain_test$branch_delta` SET tblproperties ("
+ + "'scan.fallback-snapshot-branch' = 'snapshot',"
+ + "'scan.fallback-delta-branch' = 'delta')");
+ spark.close();
+ spark = builder.getOrCreate();
+
+ /** Write main branch */
+ spark.sql(
+ "insert overwrite table `my_db1`.`chain_test` partition (region = 'CN', dt = '20250810') values (1, 1, '1'),(2, 1, '1');");
+ spark.sql(
+ "insert overwrite table `my_db1`.`chain_test` partition (region = 'US', dt = '20250810') values (11, 1, '1'),(12, 1, '1');");
+
+ /** Write delta branch */
+ spark.sql("set spark.paimon.branch=delta;");
+
+ spark.sql(
+ "insert overwrite table `my_db1`.`chain_test` partition (region = 'CN', dt = '20250809') values (1, 1, '1'),(2, 1, '1');");
+ spark.sql(
+ "insert overwrite table `my_db1`.`chain_test` partition (region = 'CN', dt = '20250810') values (1, 2, '1-1' ),(3, 1, '1' );");
+ spark.sql(
+ "insert overwrite table `my_db1`.`chain_test` partition (region = 'CN', dt = '20250811') values (2, 2, '1-1' ),(4, 1, '1' );");
+ spark.sql(
+ "insert overwrite table `my_db1`.`chain_test` partition (region = 'CN', dt = '20250812') values (3, 2, '1-1' ),(4, 2, '1-1' );");
+ spark.sql(
+ "insert overwrite table `my_db1`.`chain_test` partition (region = 'CN', dt = '20250813') values (5, 1, '1' ),(6, 1, '1' );");
+ spark.sql(
+ "insert overwrite table `my_db1`.`chain_test` partition (region = 'CN', dt = '20250814') values (5, 2, '1-1' ),(6, 2, '1-1' );");
+
+ spark.sql(
+ "insert overwrite table `my_db1`.`chain_test` partition (region = 'US', dt = '20250809') values (11, 1, '1'),(12, 1, '1');");
+ spark.sql(
+ "insert overwrite table `my_db1`.`chain_test` partition (region = 'US', dt = '20250810') values (11, 2, '1-1' ),(13, 1, '1' );");
+ spark.sql(
+ "insert overwrite table `my_db1`.`chain_test` partition (region = 'US', dt = '20250811') values (12, 2, '1-1' ),(14, 1, '1' );");
+ spark.sql(
+ "insert overwrite table `my_db1`.`chain_test` partition (region = 'US', dt = '20250812') values (13, 2, '1-1' ),(14, 2, '1-1' );");
+ spark.sql(
+ "insert overwrite table `my_db1`.`chain_test` partition (region = 'US', dt = '20250813') values (15, 1, '1' ),(16, 1, '1' );");
+ spark.sql(
+ "insert overwrite table `my_db1`.`chain_test` partition (region = 'US', dt = '20250814') values (15, 2, '1-1' ),(16, 2, '1-1' );");
+
+ /** Write snapshot branch */
+ spark.sql("set spark.paimon.branch=snapshot;");
+ spark.sql(
+ "insert overwrite table `my_db1`.`chain_test` partition (region = 'CN', dt = '20250810') values (1, 2, '1-1'),(2, 1, '1'),(3, 1, '1');");
+ spark.sql(
+ "insert overwrite table `my_db1`.`chain_test` partition (region = 'CN', dt = '20250812') values (1, 2, '1-1'),(2, 2, '1-1'),(3, 2, '1-1'), (4, 2, '1-1');");
+ spark.sql(
+ "insert overwrite table `my_db1`.`chain_test` partition (region = 'CN', dt = '20250814') values (1, 2, '1-1'),(2, 2, '1-1'),(3, 2, '1-1'), (4, 2, '1-1'), (5, 1, '1' ), (6, 1, '1');");
+
+ spark.sql(
+ "insert overwrite table `my_db1`.`chain_test` partition (region = 'US', dt = '20250810') values (11, 2, '1-1'),(12, 1, '1'),(13, 1, '1');");
+ spark.sql(
+ "insert overwrite table `my_db1`.`chain_test` partition (region = 'US', dt = '20250812') values (11, 2, '1-1'),(12, 2, '1-1'),(13, 2, '1-1'), (14, 2, '1-1');");
+ spark.sql(
+ "insert overwrite table `my_db1`.`chain_test` partition (region = 'US', dt = '20250814') values (11, 2, '1-1'),(12, 2, '1-1'),(13, 2, '1-1'), (14, 2, '1-1'), (15, 1, '1' ), (16, 1, '1');");
+
+ spark.close();
+ spark = builder.getOrCreate();
+ /** Main read */
+ assertThat(
+ spark.sql("SELECT * FROM `my_db1`.`chain_test` where dt = '20250810'")
+ .collectAsList().stream()
+ .map(Row::toString)
+ .collect(Collectors.toList()))
+ .containsExactlyInAnyOrder(
+ "[1,1,1,CN,20250810]",
+ "[2,1,1,CN,20250810]",
+ "[11,1,1,US,20250810]",
+ "[12,1,1,US,20250810]");
+
+ /** Snapshot read */
+ assertThat(
+ spark.sql("SELECT * FROM `my_db1`.`chain_test` where dt = '20250814'")
+ .collectAsList().stream()
+ .map(Row::toString)
+ .collect(Collectors.toList()))
+ .containsExactlyInAnyOrder(
+ "[1,2,1-1,CN,20250814]",
+ "[2,2,1-1,CN,20250814]",
+ "[3,2,1-1,CN,20250814]",
+ "[4,2,1-1,CN,20250814]",
+ "[5,1,1,CN,20250814]",
+ "[6,1,1,CN,20250814]",
+ "[11,2,1-1,US,20250814]",
+ "[12,2,1-1,US,20250814]",
+ "[13,2,1-1,US,20250814]",
+ "[14,2,1-1,US,20250814]",
+ "[15,1,1,US,20250814]",
+ "[16,1,1,US,20250814]");
+
+ assertThat(
+ spark
+ .sql(
+ "SELECT * FROM `my_db1`.`chain_test` where region = 'CN' and dt = '20250814'")
+ .collectAsList().stream()
+ .map(Row::toString)
+ .collect(Collectors.toList()))
+ .containsExactlyInAnyOrder(
+ "[1,2,1-1,CN,20250814]",
+ "[2,2,1-1,CN,20250814]",
+ "[3,2,1-1,CN,20250814]",
+ "[4,2,1-1,CN,20250814]",
+ "[5,1,1,CN,20250814]",
+ "[6,1,1,CN,20250814]");
+
+ assertThat(
+ spark
+ .sql(
+ "SELECT * FROM `my_db1`.`chain_test` where region = 'US' and dt = '20250814'")
+ .collectAsList().stream()
+ .map(Row::toString)
+ .collect(Collectors.toList()))
+ .containsExactlyInAnyOrder(
+ "[11,2,1-1,US,20250814]",
+ "[12,2,1-1,US,20250814]",
+ "[13,2,1-1,US,20250814]",
+ "[14,2,1-1,US,20250814]",
+ "[15,1,1,US,20250814]",
+ "[16,1,1,US,20250814]");
+
+ /** Chain read */
+ /** 1. non pre snapshot */
+ assertThat(
+ spark.sql("SELECT * FROM `my_db1`.`chain_test` where dt = '20250809'")
+ .collectAsList().stream()
+ .map(Row::toString)
+ .collect(Collectors.toList()))
+ .containsExactlyInAnyOrder(
+ "[1,1,1,CN,20250809]",
+ "[2,1,1,CN,20250809]",
+ "[11,1,1,US,20250809]",
+ "[12,1,1,US,20250809]");
+
+ assertThat(
+ spark
+ .sql(
+ "SELECT * FROM `my_db1`.`chain_test` where region = 'CN' and dt = '20250809'")
+ .collectAsList().stream()
+ .map(Row::toString)
+ .collect(Collectors.toList()))
+ .containsExactlyInAnyOrder("[1,1,1,CN,20250809]", "[2,1,1,CN,20250809]");
+
+ assertThat(
+ spark
+ .sql(
+ "SELECT * FROM `my_db1`.`chain_test` where region = 'US' and dt = '20250809'")
+ .collectAsList().stream()
+ .map(Row::toString)
+ .collect(Collectors.toList()))
+ .containsExactlyInAnyOrder("[11,1,1,US,20250809]", "[12,1,1,US,20250809]");
+ /** 2. has pre snapshot */
+ assertThat(
+ spark.sql("SELECT * FROM `my_db1`.`chain_test` where dt = '20250811'")
+ .collectAsList().stream()
+ .map(Row::toString)
+ .collect(Collectors.toList()))
+ .containsExactlyInAnyOrder(
+ "[1,2,1-1,CN,20250811]",
+ "[2,2,1-1,CN,20250811]",
+ "[3,1,1,CN,20250811]",
+ "[4,1,1,CN,20250811]",
+ "[11,2,1-1,US,20250811]",
+ "[12,2,1-1,US,20250811]",
+ "[13,1,1,US,20250811]",
+ "[14,1,1,US,20250811]");
+
+ assertThat(
+ spark
+ .sql(
+ "SELECT * FROM `my_db1`.`chain_test` where region = 'CN' and dt = '20250811'")
+ .collectAsList().stream()
+ .map(Row::toString)
+ .collect(Collectors.toList()))
+ .containsExactlyInAnyOrder(
+ "[1,2,1-1,CN,20250811]",
+ "[2,2,1-1,CN,20250811]",
+ "[3,1,1,CN,20250811]",
+ "[4,1,1,CN,20250811]");
+
+ assertThat(
+ spark
+ .sql(
+ "SELECT * FROM `my_db1`.`chain_test` where region = 'US' and dt = '20250811'")
+ .collectAsList().stream()
+ .map(Row::toString)
+ .collect(Collectors.toList()))
+ .containsExactlyInAnyOrder(
+ "[11,2,1-1,US,20250811]",
+ "[12,2,1-1,US,20250811]",
+ "[13,1,1,US,20250811]",
+ "[14,1,1,US,20250811]");
+
+ /** Multi partition Read */
+ assertThat(
+ spark
+ .sql(
+ "SELECT * FROM `my_db1`.`chain_test` where dt in ('20250810', '20250811', '20250812');")
+ .collectAsList().stream()
+ .map(Row::toString)
+ .collect(Collectors.toList()))
+ .containsExactlyInAnyOrder(
+ "[1,1,1,CN,20250810]",
+ "[2,1,1,CN,20250810]",
+ "[1,2,1-1,CN,20250811]",
+ "[2,2,1-1,CN,20250811]",
+ "[3,1,1,CN,20250811]",
+ "[4,1,1,CN,20250811]",
+ "[1,2,1-1,CN,20250812]",
+ "[2,2,1-1,CN,20250812]",
+ "[3,2,1-1,CN,20250812]",
+ "[4,2,1-1,CN,20250812]",
+ "[11,1,1,US,20250810]",
+ "[12,1,1,US,20250810]",
+ "[11,2,1-1,US,20250811]",
+ "[12,2,1-1,US,20250811]",
+ "[13,1,1,US,20250811]",
+ "[14,1,1,US,20250811]",
+ "[11,2,1-1,US,20250812]",
+ "[12,2,1-1,US,20250812]",
+ "[13,2,1-1,US,20250812]",
+ "[14,2,1-1,US,20250812]");
+
+ assertThat(
+ spark
+ .sql(
+ "SELECT * FROM `my_db1`.`chain_test` where region = 'CN' and dt in ('20250810', '20250811', '20250812');")
+ .collectAsList().stream()
+ .map(Row::toString)
+ .collect(Collectors.toList()))
+ .containsExactlyInAnyOrder(
+ "[1,1,1,CN,20250810]",
+ "[2,1,1,CN,20250810]",
+ "[1,2,1-1,CN,20250811]",
+ "[2,2,1-1,CN,20250811]",
+ "[3,1,1,CN,20250811]",
+ "[4,1,1,CN,20250811]",
+ "[1,2,1-1,CN,20250812]",
+ "[2,2,1-1,CN,20250812]",
+ "[3,2,1-1,CN,20250812]",
+ "[4,2,1-1,CN,20250812]");
+
+ assertThat(
+ spark
+ .sql(
+ "SELECT * FROM `my_db1`.`chain_test` where region = 'US' and dt in ('20250810', '20250811', '20250812');")
+ .collectAsList().stream()
+ .map(Row::toString)
+ .collect(Collectors.toList()))
+ .containsExactlyInAnyOrder(
+ "[11,1,1,US,20250810]",
+ "[12,1,1,US,20250810]",
+ "[11,2,1-1,US,20250811]",
+ "[12,2,1-1,US,20250811]",
+ "[13,1,1,US,20250811]",
+ "[14,1,1,US,20250811]",
+ "[11,2,1-1,US,20250812]",
+ "[12,2,1-1,US,20250812]",
+ "[13,2,1-1,US,20250812]",
+ "[14,2,1-1,US,20250812]");
+
+ /** Incremental read */
+ assertThat(
+ spark
+ .sql(
+ "SELECT * FROM `my_db1`.`chain_test$branch_delta` where dt = '20250811'")
+ .collectAsList().stream()
+ .map(Row::toString)
+ .collect(Collectors.toList()))
+ .containsExactlyInAnyOrder(
+ "[2,2,1-1,CN,20250811]",
+ "[4,1,1,CN,20250811]",
+ "[12,2,1-1,US,20250811]",
+ "[14,1,1,US,20250811]");
+
+ assertThat(
+ spark
+ .sql(
+ "SELECT * FROM `my_db1`.`chain_test$branch_delta` where region = 'CN' and dt = '20250811'")
+ .collectAsList().stream()
+ .map(Row::toString)
+ .collect(Collectors.toList()))
+ .containsExactlyInAnyOrder("[2,2,1-1,CN,20250811]", "[4,1,1,CN,20250811]");
+
+ assertThat(
+ spark
+ .sql(
+ "SELECT * FROM `my_db1`.`chain_test$branch_delta` where region = 'US' and dt = '20250811'")
+ .collectAsList().stream()
+ .map(Row::toString)
+ .collect(Collectors.toList()))
+ .containsExactlyInAnyOrder("[12,2,1-1,US,20250811]", "[14,1,1,US,20250811]");
+
+ /** Multi partition incremental read */
+ assertThat(
+ spark
+ .sql(
+ "SELECT * FROM `my_db1`.`chain_test$branch_delta` where dt in ('20250810', '20250811', '20250812');")
+ .collectAsList().stream()
+ .map(Row::toString)
+ .collect(Collectors.toList()))
+ .containsExactlyInAnyOrder(
+ "[1,2,1-1,CN,20250810]",
+ "[3,1,1,CN,20250810]",
+ "[2,2,1-1,CN,20250811]",
+ "[4,1,1,CN,20250811]",
+ "[3,2,1-1,CN,20250812]",
+ "[4,2,1-1,CN,20250812]",
+ "[11,2,1-1,US,20250810]",
+ "[13,1,1,US,20250810]",
+ "[12,2,1-1,US,20250811]",
+ "[14,1,1,US,20250811]",
+ "[13,2,1-1,US,20250812]",
+ "[14,2,1-1,US,20250812]");
+
+ assertThat(
+ spark
+ .sql(
+ "SELECT * FROM `my_db1`.`chain_test$branch_delta` where region = 'CN' and dt in ('20250810', '20250811', '20250812');")
+ .collectAsList().stream()
+ .map(Row::toString)
+ .collect(Collectors.toList()))
+ .containsExactlyInAnyOrder(
+ "[1,2,1-1,CN,20250810]",
+ "[3,1,1,CN,20250810]",
+ "[2,2,1-1,CN,20250811]",
+ "[4,1,1,CN,20250811]",
+ "[3,2,1-1,CN,20250812]",
+ "[4,2,1-1,CN,20250812]");
+
+ assertThat(
+ spark
+ .sql(
+ "SELECT * FROM `my_db1`.`chain_test$branch_delta` where region = 'US' and dt in ('20250810', '20250811', '20250812');")
+ .collectAsList().stream()
+ .map(Row::toString)
+ .collect(Collectors.toList()))
+ .containsExactlyInAnyOrder(
+ "[11,2,1-1,US,20250810]",
+ "[13,1,1,US,20250810]",
+ "[12,2,1-1,US,20250811]",
+ "[14,1,1,US,20250811]",
+ "[13,2,1-1,US,20250812]",
+ "[14,2,1-1,US,20250812]");
+
+ spark.close();
+ spark = builder.getOrCreate();
+ spark.sql("set spark.paimon.branch=delta;");
+ spark.sql(
+ "insert overwrite table `my_db1`.`chain_test` values (5, 2, '1', 'CN', '20250813'),(6, 2, '1', 'CN', '20250814'), (15, 2, '1', 'US', '20250813'),(16, 2, '1', 'US', '20250814');");
+
+ spark.close();
+ spark = builder.getOrCreate();
+ Dataset<Row> df =
+ spark.sql(
+ "SELECT t1,t2,t3 FROM `my_db1`.`chain_test$branch_snapshot` where dt = '20250814'");
+ assertThat(df.count()).isEqualTo(0);
+ df =
+ spark.sql(
+ "SELECT t1,t2,t3 FROM `my_db1`.`chain_test$branch_delta` where dt = '20250814'");
+ assertThat(df.count()).isEqualTo(2);
+
+ spark.close();
+ spark = builder.getOrCreate();
+ /** Drop table */
+ spark.sql("DROP TABLE IF EXISTS `my_db1`.`chain_test`;");
+
+ spark.close();
+ }
+
+ @Test
+ public void testHourlyChainTableWithGroupPartition(@TempDir java.nio.file.Path tempDir)
+ throws IOException {
+ Path warehousePath = new Path("file:" + tempDir.toString());
+ SparkSession.Builder builder =
+ SparkSession.builder()
+ .config("spark.sql.warehouse.dir", warehousePath.toString())
+ // with hive metastore
+ .config("spark.sql.catalogImplementation", "hive")
+ .config("hive.metastore.uris", "thrift://localhost:" + PORT)
+ .config("spark.sql.catalog.spark_catalog", SparkCatalog.class.getName())
+ .config("spark.sql.catalog.spark_catalog.metastore", "hive")
+ .config(
+ "spark.sql.catalog.spark_catalog.hive.metastore.uris",
+ "thrift://localhost:" + PORT)
+ .config("spark.sql.catalog.spark_catalog.format-table.enabled", "true")
+ .config(
+ "spark.sql.catalog.spark_catalog.warehouse",
+ warehousePath.toString())
+ .config(
+ "spark.sql.extensions",
+ "org.apache.paimon.spark.extensions.PaimonSparkSessionExtensions")
+ .master("local[2]");
+ SparkSession spark = builder.getOrCreate();
+ spark.sql("CREATE DATABASE IF NOT EXISTS my_db1");
+ spark.sql("USE spark_catalog.my_db1");
+
+ /** Create table */
+ spark.sql(
+ "CREATE TABLE IF NOT EXISTS \n"
+ + " `my_db1`.`chain_test` (\n"
+ + " `t1` BIGINT COMMENT 't1',\n"
+ + " `t2` BIGINT COMMENT 't2',\n"
+ + " `t3` STRING COMMENT 't3'\n"
+ + " ) PARTITIONED BY (`region` STRING, `dt` STRING COMMENT 'dt', `hour` STRING COMMENT 'hour') ROW FORMAT SERDE 'org.apache.paimon.hive.PaimonSerDe'\n"
+ + "WITH\n"
+ + " SERDEPROPERTIES ('serialization.format' = '1') STORED AS INPUTFORMAT 'org.apache.paimon.hive.mapred.PaimonInputFormat' OUTPUTFORMAT 'org.apache.paimon.hive.mapred.PaimonOutputFormat' TBLPROPERTIES (\n"
+ + " 'bucket-key' = 't1',\n"
+ + " 'primary-key' = 'region,dt,hour,t1',\n"
+ + " 'partition.timestamp-pattern' = '$dt $hour:00:00',\n"
+ + " 'partition.timestamp-formatter' = 'yyyyMMdd HH:mm:ss',\n"
+ + " 'chain-table.enabled' = 'true',\n"
+ + " 'bucket' = '2',\n"
+ + " 'merge-engine' = 'deduplicate', \n"
+ + " 'sequence.field' = 't2',\n"
+ + " 'chain-table.chain-partition-keys' = 'dt,hour'\n"
+ + " )");
+
+ /** Create branch */
+ spark.sql("CALL sys.create_branch('my_db1.chain_test', 'snapshot');");
+ spark.sql("CALL sys.create_branch('my_db1.chain_test', 'delta')");
+
+ /** Set branch */
+ spark.sql(
+ "ALTER TABLE my_db1.chain_test SET tblproperties ("
+ + "'scan.fallback-snapshot-branch' = 'snapshot', "
+ + "'scan.fallback-delta-branch' = 'delta')");
+ spark.sql(
+ "ALTER TABLE `my_db1`.`chain_test$branch_snapshot` SET tblproperties ("
+ + "'scan.fallback-snapshot-branch' = 'snapshot',"
+ + "'scan.fallback-delta-branch' = 'delta')");
+ spark.sql(
+ "ALTER TABLE `my_db1`.`chain_test$branch_delta` SET tblproperties ("
+ + "'scan.fallback-snapshot-branch' = 'snapshot',"
+ + "'scan.fallback-delta-branch' = 'delta')");
+ spark.close();
+ spark = builder.getOrCreate();
+
+ /** Write main branch */
+ spark.sql(
+ "insert overwrite table `my_db1`.`chain_test` partition (region = 'CN', dt = '20250810', hour = '22') values (1, 1, '1'),(2, 1, '1');");
+ spark.sql(
+ "insert overwrite table `my_db1`.`chain_test` partition (region = 'US', dt = '20250810', hour = '22') values (11, 1, '1'),(12, 1, '1');");
+
+ /** Write delta branch */
+ spark.sql("set spark.paimon.branch=delta;");
+ spark.sql(
+ "insert overwrite table `my_db1`.`chain_test` partition (region = 'CN', dt = '20250810', hour = '21') values (1, 1, '1'),(2, 1, '1');");
+ spark.sql(
+ "insert overwrite table `my_db1`.`chain_test` partition (region = 'CN', dt = '20250810', hour = '22') values (1, 2, '1-1' ),(3, 1, '1' );");
+ spark.sql(
+ "insert overwrite table `my_db1`.`chain_test` partition (region = 'CN', dt = '20250810', hour = '23') values (2, 2, '1-1' ),(4, 1, '1' );");
+ spark.sql(
+ "insert overwrite table `my_db1`.`chain_test` partition (region = 'CN', dt = '20250811', hour = '00') values (3, 2, '1-1' ),(4, 2, '1-1' );");
+ spark.sql(
+ "insert overwrite table `my_db1`.`chain_test` partition (region = 'CN', dt = '20250811', hour = '01') values (5, 1, '1' ),(6, 1, '1' );");
+ spark.sql(
+ "insert overwrite table `my_db1`.`chain_test` partition (region = 'CN', dt = '20250811', hour = '02') values (5, 2, '1-1' ),(6, 2, '1-1' );");
+
+ spark.sql(
+ "insert overwrite table `my_db1`.`chain_test` partition (region = 'US', dt = '20250810', hour = '21') values (11, 1, '1'),(12, 1, '1');");
+ spark.sql(
+ "insert overwrite table `my_db1`.`chain_test` partition (region = 'US', dt = '20250810', hour = '22') values (11, 2, '1-1' ),(13, 1, '1' );");
+ spark.sql(
+ "insert overwrite table `my_db1`.`chain_test` partition (region = 'US', dt = '20250810', hour = '23') values (12, 2, '1-1' ),(14, 1, '1' );");
+ spark.sql(
+ "insert overwrite table `my_db1`.`chain_test` partition (region = 'US', dt = '20250811', hour = '00') values (13, 2, '1-1' ),(14, 2, '1-1' );");
+ spark.sql(
+ "insert overwrite table `my_db1`.`chain_test` partition (region = 'US', dt = '20250811', hour = '01') values (15, 1, '1' ),(16, 1, '1' );");
+ spark.sql(
+ "insert overwrite table `my_db1`.`chain_test` partition (region = 'US', dt = '20250811', hour = '02') values (15, 2, '1-1' ),(16, 2, '1-1' );");
+
+ /** Write snapshot branch */
+ spark.sql("set spark.paimon.branch=snapshot;");
+ spark.sql(
+ "insert overwrite table `my_db1`.`chain_test` partition (region = 'CN', dt = '20250810', hour = '22') values (1, 2, '1-1'),(2, 1, '1'),(3, 1, '1');");
+ spark.sql(
+ "insert overwrite table `my_db1`.`chain_test` partition (region = 'CN', dt = '20250811', hour = '00') values (1, 2, '1-1'),(2, 2, '1-1'),(3, 2, '1-1'), (4, 2, '1-1');");
+ spark.sql(
+ "insert overwrite table `my_db1`.`chain_test` partition (region = 'CN', dt = '20250811', hour = '02') values (1, 2, '1-1'),(2, 2, '1-1'),(3, 2, '1-1'), (4, 2, '1-1'), (5, 1, '1' ), (6, 1, '1');");
+
+ spark.sql(
+ "insert overwrite table `my_db1`.`chain_test` partition (region = 'US', dt = '20250810', hour = '22') values (11, 2, '1-1'),(12, 1, '1'),(13, 1, '1');");
+ spark.sql(
+ "insert overwrite table `my_db1`.`chain_test` partition (region = 'US', dt = '20250811', hour = '00') values (11, 2, '1-1'),(12, 2, '1-1'),(13, 2, '1-1'), (14, 2, '1-1');");
+ spark.sql(
+ "insert overwrite table `my_db1`.`chain_test` partition (region = 'US', dt = '20250811', hour = '02') values (11, 2, '1-1'),(12, 2, '1-1'),(13, 2, '1-1'), (14, 2, '1-1'), (15, 1, '1' ), (16, 1, '1');");
+
+ spark.close();
+ spark = builder.getOrCreate();
+ /** Main read */
+ assertThat(
+ spark
+ .sql(
+ "SELECT * FROM `my_db1`.`chain_test` where dt = '20250810' and hour = '22'")
+ .collectAsList().stream()
+ .map(Row::toString)
+ .collect(Collectors.toList()))
+ .containsExactlyInAnyOrder(
+ "[1,1,1,CN,20250810,22]",
+ "[2,1,1,CN,20250810,22]",
+ "[11,1,1,US,20250810,22]",
+ "[12,1,1,US,20250810,22]");
+
+ assertThat(
+ spark
+ .sql(
+ "SELECT * FROM `my_db1`.`chain_test` where region = 'CN' and dt = '20250810' and hour = '22'")
+ .collectAsList().stream()
+ .map(Row::toString)
+ .collect(Collectors.toList()))
+ .containsExactlyInAnyOrder("[1,1,1,CN,20250810,22]", "[2,1,1,CN,20250810,22]");
+
+ assertThat(
+ spark
+ .sql(
+ "SELECT * FROM `my_db1`.`chain_test` where region = 'US' and dt = '20250810' and hour = '22'")
+ .collectAsList().stream()
+ .map(Row::toString)
+ .collect(Collectors.toList()))
+ .containsExactlyInAnyOrder("[11,1,1,US,20250810,22]", "[12,1,1,US,20250810,22]");
+
+ /** Snapshot read */
+ assertThat(
+ spark
+ .sql(
+ "SELECT * FROM `my_db1`.`chain_test` where dt = '20250811' and hour = '02'")
+ .collectAsList().stream()
+ .map(Row::toString)
+ .collect(Collectors.toList()))
+ .containsExactlyInAnyOrder(
+ "[1,2,1-1,CN,20250811,02]",
+ "[2,2,1-1,CN,20250811,02]",
+ "[3,2,1-1,CN,20250811,02]",
+ "[4,2,1-1,CN,20250811,02]",
+ "[5,1,1,CN,20250811,02]",
+ "[6,1,1,CN,20250811,02]",
+ "[11,2,1-1,US,20250811,02]",
+ "[12,2,1-1,US,20250811,02]",
+ "[13,2,1-1,US,20250811,02]",
+ "[14,2,1-1,US,20250811,02]",
+ "[15,1,1,US,20250811,02]",
+ "[16,1,1,US,20250811,02]");
+
+ assertThat(
+ spark
+ .sql(
+ "SELECT * FROM `my_db1`.`chain_test` where region = 'CN' and dt = '20250811' and hour = '02'")
+ .collectAsList().stream()
+ .map(Row::toString)
+ .collect(Collectors.toList()))
+ .containsExactlyInAnyOrder(
+ "[1,2,1-1,CN,20250811,02]",
+ "[2,2,1-1,CN,20250811,02]",
+ "[3,2,1-1,CN,20250811,02]",
+ "[4,2,1-1,CN,20250811,02]",
+ "[5,1,1,CN,20250811,02]",
+ "[6,1,1,CN,20250811,02]");
+
+ assertThat(
+ spark
+ .sql(
+ "SELECT * FROM `my_db1`.`chain_test` where region = 'US' and dt = '20250811' and hour = '02'")
+ .collectAsList().stream()
+ .map(Row::toString)
+ .collect(Collectors.toList()))
+ .containsExactlyInAnyOrder(
+ "[11,2,1-1,US,20250811,02]",
+ "[12,2,1-1,US,20250811,02]",
+ "[13,2,1-1,US,20250811,02]",
+ "[14,2,1-1,US,20250811,02]",
+ "[15,1,1,US,20250811,02]",
+ "[16,1,1,US,20250811,02]");
+
+ /** Chain read */
+ /** 1. non pre snapshot */
+ assertThat(
+ spark
+ .sql(
+ "SELECT * FROM `my_db1`.`chain_test` where dt = '20250810' and hour = '21'")
+ .collectAsList().stream()
+ .map(Row::toString)
+ .collect(Collectors.toList()))
+ .containsExactlyInAnyOrder(
+ "[1,1,1,CN,20250810,21]",
+ "[2,1,1,CN,20250810,21]",
+ "[11,1,1,US,20250810,21]",
+ "[12,1,1,US,20250810,21]");
+
+ assertThat(
+ spark
+ .sql(
+ "SELECT * FROM `my_db1`.`chain_test` where region = 'CN' and dt = '20250810' and hour = '21'")
+ .collectAsList().stream()
+ .map(Row::toString)
+ .collect(Collectors.toList()))
+ .containsExactlyInAnyOrder("[1,1,1,CN,20250810,21]", "[2,1,1,CN,20250810,21]");
+
+ assertThat(
+ spark
+ .sql(
+ "SELECT * FROM `my_db1`.`chain_test` where region = 'US' and dt = '20250810' and hour = '21'")
+ .collectAsList().stream()
+ .map(Row::toString)
+ .collect(Collectors.toList()))
+ .containsExactlyInAnyOrder("[11,1,1,US,20250810,21]", "[12,1,1,US,20250810,21]");
+ /** 2. has pre snapshot */
+ assertThat(
+ spark
+ .sql(
+ "SELECT * FROM `my_db1`.`chain_test` where dt = '20250810' and hour = '23'")
+ .collectAsList().stream()
+ .map(Row::toString)
+ .collect(Collectors.toList()))
+ .containsExactlyInAnyOrder(
+ "[1,2,1-1,CN,20250810,23]",
+ "[2,2,1-1,CN,20250810,23]",
+ "[3,1,1,CN,20250810,23]",
+ "[4,1,1,CN,20250810,23]",
+ "[11,2,1-1,US,20250810,23]",
+ "[12,2,1-1,US,20250810,23]",
+ "[13,1,1,US,20250810,23]",
+ "[14,1,1,US,20250810,23]");
+
+ assertThat(
+ spark
+ .sql(
+ "SELECT * FROM `my_db1`.`chain_test` where region = 'CN' and dt = '20250810' and hour = '23'")
+ .collectAsList().stream()
+ .map(Row::toString)
+ .collect(Collectors.toList()))
+ .containsExactlyInAnyOrder(
+ "[1,2,1-1,CN,20250810,23]",
+ "[2,2,1-1,CN,20250810,23]",
+ "[3,1,1,CN,20250810,23]",
+ "[4,1,1,CN,20250810,23]");
+
+ assertThat(
+ spark
+ .sql(
+ "SELECT * FROM `my_db1`.`chain_test` where region = 'US' and dt = '20250810' and hour = '23'")
+ .collectAsList().stream()
+ .map(Row::toString)
+ .collect(Collectors.toList()))
+ .containsExactlyInAnyOrder(
+ "[11,2,1-1,US,20250810,23]",
+ "[12,2,1-1,US,20250810,23]",
+ "[13,1,1,US,20250810,23]",
+ "[14,1,1,US,20250810,23]");
+
+ /** Multi partition Read */
+ assertThat(
+ spark
+ .sql(
+ "SELECT * FROM `my_db1`.`chain_test` where dt = '20250810' and hour in ('22', '23');")
+ .collectAsList().stream()
+ .map(Row::toString)
+ .collect(Collectors.toList()))
+ .containsExactlyInAnyOrder(
+ "[1,1,1,CN,20250810,22]",
+ "[2,1,1,CN,20250810,22]",
+ "[1,2,1-1,CN,20250810,23]",
+ "[2,2,1-1,CN,20250810,23]",
+ "[3,1,1,CN,20250810,23]",
+ "[4,1,1,CN,20250810,23]",
+ "[11,1,1,US,20250810,22]",
+ "[12,1,1,US,20250810,22]",
+ "[11,2,1-1,US,20250810,23]",
+ "[12,2,1-1,US,20250810,23]",
+ "[13,1,1,US,20250810,23]",
+ "[14,1,1,US,20250810,23]");
+
+ assertThat(
+ spark
+ .sql(
+ "SELECT * FROM `my_db1`.`chain_test` where region = 'CN' and dt = '20250810' and hour in ('22', '23');")
+ .collectAsList().stream()
+ .map(Row::toString)
+ .collect(Collectors.toList()))
+ .containsExactlyInAnyOrder(
+ "[1,1,1,CN,20250810,22]",
+ "[2,1,1,CN,20250810,22]",
+ "[1,2,1-1,CN,20250810,23]",
+ "[2,2,1-1,CN,20250810,23]",
+ "[3,1,1,CN,20250810,23]",
+ "[4,1,1,CN,20250810,23]");
+
+ assertThat(
+ spark
+ .sql(
+ "SELECT * FROM `my_db1`.`chain_test` where region = 'US' and dt = '20250810' and hour in ('22', '23');")
+ .collectAsList().stream()
+ .map(Row::toString)
+ .collect(Collectors.toList()))
+ .containsExactlyInAnyOrder(
+ "[11,1,1,US,20250810,22]",
+ "[12,1,1,US,20250810,22]",
+ "[11,2,1-1,US,20250810,23]",
+ "[12,2,1-1,US,20250810,23]",
+ "[13,1,1,US,20250810,23]",
+ "[14,1,1,US,20250810,23]");
+
+ /** Incremental read */
+ assertThat(
+ spark
+ .sql(
+ "SELECT * FROM `my_db1`.`chain_test$branch_delta` where dt = '20250810' and hour = '23'")
+ .collectAsList().stream()
+ .map(Row::toString)
+ .collect(Collectors.toList()))
+ .containsExactlyInAnyOrder(
+ "[2,2,1-1,CN,20250810,23]",
+ "[4,1,1,CN,20250810,23]",
+ "[12,2,1-1,US,20250810,23]",
+ "[14,1,1,US,20250810,23]");
+
+ assertThat(
+ spark
+ .sql(
+ "SELECT * FROM `my_db1`.`chain_test$branch_delta` where region = 'CN' and dt = '20250810' and hour = '23'")
+ .collectAsList().stream()
+ .map(Row::toString)
+ .collect(Collectors.toList()))
+ .containsExactlyInAnyOrder("[2,2,1-1,CN,20250810,23]", "[4,1,1,CN,20250810,23]");
+
+ assertThat(
+ spark
+ .sql(
+ "SELECT * FROM `my_db1`.`chain_test$branch_delta` where region = 'US' and dt = '20250810' and hour = '23'")
+ .collectAsList().stream()
+ .map(Row::toString)
+ .collect(Collectors.toList()))
+ .containsExactlyInAnyOrder("[12,2,1-1,US,20250810,23]", "[14,1,1,US,20250810,23]");
+
+ /** Multi partition incremental read */
+ assertThat(
+ spark
+ .sql(
+ "SELECT * FROM `my_db1`.`chain_test$branch_delta` where dt = '20250810' and hour in ('22', '23');")
+ .collectAsList().stream()
+ .map(Row::toString)
+ .collect(Collectors.toList()))
+ .containsExactlyInAnyOrder(
+ "[1,2,1-1,CN,20250810,22]",
+ "[3,1,1,CN,20250810,22]",
+ "[2,2,1-1,CN,20250810,23]",
+ "[4,1,1,CN,20250810,23]",
+ "[11,2,1-1,US,20250810,22]",
+ "[13,1,1,US,20250810,22]",
+ "[12,2,1-1,US,20250810,23]",
+ "[14,1,1,US,20250810,23]");
+
+ assertThat(
+ spark
+ .sql(
+ "SELECT * FROM `my_db1`.`chain_test$branch_delta` where region = 'CN' and dt = '20250810' and hour in ('22', '23');")
+ .collectAsList().stream()
+ .map(Row::toString)
+ .collect(Collectors.toList()))
+ .containsExactlyInAnyOrder(
+ "[1,2,1-1,CN,20250810,22]",
+ "[3,1,1,CN,20250810,22]",
+ "[2,2,1-1,CN,20250810,23]",
+ "[4,1,1,CN,20250810,23]");
+
+ assertThat(
+ spark
+ .sql(
+ "SELECT * FROM `my_db1`.`chain_test$branch_delta` where region = 'US' and dt = '20250810' and hour in ('22', '23');")
+ .collectAsList().stream()
+ .map(Row::toString)
+ .collect(Collectors.toList()))
+ .containsExactlyInAnyOrder(
+ "[11,2,1-1,US,20250810,22]",
+ "[13,1,1,US,20250810,22]",
+ "[12,2,1-1,US,20250810,23]",
+ "[14,1,1,US,20250810,23]");
+
+ /** Hybrid read */
+ assertThat(
+ spark
+ .sql(
+ "select * from `my_db1`.`chain_test` where dt = '20250810' and hour = '23'\n"
+ + "union all\n"
+ + "select * from `my_db1`.`chain_test$branch_delta` where dt = '20250810' and hour = '23'")
+ .collectAsList().stream()
+ .map(Row::toString)
+ .collect(Collectors.toList()))
+ .containsExactlyInAnyOrder(
+ "[1,2,1-1,CN,20250810,23]",
+ "[2,2,1-1,CN,20250810,23]",
+ "[3,1,1,CN,20250810,23]",
+ "[4,1,1,CN,20250810,23]",
+ "[2,2,1-1,CN,20250810,23]",
+ "[4,1,1,CN,20250810,23]",
+ "[11,2,1-1,US,20250810,23]",
+ "[12,2,1-1,US,20250810,23]",
+ "[13,1,1,US,20250810,23]",
+ "[14,1,1,US,20250810,23]",
+ "[12,2,1-1,US,20250810,23]",
+ "[14,1,1,US,20250810,23]");
+
+ spark.close();
+ spark = builder.getOrCreate();
+ spark.sql("set spark.paimon.branch=delta;");
+ spark.sql(
+ "insert overwrite table `my_db1`.`chain_test` values (6, 2, '1', 'CN', '20250811', '02'), (16, 2, '1', 'US', '20250811', '02');");
+
+ spark.close();
+ spark = builder.getOrCreate();
+ Dataset<Row> df =
+ spark.sql(
+ "SELECT t1,t2,t3 FROM `my_db1`.`chain_test$branch_snapshot` where dt = '20250811' and hour = '02'");
+ assertThat(df.count()).isEqualTo(0);
+ df =
+ spark.sql(
+ "SELECT t1,t2,t3 FROM `my_db1`.`chain_test$branch_delta` where dt = '20250811' and hour = '02'");
+ assertThat(df.count()).isEqualTo(2);
+
+ spark.close();
+ spark = builder.getOrCreate();
+ /** Drop table */
+ spark.sql("DROP TABLE IF EXISTS `my_db1`.`chain_test`;");
+
+ spark.close();
+ }
+
+ @Test
+ public void testChainTableWithMultiGroupPartition(@TempDir java.nio.file.Path tempDir)
+ throws IOException {
+ Path warehousePath = new Path("file:" + tempDir.toString());
+ SparkSession.Builder builder =
+ SparkSession.builder()
+ .config("spark.sql.warehouse.dir", warehousePath.toString())
+ // with hive metastore
+ .config("spark.sql.catalogImplementation", "hive")
+ .config("hive.metastore.uris", "thrift://localhost:" + PORT)
+ .config("spark.sql.catalog.spark_catalog", SparkCatalog.class.getName())
+ .config("spark.sql.catalog.spark_catalog.metastore", "hive")
+ .config(
+ "spark.sql.catalog.spark_catalog.hive.metastore.uris",
+ "thrift://localhost:" + PORT)
+ .config("spark.sql.catalog.spark_catalog.format-table.enabled", "true")
+ .config(
+ "spark.sql.catalog.spark_catalog.warehouse",
+ warehousePath.toString())
+ .config(
+ "spark.sql.extensions",
+ "org.apache.paimon.spark.extensions.PaimonSparkSessionExtensions")
+ .master("local[2]");
+ SparkSession spark = builder.getOrCreate();
+ spark.sql("CREATE DATABASE IF NOT EXISTS my_db1");
+ spark.sql("USE spark_catalog.my_db1");
+
+ /** Create table */
+ spark.sql(
+ "CREATE TABLE IF NOT EXISTS \n"
+ + " `my_db1`.`chain_test` (\n"
+ + " `t1` BIGINT COMMENT 't1',\n"
+ + " `t2` BIGINT COMMENT 't2',\n"
+ + " `t3` STRING COMMENT 't3'\n"
+ + " ) PARTITIONED BY (`region` STRING, `biz_type` STRING COMMENT 'biz_type', `dt` STRING COMMENT 'dt') ROW FORMAT SERDE 'org.apache.paimon.hive.PaimonSerDe'\n"
+ + "WITH\n"
+ + " SERDEPROPERTIES ('serialization.format' = '1') STORED AS INPUTFORMAT 'org.apache.paimon.hive.mapred.PaimonInputFormat' OUTPUTFORMAT 'org.apache.paimon.hive.mapred.PaimonOutputFormat' TBLPROPERTIES (\n"
+ + " 'bucket-key' = 't1',\n"
+ + " 'primary-key' = 'region,biz_type,dt,t1',\n"
+ + " 'partition.timestamp-pattern' = '$dt',\n"
+ + " 'partition.timestamp-formatter' = 'yyyyMMdd',\n"
+ + " 'chain-table.enabled' = 'true',\n"
+ + " 'bucket' = '2',\n"
+ + " 'merge-engine' = 'deduplicate', \n"
+ + " 'sequence.field' = 't2',\n"
+ + " 'chain-table.chain-partition-keys' = 'dt'\n"
+ + " )");
+
+ /** Create branch */
+ spark.sql("CALL sys.create_branch('my_db1.chain_test', 'snapshot');");
+ spark.sql("CALL sys.create_branch('my_db1.chain_test', 'delta')");
+
+ /** Set branch */
+ spark.sql(
+ "ALTER TABLE my_db1.chain_test SET tblproperties ("
+ + "'scan.fallback-snapshot-branch' = 'snapshot', "
+ + "'scan.fallback-delta-branch' = 'delta')");
+ spark.sql(
+ "ALTER TABLE `my_db1`.`chain_test$branch_snapshot` SET tblproperties ("
+ + "'scan.fallback-snapshot-branch' = 'snapshot',"
+ + "'scan.fallback-delta-branch' = 'delta')");
+ spark.sql(
+ "ALTER TABLE `my_db1`.`chain_test$branch_delta` SET tblproperties ("
+ + "'scan.fallback-snapshot-branch' = 'snapshot',"
+ + "'scan.fallback-delta-branch' = 'delta')");
+ spark.close();
+ spark = builder.getOrCreate();
+
+ /** Write main branch */
+ spark.sql(
+ "insert overwrite table `my_db1`.`chain_test` partition (region = 'CN', biz_type = '0', dt = '20250810') values (1, 1, '1'),(2, 1, '1');");
+ spark.sql(
+ "insert overwrite table `my_db1`.`chain_test` partition (region = 'CN', biz_type = '1', dt = '20250810') values (1, 1, '1'),(2, 1, '1');");
+ spark.sql(
+ "insert overwrite table `my_db1`.`chain_test` partition (region = 'US', biz_type = '1', dt = '20250810') values (11, 1, '1'),(12, 1, '1');");
+
+ /** Write delta branch */
+ spark.sql("set spark.paimon.branch=delta;");
+
+ spark.sql(
+ "insert overwrite table `my_db1`.`chain_test` partition (region = 'CN', biz_type = '0', dt = '20250809') values (1, 1, '1'),(2, 1, '1');");
+ spark.sql(
+ "insert overwrite table `my_db1`.`chain_test` partition (region = 'CN', biz_type = '0', dt = '20250810') values (1, 2, '1-1' ),(3, 1, '1' );");
+ spark.sql(
+ "insert overwrite table `my_db1`.`chain_test` partition (region = 'CN', biz_type = '0', dt = '20250811') values (2, 2, '1-1' ),(4, 1, '1' );");
+ spark.sql(
+ "insert overwrite table `my_db1`.`chain_test` partition (region = 'CN', biz_type = '0', dt = '20250812') values (3, 2, '1-1' ),(4, 2, '1-1' );");
+ spark.sql(
+ "insert overwrite table `my_db1`.`chain_test` partition (region = 'CN', biz_type = '0', dt = '20250813') values (5, 1, '1' ),(6, 1, '1' );");
+ spark.sql(
+ "insert overwrite table `my_db1`.`chain_test` partition (region = 'CN', biz_type = '0', dt = '20250814') values (5, 2, '1-1' ),(6, 2, '1-1' );");
+
+ spark.sql(
+ "insert overwrite table `my_db1`.`chain_test` partition (region = 'CN', biz_type = '1', dt = '20250809') values (1, 1, '1'),(2, 1, '1');");
+ spark.sql(
+ "insert overwrite table `my_db1`.`chain_test` partition (region = 'CN', biz_type = '1', dt = '20250810') values (1, 2, '1-1' ),(3, 1, '1' );");
+ spark.sql(
+ "insert overwrite table `my_db1`.`chain_test` partition (region = 'CN', biz_type = '1', dt = '20250811') values (2, 2, '1-1' ),(4, 1, '1' );");
+ spark.sql(
+ "insert overwrite table `my_db1`.`chain_test` partition (region = 'CN', biz_type = '1', dt = '20250812') values (3, 2, '1-1' ),(4, 2, '1-1' );");
+ spark.sql(
+ "insert overwrite table `my_db1`.`chain_test` partition (region = 'CN', biz_type = '1', dt = '20250813') values (5, 1, '1' ),(6, 1, '1' );");
+ spark.sql(
+ "insert overwrite table `my_db1`.`chain_test` partition (region = 'CN', biz_type = '1', dt = '20250814') values (5, 2, '1-1' ),(6, 2, '1-1' );");
+
+ spark.sql(
+ "insert overwrite table `my_db1`.`chain_test` partition (region = 'US', biz_type = '1', dt = '20250809') values (11, 1, '1'),(12, 1, '1');");
+ spark.sql(
+ "insert overwrite table `my_db1`.`chain_test` partition (region = 'US', biz_type = '1', dt = '20250810') values (11, 2, '1-1' ),(13, 1, '1' );");
+ spark.sql(
+ "insert overwrite table `my_db1`.`chain_test` partition (region = 'US', biz_type = '1', dt = '20250811') values (12, 2, '1-1' ),(14, 1, '1' );");
+ spark.sql(
+ "insert overwrite table `my_db1`.`chain_test` partition (region = 'US', biz_type = '1', dt = '20250812') values (13, 2, '1-1' ),(14, 2, '1-1' );");
+ spark.sql(
+ "insert overwrite table `my_db1`.`chain_test` partition (region = 'US', biz_type = '1', dt = '20250813') values (15, 1, '1' ),(16, 1, '1' );");
+ spark.sql(
+ "insert overwrite table `my_db1`.`chain_test` partition (region = 'US', biz_type = '1', dt = '20250814') values (15, 2, '1-1' ),(16, 2, '1-1' );");
+
+ /** Write snapshot branch */
+ spark.sql("set spark.paimon.branch=snapshot;");
+ spark.sql(
+ "insert overwrite table `my_db1`.`chain_test` partition (region = 'CN', biz_type = '0', dt = '20250810') values (1, 2, '1-1'),(2, 1, '1'),(3, 1, '1');");
+ spark.sql(
+ "insert overwrite table `my_db1`.`chain_test` partition (region = 'CN', biz_type = '0', dt = '20250812') values (1, 2, '1-1'),(2, 2, '1-1'),(3, 2, '1-1'), (4, 2, '1-1');");
+ spark.sql(
+ "insert overwrite table `my_db1`.`chain_test` partition (region = 'CN', biz_type = '0', dt = '20250814') values (1, 2, '1-1'),(2, 2, '1-1'),(3, 2, '1-1'), (4, 2, '1-1'), (5, 1, '1' ), (6, 1, '1');");
+
+ spark.sql(
+ "insert overwrite table `my_db1`.`chain_test` partition (region = 'CN', biz_type = '1', dt = '20250810') values (1, 2, '1-1'),(2, 1, '1'),(3, 1, '1');");
+ spark.sql(
+ "insert overwrite table `my_db1`.`chain_test` partition (region = 'CN', biz_type = '1', dt = '20250812') values (1, 2, '1-1'),(2, 2, '1-1'),(3, 2, '1-1'), (4, 2, '1-1');");
+ spark.sql(
+ "insert overwrite table `my_db1`.`chain_test` partition (region = 'CN', biz_type = '1', dt = '20250814') values (1, 2, '1-1'),(2, 2, '1-1'),(3, 2, '1-1'), (4, 2, '1-1'), (5, 1, '1' ), (6, 1, '1');");
+
+ spark.sql(
+ "insert overwrite table `my_db1`.`chain_test` partition (region = 'US', biz_type = '1', dt = '20250810') values (11, 2, '1-1'),(12, 1, '1'),(13, 1, '1');");
+ spark.sql(
+ "insert overwrite table `my_db1`.`chain_test` partition (region = 'US', biz_type = '1', dt = '20250812') values (11, 2, '1-1'),(12, 2, '1-1'),(13, 2, '1-1'), (14, 2, '1-1');");
+ spark.sql(
+ "insert overwrite table `my_db1`.`chain_test` partition (region = 'US', biz_type = '1', dt = '20250814') values (11, 2, '1-1'),(12, 2, '1-1'),(13, 2, '1-1'), (14, 2, '1-1'), (15, 1, '1' ), (16, 1, '1');");
+
+ spark.close();
+ spark = builder.getOrCreate();
+ /** Main read */
+ assertThat(
+ spark.sql("SELECT * FROM `my_db1`.`chain_test` where dt = '20250810'")
+ .collectAsList().stream()
+ .map(Row::toString)
+ .collect(Collectors.toList()))
+ .containsExactlyInAnyOrder(
+ "[1,1,1,CN,0,20250810]",
+ "[2,1,1,CN,0,20250810]",
+ "[1,1,1,CN,1,20250810]",
+ "[2,1,1,CN,1,20250810]",
+ "[11,1,1,US,1,20250810]",
+ "[12,1,1,US,1,20250810]");
+
+ /** Snapshot read */
+ assertThat(
+ spark.sql("SELECT * FROM `my_db1`.`chain_test` where dt = '20250814'")
+ .collectAsList().stream()
+ .map(Row::toString)
+ .collect(Collectors.toList()))
+ .containsExactlyInAnyOrder(
+ "[1,2,1-1,CN,0,20250814]",
+ "[2,2,1-1,CN,0,20250814]",
+ "[3,2,1-1,CN,0,20250814]",
+ "[4,2,1-1,CN,0,20250814]",
+ "[5,1,1,CN,0,20250814]",
+ "[6,1,1,CN,0,20250814]",
+ "[1,2,1-1,CN,1,20250814]",
+ "[2,2,1-1,CN,1,20250814]",
+ "[3,2,1-1,CN,1,20250814]",
+ "[4,2,1-1,CN,1,20250814]",
+ "[5,1,1,CN,1,20250814]",
+ "[6,1,1,CN,1,20250814]",
+ "[11,2,1-1,US,1,20250814]",
+ "[12,2,1-1,US,1,20250814]",
+ "[13,2,1-1,US,1,20250814]",
+ "[14,2,1-1,US,1,20250814]",
+ "[15,1,1,US,1,20250814]",
+ "[16,1,1,US,1,20250814]");
+
+ assertThat(
+ spark
+ .sql(
+ "SELECT * FROM `my_db1`.`chain_test` where region = 'CN' and biz_type = '0' and dt = '20250814'")
+ .collectAsList().stream()
+ .map(Row::toString)
+ .collect(Collectors.toList()))
+ .containsExactlyInAnyOrder(
+ "[1,2,1-1,CN,0,20250814]",
+ "[2,2,1-1,CN,0,20250814]",
+ "[3,2,1-1,CN,0,20250814]",
+ "[4,2,1-1,CN,0,20250814]",
+ "[5,1,1,CN,0,20250814]",
+ "[6,1,1,CN,0,20250814]");
+
+ assertThat(
+ spark
+ .sql(
+ "SELECT * FROM `my_db1`.`chain_test` where region = 'CN' and biz_type = '1' and dt = '20250814'")
+ .collectAsList().stream()
+ .map(Row::toString)
+ .collect(Collectors.toList()))
+ .containsExactlyInAnyOrder(
+ "[1,2,1-1,CN,1,20250814]",
+ "[2,2,1-1,CN,1,20250814]",
+ "[3,2,1-1,CN,1,20250814]",
+ "[4,2,1-1,CN,1,20250814]",
+ "[5,1,1,CN,1,20250814]",
+ "[6,1,1,CN,1,20250814]");
+
+ assertThat(
+ spark
+ .sql(
+ "SELECT * FROM `my_db1`.`chain_test` where region = 'US' and dt = '20250814'")
+ .collectAsList().stream()
+ .map(Row::toString)
+ .collect(Collectors.toList()))
+ .containsExactlyInAnyOrder(
+ "[11,2,1-1,US,1,20250814]",
+ "[12,2,1-1,US,1,20250814]",
+ "[13,2,1-1,US,1,20250814]",
+ "[14,2,1-1,US,1,20250814]",
+ "[15,1,1,US,1,20250814]",
+ "[16,1,1,US,1,20250814]");
+
+ /** Chain read */
+ /** 1. non pre snapshot */
+ assertThat(
+ spark.sql("SELECT * FROM `my_db1`.`chain_test` where dt = '20250809'")
+ .collectAsList().stream()
+ .map(Row::toString)
+ .collect(Collectors.toList()))
+ .containsExactlyInAnyOrder(
+ "[1,1,1,CN,0,20250809]",
+ "[2,1,1,CN,0,20250809]",
+ "[1,1,1,CN,1,20250809]",
+ "[2,1,1,CN,1,20250809]",
+ "[11,1,1,US,1,20250809]",
+ "[12,1,1,US,1,20250809]");
+
+ assertThat(
+ spark
+ .sql(
+ "SELECT * FROM `my_db1`.`chain_test` where region = 'CN' and biz_type = '0' and dt = '20250809'")
+ .collectAsList().stream()
+ .map(Row::toString)
+ .collect(Collectors.toList()))
+ .containsExactlyInAnyOrder("[1,1,1,CN,0,20250809]", "[2,1,1,CN,0,20250809]");
+
+ assertThat(
+ spark
+ .sql(
+ "SELECT * FROM `my_db1`.`chain_test` where region = 'CN' and biz_type = '1' and dt = '20250809'")
+ .collectAsList().stream()
+ .map(Row::toString)
+ .collect(Collectors.toList()))
+ .containsExactlyInAnyOrder("[1,1,1,CN,1,20250809]", "[2,1,1,CN,1,20250809]");
+
+ assertThat(
+ spark
+ .sql(
+ "SELECT * FROM `my_db1`.`chain_test` where region = 'US' and dt = '20250809'")
+ .collectAsList().stream()
+ .map(Row::toString)
+ .collect(Collectors.toList()))
+ .containsExactlyInAnyOrder("[11,1,1,US,1,20250809]", "[12,1,1,US,1,20250809]");
+
+ /** 2. has pre snapshot */
+ assertThat(
+ spark.sql("SELECT * FROM `my_db1`.`chain_test` where dt = '20250811'")
+ .collectAsList().stream()
+ .map(Row::toString)
+ .collect(Collectors.toList()))
+ .containsExactlyInAnyOrder(
+ "[1,2,1-1,CN,0,20250811]",
+ "[2,2,1-1,CN,0,20250811]",
+ "[3,1,1,CN,0,20250811]",
+ "[4,1,1,CN,0,20250811]",
+ "[1,2,1-1,CN,1,20250811]",
+ "[2,2,1-1,CN,1,20250811]",
+ "[3,1,1,CN,1,20250811]",
+ "[4,1,1,CN,1,20250811]",
+ "[11,2,1-1,US,1,20250811]",
+ "[12,2,1-1,US,1,20250811]",
+ "[13,1,1,US,1,20250811]",
+ "[14,1,1,US,1,20250811]");
+
+ assertThat(
+ spark
+ .sql(
+ "SELECT * FROM `my_db1`.`chain_test` where region = 'CN' and biz_type = '0' and dt = '20250811'")
+ .collectAsList().stream()
+ .map(Row::toString)
+ .collect(Collectors.toList()))
+ .containsExactlyInAnyOrder(
+ "[1,2,1-1,CN,0,20250811]",
+ "[2,2,1-1,CN,0,20250811]",
+ "[3,1,1,CN,0,20250811]",
+ "[4,1,1,CN,0,20250811]");
+
+ assertThat(
+ spark
+ .sql(
+ "SELECT * FROM `my_db1`.`chain_test` where region = 'CN' and biz_type = '1' and dt = '20250811'")
+ .collectAsList().stream()
+ .map(Row::toString)
+ .collect(Collectors.toList()))
+ .containsExactlyInAnyOrder(
+ "[1,2,1-1,CN,1,20250811]",
+ "[2,2,1-1,CN,1,20250811]",
+ "[3,1,1,CN,1,20250811]",
+ "[4,1,1,CN,1,20250811]");
+
+ assertThat(
+ spark
+ .sql(
+ "SELECT * FROM `my_db1`.`chain_test` where region = 'US' and dt = '20250811'")
+ .collectAsList().stream()
+ .map(Row::toString)
+ .collect(Collectors.toList()))
+ .containsExactlyInAnyOrder(
+ "[11,2,1-1,US,1,20250811]",
+ "[12,2,1-1,US,1,20250811]",
+ "[13,1,1,US,1,20250811]",
+ "[14,1,1,US,1,20250811]");
+
+ /** Multi partition Read */
+ assertThat(
+ spark
+ .sql(
+ "SELECT * FROM `my_db1`.`chain_test` where dt in ('20250810', '20250811', '20250812');")
+ .collectAsList().stream()
+ .map(Row::toString)
+ .collect(Collectors.toList()))
+ .containsExactlyInAnyOrder(
+ "[1,1,1,CN,0,20250810]",
+ "[2,1,1,CN,0,20250810]",
+ "[1,2,1-1,CN,0,20250811]",
+ "[2,2,1-1,CN,0,20250811]",
+ "[3,1,1,CN,0,20250811]",
+ "[4,1,1,CN,0,20250811]",
+ "[1,2,1-1,CN,0,20250812]",
+ "[2,2,1-1,CN,0,20250812]",
+ "[3,2,1-1,CN,0,20250812]",
+ "[4,2,1-1,CN,0,20250812]",
+ "[1,1,1,CN,1,20250810]",
+ "[2,1,1,CN,1,20250810]",
+ "[1,2,1-1,CN,1,20250811]",
+ "[2,2,1-1,CN,1,20250811]",
+ "[3,1,1,CN,1,20250811]",
+ "[4,1,1,CN,1,20250811]",
+ "[1,2,1-1,CN,1,20250812]",
+ "[2,2,1-1,CN,1,20250812]",
+ "[3,2,1-1,CN,1,20250812]",
+ "[4,2,1-1,CN,1,20250812]",
+ "[11,1,1,US,1,20250810]",
+ "[12,1,1,US,1,20250810]",
+ "[11,2,1-1,US,1,20250811]",
+ "[12,2,1-1,US,1,20250811]",
+ "[13,1,1,US,1,20250811]",
+ "[14,1,1,US,1,20250811]",
+ "[11,2,1-1,US,1,20250812]",
+ "[12,2,1-1,US,1,20250812]",
+ "[13,2,1-1,US,1,20250812]",
+ "[14,2,1-1,US,1,20250812]");
+
+ assertThat(
+ spark
+ .sql(
+ "SELECT * FROM `my_db1`.`chain_test` where region = 'CN' and biz_type = '0' and dt in ('20250810', '20250811', '20250812');")
+ .collectAsList().stream()
+ .map(Row::toString)
+ .collect(Collectors.toList()))
+ .containsExactlyInAnyOrder(
+ "[1,1,1,CN,0,20250810]",
+ "[2,1,1,CN,0,20250810]",
+ "[1,2,1-1,CN,0,20250811]",
+ "[2,2,1-1,CN,0,20250811]",
+ "[3,1,1,CN,0,20250811]",
+ "[4,1,1,CN,0,20250811]",
+ "[1,2,1-1,CN,0,20250812]",
+ "[2,2,1-1,CN,0,20250812]",
+ "[3,2,1-1,CN,0,20250812]",
+ "[4,2,1-1,CN,0,20250812]");
+
+ assertThat(
+ spark
+ .sql(
+ "SELECT * FROM `my_db1`.`chain_test` where region = 'CN' and biz_type = '1' and dt in ('20250810', '20250811', '20250812');")
+ .collectAsList().stream()
+ .map(Row::toString)
+ .collect(Collectors.toList()))
+ .containsExactlyInAnyOrder(
+ "[1,1,1,CN,1,20250810]",
+ "[2,1,1,CN,1,20250810]",
+ "[1,2,1-1,CN,1,20250811]",
+ "[2,2,1-1,CN,1,20250811]",
+ "[3,1,1,CN,1,20250811]",
+ "[4,1,1,CN,1,20250811]",
+ "[1,2,1-1,CN,1,20250812]",
+ "[2,2,1-1,CN,1,20250812]",
+ "[3,2,1-1,CN,1,20250812]",
+ "[4,2,1-1,CN,1,20250812]");
+
+ assertThat(
+ spark
+ .sql(
+ "SELECT * FROM `my_db1`.`chain_test` where region = 'US' and dt in ('20250810', '20250811', '20250812');")
+ .collectAsList().stream()
+ .map(Row::toString)
+ .collect(Collectors.toList()))
+ .containsExactlyInAnyOrder(
+ "[11,1,1,US,1,20250810]",
+ "[12,1,1,US,1,20250810]",
+ "[11,2,1-1,US,1,20250811]",
+ "[12,2,1-1,US,1,20250811]",
+ "[13,1,1,US,1,20250811]",
+ "[14,1,1,US,1,20250811]",
+ "[11,2,1-1,US,1,20250812]",
+ "[12,2,1-1,US,1,20250812]",
+ "[13,2,1-1,US,1,20250812]",
+ "[14,2,1-1,US,1,20250812]");
+
+ /** Incremental read */
+ assertThat(
+ spark
+ .sql(
+ "SELECT * FROM `my_db1`.`chain_test$branch_delta` where dt = '20250811'")
+ .collectAsList().stream()
+ .map(Row::toString)
+ .collect(Collectors.toList()))
+ .containsExactlyInAnyOrder(
+ "[2,2,1-1,CN,0,20250811]",
+ "[4,1,1,CN,0,20250811]",
+ "[2,2,1-1,CN,1,20250811]",
+ "[4,1,1,CN,1,20250811]",
+ "[12,2,1-1,US,1,20250811]",
+ "[14,1,1,US,1,20250811]");
+
+ assertThat(
+ spark
+ .sql(
+ "SELECT * FROM `my_db1`.`chain_test$branch_delta` where region = 'CN' and biz_type = '0' and dt = '20250811'")
+ .collectAsList().stream()
+ .map(Row::toString)
+ .collect(Collectors.toList()))
+ .containsExactlyInAnyOrder("[2,2,1-1,CN,0,20250811]", "[4,1,1,CN,0,20250811]");
+
+ assertThat(
+ spark
+ .sql(
+ "SELECT * FROM `my_db1`.`chain_test$branch_delta` where region = 'CN' and biz_type = '1' and dt = '20250811'")
+ .collectAsList().stream()
+ .map(Row::toString)
+ .collect(Collectors.toList()))
+ .containsExactlyInAnyOrder("[2,2,1-1,CN,1,20250811]", "[4,1,1,CN,1,20250811]");
+
+ assertThat(
+ spark
+ .sql(
+ "SELECT * FROM `my_db1`.`chain_test$branch_delta` where region = 'US' and dt = '20250811'")
+ .collectAsList().stream()
+ .map(Row::toString)
+ .collect(Collectors.toList()))
+ .containsExactlyInAnyOrder("[12,2,1-1,US,1,20250811]", "[14,1,1,US,1,20250811]");
+
+ /** Multi partition incremental read */
+ assertThat(
+ spark
+ .sql(
+ "SELECT * FROM `my_db1`.`chain_test$branch_delta` where dt in ('20250810', '20250811', '20250812');")
+ .collectAsList().stream()
+ .map(Row::toString)
+ .collect(Collectors.toList()))
+ .containsExactlyInAnyOrder(
+ "[1,2,1-1,CN,0,20250810]",
+ "[3,1,1,CN,0,20250810]",
+ "[2,2,1-1,CN,0,20250811]",
+ "[4,1,1,CN,0,20250811]",
+ "[3,2,1-1,CN,0,20250812]",
+ "[4,2,1-1,CN,0,20250812]",
+ "[1,2,1-1,CN,1,20250810]",
+ "[3,1,1,CN,1,20250810]",
+ "[2,2,1-1,CN,1,20250811]",
+ "[4,1,1,CN,1,20250811]",
+ "[3,2,1-1,CN,1,20250812]",
+ "[4,2,1-1,CN,1,20250812]",
+ "[11,2,1-1,US,1,20250810]",
+ "[13,1,1,US,1,20250810]",
+ "[12,2,1-1,US,1,20250811]",
+ "[14,1,1,US,1,20250811]",
+ "[13,2,1-1,US,1,20250812]",
+ "[14,2,1-1,US,1,20250812]");
+
+ assertThat(
+ spark
+ .sql(
+ "SELECT * FROM `my_db1`.`chain_test$branch_delta` where region = 'CN' and biz_type = '0' and dt in ('20250810', '20250811', '20250812');")
+ .collectAsList().stream()
+ .map(Row::toString)
+ .collect(Collectors.toList()))
+ .containsExactlyInAnyOrder(
+ "[1,2,1-1,CN,0,20250810]",
+ "[3,1,1,CN,0,20250810]",
+ "[2,2,1-1,CN,0,20250811]",
+ "[4,1,1,CN,0,20250811]",
+ "[3,2,1-1,CN,0,20250812]",
+ "[4,2,1-1,CN,0,20250812]");
+
+ assertThat(
+ spark
+ .sql(
+ "SELECT * FROM `my_db1`.`chain_test$branch_delta` where region = 'CN' and biz_type = '1' and dt in ('20250810', '20250811', '20250812');")
+ .collectAsList().stream()
+ .map(Row::toString)
+ .collect(Collectors.toList()))
+ .containsExactlyInAnyOrder(
+ "[1,2,1-1,CN,1,20250810]",
+ "[3,1,1,CN,1,20250810]",
+ "[2,2,1-1,CN,1,20250811]",
+ "[4,1,1,CN,1,20250811]",
+ "[3,2,1-1,CN,1,20250812]",
+ "[4,2,1-1,CN,1,20250812]");
+
+ assertThat(
+ spark
+ .sql(
+ "SELECT * FROM `my_db1`.`chain_test$branch_delta` where region = 'US' and dt in ('20250810', '20250811', '20250812');")
+ .collectAsList().stream()
+ .map(Row::toString)
+ .collect(Collectors.toList()))
+ .containsExactlyInAnyOrder(
+ "[11,2,1-1,US,1,20250810]",
+ "[13,1,1,US,1,20250810]",
+ "[12,2,1-1,US,1,20250811]",
+ "[14,1,1,US,1,20250811]",
+ "[13,2,1-1,US,1,20250812]",
+ "[14,2,1-1,US,1,20250812]");
+
+ spark.close();
+ spark = builder.getOrCreate();
+ spark.sql("set spark.paimon.branch=delta;");
+ spark.sql(
+ "insert overwrite table `my_db1`.`chain_test` values (5, 2, '1', 'CN', '0', '20250813'),(6, 2, '1', 'CN', '0', '20250814'), (15, 2, '1', 'US', '1', '20250813'),(16, 2, '1', 'US', '1', '20250814');");
+
+ spark.close();
+ spark = builder.getOrCreate();
+ Dataset<Row> df =
+ spark.sql(
+ "SELECT t1,t2,t3 FROM `my_db1`.`chain_test$branch_snapshot` where dt = '20250814'");
+ assertThat(df.count()).isEqualTo(0);
+ df =
+ spark.sql(
+ "SELECT t1,t2,t3 FROM `my_db1`.`chain_test$branch_delta` where dt = '20250814'");
+ assertThat(df.count()).isEqualTo(2);
+
+ spark.close();
+ spark = builder.getOrCreate();
+ /** Drop table */
+ spark.sql("DROP TABLE IF EXISTS `my_db1`.`chain_test`;");
+
+ spark.close();
+ }
}
(paimon) 16/37: [doc] Modify scenario-guide to changelog-producer
Posted by ho...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hope pushed a commit to branch release-1.4
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit 94222cffafa7c3681df3dfcccd3ac318899108b5
Author: JingsongLi <ji...@gmail.com>
AuthorDate: Thu Mar 26 18:05:50 2026 +0800
[doc] Modify scenario-guide to changelog-producer
---
docs/content/learn-paimon/scenario-guide.md | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/docs/content/learn-paimon/scenario-guide.md b/docs/content/learn-paimon/scenario-guide.md
index 9cb7c76bb6..9fe0f13dff 100644
--- a/docs/content/learn-paimon/scenario-guide.md
+++ b/docs/content/learn-paimon/scenario-guide.md
@@ -83,8 +83,10 @@ CREATE TABLE orders (
This mode gives you the best balance of write and read performance. Compared to the default MOR mode, MOW
avoids merging at read time, which greatly improves OLAP query performance.
- **`changelog-producer = lookup`**: Generates a complete [changelog]({{< ref "primary-key-table/changelog-producer#lookup" >}})
- for downstream streaming consumers. If no downstream streaming read is needed, you can omit this to save
- compaction resources.
+ for downstream streaming consumers. If your CDC source is directly connected to a database (e.g., MySQL CDC, Postgres CDC),
+ you can use `changelog-producer = input` instead, since the database CDC stream already provides a complete changelog.
+ However, if your CDC source comes from Kafka (or other message queues), `input` may not be reliable — use `lookup` to
+ ensure changelog correctness. If no downstream streaming read is needed, you can omit this to save compaction resources.
- **`sequence.field = update_time`**: Guarantees correct update ordering even when data arrives out of order.
- **Bucketing**: Use the default Dynamic Bucket (`bucket = -1`). The system automatically adjusts bucket count based
on data volume. If you are sensitive to data visibility latency, set a fixed bucket number (e.g. `'bucket' = '5'`)
(paimon) 30/37: [Python] Fix data evolution read IndexError when file has no write_cols (#7556)
Posted by ho...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hope pushed a commit to branch release-1.4
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit cb59e904aec313fe02b465ddd6285689abcf5866
Author: XiaoHongbo <13...@qq.com>
AuthorDate: Mon Mar 30 20:40:57 2026 +0800
[Python] Fix data evolution read IndexError when file has no write_cols (#7556)
When a data file has no write_cols (written with all columns before
schema evolution), `_create_union_reader` wrongly assumes it contains
all current table fields including columns added later, causing
IndexError on read. This PR fixes it.
---
paimon-python/pypaimon/read/split_read.py | 9 ++-
.../pypaimon/tests/data_evolution_test.py | 81 ++++++++++++++++++++++
2 files changed, 88 insertions(+), 2 deletions(-)
diff --git a/paimon-python/pypaimon/read/split_read.py b/paimon-python/pypaimon/read/split_read.py
index f9adee4068..25324bdcbb 100644
--- a/paimon-python/pypaimon/read/split_read.py
+++ b/paimon-python/pypaimon/read/split_read.py
@@ -609,8 +609,13 @@ class DataEvolutionSplitRead(SplitRead):
elif first_file.write_cols:
field_ids = self._get_field_ids_from_write_cols(first_file.write_cols)
else:
- # For regular files, get all field IDs from the schema
- field_ids = [field.id for field in self.table.fields]
+ # For regular files without write_cols, derive field IDs from
+ # the file's schema version, not the current table schema.
+ # The file only contains columns from when it was written.
+ file_schema = self.table.schema_manager.get_schema(first_file.schema_id)
+ field_ids = [field.id for field in file_schema.fields]
+ field_ids.append(SpecialFields.ROW_ID.id)
+ field_ids.append(SpecialFields.SEQUENCE_NUMBER.id)
read_fields = []
for j, read_field_id in enumerate(read_field_index):
diff --git a/paimon-python/pypaimon/tests/data_evolution_test.py b/paimon-python/pypaimon/tests/data_evolution_test.py
index 796625f6a4..266041eec5 100644
--- a/paimon-python/pypaimon/tests/data_evolution_test.py
+++ b/paimon-python/pypaimon/tests/data_evolution_test.py
@@ -1356,3 +1356,84 @@ class DataEvolutionTest(unittest.TestCase):
rebuilt = pa.RecordBatch.from_arrays(arrays, names=batch.schema.names)
self.assertTrue(rebuilt.schema.field('_ROW_ID').nullable)
self.assertTrue(rebuilt.schema.field('_SEQUENCE_NUMBER').nullable)
+
+ def test_read_full_schema_on_write_before_evolution(self):
+ from pypaimon.schema.schema_change import SchemaChange
+ from pypaimon.schema.data_types import AtomicType
+
+ # Step 1: Create table with [f0, f1]
+ initial_schema = pa.schema([
+ ('f0', pa.int32()),
+ ('f1', pa.string()),
+ ])
+ schema = Schema.from_pyarrow_schema(
+ initial_schema,
+ options={'row-tracking.enabled': 'true', 'data-evolution.enabled': 'true'},
+ )
+ table_name = 'default.test_no_write_cols_schema_evo'
+ self.catalog.create_table(table_name, schema, False)
+ table = self.catalog.get_table(table_name)
+
+ # Step 2: Write data with ALL columns of old schema → write_cols = None
+ write_builder = table.new_batch_write_builder()
+ table_write = write_builder.new_write()
+ table_commit = write_builder.new_commit()
+ table_write.write_arrow(pa.Table.from_pydict(
+ {'f0': [1, 2], 'f1': ['a', 'b']},
+ schema=initial_schema,
+ ))
+ cmts = table_write.prepare_commit()
+ for c in cmts:
+ for nf in c.new_files:
+ self.assertIsNone(nf.write_cols)
+ table_commit.commit(cmts)
+ table_write.close()
+ table_commit.close()
+
+ # Step 3: Schema evolution - add column f2
+ self.catalog.alter_table(
+ table_name,
+ [SchemaChange.add_column('f2', AtomicType('STRING'))],
+ )
+ table = self.catalog.get_table(table_name)
+
+ # Step 4: Write f2 only for same rows (first_row_id = 0)
+ write_builder = table.new_batch_write_builder()
+ table_write = write_builder.new_write().with_write_type(['f2'])
+ table_commit = write_builder.new_commit()
+ table_write.write_arrow(pa.Table.from_pydict(
+ {'f2': ['x', 'y']},
+ schema=pa.schema([('f2', pa.string())]),
+ ))
+ cmts_f2 = table_write.prepare_commit()
+ for c in cmts_f2:
+ for nf in c.new_files:
+ nf.first_row_id = 0
+ table_commit.commit(cmts_f2)
+ table_write.close()
+ table_commit.close()
+
+ # Step 5: Read all columns
+ read_builder = table.new_read_builder()
+ table_scan = read_builder.new_scan()
+ table_read = read_builder.new_read()
+ splits = table_scan.plan().splits()
+
+ for split in splits:
+ for f in split.files:
+ if f.write_cols is None:
+ f.max_sequence_number = 999999
+
+ actual = table_read.to_arrow(splits)
+
+ expect = pa.Table.from_pydict({
+ 'f0': [1, 2],
+ 'f1': ['a', 'b'],
+ 'f2': ['x', 'y'],
+ }, schema=pa.schema([
+ ('f0', pa.int32()),
+ ('f1', pa.string()),
+ ('f2', pa.string()),
+ ]))
+ self.assertEqual(actual.num_rows, 2)
+ self.assertEqual(actual, expect)
(paimon) 27/37: [python] Support PyJindo in pyarrow_file_io (#7410)
Posted by ho...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hope pushed a commit to branch release-1.4
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit c918c47f38d0ed7b17569e2ea0bf15c0d411c5b2
Author: timmyyao <19...@qq.com>
AuthorDate: Mon Mar 30 18:05:43 2026 +0800
[python] Support PyJindo in pyarrow_file_io (#7410)
Implement a filesystem for OSS backed by PyJindo
(https://aliyun.github.io/alibabacloud-jindodata/jindosdk/jindosdk_download/),
which provides higher performance and stability for visiting OSS.
---
paimon-python/pypaimon/common/options/config.py | 2 +
.../filesystem/jindo_file_system_handler.py | 283 +++++++++++
.../pypaimon/filesystem/pyarrow_file_io.py | 23 +-
paimon-python/pypaimon/tests/file_io_test.py | 8 +-
.../pypaimon/tests/jindo_file_system_test.py | 524 +++++++++++++++++++++
paimon-python/pypaimon/tests/oss_file_io_test.py | 362 ++++++++++++++
6 files changed, 1198 insertions(+), 4 deletions(-)
diff --git a/paimon-python/pypaimon/common/options/config.py b/paimon-python/pypaimon/common/options/config.py
index fb6a46446e..8b0230da64 100644
--- a/paimon-python/pypaimon/common/options/config.py
+++ b/paimon-python/pypaimon/common/options/config.py
@@ -18,6 +18,8 @@ from pypaimon.common.options.config_options import ConfigOptions
class OssOptions:
+ OSS_IMPL = ConfigOptions.key("fs.oss.impl").string_type().default_value("default").with_description(
+ "OSS filesystem implementation: default or jindo")
OSS_ACCESS_KEY_ID = ConfigOptions.key("fs.oss.accessKeyId").string_type().no_default_value().with_description(
"OSS access key ID")
OSS_ACCESS_KEY_SECRET = ConfigOptions.key(
diff --git a/paimon-python/pypaimon/filesystem/jindo_file_system_handler.py b/paimon-python/pypaimon/filesystem/jindo_file_system_handler.py
new file mode 100644
index 0000000000..693416cc57
--- /dev/null
+++ b/paimon-python/pypaimon/filesystem/jindo_file_system_handler.py
@@ -0,0 +1,283 @@
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+import logging
+
+import pyarrow as pa
+from pyarrow import PythonFile
+from pyarrow._fs import FileSystemHandler
+from pyarrow.fs import FileInfo, FileSelector, FileType
+
+try:
+ import pyjindo.fs as jfs
+ import pyjindo.util as jutil
+ JINDO_AVAILABLE = True
+except ImportError:
+ JINDO_AVAILABLE = False
+ jfs = None
+ jutil = None
+
+from pypaimon.common.options import Options
+from pypaimon.common.options.config import OssOptions
+
+
+class JindoInputFile:
+ def __init__(self, jindo_stream):
+ self._stream = jindo_stream
+ self._closed = False
+
+ @property
+ def closed(self):
+ if hasattr(self._stream, 'closed'):
+ return self._stream.closed
+ return self._closed
+
+ def read(self, nbytes: int = -1):
+ if self.closed:
+ raise ValueError("I/O operation on closed file")
+ if nbytes is None or nbytes < 0:
+ return self._stream.read()
+ return self._stream.read(nbytes)
+
+ def seek(self, position: int, whence: int = 0):
+ if self.closed:
+ raise ValueError("I/O operation on closed file")
+ self._stream.seek(position, whence)
+
+ def tell(self) -> int:
+ if self.closed:
+ raise ValueError("I/O operation on closed file")
+ return self._stream.tell()
+
+ def read_at(self, nbytes: int, offset: int):
+ if self.closed:
+ raise ValueError("I/O operation on closed file")
+ return self._stream.pread(nbytes, offset)
+
+ def close(self):
+ if not self._closed:
+ self._stream.close()
+ self._closed = True
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, exc_type, exc_val, exc_tb):
+ self.close()
+ return False
+
+
+class JindoOutputFile:
+ def __init__(self, jindo_stream):
+ self._stream = jindo_stream
+ self._closed = False
+
+ @property
+ def closed(self):
+ if hasattr(self._stream, 'closed'):
+ return self._stream.closed
+ return self._closed
+
+ def write(self, data: bytes) -> int:
+ if self.closed:
+ raise ValueError("I/O operation on closed file")
+ if isinstance(data, pa.Buffer):
+ data = data.to_pybytes()
+ elif not isinstance(data, bytes):
+ raise TypeError("Unsupported data type")
+ return self._stream.write(data)
+
+ def flush(self):
+ if self.closed:
+ raise ValueError("I/O operation on closed file")
+ if hasattr(self._stream, 'flush'):
+ self._stream.flush()
+
+ def close(self):
+ if not self._closed:
+ self._stream.close()
+ self._closed = True
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, exc_type, exc_val, exc_tb):
+ self.close()
+ return False
+
+
+class JindoFileSystemHandler(FileSystemHandler):
+ def __init__(self, root_path: str, catalog_options: Options):
+ if not JINDO_AVAILABLE:
+ raise ImportError("Module pyjindo is not available. Please install pyjindosdk.")
+
+ self.logger = logging.getLogger(__name__)
+ self.root_path = root_path
+ self.properties = catalog_options
+
+ # Build jindo config from catalog_options
+ config = jutil.Config()
+
+ access_key_id = catalog_options.get(OssOptions.OSS_ACCESS_KEY_ID)
+ access_key_secret = catalog_options.get(OssOptions.OSS_ACCESS_KEY_SECRET)
+ security_token = catalog_options.get(OssOptions.OSS_SECURITY_TOKEN)
+ endpoint = catalog_options.get(OssOptions.OSS_ENDPOINT)
+ region = catalog_options.get(OssOptions.OSS_REGION)
+
+ if access_key_id:
+ config.set("fs.oss.accessKeyId", access_key_id)
+ if access_key_secret:
+ config.set("fs.oss.accessKeySecret", access_key_secret)
+ if security_token:
+ config.set("fs.oss.securityToken", security_token)
+ if endpoint:
+ endpoint_clean = endpoint.replace('http://', '').replace('https://', '')
+ config.set("fs.oss.endpoint", endpoint_clean)
+ if region:
+ config.set("fs.oss.region", region)
+ config.set("fs.oss.user.agent.features", "pypaimon")
+
+ self._jindo_fs = jfs.connect(self.root_path, "root", config)
+
+ def __eq__(self, other):
+ if isinstance(other, JindoFileSystemHandler):
+ return self.root_path == other.root_path
+ return NotImplemented
+
+ def __ne__(self, other):
+ if isinstance(other, JindoFileSystemHandler):
+ return not self.__eq__(other)
+ return NotImplemented
+
+ def _normalize_path(self, path: str) -> str:
+ if path.startswith('oss://'):
+ return path
+
+ if not path or path == '.':
+ return self.root_path.rstrip('/') + '/'
+
+ path_clean = path.lstrip('/')
+ return self.root_path.rstrip('/') + '/' + path_clean
+
+ def _convert_file_type(self, jindo_type) -> FileType:
+ if jindo_type == jfs.FileType.File:
+ return FileType.File
+ elif jindo_type == jfs.FileType.Directory:
+ return FileType.Directory
+ else:
+ return FileType.Unknown
+
+ def _convert_file_info(self, jindo_info) -> FileInfo:
+ pa_type = self._convert_file_type(jindo_info.type)
+ return FileInfo(
+ path=jindo_info.path,
+ type=pa_type,
+ size=jindo_info.size if jindo_info.type == jfs.FileType.File else None,
+ mtime=jindo_info.mtime if hasattr(jindo_info, 'mtime') else None,
+ )
+
+ def get_type_name(self) -> str:
+ return "jindo"
+
+ def get_file_info(self, paths) -> list:
+ infos = []
+ for path in paths:
+ normalized = self._normalize_path(path)
+ try:
+ jindo_info = self._jindo_fs.get_file_info(normalized)
+ infos.append(self._convert_file_info(jindo_info))
+ except FileNotFoundError:
+ infos.append(FileInfo(normalized, FileType.NotFound))
+ return infos
+
+ def get_file_info_selector(self, selector: FileSelector) -> list:
+ normalized = self._normalize_path(selector.base_dir)
+ try:
+ items = self._jindo_fs.listdir(normalized, recursive=selector.recursive)
+ return [self._convert_file_info(item) for item in items]
+ except FileNotFoundError:
+ if selector.allow_not_found:
+ return []
+ raise
+
+ def create_dir(self, path: str, recursive: bool):
+ normalized = self._normalize_path(path)
+ self._jindo_fs.mkdir(normalized)
+
+ def delete_dir(self, path: str):
+ normalized = self._normalize_path(path)
+ self._jindo_fs.remove(normalized)
+
+ def delete_dir_contents(self, path: str, missing_dir_ok: bool = False):
+ normalized = self._normalize_path(path)
+ if normalized == self.root_path:
+ raise ValueError(
+ "delete_dir_contents() does not accept root path"
+ )
+ self._delete_dir_contents(path, missing_dir_ok)
+
+ def delete_root_dir_contents(self):
+ self._delete_dir_contents("/", missing_dir_ok=False)
+
+ def _delete_dir_contents(self, path: str, missing_dir_ok: bool):
+ normalized = self._normalize_path(path)
+ try:
+ items = self._jindo_fs.listdir(normalized, recursive=False)
+ except FileNotFoundError:
+ if missing_dir_ok:
+ return
+ raise
+ except Exception as e:
+ self.logger.warning(f"Error listing {path}: {e}")
+ raise
+ for item in items:
+ self._jindo_fs.remove(item.path)
+
+ def delete_file(self, path: str):
+ normalized = self._normalize_path(path)
+ self._jindo_fs.remove(normalized)
+
+ def move(self, src: str, dest: str):
+ src_norm = self._normalize_path(src)
+ dst_norm = self._normalize_path(dest)
+ self._jindo_fs.rename(src_norm, dst_norm)
+
+ def copy_file(self, src: str, dest: str):
+ src_norm = self._normalize_path(src)
+ dst_norm = self._normalize_path(dest)
+ self._jindo_fs.copy_file(src_norm, dst_norm)
+
+ def open_input_stream(self, path: str):
+ normalized = self._normalize_path(path)
+ jindo_stream = self._jindo_fs.open(normalized, "rb")
+ return PythonFile(JindoInputFile(jindo_stream), mode="r")
+
+ def open_input_file(self, path: str):
+ normalized = self._normalize_path(path)
+ jindo_stream = self._jindo_fs.open(normalized, "rb")
+ return PythonFile(JindoInputFile(jindo_stream), mode="r")
+
+ def open_output_stream(self, path: str, metadata):
+ normalized = self._normalize_path(path)
+ jindo_stream = self._jindo_fs.open(normalized, "wb")
+ return PythonFile(JindoOutputFile(jindo_stream), mode="w")
+
+ def open_append_stream(self, path: str, metadata):
+ raise IOError("append mode is not supported")
+
+ def normalize_path(self, path: str) -> str:
+ return self._normalize_path(path)
diff --git a/paimon-python/pypaimon/filesystem/pyarrow_file_io.py b/paimon-python/pypaimon/filesystem/pyarrow_file_io.py
index 18e54a0b70..6cf2faabb2 100644
--- a/paimon-python/pypaimon/filesystem/pyarrow_file_io.py
+++ b/paimon-python/pypaimon/filesystem/pyarrow_file_io.py
@@ -34,6 +34,7 @@ from pypaimon.common.file_io import FileIO
from pypaimon.common.options import Options
from pypaimon.common.options.config import OssOptions, S3Options
from pypaimon.common.uri_reader import UriReaderFactory
+from pypaimon.filesystem.jindo_file_system_handler import JindoFileSystemHandler
from pypaimon.schema.data_types import (AtomicType, DataField,
PyarrowFieldParser)
from pypaimon.table.row.blob import Blob, BlobData, BlobDescriptor
@@ -56,9 +57,13 @@ class PyArrowFileIO(FileIO):
self.uri_reader_factory = UriReaderFactory(catalog_options)
self._is_oss = scheme in {"oss"}
self._oss_bucket = None
+ self._oss_impl = self.properties.get(OssOptions.OSS_IMPL)
if self._is_oss:
self._oss_bucket = self._extract_oss_bucket(path)
- self.filesystem = self._initialize_oss_fs(path)
+ if self._oss_impl == "jindo":
+ self.filesystem = self._initialize_jindo_fs(path)
+ else:
+ self.filesystem = self._initialize_oss_fs(path)
elif scheme in {"s3", "s3a", "s3n"}:
self.filesystem = self._initialize_s3_fs()
elif scheme in {"hdfs", "viewfs"}:
@@ -116,6 +121,13 @@ class PyArrowFileIO(FileIO):
raise ValueError("Invalid OSS URI without bucket: {}".format(location))
return bucket
+ def _initialize_jindo_fs(self, path) -> FileSystem:
+ """Initialize JindoFileSystem for OSS access."""
+ self.logger.info(f"Initializing JindoFileSystem for OSS access: {path}")
+ root_path = f"oss://{self._oss_bucket}/"
+ fs_handler = JindoFileSystemHandler(root_path, self.properties)
+ return pafs.PyFileSystem(fs_handler)
+
def _initialize_oss_fs(self, path) -> FileSystem:
if self.properties.get(OssOptions.OSS_ACCESS_KEY_ID):
# When explicit credentials are provided, disable the EC2 Instance Metadata
@@ -198,7 +210,9 @@ class PyArrowFileIO(FileIO):
def new_output_stream(self, path: str):
path_str = self.to_filesystem_path(path)
- if self._is_oss and not self._pyarrow_gte_7:
+ if self._oss_impl == "jindo":
+ pass
+ elif self._is_oss and not self._pyarrow_gte_7:
# For PyArrow 6.x + OSS, path_str is already just the key part
if '/' in path_str:
parent_dir = '/'.join(path_str.split('/')[:-1])
@@ -560,6 +574,11 @@ class PyArrowFileIO(FileIO):
path_part = normalized_path.lstrip('/')
return f"{drive_letter}:/{path_part}" if path_part else f"{drive_letter}:"
+ if self._oss_impl == "jindo":
+ # For JindoFileSystem, pass key only
+ path_part = normalized_path.lstrip('/')
+ return path_part if path_part else '.'
+
if isinstance(self.filesystem, S3FileSystem):
if parsed.scheme:
if parsed.netloc:
diff --git a/paimon-python/pypaimon/tests/file_io_test.py b/paimon-python/pypaimon/tests/file_io_test.py
index ba91f94a18..5cc4d7a821 100644
--- a/paimon-python/pypaimon/tests/file_io_test.py
+++ b/paimon-python/pypaimon/tests/file_io_test.py
@@ -67,7 +67,9 @@ class FileIOTest(unittest.TestCase):
lt7 = _pyarrow_lt_7()
oss_io = PyArrowFileIO("oss://test-bucket/warehouse", Options({
- OssOptions.OSS_ENDPOINT.key(): 'oss-cn-hangzhou.aliyuncs.com'
+ OssOptions.OSS_ENDPOINT.key(): 'oss-cn-hangzhou.aliyuncs.com',
+ OssOptions.OSS_ACCESS_KEY_ID.key(): 'test-key',
+ OssOptions.OSS_ACCESS_KEY_SECRET.key(): 'test-secret',
}))
got = oss_io.to_filesystem_path("oss://test-bucket/path/to/file.txt")
self.assertEqual(got, "path/to/file.txt" if lt7 else "test-bucket/path/to/file.txt")
@@ -286,7 +288,9 @@ class FileIOTest(unittest.TestCase):
file_io.delete_directory_quietly("file:///some/path")
oss_io = PyArrowFileIO("oss://test-bucket/warehouse", Options({
- OssOptions.OSS_ENDPOINT.key(): 'oss-cn-hangzhou.aliyuncs.com'
+ OssOptions.OSS_ENDPOINT.key(): 'oss-cn-hangzhou.aliyuncs.com',
+ OssOptions.OSS_ACCESS_KEY_ID.key(): 'test-key',
+ OssOptions.OSS_ACCESS_KEY_SECRET.key(): 'test-secret',
}))
mock_fs = MagicMock()
mock_fs.get_file_info.return_value = [
diff --git a/paimon-python/pypaimon/tests/jindo_file_system_test.py b/paimon-python/pypaimon/tests/jindo_file_system_test.py
new file mode 100644
index 0000000000..b6f168ac58
--- /dev/null
+++ b/paimon-python/pypaimon/tests/jindo_file_system_test.py
@@ -0,0 +1,524 @@
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+import os
+import unittest
+import uuid
+
+import pyarrow.fs as pafs
+
+from pyarrow.fs import PyFileSystem
+from pypaimon.common.options import Options
+from pypaimon.common.options.config import OssOptions
+from pypaimon.filesystem.jindo_file_system_handler import JindoFileSystemHandler, JINDO_AVAILABLE
+
+
+class JindoFileSystemTest(unittest.TestCase):
+ """Test cases for JindoFileSystem."""
+
+ def setUp(self):
+ """Set up test fixtures."""
+ if not JINDO_AVAILABLE:
+ self.skipTest("pyjindo is not available")
+
+ # Get OSS credentials from environment variables or use test values
+ bucket = os.environ.get("OSS_TEST_BUCKET")
+ access_key_id = os.environ.get("OSS_ACCESS_KEY_ID")
+ access_key_secret = os.environ.get("OSS_ACCESS_KEY_SECRET")
+ endpoint = os.environ.get("OSS_ENDPOINT")
+ if not bucket:
+ self.skipTest("test bucket is not configured")
+ return
+ if not access_key_id:
+ self.skipTest("test access key id is not configured")
+ return
+ if not access_key_secret:
+ self.skipTest("test access key secret is not configured")
+ return
+ if not endpoint:
+ self.skipTest("test endpoint is not configured")
+ return
+ self.root_path = f"oss://{bucket}/"
+
+ self.catalog_options = Options({
+ OssOptions.OSS_ACCESS_KEY_ID.key(): access_key_id,
+ OssOptions.OSS_ACCESS_KEY_SECRET.key(): access_key_secret,
+ OssOptions.OSS_ENDPOINT.key(): endpoint,
+ })
+
+ # Create JindoFileSystemHandler instance
+ fs_handler = JindoFileSystemHandler(self.root_path, self.catalog_options)
+ self.fs = PyFileSystem(fs_handler)
+
+ # Create unique test prefix to avoid conflicts
+ self.test_prefix = f"test-{uuid.uuid4().hex[:8]}/"
+
+ def tearDown(self):
+ """Clean up test files and directories."""
+ # Delete the entire test prefix directory
+ test_dir = f"{self.root_path}{self.test_prefix}"
+ try:
+ file_info = self.fs.get_file_info(test_dir)
+ if file_info.type == pafs.FileType.Directory:
+ try:
+ self.fs.delete_dir(test_dir)
+ except Exception:
+ pass
+ except Exception:
+ pass # Ignore cleanup errors
+
+ def _get_test_path(self, name: str) -> str:
+ """Get a test path under test_prefix."""
+ return f"{self.root_path}{self.test_prefix}{name}"
+
+ def test_get_root_file_info(self):
+ """Test get_file_info for root path."""
+ # Verify directory exists using get_file_info
+ file_info = self.fs.get_file_info(self.root_path)
+
+ # Verify the returned file info
+ self.assertIsInstance(file_info, pafs.FileInfo)
+ self.assertEqual(file_info.type, pafs.FileType.Directory)
+
+ def test_create_dir_recursive(self):
+ """Test create_dir with recursive=True."""
+ test_dir = self._get_test_path("nested/deep/dir/")
+
+ # Create nested directory
+ self.fs.create_dir(test_dir, recursive=True)
+
+ # Verify directory exists
+ file_info = self.fs.get_file_info(test_dir)
+ self.assertEqual(file_info.type, pafs.FileType.Directory)
+
+ def test_write_and_read_small_file(self):
+ """Test writing and reading a small file (10 bytes) with random data."""
+ test_file = self._get_test_path("small-file.txt")
+ test_data = os.urandom(10)
+
+ # Write file
+ out_stream = self.fs.open_output_stream(test_file)
+ out_stream.write(test_data)
+ out_stream.close()
+
+ # Read file
+ with self.fs.open_input_file(test_file) as in_file:
+ read_data = in_file.read()
+
+ # Verify data correctness
+ self.assertEqual(test_data, read_data)
+ self.assertEqual(len(read_data), 10)
+
+ # Verify file info
+ file_info = self.fs.get_file_info(test_file)
+ self.assertEqual(file_info.type, pafs.FileType.File)
+ self.assertEqual(file_info.size, 10)
+
+ def test_write_and_read_small_file_with_context(self):
+ """Test writing and reading a small file (10 bytes) with random data using context manager."""
+ test_file = self._get_test_path("small-file-with-context.txt")
+ test_data = os.urandom(10)
+
+ # Write file using context manager
+ with self.fs.open_output_stream(test_file) as out_stream:
+ out_stream.write(test_data)
+
+ # Read file
+ with self.fs.open_input_file(test_file) as in_file:
+ read_data = in_file.read()
+
+ # Verify data correctness
+ self.assertEqual(test_data, read_data)
+ self.assertEqual(len(read_data), 10)
+
+ # Verify file info
+ file_info = self.fs.get_file_info(test_file)
+ self.assertEqual(file_info.type, pafs.FileType.File)
+ self.assertEqual(file_info.size, 10)
+
+ def test_write_and_read_large_file(self):
+ """Test writing and reading a large file (20MB) with random data."""
+ test_file = self._get_test_path("large-file.bin")
+ file_size = 20 * 1024 * 1024 # 20MB
+ test_data = os.urandom(file_size)
+
+ # Write file in chunks
+ chunk_size = 1024 * 1024 # 1MB chunks
+ out_stream = self.fs.open_output_stream(test_file)
+ for i in range(0, len(test_data), chunk_size):
+ chunk = test_data[i:i + chunk_size]
+ out_stream.write(chunk)
+ out_stream.close()
+
+ # Verify file info
+ file_info = self.fs.get_file_info(test_file)
+ self.assertEqual(file_info.type, pafs.FileType.File)
+ self.assertEqual(file_info.size, file_size)
+
+ # Test JindoInputFile all methods
+ with self.fs.open_input_file(test_file) as in_file:
+ # Test tell() - should be at position 0 initially
+ self.assertEqual(in_file.tell(), 0)
+
+ # Test read() with specific size
+ chunk1 = in_file.read(1024)
+ self.assertEqual(len(chunk1), 1024)
+ self.assertEqual(in_file.tell(), 1024)
+ self.assertEqual(chunk1, test_data[0:1024])
+
+ # Test seek() - seek to middle of file
+ middle_pos = file_size // 2
+ in_file.seek(middle_pos)
+ self.assertEqual(in_file.tell(), middle_pos)
+
+ # Test read_at()
+ read_at_data = in_file.read_at(1024, 0)
+ self.assertEqual(len(read_at_data), 1024)
+ self.assertEqual(read_at_data, test_data[0:1024])
+
+ # Test read_at() at different offset
+ offset = file_size - 2048
+ read_at_data2 = in_file.read_at(1024, offset)
+ self.assertEqual(len(read_at_data2), 1024)
+ self.assertEqual(read_at_data2, test_data[offset:offset + 1024])
+
+ # Test seek() to end
+ in_file.seek(file_size)
+ self.assertEqual(in_file.tell(), file_size)
+
+ # Test read() at end - should return empty bytes
+ empty_data = in_file.read(100)
+ self.assertEqual(len(empty_data), 0)
+
+ # Test read() without size - read remaining
+ in_file.seek(0)
+ all_data = in_file.read()
+ self.assertEqual(len(all_data), file_size)
+ self.assertEqual(all_data, test_data)
+
+ # Verify complete file read
+ with self.fs.open_input_file(test_file) as in_file:
+ complete_data = in_file.read()
+ self.assertEqual(complete_data, test_data)
+ self.assertEqual(len(complete_data), file_size)
+
+ def test_get_file_info_single_path(self):
+ """Test get_file_info with single path."""
+ test_file = self._get_test_path("info-test.txt")
+ test_data = b"test content"
+
+ # Write file
+ out_stream = self.fs.open_output_stream(test_file)
+ out_stream.write(test_data)
+ out_stream.close()
+
+ # Get file info
+ file_info = self.fs.get_file_info(test_file)
+ self.assertIsInstance(file_info, pafs.FileInfo)
+ self.assertEqual(file_info.type, pafs.FileType.File)
+ self.assertEqual(file_info.size, len(test_data))
+
+ def test_get_file_info_list(self):
+ """Test get_file_info with list of paths."""
+ test_file = self._get_test_path("info-test1.txt")
+ test_dir = self._get_test_path("dir1")
+
+ # Write files
+ out_stream = self.fs.open_output_stream(test_file)
+ out_stream.write(b"content1")
+ out_stream.close()
+ self.fs.create_dir(test_dir)
+
+ # Get file info for list
+ file_infos = self.fs.get_file_info([test_file, test_dir])
+ self.assertIsInstance(file_infos, list)
+ self.assertEqual(len(file_infos), 2)
+ self.assertEqual(file_infos[0].type, pafs.FileType.File)
+ self.assertTrue(test_file in file_infos[0].path)
+ self.assertEqual(file_infos[1].type, pafs.FileType.Directory)
+ self.assertTrue(test_dir in file_infos[1].path)
+
+ def test_get_file_info_with_selector(self):
+ """Test get_file_info with FileSelector."""
+ test_dir = self._get_test_path("selector-test/")
+ test_file1 = self._get_test_path("selector-test/file1.txt")
+ test_file2 = self._get_test_path("selector-test/file2.txt")
+
+ # Create files
+ self.fs.create_dir(test_dir)
+ out_stream1 = self.fs.open_output_stream(test_file1)
+ out_stream1.write(b"content1")
+ out_stream1.close()
+ out_stream2 = self.fs.open_output_stream(test_file2)
+ out_stream2.write(b"content2")
+ out_stream2.close()
+
+ # Test non-recursive listing
+ selector = pafs.FileSelector(test_dir, recursive=False, allow_not_found=False)
+ file_infos = self.fs.get_file_info(selector)
+ self.assertIsInstance(file_infos, list)
+ self.assertEqual(len(file_infos), 2)
+
+ # Verify we got the files
+ file_names = [info.path for info in file_infos]
+ self.assertTrue(any("file1.txt" in name for name in file_names))
+ self.assertTrue(any("file2.txt" in name for name in file_names))
+
+ def test_get_file_info_with_selector_recursive(self):
+ """Test get_file_info with FileSelector recursive=True."""
+ test_dir = self._get_test_path("selector-recursive/")
+ test_subdir = self._get_test_path("selector-recursive/subdir/")
+ test_file1 = self._get_test_path("selector-recursive/file1.txt")
+ test_file2 = self._get_test_path("selector-recursive/subdir/file2.txt")
+
+ # Create directory structure
+ self.fs.create_dir(test_dir)
+ self.fs.create_dir(test_subdir)
+ out_stream1 = self.fs.open_output_stream(test_file1)
+ out_stream1.write(b"content1")
+ out_stream1.close()
+ out_stream2 = self.fs.open_output_stream(test_file2)
+ out_stream2.write(b"content2")
+ out_stream2.close()
+
+ # Test recursive listing
+ selector = pafs.FileSelector(test_dir, recursive=True, allow_not_found=False)
+ file_infos = self.fs.get_file_info(selector)
+ self.assertIsInstance(file_infos, list)
+ self.assertEqual(len(file_infos), 3)
+
+ def test_get_file_info_not_found(self):
+ """Test get_file_info for non-existent file."""
+ non_existent = self._get_test_path("non-existent-file.txt")
+
+ file_info = self.fs.get_file_info(non_existent)
+ self.assertEqual(file_info.type, pafs.FileType.NotFound)
+
+ # Try to open non-existent file should raise FileNotFoundError
+ with self.assertRaises(FileNotFoundError):
+ with self.fs.open_input_file(non_existent) as in_file:
+ in_file.read()
+
+ def test_get_file_info_selector_allow_not_found(self):
+ """Test get_file_info with FileSelector allow_not_found=True."""
+ non_existent_dir = self._get_test_path("non-existent-dir/")
+
+ # Test with allow_not_found=True
+ selector = pafs.FileSelector(non_existent_dir, recursive=False, allow_not_found=True)
+ file_infos = self.fs.get_file_info(selector)
+ self.assertIsInstance(file_infos, list)
+ self.assertEqual(len(file_infos), 0)
+
+ def test_delete_file(self):
+ """Test delete_file method."""
+ test_file = self._get_test_path("delete-test.txt")
+
+ # Create file
+ out_stream = self.fs.open_output_stream(test_file)
+ out_stream.write(b"test content")
+ out_stream.close()
+
+ # Verify file exists
+ file_info = self.fs.get_file_info(test_file)
+ self.assertEqual(file_info.type, pafs.FileType.File)
+
+ # Delete file
+ self.fs.delete_file(test_file)
+
+ # Verify file is deleted - should raise FileNotFoundError when accessing
+ with self.assertRaises(FileNotFoundError):
+ with self.fs.open_input_file(test_file) as in_file:
+ in_file.read()
+
+ def test_delete_dir(self):
+ """Test delete_dir method."""
+ test_dir = self._get_test_path("delete-dir/")
+
+ # Create directory
+ self.fs.create_dir(test_dir)
+
+ # Verify directory exists
+ file_info = self.fs.get_file_info(test_dir)
+ self.assertEqual(file_info.type, pafs.FileType.Directory)
+
+ # Delete directory
+ self.fs.delete_dir(test_dir)
+
+ # Verify directory is deleted - should raise FileNotFoundError when accessing
+ with self.assertRaises(FileNotFoundError):
+ # Try to list directory contents
+ selector = pafs.FileSelector(test_dir, recursive=False, allow_not_found=False)
+ self.fs.get_file_info(selector)
+
+ def test_delete_dir_contents(self):
+ """Test delete_dir_contents method."""
+ test_dir = self._get_test_path("delete-contents/")
+ test_file1 = self._get_test_path("delete-contents/file1.txt")
+ test_file2 = self._get_test_path("delete-contents/file2.txt")
+ test_subdir = self._get_test_path("delete-contents/subdir/")
+ test_file3 = self._get_test_path("delete-contents/subdir/file3.txt")
+
+ # Create directory structure
+ self.fs.create_dir(test_dir)
+ out_stream1 = self.fs.open_output_stream(test_file1)
+ out_stream1.write(b"content1")
+ out_stream1.close()
+ out_stream2 = self.fs.open_output_stream(test_file2)
+ out_stream2.write(b"content2")
+ out_stream2.close()
+ self.fs.create_dir(test_subdir)
+ out_stream3 = self.fs.open_output_stream(test_file3)
+ out_stream3.write(b"content3")
+ out_stream3.close()
+
+ # Delete directory contents
+ self.fs.delete_dir_contents(test_dir)
+
+ # Verify directory still exists but is empty
+ file_info = self.fs.get_file_info(test_dir)
+ self.assertEqual(file_info.type, pafs.FileType.Directory)
+ selector = pafs.FileSelector(test_dir, recursive=False, allow_not_found=False)
+ file_infos = self.fs.get_file_info(selector)
+ self.assertEqual(len(file_infos), 0)
+
+ # Verify files are deleted - should raise FileNotFoundError when accessing
+ with self.assertRaises(FileNotFoundError):
+ with self.fs.open_input_file(test_file1) as in_file:
+ in_file.read()
+ with self.assertRaises(FileNotFoundError):
+ with self.fs.open_input_file(test_file2) as in_file:
+ in_file.read()
+ with self.assertRaises(FileNotFoundError):
+ with self.fs.open_input_file(test_file3) as in_file:
+ in_file.read()
+
+ def test_move_file(self):
+ """Test move method for file."""
+ src_file = self._get_test_path("move-src.txt")
+ dst_file = self._get_test_path("move-dst.txt")
+ test_data = b"move test content"
+
+ # Create source file
+ out_stream = self.fs.open_output_stream(src_file)
+ out_stream.write(test_data)
+ out_stream.close()
+
+ # Move file
+ self.fs.move(src_file, dst_file)
+
+ # Verify source is gone - should raise FileNotFoundError when accessing
+ with self.assertRaises(FileNotFoundError):
+ with self.fs.open_input_file(src_file) as in_file:
+ in_file.read()
+
+ # Verify destination exists with correct content
+ dst_info = self.fs.get_file_info(dst_file)
+ self.assertEqual(dst_info.type, pafs.FileType.File)
+
+ with self.fs.open_input_file(dst_file) as in_file:
+ read_data = in_file.read()
+ self.assertEqual(read_data, test_data)
+
+ def test_move_directory(self):
+ """Test move method for directory."""
+ src_dir = self._get_test_path("move-src-dir/")
+ dst_dir = self._get_test_path("move-dst-dir/")
+ src_file = self._get_test_path("move-src-dir/file.txt")
+
+ # Create source directory and file
+ self.fs.create_dir(src_dir)
+ out_stream = self.fs.open_output_stream(src_file)
+ out_stream.write(b"test content")
+ out_stream.close()
+
+ # Move directory
+ self.fs.move(src_dir, dst_dir)
+
+ # Verify source is gone - should raise FileNotFoundError when accessing
+ with self.assertRaises(FileNotFoundError):
+ # Try to list directory contents
+ selector = pafs.FileSelector(src_dir, recursive=False, allow_not_found=False)
+ self.fs.get_file_info(selector)
+
+ # Verify destination exists
+ dst_info = self.fs.get_file_info(dst_dir)
+ self.assertEqual(dst_info.type, pafs.FileType.Directory)
+
+ def test_copy_file(self):
+ """Test copy_file method."""
+ src_file = self._get_test_path("copy-src.txt")
+ dst_file = self._get_test_path("copy-dst.txt")
+ test_data = os.urandom(1024 * 1024) # 1MB random data
+
+ # Create source file
+ out_stream = self.fs.open_output_stream(src_file)
+ out_stream.write(test_data)
+ out_stream.close()
+
+ # Copy file
+ self.fs.copy_file(src_file, dst_file)
+
+ # Verify both files exist
+ src_info = self.fs.get_file_info(src_file)
+ self.assertEqual(src_info.type, pafs.FileType.File)
+ dst_info = self.fs.get_file_info(dst_file)
+ self.assertEqual(dst_info.type, pafs.FileType.File)
+
+ # Verify destination content matches source
+ with self.fs.open_input_file(dst_file) as in_file:
+ read_data = in_file.read()
+ self.assertEqual(read_data, test_data)
+ self.assertEqual(len(read_data), len(test_data))
+
+ def test_delete_nonexistent_file(self):
+ """Test deleting non-existent file."""
+ non_existent = self._get_test_path("non-existent-delete.txt")
+
+ # Try to delete non-existent file
+ with self.assertRaises(IOError):
+ self.fs.delete_file(non_existent)
+
+ def test_multiple_sequential_reads(self):
+ """Test multiple sequential reads from same file."""
+ test_file = self._get_test_path("sequential-read.txt")
+ test_data = os.urandom(10000) # 10KB
+
+ # Write file
+ out_stream = self.fs.open_output_stream(test_file)
+ out_stream.write(test_data)
+ out_stream.close()
+
+ # Read in chunks sequentially
+ with self.fs.open_input_file(test_file) as in_file:
+ chunk1 = in_file.read(1000)
+ chunk2 = in_file.read(2000)
+ chunk3 = in_file.read(3000)
+ chunk4 = in_file.read() # Read rest
+
+ # Verify all chunks
+ self.assertEqual(chunk1, test_data[0:1000])
+ self.assertEqual(chunk2, test_data[1000:3000])
+ self.assertEqual(chunk3, test_data[3000:6000])
+ self.assertEqual(chunk4, test_data[6000:])
+
+ # Verify total length
+ total_read = len(chunk1) + len(chunk2) + len(chunk3) + len(chunk4)
+ self.assertEqual(total_read, len(test_data))
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/paimon-python/pypaimon/tests/oss_file_io_test.py b/paimon-python/pypaimon/tests/oss_file_io_test.py
new file mode 100644
index 0000000000..f9602c975e
--- /dev/null
+++ b/paimon-python/pypaimon/tests/oss_file_io_test.py
@@ -0,0 +1,362 @@
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+import os
+import unittest
+import uuid
+
+import pyarrow.fs as pafs
+
+from pypaimon.common.options import Options
+from pypaimon.common.options.config import OssOptions
+from pypaimon.filesystem.pyarrow_file_io import PyArrowFileIO
+
+
+class OSSFileIOTest(unittest.TestCase):
+ """Test cases for PyArrowFileIO with OSS."""
+
+ def setUp(self):
+ """Set up test fixtures."""
+ # Get OSS credentials from environment variables
+ self.bucket = os.environ.get("OSS_TEST_BUCKET")
+ access_key_id = os.environ.get("OSS_ACCESS_KEY_ID")
+ access_key_secret = os.environ.get("OSS_ACCESS_KEY_SECRET")
+ endpoint = os.environ.get("OSS_ENDPOINT")
+ oss_impl = os.environ.get("OSS_IMPL", "default")
+
+ if not self.bucket:
+ self.skipTest("test bucket is not configured")
+ return
+ if not access_key_id:
+ self.skipTest("test access key id is not configured")
+ return
+ if not access_key_secret:
+ self.skipTest("test access key secret is not configured")
+ return
+ if not endpoint:
+ self.skipTest("test endpoint is not configured")
+ return
+
+ self.root_path = f"oss://{self.bucket}/"
+
+ self.catalog_options = Options({
+ OssOptions.OSS_ACCESS_KEY_ID.key(): access_key_id,
+ OssOptions.OSS_ACCESS_KEY_SECRET.key(): access_key_secret,
+ OssOptions.OSS_ENDPOINT.key(): endpoint,
+ OssOptions.OSS_IMPL.key(): oss_impl,
+ })
+
+ # Create PyArrowFileIO instance
+ self.file_io = PyArrowFileIO(self.root_path, self.catalog_options)
+
+ # Create unique test prefix to avoid conflicts
+ self.test_prefix = f"test-{uuid.uuid4().hex[:8]}/"
+
+ def tearDown(self):
+ """Clean up test files and directories."""
+ # Delete the entire test prefix directory
+ test_dir = f"{self.root_path}{self.test_prefix}"
+ try:
+ if self.file_io.exists(test_dir):
+ self.file_io.delete(test_dir, recursive=True)
+ except Exception:
+ pass # Ignore cleanup errors
+
+ def _get_test_path(self, name: str) -> str:
+ """Get a test path under test_prefix."""
+ return f"{self.root_path}{self.test_prefix}{name}"
+
+ def test_get_file_status_directory(self):
+ """Test get_file_status for a directory."""
+ # Create a test directory
+ test_dir = self._get_test_path("test-dir/")
+ self.file_io.mkdirs(test_dir)
+ # Get file status
+ file_status = self.file_io.get_file_status(test_dir)
+ # Verify the returned file status
+ self.assertIsNotNone(file_status)
+ self.assertEqual(file_status.type, pafs.FileType.Directory)
+
+ def test_new_input_stream_read(self):
+ """Test new_input_stream and read method."""
+ # Create test data
+ test_data = b"Hello, World! This is a test file for OSS input stream."
+ test_file = self._get_test_path("test-input-stream.txt")
+
+ # Write test data to file
+ with self.file_io.new_output_stream(test_file) as out_stream:
+ out_stream.write(test_data)
+
+ # Test new_input_stream
+ input_stream = self.file_io.new_input_stream(test_file)
+ self.assertIsNotNone(input_stream)
+
+ # Test read without nbytes (read all)
+ input_stream.seek(0)
+ read_data = input_stream.read()
+ self.assertEqual(read_data, test_data)
+
+ # Test read with nbytes
+ input_stream.seek(0)
+ read_partial = input_stream.read(5)
+ self.assertEqual(read_partial, b"Hello")
+
+ # Test read more bytes
+ read_partial2 = input_stream.read(7)
+ self.assertEqual(read_partial2, b", World")
+
+ # Test read remaining
+ read_remaining = input_stream.read()
+ self.assertEqual(read_remaining, b"! This is a test file for OSS input stream.")
+
+ # Verify complete data
+ input_stream.seek(0)
+ complete_data = input_stream.read()
+ self.assertEqual(complete_data, test_data)
+
+ # Close the stream
+ input_stream.close()
+
+ # Test context manager
+ with self.file_io.new_input_stream(test_file) as input_stream2:
+ data = input_stream2.read()
+ self.assertEqual(data, test_data)
+
+ def test_new_input_stream_read_large_file(self):
+ """Test new_input_stream with larger file and read operations."""
+ # Create larger test data (1MB)
+ test_data = b"X" * (1024 * 1024)
+ test_file = self._get_test_path("test-large-input-stream.bin")
+
+ # Write test data
+ with self.file_io.new_output_stream(test_file) as out_stream:
+ out_stream.write(test_data)
+
+ # Test reading in chunks
+ chunk_size = 64 * 1024 # 64KB chunks
+ with self.file_io.new_input_stream(test_file) as input_stream:
+ read_chunks = []
+ while True:
+ chunk = input_stream.read(chunk_size)
+ if not chunk:
+ break
+ read_chunks.append(chunk)
+
+ # Verify all data was read
+ read_data = b''.join(read_chunks)
+ self.assertEqual(len(read_data), len(test_data))
+ self.assertEqual(read_data, test_data)
+
+ # Test read_at method if available
+ with self.file_io.new_input_stream(test_file) as input_stream:
+ if hasattr(input_stream, 'read_at'):
+ # Read from middle of file
+ offset = len(test_data) // 2
+ read_at_data = input_stream.read_at(1024, offset)
+ self.assertEqual(len(read_at_data), 1024)
+ self.assertEqual(read_at_data, test_data[offset:offset+1024])
+
+ def test_new_input_stream_file_not_found(self):
+ """Test new_input_stream with non-existent file."""
+ non_existent_file = self._get_test_path("non-existent-file.txt")
+
+ with self.assertRaises(FileNotFoundError):
+ self.file_io.new_input_stream(non_existent_file)
+
+ def test_exists(self):
+ """Test exists method."""
+ missing_uri = self._get_test_path("nonexistent_xyz")
+ self.assertFalse(self.file_io.exists(missing_uri))
+ with self.assertRaises(FileNotFoundError):
+ self.file_io.get_file_status(missing_uri)
+
+ def test_write_file_with_overwrite_flag(self):
+ """Test write_file with overwrite flag."""
+ test_file_uri = self._get_test_path("overwrite_test.txt")
+
+ # 1) Write to a new file with default overwrite=False
+ self.file_io.write_file(test_file_uri, "first content")
+ self.assertTrue(self.file_io.exists(test_file_uri))
+ content = self.file_io.read_file_utf8(test_file_uri)
+ self.assertEqual(content, "first content")
+
+ # 2) Attempt to write again with overwrite=False should raise FileExistsError
+ with self.assertRaises(FileExistsError):
+ self.file_io.write_file(test_file_uri, "second content", overwrite=False)
+
+ # Ensure content is unchanged
+ content = self.file_io.read_file_utf8(test_file_uri)
+ self.assertEqual(content, "first content")
+
+ # 3) Write with overwrite=True should replace the content
+ self.file_io.write_file(test_file_uri, "overwritten content", overwrite=True)
+ content = self.file_io.read_file_utf8(test_file_uri)
+ self.assertEqual(content, "overwritten content")
+
+ def test_exists_does_not_catch_exception(self):
+ """Test that exists does not catch exceptions."""
+ test_file = self._get_test_path("test_file.txt")
+
+ # Write a test file
+ with self.file_io.new_output_stream(test_file) as out_stream:
+ out_stream.write(b"test")
+ self.assertTrue(self.file_io.exists(test_file))
+ self.assertFalse(self.file_io.exists(self._get_test_path("nonexistent.txt")))
+
+ def test_delete_non_empty_directory_raises_error(self):
+ """Test that delete raises error for non-empty directory."""
+ test_dir = self._get_test_path("test_dir/")
+ test_file = self._get_test_path("test_dir/test_file.txt")
+
+ # Create directory and file
+ self.file_io.mkdirs(test_dir)
+ with self.file_io.new_output_stream(test_file) as out_stream:
+ out_stream.write(b"test")
+
+ with self.assertRaises(OSError) as context:
+ self.file_io.delete(test_dir, recursive=False)
+ self.assertIn("is not empty", str(context.exception))
+
+ def test_delete_returns_false_when_file_not_exists(self):
+ """Test that delete returns False when file does not exist."""
+ result = self.file_io.delete(self._get_test_path("nonexistent.txt"))
+ self.assertFalse(result, "delete() should return False when file does not exist")
+
+ result = self.file_io.delete(self._get_test_path("nonexistent_dir"), recursive=False)
+ self.assertFalse(result, "delete() should return False when directory does not exist")
+
+ def test_mkdirs_raises_error_when_path_is_file(self):
+ """Test that mkdirs raises error when path is a file."""
+ test_file = self._get_test_path("test_file.txt")
+
+ # Create a file
+ with self.file_io.new_output_stream(test_file) as out_stream:
+ out_stream.write(b"test")
+
+ with self.assertRaises(FileExistsError) as context:
+ self.file_io.mkdirs(test_file)
+ self.assertIn("is not a directory", str(context.exception))
+
+ def test_rename_returns_false_when_dst_exists(self):
+ """Test that rename returns False when destination exists."""
+ src_file = self._get_test_path("src.txt")
+ dst_file = self._get_test_path("dst.txt")
+
+ # Create source and destination files
+ with self.file_io.new_output_stream(src_file) as out_stream:
+ out_stream.write(b"src")
+ with self.file_io.new_output_stream(dst_file) as out_stream:
+ out_stream.write(b"dst")
+
+ result = self.file_io.rename(src_file, dst_file)
+ self.assertFalse(result)
+
+ def test_get_file_status_raises_error_when_file_not_exists(self):
+ """Test that get_file_status raises error when file does not exist."""
+ with self.assertRaises(FileNotFoundError) as context:
+ self.file_io.get_file_status(self._get_test_path("nonexistent.txt"))
+ self.assertIn("does not exist", str(context.exception))
+
+ test_file = self._get_test_path("test_file.txt")
+ with self.file_io.new_output_stream(test_file) as out_stream:
+ out_stream.write(b"test content")
+
+ file_info = self.file_io.get_file_status(test_file)
+ self.assertEqual(file_info.type, pafs.FileType.File)
+ self.assertIsNotNone(file_info.size)
+
+ with self.assertRaises(FileNotFoundError) as context:
+ self.file_io.get_file_size(self._get_test_path("nonexistent.txt"))
+ self.assertIn("does not exist", str(context.exception))
+
+ with self.assertRaises(FileNotFoundError) as context:
+ self.file_io.is_dir(self._get_test_path("nonexistent_dir"))
+ self.assertIn("does not exist", str(context.exception))
+
+ def test_copy_file(self):
+ """Test copy_file method."""
+ source_file = self._get_test_path("source.txt")
+ target_file = self._get_test_path("target.txt")
+
+ # Create source file
+ with self.file_io.new_output_stream(source_file) as out_stream:
+ out_stream.write(b"source content")
+
+ # Test 1: Raises FileExistsError when target exists and overwrite=False
+ with self.file_io.new_output_stream(target_file) as out_stream:
+ out_stream.write(b"target content")
+
+ with self.assertRaises(FileExistsError) as context:
+ self.file_io.copy_file(source_file, target_file, overwrite=False)
+ self.assertIn("already exists", str(context.exception))
+
+ # Verify target content unchanged
+ with self.file_io.new_input_stream(target_file) as in_stream:
+ content = in_stream.read()
+ self.assertEqual(content, b"target content")
+
+ # Test 2: Overwrites when overwrite=True
+ self.file_io.copy_file(source_file, target_file, overwrite=True)
+ with self.file_io.new_input_stream(target_file) as in_stream:
+ content = in_stream.read()
+ self.assertEqual(content, b"source content")
+
+ # Test 3: Creates parent directory if it doesn't exist
+ target_file_in_subdir = self._get_test_path("subdir/target.txt")
+ self.file_io.copy_file(source_file, target_file_in_subdir, overwrite=False)
+ self.assertTrue(self.file_io.exists(target_file_in_subdir))
+ with self.file_io.new_input_stream(target_file_in_subdir) as in_stream:
+ content = in_stream.read()
+ self.assertEqual(content, b"source content")
+
+ def test_try_to_write_atomic(self):
+ """Test try_to_write_atomic method."""
+ target_dir = self._get_test_path("target_dir/")
+ normal_file = self._get_test_path("normal_file.txt")
+
+ # Create target directory
+ self.file_io.mkdirs(target_dir)
+ self.assertFalse(
+ self.file_io.try_to_write_atomic(target_dir, "test content"),
+ "PyArrowFileIO should return False when target is a directory")
+
+ # Verify no file was created inside the directory
+ # List directory contents to verify it's empty
+ selector = pafs.FileSelector(self.file_io.to_filesystem_path(target_dir), recursive=False, allow_not_found=True)
+ dir_contents = self.file_io.filesystem.get_file_info(selector)
+ self.assertEqual(len(dir_contents), 0, "No file should be created inside the directory")
+
+ self.assertTrue(self.file_io.try_to_write_atomic(normal_file, "test content"))
+ content = self.file_io.read_file_utf8(normal_file)
+ self.assertEqual(content, "test content")
+
+ # Delete and test again
+ self.file_io.delete(normal_file)
+ self.assertFalse(
+ self.file_io.try_to_write_atomic(target_dir, "test content"),
+ "PyArrowFileIO should return False when target is a directory")
+
+ # Verify no file was created inside the directory
+ dir_contents = self.file_io.filesystem.get_file_info(selector)
+ self.assertEqual(len(dir_contents), 0, "No file should be created inside the directory")
+
+ self.assertTrue(self.file_io.try_to_write_atomic(normal_file, "test content"))
+ content = self.file_io.read_file_utf8(normal_file)
+ self.assertEqual(content, "test content")
+
+if __name__ == '__main__':
+ unittest.main()
(paimon) 07/37: [flink][spark] path_to_descriptor function supports http url (#7529)
Posted by ho...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hope pushed a commit to branch release-1.4
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit d29f5a685d040709d3bc76c7b55a25fc7a277edb
Author: Faiz <wx...@antgroup.com>
AuthorDate: Wed Mar 25 22:06:20 2026 +0800
[flink][spark] path_to_descriptor function supports http url (#7529)
---
.../paimon/flink/function/PathToDescriptor.java | 2 +-
.../org/apache/paimon/flink/BlobTableITCase.java | 31 ++++++++++++++++++++++
.../spark/function/PathToDescriptorFunction.java | 2 +-
3 files changed, 33 insertions(+), 2 deletions(-)
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/function/PathToDescriptor.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/function/PathToDescriptor.java
index 87d136f717..210116b9ca 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/function/PathToDescriptor.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/function/PathToDescriptor.java
@@ -30,7 +30,7 @@ public class PathToDescriptor extends ScalarFunction {
return null;
}
- BlobDescriptor descriptor = new BlobDescriptor(path, 0, Long.MAX_VALUE);
+ BlobDescriptor descriptor = new BlobDescriptor(path, 0, -1);
return descriptor.serialize();
}
}
diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/BlobTableITCase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/BlobTableITCase.java
index 29938f36be..e490020709 100644
--- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/BlobTableITCase.java
+++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/BlobTableITCase.java
@@ -25,6 +25,7 @@ import org.apache.paimon.data.BlobRef;
import org.apache.paimon.fs.FileIO;
import org.apache.paimon.fs.local.LocalFileIO;
import org.apache.paimon.options.Options;
+import org.apache.paimon.rest.TestHttpWebServer;
import org.apache.paimon.table.FileStoreTable;
import org.apache.paimon.types.RowType;
import org.apache.paimon.utils.UriReader;
@@ -299,6 +300,36 @@ public class BlobTableITCase extends CatalogITCaseBase {
assertThat(allColumns).hasSize(1);
}
+ @Test
+ public void testWriteBlobWithHttpUrlDescriptor() throws Exception {
+ TestHttpWebServer httpServer = new TestHttpWebServer("/blob_data");
+ httpServer.start();
+ try {
+ String blobContent = "hello-http-blob";
+ String httpUrl = httpServer.getBaseUrl();
+
+ // Enqueue response for the write phase
+ httpServer.enqueueResponse(blobContent, 200);
+
+ // Use sys.path_to_descriptor with HTTP URL
+ batchSql(
+ "INSERT INTO blob_table_descriptor VALUES (1, 'http-blob', sys.path_to_descriptor('"
+ + httpUrl
+ + "'))");
+
+ // Read back with blob-as-descriptor=false to get raw data
+ batchSql("ALTER TABLE blob_table_descriptor SET ('blob-as-descriptor'='false')");
+ List<Row> result = batchSql("SELECT * FROM blob_table_descriptor");
+ assertThat(result).hasSize(1);
+ assertThat(result.get(0).getField(0)).isEqualTo(1);
+ assertThat(result.get(0).getField(1)).isEqualTo("http-blob");
+ assertThat((byte[]) result.get(0).getField(2))
+ .isEqualTo(blobContent.getBytes(java.nio.charset.StandardCharsets.UTF_8));
+ } finally {
+ httpServer.stop();
+ }
+ }
+
@Test
public void testBlobTypeSchemaEquals() throws Exception {
// Step 1: Create a Paimon table with blob field via Flink SQL
diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/function/PathToDescriptorFunction.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/function/PathToDescriptorFunction.java
index ea6f306cc3..8c26b5eefd 100644
--- a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/function/PathToDescriptorFunction.java
+++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/function/PathToDescriptorFunction.java
@@ -45,7 +45,7 @@ public class PathToDescriptorFunction implements ScalarFunction<byte[]>, Seriali
return null;
}
- BlobDescriptor descriptor = new BlobDescriptor(path.toString(), 0, Long.MAX_VALUE);
+ BlobDescriptor descriptor = new BlobDescriptor(path.toString(), 0, -1);
return descriptor.serialize();
}
(paimon) 34/37: [python] Fix token cache pollution in Python `RESTTokenFileIO` by aligning with the Java implementation. (#7562)
Posted by ho...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hope pushed a commit to branch release-1.4
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit a413c697c42d5a4739674777d96b712b233366f0
Author: shyjsarah <44...@users.noreply.github.com>
AuthorDate: Tue Mar 31 00:52:57 2026 -0700
[python] Fix token cache pollution in Python `RESTTokenFileIO` by aligning with the Java implementation. (#7562)
Fix token cache pollution in Python `RESTTokenFileIO` by aligning with
the Java implementation.
**Problem:**
Python's `RESTTokenFileIO` had a class-level `_TOKEN_CACHE` (keyed by
table identifier string) that caused token sharing across different
catalog instances. When two catalogs with different AK/SK credentials
operated on the same table within one process, the second catalog would
reuse the first catalog's token — leading to permission errors (e.g.,
read AK/SK token used for write operations).
**Root Cause:**
Java's `RESTTokenFileIO` has **no token cache** — each instance manages
its own `token` field independently. Python added an extra
`_TOKEN_CACHE` class-level dict that Java never had.
**Changes:**
- Remove class-level `_TOKEN_CACHE`, `_TOKEN_LOCKS`, `_TOKEN_LOCKS_LOCK`
and their associated methods (`_get_cached_token`, `_set_cached_token`,
`_get_global_token_lock`, `_is_token_expired`)
- Simplify `try_to_refresh_token()` to use instance-level lock with
double-check pattern, aligned with Java's `tryToRefreshToken()`
- Merge `should_refresh()` and `_is_token_expired()` into a single
`_should_refresh()` method
- Add system table identifier handling in `refresh_token()` (strip
`$snapshots` suffix before requesting token), aligned with Java
The `FILE_IO_CACHE` (keyed by `RESTToken` object) is kept unchanged — it
correctly isolates different credentials since different tokens produce
different cache keys.
---
.../pypaimon/catalog/rest/rest_token_file_io.py | 95 ++++--------
.../pypaimon/tests/rest/rest_token_file_io_test.py | 164 ++++++++++++++++++---
2 files changed, 177 insertions(+), 82 deletions(-)
diff --git a/paimon-python/pypaimon/catalog/rest/rest_token_file_io.py b/paimon-python/pypaimon/catalog/rest/rest_token_file_io.py
index 7bf984d1f5..16b16e6972 100644
--- a/paimon-python/pypaimon/catalog/rest/rest_token_file_io.py
+++ b/paimon-python/pypaimon/catalog/rest/rest_token_file_io.py
@@ -27,7 +27,7 @@ from pypaimon.api.rest_util import RESTUtil
from pypaimon.catalog.rest.rest_token import RESTToken
from pypaimon.common.file_io import FileIO
from pypaimon.filesystem.pyarrow_file_io import PyArrowFileIO
-from pypaimon.common.identifier import Identifier
+from pypaimon.common.identifier import Identifier, SYSTEM_TABLE_SPLITTER
from pypaimon.common.options import Options
from pypaimon.common.options.config import CatalogOptions, OssOptions
from pypaimon.common.uri_reader import UriReaderFactory
@@ -37,17 +37,13 @@ class RESTTokenFileIO(FileIO):
"""
A FileIO to support getting token from REST Server.
"""
-
+
_FILE_IO_CACHE_MAXSIZE = 1000
_FILE_IO_CACHE_TTL = 36000 # 10 hours in seconds
-
+
_FILE_IO_CACHE: TTLCache = None
_FILE_IO_CACHE_LOCK = threading.Lock()
-
- _TOKEN_CACHE: dict = {}
- _TOKEN_LOCKS: dict = {}
- _TOKEN_LOCKS_LOCK = threading.Lock()
-
+
@classmethod
def _get_file_io_cache(cls) -> TTLCache:
if cls._FILE_IO_CACHE is None:
@@ -58,7 +54,7 @@ class RESTTokenFileIO(FileIO):
ttl=cls._FILE_IO_CACHE_TTL
)
return cls._FILE_IO_CACHE
-
+
def __init__(self, identifier: Identifier, path: str,
catalog_options: Optional[Union[dict, Options]] = None):
self.identifier = identifier
@@ -99,26 +95,26 @@ class RESTTokenFileIO(FileIO):
if self.token is None:
return FileIO.get(self.path, self.catalog_options or Options({}))
-
+
cache_key = self.token
cache = self._get_file_io_cache()
-
+
file_io = cache.get(cache_key)
if file_io is not None:
return file_io
-
+
with self._FILE_IO_CACHE_LOCK:
self.try_to_refresh_token()
-
+
if self.token is None:
return FileIO.get(self.path, self.catalog_options or Options({}))
-
+
cache_key = self.token
cache = self._get_file_io_cache()
file_io = cache.get(cache_key)
if file_io is not None:
return file_io
-
+
merged_properties = RESTUtil.merge(
self.catalog_options.to_map() if self.catalog_options else {},
self.token.token
@@ -128,7 +124,7 @@ class RESTTokenFileIO(FileIO):
if dlf_oss_endpoint and dlf_oss_endpoint.strip():
merged_properties[OssOptions.OSS_ENDPOINT.key()] = dlf_oss_endpoint
merged_options = Options(merged_properties)
-
+
file_io = PyArrowFileIO(self.path, merged_options)
cache[cache_key] = file_io
return file_io
@@ -198,7 +194,7 @@ class RESTTokenFileIO(FileIO):
if self._uri_reader_factory_cache is None:
catalog_options = self.catalog_options or Options({})
self._uri_reader_factory_cache = UriReaderFactory(catalog_options)
-
+
return self._uri_reader_factory_cache
@property
@@ -206,66 +202,35 @@ class RESTTokenFileIO(FileIO):
return self.file_io().filesystem
def try_to_refresh_token(self):
- identifier_str = str(self.identifier)
-
- if self.token is not None and not self._is_token_expired(self.token):
- return
-
- cached_token = self._get_cached_token(identifier_str)
- if cached_token and not self._is_token_expired(cached_token):
- self.token = cached_token
- return
-
- global_lock = self._get_global_token_lock(identifier_str)
-
- with global_lock:
- cached_token = self._get_cached_token(identifier_str)
- if cached_token and not self._is_token_expired(cached_token):
- self.token = cached_token
- return
-
- token_to_check = cached_token if cached_token else self.token
- if token_to_check is None or self._is_token_expired(token_to_check):
- self.refresh_token()
- self._set_cached_token(identifier_str, self.token)
-
- def _get_cached_token(self, identifier_str: str) -> Optional[RESTToken]:
- with self._TOKEN_LOCKS_LOCK:
- return self._TOKEN_CACHE.get(identifier_str)
-
- def _set_cached_token(self, identifier_str: str, token: RESTToken):
- with self._TOKEN_LOCKS_LOCK:
- self._TOKEN_CACHE[identifier_str] = token
-
- def _is_token_expired(self, token: Optional[RESTToken]) -> bool:
- if token is None:
- return True
- current_time = int(time.time() * 1000)
- return (token.expire_at_millis - current_time) < RESTApi.TOKEN_EXPIRATION_SAFE_TIME_MILLIS
-
- def _get_global_token_lock(self, identifier_str: str) -> threading.Lock:
- with self._TOKEN_LOCKS_LOCK:
- if identifier_str not in self._TOKEN_LOCKS:
- self._TOKEN_LOCKS[identifier_str] = threading.Lock()
- return self._TOKEN_LOCKS[identifier_str]
-
- def should_refresh(self):
+ if self._should_refresh():
+ with self.lock:
+ if self._should_refresh():
+ self.refresh_token()
+
+ def _should_refresh(self):
if self.token is None:
return True
current_time = int(time.time() * 1000)
- time_until_expiry = self.token.expire_at_millis - current_time
- return time_until_expiry < RESTApi.TOKEN_EXPIRATION_SAFE_TIME_MILLIS
+ return (self.token.expire_at_millis - current_time) < RESTApi.TOKEN_EXPIRATION_SAFE_TIME_MILLIS
def refresh_token(self):
self.log.info(f"begin refresh data token for identifier [{self.identifier}]")
if self.api_instance is None:
self.api_instance = RESTApi(self.properties, False)
- response = self.api_instance.load_table_token(self.identifier)
+ table_identifier = self.identifier
+ if SYSTEM_TABLE_SPLITTER in self.identifier.get_object_name():
+ base_table = self.identifier.get_object_name().split(SYSTEM_TABLE_SPLITTER)[0]
+ table_identifier = Identifier(
+ database=self.identifier.get_database_name(),
+ object=base_table,
+ branch=self.identifier.get_branch_name())
+
+ response = self.api_instance.load_table_token(table_identifier)
self.log.info(
f"end refresh data token for identifier [{self.identifier}] expiresAtMillis [{response.expires_at_millis}]"
)
-
+
merged_token_dict = self._merge_token_with_catalog_options(response.token)
new_token = RESTToken(merged_token_dict, response.expires_at_millis)
self.token = new_token
diff --git a/paimon-python/pypaimon/tests/rest/rest_token_file_io_test.py b/paimon-python/pypaimon/tests/rest/rest_token_file_io_test.py
index 47ea8e6cb6..cdcd5ed36c 100644
--- a/paimon-python/pypaimon/tests/rest/rest_token_file_io_test.py
+++ b/paimon-python/pypaimon/tests/rest/rest_token_file_io_test.py
@@ -18,10 +18,12 @@
import os
import pickle
import tempfile
+import time
import unittest
-from unittest.mock import patch
+from unittest.mock import patch, MagicMock
from pypaimon.catalog.rest.rest_token_file_io import RESTTokenFileIO
+from pypaimon.catalog.rest.rest_token import RESTToken
from pypaimon.common.identifier import Identifier
from pypaimon.common.options import Options
@@ -101,13 +103,13 @@ class RESTTokenFileIOTest(unittest.TestCase):
target_dir = os.path.join(self.temp_dir, "target_dir")
os.makedirs(target_dir)
-
+
result = file_io.try_to_write_atomic(f"file://{target_dir}", "test content")
self.assertFalse(result, "try_to_write_atomic should return False when target is a directory")
-
+
self.assertTrue(os.path.isdir(target_dir))
self.assertEqual(len(os.listdir(target_dir)), 0, "No file should be created inside the directory")
-
+
normal_file = os.path.join(self.temp_dir, "normal_file.txt")
result = file_io.try_to_write_atomic(f"file://{normal_file}", "test content")
self.assertTrue(result, "try_to_write_atomic should succeed for a normal file path")
@@ -223,35 +225,35 @@ class RESTTokenFileIOTest(unittest.TestCase):
CatalogOptions.URI.key(): "http://test-uri",
"custom.key": "custom.value"
})
-
+
catalog_options_copy = Options(original_catalog_options.to_map())
-
+
with patch.object(RESTTokenFileIO, 'try_to_refresh_token'):
file_io = RESTTokenFileIO(
self.identifier,
self.warehouse_path,
original_catalog_options
)
-
+
token_dict = {
OssOptions.OSS_ACCESS_KEY_ID.key(): "token-access-key",
OssOptions.OSS_ACCESS_KEY_SECRET.key(): "token-secret-key",
OssOptions.OSS_ENDPOINT.key(): "token-endpoint"
}
-
+
merged_token = file_io._merge_token_with_catalog_options(token_dict)
-
+
self.assertEqual(
original_catalog_options.to_map(),
catalog_options_copy.to_map(),
"Original catalog_options should not be modified"
)
-
+
merged_properties = RESTUtil.merge(
original_catalog_options.to_map(),
merged_token
)
-
+
self.assertIn("custom.key", merged_properties)
self.assertEqual(merged_properties["custom.key"], "custom.value")
self.assertIn(OssOptions.OSS_ACCESS_KEY_ID.key(), merged_properties)
@@ -264,11 +266,11 @@ class RESTTokenFileIOTest(unittest.TestCase):
self.warehouse_path,
self.catalog_options
)
-
+
self.assertTrue(hasattr(file_io, 'filesystem'), "RESTTokenFileIO should have filesystem property")
filesystem = file_io.filesystem
self.assertIsNotNone(filesystem, "filesystem should not be None")
-
+
self.assertTrue(hasattr(filesystem, 'open_input_file'),
"filesystem should support open_input_file method")
@@ -279,12 +281,12 @@ class RESTTokenFileIOTest(unittest.TestCase):
self.warehouse_path,
self.catalog_options
)
-
+
self.assertTrue(hasattr(file_io, 'uri_reader_factory'),
"RESTTokenFileIO should have uri_reader_factory property")
uri_reader_factory = file_io.uri_reader_factory
self.assertIsNotNone(uri_reader_factory, "uri_reader_factory should not be None")
-
+
self.assertTrue(hasattr(uri_reader_factory, 'create'),
"uri_reader_factory should support create method")
@@ -295,15 +297,143 @@ class RESTTokenFileIOTest(unittest.TestCase):
self.warehouse_path,
self.catalog_options
)
-
+
pickled = pickle.dumps(original_file_io)
restored_file_io = pickle.loads(pickled)
-
+
self.assertIsNotNone(restored_file_io.filesystem,
"filesystem should work after deserialization")
self.assertIsNotNone(restored_file_io.uri_reader_factory,
"uri_reader_factory should work after deserialization")
+ def test_should_refresh_when_token_is_none(self):
+ """_should_refresh() returns True when token is None."""
+ with patch.object(RESTTokenFileIO, 'try_to_refresh_token'):
+ file_io = RESTTokenFileIO(
+ self.identifier, self.warehouse_path, self.catalog_options)
+ self.assertIsNone(file_io.token)
+ self.assertTrue(file_io._should_refresh())
+
+ def test_should_refresh_when_token_not_expired(self):
+ """_should_refresh() returns False when token is far from expiry."""
+ with patch.object(RESTTokenFileIO, 'try_to_refresh_token'):
+ file_io = RESTTokenFileIO(
+ self.identifier, self.warehouse_path, self.catalog_options)
+ # Token that expires 2 hours from now (well beyond the 1-hour safe margin)
+ future_millis = int(time.time() * 1000) + 7200_000
+ file_io.token = RESTToken({'ak': 'v'}, future_millis)
+ self.assertFalse(file_io._should_refresh())
+
+ def test_should_refresh_when_token_expired(self):
+ """_should_refresh() returns True when token is already expired."""
+ with patch.object(RESTTokenFileIO, 'try_to_refresh_token'):
+ file_io = RESTTokenFileIO(
+ self.identifier, self.warehouse_path, self.catalog_options)
+ # Token that expired 1 second ago
+ past_millis = int(time.time() * 1000) - 1000
+ file_io.token = RESTToken({'ak': 'v'}, past_millis)
+ self.assertTrue(file_io._should_refresh())
+
+ def test_try_to_refresh_token_calls_refresh_once(self):
+ """try_to_refresh_token() calls refresh_token() exactly once via double-check."""
+ file_io = RESTTokenFileIO(
+ self.identifier, self.warehouse_path, self.catalog_options)
+ self.assertIsNone(file_io.token)
+
+ mock_response = MagicMock()
+ mock_response.token = {'ak': 'test-ak'}
+ mock_response.expires_at_millis = int(time.time() * 1000) + 7200_000
+
+ mock_api = MagicMock()
+ mock_api.load_table_token.return_value = mock_response
+ file_io.api_instance = mock_api
+
+ file_io.try_to_refresh_token()
+
+ mock_api.load_table_token.assert_called_once()
+ self.assertIsNotNone(file_io.token)
+
+ # Second call should NOT trigger refresh again (token is valid)
+ file_io.try_to_refresh_token()
+ mock_api.load_table_token.assert_called_once()
+
+ def test_refresh_token_strips_system_table_suffix(self):
+ """refresh_token() strips $snapshots suffix before requesting token."""
+ system_identifier = Identifier.create("db", "my_table$snapshots")
+ file_io = RESTTokenFileIO(
+ system_identifier, self.warehouse_path, self.catalog_options)
+
+ mock_response = MagicMock()
+ mock_response.token = {'ak': 'test-ak'}
+ mock_response.expires_at_millis = int(time.time() * 1000) + 7200_000
+
+ mock_api = MagicMock()
+ mock_api.load_table_token.return_value = mock_response
+ file_io.api_instance = mock_api
+
+ file_io.refresh_token()
+
+ # Verify load_table_token was called with base table identifier (no $snapshots)
+ called_identifier = mock_api.load_table_token.call_args[0][0]
+ self.assertEqual(called_identifier.get_database_name(), "db")
+ self.assertEqual(called_identifier.get_object_name(), "my_table")
+
+ def test_refresh_token_keeps_normal_identifier(self):
+ """refresh_token() does not modify normal (non-system) identifiers."""
+ normal_identifier = Identifier.create("db", "my_table")
+ file_io = RESTTokenFileIO(
+ normal_identifier, self.warehouse_path, self.catalog_options)
+
+ mock_response = MagicMock()
+ mock_response.token = {'ak': 'test-ak'}
+ mock_response.expires_at_millis = int(time.time() * 1000) + 7200_000
+
+ mock_api = MagicMock()
+ mock_api.load_table_token.return_value = mock_response
+ file_io.api_instance = mock_api
+
+ file_io.refresh_token()
+
+ called_identifier = mock_api.load_table_token.call_args[0][0]
+ self.assertEqual(called_identifier.get_object_name(), "my_table")
+
+ def test_different_instances_do_not_share_token(self):
+ """Two instances with same identifier get independent tokens (no class-level cache)."""
+ same_identifier = Identifier.from_string("db.shared_table")
+
+ file_io_a = RESTTokenFileIO(
+ same_identifier, self.warehouse_path, self.catalog_options)
+ file_io_b = RESTTokenFileIO(
+ same_identifier, self.warehouse_path, self.catalog_options)
+
+ token_a = RESTToken({'ak': 'ak-A'}, int(time.time() * 1000) + 7200_000)
+ token_b = RESTToken({'ak': 'ak-B'}, int(time.time() * 1000) + 7200_000)
+
+ mock_response_a = MagicMock()
+ mock_response_a.token = token_a.token
+ mock_response_a.expires_at_millis = token_a.expire_at_millis
+
+ mock_response_b = MagicMock()
+ mock_response_b.token = token_b.token
+ mock_response_b.expires_at_millis = token_b.expire_at_millis
+
+ mock_api_a = MagicMock()
+ mock_api_a.load_table_token.return_value = mock_response_a
+ file_io_a.api_instance = mock_api_a
+
+ mock_api_b = MagicMock()
+ mock_api_b.load_table_token.return_value = mock_response_b
+ file_io_b.api_instance = mock_api_b
+
+ # Refresh both
+ file_io_a.try_to_refresh_token()
+ file_io_b.try_to_refresh_token()
+
+ # Each instance should hold its own token
+ self.assertEqual(file_io_a.token.token['ak'], 'ak-A')
+ self.assertEqual(file_io_b.token.token['ak'], 'ak-B')
+ self.assertIsNot(file_io_a.token, file_io_b.token)
+
if __name__ == '__main__':
unittest.main()
(paimon) 24/37: [core] Ensuring JDBC connection is not stale before using it (#7470)
Posted by ho...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hope pushed a commit to branch release-1.4
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit 9eee82f6e12c4ffe87decaf29dd2d33423bef9e6
Author: junmuz <47...@users.noreply.github.com>
AuthorDate: Sun Mar 29 13:05:34 2026 +0100
[core] Ensuring JDBC connection is not stale before using it (#7470)
---
.../java/org/apache/paimon/client/ClientPool.java | 9 ++
.../org/apache/paimon/jdbc/JdbcClientPool.java | 34 +++++
.../org/apache/paimon/jdbc/JdbcClientPoolTest.java | 152 +++++++++++++++++++++
3 files changed, 195 insertions(+)
diff --git a/paimon-common/src/main/java/org/apache/paimon/client/ClientPool.java b/paimon-common/src/main/java/org/apache/paimon/client/ClientPool.java
index c42a30b353..b635a0882c 100644
--- a/paimon-common/src/main/java/org/apache/paimon/client/ClientPool.java
+++ b/paimon-common/src/main/java/org/apache/paimon/client/ClientPool.java
@@ -65,6 +65,7 @@ public interface ClientPool<C, E extends Exception> {
continue;
}
try {
+ client = ensureActiveClient(client);
return action.run(client);
} finally {
clients.addFirst(client);
@@ -72,6 +73,14 @@ public interface ClientPool<C, E extends Exception> {
}
}
+ /**
+ * Validates that the client is still active before use. Subclasses should override this
+ * method to provide protocol-specific validation and reconnection logic.
+ */
+ protected C ensureActiveClient(C client) {
+ return client;
+ }
+
@Override
public void execute(ExecuteAction<C, E> action) throws E, InterruptedException {
run(
diff --git a/paimon-core/src/main/java/org/apache/paimon/jdbc/JdbcClientPool.java b/paimon-core/src/main/java/org/apache/paimon/jdbc/JdbcClientPool.java
index d2ef4ecf73..80c97b4a1b 100644
--- a/paimon-core/src/main/java/org/apache/paimon/jdbc/JdbcClientPool.java
+++ b/paimon-core/src/main/java/org/apache/paimon/jdbc/JdbcClientPool.java
@@ -20,6 +20,9 @@ package org.apache.paimon.jdbc;
import org.apache.paimon.client.ClientPool;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
@@ -32,12 +35,16 @@ import java.util.regex.Pattern;
/** Client pool for jdbc. */
public class JdbcClientPool extends ClientPool.ClientPoolImpl<Connection, SQLException> {
+ private static final Logger LOG = LoggerFactory.getLogger(JdbcClientPool.class);
private static final Pattern PROTOCOL_PATTERN = Pattern.compile("jdbc:([^:]+):(.*)");
+ private static final int CONNECTION_VALIDATION_TIMEOUT_SECONDS = 5;
private final String protocol;
+ private final Supplier<Connection> connectionSupplier;
public JdbcClientPool(int poolSize, String dbUrl, Map<String, String> props) {
super(poolSize, clientSupplier(dbUrl, props));
+ this.connectionSupplier = clientSupplier(dbUrl, props);
Matcher matcher = PROTOCOL_PATTERN.matcher(dbUrl);
if (matcher.matches()) {
this.protocol = matcher.group(1);
@@ -46,6 +53,23 @@ public class JdbcClientPool extends ClientPool.ClientPoolImpl<Connection, SQLExc
}
}
+ @Override
+ protected Connection ensureActiveClient(Connection connection) {
+ try {
+ if (connection.isClosed()
+ || !connection.isValid(CONNECTION_VALIDATION_TIMEOUT_SECONDS)) {
+ LOG.warn("Stale JDBC connection detected, creating a new connection.");
+ closeQuietly(connection);
+ return connectionSupplier.get();
+ }
+ } catch (SQLException e) {
+ LOG.warn("Failed to validate JDBC connection, creating a new connection.", e);
+ closeQuietly(connection);
+ return connectionSupplier.get();
+ }
+ return connection;
+ }
+
private static Supplier<Connection> clientSupplier(String dbUrl, Map<String, String> props) {
return () -> {
try {
@@ -70,4 +94,14 @@ public class JdbcClientPool extends ClientPool.ClientPoolImpl<Connection, SQLExc
throw new RuntimeException("Failed to close connection", e);
}
}
+
+ private static void closeQuietly(Connection connection) {
+ try {
+ if (connection != null && !connection.isClosed()) {
+ connection.close();
+ }
+ } catch (SQLException e) {
+ LOG.debug("Failed to close stale connection", e);
+ }
+ }
}
diff --git a/paimon-core/src/test/java/org/apache/paimon/jdbc/JdbcClientPoolTest.java b/paimon-core/src/test/java/org/apache/paimon/jdbc/JdbcClientPoolTest.java
new file mode 100644
index 0000000000..a96a2170bc
--- /dev/null
+++ b/paimon-core/src/test/java/org/apache/paimon/jdbc/JdbcClientPoolTest.java
@@ -0,0 +1,152 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.jdbc;
+
+import org.junit.jupiter.api.Test;
+
+import java.sql.Connection;
+import java.sql.SQLException;
+import java.util.Collections;
+import java.util.UUID;
+import java.util.concurrent.atomic.AtomicReference;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/** Tests for {@link JdbcClientPool} connection validation. */
+public class JdbcClientPoolTest {
+
+ private JdbcClientPool createPool(int poolSize) {
+ String dbUrl =
+ "jdbc:sqlite:file::memory:?ic" + UUID.randomUUID().toString().replace("-", "");
+ return new JdbcClientPool(poolSize, dbUrl, Collections.emptyMap());
+ }
+
+ @Test
+ public void testValidConnectionIsReused() throws SQLException, InterruptedException {
+ JdbcClientPool pool = createPool(1);
+ try {
+ AtomicReference<Connection> firstConn = new AtomicReference<>();
+ AtomicReference<Connection> secondConn = new AtomicReference<>();
+
+ pool.run(
+ connection -> {
+ firstConn.set(connection);
+ return null;
+ });
+
+ pool.run(
+ connection -> {
+ secondConn.set(connection);
+ return null;
+ });
+
+ assertThat(secondConn.get()).isSameAs(firstConn.get());
+ } finally {
+ pool.close();
+ }
+ }
+
+ @Test
+ public void testClosedConnectionIsReplaced() throws SQLException, InterruptedException {
+ JdbcClientPool pool = createPool(1);
+ try {
+ AtomicReference<Connection> firstConn = new AtomicReference<>();
+ AtomicReference<Connection> secondConn = new AtomicReference<>();
+
+ // Get the connection and close it to simulate a stale connection
+ pool.run(
+ connection -> {
+ firstConn.set(connection);
+ connection.close();
+ return null;
+ });
+
+ // The pool should detect the closed connection and create a new one
+ pool.run(
+ connection -> {
+ secondConn.set(connection);
+ return null;
+ });
+
+ assertThat(secondConn.get()).isNotSameAs(firstConn.get());
+ assertThat(secondConn.get().isClosed()).isFalse();
+ } finally {
+ pool.close();
+ }
+ }
+
+ @Test
+ public void testReplacedConnectionIsReturnedToPool() throws SQLException, InterruptedException {
+ JdbcClientPool pool = createPool(1);
+ try {
+ AtomicReference<Connection> replacedConn = new AtomicReference<>();
+ AtomicReference<Connection> thirdConn = new AtomicReference<>();
+
+ // Close the connection to trigger replacement
+ pool.run(
+ connection -> {
+ connection.close();
+ return null;
+ });
+
+ // This call gets the replacement connection
+ pool.run(
+ connection -> {
+ replacedConn.set(connection);
+ return null;
+ });
+
+ // The replacement should be reused since it's valid
+ pool.run(
+ connection -> {
+ thirdConn.set(connection);
+ return null;
+ });
+
+ assertThat(thirdConn.get()).isSameAs(replacedConn.get());
+ } finally {
+ pool.close();
+ }
+ }
+
+ @Test
+ public void testActionIsExecutedOnValidConnection() throws SQLException, InterruptedException {
+ JdbcClientPool pool = createPool(1);
+ try {
+ // Close the connection to simulate staleness
+ pool.run(
+ connection -> {
+ connection.close();
+ return null;
+ });
+
+ // The action should receive a valid connection and succeed
+ boolean result =
+ pool.run(
+ connection -> {
+ // Execute a real SQL statement to verify the connection works
+ return connection.prepareStatement("SELECT 1").execute();
+ });
+
+ assertThat(result).isTrue();
+ } finally {
+ pool.close();
+ }
+ }
+}
(paimon) 20/37: [core] Fix frequent compact triggers during streaming writes for pk clustering override (#7540)
Posted by ho...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hope pushed a commit to branch release-1.4
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit 96f476c16097df34420ad6283a1636bce2b36218
Author: Jingsong Lee <ji...@gmail.com>
AuthorDate: Thu Mar 26 22:17:27 2026 +0800
[core] Fix frequent compact triggers during streaming writes for pk clustering override (#7540)
---
.../clustering/ClusteringCompactManager.java | 3 +
.../paimon/separated/ClusteringTableTest.java | 76 ++++++++++++++++++++++
2 files changed, 79 insertions(+)
diff --git a/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/clustering/ClusteringCompactManager.java b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/clustering/ClusteringCompactManager.java
index 3e288b22ca..1e9b02b32b 100644
--- a/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/clustering/ClusteringCompactManager.java
+++ b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/clustering/ClusteringCompactManager.java
@@ -180,6 +180,9 @@ public class ClusteringCompactManager extends CompactFutureManager {
@Override
public void triggerCompaction(boolean fullCompaction) {
+ if (taskFuture != null) {
+ return;
+ }
taskFuture =
executor.submit(
new CompactTask(metricsReporter) {
diff --git a/paimon-core/src/test/java/org/apache/paimon/separated/ClusteringTableTest.java b/paimon-core/src/test/java/org/apache/paimon/separated/ClusteringTableTest.java
index e33afb7927..e96a18cdd6 100644
--- a/paimon-core/src/test/java/org/apache/paimon/separated/ClusteringTableTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/separated/ClusteringTableTest.java
@@ -23,6 +23,7 @@ import org.apache.paimon.catalog.Catalog;
import org.apache.paimon.catalog.CatalogContext;
import org.apache.paimon.catalog.CatalogFactory;
import org.apache.paimon.catalog.Identifier;
+import org.apache.paimon.data.BinaryRow;
import org.apache.paimon.data.GenericRow;
import org.apache.paimon.data.InternalRow;
import org.apache.paimon.disk.IOManager;
@@ -35,6 +36,9 @@ import org.apache.paimon.table.Table;
import org.apache.paimon.table.sink.BatchTableCommit;
import org.apache.paimon.table.sink.BatchTableWrite;
import org.apache.paimon.table.sink.BatchWriteBuilder;
+import org.apache.paimon.table.sink.StreamTableCommit;
+import org.apache.paimon.table.sink.StreamTableWrite;
+import org.apache.paimon.table.sink.StreamWriteBuilder;
import org.apache.paimon.table.source.DataSplit;
import org.apache.paimon.table.source.RawFile;
import org.apache.paimon.table.source.ReadBuilder;
@@ -787,6 +791,78 @@ class ClusteringTableTest {
GenericRow.of(4, 44));
}
+ // ==================== Stream Write with Frequent Compact Tests ====================
+
+ /**
+ * Test that frequent compact triggers during streaming writes don't corrupt data. This
+ * exercises the {@code if (taskFuture != null) return} guard in {@code
+ * ClusteringCompactManager.triggerCompaction}.
+ */
+ @Test
+ public void testStreamWriteFrequentCompact() throws Exception {
+ StreamWriteBuilder writeBuilder = table.newStreamWriteBuilder();
+ try (StreamTableWrite write =
+ (StreamTableWrite) writeBuilder.newWrite().withIOManager(ioManager);
+ StreamTableCommit commit = writeBuilder.newCommit()) {
+ long commitId = 0;
+ for (int round = 0; round < 20; round++) {
+ write.write(GenericRow.of(round, round * 10));
+ // Trigger compact on every round for both buckets
+ write.compact(BinaryRow.EMPTY_ROW, 0, false);
+ write.compact(BinaryRow.EMPTY_ROW, 1, false);
+ commit.commit(commitId, write.prepareCommit(false, commitId));
+ commitId++;
+ }
+
+ // Final commit with waitCompaction to ensure all compactions finish
+ commit.commit(commitId, write.prepareCommit(true, commitId));
+
+ // All 20 distinct keys should be present
+ List<GenericRow> expected = new ArrayList<>();
+ for (int i = 0; i < 20; i++) {
+ expected.add(GenericRow.of(i, i * 10));
+ }
+ assertThat(readRows()).containsExactlyInAnyOrderElementsOf(expected);
+ }
+ }
+
+ /**
+ * Test streaming writes with overlapping keys and frequent compaction. The {@code taskFuture !=
+ * null} guard should safely skip redundant compaction triggers while still producing correct
+ * deduplicated results.
+ */
+ @Test
+ public void testStreamWriteFrequentCompactWithOverlappingKeys() throws Exception {
+ StreamWriteBuilder writeBuilder = table.newStreamWriteBuilder();
+ try (StreamTableWrite write =
+ (StreamTableWrite) writeBuilder.newWrite().withIOManager(ioManager);
+ StreamTableCommit commit = writeBuilder.newCommit()) {
+ long commitId = 0;
+ for (int round = 0; round < 20; round++) {
+ // Write overlapping keys: keys 0-4 are updated every round
+ for (int i = 0; i < 5; i++) {
+ write.write(GenericRow.of(i, round * 100 + i));
+ }
+ write.compact(BinaryRow.EMPTY_ROW, 0, false);
+ write.compact(BinaryRow.EMPTY_ROW, 1, false);
+ commit.commit(commitId, write.prepareCommit(false, commitId));
+ commitId++;
+ }
+
+ // Final wait for compaction
+ commit.commit(commitId, write.prepareCommit(true, commitId));
+
+ // Should see only the latest values from round 19
+ assertThat(readRows())
+ .containsExactlyInAnyOrder(
+ GenericRow.of(0, 1900),
+ GenericRow.of(1, 1901),
+ GenericRow.of(2, 1902),
+ GenericRow.of(3, 1903),
+ GenericRow.of(4, 1904));
+ }
+ }
+
private Table createFirstRowTableWithLowSpillThreshold() throws Exception {
Identifier identifier = Identifier.create("default", "first_row_spill_table");
Schema schema =
(paimon) 14/37: [python] Include LICENSE and NOTICE in sdist for Apache release compliance (#7533)
Posted by ho...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hope pushed a commit to branch release-1.4
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit 78dcf0be5649d7eb167427fd3004f3636191ed5b
Author: umi <55...@users.noreply.github.com>
AuthorDate: Thu Mar 26 15:23:42 2026 +0800
[python] Include LICENSE and NOTICE in sdist for Apache release compliance (#7533)
---
paimon-python/MANIFEST.in | 2 ++
paimon-python/NOTICE | 5 +++++
2 files changed, 7 insertions(+)
diff --git a/paimon-python/MANIFEST.in b/paimon-python/MANIFEST.in
index fa6d79637e..4d3df3ade9 100644
--- a/paimon-python/MANIFEST.in
+++ b/paimon-python/MANIFEST.in
@@ -15,6 +15,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
################################################################################
+include LICENSE
+include NOTICE
include dev/requirements.txt
include README.md
recursive-include pypaimon *.py
\ No newline at end of file
diff --git a/paimon-python/NOTICE b/paimon-python/NOTICE
new file mode 100644
index 0000000000..cc3b41cc37
--- /dev/null
+++ b/paimon-python/NOTICE
@@ -0,0 +1,5 @@
+Apache Paimon (PyPaimon)
+Copyright 2023-2026 The Apache Software Foundation
+
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).
(paimon) 13/37: [doc] Introduce Scenario Guide in Learn Paimon
Posted by ho...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hope pushed a commit to branch release-1.4
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit 77e6437a6320dcedcce411cc71d5961aad9b82b7
Author: JingsongLi <ji...@gmail.com>
AuthorDate: Thu Mar 26 12:09:03 2026 +0800
[doc] Introduce Scenario Guide in Learn Paimon
---
docs/content/learn-paimon/scenario-guide.md | 598 ++++++++++++++++++++++++++++
1 file changed, 598 insertions(+)
diff --git a/docs/content/learn-paimon/scenario-guide.md b/docs/content/learn-paimon/scenario-guide.md
new file mode 100644
index 0000000000..9cb7c76bb6
--- /dev/null
+++ b/docs/content/learn-paimon/scenario-guide.md
@@ -0,0 +1,598 @@
+---
+title: "Scenario Guide"
+weight: 2
+type: docs
+aliases:
+- /learn-paimon/scenario-guide.html
+---
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# Scenario Guide
+
+This guide helps you choose the right Paimon table type and configuration for your specific use case. Paimon provides
+**Primary Key Table**, **Append Table**, and **Multimodal Data Lake** capabilities — each with different modes and
+configurations that are suited for different scenarios.
+
+## Quick Decision
+
+| Scenario | Table Type | Key Configuration |
+|---|---|---|
+| CDC real-time sync from database | Primary Key Table | `deletion-vectors.enabled = true` |
+| Streaming aggregation / metrics | Primary Key Table | `merge-engine = aggregation` |
+| Multi-stream partial column updates | Primary Key Table | `merge-engine = partial-update` |
+| Log deduplication (keep first) | Primary Key Table | `merge-engine = first-row` |
+| Batch ETL / data warehouse layers | Append Table | Default (unaware-bucket) |
+| High-frequency point queries on key | Append Table | `bucket = N, bucket-key = col` |
+| Queue-like ordered streaming | Append Table | `bucket = N, bucket-key = col` |
+| Large-scale OLAP with ad-hoc queries | Append Table | Incremental Clustering |
+| Store images / videos / documents | Append Table (Blob) | `blob-field`, Data Evolution enabled |
+| AI vector search / RAG | Append Table (Vector) | `VECTOR` type, Global Index (DiskANN) |
+| AI feature engineering & column evolution | Append Table | `data-evolution.enabled = true` |
+| Python AI pipeline (Ray / PyTorch) | Append Table | PyPaimon SDK |
+
+---
+
+## Primary Key Table
+
+Use a Primary Key Table when your data has a natural unique key and you need **real-time updates** (insert, update, delete).
+See [Primary Key Table Overview]({{< ref "primary-key-table/overview" >}}).
+
+### Scenario 1: CDC Real-Time Sync
+
+**When:** You want to synchronize a MySQL / PostgreSQL / MongoDB table to the data lake in real-time with upsert
+semantics. This is the most common use case for Primary Key Tables.
+
+**Recommended Configuration:**
+
+```sql
+CREATE TABLE orders (
+ order_id BIGINT,
+ user_id BIGINT,
+ amount DECIMAL(10,2),
+ status STRING,
+ update_time TIMESTAMP,
+ dt STRING,
+ PRIMARY KEY (order_id, dt) NOT ENFORCED
+) PARTITIONED BY (dt) WITH (
+ 'deletion-vectors.enabled' = 'true',
+ 'changelog-producer' = 'lookup',
+ 'sequence.field' = 'update_time'
+);
+```
+
+**Why this configuration:**
+
+- **`deletion-vectors.enabled = true`** (MOW mode): Enables [Merge On Write]({{< ref "primary-key-table/table-mode#merge-on-write" >}}) with Deletion Vectors.
+ This mode gives you the best balance of write and read performance. Compared to the default MOR mode, MOW
+ avoids merging at read time, which greatly improves OLAP query performance.
+- **`changelog-producer = lookup`**: Generates a complete [changelog]({{< ref "primary-key-table/changelog-producer#lookup" >}})
+ for downstream streaming consumers. If no downstream streaming read is needed, you can omit this to save
+ compaction resources.
+- **`sequence.field = update_time`**: Guarantees correct update ordering even when data arrives out of order.
+- **Bucketing**: Use the default Dynamic Bucket (`bucket = -1`). The system automatically adjusts bucket count based
+ on data volume. If you are sensitive to data visibility latency, set a fixed bucket number (e.g. `'bucket' = '5'`)
+ — roughly 1 bucket per 1GB of data in a partition.
+
+**CDC Ingestion Tip:** Use [Paimon CDC Ingestion]({{< ref "cdc-ingestion/overview" >}}) for whole-database sync with
+automatic table creation and schema evolution support.
+
+### Scenario 2: Multi-Stream Partial Column Updates
+
+**When:** Multiple data sources each contribute different columns to the same record, and you want to progressively
+merge them into a complete wide table (e.g., orders from one stream + logistics info from another).
+
+**Recommended Configuration:**
+
+```sql
+CREATE TABLE order_wide (
+ order_id BIGINT PRIMARY KEY NOT ENFORCED,
+ -- from order stream
+ user_name STRING,
+ amount DECIMAL(10,2),
+ order_time TIMESTAMP,
+ -- from logistics stream
+ tracking_no STRING,
+ delivery_status STRING,
+ delivery_time TIMESTAMP
+) WITH (
+ 'merge-engine' = 'partial-update',
+ 'fields.order_time.sequence-group' = 'user_name,amount',
+ 'fields.delivery_time.sequence-group' = 'tracking_no,delivery_status',
+ 'deletion-vectors.enabled' = 'true',
+ 'changelog-producer' = 'lookup'
+);
+```
+
+**Why:** The [partial-update]({{< ref "primary-key-table/merge-engine/partial-update" >}}) merge engine allows each
+stream to update only its own columns without overwriting the others. `sequence-group` ensures ordering within each
+stream independently.
+
+### Scenario 3: Streaming Aggregation / Metrics
+
+**When:** You need to pre-aggregate metrics in real-time (e.g., page views, total sales, UV count). Each incoming record
+should be aggregated with the existing value, not replace it.
+
+**Recommended Configuration:**
+
+```sql
+CREATE TABLE product_metrics (
+ product_id BIGINT,
+ dt STRING,
+ total_sales BIGINT,
+ max_price DOUBLE,
+ uv VARBINARY,
+ PRIMARY KEY (product_id, dt) NOT ENFORCED
+) PARTITIONED BY (dt) WITH (
+ 'merge-engine' = 'aggregation',
+ 'fields.total_sales.aggregate-function' = 'sum',
+ 'fields.max_price.aggregate-function' = 'max',
+ 'fields.uv.aggregate-function' = 'hll_sketch',
+ 'deletion-vectors.enabled' = 'true',
+ 'changelog-producer' = 'lookup'
+);
+```
+
+**Why:** The [aggregation]({{< ref "primary-key-table/merge-engine/aggregation" >}}) merge engine supports 20+
+aggregate functions (`sum`, `max`, `min`, `count`, `hll_sketch`, `theta_sketch`, `collect`, `merge_map`, etc.), ideal
+for real-time metric accumulation.
+
+### Scenario 4: Log Deduplication (Keep First Record)
+
+**When:** You receive a high-volume log stream with possible duplicates and only want to keep the first occurrence of
+each key (e.g., first login event per user per day).
+
+**Recommended Configuration:**
+
+```sql
+CREATE TABLE first_login (
+ user_id BIGINT,
+ dt STRING,
+ login_time TIMESTAMP,
+ device STRING,
+ ip STRING,
+ PRIMARY KEY (user_id, dt) NOT ENFORCED
+) PARTITIONED BY (dt) WITH (
+ 'merge-engine' = 'first-row',
+ 'changelog-producer' = 'lookup'
+);
+```
+
+**Why:** The [first-row]({{< ref "primary-key-table/merge-engine/first-row" >}}) merge engine keeps only the earliest
+record for each primary key and produces insert-only changelog, making it perfect for streaming log deduplication.
+
+### Primary Key Table: Bucket Mode Comparison
+
+| Mode | Config | Best For | Trade-off |
+|---|---|---|---|
+| Dynamic Bucket (default) | `bucket = -1` | Most scenarios, auto-scaling | Requires single write job |
+| Fixed Bucket | `bucket = N` | Stable workloads, bucketed join | Manual rescaling needed |
+| Postpone Bucket | `bucket = -2` | Adaptive Partition Level Bucket | New data not visible until compaction |
+
+**General guideline:** ~1 bucket per 1GB of data in a partition, with each bucket containing 200MB–1GB of data.
+
+### Primary Key Table: Table Mode Comparison
+
+| Mode | Config | Write Perf | Read Perf | Best For |
+|---|---|---|---|---|
+| MOR (default) | — | Very good | Not so good | Write-heavy, less query |
+| COW | `full-compaction.delta-commits = 1` | Very bad | Very good | Read-heavy, batch jobs |
+| MOW | `deletion-vectors.enabled = true` | Good | Good | Balanced (recommended for most) |
+
+---
+
+## Append Table
+
+Use an Append Table when your data **has no natural primary key**, or you are working with **batch ETL** pipelines
+where data is only inserted and does not need upsert semantics.
+See [Append Table Overview]({{< ref "append-table/overview" >}}).
+
+Compared to Primary Key Tables, Append Tables have much better batch read/write performance, simpler design, and lower
+resource consumption. **We recommend using Append Tables for most batch processing scenarios.**
+
+### Scenario 5: Batch ETL / Data Warehouse Layers (Unaware-Bucket)
+
+**When:** Standard data warehouse layering (ODS → DWD → DWS → ADS), bulk INSERT OVERWRITE, or Spark / Hive style
+batch processing.
+
+**Recommended Configuration:**
+
+```sql
+CREATE TABLE dwd_events (
+ event_id BIGINT,
+ user_id BIGINT,
+ event_type STRING,
+ event_time TIMESTAMP,
+ dt STRING
+) PARTITIONED BY (dt);
+```
+
+No bucket configuration needed. This is an **unaware-bucket** append table — the simplest and most commonly used form.
+Paimon automatically handles small file merging and supports:
+
+- **Time travel** and version rollback.
+- **Schema evolution** (add/drop/rename columns).
+- **Data skipping** via min-max stats in manifest files.
+- **File Index** (BloomFilter, Bitmap, Range Bitmap) for further query acceleration.
+- **Row-level operations** (DELETE / UPDATE / MERGE INTO in Spark SQL).
+- **Incremental Clustering** for advanced data layout optimization.
+
+**Query Optimization Tip:** If your queries frequently filter on specific columns, consider using
+[Incremental Clustering]({{< ref "append-table/incremental-clustering" >}}) to sort data by those columns:
+
+```sql
+ALTER TABLE dwd_events SET (
+ 'clustering.incremental' = 'true',
+ 'clustering.columns' = 'user_id'
+);
+```
+
+Or define a File Index for point lookups:
+
+```sql
+ALTER TABLE dwd_events SET (
+ 'file-index.bloom-filter.columns' = 'user_id'
+);
+```
+
+### Scenario 6: High-Frequency Point Queries (Bucketed Append)
+
+**When:** Your append table is frequently queried with equality or IN filters on a specific column
+(e.g., `WHERE product_id = xxx`). This is the **most impactful** advantage of a bucketed append table.
+
+**Recommended Configuration:**
+
+```sql
+CREATE TABLE product_logs (
+ product_id BIGINT,
+ log_time TIMESTAMP,
+ message STRING,
+ dt STRING
+) PARTITIONED BY (dt) WITH (
+ 'bucket' = '16',
+ 'bucket-key' = 'product_id'
+);
+```
+
+**Why this is powerful:** The `bucket-key` enables **data skipping** — when a query contains `=` or `IN` conditions on
+the bucket-key, Paimon pushes these predicates down and prunes all irrelevant bucket files entirely. With 16 buckets,
+a point query on `product_id` only reads ~1/16 of the data.
+
+```sql
+-- Only reads the bucket containing product_id=12345, skips all other 15 buckets
+SELECT * FROM product_logs WHERE product_id = 12345;
+
+-- Only reads buckets for these 3 values
+SELECT * FROM product_logs WHERE product_id IN (1, 2, 3);
+```
+
+See [Bucketed Append — Data Skipping]({{< ref "append-table/bucketed#data-skipping" >}}).
+
+**Bucketed Join Bonus:** If two bucketed tables share the same `bucket-key` and bucket count, Spark can join them
+**without shuffle**, significantly accelerating batch join queries:
+
+```sql
+SET spark.sql.sources.v2.bucketing.enabled = true;
+
+-- Both tables have bucket=16, bucket-key=product_id
+SELECT * FROM product_logs JOIN product_dim
+ON product_logs.product_id = product_dim.product_id;
+```
+
+See [Bucketed Join]({{< ref "append-table/bucketed#bucketed-join" >}}).
+
+### Scenario 7: Queue-Like Ordered Streaming (Bucketed Append)
+
+**When:** You want to use Paimon as a message queue replacement with strict ordering guarantees per key (similar to
+Kafka partitioning), with the benefits of filter push-down and lower cost.
+
+**Recommended Configuration:**
+
+```sql
+CREATE TABLE event_stream (
+ user_id BIGINT,
+ event_type STRING,
+ event_time TIMESTAMP(3),
+ payload STRING,
+ WATERMARK FOR event_time AS event_time - INTERVAL '5' SECOND
+) WITH (
+ 'bucket' = '8',
+ 'bucket-key' = 'user_id'
+);
+```
+
+**Why:** Within the same bucket, records are strictly ordered by write time. Streaming reads deliver records in exact
+write order per bucket. This gives you Kafka-like partitioned ordering at data lake cost.
+
+See [Bucketed Streaming]({{< ref "append-table/bucketed#bucketed-streaming" >}}).
+
+### Append Table: Bucket Mode Comparison
+
+| Mode | Config | Data Skipping | Bucketed Join | Ordered Streaming | Incremental Clustering |
+|---|---|---|---|---|---|
+| Unaware-Bucket (default) | No bucket config | Via min-max / file index | No | No | Yes |
+| Bucketed | `bucket = N, bucket-key = col` | **Bucket-key filter pushdown** | Yes | Yes | No |
+
+---
+
+## Multimodal Data Lake
+
+Paimon is a multimodal lakehouse for AI. You can keep multimodal data, metadata, and embeddings in the same table and
+query them via vector search, full-text search, or SQL. All multimodal features are built on top of Append Tables with
+[Data Evolution]({{< ref "append-table/data-evolution" >}}) mode enabled.
+
+### Scenario 8: Storing Multimodal Data (Blob Table)
+
+**When:** You need to store images, videos, audio files, documents, or model weights alongside structured metadata in
+the data lake, and want efficient column projection without loading large binary data.
+
+**Recommended Configuration:**
+
+{{< tabs "blob-scenario" >}}
+{{< tab "Flink SQL" >}}
+```sql
+CREATE TABLE image_table (
+ id INT,
+ name STRING,
+ label STRING,
+ image BYTES
+) WITH (
+ 'row-tracking.enabled' = 'true',
+ 'data-evolution.enabled' = 'true',
+ 'blob-field' = 'image'
+);
+```
+{{< /tab >}}
+{{< tab "Spark SQL" >}}
+```sql
+CREATE TABLE image_table (
+ id INT,
+ name STRING,
+ label STRING,
+ image BINARY
+) TBLPROPERTIES (
+ 'row-tracking.enabled' = 'true',
+ 'data-evolution.enabled' = 'true',
+ 'blob-field' = 'image'
+);
+```
+{{< /tab >}}
+{{< /tabs >}}
+
+**Why:** The [Blob Storage]({{< ref "append-table/blob" >}}) separates large binary data into dedicated `.blob` files
+while metadata stays in standard columnar files (Parquet/ORC). This means:
+
+- `SELECT id, name, label FROM image_table` does **not** load any blob data — very fast.
+- Blob data supports streaming reads for large objects (videos, model weights) without loading entire files into memory.
+- Supports multiple input methods: local files, HTTP URLs, InputStreams, and byte arrays.
+
+**For external storage (e.g., blobs already in S3):**
+
+```sql
+CREATE TABLE video_table (
+ id INT,
+ title STRING,
+ video BYTES
+) WITH (
+ 'row-tracking.enabled' = 'true',
+ 'data-evolution.enabled' = 'true',
+ 'blob-descriptor-field' = 'video',
+ 'blob-external-storage-field' = 'video',
+ 'blob-external-storage-path' = 's3://my-bucket/paimon-blobs/'
+);
+```
+
+This stores only descriptor references inline, while the actual blob data resides in external storage.
+
+### Scenario 9: Vector Search / RAG Applications
+
+**When:** You are building a recommendation system, image retrieval, or RAG (Retrieval Augmented Generation)
+application that needs approximate nearest neighbor (ANN) search on embeddings.
+
+**Recommended Configuration:**
+
+{{< tabs "vector-scenario" >}}
+{{< tab "Spark SQL" >}}
+```sql
+CREATE TABLE doc_embeddings (
+ doc_id INT,
+ title STRING,
+ content STRING,
+ embedding ARRAY<FLOAT>
+) TBLPROPERTIES (
+ 'row-tracking.enabled' = 'true',
+ 'data-evolution.enabled' = 'true',
+ 'global-index.enabled' = 'true',
+ 'vector-field' = 'embedding',
+ 'field.embedding.vector-dim' = '768',
+ 'vector.file.format' = 'lance'
+);
+```
+{{< /tab >}}
+{{< tab "Java API" >}}
+```java
+Schema schema = Schema.newBuilder()
+ .column("doc_id", DataTypes.INT())
+ .column("title", DataTypes.STRING())
+ .column("content", DataTypes.STRING())
+ .column("embedding", DataTypes.VECTOR(768, DataTypes.FLOAT()))
+ .option("bucket", "-1")
+ .option("row-tracking.enabled", "true")
+ .option("data-evolution.enabled", "true")
+ .option("global-index.enabled", "true")
+ .option("vector.file.format", "lance")
+ .build();
+```
+{{< /tab >}}
+{{< /tabs >}}
+
+**Build the vector index and search:**
+
+```sql
+-- Build DiskANN vector index
+CALL sys.create_global_index(
+ table => 'db.doc_embeddings',
+ index_column => 'embedding',
+ index_type => 'lumina-vector-ann',
+ options => 'lumina.index.dimension=768'
+);
+
+-- Search for top-5 nearest neighbors
+SELECT * FROM vector_search('doc_embeddings', 'embedding', array(0.1f, 0.2f, ...), 5);
+```
+
+**Why:** The [Global Index]({{< ref "append-table/global-index" >}}) with DiskANN provides high-performance ANN search.
+Vector data is stored in dedicated `.vector.lance` files optimized for dense vectors, while scalar columns stay in
+Parquet. You can also build a **BTree Index** on scalar columns for efficient filtering:
+
+```sql
+-- Build BTree index for scalar filtering
+CALL sys.create_global_index(
+ table => 'db.doc_embeddings',
+ index_column => 'title',
+ index_type => 'btree'
+);
+
+-- Scalar lookup is accelerated by BTree index
+SELECT * FROM doc_embeddings WHERE title IN ('doc_a', 'doc_b');
+```
+
+### Scenario 10: AI Feature Engineering with Data Evolution
+
+**When:** You have a feature store or data pipeline where new feature columns are added frequently, and you want to
+backfill or update specific columns without rewriting entire data files.
+
+**Recommended Configuration:**
+
+```sql
+CREATE TABLE feature_store (
+ user_id INT,
+ age INT,
+ gender STRING,
+ purchase_count INT
+) TBLPROPERTIES (
+ 'row-tracking.enabled' = 'true',
+ 'data-evolution.enabled' = 'true'
+);
+```
+
+**Update only specific columns via MERGE INTO:**
+
+```sql
+-- Later, add a new feature column
+ALTER TABLE feature_store ADD COLUMNS (embedding ARRAY<FLOAT>);
+
+-- Backfill only the new column — no full file rewrite!
+MERGE INTO feature_store AS t
+USING embedding_source AS s
+ON t.user_id = s.user_id
+WHEN MATCHED THEN UPDATE SET t.embedding = s.embedding;
+```
+
+**Why:** [Data Evolution]({{< ref "append-table/data-evolution" >}}) mode writes only the updated columns to new files
+and merges them at read time. This is ideal for:
+
+- Adding new feature columns and backfilling data without rewriting the entire table.
+- Iterative ML feature engineering — add, update, or refine features as your model evolves.
+- Reducing I/O cost and storage overhead for frequent partial column updates.
+
+### Scenario 11: Python AI Pipeline (PyPaimon)
+
+**When:** You are building ML training or inference pipelines in Python and need to read/write Paimon tables natively
+without JDK dependency.
+
+**Example: Read data for model training:**
+
+```python
+from pypaimon import CatalogFactory
+from torch.utils.data import DataLoader
+
+# Connect to Paimon
+catalog = CatalogFactory.create({'warehouse': 's3://my-bucket/warehouse'})
+table = catalog.get_table('db.feature_store')
+
+# Read with filter and projection
+read_builder = table.new_read_builder()
+read_builder = read_builder.with_projection(['user_id', 'embedding'])
+read_builder = read_builder.with_filter(
+ read_builder.new_predicate_builder().equal('gender', 'M')
+)
+splits = read_builder.new_scan().plan().splits()
+table_read = read_builder.new_read()
+
+# Option 1: Load into PyTorch DataLoader
+dataset = table_read.to_torch(splits, streaming=True, prefetch_concurrency=2)
+dataloader = DataLoader(dataset, batch_size=32, num_workers=4)
+for batch in dataloader:
+ # Training loop
+ pass
+
+# Option 2: Load into Ray for distributed processing
+ray_dataset = table_read.to_ray(splits, override_num_blocks=8)
+mapped = ray_dataset.map(lambda row: {'feat': row['embedding']})
+
+# Option 3: Load into Pandas / PyArrow
+df = table_read.to_pandas(splits)
+arrow_table = table_read.to_arrow(splits)
+```
+
+**Why:** [PyPaimon]({{< ref "pypaimon/overview" >}}) is a pure Python SDK (no JDK required) that integrates seamlessly
+with the Python AI ecosystem:
+
+- **PyTorch**: Direct `DataLoader` integration with streaming and prefetch support.
+- **Ray**: Distributed data processing with configurable parallelism.
+- **Pandas / PyArrow**: Native DataFrame and Arrow Table support for data science workflows.
+- **Data Evolution**: Python API supports `update_by_arrow_with_row_id` and `upsert_by_arrow_with_key` for
+ row-level updates from Python. See [PyPaimon Data Evolution]({{< ref "pypaimon/data-evolution" >}}).
+
+---
+
+## Summary: Table Type Decision Tree
+
+```
+Do you need upsert / update / delete?
+├── YES → Primary Key Table
+│ ├── Simple upsert (keep latest)? → merge-engine = deduplicate (default)
+│ ├── Progressive multi-column updates? → merge-engine = partial-update
+│ ├── Pre-aggregate metrics? → merge-engine = aggregation
+│ └── Dedup keep first? → merge-engine = first-row
+│
+│ Table mode:
+│ ├── Most scenarios → deletion-vectors.enabled = true (MOW, recommended)
+│ ├── Write-heavy, query-light → default MOR
+│ └── Read-heavy, batch → full-compaction.delta-commits = 1 (COW)
+│
+│ Bucket mode:
+│ ├── Most scenarios → bucket = -1 (Dynamic, default)
+│ ├── Stable workload, need bucketed join → bucket = N (Fixed)
+│ └── Unknown distribution → bucket = -2 (Postpone)
+│
+└── NO → Append Table
+ ├── Standard batch ETL? → No bucket config (unaware-bucket)
+ │ └── Need query acceleration? → Incremental Clustering or File Index
+ │
+ ├── Need bucket-key filter pushdown / join / ordered streaming?
+ │ → bucket = N, bucket-key = col (Bucketed Append)
+ │
+ └── AI / Multimodal scenarios? → Enable Data Evolution
+ ├── Store images / videos / docs? → Blob Table (blob-field)
+ ├── Vector search / RAG? → VECTOR type + Global Index (DiskANN)
+ ├── Feature engineering? → Data Evolution (MERGE INTO partial columns)
+ └── Python pipeline? → PyPaimon (Ray / PyTorch / Pandas)
+```
(paimon) 23/37: [python] Fix data evolution merge read with disordered projection (#7544)
Posted by ho...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hope pushed a commit to branch release-1.4
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit bfee09f7d22206920a2e96beac8f02aa80709671
Author: XiaoHongbo <ho...@apache.org>
AuthorDate: Fri Mar 27 14:40:25 2026 +0800
[python] Fix data evolution merge read with disordered projection (#7544)
In the data-evolution merge-read path, when the projection order differs
from the table schema order, there is `ArrowInvalid: Failed to parse
string as a scalar of type int32`. `upsert_by_key` is one path that can
trigger this bug.This PR fixes the above bug.
---
paimon-python/pypaimon/read/split_read.py | 8 +++++++-
paimon-python/pypaimon/tests/data_evolution_test.py | 8 ++++++++
2 files changed, 15 insertions(+), 1 deletion(-)
diff --git a/paimon-python/pypaimon/read/split_read.py b/paimon-python/pypaimon/read/split_read.py
index 498e53e9c0..f9adee4068 100644
--- a/paimon-python/pypaimon/read/split_read.py
+++ b/paimon-python/pypaimon/read/split_read.py
@@ -597,6 +597,7 @@ class DataEvolutionSplitRead(SplitRead):
# Initialize offsets
row_offsets = [-1] * len(all_read_fields)
field_offsets = [-1] * len(all_read_fields)
+ schema_pos = {f.id: p for p, f in enumerate(self.table.fields)}
for i, bunch in enumerate(fields_files):
first_file = bunch.files()[0]
@@ -617,13 +618,18 @@ class DataEvolutionSplitRead(SplitRead):
if read_field_id == field_id:
if row_offsets[j] == -1:
row_offsets[j] = i
- field_offsets[j] = len(read_fields)
read_fields.append(all_read_fields[j])
break
if not read_fields:
file_record_readers[i] = None
else:
+ read_fields.sort(key=lambda f: schema_pos.get(f.id, float('inf')))
+ id_to_pos = {f.id: p for p, f in enumerate(read_fields)}
+ for j in range(len(read_field_index)):
+ if row_offsets[j] == i:
+ field_offsets[j] = id_to_pos[read_field_index[j]]
+
read_field_names = self._remove_partition_fields(read_fields)
table_fields = self.read_fields
self.read_fields = read_fields # create reader based on read_fields
diff --git a/paimon-python/pypaimon/tests/data_evolution_test.py b/paimon-python/pypaimon/tests/data_evolution_test.py
index afb12cd948..796625f6a4 100644
--- a/paimon-python/pypaimon/tests/data_evolution_test.py
+++ b/paimon-python/pypaimon/tests/data_evolution_test.py
@@ -558,6 +558,14 @@ class DataEvolutionTest(unittest.TestCase):
}, schema=simple_pa_schema)
self.assertEqual(actual, expect)
+ read_builder2 = table.new_read_builder()
+ read_builder2.with_projection(['f2', 'f0', 'f1'])
+ actual2 = read_builder2.new_read().to_arrow(
+ read_builder2.new_scan().plan().splits())
+ self.assertEqual(actual2.column('f0').to_pylist(), [2] * num_rows)
+ self.assertEqual(actual2.column('f1').to_pylist(), ['x'] * num_rows)
+ self.assertEqual(actual2.column('f2').to_pylist(), ['y'] * num_rows)
+
def test_only_some_columns(self):
simple_pa_schema = pa.schema([
('f0', pa.int32()),
(paimon) 31/37: [doc] Bucket pruning also works for pk clustering override (#7554)
Posted by ho...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hope pushed a commit to branch release-1.4
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit e5aa9566568839309f2fe9c3abe24e87225a02fc
Author: Jingsong Lee <ji...@gmail.com>
AuthorDate: Mon Mar 30 21:41:06 2026 +0800
[doc] Bucket pruning also works for pk clustering override (#7554)
### Purpose
Although data files are no longer sorted by the primary key, filtering
on bucket-key fields (which default to the primary key) still benefits
from bucket pruning. The query engine can skip entire buckets that do
not contain matching values, so queries like WHERE id = 12345 remain
efficient.
### Tests
---
.../primary-key-table/pk-clustering-override.md | 36 ++++++----------------
1 file changed, 10 insertions(+), 26 deletions(-)
diff --git a/docs/content/primary-key-table/pk-clustering-override.md b/docs/content/primary-key-table/pk-clustering-override.md
index 534c3c3785..3348bc0482 100644
--- a/docs/content/primary-key-table/pk-clustering-override.md
+++ b/docs/content/primary-key-table/pk-clustering-override.md
@@ -53,26 +53,6 @@ CREATE TABLE my_table (
After this, data files within each bucket will be physically sorted by `city` instead of `id`. Queries like
`SELECT * FROM my_table WHERE city = 'Beijing'` can skip irrelevant data files by checking their min/max statistics
on the clustering column.
-s
-## How It Works
-
-PK Clustering Override replaces the default LSM compaction with a two-phase clustering compaction:
-
-**Phase 1 — Sort by Clustering Columns**: Newly flushed (level 0) files are read, sorted by the configured clustering
-columns, and rewritten as sorted (level 1) files. A key index tracks each primary key's file and row position to
-maintain uniqueness.
-
-**Phase 2 — Merge Overlapping Sections**: Sorted files are grouped into sections based on clustering column range
-overlap. Overlapping sections are merged together. Adjacent small sections are also consolidated to reduce file count
-and IO amplification. Non-overlapping large files are left untouched.
-
-During both phases, deduplication is handled via deletion vectors:
-
-- **Deduplicate mode**: When a key already exists in an older file, the old row is marked as deleted.
-- **First-row mode**: When a key already exists, the new row is marked as deleted, keeping the first-seen value.
-
-When the number of files to merge exceeds `sort-spill-threshold`, smaller files are first spilled to row-based
-temporary files to reduce memory consumption, preventing OOM during multi-way merge.
## Requirements
@@ -89,11 +69,15 @@ PK Clustering Override is beneficial when:
- Analytical queries frequently filter or aggregate on non-primary-key columns (e.g., `WHERE city = 'Beijing'`).
- The table uses `deduplicate` or `first-row` merge engine.
-- You want data files physically co-located by a business dimension rather than the primary key.
-It is **not** suitable when:
+{{< hint info >}}
+Although data files are no longer sorted by the primary key, filtering on bucket-key fields (which default to the
+primary key) still benefits from bucket pruning. The query engine can skip entire buckets that do not contain matching
+values, so queries like `WHERE id = 12345` remain efficient.
+{{< /hint >}}
+
+**Unsupported modes**:
-- Point lookups by primary key are the dominant access pattern (default LSM sort is already optimal).
-- You need `partial-update` or `aggregation` merge engine.
-- `sequence.fields` or `record-level.expire-time` is required.
-- Changelog producer`lookup` or `full-compaction` is required.
+- Merge engine: `partial-update` or `aggregation`.
+- Changelog producer: `lookup` or `full-compaction`.
+- Configue: `sequence.fields` or `record-level.expire-time`.
(paimon) 29/37: [flink] support vector search procedure for flink (#7550)
Posted by ho...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hope pushed a commit to branch release-1.4
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit 53e674ed481e7154c98d23a1ecda79e9a53e78b0
Author: Faiz <wx...@antgroup.com>
AuthorDate: Mon Mar 30 20:00:01 2026 +0800
[flink] support vector search procedure for flink (#7550)
---
docs/content/append-table/global-index.md | 26 +++
docs/content/flink/procedures.md | 34 ++++
.../flink/procedure/VectorSearchProcedure.java | 216 +++++++++++++++++++++
.../services/org.apache.paimon.factories.Factory | 3 +-
.../procedure/VectorSearchProcedureITCase.java | 216 +++++++++++++++++++++
5 files changed, 494 insertions(+), 1 deletion(-)
diff --git a/docs/content/append-table/global-index.md b/docs/content/append-table/global-index.md
index dc90a4664f..f00462492f 100644
--- a/docs/content/append-table/global-index.md
+++ b/docs/content/append-table/global-index.md
@@ -109,6 +109,32 @@ SELECT * FROM vector_search('my_table', 'embedding', array(1.0f, 2.0f, 3.0f), 5)
```
{{< /tab >}}
+{{< tab "Flink SQL (Procedure)" >}}
+
+Unlike Spark's table-valued function, Flink uses a `CALL` procedure to perform vector search.
+The procedure returns JSON-serialized rows as strings.
+
+```sql
+-- Search for top-5 nearest neighbors
+CALL sys.vector_search(
+ `table` => 'db.my_table',
+ vector_column => 'embedding',
+ query_vector => '1.0,2.0,3.0',
+ top_k => 5
+);
+
+-- With projection (only return specific columns)
+CALL sys.vector_search(
+ `table` => 'db.my_table',
+ vector_column => 'embedding',
+ query_vector => '1.0,2.0,3.0',
+ top_k => 5,
+ projection => 'id,name'
+);
+```
+
+{{< /tab >}}
+
{{< tab "Java API" >}}
```java
Table table = catalog.getTable(identifier);
diff --git a/docs/content/flink/procedures.md b/docs/content/flink/procedures.md
index 49e65d705b..1c982c3474 100644
--- a/docs/content/flink/procedures.md
+++ b/docs/content/flink/procedures.md
@@ -951,5 +951,39 @@ All available procedures are listed below.
CALL sys.drop_function(`function` => 'function_identifier')<br/>
</td>
</tr>
+ <tr>
+ <td>vector_search</td>
+ <td>
+ CALL [catalog.]sys.vector_search(<br/>
+ `table` => 'identifier',<br/>
+ vector_column => 'columnName',<br/>
+ query_vector => 'v1,v2,...',<br/>
+ top_k => topK,<br/>
+ projection => 'col1,col2',<br/>
+ options => 'key1=value1;key2=value2')<br/>
+ </td>
+ <td>
+ To perform vector similarity search on a table with a global vector index. Returns JSON-serialized rows. Arguments:
+ <li>table(required): the target table identifier.</li>
+ <li>vector_column(required): the name of the vector column to search.</li>
+ <li>query_vector(required): comma-separated float values representing the query vector, e.g. '1.0,2.0,3.0'.</li>
+ <li>top_k(required): the number of nearest neighbors to return.</li>
+ <li>projection(optional): comma-separated column names to include in the result. If omitted, all columns are returned.</li>
+ <li>options(optional): additional dynamic options of the table.</li>
+ </td>
+ <td>
+ CALL sys.vector_search(<br/>
+ `table` => 'default.T',<br/>
+ vector_column => 'embedding',<br/>
+ query_vector => '1.0,2.0,3.0',<br/>
+ top_k => 5)<br/><br/>
+ CALL sys.vector_search(<br/>
+ `table` => 'default.T',<br/>
+ vector_column => 'embedding',<br/>
+ query_vector => '1.0,2.0,3.0',<br/>
+ top_k => 5,<br/>
+ projection => 'id,name')
+ </td>
+ </tr>
</tbody>
</table>
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/VectorSearchProcedure.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/VectorSearchProcedure.java
new file mode 100644
index 0000000000..2b5df413f8
--- /dev/null
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/VectorSearchProcedure.java
@@ -0,0 +1,216 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.flink.procedure;
+
+import org.apache.paimon.data.InternalRow;
+import org.apache.paimon.format.json.JsonFormatWriter;
+import org.apache.paimon.format.json.JsonOptions;
+import org.apache.paimon.fs.PositionOutputStream;
+import org.apache.paimon.globalindex.GlobalIndexResult;
+import org.apache.paimon.options.Options;
+import org.apache.paimon.reader.RecordReader;
+import org.apache.paimon.table.Table;
+import org.apache.paimon.table.source.ReadBuilder;
+import org.apache.paimon.table.source.TableScan;
+import org.apache.paimon.types.RowType;
+import org.apache.paimon.utils.StringUtils;
+
+import org.apache.flink.table.annotation.ArgumentHint;
+import org.apache.flink.table.annotation.DataTypeHint;
+import org.apache.flink.table.annotation.ProcedureHint;
+import org.apache.flink.table.procedure.ProcedureContext;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+/**
+ * Vector search procedure. This procedure takes one vector and searches for topK nearest vectors.
+ * Usage:
+ *
+ * <pre><code>
+ * CALL sys.vector_search(
+ * `table` => 'tableId',
+ * vector_column => 'v',
+ * query_vector => '1.0,2.0,3.0',
+ * top_k => 5
+ * )
+ *
+ * -- with projection and options
+ * CALL sys.vector_search(
+ * `table` => 'tableId',
+ * vector_column => 'v',
+ * query_vector => '1.0,2.0,3.0',
+ * top_k => 5,
+ * projection => 'id,name',
+ * options => 'k1=v1;k2=v2'
+ * )
+ * </code></pre>
+ */
+public class VectorSearchProcedure extends ProcedureBase {
+
+ public static final String IDENTIFIER = "vector_search";
+
+ @ProcedureHint(
+ argument = {
+ @ArgumentHint(name = "table", type = @DataTypeHint("STRING")),
+ @ArgumentHint(name = "vector_column", type = @DataTypeHint("STRING")),
+ @ArgumentHint(name = "query_vector", type = @DataTypeHint("STRING")),
+ @ArgumentHint(name = "top_k", type = @DataTypeHint("INT")),
+ @ArgumentHint(
+ name = "projection",
+ type = @DataTypeHint("STRING"),
+ isOptional = true),
+ @ArgumentHint(name = "options", type = @DataTypeHint("STRING"), isOptional = true)
+ })
+ public String[] call(
+ ProcedureContext procedureContext,
+ String tableId,
+ String vectorColumn,
+ String queryVectorStr,
+ Integer topK,
+ String projection,
+ String options)
+ throws Exception {
+ Table table = table(tableId);
+
+ Map<String, String> optionsMap = optionalConfigMap(options);
+ if (!optionsMap.isEmpty()) {
+ table = table.copy(optionsMap);
+ }
+
+ float[] queryVector = parseVector(queryVectorStr);
+
+ GlobalIndexResult result =
+ table.newVectorSearchBuilder()
+ .withVector(queryVector)
+ .withVectorColumn(vectorColumn)
+ .withLimit(topK)
+ .executeLocal();
+
+ RowType tableRowType = table.rowType();
+ int[] projectionIndices = parseProjection(projection, tableRowType);
+
+ ReadBuilder readBuilder = table.newReadBuilder();
+ if (projectionIndices != null) {
+ readBuilder.withProjection(projectionIndices);
+ }
+
+ TableScan.Plan plan = readBuilder.newScan().withGlobalIndexResult(result).plan();
+
+ RowType readType =
+ projectionIndices != null ? tableRowType.project(projectionIndices) : tableRowType;
+
+ ByteArrayOutputStream byteOut = new ByteArrayOutputStream(1024);
+ JsonOptions jsonOptions = new JsonOptions(new Options());
+ try (JsonFormatWriter jsonWriter =
+ new JsonFormatWriter(
+ new ByteArrayPositionOutputStream(byteOut),
+ readType,
+ jsonOptions,
+ "none");
+ RecordReader<InternalRow> reader = readBuilder.newRead().createReader(plan)) {
+ reader.forEachRemaining(
+ row -> {
+ try {
+ jsonWriter.addElement(row);
+ } catch (Exception e) {
+ throw new RuntimeException("Failed to convert row to JSON string", e);
+ }
+ });
+ }
+
+ String[] lines =
+ StringUtils.split(byteOut.toString("UTF-8"), jsonOptions.getLineDelimiter());
+ List<String> rows = new ArrayList<>(lines.length);
+ for (String line : lines) {
+ String trimmed = line.trim();
+ if (!trimmed.isEmpty()) {
+ rows.add(trimmed);
+ }
+ }
+ return rows.toArray(new String[0]);
+ }
+
+ private static float[] parseVector(String vectorStr) {
+ String[] parts = StringUtils.split(vectorStr, ",");
+ float[] vector = new float[parts.length];
+ for (int i = 0; i < parts.length; i++) {
+ vector[i] = Float.parseFloat(parts[i].trim());
+ }
+ return vector;
+ }
+
+ private static int[] parseProjection(String projection, RowType rowType) {
+ if (StringUtils.isNullOrWhitespaceOnly(projection)) {
+ return null;
+ }
+ String[] projectionNames = StringUtils.split(projection, ",");
+ return rowType.getFieldIndices(Arrays.stream(projectionNames).collect(Collectors.toList()));
+ }
+
+ @Override
+ public String identifier() {
+ return IDENTIFIER;
+ }
+
+ /** A {@link PositionOutputStream} wrapping a {@link ByteArrayOutputStream}. */
+ private static class ByteArrayPositionOutputStream extends PositionOutputStream {
+
+ private final ByteArrayOutputStream out;
+
+ private ByteArrayPositionOutputStream(ByteArrayOutputStream out) {
+ this.out = out;
+ }
+
+ @Override
+ public long getPos() {
+ return out.size();
+ }
+
+ @Override
+ public void write(int b) {
+ out.write(b);
+ }
+
+ @Override
+ public void write(byte[] b) throws IOException {
+ out.write(b);
+ }
+
+ @Override
+ public void write(byte[] b, int off, int len) {
+ out.write(b, off, len);
+ }
+
+ @Override
+ public void flush() throws IOException {
+ out.flush();
+ }
+
+ @Override
+ public void close() throws IOException {
+ out.close();
+ }
+ }
+}
diff --git a/paimon-flink/paimon-flink-common/src/main/resources/META-INF/services/org.apache.paimon.factories.Factory b/paimon-flink/paimon-flink-common/src/main/resources/META-INF/services/org.apache.paimon.factories.Factory
index f87a250ec2..d687fe2244 100644
--- a/paimon-flink/paimon-flink-common/src/main/resources/META-INF/services/org.apache.paimon.factories.Factory
+++ b/paimon-flink/paimon-flink-common/src/main/resources/META-INF/services/org.apache.paimon.factories.Factory
@@ -100,4 +100,5 @@ org.apache.paimon.flink.procedure.AlterColumnDefaultValueProcedure
org.apache.paimon.flink.procedure.TriggerTagAutomaticCreationProcedure
org.apache.paimon.flink.procedure.RemoveUnexistingManifestsProcedure
org.apache.paimon.flink.procedure.DataEvolutionMergeIntoProcedure
-org.apache.paimon.flink.procedure.CreateGlobalIndexProcedure
\ No newline at end of file
+org.apache.paimon.flink.procedure.CreateGlobalIndexProcedure
+org.apache.paimon.flink.procedure.VectorSearchProcedure
\ No newline at end of file
diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/procedure/VectorSearchProcedureITCase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/procedure/VectorSearchProcedureITCase.java
new file mode 100644
index 0000000000..935da25330
--- /dev/null
+++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/procedure/VectorSearchProcedureITCase.java
@@ -0,0 +1,216 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.flink.procedure;
+
+import org.apache.paimon.data.BinaryRow;
+import org.apache.paimon.data.GenericArray;
+import org.apache.paimon.data.GenericRow;
+import org.apache.paimon.flink.CatalogITCaseBase;
+import org.apache.paimon.globalindex.GlobalIndexBuilderUtils;
+import org.apache.paimon.globalindex.GlobalIndexSingletonWriter;
+import org.apache.paimon.globalindex.ResultEntry;
+import org.apache.paimon.globalindex.testvector.TestVectorGlobalIndexerFactory;
+import org.apache.paimon.index.IndexFileMeta;
+import org.apache.paimon.io.CompactIncrement;
+import org.apache.paimon.io.DataIncrement;
+import org.apache.paimon.options.Options;
+import org.apache.paimon.table.FileStoreTable;
+import org.apache.paimon.table.sink.BatchTableCommit;
+import org.apache.paimon.table.sink.BatchTableWrite;
+import org.apache.paimon.table.sink.BatchWriteBuilder;
+import org.apache.paimon.table.sink.CommitMessage;
+import org.apache.paimon.table.sink.CommitMessageImpl;
+import org.apache.paimon.types.DataField;
+import org.apache.paimon.utils.Range;
+
+import org.apache.flink.types.Row;
+import org.junit.jupiter.api.Test;
+
+import java.util.Collections;
+import java.util.List;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/** IT cases for {@link VectorSearchProcedure}. */
+public class VectorSearchProcedureITCase extends CatalogITCaseBase {
+
+ private static final String VECTOR_FIELD = "vec";
+ private static final int DIMENSION = 2;
+
+ @Test
+ public void testVectorSearchBasic() throws Exception {
+ createVectorTable("T");
+ FileStoreTable table = paimonTable("T");
+
+ float[][] vectors = {
+ {1.0f, 0.0f}, // row 0
+ {0.95f, 0.1f}, // row 1
+ {0.1f, 0.95f}, // row 2
+ {0.98f, 0.05f}, // row 3
+ {0.0f, 1.0f}, // row 4
+ {0.05f, 0.98f} // row 5
+ };
+
+ writeVectors(table, vectors);
+ buildAndCommitVectorIndex(table, vectors);
+
+ // Search for vectors close to (1.0, 0.0)
+ List<Row> result =
+ sql(
+ "CALL sys.vector_search("
+ + "`table` => 'default.T', "
+ + "vector_column => 'vec', "
+ + "query_vector => '1.0 ,0.0', "
+ + "top_k => 3)");
+
+ assertThat(result).isNotEmpty();
+ assertThat(result.size()).isLessThanOrEqualTo(3);
+
+ // Verify results contain JSON strings
+ for (Row row : result) {
+ String json = row.getField(0).toString();
+ assertThat(json).contains("\"id\"");
+ assertThat(json).contains("\"vec\"");
+ }
+ }
+
+ @Test
+ public void testVectorSearchWithProjection() throws Exception {
+ createVectorTable("T2");
+ FileStoreTable table = paimonTable("T2");
+
+ float[][] vectors = {
+ {1.0f, 0.0f}, // row 0
+ {0.0f, 1.0f}, // row 1
+ };
+
+ writeVectors(table, vectors);
+ buildAndCommitVectorIndex(table, vectors);
+
+ List<Row> result =
+ sql(
+ "CALL sys.vector_search("
+ + "`table` => 'default.T2', "
+ + "vector_column => 'vec', "
+ + "query_vector => '1.0,0.0', "
+ + "top_k => 2, "
+ + "projection => 'id')");
+
+ assertThat(result).isNotEmpty();
+ assertThat(result.size()).isLessThanOrEqualTo(2);
+
+ for (Row row : result) {
+ String json = row.getField(0).toString();
+ assertThat(json).contains("\"id\"");
+ // projection only selects 'id', so 'vec' should not appear
+ assertThat(json).doesNotContain("\"vec\"");
+ }
+ }
+
+ @Test
+ public void testVectorSearchTopK() throws Exception {
+ createVectorTable("T3");
+ FileStoreTable table = paimonTable("T3");
+
+ float[][] vectors = new float[10][];
+ for (int i = 0; i < 10; i++) {
+ vectors[i] = new float[] {(float) Math.cos(i * 0.3), (float) Math.sin(i * 0.3)};
+ }
+
+ writeVectors(table, vectors);
+ buildAndCommitVectorIndex(table, vectors);
+
+ List<Row> result =
+ sql(
+ "CALL sys.vector_search("
+ + "`table` => 'default.T3', "
+ + "vector_column => 'vec', "
+ + "query_vector => '1.0,0.0', "
+ + "top_k => 3)");
+
+ assertThat(result.size()).isLessThanOrEqualTo(3);
+ }
+
+ private void createVectorTable(String tableName) {
+ sql(
+ "CREATE TABLE %s ("
+ + "id INT, "
+ + "vec ARRAY<FLOAT>"
+ + ") WITH ("
+ + "'bucket' = '-1', "
+ + "'row-tracking.enabled' = 'true', "
+ + "'data-evolution.enabled' = 'true', "
+ + "'test.vector.dimension' = '%d', "
+ + "'test.vector.metric' = 'l2'"
+ + ")",
+ tableName, DIMENSION);
+ }
+
+ private void writeVectors(FileStoreTable table, float[][] vectors) throws Exception {
+ BatchWriteBuilder writeBuilder = table.newBatchWriteBuilder();
+ try (BatchTableWrite write = writeBuilder.newWrite();
+ BatchTableCommit commit = writeBuilder.newCommit()) {
+ for (int i = 0; i < vectors.length; i++) {
+ write.write(GenericRow.of(i, new GenericArray(vectors[i])));
+ }
+ commit.commit(write.prepareCommit());
+ }
+ }
+
+ private void buildAndCommitVectorIndex(FileStoreTable table, float[][] vectors)
+ throws Exception {
+ Options options = table.coreOptions().toConfiguration();
+ DataField vectorField = table.rowType().getField(VECTOR_FIELD);
+
+ GlobalIndexSingletonWriter writer =
+ (GlobalIndexSingletonWriter)
+ GlobalIndexBuilderUtils.createIndexWriter(
+ table,
+ TestVectorGlobalIndexerFactory.IDENTIFIER,
+ vectorField,
+ options);
+ for (float[] vec : vectors) {
+ writer.write(vec);
+ }
+ List<ResultEntry> entries = writer.finish();
+
+ Range rowRange = new Range(0, vectors.length - 1);
+ List<IndexFileMeta> indexFiles =
+ GlobalIndexBuilderUtils.toIndexFileMetas(
+ table.fileIO(),
+ table.store().pathFactory().globalIndexFileFactory(),
+ table.coreOptions(),
+ rowRange,
+ vectorField.id(),
+ TestVectorGlobalIndexerFactory.IDENTIFIER,
+ entries);
+
+ DataIncrement dataIncrement = DataIncrement.indexIncrement(indexFiles);
+ CommitMessage message =
+ new CommitMessageImpl(
+ BinaryRow.EMPTY_ROW,
+ 0,
+ null,
+ dataIncrement,
+ CompactIncrement.emptyIncrement());
+ try (BatchTableCommit commit = table.newBatchWriteBuilder().newCommit()) {
+ commit.commit(Collections.singletonList(message));
+ }
+ }
+}
(paimon) 33/37: Introduce scan.primary-branch which is symmetrical to scan.fallback-branch (#7553)
Posted by ho...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hope pushed a commit to branch release-1.4
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit 57f6db7960fa023013a9e377f58394d871ba6330
Author: tsreaper <ts...@gmail.com>
AuthorDate: Tue Mar 31 15:27:00 2026 +0800
Introduce scan.primary-branch which is symmetrical to scan.fallback-branch (#7553)
### Purpose
Some users may have accumulated large amounts of data in their main
primary-key branch, and they want to organize the data into another
non-primary-key branch to speed up querying. This PR introduces
`scan.primary-branch` which is symmetrical to `scan.fallback-branch`.
### Tests
* `FallbackReadFileStoreTableTest`
---
.../shortcodes/generated/core_configuration.html | 18 ++-
.../generated/flink_connector_configuration.html | 2 +-
.../main/java/org/apache/paimon/CoreOptions.java | 9 ++
.../org/apache/paimon/schema/SchemaValidation.java | 22 +++-
.../paimon/table/AbstractFileStoreTable.java | 17 ++-
.../apache/paimon/table/ChainGroupReadTable.java | 38 ++----
.../paimon/table/FallbackReadFileStoreTable.java | 137 ++++++++++++---------
.../apache/paimon/table/FileStoreTableFactory.java | 66 ++++++----
.../paimon/table/system/ReadOptimizedTable.java | 9 +-
.../org/apache/paimon/utils/ChainTableUtils.java | 3 +-
.../table/FallbackReadFileStoreTableTest.java | 118 ++++++++++++------
.../apache/paimon/table/SimpleTableTestBase.java | 5 +-
.../org/apache/paimon/flink/BranchSqlITCase.java | 2 +-
.../flink/PrimaryKeyFileStoreTableITCase.java | 3 +-
14 files changed, 274 insertions(+), 175 deletions(-)
diff --git a/docs/layouts/shortcodes/generated/core_configuration.html b/docs/layouts/shortcodes/generated/core_configuration.html
index 6dd621fcac..0071c26437 100644
--- a/docs/layouts/shortcodes/generated/core_configuration.html
+++ b/docs/layouts/shortcodes/generated/core_configuration.html
@@ -128,6 +128,12 @@ under the License.
<td>MemorySize</td>
<td>Memory page size for caching.</td>
</tr>
+ <tr>
+ <td><h5>chain-table.chain-partition-keys</h5></td>
+ <td style="word-wrap: break-word;">(none)</td>
+ <td>String</td>
+ <td>Partition keys that participate in chain logic. Must be a contiguous suffix of the table's partition keys. Comma-separated. If not set, all partition keys participate in chain.</td>
+ </tr>
<tr>
<td><h5>chain-table.enabled</h5></td>
<td style="word-wrap: break-word;">false</td>
@@ -1187,12 +1193,6 @@ This config option does not affect the default filesystem metastore.</td>
<td>String</td>
<td>When a batch job queries from a chain table, if a partition does not exist in the main branch, the reader will try to get this partition from chain snapshot branch.</td>
</tr>
- <tr>
- <td><h5>chain-table.chain-partition-keys</h5></td>
- <td style="word-wrap: break-word;">(none)</td>
- <td>String</td>
- <td>Partition keys that participate in chain logic. Must be a contiguous suffix of the table's partition keys. Comma-separated. If not set, all partition keys participate in chain.</td>
- </tr>
<tr>
<td><h5>scan.file-creation-time-millis</h5></td>
<td style="word-wrap: break-word;">(none)</td>
@@ -1235,6 +1235,12 @@ This config option does not affect the default filesystem metastore.</td>
<td>Boolean</td>
<td>Whether to sort plan files by partition fields, this allows you to read according to the partition order, even if your partition writes are out of order.<br />It is recommended that you use this for streaming read of the 'append-only' table. By default, streaming read will read the full snapshot first. In order to avoid the disorder reading for partitions, you can open this option.</td>
</tr>
+ <tr>
+ <td><h5>scan.primary-branch</h5></td>
+ <td style="word-wrap: break-word;">(none)</td>
+ <td>String</td>
+ <td>When a batch job queries from a table, if a partition exists in the primary branch, the reader will read this partition from the primary branch. Otherwise, the reader will read this partition from the current branch.</td>
+ </tr>
<tr>
<td><h5>scan.snapshot-id</h5></td>
<td style="word-wrap: break-word;">(none)</td>
diff --git a/docs/layouts/shortcodes/generated/flink_connector_configuration.html b/docs/layouts/shortcodes/generated/flink_connector_configuration.html
index d4c8b5a167..253652cb78 100644
--- a/docs/layouts/shortcodes/generated/flink_connector_configuration.html
+++ b/docs/layouts/shortcodes/generated/flink_connector_configuration.html
@@ -363,4 +363,4 @@ under the License.
<td>Defines a custom parallelism for the unaware-bucket table compaction job. By default, if this option is not defined, the planner will derive the parallelism for each statement individually by also considering the global configuration.</td>
</tr>
</tbody>
-</table>
\ No newline at end of file
+</table>
diff --git a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java
index 7c054b8c8a..897bebcf68 100644
--- a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java
+++ b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java
@@ -1961,6 +1961,15 @@ public class CoreOptions implements Serializable {
"When a batch job queries from a table, if a partition does not exist in the current branch, "
+ "the reader will try to get this partition from this fallback branch.");
+ public static final ConfigOption<String> SCAN_PRIMARY_BRANCH =
+ key("scan.primary-branch")
+ .stringType()
+ .noDefaultValue()
+ .withDescription(
+ "When a batch job queries from a table, if a partition exists in the primary branch, "
+ + "the reader will read this partition from the primary branch. "
+ + "Otherwise, the reader will read this partition from the current branch.");
+
public static final ConfigOption<Boolean> ASYNC_FILE_WRITE =
key("async-file-write")
.booleanType()
diff --git a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java
index b9b68fc647..2ff1080c4a 100644
--- a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java
+++ b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java
@@ -288,14 +288,34 @@ public class SchemaValidation {
public static void validateFallbackBranch(SchemaManager schemaManager, TableSchema schema) {
String fallbackBranch = schema.options().get(CoreOptions.SCAN_FALLBACK_BRANCH.key());
+ String primaryBranch = schema.options().get(CoreOptions.SCAN_PRIMARY_BRANCH.key());
+
+ if (!StringUtils.isNullOrWhitespaceOnly(fallbackBranch)
+ && !StringUtils.isNullOrWhitespaceOnly(primaryBranch)) {
+ throw new IllegalArgumentException(
+ String.format(
+ "Cannot set both '%s' and '%s' at the same time.",
+ CoreOptions.SCAN_FALLBACK_BRANCH.key(),
+ CoreOptions.SCAN_PRIMARY_BRANCH.key()));
+ }
+
if (!StringUtils.isNullOrWhitespaceOnly(fallbackBranch)) {
checkArgument(
schemaManager.copyWithBranch(fallbackBranch).latest().isPresent(),
- "Cannot set '%s' = '%s' because the branch '%s' isn't existed.",
+ "Cannot set '%s' = '%s' because the branch '%s' does not exist.",
CoreOptions.SCAN_FALLBACK_BRANCH.key(),
fallbackBranch,
fallbackBranch);
}
+
+ if (!StringUtils.isNullOrWhitespaceOnly(primaryBranch)) {
+ checkArgument(
+ schemaManager.copyWithBranch(primaryBranch).latest().isPresent(),
+ "Cannot set '%s' = '%s' because the branch '%s' does not exist.",
+ CoreOptions.SCAN_PRIMARY_BRANCH.key(),
+ primaryBranch,
+ primaryBranch);
+ }
}
private static void validateOnlyContainPrimitiveType(
diff --git a/paimon-core/src/main/java/org/apache/paimon/table/AbstractFileStoreTable.java b/paimon-core/src/main/java/org/apache/paimon/table/AbstractFileStoreTable.java
index 3583d17c08..f8a6b1722f 100644
--- a/paimon-core/src/main/java/org/apache/paimon/table/AbstractFileStoreTable.java
+++ b/paimon-core/src/main/java/org/apache/paimon/table/AbstractFileStoreTable.java
@@ -709,10 +709,19 @@ abstract class AbstractFileStoreTable implements FileStoreTable {
&& branchName.equals(fallbackBranch)) {
throw new IllegalArgumentException(
String.format(
- "can not delete the fallback branch. "
- + "branchName to be deleted is %s. you have set 'scan.fallback-branch' = '%s'. "
- + "you should reset 'scan.fallback-branch' before deleting this branch.",
- branchName, fallbackBranch));
+ "Cannot delete branch '%s' because it is configured as"
+ + " 'scan.fallback-branch'. Unset 'scan.fallback-branch' first.",
+ branchName));
+ }
+
+ String primaryBranch = coreOptions().toConfiguration().get(CoreOptions.SCAN_PRIMARY_BRANCH);
+ if (!StringUtils.isNullOrWhitespaceOnly(primaryBranch)
+ && branchName.equals(primaryBranch)) {
+ throw new IllegalArgumentException(
+ String.format(
+ "Cannot delete branch '%s' because it is configured as"
+ + " 'scan.primary-branch'. Unset 'scan.primary-branch' first.",
+ branchName));
}
branchManager().dropBranch(branchName);
diff --git a/paimon-core/src/main/java/org/apache/paimon/table/ChainGroupReadTable.java b/paimon-core/src/main/java/org/apache/paimon/table/ChainGroupReadTable.java
index 78752acfc6..b178715a2c 100644
--- a/paimon-core/src/main/java/org/apache/paimon/table/ChainGroupReadTable.java
+++ b/paimon-core/src/main/java/org/apache/paimon/table/ChainGroupReadTable.java
@@ -67,7 +67,7 @@ import static org.apache.paimon.utils.Preconditions.checkNotNull;
public class ChainGroupReadTable extends FallbackReadFileStoreTable {
public ChainGroupReadTable(FileStoreTable snapshotStoreTable, FileStoreTable deltaStoreTable) {
- super(snapshotStoreTable, deltaStoreTable);
+ super(snapshotStoreTable, deltaStoreTable, true);
checkArgument(snapshotStoreTable instanceof PrimaryKeyFileStoreTable);
checkArgument(deltaStoreTable instanceof PrimaryKeyFileStoreTable);
}
@@ -75,11 +75,7 @@ public class ChainGroupReadTable extends FallbackReadFileStoreTable {
@Override
public DataTableScan newScan() {
super.validateSchema();
- return new ChainTableBatchScan(
- wrapped.newScan(),
- fallback().newScan(),
- ((AbstractFileStoreTable) wrapped).tableSchema,
- this);
+ return new ChainTableBatchScan(((AbstractFileStoreTable) wrapped).tableSchema, this);
}
private DataTableScan newSnapshotScan() {
@@ -87,42 +83,38 @@ public class ChainGroupReadTable extends FallbackReadFileStoreTable {
}
private DataTableScan newDeltaScan() {
- return fallback().newScan();
+ return other().newScan();
}
@Override
public FileStoreTable copy(Map<String, String> dynamicOptions) {
return new ChainGroupReadTable(
- wrapped.copy(dynamicOptions),
- fallback().copy(rewriteFallbackOptions(dynamicOptions)));
+ wrapped.copy(dynamicOptions), other().copy(rewriteOtherOptions(dynamicOptions)));
}
@Override
public FileStoreTable copy(TableSchema newTableSchema) {
return new ChainGroupReadTable(
wrapped.copy(newTableSchema),
- fallback()
- .copy(
- newTableSchema.copy(
- rewriteFallbackOptions(newTableSchema.options()))));
+ other().copy(newTableSchema.copy(rewriteOtherOptions(newTableSchema.options()))));
}
@Override
public FileStoreTable copyWithoutTimeTravel(Map<String, String> dynamicOptions) {
return new ChainGroupReadTable(
wrapped.copyWithoutTimeTravel(dynamicOptions),
- fallback().copyWithoutTimeTravel(rewriteFallbackOptions(dynamicOptions)));
+ other().copyWithoutTimeTravel(rewriteOtherOptions(dynamicOptions)));
}
@Override
public FileStoreTable copyWithLatestSchema() {
return new ChainGroupReadTable(
- wrapped.copyWithLatestSchema(), fallback().copyWithLatestSchema());
+ wrapped.copyWithLatestSchema(), other().copyWithLatestSchema());
}
@Override
public FileStoreTable switchToBranch(String branchName) {
- return new ChainGroupReadTable(switchWrappedToBranch(branchName), fallback());
+ return new ChainGroupReadTable(switchWrappedToBranch(branchName), other());
}
/** Scan implementation for {@link ChainGroupReadTable}. */
@@ -138,16 +130,12 @@ public class ChainGroupReadTable extends FallbackReadFileStoreTable {
private Filter<Integer> bucketFilter;
public ChainTableBatchScan(
- DataTableScan mainScan,
- DataTableScan fallbackScan,
- TableSchema tableSchema,
- ChainGroupReadTable chainGroupReadTable) {
+ TableSchema tableSchema, ChainGroupReadTable chainGroupReadTable) {
super(
- mainScan,
- fallbackScan,
chainGroupReadTable.wrapped,
- chainGroupReadTable.fallback(),
- tableSchema);
+ chainGroupReadTable.other(),
+ tableSchema,
+ FileStoreTable::newScan);
this.options = CoreOptions.fromMap(tableSchema.options());
this.chainGroupReadTable = chainGroupReadTable;
this.partitionConverter =
@@ -500,7 +488,7 @@ public class ChainGroupReadTable extends FallbackReadFileStoreTable {
private Read() {
this.mainRead = wrapped.newRead();
- this.fallbackRead = fallback().newRead();
+ this.fallbackRead = other().newRead();
}
@Override
diff --git a/paimon-core/src/main/java/org/apache/paimon/table/FallbackReadFileStoreTable.java b/paimon-core/src/main/java/org/apache/paimon/table/FallbackReadFileStoreTable.java
index aff2aa9a23..5c839a5581 100644
--- a/paimon-core/src/main/java/org/apache/paimon/table/FallbackReadFileStoreTable.java
+++ b/paimon-core/src/main/java/org/apache/paimon/table/FallbackReadFileStoreTable.java
@@ -66,73 +66,82 @@ import java.util.Objects;
import java.util.Optional;
import java.util.OptionalLong;
import java.util.Set;
+import java.util.function.Function;
import java.util.stream.Collectors;
/**
- * A {@link FileStoreTable} which mainly read from the current branch. However, if the current
- * branch does not have a partition, it will read that partition from the fallback branch.
+ * A {@link FileStoreTable} which reads from two branches with partition-level merging. When {@code
+ * wrappedFirst} is true, the wrapped table has read priority (fallback mode). When {@code
+ * wrappedFirst} is false, the other table has read priority (primary mode). Write operations are
+ * always delegated to the wrapped table (current branch).
*/
public class FallbackReadFileStoreTable extends DelegatedFileStoreTable {
private static final Logger LOG = LoggerFactory.getLogger(FallbackReadFileStoreTable.class);
- private final FileStoreTable fallback;
+ private final FileStoreTable other;
+ private final boolean wrappedFirst;
- public FallbackReadFileStoreTable(FileStoreTable wrapped, FileStoreTable fallback) {
+ public FallbackReadFileStoreTable(
+ FileStoreTable wrapped, FileStoreTable other, boolean wrappedFirst) {
super(wrapped);
- this.fallback = fallback;
+ this.other = other;
+ this.wrappedFirst = wrappedFirst;
Preconditions.checkArgument(!(wrapped instanceof FallbackReadFileStoreTable));
- if (fallback instanceof FallbackReadFileStoreTable) {
+ if (other instanceof FallbackReadFileStoreTable) {
// ChainGroupReadTable need to be wrapped again
- if (!(fallback instanceof ChainGroupReadTable)) {
+ if (!(other instanceof ChainGroupReadTable)) {
throw new IllegalArgumentException(
"This is a bug, perhaps there is a recursive call.");
}
}
}
- public FileStoreTable fallback() {
- return fallback;
+ public FileStoreTable other() {
+ return other;
}
@Override
public FileStoreTable copy(Map<String, String> dynamicOptions) {
return new FallbackReadFileStoreTable(
wrapped.copy(dynamicOptions),
- fallback.copy(rewriteFallbackOptions(dynamicOptions)));
+ other.copy(rewriteOtherOptions(dynamicOptions)),
+ wrappedFirst);
}
@Override
public FileStoreTable copy(TableSchema newTableSchema) {
return new FallbackReadFileStoreTable(
wrapped.copy(newTableSchema),
- fallback.copy(
- newTableSchema.copy(rewriteFallbackOptions(newTableSchema.options()))));
+ other.copy(newTableSchema.copy(rewriteOtherOptions(newTableSchema.options()))),
+ wrappedFirst);
}
@Override
public FileStoreTable copyWithoutTimeTravel(Map<String, String> dynamicOptions) {
return new FallbackReadFileStoreTable(
wrapped.copyWithoutTimeTravel(dynamicOptions),
- fallback.copyWithoutTimeTravel(rewriteFallbackOptions(dynamicOptions)));
+ other.copyWithoutTimeTravel(rewriteOtherOptions(dynamicOptions)),
+ wrappedFirst);
}
@Override
public FileStoreTable copyWithLatestSchema() {
return new FallbackReadFileStoreTable(
- wrapped.copyWithLatestSchema(), fallback.copyWithLatestSchema());
+ wrapped.copyWithLatestSchema(), other.copyWithLatestSchema(), wrappedFirst);
}
@Override
public FileStoreTable switchToBranch(String branchName) {
- return new FallbackReadFileStoreTable(switchWrappedToBranch(branchName), fallback);
+ return new FallbackReadFileStoreTable(
+ switchWrappedToBranch(branchName), other, wrappedFirst);
}
@Override
public void setManifestCache(SegmentsCache<Path> manifestCache) {
super.setManifestCache(manifestCache);
- fallback.setManifestCache(manifestCache);
+ other.setManifestCache(manifestCache);
}
protected FileStoreTable switchWrappedToBranch(String branchName) {
@@ -153,35 +162,35 @@ public class FallbackReadFileStoreTable extends DelegatedFileStoreTable {
wrapped.catalogEnvironment());
}
- protected Map<String, String> rewriteFallbackOptions(Map<String, String> options) {
+ protected Map<String, String> rewriteOtherOptions(Map<String, String> options) {
Map<String, String> result = new HashMap<>(options);
- // branch of fallback table should never change
+ // branch of other table should never change
String branchKey = CoreOptions.BRANCH.key();
if (options.containsKey(branchKey)) {
- result.put(branchKey, fallback.options().get(branchKey));
+ result.put(branchKey, other.options().get(branchKey));
}
- // snapshot ids may be different between the main branch and the fallback branch,
+ // snapshot ids may be different between the main branch and the other branch,
// so we need to convert main branch snapshot id to millisecond,
- // then convert millisecond to fallback branch snapshot id
+ // then convert millisecond to other branch snapshot id
String scanSnapshotIdOptionKey = CoreOptions.SCAN_SNAPSHOT_ID.key();
String scanSnapshotId = options.get(scanSnapshotIdOptionKey);
if (scanSnapshotId != null) {
long id = Long.parseLong(scanSnapshotId);
long millis = wrapped.snapshotManager().snapshot(id).timeMillis();
- Snapshot fallbackSnapshot = fallback.snapshotManager().earlierOrEqualTimeMills(millis);
- long fallbackId;
- if (fallbackSnapshot == null) {
- fallbackId = Snapshot.FIRST_SNAPSHOT_ID;
+ Snapshot otherSnapshot = other.snapshotManager().earlierOrEqualTimeMills(millis);
+ long otherId;
+ if (otherSnapshot == null) {
+ otherId = Snapshot.FIRST_SNAPSHOT_ID;
} else {
- fallbackId = fallbackSnapshot.id();
+ otherId = otherSnapshot.id();
}
- result.put(scanSnapshotIdOptionKey, String.valueOf(fallbackId));
+ result.put(scanSnapshotIdOptionKey, String.valueOf(otherId));
}
- // bucket number of main branch and fallback branch are very likely different,
- // so we remove bucket in options to use fallback branch's bucket number
+ // bucket number of main branch and other branch are very likely different,
+ // so we remove bucket in options to use other branch's bucket number
result.remove(CoreOptions.BUCKET.key());
return result;
@@ -189,61 +198,66 @@ public class FallbackReadFileStoreTable extends DelegatedFileStoreTable {
@Override
public DataTableScan newScan() {
+ return newScan(FileStoreTable::newScan);
+ }
+
+ public DataTableScan newScan(Function<FileStoreTable, DataTableScan> scanCreator) {
validateSchema();
- return new FallbackReadScan(
- wrapped.newScan(), fallback.newScan(), wrapped, fallback, wrapped.schema());
+ FileStoreTable first = wrappedFirst ? wrapped : other;
+ FileStoreTable second = wrappedFirst ? other : wrapped;
+ return new FallbackReadScan(first, second, wrapped.schema(), scanCreator);
}
protected void validateSchema() {
String mainBranch = wrapped.coreOptions().branch();
- String fallbackBranch = fallback.coreOptions().branch();
+ String otherBranch = other.coreOptions().branch();
RowType mainRowType = wrapped.schema().logicalRowType();
- RowType fallbackRowType = fallback.schema().logicalRowType();
+ RowType otherRowType = other.schema().logicalRowType();
Preconditions.checkArgument(
- sameRowTypeIgnoreNullable(mainRowType, fallbackRowType),
+ sameRowTypeIgnoreNullable(mainRowType, otherRowType),
"Branch %s and %s does not have the same row type.\n"
+ "Row type of branch %s is %s.\n"
+ "Row type of branch %s is %s.",
mainBranch,
- fallbackBranch,
+ otherBranch,
mainBranch,
mainRowType,
- fallbackBranch,
- fallbackRowType);
+ otherBranch,
+ otherRowType);
List<String> mainPrimaryKeys = wrapped.schema().primaryKeys();
- List<String> fallbackPrimaryKeys = fallback.schema().primaryKeys();
+ List<String> otherPrimaryKeys = other.schema().primaryKeys();
if (!mainPrimaryKeys.isEmpty()) {
- if (fallbackPrimaryKeys.isEmpty()) {
+ if (otherPrimaryKeys.isEmpty()) {
throw new IllegalArgumentException(
"Branch "
+ mainBranch
- + " has primary keys while fallback branch "
- + fallbackBranch
+ + " has primary keys while branch "
+ + otherBranch
+ " does not. This is not allowed.");
}
Preconditions.checkArgument(
- mainPrimaryKeys.equals(fallbackPrimaryKeys),
+ mainPrimaryKeys.equals(otherPrimaryKeys),
"Branch %s and %s both have primary keys but are not the same.\n"
+ "Primary keys of %s are %s.\n"
+ "Primary keys of %s are %s.",
mainBranch,
- fallbackBranch,
+ otherBranch,
mainBranch,
mainPrimaryKeys,
- fallbackBranch,
- fallbackPrimaryKeys);
+ otherBranch,
+ otherPrimaryKeys);
}
}
- private boolean sameRowTypeIgnoreNullable(RowType mainRowType, RowType fallbackRowType) {
- if (mainRowType.getFieldCount() != fallbackRowType.getFieldCount()) {
+ private boolean sameRowTypeIgnoreNullable(RowType mainRowType, RowType otherRowType) {
+ if (mainRowType.getFieldCount() != otherRowType.getFieldCount()) {
return false;
}
for (int i = 0; i < mainRowType.getFieldCount(); i++) {
DataType mainType = mainRowType.getFields().get(i).type();
- DataType fallbackType = fallbackRowType.getFields().get(i).type();
- if (!mainType.equalsIgnoreNullable(fallbackType)) {
+ DataType otherType = otherRowType.getFields().get(i).type();
+ if (!mainType.equalsIgnoreNullable(otherType)) {
return false;
}
}
@@ -356,24 +370,25 @@ public class FallbackReadFileStoreTable extends DelegatedFileStoreTable {
/** Scan implementation for {@link FallbackReadFileStoreTable}. */
public static class FallbackReadScan implements DataTableScan {
- protected final DataTableScan mainScan;
- protected final DataTableScan fallbackScan;
protected final FileStoreTable wrappedTable;
protected final FileStoreTable fallbackTable;
protected final TableSchema tableSchema;
+ protected final Function<FileStoreTable, DataTableScan> scanCreator;
+ protected final DataTableScan mainScan;
+ protected final DataTableScan fallbackScan;
private PartitionPredicate partitionPredicate;
public FallbackReadScan(
- DataTableScan mainScan,
- DataTableScan fallbackScan,
FileStoreTable wrappedTable,
FileStoreTable fallbackTable,
- TableSchema tableSchema) {
- this.mainScan = mainScan;
- this.fallbackScan = fallbackScan;
+ TableSchema tableSchema,
+ Function<FileStoreTable, DataTableScan> scanCreator) {
this.wrappedTable = wrappedTable;
this.fallbackTable = fallbackTable;
this.tableSchema = tableSchema;
+ this.scanCreator = scanCreator;
+ this.mainScan = scanCreator.apply(wrappedTable);
+ this.fallbackScan = scanCreator.apply(fallbackTable);
}
@Override
@@ -561,7 +576,7 @@ public class FallbackReadFileStoreTable extends DelegatedFileStoreTable {
private DataTableScan newPartitionListingScan(
boolean isMain, PartitionPredicate scanPartitionPredicate) {
- DataTableScan scan = isMain ? wrappedTable.newScan() : fallbackTable.newScan();
+ DataTableScan scan = scanCreator.apply(isMain ? wrappedTable : fallbackTable);
if (scanPartitionPredicate != null) {
scan.withPartitionFilter(scanPartitionPredicate);
}
@@ -580,8 +595,10 @@ public class FallbackReadFileStoreTable extends DelegatedFileStoreTable {
private final InnerTableRead fallbackRead;
private Read() {
- this.mainRead = wrapped.newRead();
- this.fallbackRead = fallback.newRead();
+ FileStoreTable first = wrappedFirst ? wrapped : other;
+ FileStoreTable second = wrappedFirst ? other : wrapped;
+ this.mainRead = first.newRead();
+ this.fallbackRead = second.newRead();
}
@Override
@@ -628,7 +645,7 @@ public class FallbackReadFileStoreTable extends DelegatedFileStoreTable {
return fallbackRead.createReader(fallbackSplit.wrapped());
} catch (Exception ignored) {
LOG.error(
- "Reading from fallback branch has problems: {}",
+ "Reading from supplemental branch has problems: {}",
fallbackSplit.wrapped());
}
}
diff --git a/paimon-core/src/main/java/org/apache/paimon/table/FileStoreTableFactory.java b/paimon-core/src/main/java/org/apache/paimon/table/FileStoreTableFactory.java
index 73ae6d2dd1..e4a8656799 100644
--- a/paimon-core/src/main/java/org/apache/paimon/table/FileStoreTableFactory.java
+++ b/paimon-core/src/main/java/org/apache/paimon/table/FileStoreTableFactory.java
@@ -99,35 +99,27 @@ public class FileStoreTableFactory {
Options options = new Options(table.options());
String fallbackBranch = options.get(CoreOptions.SCAN_FALLBACK_BRANCH);
+ String primaryBranch = options.get(CoreOptions.SCAN_PRIMARY_BRANCH);
if (ChainTableUtils.isChainTable(options.toMap())) {
table = createChainTable(table, fileIO, tablePath, dynamicOptions, catalogEnvironment);
} else if (!StringUtils.isNullOrWhitespaceOnly(fallbackBranch)) {
- Options branchOptions = new Options(dynamicOptions.toMap());
- branchOptions.set(CoreOptions.BRANCH, fallbackBranch);
- Optional<TableSchema> schema =
- new SchemaManager(fileIO, tablePath, fallbackBranch).latest();
- if (schema.isPresent()) {
- Identifier identifier = catalogEnvironment.identifier();
- CatalogEnvironment fallbackCatalogEnvironment = catalogEnvironment;
- if (identifier != null) {
- fallbackCatalogEnvironment =
- catalogEnvironment.copy(
- new Identifier(
- identifier.getDatabaseName(),
- identifier.getObjectName(),
- fallbackBranch));
- }
- FileStoreTable fallbackTable =
- createWithoutFallbackBranch(
- fileIO,
- tablePath,
- schema.get(),
- branchOptions,
- fallbackCatalogEnvironment);
- table = new FallbackReadFileStoreTable(table, fallbackTable);
+ FileStoreTable otherTable =
+ createOtherBranchTable(
+ fileIO, tablePath, fallbackBranch, dynamicOptions, catalogEnvironment);
+ if (otherTable != null) {
+ table = new FallbackReadFileStoreTable(table, otherTable, true);
} else {
LOG.error("Fallback branch {} not found for table {}", fallbackBranch, tablePath);
}
+ } else if (!StringUtils.isNullOrWhitespaceOnly(primaryBranch)) {
+ FileStoreTable otherTable =
+ createOtherBranchTable(
+ fileIO, tablePath, primaryBranch, dynamicOptions, catalogEnvironment);
+ if (otherTable != null) {
+ table = new FallbackReadFileStoreTable(table, otherTable, false);
+ } else {
+ LOG.error("Primary branch {} not found for table {}", primaryBranch, tablePath);
+ }
}
return table;
@@ -189,7 +181,33 @@ public class FileStoreTableFactory {
catalogEnvironment);
FileStoreTable chainGroupFileStoreTable =
new ChainGroupReadTable(snapshotTable, deltaTable);
- return new FallbackReadFileStoreTable(table, chainGroupFileStoreTable);
+ return new FallbackReadFileStoreTable(table, chainGroupFileStoreTable, true);
+ }
+
+ private static FileStoreTable createOtherBranchTable(
+ FileIO fileIO,
+ Path tablePath,
+ String branchName,
+ Options dynamicOptions,
+ CatalogEnvironment catalogEnvironment) {
+ Options branchOptions = new Options(dynamicOptions.toMap());
+ branchOptions.set(CoreOptions.BRANCH, branchName);
+ Optional<TableSchema> schema = new SchemaManager(fileIO, tablePath, branchName).latest();
+ if (schema.isPresent()) {
+ Identifier identifier = catalogEnvironment.identifier();
+ CatalogEnvironment branchCatalogEnvironment = catalogEnvironment;
+ if (identifier != null) {
+ branchCatalogEnvironment =
+ catalogEnvironment.copy(
+ new Identifier(
+ identifier.getDatabaseName(),
+ identifier.getObjectName(),
+ branchName));
+ }
+ return createWithoutFallbackBranch(
+ fileIO, tablePath, schema.get(), branchOptions, branchCatalogEnvironment);
+ }
+ return null;
}
public static FileStoreTable createWithoutFallbackBranch(
diff --git a/paimon-core/src/main/java/org/apache/paimon/table/system/ReadOptimizedTable.java b/paimon-core/src/main/java/org/apache/paimon/table/system/ReadOptimizedTable.java
index 64ef8aa34e..0a96533a99 100644
--- a/paimon-core/src/main/java/org/apache/paimon/table/system/ReadOptimizedTable.java
+++ b/paimon-core/src/main/java/org/apache/paimon/table/system/ReadOptimizedTable.java
@@ -29,7 +29,6 @@ import org.apache.paimon.manifest.ManifestFileMeta;
import org.apache.paimon.schema.SchemaManager;
import org.apache.paimon.table.DataTable;
import org.apache.paimon.table.FallbackReadFileStoreTable;
-import org.apache.paimon.table.FallbackReadFileStoreTable.FallbackReadScan;
import org.apache.paimon.table.FileStoreTable;
import org.apache.paimon.table.ReadonlyTable;
import org.apache.paimon.table.Table;
@@ -139,13 +138,7 @@ public class ReadOptimizedTable implements DataTable, ReadonlyTable {
@Override
public DataTableScan newScan() {
if (wrapped instanceof FallbackReadFileStoreTable) {
- FallbackReadFileStoreTable table = (FallbackReadFileStoreTable) wrapped;
- return new FallbackReadScan(
- newScan(table.wrapped()),
- newScan(table.fallback()),
- table.wrapped(),
- table.fallback(),
- table.wrapped().schema());
+ return ((FallbackReadFileStoreTable) wrapped).newScan(this::newScan);
}
return newScan(wrapped);
}
diff --git a/paimon-core/src/main/java/org/apache/paimon/utils/ChainTableUtils.java b/paimon-core/src/main/java/org/apache/paimon/utils/ChainTableUtils.java
index 6ccbbbc8fc..99e25cc2ce 100644
--- a/paimon-core/src/main/java/org/apache/paimon/utils/ChainTableUtils.java
+++ b/paimon-core/src/main/java/org/apache/paimon/utils/ChainTableUtils.java
@@ -220,8 +220,7 @@ public class ChainTableUtils {
public static FileStoreTable resolveChainPrimaryTable(FileStoreTable table) {
if (table.coreOptions().isChainTable() && table instanceof FallbackReadFileStoreTable) {
- return ((ChainGroupReadTable) ((FallbackReadFileStoreTable) table).fallback())
- .wrapped();
+ return ((ChainGroupReadTable) ((FallbackReadFileStoreTable) table).other()).wrapped();
}
return table;
}
diff --git a/paimon-core/src/test/java/org/apache/paimon/table/FallbackReadFileStoreTableTest.java b/paimon-core/src/test/java/org/apache/paimon/table/FallbackReadFileStoreTableTest.java
index 7163202a8e..6a1f782fd2 100644
--- a/paimon-core/src/test/java/org/apache/paimon/table/FallbackReadFileStoreTableTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/table/FallbackReadFileStoreTableTest.java
@@ -42,8 +42,9 @@ import org.apache.paimon.utils.Pair;
import org.apache.paimon.utils.TraceableFileIO;
import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
import java.util.Collections;
import java.util.List;
@@ -77,8 +78,9 @@ public class FallbackReadFileStoreTableTest {
fileIO = FileIOFinder.find(tablePath);
}
- @Test
- public void testListPartitions() throws Exception {
+ @ParameterizedTest
+ @ValueSource(booleans = {false, true})
+ public void testListPartitions(boolean wrappedFirst) throws Exception {
String branchName = "bc";
FileStoreTable mainTable = createTable();
@@ -93,25 +95,20 @@ public class FallbackReadFileStoreTableTest {
// write data into partition for branch only
writeDataIntoTable(branchTable, 0, rowData(3, 60));
- FallbackReadFileStoreTable fallbackTable =
- new FallbackReadFileStoreTable(mainTable, branchTable);
-
- List<Integer> partitionsFromMainTable =
- mainTable.newScan().listPartitions().stream()
- .map(row -> row.getInt(0))
- .collect(Collectors.toList());
- assertThat(partitionsFromMainTable).containsExactlyInAnyOrder(1, 2);
+ FallbackReadFileStoreTable table =
+ new FallbackReadFileStoreTable(mainTable, branchTable, wrappedFirst);
- List<Integer> partitionsFromFallbackTable =
- fallbackTable.newScan().listPartitions().stream()
+ List<Integer> partitions =
+ table.newScan().listPartitions().stream()
.map(row -> row.getInt(0))
.collect(Collectors.toList());
// this should contain all partitions
- assertThat(partitionsFromFallbackTable).containsExactlyInAnyOrder(1, 2, 3);
+ assertThat(partitions).containsExactlyInAnyOrder(1, 2, 3);
}
- @Test
- public void testListPartitionEntries() throws Exception {
+ @ParameterizedTest
+ @ValueSource(booleans = {false, true})
+ public void testListPartitionEntries(boolean wrappedFirst) throws Exception {
String branchName = "bc";
FileStoreTable mainTable = createTable();
@@ -126,40 +123,38 @@ public class FallbackReadFileStoreTableTest {
// write data into partition for branch only
writeDataIntoTable(branchTable, 0, rowData(1, 50), rowData(3, 60), rowData(4, 70));
- FallbackReadFileStoreTable fallbackTable =
- new FallbackReadFileStoreTable(mainTable, branchTable);
-
- List<PartitionEntry> partitionEntries = mainTable.newScan().listPartitionEntries();
- assertThat(partitionEntries)
- .map(e -> e.partition().getInt(0))
- .containsExactlyInAnyOrder(1, 2);
+ FallbackReadFileStoreTable table =
+ new FallbackReadFileStoreTable(mainTable, branchTable, wrappedFirst);
- List<PartitionEntry> fallbackPartitionEntries =
- fallbackTable.newScan().listPartitionEntries();
- assertThat(fallbackPartitionEntries)
+ List<PartitionEntry> entries = table.newScan().listPartitionEntries();
+ // partition 1 exists in both: record count depends on which table has priority
+ // wrappedFirst=true → mainTable has priority (2 records), false → branchTable (1 record)
+ long expectedPt1Count = wrappedFirst ? 2L : 1L;
+ assertThat(entries)
.map(e -> Pair.of(e.partition().getInt(0), e.recordCount()))
.containsExactlyInAnyOrder(
- Pair.of(1, 2L), Pair.of(2, 1L), Pair.of(3, 1L), Pair.of(4, 1L));
+ Pair.of(1, expectedPt1Count),
+ Pair.of(2, 1L),
+ Pair.of(3, 1L),
+ Pair.of(4, 1L));
}
/**
* Test that FallbackReadScan.plan() determines partition ownership based on partition
- * predicates only, not mixed with data filters. If a partition exists in the main branch, it
- * should never be read from fallback, regardless of the data filter.
- *
- * <p>Without the fix, the old code built completePartitions from mainScan.plan() results which
- * already had data filters applied. When the data filter excluded all files of a main partition
- * via filterByStats, that partition was incorrectly treated as "not in main" and read from
- * fallback.
+ * predicates only, not mixed with data filters. If a partition exists in the priority table, it
+ * should never be read from the supplemental table, regardless of the data filter.
*/
- @Test
- public void testPlanWithDataFilter() throws Exception {
+ @ParameterizedTest
+ @ValueSource(booleans = {false, true})
+ public void testPlanWithDataFilter(boolean wrappedFirst) throws Exception {
String branchName = "bc";
+ InternalRow[] firstValues = new InternalRow[] {rowData(1, 10), rowData(2, 20)};
+ InternalRow[] secondValues = new InternalRow[] {rowData(1, 100), rowData(3, 30)};
FileStoreTable mainTable = createTable();
// Main branch: partition 1 (a=10), partition 2 (a=20)
- writeDataIntoTable(mainTable, 0, rowData(1, 10), rowData(2, 20));
+ writeDataIntoTable(mainTable, 0, wrappedFirst ? firstValues : secondValues);
mainTable.createBranch(branchName);
@@ -167,10 +162,10 @@ public class FallbackReadFileStoreTableTest {
// Fallback branch: partition 1 already has a=10 (inherited), add a=100.
// Also add partition 3 (a=30) which is fallback-only.
- writeDataIntoTable(branchTable, 1, rowData(1, 100), rowData(3, 30));
+ writeDataIntoTable(branchTable, 1, wrappedFirst ? secondValues : firstValues);
FallbackReadFileStoreTable fallbackTable =
- new FallbackReadFileStoreTable(mainTable, branchTable);
+ new FallbackReadFileStoreTable(mainTable, branchTable, wrappedFirst);
PredicateBuilder builder = new PredicateBuilder(ROW_TYPE);
// Case 1: WHERE pt = 1 AND a = 100
@@ -202,6 +197,51 @@ public class FallbackReadFileStoreTableTest {
.isTrue();
}
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ public void testWriteGoesToWrapped(boolean wrappedFirst) throws Exception {
+ String branchName = "bc";
+
+ FileStoreTable mainTable = createTable();
+
+ // write data into partition 1 for main
+ writeDataIntoTable(mainTable, 0, rowData(1, 10));
+
+ mainTable.createBranch(branchName);
+
+ FileStoreTable branchTable = createTableFromBranch(mainTable, branchName);
+
+ // write data into partition 2 for branch
+ writeDataIntoTable(branchTable, 0, rowData(2, 20));
+
+ FallbackReadFileStoreTable table =
+ new FallbackReadFileStoreTable(mainTable, branchTable, wrappedFirst);
+
+ // write through the merged table — should go to mainTable (wrapped)
+ writeDataIntoTable(table, 1, rowData(3, 30));
+
+ // verify: main branch should now have partition 1 and 3
+ List<Integer> mainPartitions =
+ mainTable.newScan().listPartitions().stream()
+ .map(row -> row.getInt(0))
+ .collect(Collectors.toList());
+ assertThat(mainPartitions).containsExactlyInAnyOrder(1, 3);
+
+ // verify: branch should still only have partition 2
+ List<Integer> branchPartitions =
+ branchTable.newScan().listPartitions().stream()
+ .map(row -> row.getInt(0))
+ .collect(Collectors.toList());
+ assertThat(branchPartitions).containsExactlyInAnyOrder(2);
+
+ // verify: merged read should see all three partitions
+ List<Integer> mergedPartitions =
+ table.newScan().listPartitions().stream()
+ .map(row -> row.getInt(0))
+ .collect(Collectors.toList());
+ assertThat(mergedPartitions).containsExactlyInAnyOrder(1, 2, 3);
+ }
+
private void writeDataIntoTable(
FileStoreTable table, long commitIdentifier, InternalRow... allData) throws Exception {
StreamTableWrite write = table.newWrite(commitUser);
diff --git a/paimon-core/src/test/java/org/apache/paimon/table/SimpleTableTestBase.java b/paimon-core/src/test/java/org/apache/paimon/table/SimpleTableTestBase.java
index f50e9922f5..c4ff1ca51a 100644
--- a/paimon-core/src/test/java/org/apache/paimon/table/SimpleTableTestBase.java
+++ b/paimon-core/src/test/java/org/apache/paimon/table/SimpleTableTestBase.java
@@ -1218,9 +1218,8 @@ public abstract class SimpleTableTestBase {
.satisfies(
anyCauseMatches(
IllegalArgumentException.class,
- "can not delete the fallback branch. "
- + "branchName to be deleted is fallback. you have set 'scan.fallback-branch' = 'fallback'. "
- + "you should reset 'scan.fallback-branch' before deleting this branch."));
+ "Cannot delete branch 'fallback' because it is configured as"
+ + " 'scan.fallback-branch'. Unset 'scan.fallback-branch' first."));
table.deleteBranch("fallback");
}
diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/BranchSqlITCase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/BranchSqlITCase.java
index 1b7d50c852..641efb7335 100644
--- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/BranchSqlITCase.java
+++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/BranchSqlITCase.java
@@ -760,7 +760,7 @@ public class BranchSqlITCase extends CatalogITCaseBase {
@Test
public void testCannotSetEmptyFallbackBranch() {
String errMsg =
- "Cannot set 'scan.fallback-branch' = 'test' because the branch 'test' isn't existed.";
+ "Cannot set 'scan.fallback-branch' = 'test' because the branch 'test' does not exist.";
assertThatThrownBy(
() ->
sql(
diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/PrimaryKeyFileStoreTableITCase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/PrimaryKeyFileStoreTableITCase.java
index 06afcce3bb..94bd5a30e7 100644
--- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/PrimaryKeyFileStoreTableITCase.java
+++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/PrimaryKeyFileStoreTableITCase.java
@@ -966,7 +966,8 @@ public class PrimaryKeyFileStoreTableITCase extends AbstractTestBase {
// branch1 is fallback branch, can not be deleted
assertThatCode(() -> bEnv.executeSql("CALL sys.delete_branch('default.t', 'branch1')"))
.rootCause()
- .hasMessageContaining("can not delete the fallback branch.");
+ .hasMessageContaining(
+ "Cannot delete branch 'branch1' because it is configured as 'scan.fallback-branch'.");
// reset scan.fallback-branch
bEnv.executeSql("ALTER TABLE t RESET ('scan.fallback-branch')");
(paimon) 19/37: [spark] Support merge schema through catalog for REST catalog (#7539)
Posted by ho...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hope pushed a commit to branch release-1.4
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit 3e0b96b6e972c28004d3e2d2928e08c170412585
Author: Zouxxyy <zo...@alibaba-inc.com>
AuthorDate: Thu Mar 26 20:02:58 2026 +0800
[spark] Support merge schema through catalog for REST catalog (#7539)
When using REST catalog, schema merge during write bypasses the catalog
and directly commits to SchemaManager, which doesn't work for REST
catalog. This PR introduces SchemaModification to route schema changes
through Catalog.alterTable(), and adds diffSchemaChanges() to convert
merged schemas into SchemaChange lists.
---
.../java/org/apache/paimon/AbstractFileStore.java | 3 +-
.../org/apache/paimon/schema/SchemaManager.java | 19 ++++--
.../apache/paimon/schema/SchemaMergingUtils.java | 54 ++++++++++++++++
.../apache/paimon/table/CatalogEnvironment.java | 9 +++
.../apache/paimon/table/SchemaModification.java | 46 ++++++++++++++
.../sql/WriteMergeSchemaWithRestCatalogTest.scala | 74 ++++++++++++++++++++++
6 files changed, 199 insertions(+), 6 deletions(-)
diff --git a/paimon-core/src/main/java/org/apache/paimon/AbstractFileStore.java b/paimon-core/src/main/java/org/apache/paimon/AbstractFileStore.java
index 388b7eb9de..9aad0259cd 100644
--- a/paimon-core/src/main/java/org/apache/paimon/AbstractFileStore.java
+++ b/paimon-core/src/main/java/org/apache/paimon/AbstractFileStore.java
@@ -267,7 +267,8 @@ abstract class AbstractFileStore<T> implements FileStore<T> {
@Override
public boolean mergeSchema(RowType rowType, boolean allowExplicitCast) {
- return schemaManager.mergeSchema(rowType, allowExplicitCast);
+ return schemaManager.mergeSchema(
+ rowType, allowExplicitCast, catalogEnvironment.schemaModification());
}
@Override
diff --git a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaManager.java b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaManager.java
index dbc605b96c..13da6677f7 100644
--- a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaManager.java
+++ b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaManager.java
@@ -37,6 +37,7 @@ import org.apache.paimon.schema.SchemaChange.UpdateColumnPosition;
import org.apache.paimon.schema.SchemaChange.UpdateColumnType;
import org.apache.paimon.schema.SchemaChange.UpdateComment;
import org.apache.paimon.table.FileStoreTableFactory;
+import org.apache.paimon.table.SchemaModification;
import org.apache.paimon.types.ArrayType;
import org.apache.paimon.types.DataField;
import org.apache.paimon.types.DataType;
@@ -706,7 +707,10 @@ public class SchemaManager implements Serializable {
}
}
- public boolean mergeSchema(RowType rowType, boolean allowExplicitCast) {
+ public boolean mergeSchema(
+ RowType rowType,
+ boolean allowExplicitCast,
+ @Nullable SchemaModification schemaModification) {
TableSchema current =
latest().orElseThrow(
() ->
@@ -715,12 +719,17 @@ public class SchemaManager implements Serializable {
TableSchema update = SchemaMergingUtils.mergeSchemas(current, rowType, allowExplicitCast);
if (current.equals(update)) {
return false;
- } else {
- try {
+ }
+ try {
+ if (schemaModification != null) {
+ List<SchemaChange> changes = SchemaMergingUtils.diffSchemaChanges(current, update);
+ schemaModification.alterSchema(changes);
+ return true;
+ } else {
return commit(update);
- } catch (Exception e) {
- throw new RuntimeException("Failed to commit the schema.", e);
}
+ } catch (Exception e) {
+ throw new RuntimeException("Failed to commit the schema.", e);
}
}
diff --git a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaMergingUtils.java b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaMergingUtils.java
index 82c61adf6c..1ed9bad7e6 100644
--- a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaMergingUtils.java
+++ b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaMergingUtils.java
@@ -29,6 +29,7 @@ import org.apache.paimon.types.MultisetType;
import org.apache.paimon.types.ReassignFieldId;
import org.apache.paimon.types.RowType;
+import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
@@ -233,4 +234,57 @@ public class SchemaMergingUtils {
field.description(),
field.defaultValue());
}
+
+ /**
+ * Generate a list of {@link SchemaChange} by comparing the old and new {@link TableSchema}.
+ * This supports detecting added columns and type changes (including nested structs).
+ */
+ public static List<SchemaChange> diffSchemaChanges(
+ TableSchema oldSchema, TableSchema newSchema) {
+ List<SchemaChange> changes = new ArrayList<>();
+ diffFields(
+ oldSchema.logicalRowType().getFields(),
+ newSchema.logicalRowType().getFields(),
+ new String[0],
+ changes);
+ return changes;
+ }
+
+ private static void diffFields(
+ List<DataField> oldFields,
+ List<DataField> newFields,
+ String[] parentNames,
+ List<SchemaChange> changes) {
+ Map<String, DataField> oldFieldMap =
+ oldFields.stream().collect(Collectors.toMap(DataField::name, Function.identity()));
+
+ for (DataField newField : newFields) {
+ String[] fieldNames = appendFieldName(parentNames, newField.name());
+ DataField oldField = oldFieldMap.get(newField.name());
+ if (oldField == null) {
+ // new column added
+ changes.add(
+ SchemaChange.addColumn(
+ fieldNames, newField.type(), newField.description(), null));
+ } else if (!oldField.type().equals(newField.type())) {
+ // type changed — check if it's a nested struct change
+ if (oldField.type() instanceof RowType && newField.type() instanceof RowType) {
+ diffFields(
+ ((RowType) oldField.type()).getFields(),
+ ((RowType) newField.type()).getFields(),
+ fieldNames,
+ changes);
+ } else {
+ changes.add(SchemaChange.updateColumnType(fieldNames, newField.type(), true));
+ }
+ }
+ }
+ }
+
+ private static String[] appendFieldName(String[] parentNames, String fieldName) {
+ String[] result = new String[parentNames.length + 1];
+ System.arraycopy(parentNames, 0, result, 0, parentNames.length);
+ result[parentNames.length] = fieldName;
+ return result;
+ }
}
diff --git a/paimon-core/src/main/java/org/apache/paimon/table/CatalogEnvironment.java b/paimon-core/src/main/java/org/apache/paimon/table/CatalogEnvironment.java
index dac63fc178..cb40c4447e 100644
--- a/paimon-core/src/main/java/org/apache/paimon/table/CatalogEnvironment.java
+++ b/paimon-core/src/main/java/org/apache/paimon/table/CatalogEnvironment.java
@@ -113,6 +113,15 @@ public class CatalogEnvironment implements Serializable {
return supportsVersionManagement;
}
+ @Nullable
+ public SchemaModification schemaModification() {
+ if (catalogLoader == null) {
+ return null;
+ }
+ Catalog catalog = catalogLoader.load();
+ return SchemaModification.create(catalog, identifier);
+ }
+
@Nullable
public SnapshotCommit snapshotCommit(SnapshotManager snapshotManager) {
SnapshotCommit snapshotCommit;
diff --git a/paimon-core/src/main/java/org/apache/paimon/table/SchemaModification.java b/paimon-core/src/main/java/org/apache/paimon/table/SchemaModification.java
new file mode 100644
index 0000000000..2f024811b5
--- /dev/null
+++ b/paimon-core/src/main/java/org/apache/paimon/table/SchemaModification.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.table;
+
+import org.apache.paimon.catalog.Catalog;
+import org.apache.paimon.catalog.Identifier;
+import org.apache.paimon.schema.SchemaChange;
+
+import java.util.List;
+
+/** Handler to alter table schema via catalog. */
+public interface SchemaModification extends AutoCloseable {
+
+ void alterSchema(List<SchemaChange> changes) throws Exception;
+
+ static SchemaModification create(Catalog catalog, Identifier identifier) {
+ return new SchemaModification() {
+
+ @Override
+ public void alterSchema(List<SchemaChange> changes) throws Exception {
+ catalog.alterTable(identifier, changes, false);
+ }
+
+ @Override
+ public void close() throws Exception {
+ catalog.close();
+ }
+ };
+ }
+}
diff --git a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/WriteMergeSchemaWithRestCatalogTest.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/WriteMergeSchemaWithRestCatalogTest.scala
new file mode 100644
index 0000000000..caaea22766
--- /dev/null
+++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/WriteMergeSchemaWithRestCatalogTest.scala
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.spark.sql
+
+import org.apache.paimon.spark.PaimonSparkTestWithRestCatalogBase
+
+import org.apache.spark.SparkConf
+import org.apache.spark.sql.Row
+
+class WriteMergeSchemaWithRestCatalogTest extends PaimonSparkTestWithRestCatalogBase {
+
+ import testImplicits._
+
+ override protected def sparkConf: SparkConf = {
+ super.sparkConf.set("spark.sql.catalog.paimon.cache-enabled", "false")
+ }
+
+ test("Write merge schema with REST catalog: dataframe write") {
+ withTable("t") {
+ sql("CREATE TABLE t (a INT, b STRING)")
+ Seq((1, "1"), (2, "2"))
+ .toDF("a", "b")
+ .write
+ .format("paimon")
+ .mode("append")
+ .saveAsTable("t")
+
+ // new columns
+ Seq((3, "3", 3))
+ .toDF("a", "b", "c")
+ .write
+ .format("paimon")
+ .mode("append")
+ .option("write.merge-schema", "true")
+ .saveAsTable("t")
+ checkAnswer(
+ sql("SELECT * FROM t ORDER BY a"),
+ Seq(Row(1, "1", null), Row(2, "2", null), Row(3, "3", 3))
+ )
+ }
+ }
+
+ test("Write merge schema with REST catalog: sql write") {
+ withTable("t") {
+ withSparkSQLConf("spark.paimon.write.merge-schema" -> "true") {
+ sql("CREATE TABLE t (a INT, b STRING)")
+ sql("INSERT INTO t VALUES (1, '1'), (2, '2')")
+
+ // new columns
+ sql("INSERT INTO t BY NAME SELECT 3 AS a, '3' AS b, 3 AS c")
+ checkAnswer(
+ sql("SELECT * FROM t ORDER BY a"),
+ Seq(Row(1, "1", null), Row(2, "2", null), Row(3, "3", 3))
+ )
+ }
+ }
+ }
+}
(paimon) 18/37: [spark] Support reading ObjectTable via Spark SQL (#7535)
Posted by ho...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hope pushed a commit to branch release-1.4
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit b6fc657b4e85696223a1244ff9d61c4cf9f5e7ed
Author: Zouxxyy <zo...@alibaba-inc.com>
AuthorDate: Thu Mar 26 19:54:06 2026 +0800
[spark] Support reading ObjectTable via Spark SQL (#7535)
Support reading ObjectTable via Spark SQL by implementing `SupportsRead`
for `SparkObjectTable` and adding `ObjectTableScan` /
`ObjectTableScanBuilder`.
---
.../java/org/apache/paimon/spark/SparkCatalog.java | 2 +-
.../scala/org/apache/paimon/spark/SparkTable.scala | 19 +++++-
.../apache/paimon/spark/read/ObjectTableScan.scala | 52 ++++++++++++++++
.../paimon/spark/table/PaimonObjectTableTest.scala | 72 ++++++++++++++++++++++
4 files changed, 142 insertions(+), 3 deletions(-)
diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkCatalog.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkCatalog.java
index 9d12201c91..6ef853eda8 100644
--- a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkCatalog.java
+++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkCatalog.java
@@ -662,7 +662,7 @@ public class SparkCatalog extends SparkBaseCatalog
} else if (table instanceof LanceTable) {
return new SparkLanceTable(table);
} else if (table instanceof ObjectTable) {
- return new SparkObjectTable(table);
+ return new SparkObjectTable((ObjectTable) table);
} else {
return new SparkTable(table);
}
diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/SparkTable.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/SparkTable.scala
index 740e7b5994..55b21263df 100644
--- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/SparkTable.scala
+++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/SparkTable.scala
@@ -18,11 +18,17 @@
package org.apache.paimon.spark
+import org.apache.paimon.spark.read.ObjectTableScanBuilder
import org.apache.paimon.spark.rowops.PaimonSparkCopyOnWriteOperation
+import org.apache.paimon.table.`object`.ObjectTable
import org.apache.paimon.table.{FileStoreTable, Table}
-import org.apache.spark.sql.connector.catalog.SupportsRowLevelOperations
+import org.apache.spark.sql.connector.catalog.{SupportsRead, SupportsRowLevelOperations, TableCapability}
+import org.apache.spark.sql.connector.read.ScanBuilder
import org.apache.spark.sql.connector.write.{RowLevelOperationBuilder, RowLevelOperationInfo}
+import org.apache.spark.sql.util.CaseInsensitiveStringMap
+
+import java.util.{EnumSet => JEnumSet, Set => JSet}
/** A spark [[org.apache.spark.sql.connector.catalog.Table]] for paimon. */
case class SparkTable(override val table: Table)
@@ -46,4 +52,13 @@ case class SparkIcebergTable(table: Table) extends BaseTable
case class SparkLanceTable(table: Table) extends BaseTable
-case class SparkObjectTable(table: Table) extends BaseTable
+case class SparkObjectTable(override val table: ObjectTable) extends BaseTable with SupportsRead {
+
+ override def capabilities(): JSet[TableCapability] = {
+ JEnumSet.of(TableCapability.BATCH_READ)
+ }
+
+ override def newScanBuilder(options: CaseInsensitiveStringMap): ScanBuilder = {
+ new ObjectTableScanBuilder(table.copy(options.asCaseSensitiveMap))
+ }
+}
diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/read/ObjectTableScan.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/read/ObjectTableScan.scala
new file mode 100644
index 0000000000..4542b4d2a4
--- /dev/null
+++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/read/ObjectTableScan.scala
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.spark.read
+
+import org.apache.paimon.partition.PartitionPredicate
+import org.apache.paimon.predicate.Predicate
+import org.apache.paimon.spark.PaimonBaseScanBuilder
+import org.apache.paimon.table.`object`.ObjectTable
+import org.apache.paimon.table.source.Split
+
+import org.apache.spark.sql.types.StructType
+
+import scala.collection.JavaConverters._
+
+class ObjectTableScanBuilder(val table: ObjectTable) extends PaimonBaseScanBuilder {
+
+ override def build(): ObjectTableScan =
+ ObjectTableScan(table, requiredSchema)
+}
+
+/** Scan implementation for [[ObjectTable]] */
+case class ObjectTableScan(table: ObjectTable, requiredSchema: StructType) extends BaseScan {
+
+ override val pushedPartitionFilters: Seq[PartitionPredicate] = Nil
+ override val pushedDataFilters: Seq[Predicate] = Nil
+ override val pushedLimit: Option[Int] = None
+
+ protected def getInputSplits: Array[Split] = {
+ readBuilder
+ .newScan()
+ .plan()
+ .splits()
+ .asScala
+ .toArray
+ }
+}
diff --git a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/table/PaimonObjectTableTest.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/table/PaimonObjectTableTest.scala
new file mode 100644
index 0000000000..3c3ecee016
--- /dev/null
+++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/table/PaimonObjectTableTest.scala
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.spark.table
+
+import org.apache.paimon.catalog.Identifier
+import org.apache.paimon.fs.Path
+import org.apache.paimon.spark.PaimonSparkTestWithRestCatalogBase
+import org.apache.paimon.table.`object`.ObjectTable
+
+import org.apache.spark.sql.Row
+
+class PaimonObjectTableTest extends PaimonSparkTestWithRestCatalogBase {
+
+ override protected def beforeEach(): Unit = {
+ super.beforeEach()
+ sql("USE paimon")
+ sql("CREATE DATABASE IF NOT EXISTS test_db")
+ sql("USE test_db")
+ }
+
+ test("ObjectTable: read file metadata") {
+ val tableName = "object_table_test"
+ withTable(tableName) {
+ sql(
+ s"CREATE TABLE $tableName TBLPROPERTIES (" +
+ s"'type' = 'object-table')")
+
+ val objectTable =
+ paimonCatalog
+ .getTable(Identifier.create("test_db", tableName))
+ .asInstanceOf[ObjectTable]
+
+ val fileIO = objectTable.fileIO()
+ val basePath = new Path(objectTable.location())
+ fileIO.mkdirs(basePath)
+ fileIO.writeFile(new Path(basePath, "file1.txt"), "content1", false)
+ fileIO.writeFile(new Path(basePath, "file2.txt"), "content2", false)
+ fileIO.mkdirs(new Path(basePath, "subdir"))
+ fileIO.writeFile(new Path(basePath, "subdir/file3.txt"), "content3", false)
+
+ checkAnswer(
+ sql(s"SELECT path, name FROM $tableName ORDER BY path"),
+ Seq(
+ Row("file1.txt", "file1.txt"),
+ Row("file2.txt", "file2.txt"),
+ Row("subdir/file3.txt", "file3.txt"))
+ )
+
+ // Verify schema has expected columns
+ checkAnswer(
+ sql(s"SELECT COUNT(*) FROM $tableName"),
+ Seq(Row(3))
+ )
+ }
+ }
+}
(paimon) 06/37: [spark] Add OVERWRITE_BY_FILTER capabilities to PartitionedFormatTable for Spark 4 compatibility (#7517)
Posted by ho...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hope pushed a commit to branch release-1.4
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit 0fec396774ef26d9c9127cfe12ec89b382473814
Author: Kerwin Zhang <xi...@alibaba-inc.com>
AuthorDate: Wed Mar 25 19:19:41 2026 +0800
[spark] Add OVERWRITE_BY_FILTER capabilities to PartitionedFormatTable for Spark 4 compatibility (#7517)
---
.../spark/sql/execution/SparkFormatTable.scala | 36 ++++++++++++++++++++
.../paimon/spark/table/PaimonFormatTableTest.scala | 38 ++++++++++++++++++++++
2 files changed, 74 insertions(+)
diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/execution/SparkFormatTable.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/execution/SparkFormatTable.scala
index 2cb0101653..9d0983ed0b 100644
--- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/execution/SparkFormatTable.scala
+++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/execution/SparkFormatTable.scala
@@ -25,7 +25,11 @@ import org.apache.spark.sql.SparkSession
import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.expressions.{AttributeReference, EqualTo, Literal}
import org.apache.spark.sql.connector.catalog.SupportsPartitionManagement
+import org.apache.spark.sql.connector.catalog.TableCapability
+import org.apache.spark.sql.connector.catalog.TableCapability._
import org.apache.spark.sql.connector.expressions.{Expressions, Transform}
+import org.apache.spark.sql.connector.expressions.filter.Predicate
+import org.apache.spark.sql.connector.write.{LogicalWriteInfo, SupportsOverwriteV2, Write, WriteBuilder}
import org.apache.spark.sql.execution.datasources._
import org.apache.spark.sql.execution.datasources.v2.csv.{CSVScanBuilder, CSVTable}
import org.apache.spark.sql.execution.datasources.v2.json.JsonTable
@@ -123,6 +127,18 @@ trait PartitionedFormatTable extends SupportsPartitionManagement {
val fileIndex: PartitioningAwareFileIndex
+ override def capabilities(): util.Set[TableCapability] = {
+ util.EnumSet.of(BATCH_READ, BATCH_WRITE, OVERWRITE_DYNAMIC, OVERWRITE_BY_FILTER)
+ }
+
+ protected def wrapWriteBuilderWithOverwrite(original: WriteBuilder): WriteBuilder = {
+ new WriteBuilder with SupportsOverwriteV2 {
+ override def build(): Write = original.build()
+ override def canOverwrite(predicates: Array[Predicate]): Boolean = true
+ override def overwrite(predicates: Array[Predicate]): WriteBuilder = this
+ }
+ }
+
override def partitionSchema(): StructType = partitionSchema_
override def partitioning(): Array[Transform] = {
@@ -172,6 +188,10 @@ class PartitionedCSVTable(
extends CSVTable(name, sparkSession, options, paths, userSpecifiedSchema, fallbackFileFormat)
with PartitionedFormatTable {
+ override def newWriteBuilder(info: LogicalWriteInfo): WriteBuilder = {
+ wrapWriteBuilderWithOverwrite(super.newWriteBuilder(info))
+ }
+
override def newScanBuilder(options: CaseInsensitiveStringMap): CSVScanBuilder = {
val mergedOptions =
this.options.asCaseSensitiveMap().asScala ++ options.asCaseSensitiveMap().asScala
@@ -204,6 +224,10 @@ class PartitionedTextTable(
extends TextTable(name, sparkSession, options, paths, userSpecifiedSchema, fallbackFileFormat)
with PartitionedFormatTable {
+ override def newWriteBuilder(info: LogicalWriteInfo): WriteBuilder = {
+ wrapWriteBuilderWithOverwrite(super.newWriteBuilder(info))
+ }
+
override def newScanBuilder(options: CaseInsensitiveStringMap): TextScanBuilder = {
val mergedOptions =
this.options.asCaseSensitiveMap().asScala ++ options.asCaseSensitiveMap().asScala
@@ -236,6 +260,10 @@ class PartitionedOrcTable(
) extends OrcTable(name, sparkSession, options, paths, userSpecifiedSchema, fallbackFileFormat)
with PartitionedFormatTable {
+ override def newWriteBuilder(info: LogicalWriteInfo): WriteBuilder = {
+ wrapWriteBuilderWithOverwrite(super.newWriteBuilder(info))
+ }
+
override lazy val fileIndex: PartitioningAwareFileIndex = {
SparkFormatTable.createFileIndex(
options,
@@ -257,6 +285,10 @@ class PartitionedParquetTable(
) extends ParquetTable(name, sparkSession, options, paths, userSpecifiedSchema, fallbackFileFormat)
with PartitionedFormatTable {
+ override def newWriteBuilder(info: LogicalWriteInfo): WriteBuilder = {
+ wrapWriteBuilderWithOverwrite(super.newWriteBuilder(info))
+ }
+
override lazy val fileIndex: PartitioningAwareFileIndex = {
SparkFormatTable.createFileIndex(
options,
@@ -278,6 +310,10 @@ class PartitionedJsonTable(
extends JsonTable(name, sparkSession, options, paths, userSpecifiedSchema, fallbackFileFormat)
with PartitionedFormatTable {
+ override def newWriteBuilder(info: LogicalWriteInfo): WriteBuilder = {
+ wrapWriteBuilderWithOverwrite(super.newWriteBuilder(info))
+ }
+
override lazy val fileIndex: PartitioningAwareFileIndex = {
SparkFormatTable.createFileIndex(
options,
diff --git a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/table/PaimonFormatTableTest.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/table/PaimonFormatTableTest.scala
index 374383d316..6fa31f5e2c 100644
--- a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/table/PaimonFormatTableTest.scala
+++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/table/PaimonFormatTableTest.scala
@@ -24,6 +24,8 @@ import org.apache.paimon.spark.PaimonSparkTestWithRestCatalogBase
import org.apache.paimon.table.FormatTable
import org.apache.spark.sql.Row
+import org.apache.spark.sql.connector.catalog.TableCapability
+import org.apache.spark.sql.connector.catalog.TableCatalog
class PaimonFormatTableTest extends PaimonSparkTestWithRestCatalogBase {
@@ -367,6 +369,42 @@ class PaimonFormatTableTest extends PaimonSparkTestWithRestCatalogBase {
}
}
+ test("PartitionedFormatTable: external table saveAsTable should support overwrite") {
+ val tableName = "paimon_format_external_overwrite_test"
+ Seq("paimon", "engine").foreach {
+ impl =>
+ withTable(tableName) {
+ val location = s"${tempDBDir.getCanonicalPath}/external_overwrite_test"
+
+ sql(s"""CREATE TABLE $tableName (a INT, b STRING, pt STRING)
+ |USING CSV PARTITIONED BY (pt) LOCATION '$location'
+ |TBLPROPERTIES ('format-table.implementation'='$impl')
+ |""".stripMargin)
+
+ // Verify external format table has OVERWRITE_BY_FILTER capability
+ val catalog = spark.sessionState.catalogManager.currentCatalog
+ .asInstanceOf[TableCatalog]
+ val v2Table = catalog.loadTable(
+ org.apache.spark.sql.connector.catalog.Identifier.of(Array("test_db"), tableName))
+ val caps = v2Table.capabilities()
+ assert(
+ caps.contains(TableCapability.OVERWRITE_BY_FILTER),
+ s"External format table ($impl) should have OVERWRITE_BY_FILTER capability, but capabilities are: $caps"
+ )
+
+ spark
+ .createDataFrame(Seq((5, "x5", "p1")))
+ .toDF("a", "b", "pt")
+ .write
+ .format("csv")
+ .option("path", location)
+ .partitionBy("pt")
+ .mode("overwrite")
+ .saveAsTable(tableName)
+ }
+ }
+ }
+
test("Paimon format table: show partitions") {
withTable("t") {
sql("""
(paimon) 25/37: [core][spark] Introduce Full Text Search interfaces and Spark function (#7548)
Posted by ho...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hope pushed a commit to branch release-1.4
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit e6231d01c6a828e7677073c821d79349f3be5dda
Author: Jingsong Lee <ji...@gmail.com>
AuthorDate: Mon Mar 30 14:34:30 2026 +0800
[core][spark] Introduce Full Text Search interfaces and Spark function (#7548)
Symmetric implementation of full text search based on the existing
VectorSearch pattern.
---
.../paimon/globalindex/GlobalIndexReader.java | 5 +
.../globalindex/OffsetGlobalIndexReader.java | 6 +
.../apache/paimon/predicate/FullTextSearch.java | 72 ++++
.../TestFullTextGlobalIndexReader.java | 254 ++++++++++++
.../TestFullTextGlobalIndexWriter.java | 110 ++++++
.../testfulltext/TestFullTextGlobalIndexer.java | 62 +++
.../TestFullTextGlobalIndexerFactory.java | 37 +-
....apache.paimon.globalindex.GlobalIndexerFactory | 1 +
.../java/org/apache/paimon/table/FormatTable.java | 7 +
.../apache/paimon/table/FullTextSearchTable.java | 101 +++++
.../java/org/apache/paimon/table/InnerTable.java | 7 +
.../main/java/org/apache/paimon/table/Table.java | 4 +
.../apache/paimon/table/source/FullTextRead.java | 27 +-
.../paimon/table/source/FullTextReadImpl.java | 127 ++++++
.../apache/paimon/table/source/FullTextScan.java | 26 +-
.../paimon/table/source/FullTextScanImpl.java | 88 +++++
.../paimon/table/source/FullTextSearchBuilder.java | 47 +++
.../table/source/FullTextSearchBuilderImpl.java | 74 ++++
.../paimon/table/source/FullTextSearchSplit.java | 82 ++++
.../table/source/FullTextSearchBuilderTest.java | 424 +++++++++++++++++++++
.../paimon/spark/PaimonBaseScanBuilder.scala | 2 +
.../scala/org/apache/paimon/spark/PaimonScan.scala | 3 +-
.../scala/org/apache/paimon/spark/PaimonScan.scala | 3 +-
.../org/apache/paimon/spark/PaimonBaseScan.scala | 28 +-
.../paimon/spark/PaimonBaseScanBuilder.scala | 2 +
.../scala/org/apache/paimon/spark/PaimonScan.scala | 3 +-
.../apache/paimon/spark/PaimonScanBuilder.scala | 11 +-
.../plans/logical/PaimonTableValuedFunctions.scala | 89 ++++-
.../org/apache/paimon/spark/read/BaseScan.scala | 6 +-
.../paimon/spark/sql/FullTextSearchTest.scala | 223 +++++++++++
30 files changed, 1869 insertions(+), 62 deletions(-)
diff --git a/paimon-common/src/main/java/org/apache/paimon/globalindex/GlobalIndexReader.java b/paimon-common/src/main/java/org/apache/paimon/globalindex/GlobalIndexReader.java
index 1b8c17bf66..f639273575 100644
--- a/paimon-common/src/main/java/org/apache/paimon/globalindex/GlobalIndexReader.java
+++ b/paimon-common/src/main/java/org/apache/paimon/globalindex/GlobalIndexReader.java
@@ -18,6 +18,7 @@
package org.apache.paimon.globalindex;
+import org.apache.paimon.predicate.FullTextSearch;
import org.apache.paimon.predicate.FunctionVisitor;
import org.apache.paimon.predicate.LeafPredicate;
import org.apache.paimon.predicate.VectorSearch;
@@ -47,4 +48,8 @@ public interface GlobalIndexReader extends FunctionVisitor<Optional<GlobalIndexR
default Optional<ScoredGlobalIndexResult> visitVectorSearch(VectorSearch vectorSearch) {
throw new UnsupportedOperationException();
}
+
+ default Optional<ScoredGlobalIndexResult> visitFullTextSearch(FullTextSearch fullTextSearch) {
+ throw new UnsupportedOperationException();
+ }
}
diff --git a/paimon-common/src/main/java/org/apache/paimon/globalindex/OffsetGlobalIndexReader.java b/paimon-common/src/main/java/org/apache/paimon/globalindex/OffsetGlobalIndexReader.java
index 21f06f59f3..0122060dfb 100644
--- a/paimon-common/src/main/java/org/apache/paimon/globalindex/OffsetGlobalIndexReader.java
+++ b/paimon-common/src/main/java/org/apache/paimon/globalindex/OffsetGlobalIndexReader.java
@@ -19,6 +19,7 @@
package org.apache.paimon.globalindex;
import org.apache.paimon.predicate.FieldRef;
+import org.apache.paimon.predicate.FullTextSearch;
import org.apache.paimon.predicate.VectorSearch;
import java.io.IOException;
@@ -122,6 +123,11 @@ public class OffsetGlobalIndexReader implements GlobalIndexReader {
.map(r -> r.offset(offset));
}
+ @Override
+ public Optional<ScoredGlobalIndexResult> visitFullTextSearch(FullTextSearch fullTextSearch) {
+ return wrapped.visitFullTextSearch(fullTextSearch).map(r -> r.offset(offset));
+ }
+
private Optional<GlobalIndexResult> applyOffset(Optional<GlobalIndexResult> result) {
return result.map(r -> r.offset(offset));
}
diff --git a/paimon-common/src/main/java/org/apache/paimon/predicate/FullTextSearch.java b/paimon-common/src/main/java/org/apache/paimon/predicate/FullTextSearch.java
new file mode 100644
index 0000000000..5b91d0bbc6
--- /dev/null
+++ b/paimon-common/src/main/java/org/apache/paimon/predicate/FullTextSearch.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.predicate;
+
+import org.apache.paimon.globalindex.GlobalIndexReader;
+import org.apache.paimon.globalindex.ScoredGlobalIndexResult;
+
+import java.io.Serializable;
+import java.util.Optional;
+
+/** FullTextSearch to perform full-text search on a text column. */
+public class FullTextSearch implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ private final String queryText;
+ private final String fieldName;
+ private final int limit;
+
+ public FullTextSearch(String queryText, int limit, String fieldName) {
+ if (queryText == null || queryText.isEmpty()) {
+ throw new IllegalArgumentException("Query text cannot be null or empty");
+ }
+ if (limit <= 0) {
+ throw new IllegalArgumentException("Limit must be positive, got: " + limit);
+ }
+ if (fieldName == null || fieldName.isEmpty()) {
+ throw new IllegalArgumentException("Field name cannot be null or empty");
+ }
+ this.queryText = queryText;
+ this.limit = limit;
+ this.fieldName = fieldName;
+ }
+
+ public String queryText() {
+ return queryText;
+ }
+
+ public int limit() {
+ return limit;
+ }
+
+ public String fieldName() {
+ return fieldName;
+ }
+
+ public Optional<ScoredGlobalIndexResult> visit(GlobalIndexReader visitor) {
+ return visitor.visitFullTextSearch(this);
+ }
+
+ @Override
+ public String toString() {
+ return String.format(
+ "FullTextSearch{field=%s, query='%s', limit=%d}", fieldName, queryText, limit);
+ }
+}
diff --git a/paimon-common/src/test/java/org/apache/paimon/globalindex/testfulltext/TestFullTextGlobalIndexReader.java b/paimon-common/src/test/java/org/apache/paimon/globalindex/testfulltext/TestFullTextGlobalIndexReader.java
new file mode 100644
index 0000000000..affc79e916
--- /dev/null
+++ b/paimon-common/src/test/java/org/apache/paimon/globalindex/testfulltext/TestFullTextGlobalIndexReader.java
@@ -0,0 +1,254 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.globalindex.testfulltext;
+
+import org.apache.paimon.fs.SeekableInputStream;
+import org.apache.paimon.globalindex.GlobalIndexIOMeta;
+import org.apache.paimon.globalindex.GlobalIndexReader;
+import org.apache.paimon.globalindex.GlobalIndexResult;
+import org.apache.paimon.globalindex.ScoredGlobalIndexResult;
+import org.apache.paimon.globalindex.io.GlobalIndexFileReader;
+import org.apache.paimon.predicate.FieldRef;
+import org.apache.paimon.predicate.FullTextSearch;
+import org.apache.paimon.utils.RoaringNavigableMap64;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.nio.charset.StandardCharsets;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Optional;
+import java.util.PriorityQueue;
+
+/**
+ * Test full-text index reader that performs brute-force text matching. Loads all documents into
+ * memory and scores them against the query using simple term-frequency matching.
+ *
+ * <p>Scoring: for each query term found in the document (case-insensitive), score += 1.0 /
+ * queryTermCount. A document that contains all query terms scores 1.0.
+ */
+public class TestFullTextGlobalIndexReader implements GlobalIndexReader {
+
+ private final GlobalIndexFileReader fileReader;
+ private final GlobalIndexIOMeta ioMeta;
+
+ private String[] documents;
+ private int count;
+
+ public TestFullTextGlobalIndexReader(
+ GlobalIndexFileReader fileReader, GlobalIndexIOMeta ioMeta) {
+ this.fileReader = fileReader;
+ this.ioMeta = ioMeta;
+ }
+
+ @Override
+ public Optional<ScoredGlobalIndexResult> visitFullTextSearch(FullTextSearch fullTextSearch) {
+ try {
+ ensureLoaded();
+ } catch (IOException e) {
+ throw new RuntimeException("Failed to load test full-text index", e);
+ }
+
+ String queryText = fullTextSearch.queryText();
+ int limit = fullTextSearch.limit();
+ int effectiveK = Math.min(limit, count);
+ if (effectiveK <= 0) {
+ return Optional.empty();
+ }
+
+ String[] queryTerms = queryText.toLowerCase(Locale.ROOT).split("\\s+");
+
+ // Min-heap: smallest score at head, so we evict the weakest candidate.
+ PriorityQueue<ScoredRow> topK =
+ new PriorityQueue<>(effectiveK + 1, Comparator.comparingDouble(s -> s.score));
+
+ for (int i = 0; i < count; i++) {
+ float score = computeScore(documents[i], queryTerms);
+ if (score <= 0) {
+ continue;
+ }
+ if (topK.size() < effectiveK) {
+ topK.offer(new ScoredRow(i, score));
+ } else if (score > topK.peek().score) {
+ topK.poll();
+ topK.offer(new ScoredRow(i, score));
+ }
+ }
+
+ if (topK.isEmpty()) {
+ return Optional.empty();
+ }
+
+ RoaringNavigableMap64 resultBitmap = new RoaringNavigableMap64();
+ Map<Long, Float> scoreMap = new HashMap<>(topK.size());
+ for (ScoredRow row : topK) {
+ resultBitmap.add(row.rowId);
+ scoreMap.put(row.rowId, row.score);
+ }
+
+ return Optional.of(ScoredGlobalIndexResult.create(() -> resultBitmap, scoreMap::get));
+ }
+
+ private static float computeScore(String document, String[] queryTerms) {
+ String lowerDoc = document.toLowerCase(Locale.ROOT);
+ float score = 0;
+ for (String term : queryTerms) {
+ if (lowerDoc.contains(term)) {
+ score += 1.0f / queryTerms.length;
+ }
+ }
+ return score;
+ }
+
+ private void ensureLoaded() throws IOException {
+ if (documents != null) {
+ return;
+ }
+
+ try (SeekableInputStream in = fileReader.getInputStream(ioMeta)) {
+ // Read header: count (4 bytes)
+ byte[] headerBytes = new byte[4];
+ readFully(in, headerBytes);
+ ByteBuffer header = ByteBuffer.wrap(headerBytes);
+ header.order(ByteOrder.LITTLE_ENDIAN);
+ count = header.getInt();
+
+ // Read documents
+ documents = new String[count];
+ for (int i = 0; i < count; i++) {
+ byte[] lenBytes = new byte[4];
+ readFully(in, lenBytes);
+ ByteBuffer lenBuf = ByteBuffer.wrap(lenBytes);
+ lenBuf.order(ByteOrder.LITTLE_ENDIAN);
+ int textLen = lenBuf.getInt();
+
+ byte[] textBytes = new byte[textLen];
+ readFully(in, textBytes);
+ documents[i] = new String(textBytes, StandardCharsets.UTF_8);
+ }
+ }
+ }
+
+ private static void readFully(SeekableInputStream in, byte[] buf) throws IOException {
+ int offset = 0;
+ while (offset < buf.length) {
+ int bytesRead = in.read(buf, offset, buf.length - offset);
+ if (bytesRead < 0) {
+ throw new IOException(
+ "Unexpected end of stream: read "
+ + offset
+ + " bytes but expected "
+ + buf.length);
+ }
+ offset += bytesRead;
+ }
+ }
+
+ @Override
+ public void close() throws IOException {
+ documents = null;
+ }
+
+ // =================== unsupported predicate operations =====================
+
+ @Override
+ public Optional<GlobalIndexResult> visitIsNotNull(FieldRef fieldRef) {
+ return Optional.empty();
+ }
+
+ @Override
+ public Optional<GlobalIndexResult> visitIsNull(FieldRef fieldRef) {
+ return Optional.empty();
+ }
+
+ @Override
+ public Optional<GlobalIndexResult> visitStartsWith(FieldRef fieldRef, Object literal) {
+ return Optional.empty();
+ }
+
+ @Override
+ public Optional<GlobalIndexResult> visitEndsWith(FieldRef fieldRef, Object literal) {
+ return Optional.empty();
+ }
+
+ @Override
+ public Optional<GlobalIndexResult> visitContains(FieldRef fieldRef, Object literal) {
+ return Optional.empty();
+ }
+
+ @Override
+ public Optional<GlobalIndexResult> visitLike(FieldRef fieldRef, Object literal) {
+ return Optional.empty();
+ }
+
+ @Override
+ public Optional<GlobalIndexResult> visitLessThan(FieldRef fieldRef, Object literal) {
+ return Optional.empty();
+ }
+
+ @Override
+ public Optional<GlobalIndexResult> visitGreaterOrEqual(FieldRef fieldRef, Object literal) {
+ return Optional.empty();
+ }
+
+ @Override
+ public Optional<GlobalIndexResult> visitNotEqual(FieldRef fieldRef, Object literal) {
+ return Optional.empty();
+ }
+
+ @Override
+ public Optional<GlobalIndexResult> visitLessOrEqual(FieldRef fieldRef, Object literal) {
+ return Optional.empty();
+ }
+
+ @Override
+ public Optional<GlobalIndexResult> visitEqual(FieldRef fieldRef, Object literal) {
+ return Optional.empty();
+ }
+
+ @Override
+ public Optional<GlobalIndexResult> visitGreaterThan(FieldRef fieldRef, Object literal) {
+ return Optional.empty();
+ }
+
+ @Override
+ public Optional<GlobalIndexResult> visitIn(FieldRef fieldRef, List<Object> literals) {
+ return Optional.empty();
+ }
+
+ @Override
+ public Optional<GlobalIndexResult> visitNotIn(FieldRef fieldRef, List<Object> literals) {
+ return Optional.empty();
+ }
+
+ /** A row ID paired with its similarity score, used in the top-k min-heap. */
+ private static class ScoredRow {
+ final long rowId;
+ final float score;
+
+ ScoredRow(long rowId, float score) {
+ this.rowId = rowId;
+ this.score = score;
+ }
+ }
+}
diff --git a/paimon-common/src/test/java/org/apache/paimon/globalindex/testfulltext/TestFullTextGlobalIndexWriter.java b/paimon-common/src/test/java/org/apache/paimon/globalindex/testfulltext/TestFullTextGlobalIndexWriter.java
new file mode 100644
index 0000000000..cf8d2de24a
--- /dev/null
+++ b/paimon-common/src/test/java/org/apache/paimon/globalindex/testfulltext/TestFullTextGlobalIndexWriter.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.globalindex.testfulltext;
+
+import org.apache.paimon.data.BinaryString;
+import org.apache.paimon.fs.PositionOutputStream;
+import org.apache.paimon.globalindex.GlobalIndexSingletonWriter;
+import org.apache.paimon.globalindex.ResultEntry;
+import org.apache.paimon.globalindex.io.GlobalIndexFileWriter;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * Test full-text index writer that stores all text documents in memory and writes them to a single
+ * binary file on {@link #finish()}.
+ *
+ * <p>Binary format (all values little-endian):
+ *
+ * <pre>
+ * [4 bytes] count (int)
+ * For each document:
+ * [4 bytes] text length in bytes (int)
+ * [N bytes] UTF-8 text
+ * </pre>
+ */
+public class TestFullTextGlobalIndexWriter implements GlobalIndexSingletonWriter {
+
+ private static final String FILE_NAME_PREFIX = "test-fulltext";
+
+ private final GlobalIndexFileWriter fileWriter;
+ private final List<String> documents;
+
+ public TestFullTextGlobalIndexWriter(GlobalIndexFileWriter fileWriter) {
+ this.fileWriter = fileWriter;
+ this.documents = new ArrayList<>();
+ }
+
+ @Override
+ public void write(Object fieldData) {
+ if (fieldData == null) {
+ throw new IllegalArgumentException("Text field data must not be null");
+ }
+
+ String text;
+ if (fieldData instanceof String) {
+ text = (String) fieldData;
+ } else if (fieldData instanceof BinaryString) {
+ text = fieldData.toString();
+ } else {
+ throw new IllegalArgumentException(
+ "Unsupported text type: " + fieldData.getClass().getName());
+ }
+ documents.add(text);
+ }
+
+ @Override
+ public List<ResultEntry> finish() {
+ if (documents.isEmpty()) {
+ return Collections.emptyList();
+ }
+
+ try {
+ String fileName = fileWriter.newFileName(FILE_NAME_PREFIX);
+ try (PositionOutputStream out = fileWriter.newOutputStream(fileName)) {
+ // Header: count
+ ByteBuffer header = ByteBuffer.allocate(4);
+ header.order(ByteOrder.LITTLE_ENDIAN);
+ header.putInt(documents.size());
+ out.write(header.array());
+
+ // Documents
+ for (String doc : documents) {
+ byte[] textBytes = doc.getBytes(StandardCharsets.UTF_8);
+ ByteBuffer lenBuf = ByteBuffer.allocate(4);
+ lenBuf.order(ByteOrder.LITTLE_ENDIAN);
+ lenBuf.putInt(textBytes.length);
+ out.write(lenBuf.array());
+ out.write(textBytes);
+ }
+ out.flush();
+ }
+
+ return Collections.singletonList(new ResultEntry(fileName, documents.size(), null));
+ } catch (IOException e) {
+ throw new RuntimeException("Failed to write test full-text index", e);
+ }
+ }
+}
diff --git a/paimon-common/src/test/java/org/apache/paimon/globalindex/testfulltext/TestFullTextGlobalIndexer.java b/paimon-common/src/test/java/org/apache/paimon/globalindex/testfulltext/TestFullTextGlobalIndexer.java
new file mode 100644
index 0000000000..d9d3d14ff7
--- /dev/null
+++ b/paimon-common/src/test/java/org/apache/paimon/globalindex/testfulltext/TestFullTextGlobalIndexer.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.globalindex.testfulltext;
+
+import org.apache.paimon.globalindex.GlobalIndexIOMeta;
+import org.apache.paimon.globalindex.GlobalIndexReader;
+import org.apache.paimon.globalindex.GlobalIndexWriter;
+import org.apache.paimon.globalindex.GlobalIndexer;
+import org.apache.paimon.globalindex.io.GlobalIndexFileReader;
+import org.apache.paimon.globalindex.io.GlobalIndexFileWriter;
+import org.apache.paimon.options.Options;
+import org.apache.paimon.types.DataType;
+import org.apache.paimon.types.VarCharType;
+
+import java.io.IOException;
+import java.util.List;
+
+import static org.apache.paimon.utils.Preconditions.checkArgument;
+
+/**
+ * A test-only {@link GlobalIndexer} for full-text search. Uses brute-force in-memory inverted index
+ * for text queries. No native library dependency required.
+ */
+public class TestFullTextGlobalIndexer implements GlobalIndexer {
+
+ private final DataType fieldType;
+
+ public TestFullTextGlobalIndexer(DataType fieldType, Options options) {
+ checkArgument(
+ fieldType instanceof VarCharType,
+ "TestFullTextGlobalIndexer only supports VARCHAR/STRING, but got: " + fieldType);
+ this.fieldType = fieldType;
+ }
+
+ @Override
+ public GlobalIndexWriter createWriter(GlobalIndexFileWriter fileWriter) throws IOException {
+ return new TestFullTextGlobalIndexWriter(fileWriter);
+ }
+
+ @Override
+ public GlobalIndexReader createReader(
+ GlobalIndexFileReader fileReader, List<GlobalIndexIOMeta> files) throws IOException {
+ checkArgument(files.size() == 1, "Expected exactly one index file per shard");
+ return new TestFullTextGlobalIndexReader(fileReader, files.get(0));
+ }
+}
diff --git a/paimon-spark/paimon-spark-3.2/src/main/scala/org/apache/paimon/spark/PaimonScan.scala b/paimon-common/src/test/java/org/apache/paimon/globalindex/testfulltext/TestFullTextGlobalIndexerFactory.java
similarity index 50%
copy from paimon-spark/paimon-spark-3.2/src/main/scala/org/apache/paimon/spark/PaimonScan.scala
copy to paimon-common/src/test/java/org/apache/paimon/globalindex/testfulltext/TestFullTextGlobalIndexerFactory.java
index 2ad14bd055..b501c2df10 100644
--- a/paimon-spark/paimon-spark-3.2/src/main/scala/org/apache/paimon/spark/PaimonScan.scala
+++ b/paimon-common/src/test/java/org/apache/paimon/globalindex/testfulltext/TestFullTextGlobalIndexerFactory.java
@@ -16,21 +16,28 @@
* limitations under the License.
*/
-package org.apache.paimon.spark
+package org.apache.paimon.globalindex.testfulltext;
-import org.apache.paimon.partition.PartitionPredicate
-import org.apache.paimon.predicate.{Predicate, TopN, VectorSearch}
-import org.apache.paimon.table.InnerTable
+import org.apache.paimon.globalindex.GlobalIndexer;
+import org.apache.paimon.globalindex.GlobalIndexerFactory;
+import org.apache.paimon.options.Options;
+import org.apache.paimon.types.DataField;
-import org.apache.spark.sql.types.StructType
+/**
+ * A test-only {@link GlobalIndexerFactory} for full-text search. Uses brute-force in-memory
+ * inverted index, no native dependencies required.
+ */
+public class TestFullTextGlobalIndexerFactory implements GlobalIndexerFactory {
+
+ public static final String IDENTIFIER = "test-fulltext";
+
+ @Override
+ public String identifier() {
+ return IDENTIFIER;
+ }
-case class PaimonScan(
- table: InnerTable,
- requiredSchema: StructType,
- pushedPartitionFilters: Seq[PartitionPredicate],
- pushedDataFilters: Seq[Predicate],
- override val pushedLimit: Option[Int] = None,
- override val pushedTopN: Option[TopN] = None,
- override val pushedVectorSearch: Option[VectorSearch] = None,
- bucketedScanDisabled: Boolean = true)
- extends PaimonBaseScan(table) {}
+ @Override
+ public GlobalIndexer create(DataField field, Options options) {
+ return new TestFullTextGlobalIndexer(field.type(), options);
+ }
+}
diff --git a/paimon-common/src/test/resources/META-INF/services/org.apache.paimon.globalindex.GlobalIndexerFactory b/paimon-common/src/test/resources/META-INF/services/org.apache.paimon.globalindex.GlobalIndexerFactory
index 3fa7826081..841301f1b2 100644
--- a/paimon-common/src/test/resources/META-INF/services/org.apache.paimon.globalindex.GlobalIndexerFactory
+++ b/paimon-common/src/test/resources/META-INF/services/org.apache.paimon.globalindex.GlobalIndexerFactory
@@ -14,3 +14,4 @@
# limitations under the License.
org.apache.paimon.globalindex.testvector.TestVectorGlobalIndexerFactory
+org.apache.paimon.globalindex.testfulltext.TestFullTextGlobalIndexerFactory
diff --git a/paimon-core/src/main/java/org/apache/paimon/table/FormatTable.java b/paimon-core/src/main/java/org/apache/paimon/table/FormatTable.java
index 8f298ff9bf..d7af8510a9 100644
--- a/paimon-core/src/main/java/org/apache/paimon/table/FormatTable.java
+++ b/paimon-core/src/main/java/org/apache/paimon/table/FormatTable.java
@@ -31,6 +31,7 @@ import org.apache.paimon.table.format.FormatBatchWriteBuilder;
import org.apache.paimon.table.format.FormatReadBuilder;
import org.apache.paimon.table.sink.BatchWriteBuilder;
import org.apache.paimon.table.sink.StreamWriteBuilder;
+import org.apache.paimon.table.source.FullTextSearchBuilder;
import org.apache.paimon.table.source.ReadBuilder;
import org.apache.paimon.table.source.VectorSearchBuilder;
import org.apache.paimon.types.RowType;
@@ -278,6 +279,12 @@ public interface FormatTable extends Table {
throw new UnsupportedOperationException("FormatTable does not support vector search.");
}
+ @Override
+ public FullTextSearchBuilder newFullTextSearchBuilder() {
+ throw new UnsupportedOperationException(
+ "FormatTable does not support full-text search.");
+ }
+
@Override
public CatalogContext catalogContext() {
return this.catalogContext;
diff --git a/paimon-core/src/main/java/org/apache/paimon/table/FullTextSearchTable.java b/paimon-core/src/main/java/org/apache/paimon/table/FullTextSearchTable.java
new file mode 100644
index 0000000000..efe4a9f3d9
--- /dev/null
+++ b/paimon-core/src/main/java/org/apache/paimon/table/FullTextSearchTable.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.table;
+
+import org.apache.paimon.fs.FileIO;
+import org.apache.paimon.predicate.FullTextSearch;
+import org.apache.paimon.table.source.InnerTableRead;
+import org.apache.paimon.table.source.InnerTableScan;
+import org.apache.paimon.types.RowType;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * A table wrapper to hold full-text search information. This is used to pass full-text search
+ * pushdown information from logical plan optimization to physical plan execution. For now, it is
+ * only used by internal for Spark engine.
+ */
+public class FullTextSearchTable implements ReadonlyTable {
+
+ private final InnerTable origin;
+ private final FullTextSearch fullTextSearch;
+
+ private FullTextSearchTable(InnerTable origin, FullTextSearch fullTextSearch) {
+ this.origin = origin;
+ this.fullTextSearch = fullTextSearch;
+ }
+
+ public static FullTextSearchTable create(InnerTable origin, FullTextSearch fullTextSearch) {
+ return new FullTextSearchTable(origin, fullTextSearch);
+ }
+
+ public FullTextSearch fullTextSearch() {
+ return fullTextSearch;
+ }
+
+ public InnerTable origin() {
+ return origin;
+ }
+
+ @Override
+ public String name() {
+ return origin.name();
+ }
+
+ @Override
+ public RowType rowType() {
+ return origin.rowType();
+ }
+
+ @Override
+ public List<String> primaryKeys() {
+ return origin.primaryKeys();
+ }
+
+ @Override
+ public List<String> partitionKeys() {
+ return origin.partitionKeys();
+ }
+
+ @Override
+ public Map<String, String> options() {
+ return origin.options();
+ }
+
+ @Override
+ public FileIO fileIO() {
+ return origin.fileIO();
+ }
+
+ @Override
+ public InnerTableRead newRead() {
+ return origin.newRead();
+ }
+
+ @Override
+ public InnerTableScan newScan() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Table copy(Map<String, String> dynamicOptions) {
+ return new FullTextSearchTable((InnerTable) origin.copy(dynamicOptions), fullTextSearch);
+ }
+}
diff --git a/paimon-core/src/main/java/org/apache/paimon/table/InnerTable.java b/paimon-core/src/main/java/org/apache/paimon/table/InnerTable.java
index c404672e43..d360597744 100644
--- a/paimon-core/src/main/java/org/apache/paimon/table/InnerTable.java
+++ b/paimon-core/src/main/java/org/apache/paimon/table/InnerTable.java
@@ -25,6 +25,8 @@ import org.apache.paimon.table.sink.InnerTableWrite;
import org.apache.paimon.table.sink.StreamWriteBuilder;
import org.apache.paimon.table.sink.StreamWriteBuilderImpl;
import org.apache.paimon.table.sink.WriteSelector;
+import org.apache.paimon.table.source.FullTextSearchBuilder;
+import org.apache.paimon.table.source.FullTextSearchBuilderImpl;
import org.apache.paimon.table.source.InnerTableRead;
import org.apache.paimon.table.source.InnerTableScan;
import org.apache.paimon.table.source.ReadBuilder;
@@ -60,6 +62,11 @@ public interface InnerTable extends Table {
return new VectorSearchBuilderImpl(this);
}
+ @Override
+ default FullTextSearchBuilder newFullTextSearchBuilder() {
+ return new FullTextSearchBuilderImpl(this);
+ }
+
@Override
default BatchWriteBuilder newBatchWriteBuilder() {
return new BatchWriteBuilderImpl(this);
diff --git a/paimon-core/src/main/java/org/apache/paimon/table/Table.java b/paimon-core/src/main/java/org/apache/paimon/table/Table.java
index 938718db98..13d56d6849 100644
--- a/paimon-core/src/main/java/org/apache/paimon/table/Table.java
+++ b/paimon-core/src/main/java/org/apache/paimon/table/Table.java
@@ -28,6 +28,7 @@ import org.apache.paimon.manifest.ManifestFileMeta;
import org.apache.paimon.stats.Statistics;
import org.apache.paimon.table.sink.BatchWriteBuilder;
import org.apache.paimon.table.sink.StreamWriteBuilder;
+import org.apache.paimon.table.source.FullTextSearchBuilder;
import org.apache.paimon.table.source.ReadBuilder;
import org.apache.paimon.table.source.VectorSearchBuilder;
import org.apache.paimon.types.RowType;
@@ -217,6 +218,9 @@ public interface Table extends Serializable {
/** Returns a new vector search builder. */
VectorSearchBuilder newVectorSearchBuilder();
+ /** Returns a new full-text search builder. */
+ FullTextSearchBuilder newFullTextSearchBuilder();
+
/** Returns a new read builder. */
ReadBuilder newReadBuilder();
diff --git a/paimon-spark/paimon-spark-3.2/src/main/scala/org/apache/paimon/spark/PaimonScan.scala b/paimon-core/src/main/java/org/apache/paimon/table/source/FullTextRead.java
similarity index 55%
copy from paimon-spark/paimon-spark-3.2/src/main/scala/org/apache/paimon/spark/PaimonScan.scala
copy to paimon-core/src/main/java/org/apache/paimon/table/source/FullTextRead.java
index 2ad14bd055..77cc04b16d 100644
--- a/paimon-spark/paimon-spark-3.2/src/main/scala/org/apache/paimon/spark/PaimonScan.scala
+++ b/paimon-core/src/main/java/org/apache/paimon/table/source/FullTextRead.java
@@ -16,21 +16,18 @@
* limitations under the License.
*/
-package org.apache.paimon.spark
+package org.apache.paimon.table.source;
-import org.apache.paimon.partition.PartitionPredicate
-import org.apache.paimon.predicate.{Predicate, TopN, VectorSearch}
-import org.apache.paimon.table.InnerTable
+import org.apache.paimon.globalindex.GlobalIndexResult;
-import org.apache.spark.sql.types.StructType
+import java.util.List;
-case class PaimonScan(
- table: InnerTable,
- requiredSchema: StructType,
- pushedPartitionFilters: Seq[PartitionPredicate],
- pushedDataFilters: Seq[Predicate],
- override val pushedLimit: Option[Int] = None,
- override val pushedTopN: Option[TopN] = None,
- override val pushedVectorSearch: Option[VectorSearch] = None,
- bucketedScanDisabled: Boolean = true)
- extends PaimonBaseScan(table) {}
+/** Full-text read to read index files. */
+public interface FullTextRead {
+
+ default GlobalIndexResult read(FullTextScan.Plan plan) {
+ return read(plan.splits());
+ }
+
+ GlobalIndexResult read(List<FullTextSearchSplit> splits);
+}
diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/FullTextReadImpl.java b/paimon-core/src/main/java/org/apache/paimon/table/source/FullTextReadImpl.java
new file mode 100644
index 0000000000..f58f8f26ea
--- /dev/null
+++ b/paimon-core/src/main/java/org/apache/paimon/table/source/FullTextReadImpl.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.table.source;
+
+import org.apache.paimon.fs.FileIO;
+import org.apache.paimon.globalindex.GlobalIndexIOMeta;
+import org.apache.paimon.globalindex.GlobalIndexReader;
+import org.apache.paimon.globalindex.GlobalIndexResult;
+import org.apache.paimon.globalindex.GlobalIndexer;
+import org.apache.paimon.globalindex.GlobalIndexerFactoryUtils;
+import org.apache.paimon.globalindex.OffsetGlobalIndexReader;
+import org.apache.paimon.globalindex.ScoredGlobalIndexResult;
+import org.apache.paimon.globalindex.io.GlobalIndexFileReader;
+import org.apache.paimon.index.GlobalIndexMeta;
+import org.apache.paimon.index.IndexFileMeta;
+import org.apache.paimon.index.IndexPathFactory;
+import org.apache.paimon.predicate.FullTextSearch;
+import org.apache.paimon.table.FileStoreTable;
+import org.apache.paimon.types.DataField;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Optional;
+
+import static java.util.Collections.singletonList;
+import static org.apache.paimon.utils.ManifestReadThreadPool.randomlyExecuteSequentialReturn;
+import static org.apache.paimon.utils.Preconditions.checkNotNull;
+
+/** Implementation for {@link FullTextRead}. */
+public class FullTextReadImpl implements FullTextRead {
+
+ private final FileStoreTable table;
+ private final int limit;
+ private final DataField textColumn;
+ private final String queryText;
+
+ public FullTextReadImpl(
+ FileStoreTable table, int limit, DataField textColumn, String queryText) {
+ this.table = table;
+ this.limit = limit;
+ this.textColumn = textColumn;
+ this.queryText = queryText;
+ }
+
+ @Override
+ public GlobalIndexResult read(List<FullTextSearchSplit> splits) {
+ if (splits.isEmpty()) {
+ return GlobalIndexResult.createEmpty();
+ }
+
+ Integer threadNum = table.coreOptions().globalIndexThreadNum();
+
+ String indexType = splits.get(0).fullTextIndexFiles().get(0).indexType();
+ GlobalIndexer globalIndexer =
+ GlobalIndexerFactoryUtils.load(indexType)
+ .create(textColumn, table.coreOptions().toConfiguration());
+ IndexPathFactory indexPathFactory = table.store().pathFactory().globalIndexFileFactory();
+ Iterator<Optional<ScoredGlobalIndexResult>> resultIterators =
+ randomlyExecuteSequentialReturn(
+ split ->
+ singletonList(
+ eval(
+ globalIndexer,
+ indexPathFactory,
+ split.rowRangeStart(),
+ split.rowRangeEnd(),
+ split.fullTextIndexFiles())),
+ splits,
+ threadNum);
+
+ ScoredGlobalIndexResult result = ScoredGlobalIndexResult.createEmpty();
+ while (resultIterators.hasNext()) {
+ Optional<ScoredGlobalIndexResult> next = resultIterators.next();
+ if (next.isPresent()) {
+ result = result.or(next.get());
+ }
+ }
+
+ return result.topK(limit);
+ }
+
+ private Optional<ScoredGlobalIndexResult> eval(
+ GlobalIndexer globalIndexer,
+ IndexPathFactory indexPathFactory,
+ long rowRangeStart,
+ long rowRangeEnd,
+ List<IndexFileMeta> fullTextIndexFiles) {
+ List<GlobalIndexIOMeta> indexIOMetaList = new ArrayList<>();
+ for (IndexFileMeta indexFile : fullTextIndexFiles) {
+ GlobalIndexMeta meta = checkNotNull(indexFile.globalIndexMeta());
+ indexIOMetaList.add(
+ new GlobalIndexIOMeta(
+ indexPathFactory.toPath(indexFile),
+ indexFile.fileSize(),
+ meta.indexMeta()));
+ }
+ @SuppressWarnings("resource")
+ FileIO fileIO = table.fileIO();
+ GlobalIndexFileReader indexFileReader = m -> fileIO.newInputStream(m.filePath());
+ try (GlobalIndexReader reader =
+ globalIndexer.createReader(indexFileReader, indexIOMetaList)) {
+ FullTextSearch fullTextSearch = new FullTextSearch(queryText, limit, textColumn.name());
+ return new OffsetGlobalIndexReader(reader, rowRangeStart, rowRangeEnd)
+ .visitFullTextSearch(fullTextSearch);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+}
diff --git a/paimon-spark/paimon-spark-3.2/src/main/scala/org/apache/paimon/spark/PaimonScan.scala b/paimon-core/src/main/java/org/apache/paimon/table/source/FullTextScan.java
similarity index 55%
copy from paimon-spark/paimon-spark-3.2/src/main/scala/org/apache/paimon/spark/PaimonScan.scala
copy to paimon-core/src/main/java/org/apache/paimon/table/source/FullTextScan.java
index 2ad14bd055..563ffe1c9d 100644
--- a/paimon-spark/paimon-spark-3.2/src/main/scala/org/apache/paimon/spark/PaimonScan.scala
+++ b/paimon-core/src/main/java/org/apache/paimon/table/source/FullTextScan.java
@@ -16,21 +16,17 @@
* limitations under the License.
*/
-package org.apache.paimon.spark
+package org.apache.paimon.table.source;
-import org.apache.paimon.partition.PartitionPredicate
-import org.apache.paimon.predicate.{Predicate, TopN, VectorSearch}
-import org.apache.paimon.table.InnerTable
+import java.util.List;
-import org.apache.spark.sql.types.StructType
+/** Full-text scan to scan index files. */
+public interface FullTextScan {
-case class PaimonScan(
- table: InnerTable,
- requiredSchema: StructType,
- pushedPartitionFilters: Seq[PartitionPredicate],
- pushedDataFilters: Seq[Predicate],
- override val pushedLimit: Option[Int] = None,
- override val pushedTopN: Option[TopN] = None,
- override val pushedVectorSearch: Option[VectorSearch] = None,
- bucketedScanDisabled: Boolean = true)
- extends PaimonBaseScan(table) {}
+ Plan scan();
+
+ /** Plan of full-text scan. */
+ interface Plan {
+ List<FullTextSearchSplit> splits();
+ }
+}
diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/FullTextScanImpl.java b/paimon-core/src/main/java/org/apache/paimon/table/source/FullTextScanImpl.java
new file mode 100644
index 0000000000..cc77d9121a
--- /dev/null
+++ b/paimon-core/src/main/java/org/apache/paimon/table/source/FullTextScanImpl.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.table.source;
+
+import org.apache.paimon.Snapshot;
+import org.apache.paimon.index.GlobalIndexMeta;
+import org.apache.paimon.index.IndexFileHandler;
+import org.apache.paimon.index.IndexFileMeta;
+import org.apache.paimon.manifest.IndexManifestEntry;
+import org.apache.paimon.table.FileStoreTable;
+import org.apache.paimon.table.source.snapshot.TimeTravelUtil;
+import org.apache.paimon.types.DataField;
+import org.apache.paimon.utils.Filter;
+import org.apache.paimon.utils.Range;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
+import static org.apache.paimon.utils.Preconditions.checkNotNull;
+
+/** Implementation for {@link FullTextScan}. */
+public class FullTextScanImpl implements FullTextScan {
+
+ private final FileStoreTable table;
+ private final DataField textColumn;
+
+ public FullTextScanImpl(FileStoreTable table, DataField textColumn) {
+ this.table = table;
+ this.textColumn = textColumn;
+ }
+
+ @Override
+ public Plan scan() {
+ Objects.requireNonNull(textColumn, "Text column must be set");
+
+ Snapshot snapshot = TimeTravelUtil.tryTravelOrLatest(table);
+ IndexFileHandler indexFileHandler = table.store().newIndexFileHandler();
+ Filter<IndexManifestEntry> indexFileFilter =
+ entry -> {
+ GlobalIndexMeta globalIndex = entry.indexFile().globalIndexMeta();
+ if (globalIndex == null) {
+ return false;
+ }
+ return textColumn.id() == globalIndex.indexFieldId();
+ };
+
+ List<IndexFileMeta> allIndexFiles =
+ indexFileHandler.scan(snapshot, indexFileFilter).stream()
+ .map(IndexManifestEntry::indexFile)
+ .collect(Collectors.toList());
+
+ // Group full-text index files by (rowRangeStart, rowRangeEnd)
+ Map<Range, List<IndexFileMeta>> byRange = new HashMap<>();
+ for (IndexFileMeta indexFile : allIndexFiles) {
+ GlobalIndexMeta meta = checkNotNull(indexFile.globalIndexMeta());
+ Range range = new Range(meta.rowRangeStart(), meta.rowRangeEnd());
+ byRange.computeIfAbsent(range, k -> new ArrayList<>()).add(indexFile);
+ }
+
+ List<FullTextSearchSplit> splits = new ArrayList<>();
+ for (Map.Entry<Range, List<IndexFileMeta>> entry : byRange.entrySet()) {
+ Range range = entry.getKey();
+ splits.add(new FullTextSearchSplit(range.from, range.to, entry.getValue()));
+ }
+
+ return () -> splits;
+ }
+}
diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/FullTextSearchBuilder.java b/paimon-core/src/main/java/org/apache/paimon/table/source/FullTextSearchBuilder.java
new file mode 100644
index 0000000000..731962d0cf
--- /dev/null
+++ b/paimon-core/src/main/java/org/apache/paimon/table/source/FullTextSearchBuilder.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.table.source;
+
+import org.apache.paimon.globalindex.GlobalIndexResult;
+
+import java.io.Serializable;
+
+/** Builder to build full-text search. */
+public interface FullTextSearchBuilder extends Serializable {
+
+ /** The top k results to return. */
+ FullTextSearchBuilder withLimit(int limit);
+
+ /** The text column to search. */
+ FullTextSearchBuilder withTextColumn(String name);
+
+ /** The query text to search. */
+ FullTextSearchBuilder withQueryText(String queryText);
+
+ /** Create full-text scan to scan index files. */
+ FullTextScan newFullTextScan();
+
+ /** Create full-text read to read index files. */
+ FullTextRead newFullTextRead();
+
+ /** Execute full-text index search in local. */
+ default GlobalIndexResult executeLocal() {
+ return newFullTextRead().read(newFullTextScan().scan());
+ }
+}
diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/FullTextSearchBuilderImpl.java b/paimon-core/src/main/java/org/apache/paimon/table/source/FullTextSearchBuilderImpl.java
new file mode 100644
index 0000000000..1165fcf205
--- /dev/null
+++ b/paimon-core/src/main/java/org/apache/paimon/table/source/FullTextSearchBuilderImpl.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.table.source;
+
+import org.apache.paimon.table.FileStoreTable;
+import org.apache.paimon.table.InnerTable;
+import org.apache.paimon.types.DataField;
+
+import static org.apache.paimon.utils.Preconditions.checkArgument;
+import static org.apache.paimon.utils.Preconditions.checkNotNull;
+
+/** Implementation for {@link FullTextSearchBuilder}. */
+public class FullTextSearchBuilderImpl implements FullTextSearchBuilder {
+
+ private static final long serialVersionUID = 1L;
+
+ private final FileStoreTable table;
+
+ private int limit;
+ private DataField textColumn;
+ private String queryText;
+
+ public FullTextSearchBuilderImpl(InnerTable table) {
+ this.table = (FileStoreTable) table;
+ }
+
+ @Override
+ public FullTextSearchBuilder withLimit(int limit) {
+ this.limit = limit;
+ return this;
+ }
+
+ @Override
+ public FullTextSearchBuilder withTextColumn(String name) {
+ this.textColumn = table.rowType().getField(name);
+ return this;
+ }
+
+ @Override
+ public FullTextSearchBuilder withQueryText(String queryText) {
+ this.queryText = queryText;
+ return this;
+ }
+
+ @Override
+ public FullTextScan newFullTextScan() {
+ checkNotNull(textColumn, "Text column must be set via withTextColumn()");
+ return new FullTextScanImpl(table, textColumn);
+ }
+
+ @Override
+ public FullTextRead newFullTextRead() {
+ checkArgument(limit > 0, "Limit must be positive, set via withLimit()");
+ checkNotNull(textColumn, "Text column must be set via withTextColumn()");
+ checkNotNull(queryText, "Query text must be set via withQueryText()");
+ return new FullTextReadImpl(table, limit, textColumn, queryText);
+ }
+}
diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/FullTextSearchSplit.java b/paimon-core/src/main/java/org/apache/paimon/table/source/FullTextSearchSplit.java
new file mode 100644
index 0000000000..d230482186
--- /dev/null
+++ b/paimon-core/src/main/java/org/apache/paimon/table/source/FullTextSearchSplit.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.table.source;
+
+import org.apache.paimon.index.IndexFileMeta;
+
+import java.io.Serializable;
+import java.util.List;
+import java.util.Objects;
+
+/** Split of full-text search. */
+public class FullTextSearchSplit implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ private final long rowRangeStart;
+ private final long rowRangeEnd;
+ private final List<IndexFileMeta> fullTextIndexFiles;
+
+ public FullTextSearchSplit(
+ long rowRangeStart, long rowRangeEnd, List<IndexFileMeta> fullTextIndexFiles) {
+ this.rowRangeStart = rowRangeStart;
+ this.rowRangeEnd = rowRangeEnd;
+ this.fullTextIndexFiles = fullTextIndexFiles;
+ }
+
+ public long rowRangeStart() {
+ return rowRangeStart;
+ }
+
+ public long rowRangeEnd() {
+ return rowRangeEnd;
+ }
+
+ public List<IndexFileMeta> fullTextIndexFiles() {
+ return fullTextIndexFiles;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ FullTextSearchSplit that = (FullTextSearchSplit) o;
+ return rowRangeStart == that.rowRangeStart
+ && rowRangeEnd == that.rowRangeEnd
+ && Objects.equals(fullTextIndexFiles, that.fullTextIndexFiles);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(rowRangeStart, rowRangeEnd, fullTextIndexFiles);
+ }
+
+ @Override
+ public String toString() {
+ return "FullTextSearchSplit{"
+ + "rowRangeStart="
+ + rowRangeStart
+ + ", rowRangeEnd="
+ + rowRangeEnd
+ + ", fullTextIndexFiles="
+ + fullTextIndexFiles
+ + '}';
+ }
+}
diff --git a/paimon-core/src/test/java/org/apache/paimon/table/source/FullTextSearchBuilderTest.java b/paimon-core/src/test/java/org/apache/paimon/table/source/FullTextSearchBuilderTest.java
new file mode 100644
index 0000000000..c52a2a46af
--- /dev/null
+++ b/paimon-core/src/test/java/org/apache/paimon/table/source/FullTextSearchBuilderTest.java
@@ -0,0 +1,424 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.table.source;
+
+import org.apache.paimon.CoreOptions;
+import org.apache.paimon.data.BinaryRow;
+import org.apache.paimon.data.BinaryString;
+import org.apache.paimon.data.GenericRow;
+import org.apache.paimon.data.InternalRow;
+import org.apache.paimon.globalindex.GlobalIndexBuilderUtils;
+import org.apache.paimon.globalindex.GlobalIndexResult;
+import org.apache.paimon.globalindex.GlobalIndexSingletonWriter;
+import org.apache.paimon.globalindex.ResultEntry;
+import org.apache.paimon.globalindex.ScoredGlobalIndexResult;
+import org.apache.paimon.globalindex.testfulltext.TestFullTextGlobalIndexerFactory;
+import org.apache.paimon.index.IndexFileMeta;
+import org.apache.paimon.io.CompactIncrement;
+import org.apache.paimon.io.DataIncrement;
+import org.apache.paimon.options.Options;
+import org.apache.paimon.reader.RecordReader;
+import org.apache.paimon.schema.Schema;
+import org.apache.paimon.table.FileStoreTable;
+import org.apache.paimon.table.TableTestBase;
+import org.apache.paimon.table.sink.BatchTableCommit;
+import org.apache.paimon.table.sink.BatchTableWrite;
+import org.apache.paimon.table.sink.BatchWriteBuilder;
+import org.apache.paimon.table.sink.CommitMessage;
+import org.apache.paimon.table.sink.CommitMessageImpl;
+import org.apache.paimon.types.DataField;
+import org.apache.paimon.types.DataTypes;
+import org.apache.paimon.utils.Range;
+
+import org.junit.jupiter.api.Test;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/** Tests for {@link FullTextSearchBuilder} using test-only brute-force full-text index. */
+public class FullTextSearchBuilderTest extends TableTestBase {
+
+ private static final String TEXT_FIELD_NAME = "content";
+
+ @Override
+ protected Schema schemaDefault() {
+ return Schema.newBuilder()
+ .column("id", DataTypes.INT())
+ .column(TEXT_FIELD_NAME, DataTypes.STRING())
+ .option(CoreOptions.BUCKET.key(), "-1")
+ .option(CoreOptions.ROW_TRACKING_ENABLED.key(), "true")
+ .option(CoreOptions.DATA_EVOLUTION_ENABLED.key(), "true")
+ .build();
+ }
+
+ @Test
+ public void testFullTextSearchEndToEnd() throws Exception {
+ createTableDefault();
+ FileStoreTable table = getTableDefault();
+
+ String[] documents = {
+ "Apache Paimon is a lake format",
+ "Paimon supports full-text search",
+ "Vector search is also supported",
+ "Paimon provides streaming and batch processing",
+ "Full-text indexing enables fast text queries",
+ "The lake format supports ACID transactions"
+ };
+
+ writeDocuments(table, documents);
+ buildAndCommitIndex(table, documents);
+
+ // Query "Paimon" - should match rows 0, 1, 3
+ GlobalIndexResult result =
+ table.newFullTextSearchBuilder()
+ .withQueryText("Paimon")
+ .withLimit(3)
+ .withTextColumn(TEXT_FIELD_NAME)
+ .executeLocal();
+
+ assertThat(result).isInstanceOf(ScoredGlobalIndexResult.class);
+ assertThat(result.results().isEmpty()).isFalse();
+
+ // Read using the search result
+ ReadBuilder readBuilder = table.newReadBuilder();
+ List<Integer> ids = new ArrayList<>();
+ TableScan.Plan plan = readBuilder.newScan().withGlobalIndexResult(result).plan();
+ try (RecordReader<InternalRow> reader = readBuilder.newRead().createReader(plan)) {
+ reader.forEachRemaining(row -> ids.add(row.getInt(0)));
+ }
+
+ assertThat(ids).isNotEmpty();
+ assertThat(ids.size()).isLessThanOrEqualTo(3);
+ // Rows 0, 1, 3 contain "Paimon"
+ assertThat(ids).containsAnyOf(0, 1, 3);
+ }
+
+ @Test
+ public void testFullTextSearchMultiTermQuery() throws Exception {
+ createTableDefault();
+ FileStoreTable table = getTableDefault();
+
+ String[] documents = {
+ "Apache Paimon lake format",
+ "Paimon full-text search support",
+ "full-text search in Apache Paimon",
+ "Vector search capabilities",
+ };
+
+ writeDocuments(table, documents);
+ buildAndCommitIndex(table, documents);
+
+ // Query "Paimon search" - row 2 matches both terms, rows 1 matches both, row 0 matches one
+ GlobalIndexResult result =
+ table.newFullTextSearchBuilder()
+ .withQueryText("Paimon search")
+ .withLimit(2)
+ .withTextColumn(TEXT_FIELD_NAME)
+ .executeLocal();
+
+ assertThat(result).isInstanceOf(ScoredGlobalIndexResult.class);
+ assertThat(result.results().isEmpty()).isFalse();
+
+ ReadBuilder readBuilder = table.newReadBuilder();
+ TableScan.Plan plan = readBuilder.newScan().withGlobalIndexResult(result).plan();
+ List<Integer> ids = new ArrayList<>();
+ try (RecordReader<InternalRow> reader = readBuilder.newRead().createReader(plan)) {
+ reader.forEachRemaining(row -> ids.add(row.getInt(0)));
+ }
+
+ assertThat(ids).hasSize(2);
+ // Rows 1 and 2 contain both "Paimon" and "search"
+ assertThat(ids).contains(1, 2);
+ }
+
+ @Test
+ public void testFullTextSearchEmptyResult() throws Exception {
+ createTableDefault();
+ FileStoreTable table = getTableDefault();
+
+ // Write data but no index - should return empty result
+ String[] documents = {"hello world", "foo bar"};
+ writeDocuments(table, documents);
+
+ GlobalIndexResult result =
+ table.newFullTextSearchBuilder()
+ .withQueryText("nonexistent")
+ .withLimit(1)
+ .withTextColumn(TEXT_FIELD_NAME)
+ .executeLocal();
+
+ assertThat(result.results().isEmpty()).isTrue();
+ }
+
+ @Test
+ public void testFullTextSearchTopKLimit() throws Exception {
+ createTableDefault();
+ FileStoreTable table = getTableDefault();
+
+ String[] documents = new String[20];
+ for (int i = 0; i < 20; i++) {
+ documents[i] = "document number " + i + " with common keyword";
+ }
+
+ writeDocuments(table, documents);
+ buildAndCommitIndex(table, documents);
+
+ // Search with limit=5
+ GlobalIndexResult result =
+ table.newFullTextSearchBuilder()
+ .withQueryText("keyword")
+ .withLimit(5)
+ .withTextColumn(TEXT_FIELD_NAME)
+ .executeLocal();
+
+ ReadBuilder readBuilder = table.newReadBuilder();
+ TableScan.Plan plan = readBuilder.newScan().withGlobalIndexResult(result).plan();
+ List<Integer> ids = new ArrayList<>();
+ try (RecordReader<InternalRow> reader = readBuilder.newRead().createReader(plan)) {
+ reader.forEachRemaining(row -> ids.add(row.getInt(0)));
+ }
+
+ assertThat(ids.size()).isLessThanOrEqualTo(5);
+ }
+
+ @Test
+ public void testFullTextSearchWithMultipleIndexFiles() throws Exception {
+ createTableDefault();
+ FileStoreTable table = getTableDefault();
+
+ String[] allDocuments = {
+ "Apache Paimon lake format", // row 0
+ "Paimon supports streaming", // row 1
+ "batch processing engine", // row 2
+ "Paimon full-text search", // row 3
+ "vector similarity search", // row 4
+ "Paimon ACID transactions" // row 5
+ };
+
+ writeDocuments(table, allDocuments);
+
+ // Build two separate index files covering different row ranges
+ buildAndCommitMultipleIndexFiles(table, allDocuments);
+
+ // Query "Paimon" - results should span across both index files
+ GlobalIndexResult result =
+ table.newFullTextSearchBuilder()
+ .withQueryText("Paimon")
+ .withLimit(4)
+ .withTextColumn(TEXT_FIELD_NAME)
+ .executeLocal();
+
+ assertThat(result).isInstanceOf(ScoredGlobalIndexResult.class);
+ assertThat(result.results().isEmpty()).isFalse();
+
+ ReadBuilder readBuilder = table.newReadBuilder();
+ TableScan.Plan plan = readBuilder.newScan().withGlobalIndexResult(result).plan();
+ List<Integer> ids = new ArrayList<>();
+ try (RecordReader<InternalRow> reader = readBuilder.newRead().createReader(plan)) {
+ reader.forEachRemaining(row -> ids.add(row.getInt(0)));
+ }
+
+ assertThat(ids).isNotEmpty();
+ // Rows 0,1 are in first index file, rows 3,5 are in second
+ assertThat(ids).containsAnyOf(0, 1);
+ assertThat(ids).containsAnyOf(3, 5);
+ }
+
+ @Test
+ public void testFullTextSearchNoMatchingDocuments() throws Exception {
+ createTableDefault();
+ FileStoreTable table = getTableDefault();
+
+ String[] documents = {
+ "Apache Paimon lake format", "streaming batch processing", "ACID transactions support"
+ };
+
+ writeDocuments(table, documents);
+ buildAndCommitIndex(table, documents);
+
+ // Query a term that doesn't exist in any document
+ GlobalIndexResult result =
+ table.newFullTextSearchBuilder()
+ .withQueryText("nonexistent")
+ .withLimit(3)
+ .withTextColumn(TEXT_FIELD_NAME)
+ .executeLocal();
+
+ assertThat(result.results().isEmpty()).isTrue();
+ }
+
+ @Test
+ public void testFullTextSearchCaseInsensitive() throws Exception {
+ createTableDefault();
+ FileStoreTable table = getTableDefault();
+
+ String[] documents = {
+ "Apache PAIMON Lake Format", "paimon supports search", "Paimon Is Great"
+ };
+
+ writeDocuments(table, documents);
+ buildAndCommitIndex(table, documents);
+
+ // Query lowercase "paimon" should match all three (case-insensitive)
+ GlobalIndexResult result =
+ table.newFullTextSearchBuilder()
+ .withQueryText("paimon")
+ .withLimit(3)
+ .withTextColumn(TEXT_FIELD_NAME)
+ .executeLocal();
+
+ assertThat(result).isInstanceOf(ScoredGlobalIndexResult.class);
+
+ ReadBuilder readBuilder = table.newReadBuilder();
+ TableScan.Plan plan = readBuilder.newScan().withGlobalIndexResult(result).plan();
+ List<Integer> ids = new ArrayList<>();
+ try (RecordReader<InternalRow> reader = readBuilder.newRead().createReader(plan)) {
+ reader.forEachRemaining(row -> ids.add(row.getInt(0)));
+ }
+
+ assertThat(ids).hasSize(3);
+ assertThat(ids).contains(0, 1, 2);
+ }
+
+ // ====================== Helper methods ======================
+
+ private void writeDocuments(FileStoreTable table, String[] documents) throws Exception {
+ BatchWriteBuilder writeBuilder = table.newBatchWriteBuilder();
+ try (BatchTableWrite write = writeBuilder.newWrite();
+ BatchTableCommit commit = writeBuilder.newCommit()) {
+ for (int i = 0; i < documents.length; i++) {
+ write.write(GenericRow.of(i, BinaryString.fromString(documents[i])));
+ }
+ commit.commit(write.prepareCommit());
+ }
+ }
+
+ private void buildAndCommitIndex(FileStoreTable table, String[] documents) throws Exception {
+ Options options = table.coreOptions().toConfiguration();
+ DataField textField = table.rowType().getField(TEXT_FIELD_NAME);
+
+ GlobalIndexSingletonWriter writer =
+ (GlobalIndexSingletonWriter)
+ GlobalIndexBuilderUtils.createIndexWriter(
+ table,
+ TestFullTextGlobalIndexerFactory.IDENTIFIER,
+ textField,
+ options);
+ for (String doc : documents) {
+ writer.write(doc);
+ }
+ List<ResultEntry> entries = writer.finish();
+
+ Range rowRange = new Range(0, documents.length - 1);
+ List<IndexFileMeta> indexFiles =
+ GlobalIndexBuilderUtils.toIndexFileMetas(
+ table.fileIO(),
+ table.store().pathFactory().globalIndexFileFactory(),
+ table.coreOptions(),
+ rowRange,
+ textField.id(),
+ TestFullTextGlobalIndexerFactory.IDENTIFIER,
+ entries);
+
+ DataIncrement dataIncrement = DataIncrement.indexIncrement(indexFiles);
+ CommitMessage message =
+ new CommitMessageImpl(
+ BinaryRow.EMPTY_ROW,
+ 0,
+ null,
+ dataIncrement,
+ CompactIncrement.emptyIncrement());
+ try (BatchTableCommit commit = table.newBatchWriteBuilder().newCommit()) {
+ commit.commit(Collections.singletonList(message));
+ }
+ }
+
+ private void buildAndCommitMultipleIndexFiles(FileStoreTable table, String[] documents)
+ throws Exception {
+ Options options = table.coreOptions().toConfiguration();
+ DataField textField = table.rowType().getField(TEXT_FIELD_NAME);
+ int mid = documents.length / 2;
+
+ // Build first index file covering rows [0, mid)
+ GlobalIndexSingletonWriter writer1 =
+ (GlobalIndexSingletonWriter)
+ GlobalIndexBuilderUtils.createIndexWriter(
+ table,
+ TestFullTextGlobalIndexerFactory.IDENTIFIER,
+ textField,
+ options);
+ for (int i = 0; i < mid; i++) {
+ writer1.write(documents[i]);
+ }
+ List<ResultEntry> entries1 = writer1.finish();
+ Range rowRange1 = new Range(0, mid - 1);
+ List<IndexFileMeta> indexFiles1 =
+ GlobalIndexBuilderUtils.toIndexFileMetas(
+ table.fileIO(),
+ table.store().pathFactory().globalIndexFileFactory(),
+ table.coreOptions(),
+ rowRange1,
+ textField.id(),
+ TestFullTextGlobalIndexerFactory.IDENTIFIER,
+ entries1);
+
+ // Build second index file covering rows [mid, end)
+ GlobalIndexSingletonWriter writer2 =
+ (GlobalIndexSingletonWriter)
+ GlobalIndexBuilderUtils.createIndexWriter(
+ table,
+ TestFullTextGlobalIndexerFactory.IDENTIFIER,
+ textField,
+ options);
+ for (int i = mid; i < documents.length; i++) {
+ writer2.write(documents[i]);
+ }
+ List<ResultEntry> entries2 = writer2.finish();
+ Range rowRange2 = new Range(mid, documents.length - 1);
+ List<IndexFileMeta> indexFiles2 =
+ GlobalIndexBuilderUtils.toIndexFileMetas(
+ table.fileIO(),
+ table.store().pathFactory().globalIndexFileFactory(),
+ table.coreOptions(),
+ rowRange2,
+ textField.id(),
+ TestFullTextGlobalIndexerFactory.IDENTIFIER,
+ entries2);
+
+ // Combine all index files and commit together
+ List<IndexFileMeta> allIndexFiles = new ArrayList<>();
+ allIndexFiles.addAll(indexFiles1);
+ allIndexFiles.addAll(indexFiles2);
+
+ DataIncrement dataIncrement = DataIncrement.indexIncrement(allIndexFiles);
+ CommitMessage message =
+ new CommitMessageImpl(
+ BinaryRow.EMPTY_ROW,
+ 0,
+ null,
+ dataIncrement,
+ CompactIncrement.emptyIncrement());
+ try (BatchTableCommit commit = table.newBatchWriteBuilder().newCommit()) {
+ commit.commit(Collections.singletonList(message));
+ }
+ }
+}
diff --git a/paimon-spark/paimon-spark-3.2/src/main/scala/org/apache/paimon/spark/PaimonBaseScanBuilder.scala b/paimon-spark/paimon-spark-3.2/src/main/scala/org/apache/paimon/spark/PaimonBaseScanBuilder.scala
index 4f5451c95d..f6e3b97c81 100644
--- a/paimon-spark/paimon-spark-3.2/src/main/scala/org/apache/paimon/spark/PaimonBaseScanBuilder.scala
+++ b/paimon-spark/paimon-spark-3.2/src/main/scala/org/apache/paimon/spark/PaimonBaseScanBuilder.scala
@@ -22,6 +22,7 @@ import org.apache.paimon.CoreOptions
import org.apache.paimon.partition.PartitionPredicate
import org.apache.paimon.partition.PartitionPredicate.splitPartitionPredicatesAndDataPredicates
import org.apache.paimon.predicate.{PartitionPredicateVisitor, Predicate, TopN, VectorSearch}
+import org.apache.paimon.predicate.FullTextSearch
import org.apache.paimon.table.SpecialFields.rowTypeWithRowTracking
import org.apache.paimon.table.Table
import org.apache.paimon.types.RowType
@@ -53,6 +54,7 @@ abstract class PaimonBaseScanBuilder
protected var pushedLimit: Option[Int] = None
protected var pushedTopN: Option[TopN] = None
protected var pushedVectorSearch: Option[VectorSearch] = None
+ protected var pushedFullTextSearch: Option[FullTextSearch] = None
protected var requiredSchema: StructType = SparkTypeUtils.fromPaimonRowType(table.rowType())
diff --git a/paimon-spark/paimon-spark-3.2/src/main/scala/org/apache/paimon/spark/PaimonScan.scala b/paimon-spark/paimon-spark-3.2/src/main/scala/org/apache/paimon/spark/PaimonScan.scala
index 2ad14bd055..f5a5c8e95e 100644
--- a/paimon-spark/paimon-spark-3.2/src/main/scala/org/apache/paimon/spark/PaimonScan.scala
+++ b/paimon-spark/paimon-spark-3.2/src/main/scala/org/apache/paimon/spark/PaimonScan.scala
@@ -19,7 +19,7 @@
package org.apache.paimon.spark
import org.apache.paimon.partition.PartitionPredicate
-import org.apache.paimon.predicate.{Predicate, TopN, VectorSearch}
+import org.apache.paimon.predicate.{FullTextSearch, Predicate, TopN, VectorSearch}
import org.apache.paimon.table.InnerTable
import org.apache.spark.sql.types.StructType
@@ -32,5 +32,6 @@ case class PaimonScan(
override val pushedLimit: Option[Int] = None,
override val pushedTopN: Option[TopN] = None,
override val pushedVectorSearch: Option[VectorSearch] = None,
+ override val pushedFullTextSearch: Option[FullTextSearch] = None,
bucketedScanDisabled: Boolean = true)
extends PaimonBaseScan(table) {}
diff --git a/paimon-spark/paimon-spark-3.3/src/main/scala/org/apache/paimon/spark/PaimonScan.scala b/paimon-spark/paimon-spark-3.3/src/main/scala/org/apache/paimon/spark/PaimonScan.scala
index 1da9b17a81..707ee13459 100644
--- a/paimon-spark/paimon-spark-3.3/src/main/scala/org/apache/paimon/spark/PaimonScan.scala
+++ b/paimon-spark/paimon-spark-3.3/src/main/scala/org/apache/paimon/spark/PaimonScan.scala
@@ -19,7 +19,7 @@
package org.apache.paimon.spark
import org.apache.paimon.partition.PartitionPredicate
-import org.apache.paimon.predicate.{Predicate, TopN, VectorSearch}
+import org.apache.paimon.predicate.{FullTextSearch, Predicate, TopN, VectorSearch}
import org.apache.paimon.table.{BucketMode, FileStoreTable, InnerTable}
import org.apache.paimon.table.source.{DataSplit, Split}
@@ -36,6 +36,7 @@ case class PaimonScan(
override val pushedLimit: Option[Int],
override val pushedTopN: Option[TopN],
override val pushedVectorSearch: Option[VectorSearch],
+ override val pushedFullTextSearch: Option[FullTextSearch] = None,
bucketedScanDisabled: Boolean = false)
extends PaimonBaseScan(table)
with SupportsReportPartitioning {
diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonBaseScan.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonBaseScan.scala
index bd884d64b1..30a1621925 100644
--- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonBaseScan.scala
+++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonBaseScan.scala
@@ -45,7 +45,7 @@ abstract class PaimonBaseScan(table: InnerTable)
protected def getInputSplits: Array[Split] = {
readBuilder
.newScan()
- .withGlobalIndexResult(evalVectorSearch())
+ .withGlobalIndexResult(evalGlobalIndexSearch())
.asInstanceOf[InnerTableScan]
.withMetricRegistry(paimonMetricsRegistry)
.plan()
@@ -54,11 +54,21 @@ abstract class PaimonBaseScan(table: InnerTable)
.toArray
}
- private def evalVectorSearch(): GlobalIndexResult = {
- if (pushedVectorSearch.isEmpty) {
- return null
+ private def evalGlobalIndexSearch(): GlobalIndexResult = {
+ if (pushedVectorSearch.isDefined && pushedFullTextSearch.isDefined) {
+ throw new UnsupportedOperationException(
+ "Cannot push down both vector search and full-text search simultaneously.")
+ }
+ if (pushedVectorSearch.isDefined) {
+ return evalVectorSearch()
}
+ if (pushedFullTextSearch.isDefined) {
+ return evalFullTextSearch()
+ }
+ null
+ }
+ private def evalVectorSearch(): GlobalIndexResult = {
val vectorSearch = pushedVectorSearch.get
val vectorBuilder = table
.newVectorSearchBuilder()
@@ -74,6 +84,16 @@ abstract class PaimonBaseScan(table: InnerTable)
vectorBuilder.newVectorRead().read(vectorBuilder.newVectorScan().scan())
}
+ private def evalFullTextSearch(): GlobalIndexResult = {
+ val fullTextSearch = pushedFullTextSearch.get
+ val ftBuilder = table
+ .newFullTextSearchBuilder()
+ .withQueryText(fullTextSearch.queryText())
+ .withTextColumn(fullTextSearch.fieldName())
+ .withLimit(fullTextSearch.limit())
+ ftBuilder.newFullTextRead().read(ftBuilder.newFullTextScan().scan())
+ }
+
override def toBatch: Batch = {
ensureNoFullScan()
super.toBatch
diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonBaseScanBuilder.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonBaseScanBuilder.scala
index 47723171e4..14a090ff14 100644
--- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonBaseScanBuilder.scala
+++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonBaseScanBuilder.scala
@@ -22,6 +22,7 @@ import org.apache.paimon.CoreOptions
import org.apache.paimon.partition.PartitionPredicate
import org.apache.paimon.partition.PartitionPredicate.splitPartitionPredicatesAndDataPredicates
import org.apache.paimon.predicate.{PartitionPredicateVisitor, Predicate, TopN, VectorSearch}
+import org.apache.paimon.predicate.FullTextSearch
import org.apache.paimon.table.{SpecialFields, Table}
import org.apache.paimon.types.RowType
@@ -53,6 +54,7 @@ abstract class PaimonBaseScanBuilder
protected var pushedLimit: Option[Int] = None
protected var pushedTopN: Option[TopN] = None
protected var pushedVectorSearch: Option[VectorSearch] = None
+ protected var pushedFullTextSearch: Option[FullTextSearch] = None
protected var requiredSchema: StructType = SparkTypeUtils.fromPaimonRowType(table.rowType())
diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonScan.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonScan.scala
index 12471f5787..0fa4d73054 100644
--- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonScan.scala
+++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonScan.scala
@@ -20,7 +20,7 @@ package org.apache.paimon.spark
import org.apache.paimon.CoreOptions.BucketFunctionType
import org.apache.paimon.partition.PartitionPredicate
-import org.apache.paimon.predicate.{Predicate, TopN, VectorSearch}
+import org.apache.paimon.predicate.{FullTextSearch, Predicate, TopN, VectorSearch}
import org.apache.paimon.spark.commands.BucketExpression.quote
import org.apache.paimon.table.{BucketMode, FileStoreTable, InnerTable}
import org.apache.paimon.table.source.{DataSplit, Split}
@@ -42,6 +42,7 @@ case class PaimonScan(
override val pushedLimit: Option[Int],
override val pushedTopN: Option[TopN],
override val pushedVectorSearch: Option[VectorSearch],
+ override val pushedFullTextSearch: Option[FullTextSearch] = None,
bucketedScanDisabled: Boolean = false)
extends PaimonBaseScan(table)
with SupportsReportPartitioning
diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonScanBuilder.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonScanBuilder.scala
index 2bf7a8ddd4..7e1989283b 100644
--- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonScanBuilder.scala
+++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonScanBuilder.scala
@@ -128,7 +128,7 @@ class PaimonScanBuilder(val table: InnerTable)
localScan match {
case Some(scan) => scan
case None =>
- val (actualTable, vectorSearch) = table match {
+ val (actualTable, vectorSearch, fullTextSearch) = table match {
case vst: org.apache.paimon.table.VectorSearchTable =>
val tableVectorSearch = Option(vst.vectorSearch())
val vs = (tableVectorSearch, pushedVectorSearch) match {
@@ -136,8 +136,10 @@ class PaimonScanBuilder(val table: InnerTable)
case (None, Some(_)) => pushedVectorSearch
case (None, None) => None
}
- (vst.origin(), vs)
- case _ => (table, pushedVectorSearch)
+ (vst.origin(), vs, None)
+ case ftst: org.apache.paimon.table.FullTextSearchTable =>
+ (ftst.origin(), None, Option(ftst.fullTextSearch()))
+ case _ => (table, pushedVectorSearch, pushedFullTextSearch)
}
PaimonScan(
@@ -147,7 +149,8 @@ class PaimonScanBuilder(val table: InnerTable)
pushedDataFilters,
pushedLimit,
pushedTopN,
- vectorSearch)
+ vectorSearch,
+ fullTextSearch)
}
}
}
diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/plans/logical/PaimonTableValuedFunctions.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/plans/logical/PaimonTableValuedFunctions.scala
index 6bb6004db8..d570f60303 100644
--- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/plans/logical/PaimonTableValuedFunctions.scala
+++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/plans/logical/PaimonTableValuedFunctions.scala
@@ -19,10 +19,10 @@
package org.apache.paimon.spark.catalyst.plans.logical
import org.apache.paimon.CoreOptions
-import org.apache.paimon.predicate.VectorSearch
+import org.apache.paimon.predicate.{FullTextSearch, VectorSearch}
import org.apache.paimon.spark.SparkTable
import org.apache.paimon.spark.catalyst.plans.logical.PaimonTableValuedFunctions._
-import org.apache.paimon.table.{DataTable, InnerTable, VectorSearchTable}
+import org.apache.paimon.table.{DataTable, FullTextSearchTable, InnerTable, VectorSearchTable}
import org.apache.paimon.table.source.snapshot.TimeTravelUtil.InconsistentTagBucketException
import org.apache.spark.sql.PaimonUtils.createDataset
@@ -44,9 +44,15 @@ object PaimonTableValuedFunctions {
val INCREMENTAL_BETWEEN_TIMESTAMP = "paimon_incremental_between_timestamp"
val INCREMENTAL_TO_AUTO_TAG = "paimon_incremental_to_auto_tag"
val VECTOR_SEARCH = "vector_search"
+ val FULL_TEXT_SEARCH = "full_text_search"
val supportedFnNames: Seq[String] =
- Seq(INCREMENTAL_QUERY, INCREMENTAL_BETWEEN_TIMESTAMP, INCREMENTAL_TO_AUTO_TAG, VECTOR_SEARCH)
+ Seq(
+ INCREMENTAL_QUERY,
+ INCREMENTAL_BETWEEN_TIMESTAMP,
+ INCREMENTAL_TO_AUTO_TAG,
+ VECTOR_SEARCH,
+ FULL_TEXT_SEARCH)
private type TableFunctionDescription = (FunctionIdentifier, ExpressionInfo, TableFunctionBuilder)
@@ -60,6 +66,8 @@ object PaimonTableValuedFunctions {
FunctionRegistryBase.build[IncrementalToAutoTag](fnName, since = None)
case VECTOR_SEARCH =>
FunctionRegistryBase.build[VectorSearchQuery](fnName, since = None)
+ case FULL_TEXT_SEARCH =>
+ FunctionRegistryBase.build[FullTextSearchQuery](fnName, since = None)
case _ =>
throw new Exception(s"Function $fnName isn't a supported table valued function.")
}
@@ -90,10 +98,12 @@ object PaimonTableValuedFunctions {
val ident: Identifier = Identifier.of(Array(dbName), tableName)
val sparkTable = sparkCatalog.loadTable(ident)
- // Handle vector_search specially
+ // Handle vector_search and full_text_search specially
tvf match {
case vsq: VectorSearchQuery =>
resolveVectorSearchQuery(sparkTable, sparkCatalog, ident, vsq, args.tail)
+ case ftsq: FullTextSearchQuery =>
+ resolveFullTextSearchQuery(sparkTable, sparkCatalog, ident, ftsq, args.tail)
case _ =>
val options = tvf.parseArgs(args.tail)
usingSparkIncrementQuery(tvf, sparkTable, options) match {
@@ -131,6 +141,28 @@ object PaimonTableValuedFunctions {
}
}
+ private def resolveFullTextSearchQuery(
+ sparkTable: Table,
+ sparkCatalog: TableCatalog,
+ ident: Identifier,
+ ftsq: FullTextSearchQuery,
+ argsWithoutTable: Seq[Expression]): LogicalPlan = {
+ sparkTable match {
+ case st @ SparkTable(innerTable: InnerTable) =>
+ val fullTextSearch = ftsq.createFullTextSearch(innerTable, argsWithoutTable)
+ val fullTextSearchTable = FullTextSearchTable.create(innerTable, fullTextSearch)
+ DataSourceV2Relation.create(
+ st.copy(table = fullTextSearchTable),
+ Some(sparkCatalog),
+ Some(ident),
+ CaseInsensitiveStringMap.empty())
+ case _ =>
+ throw new RuntimeException(
+ "full_text_search only supports Paimon SparkTable backed by InnerTable, " +
+ s"but got table implementation: ${sparkTable.getClass.getName}")
+ }
+ }
+
private def usingSparkIncrementQuery(
tvf: PaimonTableValueFunction,
sparkTable: Table,
@@ -306,3 +338,52 @@ case class VectorSearchQuery(override val args: Seq[Expression])
}
}
}
+
+/**
+ * Plan for the [[FULL_TEXT_SEARCH]] table-valued function.
+ *
+ * Usage: full_text_search(table_name, column_name, query_text, limit)
+ * - table_name: the Paimon table to search
+ * - column_name: the text column name
+ * - query_text: the query text string
+ * - limit: the number of top results to return
+ *
+ * Example: SELECT * FROM full_text_search('T', 'content', 'hello world', 10)
+ */
+case class FullTextSearchQuery(override val args: Seq[Expression])
+ extends PaimonTableValueFunction(FULL_TEXT_SEARCH) {
+
+ override def parseArgs(args: Seq[Expression]): Map[String, String] = {
+ // This method is not used for FullTextSearchQuery as we handle it specially
+ Map.empty
+ }
+
+ def createFullTextSearch(
+ innerTable: InnerTable,
+ argsWithoutTable: Seq[Expression]): FullTextSearch = {
+ if (argsWithoutTable.size != 3) {
+ throw new RuntimeException(
+ s"$FULL_TEXT_SEARCH needs three parameters after table_name: column_name, query_text, limit. " +
+ s"Got ${argsWithoutTable.size} parameters after table_name."
+ )
+ }
+ val columnName = argsWithoutTable.head.eval().toString
+ if (!innerTable.rowType().containsField(columnName)) {
+ throw new RuntimeException(
+ s"Column $columnName does not exist in table ${innerTable.name()}"
+ )
+ }
+ val queryText = argsWithoutTable(1).eval().toString
+ val limit = argsWithoutTable(2).eval() match {
+ case i: Int => i
+ case l: Long => l.toInt
+ case other => throw new RuntimeException(s"Invalid limit type: ${other.getClass.getName}")
+ }
+ if (limit <= 0) {
+ throw new IllegalArgumentException(
+ s"Limit must be a positive integer, but got: $limit"
+ )
+ }
+ new FullTextSearch(queryText, limit, columnName)
+ }
+}
diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/read/BaseScan.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/read/BaseScan.scala
index 5260ef8e09..42f724d3e6 100644
--- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/read/BaseScan.scala
+++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/read/BaseScan.scala
@@ -20,7 +20,7 @@ package org.apache.paimon.spark.read
import org.apache.paimon.CoreOptions
import org.apache.paimon.partition.PartitionPredicate
-import org.apache.paimon.predicate.{Predicate, TopN, VectorSearch}
+import org.apache.paimon.predicate.{FullTextSearch, Predicate, TopN, VectorSearch}
import org.apache.paimon.spark.{PaimonBatch, PaimonInputPartition, PaimonNumSplitMetric, PaimonPartitionSizeMetric, PaimonReadBatchTimeMetric, PaimonResultedTableFilesMetric, PaimonResultedTableFilesTaskMetric, SparkTypeUtils}
import org.apache.paimon.spark.schema.PaimonMetadataColumn
import org.apache.paimon.spark.schema.PaimonMetadataColumn._
@@ -51,6 +51,7 @@ trait BaseScan extends Scan with SupportsReportStatistics with Logging {
def pushedLimit: Option[Int] = None
def pushedTopN: Option[TopN] = None
def pushedVectorSearch: Option[VectorSearch] = None
+ def pushedFullTextSearch: Option[FullTextSearch] = None
// Runtime push down
val pushedRuntimePartitionFilters: ListBuffer[PartitionPredicate] = ListBuffer.empty
@@ -187,6 +188,7 @@ trait BaseScan extends Scan with SupportsReportStatistics with Logging {
pushedDataFiltersStr +
pushedTopN.map(topN => s", TopN: [$topN]").getOrElse("") +
pushedLimit.map(limit => s", Limit: [$limit]").getOrElse("") +
- pushedVectorSearch.map(vs => s", VectorSearch: [$vs]").getOrElse("")
+ pushedVectorSearch.map(vs => s", VectorSearch: [$vs]").getOrElse("") +
+ pushedFullTextSearch.map(fts => s", FullTextSearch: [$fts]").getOrElse("")
}
}
diff --git a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/FullTextSearchTest.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/FullTextSearchTest.scala
new file mode 100644
index 0000000000..8f6d06fd75
--- /dev/null
+++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/FullTextSearchTest.scala
@@ -0,0 +1,223 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.spark.sql
+
+import org.apache.paimon.spark.PaimonSparkTestBase
+
+import scala.collection.JavaConverters._
+
+/** Tests for full-text search read/write operations using test-only brute-force full-text index. */
+class FullTextSearchTest extends PaimonSparkTestBase {
+
+ private val indexType = "test-fulltext"
+
+ // ========== Index Creation Tests ==========
+
+ test("create full-text index - basic") {
+ withTable("T") {
+ spark.sql("""
+ |CREATE TABLE T (id INT, content STRING)
+ |TBLPROPERTIES (
+ | 'bucket' = '-1',
+ | 'global-index.row-count-per-shard' = '10000',
+ | 'row-tracking.enabled' = 'true',
+ | 'data-evolution.enabled' = 'true')
+ |""".stripMargin)
+
+ val values = (0 until 100)
+ .map(i => s"($i, 'document number $i about paimon lake format')")
+ .mkString(",")
+ spark.sql(s"INSERT INTO T VALUES $values")
+
+ val output = spark
+ .sql(
+ s"CALL sys.create_global_index(table => 'test.T', index_column => 'content', index_type => '$indexType')")
+ .collect()
+ .head
+ assert(output.getBoolean(0))
+
+ val table = loadTable("T")
+ val indexEntries = table
+ .store()
+ .newIndexFileHandler()
+ .scanEntries()
+ .asScala
+ .filter(_.indexFile().indexType() == indexType)
+
+ assert(indexEntries.nonEmpty)
+ val totalRowCount = indexEntries.map(_.indexFile().rowCount()).sum
+ assert(totalRowCount == 100L)
+ }
+ }
+
+ // ========== Index Read/Search Tests ==========
+
+ test("full-text search - basic search") {
+ withTable("T") {
+ spark.sql("""
+ |CREATE TABLE T (id INT, content STRING)
+ |TBLPROPERTIES (
+ | 'bucket' = '-1',
+ | 'global-index.row-count-per-shard' = '10000',
+ | 'row-tracking.enabled' = 'true',
+ | 'data-evolution.enabled' = 'true')
+ |""".stripMargin)
+
+ val values = (0 until 100)
+ .map(i => s"($i, 'document number $i about paimon lake format')")
+ .mkString(",")
+ spark.sql(s"INSERT INTO T VALUES $values")
+
+ spark
+ .sql(
+ s"CALL sys.create_global_index(table => 'test.T', index_column => 'content', index_type => '$indexType')")
+ .collect()
+
+ val result = spark
+ .sql("""
+ |SELECT * FROM full_text_search('T', 'content', 'paimon', 5)
+ |""".stripMargin)
+ .collect()
+ assert(result.length == 5)
+ }
+ }
+
+ test("full-text search - top-k with different k values") {
+ withTable("T") {
+ spark.sql("""
+ |CREATE TABLE T (id INT, content STRING)
+ |TBLPROPERTIES (
+ | 'bucket' = '-1',
+ | 'global-index.row-count-per-shard' = '10000',
+ | 'row-tracking.enabled' = 'true',
+ | 'data-evolution.enabled' = 'true')
+ |""".stripMargin)
+
+ val values = (0 until 200)
+ .map(i => s"($i, 'document number $i about paimon lake format')")
+ .mkString(",")
+ spark.sql(s"INSERT INTO T VALUES $values")
+
+ spark
+ .sql(
+ s"CALL sys.create_global_index(table => 'test.T', index_column => 'content', index_type => '$indexType')")
+ .collect()
+
+ // Test with k=1
+ var result = spark
+ .sql("""
+ |SELECT * FROM full_text_search('T', 'content', 'paimon', 1)
+ |""".stripMargin)
+ .collect()
+ assert(result.length == 1)
+
+ // Test with k=10
+ result = spark
+ .sql("""
+ |SELECT * FROM full_text_search('T', 'content', 'paimon', 10)
+ |""".stripMargin)
+ .collect()
+ assert(result.length == 10)
+ }
+ }
+
+ test("full-text search - multi-term query") {
+ withTable("T") {
+ spark.sql("""
+ |CREATE TABLE T (id INT, content STRING)
+ |TBLPROPERTIES (
+ | 'bucket' = '-1',
+ | 'global-index.row-count-per-shard' = '10000',
+ | 'row-tracking.enabled' = 'true',
+ | 'data-evolution.enabled' = 'true')
+ |""".stripMargin)
+
+ spark.sql("""
+ |INSERT INTO T VALUES
+ | (0, 'Apache Paimon lake format'),
+ | (1, 'Paimon supports full-text search'),
+ | (2, 'full-text search in Apache Paimon'),
+ | (3, 'vector similarity search'),
+ | (4, 'streaming batch processing')
+ |""".stripMargin)
+
+ spark
+ .sql(
+ s"CALL sys.create_global_index(table => 'test.T', index_column => 'content', index_type => '$indexType')")
+ .collect()
+
+ // Query "Paimon search" - rows 1 and 2 match both terms
+ val result = spark
+ .sql("""
+ |SELECT id FROM full_text_search('T', 'content', 'Paimon search', 2)
+ |ORDER BY id
+ |""".stripMargin)
+ .collect()
+
+ assert(result.length == 2)
+ val ids = result.map(_.getInt(0)).toSet
+ assert(ids.contains(1))
+ assert(ids.contains(2))
+ }
+ }
+
+ // ========== Integration Tests ==========
+
+ test("end-to-end: write, index, search cycle") {
+ withTable("T") {
+ spark.sql("""
+ |CREATE TABLE T (id INT, title STRING, content STRING)
+ |TBLPROPERTIES (
+ | 'bucket' = '-1',
+ | 'global-index.row-count-per-shard' = '10000',
+ | 'row-tracking.enabled' = 'true',
+ | 'data-evolution.enabled' = 'true')
+ |""".stripMargin)
+
+ val values = (0 until 1000)
+ .map(i => s"($i, 'title_$i', 'document number $i about paimon lake format')")
+ .mkString(",")
+ spark.sql(s"INSERT INTO T VALUES $values")
+
+ val indexResult = spark
+ .sql(
+ s"CALL sys.create_global_index(table => 'test.T', index_column => 'content', index_type => '$indexType')")
+ .collect()
+ .head
+ assert(indexResult.getBoolean(0))
+
+ val table = loadTable("T")
+ val indexEntries = table
+ .store()
+ .newIndexFileHandler()
+ .scanEntries()
+ .asScala
+ .filter(_.indexFile().indexType() == indexType)
+ assert(indexEntries.nonEmpty)
+
+ val searchResult = spark
+ .sql("""
+ |SELECT id, title FROM full_text_search('T', 'content', 'paimon', 10)
+ |""".stripMargin)
+ .collect()
+
+ assert(searchResult.length == 10)
+ }
+ }
+}
(paimon) 12/37: [core] Respect rowsPerShard as hard constraint in global index build (#7530)
Posted by ho...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hope pushed a commit to branch release-1.4
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit c1275261a2c37c69b0236a4cc151a0b97b98452d
Author: jerry <li...@alibaba-inc.com>
AuthorDate: Thu Mar 26 11:02:27 2026 +0800
[core] Respect rowsPerShard as hard constraint in global index build (#7530)
---
.../shortcodes/generated/core_configuration.html | 2 +-
.../main/java/org/apache/paimon/CoreOptions.java | 8 +--
.../globalindex/GlobalIndexBuilderUtils.java | 17 -----
.../globalindex/GlobalIndexBuilderUtilsTest.java | 81 ----------------------
.../flink/globalindex/GenericIndexTopoBuilder.java | 7 --
.../globalindex/DefaultGlobalIndexTopoBuilder.java | 9 ---
6 files changed, 5 insertions(+), 119 deletions(-)
diff --git a/docs/layouts/shortcodes/generated/core_configuration.html b/docs/layouts/shortcodes/generated/core_configuration.html
index 220ca4cdc4..6dd621fcac 100644
--- a/docs/layouts/shortcodes/generated/core_configuration.html
+++ b/docs/layouts/shortcodes/generated/core_configuration.html
@@ -666,7 +666,7 @@ under the License.
<td><h5>global-index.build.max-shard</h5></td>
<td style="word-wrap: break-word;">32</td>
<td>Integer</td>
- <td>The max number of shards for building global index. If the number of shards calculated by 'global-index.row-count-per-shard' exceeds this value, 'global-index.row-count-per-shard' will be recalculated as ceil(total-row-count / max-shard) to guarantee the shard count does not exceed max-shard.</td>
+ <td>The preferred max number of shards for building global index. If the number of shards calculated by 'global-index.row-count-per-shard' exceeds this value, max-shard will be automatically increased to accommodate the data volume while keeping 'global-index.row-count-per-shard' unchanged.</td>
</tr>
<tr>
<td><h5>global-index.column-update-action</h5></td>
diff --git a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java
index 4dc495c12b..7c054b8c8a 100644
--- a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java
+++ b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java
@@ -2322,11 +2322,11 @@ public class CoreOptions implements Serializable {
.intType()
.defaultValue(32)
.withDescription(
- "The max number of shards for building global index. "
+ "The preferred max number of shards for building global index. "
+ "If the number of shards calculated by 'global-index.row-count-per-shard' "
- + "exceeds this value, 'global-index.row-count-per-shard' will be "
- + "recalculated as ceil(total-row-count / max-shard) to guarantee "
- + "the shard count does not exceed max-shard.");
+ + "exceeds this value, max-shard will be automatically increased "
+ + "to accommodate the data volume while keeping "
+ + "'global-index.row-count-per-shard' unchanged.");
public static final ConfigOption<Integer> GLOBAL_INDEX_BUILD_MAX_PARALLELISM =
key("global-index.build.max-parallelism")
diff --git a/paimon-core/src/main/java/org/apache/paimon/globalindex/GlobalIndexBuilderUtils.java b/paimon-core/src/main/java/org/apache/paimon/globalindex/GlobalIndexBuilderUtils.java
index 19ffc3bb40..085423efa8 100644
--- a/paimon-core/src/main/java/org/apache/paimon/globalindex/GlobalIndexBuilderUtils.java
+++ b/paimon-core/src/main/java/org/apache/paimon/globalindex/GlobalIndexBuilderUtils.java
@@ -71,23 +71,6 @@ public class GlobalIndexBuilderUtils {
return results;
}
- /**
- * Adjust rowsPerShard if the estimated shard count exceeds maxShard. Returns the adjusted
- * rowsPerShard value.
- */
- public static long adjustRowsPerShard(long rowsPerShard, long totalRowCount, int maxShard) {
- long estimatedShards = ceilDivision(totalRowCount, rowsPerShard);
- if (estimatedShards > maxShard) {
- return ceilDivision(totalRowCount, maxShard);
- }
- return rowsPerShard;
- }
-
- /** Integer ceiling division: returns ceil(a / b) for positive a and b. */
- private static long ceilDivision(long a, long b) {
- return (a + b - 1) / b;
- }
-
public static GlobalIndexWriter createIndexWriter(
FileStoreTable table, String indexType, DataField indexField, Options options)
throws IOException {
diff --git a/paimon-core/src/test/java/org/apache/paimon/globalindex/GlobalIndexBuilderUtilsTest.java b/paimon-core/src/test/java/org/apache/paimon/globalindex/GlobalIndexBuilderUtilsTest.java
deleted file mode 100644
index 612f41e0e9..0000000000
--- a/paimon-core/src/test/java/org/apache/paimon/globalindex/GlobalIndexBuilderUtilsTest.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.paimon.globalindex;
-
-import org.junit.jupiter.api.Test;
-
-import static org.assertj.core.api.Assertions.assertThat;
-
-/** Test for {@link GlobalIndexBuilderUtils#adjustRowsPerShard}. */
-public class GlobalIndexBuilderUtilsTest {
-
- @Test
- void testAdjustRowsPerShardNoAdjustmentNeeded() {
- // 1000 rows, 100 per shard = 10 shards, maxShard = 20 -> no adjustment
- long result = GlobalIndexBuilderUtils.adjustRowsPerShard(100, 1000, 20);
- assertThat(result).isEqualTo(100);
- }
-
- @Test
- void testAdjustRowsPerShardExactMatch() {
- // 1000 rows, 100 per shard = 10 shards, maxShard = 10 -> no adjustment
- long result = GlobalIndexBuilderUtils.adjustRowsPerShard(100, 1000, 10);
- assertThat(result).isEqualTo(100);
- }
-
- @Test
- void testAdjustRowsPerShardExceedsMaxShard() {
- // 1000 rows, 100 per shard = 10 shards, maxShard = 3 -> adjust to ceil(1000/3) = 334
- long result = GlobalIndexBuilderUtils.adjustRowsPerShard(100, 1000, 3);
- assertThat(result).isEqualTo(334);
- // Verify: ceil(1000/334) = 3 shards
- assertThat((1000 + result - 1) / result).isEqualTo(3);
- }
-
- @Test
- void testAdjustRowsPerShardMaxShardOne() {
- // 1000 rows, 100 per shard = 10 shards, maxShard = 1 -> all in one shard
- long result = GlobalIndexBuilderUtils.adjustRowsPerShard(100, 1000, 1);
- assertThat(result).isEqualTo(1000);
- assertThat((1000 + result - 1) / result).isEqualTo(1);
- }
-
- @Test
- void testAdjustRowsPerShardEvenDivision() {
- // 1000 rows, 100 per shard = 10 shards, maxShard = 5 -> adjust to 200
- long result = GlobalIndexBuilderUtils.adjustRowsPerShard(100, 1000, 5);
- assertThat(result).isEqualTo(200);
- assertThat((1000 + result - 1) / result).isEqualTo(5);
- }
-
- @Test
- void testAdjustRowsPerShardLargeRowCount() {
- // 10M rows, 100K per shard = 100 shards, maxShard = 10 -> adjust to 1M
- long result = GlobalIndexBuilderUtils.adjustRowsPerShard(100000, 10000000, 10);
- assertThat(result).isEqualTo(1000000);
- assertThat((10000000 + result - 1) / result).isEqualTo(10);
- }
-
- @Test
- void testAdjustRowsPerShardTotalRowsLessThanRowsPerShard() {
- // 50 rows, 100 per shard = 1 shard, maxShard = 3 -> no adjustment
- long result = GlobalIndexBuilderUtils.adjustRowsPerShard(100, 50, 3);
- assertThat(result).isEqualTo(100);
- }
-}
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/globalindex/GenericIndexTopoBuilder.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/globalindex/GenericIndexTopoBuilder.java
index 44239b3876..865280763c 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/globalindex/GenericIndexTopoBuilder.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/globalindex/GenericIndexTopoBuilder.java
@@ -28,7 +28,6 @@ import org.apache.paimon.flink.sink.NoopCommittableStateManager;
import org.apache.paimon.flink.sink.StoreCommitter;
import org.apache.paimon.flink.utils.BoundedOneInputOperator;
import org.apache.paimon.flink.utils.JavaTypeInfo;
-import org.apache.paimon.globalindex.GlobalIndexBuilderUtils;
import org.apache.paimon.globalindex.GlobalIndexSingletonWriter;
import org.apache.paimon.globalindex.ResultEntry;
import org.apache.paimon.index.IndexFileMeta;
@@ -159,12 +158,6 @@ public class GenericIndexTopoBuilder {
rowsPerShard > 0,
"Option 'global-index.row-count-per-shard' must be greater than 0.");
- int maxShard = mergedOptions.get(CoreOptions.GLOBAL_INDEX_BUILD_MAX_SHARD);
- checkArgument(
- maxShard > 0, "Option 'global-index.build.max-shard' must be greater than 0.");
- rowsPerShard =
- GlobalIndexBuilderUtils.adjustRowsPerShard(rowsPerShard, totalRowCount, maxShard);
-
// Compute shard tasks at file level from the provided entries
List<ShardTask> shardTasks = computeShardTasks(table, entries, rowsPerShard);
if (shardTasks.isEmpty()) {
diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/globalindex/DefaultGlobalIndexTopoBuilder.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/globalindex/DefaultGlobalIndexTopoBuilder.java
index 04593c55a3..afd954c39a 100644
--- a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/globalindex/DefaultGlobalIndexTopoBuilder.java
+++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/globalindex/DefaultGlobalIndexTopoBuilder.java
@@ -21,7 +21,6 @@ package org.apache.paimon.spark.globalindex;
import org.apache.paimon.data.BinaryRow;
import org.apache.paimon.data.InternalRow;
import org.apache.paimon.fs.Path;
-import org.apache.paimon.globalindex.GlobalIndexBuilderUtils;
import org.apache.paimon.globalindex.IndexedSplit;
import org.apache.paimon.io.DataFileMeta;
import org.apache.paimon.manifest.ManifestEntry;
@@ -56,7 +55,6 @@ import java.util.Map;
import java.util.function.BiFunction;
import java.util.stream.Collectors;
-import static org.apache.paimon.CoreOptions.GLOBAL_INDEX_BUILD_MAX_SHARD;
import static org.apache.paimon.CoreOptions.GLOBAL_INDEX_ROW_COUNT_PER_SHARD;
import static org.apache.paimon.utils.Preconditions.checkArgument;
@@ -88,15 +86,8 @@ public class DefaultGlobalIndexTopoBuilder implements GlobalIndexTopologyBuilder
rowsPerShard > 0,
"Option 'global-index.row-count-per-shard' must be greater than 0.");
- int maxShard = tableOptions.get(GLOBAL_INDEX_BUILD_MAX_SHARD);
- checkArgument(
- maxShard > 0, "Option 'global-index.build.max-shard' must be greater than 0.");
List<ManifestEntry> entries =
table.store().newScan().withPartitionFilter(partitionPredicate).plan().files();
- long totalRowCount = entries.stream().mapToLong(e -> e.file().rowCount()).sum();
- rowsPerShard =
- GlobalIndexBuilderUtils.adjustRowsPerShard(rowsPerShard, totalRowCount, maxShard);
-
// generate splits for each partition && shard
Map<BinaryRow, List<IndexedSplit>> splits = split(table, entries, rowsPerShard);
(paimon) 36/37: [python] Add doc and refine config for using pyjindosdk in pypaimon (#7565)
Posted by ho...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hope pushed a commit to branch release-1.4
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit a1f79399015eadd917a5b437e2aea482f656b8d4
Author: timmyyao <19...@qq.com>
AuthorDate: Tue Mar 31 20:18:27 2026 +0800
[python] Add doc and refine config for using pyjindosdk in pypaimon (#7565)
Using pyjindosdk as the default implementation for pypaimon as long as
pyjindosdk is installed. Fall back is introduced in the doc.
---
docs/content/pypaimon/pyjindosdk-support.md | 56 ++++++++++++++++++++++
paimon-python/pypaimon/common/options/config.py | 4 +-
.../pypaimon/filesystem/pyarrow_file_io.py | 23 +++++++--
paimon-python/pypaimon/tests/file_io_test.py | 2 +
4 files changed, 78 insertions(+), 7 deletions(-)
diff --git a/docs/content/pypaimon/pyjindosdk-support.md b/docs/content/pypaimon/pyjindosdk-support.md
new file mode 100644
index 0000000000..7db012c928
--- /dev/null
+++ b/docs/content/pypaimon/pyjindosdk-support.md
@@ -0,0 +1,56 @@
+---
+title: "PyJindoSDK Support"
+weight: 8
+type: docs
+aliases:
+ - /pypaimon/pyjindosdk-support.html
+---
+
+# PyJindoSDK Support
+
+## Introduction
+
+[JindoSDK](https://github.com/aliyun/alibabacloud-jindodata) is a high-performance storage SDK developed by Alibaba Cloud for accessing OSS (Object Storage Service) and other cloud storage systems. It provides optimized I/O performance and deep integration with the Alibaba Cloud ecosystem.
+
+PyPaimon now supports using [PyJindoSDK](https://github.com/aliyun/alibabacloud-jindodata) (the Python binding of JindoSDK) to access OSS. Compared to the legacy implementation based on PyArrow's S3FileSystem, PyJindoSDK offers better performance and compatibility when working with OSS.
+
+## Usage
+
+### Installation
+
+Install `pyjindosdk` via pip:
+
+```shell
+pip install pyjindosdk
+```
+
+Once installed, PyPaimon will automatically use PyJindoSDK as the default file I/O implementation for accessing OSS. No additional configuration is required.
+
+### Fallback to Legacy Implementation
+
+Since JindoSDK is a native implementation, pre-built Python packages may not be available for all OS or platform versions. If you need to fall back to the legacy PyArrow-based implementation for any reason, there are two ways to do so:
+
+**Option 1: Set catalog option `fs.oss.impl` to `legacy`**
+
+```python
+from pypaimon import CatalogFactory
+
+catalog_options = {
+ 'metastore': 'rest',
+ 'uri': 'http://rest-server:8080',
+ 'warehouse': 'oss://my-bucket/warehouse',
+
+ # Fallback to the legacy PyArrow S3FileSystem implementation
+ 'fs.oss.impl': 'legacy',
+}
+
+catalog = CatalogFactory.create(catalog_options)
+```
+
+**Option 2: Uninstall pyjindosdk**
+
+Simply uninstalling the `pyjindosdk` package will cause PyPaimon to automatically fall back to the legacy implementation:
+
+```shell
+pip uninstall pyjindosdk
+```
diff --git a/paimon-python/pypaimon/common/options/config.py b/paimon-python/pypaimon/common/options/config.py
index 83d46c85bf..249ad810a2 100644
--- a/paimon-python/pypaimon/common/options/config.py
+++ b/paimon-python/pypaimon/common/options/config.py
@@ -18,8 +18,8 @@ from pypaimon.common.options.config_options import ConfigOptions
class OssOptions:
- OSS_IMPL = ConfigOptions.key("fs.oss.impl").string_type().default_value("default").with_description(
- "OSS filesystem implementation: default or jindo")
+ OSS_IMPL = ConfigOptions.key("fs.oss.impl").string_type().default_value("jindo").with_description(
+ "OSS filesystem implementation: legacy or jindo")
OSS_ACCESS_KEY_ID = ConfigOptions.key("fs.oss.accessKeyId").string_type().no_default_value().with_description(
"OSS access key ID")
OSS_ACCESS_KEY_SECRET = ConfigOptions.key(
diff --git a/paimon-python/pypaimon/filesystem/pyarrow_file_io.py b/paimon-python/pypaimon/filesystem/pyarrow_file_io.py
index 6cf2faabb2..87d11e55f2 100644
--- a/paimon-python/pypaimon/filesystem/pyarrow_file_io.py
+++ b/paimon-python/pypaimon/filesystem/pyarrow_file_io.py
@@ -34,7 +34,7 @@ from pypaimon.common.file_io import FileIO
from pypaimon.common.options import Options
from pypaimon.common.options.config import OssOptions, S3Options
from pypaimon.common.uri_reader import UriReaderFactory
-from pypaimon.filesystem.jindo_file_system_handler import JindoFileSystemHandler
+from pypaimon.filesystem.jindo_file_system_handler import JindoFileSystemHandler, JINDO_AVAILABLE
from pypaimon.schema.data_types import (AtomicType, DataField,
PyarrowFieldParser)
from pypaimon.table.row.blob import Blob, BlobData, BlobDescriptor
@@ -57,12 +57,24 @@ class PyArrowFileIO(FileIO):
self.uri_reader_factory = UriReaderFactory(catalog_options)
self._is_oss = scheme in {"oss"}
self._oss_bucket = None
- self._oss_impl = self.properties.get(OssOptions.OSS_IMPL)
+ _oss_impl = self.properties.get(OssOptions.OSS_IMPL)
+ self._use_jindo = False
+
if self._is_oss:
self._oss_bucket = self._extract_oss_bucket(path)
- if self._oss_impl == "jindo":
+ if _oss_impl not in ("jindo", "legacy"):
+ raise ValueError(
+ f"Unsupported fs.oss.impl value: '{_oss_impl}'. "
+ f"Supported values are 'jindo' and 'legacy'.")
+ if _oss_impl == "legacy":
+ self.filesystem = self._initialize_oss_fs(path)
+ elif JINDO_AVAILABLE:
self.filesystem = self._initialize_jindo_fs(path)
else:
+ self.logger.info(
+ "fs.oss.impl is 'jindo' but pyjindosdk is not installed. "
+ "Falling back to legacy PyArrow S3FileSystem implementation. "
+ "Install pyjindosdk for better performance: pip install pyjindosdk")
self.filesystem = self._initialize_oss_fs(path)
elif scheme in {"s3", "s3a", "s3n"}:
self.filesystem = self._initialize_s3_fs()
@@ -126,6 +138,7 @@ class PyArrowFileIO(FileIO):
self.logger.info(f"Initializing JindoFileSystem for OSS access: {path}")
root_path = f"oss://{self._oss_bucket}/"
fs_handler = JindoFileSystemHandler(root_path, self.properties)
+ self._use_jindo = True
return pafs.PyFileSystem(fs_handler)
def _initialize_oss_fs(self, path) -> FileSystem:
@@ -210,7 +223,7 @@ class PyArrowFileIO(FileIO):
def new_output_stream(self, path: str):
path_str = self.to_filesystem_path(path)
- if self._oss_impl == "jindo":
+ if self._use_jindo:
pass
elif self._is_oss and not self._pyarrow_gte_7:
# For PyArrow 6.x + OSS, path_str is already just the key part
@@ -574,7 +587,7 @@ class PyArrowFileIO(FileIO):
path_part = normalized_path.lstrip('/')
return f"{drive_letter}:/{path_part}" if path_part else f"{drive_letter}:"
- if self._oss_impl == "jindo":
+ if self._use_jindo:
# For JindoFileSystem, pass key only
path_part = normalized_path.lstrip('/')
return path_part if path_part else '.'
diff --git a/paimon-python/pypaimon/tests/file_io_test.py b/paimon-python/pypaimon/tests/file_io_test.py
index 5cc4d7a821..d39d0c6461 100644
--- a/paimon-python/pypaimon/tests/file_io_test.py
+++ b/paimon-python/pypaimon/tests/file_io_test.py
@@ -70,6 +70,7 @@ class FileIOTest(unittest.TestCase):
OssOptions.OSS_ENDPOINT.key(): 'oss-cn-hangzhou.aliyuncs.com',
OssOptions.OSS_ACCESS_KEY_ID.key(): 'test-key',
OssOptions.OSS_ACCESS_KEY_SECRET.key(): 'test-secret',
+ OssOptions.OSS_IMPL.key(): 'legacy',
}))
got = oss_io.to_filesystem_path("oss://test-bucket/path/to/file.txt")
self.assertEqual(got, "path/to/file.txt" if lt7 else "test-bucket/path/to/file.txt")
@@ -291,6 +292,7 @@ class FileIOTest(unittest.TestCase):
OssOptions.OSS_ENDPOINT.key(): 'oss-cn-hangzhou.aliyuncs.com',
OssOptions.OSS_ACCESS_KEY_ID.key(): 'test-key',
OssOptions.OSS_ACCESS_KEY_SECRET.key(): 'test-secret',
+ OssOptions.OSS_IMPL.key(): 'legacy',
}))
mock_fs = MagicMock()
mock_fs.get_file_info.return_value = [
(paimon) 01/37: [python] Deduplicate input to keep last one in TableUpsertByKey (#7526)
Posted by ho...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hope pushed a commit to branch release-1.4
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit 7cdbd23aff10b4902a9edf18d8ffc386921c0557
Author: Jingsong Lee <ji...@gmail.com>
AuthorDate: Wed Mar 25 16:23:37 2026 +0800
[python] Deduplicate input to keep last one in TableUpsertByKey (#7526)
Change the handling of duplicate keys in TableUpsertByKey from throwing
exceptions to automatic deduplication (keeping the last one).
---
docs/content/pypaimon/data-evolution.md | 2 +-
.../pypaimon/tests/table_upsert_by_key_test.py | 92 +++++++++++++++++++---
.../pypaimon/write/table_upsert_by_key.py | 21 +++--
3 files changed, 98 insertions(+), 17 deletions(-)
diff --git a/docs/content/pypaimon/data-evolution.md b/docs/content/pypaimon/data-evolution.md
index b5daafae57..068ad9b99a 100644
--- a/docs/content/pypaimon/data-evolution.md
+++ b/docs/content/pypaimon/data-evolution.md
@@ -231,7 +231,7 @@ table_commit.close()
**Notes**
- Execution is driven **partition-by-partition**: only one partition's key set is loaded into memory at a time.
-- Duplicate keys in the input data will raise an error.
+- Duplicate keys in the input data are automatically deduplicated — the **last occurrence** is kept.
- The upsert is atomic per commit — all matched updates and new appends are included in the same commit.
## Update Columns By Shards
diff --git a/paimon-python/pypaimon/tests/table_upsert_by_key_test.py b/paimon-python/pypaimon/tests/table_upsert_by_key_test.py
index 37511afd54..0fd93a3daa 100644
--- a/paimon-python/pypaimon/tests/table_upsert_by_key_test.py
+++ b/paimon-python/pypaimon/tests/table_upsert_by_key_test.py
@@ -493,21 +493,93 @@ class TableUpsertByKeyTest(unittest.TestCase):
tu.upsert_by_arrow_with_key(data, upsert_keys=['id'])
self.assertIn('empty', str(ctx.exception))
- def test_duplicate_keys_in_input_raises(self):
- """Duplicate composite keys in input data should raise ValueError."""
+ def test_duplicate_keys_in_input_keeps_last(self):
+ """Duplicate keys in input data should keep the last occurrence."""
table = self._create_table()
- data = pa.Table.from_pydict({
- 'id': [1, 1],
- 'name': ['A', 'B'],
+ self._write(table, pa.Table.from_pydict({
+ 'id': [1, 2],
+ 'name': ['Alice', 'Bob'],
'age': [25, 30],
'city': ['NYC', 'LA'],
+ }, schema=self.pa_schema))
+
+ # id=1 appears twice; the second row (name='A_last') should win
+ data = pa.Table.from_pydict({
+ 'id': [1, 1],
+ 'name': ['A_first', 'A_last'],
+ 'age': [90, 91],
+ 'city': ['X', 'Y'],
}, schema=self.pa_schema)
+ self._upsert(table, data, upsert_keys=['id'])
- with self.assertRaises(ValueError) as ctx:
- wb = table.new_batch_write_builder()
- tu = wb.new_update()
- tu.upsert_by_arrow_with_key(data, upsert_keys=['id'])
- self.assertIn('duplicate', str(ctx.exception).lower())
+ result = self._read_all(table)
+ rows = {r: (n, a, c) for r, n, a, c in zip(
+ result['id'].to_pylist(),
+ result['name'].to_pylist(),
+ result['age'].to_pylist(),
+ result['city'].to_pylist(),
+ )}
+ # id=1 updated with last duplicate row
+ self.assertEqual(rows[1], ('A_last', 91, 'Y'))
+ # id=2 unchanged
+ self.assertEqual(rows[2], ('Bob', 30, 'LA'))
+
+ def test_duplicate_keys_all_new_keeps_last(self):
+ """Duplicate keys in input on empty table keeps the last occurrence."""
+ table = self._create_table()
+
+ # id=1 appears three times; last row should win
+ data = pa.Table.from_pydict({
+ 'id': [1, 1, 1, 2],
+ 'name': ['A1', 'A2', 'A3', 'B'],
+ 'age': [10, 20, 30, 40],
+ 'city': ['X1', 'X2', 'X3', 'Y'],
+ }, schema=self.pa_schema)
+ self._upsert(table, data, upsert_keys=['id'])
+
+ result = self._read_all(table)
+ self.assertEqual(result.num_rows, 2)
+ rows = {r: (n, a, c) for r, n, a, c in zip(
+ result['id'].to_pylist(),
+ result['name'].to_pylist(),
+ result['age'].to_pylist(),
+ result['city'].to_pylist(),
+ )}
+ self.assertEqual(rows[1], ('A3', 30, 'X3'))
+ self.assertEqual(rows[2], ('B', 40, 'Y'))
+
+ def test_duplicate_keys_partitioned_keeps_last(self):
+ """Duplicate keys in a partitioned table keep the last per partition."""
+ table = self._create_table(
+ pa_schema=self.partitioned_pa_schema,
+ partition_keys=['region'],
+ )
+ self._write(table, pa.Table.from_pydict({
+ 'id': [1, 2],
+ 'name': ['Alice', 'Bob'],
+ 'age': [25, 30],
+ 'region': ['US', 'EU'],
+ }, schema=self.partitioned_pa_schema))
+
+ # id=1 duplicated in US partition; id=2 duplicated in EU partition
+ data = pa.Table.from_pydict({
+ 'id': [1, 1, 2, 2],
+ 'name': ['A_first', 'A_last', 'B_first', 'B_last'],
+ 'age': [50, 51, 60, 61],
+ 'region': ['US', 'US', 'EU', 'EU'],
+ }, schema=self.partitioned_pa_schema)
+ self._upsert(table, data, upsert_keys=['id'])
+
+ result = self._read_all(table)
+ self.assertEqual(result.num_rows, 2)
+ rows = {(r, reg): (n, a) for r, n, a, reg in zip(
+ result['id'].to_pylist(),
+ result['name'].to_pylist(),
+ result['age'].to_pylist(),
+ result['region'].to_pylist(),
+ )}
+ self.assertEqual(rows[(1, 'US')], ('A_last', 51))
+ self.assertEqual(rows[(2, 'EU')], ('B_last', 61))
def test_partitioned_table_missing_partition_col_in_data_raises(self):
"""Input data missing partition column should raise ValueError."""
diff --git a/paimon-python/pypaimon/write/table_upsert_by_key.py b/paimon-python/pypaimon/write/table_upsert_by_key.py
index 3c895e4904..d8bad25fab 100644
--- a/paimon-python/pypaimon/write/table_upsert_by_key.py
+++ b/paimon-python/pypaimon/write/table_upsert_by_key.py
@@ -164,16 +164,25 @@ class TableUpsertByKey:
for i in range(partition_data.num_rows)
]
- # 2. Per-partition duplicate check
- input_key_set = set(input_key_tuples)
- if len(input_key_tuples) != len(input_key_set):
- raise ValueError(
- f"Input data contains duplicate values in upsert_keys columns "
- f"{match_keys} within partition {partition_spec}."
+ # 2. Deduplicate: keep last occurrence of each key
+ key_to_last_idx: Dict[_KeyTuple, int] = {}
+ for i, key_tuple in enumerate(input_key_tuples):
+ key_to_last_idx[key_tuple] = i # last write wins
+
+ if len(input_key_tuples) != len(key_to_last_idx):
+ original_count = len(input_key_tuples)
+ dedup_indices = sorted(key_to_last_idx.values())
+ partition_data = partition_data.take(dedup_indices)
+ input_key_tuples = [input_key_tuples[i] for i in dedup_indices]
+ logger.warning(
+ "Deduplicated input from %d to %d rows in partition %s "
+ "(kept last occurrence).",
+ original_count, len(input_key_tuples), partition_spec,
)
# 3. Scan partition in batches, build key → _ROW_ID only for
# keys present in the input (avoids full-partition materialisation).
+ input_key_set = set(key_to_last_idx.keys())
key_to_row_id = self._build_key_to_row_id_map(
match_keys, partition_spec, input_key_set
)
(paimon) 21/37: [spark] Fix nested struct field mapping for V2 write merge schema (#7542)
Posted by ho...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hope pushed a commit to branch release-1.4
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit a657f46d517b51d8ae63606206bad8e445cbe8e0
Author: Zouxxyy <zo...@alibaba-inc.com>
AuthorDate: Fri Mar 27 08:48:51 2026 +0800
[spark] Fix nested struct field mapping for V2 write merge schema (#7542)
---
.../paimon/spark/SparkInternalRowWrapper.java | 18 +++-
.../paimon/spark/sql/V2WriteMergeSchemaTest.scala | 108 +++++++++++++++++++++
2 files changed, 123 insertions(+), 3 deletions(-)
diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkInternalRowWrapper.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkInternalRowWrapper.java
index d2e70b9ed3..ffd077741c 100644
--- a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkInternalRowWrapper.java
+++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkInternalRowWrapper.java
@@ -63,6 +63,7 @@ public class SparkInternalRowWrapper implements InternalRow, Serializable {
private final int length;
@Nullable private final UriReaderFactory uriReaderFactory;
@Nullable private final int[] fieldIndexMap;
+ @Nullable private final StructType dataSchema;
private transient org.apache.spark.sql.catalyst.InternalRow internalRow;
@@ -77,6 +78,7 @@ public class SparkInternalRowWrapper implements InternalRow, Serializable {
CatalogContext catalogContext) {
this.tableSchema = tableSchema;
this.length = length;
+ this.dataSchema = dataSchema;
this.fieldIndexMap =
dataSchema != null ? buildFieldIndexMap(tableSchema, dataSchema) : null;
this.uriReaderFactory = new UriReaderFactory(catalogContext);
@@ -240,7 +242,11 @@ public class SparkInternalRowWrapper implements InternalRow, Serializable {
@Override
public Blob getBlob(int pos) {
- byte[] bytes = internalRow.getBinary(pos);
+ int actualPos = getActualFieldPosition(pos);
+ if (actualPos == -1 || internalRow.isNullAt(actualPos)) {
+ return null;
+ }
+ byte[] bytes = internalRow.getBinary(actualPos);
boolean blobDes = BlobDescriptor.isBlobDescriptor(bytes);
if (blobDes) {
BlobDescriptor blobDescriptor = BlobDescriptor.deserialize(bytes);
@@ -284,8 +290,14 @@ public class SparkInternalRowWrapper implements InternalRow, Serializable {
if (actualPos == -1 || internalRow.isNullAt(actualPos)) {
return null;
}
- return new SparkInternalRowWrapper(
- (StructType) tableSchema.fields()[actualPos].dataType(), numFields)
+ StructType nestedTableSchema = (StructType) tableSchema.fields()[pos].dataType();
+ if (dataSchema != null) {
+ StructType nestedDataSchema = (StructType) dataSchema.fields()[actualPos].dataType();
+ int dataNumFields = nestedDataSchema.size();
+ return new SparkInternalRowWrapper(nestedTableSchema, numFields, nestedDataSchema, null)
+ .replace(internalRow.getStruct(actualPos, dataNumFields));
+ }
+ return new SparkInternalRowWrapper(nestedTableSchema, numFields)
.replace(internalRow.getStruct(actualPos, numFields));
}
diff --git a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/V2WriteMergeSchemaTest.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/V2WriteMergeSchemaTest.scala
index 0b6e589d9f..c73048f7d0 100644
--- a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/V2WriteMergeSchemaTest.scala
+++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/V2WriteMergeSchemaTest.scala
@@ -316,4 +316,112 @@ class V2WriteMergeSchemaTest extends PaimonSparkTestBase {
}
}
+ test("Write merge schema: nested struct with new fields by sql") {
+ withTable("t") {
+ sql("CREATE TABLE t (id INT, info STRUCT<key1 STRUCT<key2 STRING, key3 STRING>>)")
+ sql("INSERT INTO t VALUES (1, struct(struct('v2a', 'v3a')))")
+ sql("INSERT INTO t VALUES (2, struct(struct('v2b', 'v3b')))")
+ checkAnswer(
+ sql("SELECT * FROM t ORDER BY id"),
+ Seq(Row(1, Row(Row("v2a", "v3a"))), Row(2, Row(Row("v2b", "v3b"))))
+ )
+
+ sql(
+ "INSERT INTO t BY NAME " +
+ "SELECT 3 AS id, " +
+ "named_struct('key1', named_struct('key2', 'v2c', 'key4', 'v4c', 'key3', 'v3c')) AS info")
+ checkAnswer(
+ sql("SELECT * FROM t ORDER BY id"),
+ Seq(
+ Row(1, Row(Row("v2a", "v3a", null))),
+ Row(2, Row(Row("v2b", "v3b", null))),
+ Row(3, Row(Row("v2c", "v3c", "v4c"))))
+ )
+ }
+ }
+
+ test("Write merge schema: deeply nested struct with new fields") {
+ withTable("t") {
+ sql("CREATE TABLE t (id INT, data STRUCT<a STRUCT<b STRUCT<c1 STRING, c2 STRING>>>)")
+ sql("INSERT INTO t VALUES (1, struct(struct(struct('c1v', 'c2v'))))")
+
+ sql(
+ "INSERT INTO t BY NAME " +
+ "SELECT 2 AS id, " +
+ "named_struct('a', named_struct('b', named_struct('c1', 'c1v2', 'c3', 'c3v2', 'c2', 'c2v2'))) AS data")
+ checkAnswer(
+ sql("SELECT * FROM t ORDER BY id"),
+ Seq(
+ Row(1, Row(Row(Row("c1v", "c2v", null)))),
+ Row(2, Row(Row(Row("c1v2", "c2v2", "c3v2")))))
+ )
+ }
+ }
+
+ test("Write merge schema: nested struct new fields and top-level new column together") {
+ withTable("t") {
+ sql("CREATE TABLE t (id INT, info STRUCT<f1 STRING, f2 STRING>)")
+ sql("INSERT INTO t VALUES (1, struct('a', 'b'))")
+
+ sql(
+ "INSERT INTO t BY NAME " +
+ "SELECT 2 AS id, " +
+ "named_struct('f1', 'c', 'f3', 'd', 'f2', 'e') AS info, " +
+ "'top' AS extra")
+ checkAnswer(
+ sql("SELECT * FROM t ORDER BY id"),
+ Seq(Row(1, Row("a", "b", null), null), Row(2, Row("c", "e", "d"), "top"))
+ )
+ }
+ }
+
+ test("Write merge schema: nested struct with missing fields") {
+ withTable("t") {
+ sql("CREATE TABLE t (id INT, info STRUCT<f1 STRING, f2 STRING, f3 STRING>)")
+ sql("INSERT INTO t VALUES (1, struct('a', 'b', 'c'))")
+
+ sql(
+ "INSERT INTO t BY NAME " +
+ "SELECT 2 AS id, " +
+ "named_struct('f2', 'y', 'f3', 'z') AS info")
+ checkAnswer(
+ sql("SELECT * FROM t ORDER BY id"),
+ Seq(Row(1, Row("a", "b", "c")), Row(2, Row(null, "y", "z")))
+ )
+
+ sql(
+ "INSERT INTO t BY NAME " +
+ "SELECT 3 AS id, " +
+ "named_struct('f1', 'x', 'f4', 'w', 'f3', 'z') AS info")
+
+ sql(
+ "INSERT INTO t BY NAME " +
+ "SELECT 4 AS id, " +
+ "named_struct('f2', 'p', 'f3', 'q', 'f4', 'r') AS info")
+ checkAnswer(
+ sql("SELECT * FROM t ORDER BY id"),
+ Seq(
+ Row(1, Row("a", "b", "c", null)),
+ Row(2, Row(null, "y", "z", null)),
+ Row(3, Row("x", null, "z", "w")),
+ Row(4, Row(null, "p", "q", "r")))
+ )
+ }
+ }
+
+ test("Write merge schema: nested struct with type evolution") {
+ withTable("t") {
+ sql("CREATE TABLE t (id INT, info STRUCT<f1 INT, f2 STRING>)")
+ sql("INSERT INTO t VALUES (1, struct(10, 'a'))")
+
+ sql(
+ "INSERT INTO t BY NAME " +
+ "SELECT 2 AS id, " +
+ "named_struct('f1', cast(20 as bigint), 'f3', 'new', 'f2', 'b') AS info")
+ checkAnswer(
+ sql("SELECT * FROM t ORDER BY id"),
+ Seq(Row(1, Row(10L, "a", null)), Row(2, Row(20L, "b", "new")))
+ )
+ }
+ }
}
(paimon) 32/37: [python] Introduce file pruning for dv pk table (#7557)
Posted by ho...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hope pushed a commit to branch release-1.4
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit 2f62a4f9981827a4d89dcd86a6518829aa6ed31b
Author: Jingsong Lee <ji...@gmail.com>
AuthorDate: Mon Mar 30 21:42:26 2026 +0800
[python] Introduce file pruning for dv pk table (#7557)
### Purpose
In the primary key table scan, when DV is enabled, the logic of
filtering files by value stats has been added.
### Tests
in java_py_read_write_test.py,
test_pk_dv_read_multi_batch_with_value_filter.
---
.../pypaimon/read/scanner/file_scanner.py | 30 ++++++++++++----
.../pypaimon/tests/e2e/java_py_read_write_test.py | 42 ++++++++++++++++++++++
2 files changed, 66 insertions(+), 6 deletions(-)
diff --git a/paimon-python/pypaimon/read/scanner/file_scanner.py b/paimon-python/pypaimon/read/scanner/file_scanner.py
index a5e8576802..aea709669e 100755
--- a/paimon-python/pypaimon/read/scanner/file_scanner.py
+++ b/paimon-python/pypaimon/read/scanner/file_scanner.py
@@ -381,12 +381,30 @@ class FileScanner:
if self.table.is_primary_key_table:
if self.deletion_vectors_enabled and entry.file.level == 0: # do not read level 0 file
return False
- if not self.primary_key_predicate:
- return True
- return self.primary_key_predicate.test_by_simple_stats(
- entry.file.key_stats,
- entry.file.row_count
- )
+ if self.primary_key_predicate:
+ if not self.primary_key_predicate.test_by_simple_stats(
+ entry.file.key_stats,
+ entry.file.row_count
+ ):
+ return False
+ # In DV mode, files within a bucket don't overlap (level 0 excluded above),
+ # so we can safely filter by value stats per file.
+ if self.deletion_vectors_enabled and self.predicate_for_stats:
+ if entry.file.value_stats_cols is None and entry.file.write_cols is not None:
+ stats_fields = entry.file.write_cols
+ else:
+ stats_fields = entry.file.value_stats_cols
+ evolved_stats = evolution.evolution(
+ entry.file.value_stats,
+ entry.file.row_count,
+ stats_fields
+ )
+ if not self.predicate_for_stats.test_by_simple_stats(
+ evolved_stats,
+ entry.file.row_count
+ ):
+ return False
+ return True
else:
if not self.predicate:
return True
diff --git a/paimon-python/pypaimon/tests/e2e/java_py_read_write_test.py b/paimon-python/pypaimon/tests/e2e/java_py_read_write_test.py
index 0732925bae..4d888c3cb4 100644
--- a/paimon-python/pypaimon/tests/e2e/java_py_read_write_test.py
+++ b/paimon-python/pypaimon/tests/e2e/java_py_read_write_test.py
@@ -322,6 +322,48 @@ class JavaPyReadWriteTest(unittest.TestCase):
}, schema=pa_schema)
self.assertEqual(expected, actual)
+ def test_pk_dv_read_multi_batch_with_value_filter(self):
+ """Test that DV-enabled PK table filters files by value stats during scan."""
+ pa_schema = pa.schema([
+ pa.field('pt', pa.int32(), nullable=False),
+ pa.field('a', pa.int32(), nullable=False),
+ ('b', pa.int64())
+ ])
+ schema = Schema.from_pyarrow_schema(pa_schema,
+ partition_keys=['pt'],
+ primary_keys=['pt', 'a'],
+ options={'bucket': '1'})
+ self.catalog.create_table('default.test_pk_dv_multi_batch', schema, True)
+ table = self.catalog.get_table('default.test_pk_dv_multi_batch')
+
+ # Unfiltered scan: count total files
+ rb_all = table.new_read_builder()
+ all_splits = rb_all.new_scan().plan().splits()
+ all_files_count = sum(len(s.files) for s in all_splits)
+
+ # Filtered scan: b < 500 should only match a few rows (b in {100,200,300,400})
+ # and prune files whose value stats don't overlap
+ predicate_builder = table.new_read_builder().new_predicate_builder()
+ pred = predicate_builder.less_than('b', 500)
+ rb_filtered = table.new_read_builder().with_filter(pred)
+ filtered_splits = rb_filtered.new_scan().plan().splits()
+ filtered_files_count = sum(len(s.files) for s in filtered_splits)
+ filtered_result = table_sort_by(
+ rb_filtered.new_read().to_arrow(filtered_splits), 'a')
+
+ # Verify correctness: rows with b < 500 are a=10,b=100 / a=20,b=200 / a=30,b=300 / a=40,b=400
+ expected = pa.Table.from_pydict({
+ 'pt': [1, 1, 1, 1],
+ 'a': [10, 20, 30, 40],
+ 'b': [100, 200, 300, 400]
+ }, schema=pa_schema)
+ self.assertEqual(expected, filtered_result)
+
+ # Verify file pruning: filtered scan should read fewer files
+ self.assertLess(
+ filtered_files_count, all_files_count,
+ f"DV value filter should prune files: filtered={filtered_files_count}, all={all_files_count}")
+
def test_pk_dv_read_multi_batch_raw_convertable(self):
pa_schema = pa.schema([
pa.field('pt', pa.int32(), nullable=False),
(paimon) 08/37: [core] Add validation for 'pk-clustering-override' (#7528)
Posted by ho...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hope pushed a commit to branch release-1.4
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit bf95e9ff7baa955322314981b6298071a1c2805a
Author: Jingsong Lee <ji...@gmail.com>
AuthorDate: Wed Mar 25 22:29:10 2026 +0800
[core] Add validation for 'pk-clustering-override' (#7528)
---
.../primary-key-table/pk-clustering-override.md | 13 ++-----
.../ClusteringCompactManagerFactory.java | 28 ++-------------
.../org/apache/paimon/schema/SchemaValidation.java | 40 ++++++++++++++++++++++
3 files changed, 44 insertions(+), 37 deletions(-)
diff --git a/docs/content/primary-key-table/pk-clustering-override.md b/docs/content/primary-key-table/pk-clustering-override.md
index 0c37456a9e..534c3c3785 100644
--- a/docs/content/primary-key-table/pk-clustering-override.md
+++ b/docs/content/primary-key-table/pk-clustering-override.md
@@ -53,7 +53,7 @@ CREATE TABLE my_table (
After this, data files within each bucket will be physically sorted by `city` instead of `id`. Queries like
`SELECT * FROM my_table WHERE city = 'Beijing'` can skip irrelevant data files by checking their min/max statistics
on the clustering column.
-
+s
## How It Works
PK Clustering Override replaces the default LSM compaction with a two-phase clustering compaction:
@@ -82,16 +82,6 @@ temporary files to reduce memory consumption, preventing OOM during multi-way me
| `clustering.columns` | Must be set (one or more non-primary-key columns) |
| `deletion-vectors.enabled` | Must be `true` |
| `merge-engine` | `deduplicate` (default) or `first-row` only |
-| `sequence.fields` | Must **not** be set |
-| `record-level.expire-time` | Must **not** be set |
-
-## Related Options
-
-| Option | Default | Description |
-|--------|---------|-------------|
-| `clustering.columns` | (none) | Comma-separated column names used as the physical sort order for data files. |
-| `sort-spill-threshold` | (auto) | When the number of merge readers exceeds this value, smaller files are spilled to row-based temp files to reduce memory usage. |
-| `sort-spill-buffer-size` | `64 mb` | Buffer size used for external sort during Phase 1 rewrite. |
## When to Use
@@ -106,3 +96,4 @@ It is **not** suitable when:
- Point lookups by primary key are the dominant access pattern (default LSM sort is already optimal).
- You need `partial-update` or `aggregation` merge engine.
- `sequence.fields` or `record-level.expire-time` is required.
+- Changelog producer`lookup` or `full-compaction` is required.
diff --git a/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/clustering/ClusteringCompactManagerFactory.java b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/clustering/ClusteringCompactManagerFactory.java
index b58f5941a0..f0b117df8b 100644
--- a/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/clustering/ClusteringCompactManagerFactory.java
+++ b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/clustering/ClusteringCompactManagerFactory.java
@@ -38,6 +38,7 @@ import javax.annotation.Nullable;
import java.util.List;
import java.util.concurrent.ExecutorService;
+import static org.apache.paimon.schema.SchemaValidation.validatePkClusteringOverride;
import static org.apache.paimon.utils.Preconditions.checkNotNull;
/** Factory to create {@link ClusteringCompactManager}. */
@@ -66,32 +67,7 @@ public class ClusteringCompactManagerFactory implements KvCompactionManagerFacto
this.keyType = keyType;
this.valueType = valueType;
this.cacheManager = cacheManager;
-
- if (options.clusteringColumns().isEmpty()) {
- throw new IllegalArgumentException(
- "Cannot support 'pk-clustering-override' mode without 'clustering.columns'.");
- }
- if (!options.deletionVectorsEnabled()) {
- throw new UnsupportedOperationException(
- "Cannot support deletion-vectors disabled in 'pk-clustering-override' mode.");
- }
- if (options.recordLevelExpireTime() != null) {
- throw new UnsupportedOperationException(
- "Cannot support record level expire time enabled in 'pk-clustering-override' mode.");
- }
- if (options.mergeEngine() != CoreOptions.MergeEngine.DEDUPLICATE
- && options.mergeEngine() != CoreOptions.MergeEngine.FIRST_ROW) {
- throw new UnsupportedOperationException(
- "Cannot support merge engine: "
- + options.mergeEngine()
- + " in 'pk-clustering-override' mode.");
- }
- if (!options.sequenceField().isEmpty()) {
- throw new UnsupportedOperationException(
- "Cannot support sequence field: "
- + options.sequenceField()
- + " in 'pk-clustering-override' mode.");
- }
+ validatePkClusteringOverride(options);
}
@Override
diff --git a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java
index 689f65276b..130f9b29eb 100644
--- a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java
+++ b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java
@@ -282,6 +282,8 @@ public class SchemaValidation {
validateChainTable(schema, options);
validateChangelogReadSequenceNumber(schema, options);
+
+ validatePkClusteringOverride(options);
}
public static void validateFallbackBranch(SchemaManager schemaManager, TableSchema schema) {
@@ -795,4 +797,42 @@ public class SchemaValidation {
CoreOptions.TABLE_READ_SEQUENCE_NUMBER_ENABLED.key());
}
}
+
+ public static void validatePkClusteringOverride(CoreOptions options) {
+ if (options.pkClusteringOverride()) {
+ if (options.clusteringColumns().isEmpty()) {
+ throw new IllegalArgumentException(
+ "Cannot support 'pk-clustering-override' mode without 'clustering.columns'.");
+ }
+ if (!options.deletionVectorsEnabled()) {
+ throw new UnsupportedOperationException(
+ "Cannot support deletion-vectors disabled in 'pk-clustering-override' mode.");
+ }
+ if (options.recordLevelExpireTime() != null) {
+ throw new UnsupportedOperationException(
+ "Cannot support record level expire time enabled in 'pk-clustering-override' mode.");
+ }
+ if (options.mergeEngine() != CoreOptions.MergeEngine.DEDUPLICATE
+ && options.mergeEngine() != CoreOptions.MergeEngine.FIRST_ROW) {
+ throw new UnsupportedOperationException(
+ "Cannot support merge engine: "
+ + options.mergeEngine()
+ + " in 'pk-clustering-override' mode.");
+ }
+ if (!options.sequenceField().isEmpty()) {
+ throw new UnsupportedOperationException(
+ "Cannot support sequence field: "
+ + options.sequenceField()
+ + " in 'pk-clustering-override' mode.");
+ }
+ ChangelogProducer changelogProducer = options.changelogProducer();
+ if (changelogProducer != ChangelogProducer.NONE
+ && changelogProducer != ChangelogProducer.INPUT) {
+ throw new UnsupportedOperationException(
+ "Cannot support changelog producer: "
+ + changelogProducer
+ + " in 'pk-clustering-override' mode.");
+ }
+ }
+ }
}
(paimon) 17/37: [core] Remove stale dv files in ClusteringCompactManager (#7538)
Posted by ho...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hope pushed a commit to branch release-1.4
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit 6cf1b374a9fd9086465b240c0ad03cf4392f2c54
Author: Jingsong Lee <ji...@gmail.com>
AuthorDate: Thu Mar 26 19:49:07 2026 +0800
[core] Remove stale dv files in ClusteringCompactManager (#7538)
---
.../clustering/ClusteringCompactManager.java | 4 ++
.../paimon/separated/ClusteringTableTest.java | 56 ++++++++++++++++++++++
2 files changed, 60 insertions(+)
diff --git a/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/clustering/ClusteringCompactManager.java b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/clustering/ClusteringCompactManager.java
index 12acea3be6..3e288b22ca 100644
--- a/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/clustering/ClusteringCompactManager.java
+++ b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/clustering/ClusteringCompactManager.java
@@ -228,6 +228,10 @@ public class ClusteringCompactManager extends CompactFutureManager {
for (DataFileMeta newFile : mergedFiles) {
keyIndex.rebuildIndex(newFile);
}
+ // Remove stale deletion vectors for merged-away files
+ for (DataFileMeta file : mergeGroup) {
+ dvMaintainer.removeDeletionVectorOf(file.fileName());
+ }
result.before().addAll(mergeGroup);
result.after().addAll(mergedFiles);
}
diff --git a/paimon-core/src/test/java/org/apache/paimon/separated/ClusteringTableTest.java b/paimon-core/src/test/java/org/apache/paimon/separated/ClusteringTableTest.java
index 353dd6d704..e33afb7927 100644
--- a/paimon-core/src/test/java/org/apache/paimon/separated/ClusteringTableTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/separated/ClusteringTableTest.java
@@ -18,6 +18,7 @@
package org.apache.paimon.separated;
+import org.apache.paimon.Snapshot;
import org.apache.paimon.catalog.Catalog;
import org.apache.paimon.catalog.CatalogContext;
import org.apache.paimon.catalog.CatalogFactory;
@@ -26,6 +27,7 @@ import org.apache.paimon.data.GenericRow;
import org.apache.paimon.data.InternalRow;
import org.apache.paimon.disk.IOManager;
import org.apache.paimon.fs.Path;
+import org.apache.paimon.manifest.IndexManifestEntry;
import org.apache.paimon.predicate.Predicate;
import org.apache.paimon.predicate.PredicateBuilder;
import org.apache.paimon.schema.Schema;
@@ -34,7 +36,9 @@ import org.apache.paimon.table.sink.BatchTableCommit;
import org.apache.paimon.table.sink.BatchTableWrite;
import org.apache.paimon.table.sink.BatchWriteBuilder;
import org.apache.paimon.table.source.DataSplit;
+import org.apache.paimon.table.source.RawFile;
import org.apache.paimon.table.source.ReadBuilder;
+import org.apache.paimon.table.source.Split;
import org.apache.paimon.types.DataTypes;
import org.apache.paimon.utils.CloseableIterator;
@@ -44,7 +48,10 @@ import org.junit.jupiter.api.io.TempDir;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.HashSet;
import java.util.List;
+import java.util.Optional;
+import java.util.Set;
import static org.apache.paimon.CoreOptions.BUCKET;
import static org.apache.paimon.CoreOptions.CLUSTERING_COLUMNS;
@@ -386,6 +393,55 @@ class ClusteringTableTest {
GenericRow.of(6, 600));
}
+ /**
+ * Test that deletion vectors for merged-away files are cleaned up during Phase 2 compaction.
+ */
+ @Test
+ public void testDeletionVectorCleanupAfterMerge() throws Exception {
+ // Write many rounds of overlapping keys to trigger both Phase 1 (DV creation)
+ // and Phase 2 (merge of sorted files, DV cleanup)
+ for (int round = 0; round < 10; round++) {
+ writeRows(
+ Arrays.asList(
+ GenericRow.of(1, round * 10),
+ GenericRow.of(2, round * 10 + 1),
+ GenericRow.of(3, round * 10 + 2)));
+ }
+
+ // After many rounds of compaction with merges, only the latest values should remain
+ assertThat(readRows())
+ .containsExactlyInAnyOrder(
+ GenericRow.of(1, 90), GenericRow.of(2, 91), GenericRow.of(3, 92));
+
+ // Assert DV files are aligned with data files:
+ // every file referenced in deletion vectors must exist as an active data file
+ Snapshot snapshot = table.latestSnapshot().orElse(null);
+
+ // Collect all active data file names from manifests
+ List<Split> splits = table.newReadBuilder().newScan().plan().splits();
+ Set<String> activeDataFiles = new HashSet<>();
+ for (Split split : splits) {
+ Optional<List<RawFile>> rawFiles = split.convertToRawFiles();
+ assertThat(rawFiles.isPresent()).isTrue();
+ rawFiles.get().stream()
+ .map(RawFile::path)
+ .map(path -> new Path(path).getName())
+ .forEach(activeDataFiles::add);
+ }
+
+ // Collect all file names referenced in DV index entries
+ Set<String> dvReferencedFiles = new HashSet<>();
+ for (IndexManifestEntry indexEntry :
+ table.indexManifestFileReader().read(snapshot.indexManifest())) {
+ dvReferencedFiles.addAll(indexEntry.indexFile().dvRanges().keySet());
+ }
+
+ // Every DV-referenced file must be an active data file
+ assertThat(activeDataFiles)
+ .as("DV references should be a subset of active data files")
+ .containsAll(dvReferencedFiles);
+ }
+
// ==================== Clustering Column Filter Tests ====================
/** Test that equality filter on clustering column skips irrelevant files in the scan plan. */
(paimon) 05/37: [fix] Fix IndexFileHandler to support deleting all index types during snapshot expiration (#7523)
Posted by ho...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hope pushed a commit to branch release-1.4
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit a53aa81c59ea210b2b3e5933247ba911e6d4350e
Author: jerry <li...@alibaba-inc.com>
AuthorDate: Wed Mar 25 17:08:31 2026 +0800
[fix] Fix IndexFileHandler to support deleting all index types during snapshot expiration (#7523)
Fix IndexFileHandler to support deleting all index types during snapshot
expiration
---
.../org/apache/paimon/index/IndexFileHandler.java | 22 ++---
.../apache/paimon/index/IndexFileHandlerTest.java | 105 +++++++++++++++++++++
2 files changed, 112 insertions(+), 15 deletions(-)
diff --git a/paimon-core/src/main/java/org/apache/paimon/index/IndexFileHandler.java b/paimon-core/src/main/java/org/apache/paimon/index/IndexFileHandler.java
index 666f54f7d4..b211dd593c 100644
--- a/paimon-core/src/main/java/org/apache/paimon/index/IndexFileHandler.java
+++ b/paimon-core/src/main/java/org/apache/paimon/index/IndexFileHandler.java
@@ -33,6 +33,7 @@ import org.apache.paimon.utils.Pair;
import org.apache.paimon.utils.SnapshotManager;
import java.io.IOException;
+import java.io.UncheckedIOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
@@ -41,7 +42,6 @@ import java.util.Map;
import java.util.Optional;
import java.util.Set;
-import static org.apache.paimon.deletionvectors.DeletionVectorsIndexFile.DELETION_VECTORS_INDEX;
import static org.apache.paimon.index.HashIndexFile.HASH_INDEX;
/** Handle index files. */
@@ -205,24 +205,16 @@ public class IndexFileHandler {
return indexManifestFile.readWithIOException(indexManifest);
}
- private IndexFile indexFile(IndexManifestEntry entry) {
- IndexFileMeta file = entry.indexFile();
- switch (file.indexType()) {
- case HASH_INDEX:
- return hashIndex(entry.partition(), entry.bucket());
- case DELETION_VECTORS_INDEX:
- return dvIndex(entry.partition(), entry.bucket());
- default:
- throw new IllegalArgumentException("Unknown index type: " + file.indexType());
+ public boolean existsIndexFile(IndexManifestEntry entry) {
+ try {
+ return fileIO.exists(filePath(entry));
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
}
}
- public boolean existsIndexFile(IndexManifestEntry file) {
- return indexFile(file).exists(file.indexFile());
- }
-
public void deleteIndexFile(IndexManifestEntry entry) {
- indexFile(entry).delete(entry.indexFile());
+ fileIO.deleteQuietly(filePath(entry));
}
public void deleteManifest(String indexManifest) {
diff --git a/paimon-core/src/test/java/org/apache/paimon/index/IndexFileHandlerTest.java b/paimon-core/src/test/java/org/apache/paimon/index/IndexFileHandlerTest.java
new file mode 100644
index 0000000000..13ba3a435d
--- /dev/null
+++ b/paimon-core/src/test/java/org/apache/paimon/index/IndexFileHandlerTest.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.index;
+
+import org.apache.paimon.CoreOptions;
+import org.apache.paimon.data.BinaryRow;
+import org.apache.paimon.fs.FileIO;
+import org.apache.paimon.fs.Path;
+import org.apache.paimon.fs.local.LocalFileIO;
+import org.apache.paimon.manifest.FileKind;
+import org.apache.paimon.manifest.IndexManifestEntry;
+import org.apache.paimon.options.MemorySize;
+import org.apache.paimon.types.RowType;
+import org.apache.paimon.utils.FileStorePathFactory;
+import org.apache.paimon.utils.IndexFilePathFactories;
+
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import java.io.IOException;
+import java.util.UUID;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/**
+ * Test for {@link IndexFileHandler#deleteIndexFile} and {@link IndexFileHandler#existsIndexFile}.
+ */
+public class IndexFileHandlerTest {
+
+ @TempDir java.nio.file.Path tempPath;
+
+ private FileIO fileIO;
+ private IndexFileHandler handler;
+
+ @BeforeEach
+ void setUp() {
+ fileIO = LocalFileIO.create();
+ Path root = new Path(tempPath.toUri());
+
+ FileStorePathFactory pathFactory =
+ new FileStorePathFactory(
+ root,
+ RowType.builder().build(),
+ "default",
+ CoreOptions.FILE_FORMAT.defaultValue().toString(),
+ CoreOptions.DATA_FILE_PREFIX.defaultValue(),
+ CoreOptions.CHANGELOG_FILE_PREFIX.defaultValue(),
+ CoreOptions.PARTITION_GENERATE_LEGACY_NAME.defaultValue(),
+ CoreOptions.FILE_SUFFIX_INCLUDE_COMPRESSION.defaultValue(),
+ CoreOptions.FILE_COMPRESSION.defaultValue(),
+ null,
+ null,
+ CoreOptions.ExternalPathStrategy.NONE,
+ null,
+ false,
+ null);
+
+ handler =
+ new IndexFileHandler(
+ fileIO,
+ null,
+ null,
+ new IndexFilePathFactories(pathFactory),
+ MemorySize.ofMebiBytes(2),
+ false);
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"HASH", "DELETION_VECTORS", "btree", "bitmap", "lumina-vector-ann"})
+ void testExistsAndDeleteIndexFile(String indexType) throws IOException {
+ String fileName = "index-" + UUID.randomUUID();
+ IndexFileMeta meta =
+ new IndexFileMeta(indexType, fileName, 4, 1, (GlobalIndexMeta) null, null);
+ IndexManifestEntry entry =
+ new IndexManifestEntry(FileKind.ADD, BinaryRow.EMPTY_ROW, 0, meta);
+
+ // create the file at the expected path
+ Path filePath = handler.filePath(entry);
+ fileIO.tryToWriteAtomic(filePath, "test");
+
+ assertThat(handler.existsIndexFile(entry)).isTrue();
+
+ handler.deleteIndexFile(entry);
+
+ assertThat(handler.existsIndexFile(entry)).isFalse();
+ }
+}
(paimon) 11/37: [core][vector] add log for lumina index writer (#7512)
Posted by ho...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hope pushed a commit to branch release-1.4
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit efa88fd5009dab85f0ec84b5c6b07fbed1b24e1c
Author: jerry <li...@alibaba-inc.com>
AuthorDate: Thu Mar 26 09:47:54 2026 +0800
[core][vector] add log for lumina index writer (#7512)
---
.../index/LuminaVectorGlobalIndexWriter.java | 44 ++++++++++++++++++++--
1 file changed, 41 insertions(+), 3 deletions(-)
diff --git a/paimon-lumina/src/main/java/org/apache/paimon/lumina/index/LuminaVectorGlobalIndexWriter.java b/paimon-lumina/src/main/java/org/apache/paimon/lumina/index/LuminaVectorGlobalIndexWriter.java
index e2e257c28c..9a85fc9c60 100644
--- a/paimon-lumina/src/main/java/org/apache/paimon/lumina/index/LuminaVectorGlobalIndexWriter.java
+++ b/paimon-lumina/src/main/java/org/apache/paimon/lumina/index/LuminaVectorGlobalIndexWriter.java
@@ -196,23 +196,48 @@ public class LuminaVectorGlobalIndexWriter implements GlobalIndexSingletonWriter
private ResultEntry buildIndex() throws IOException {
configureExecutorThreadCount();
+ LOG.info(
+ "Lumina index build started: {} vectors, dim={}, type={}, metric={}",
+ count,
+ dim,
+ options.indexType(),
+ options.metric());
+ long buildStart = System.currentTimeMillis();
+
try (LuminaIndex index =
LuminaIndex.createForBuild(
options.indexType(), dim, options.metric(), options.toLuminaOptions())) {
// Pretrain and insert via streaming file-backed Dataset API
- try (FileBackedDataset ds = new FileBackedDataset(tempVectorFile, dim, count)) {
+ long phaseStart = System.currentTimeMillis();
+ LOG.info("Lumina pretrain phase started");
+ try (FileBackedDataset ds =
+ new FileBackedDataset(tempVectorFile, dim, count, "pretrain")) {
index.pretrainFrom(ds);
}
- try (FileBackedDataset ds = new FileBackedDataset(tempVectorFile, dim, count)) {
+ LOG.info(
+ "Lumina pretrain phase done in {} ms", System.currentTimeMillis() - phaseStart);
+
+ phaseStart = System.currentTimeMillis();
+ LOG.info("Lumina insert phase started");
+ try (FileBackedDataset ds =
+ new FileBackedDataset(tempVectorFile, dim, count, "insert")) {
index.insertFrom(ds);
}
+ LOG.info("Lumina insert phase done in {} ms", System.currentTimeMillis() - phaseStart);
+ phaseStart = System.currentTimeMillis();
+ LOG.info("Lumina dump phase started");
String fileName = fileWriter.newFileName(FILE_NAME_PREFIX);
try (PositionOutputStream out = fileWriter.newOutputStream(fileName)) {
index.dump(new OutputStreamFileOutput(out));
out.flush();
}
+ LOG.info("Lumina dump phase done in {} ms", System.currentTimeMillis() - phaseStart);
+
+ LOG.info(
+ "Lumina index build completed in {} ms",
+ System.currentTimeMillis() - buildStart);
LuminaIndexMeta meta = new LuminaIndexMeta(options.toLuminaOptions());
return new ResultEntry(fileName, count, meta.serialize());
@@ -308,8 +333,10 @@ public class LuminaVectorGlobalIndexWriter implements GlobalIndexSingletonWriter
private final int totalCount;
private int cursor;
private final ByteBuffer readBuf;
+ private final String phase;
+ private int lastLoggedPercent;
- FileBackedDataset(File file, int dim, int totalCount) throws IOException {
+ FileBackedDataset(File file, int dim, int totalCount, String phase) throws IOException {
this.raf = new RandomAccessFile(file, "r");
this.channel = raf.getChannel();
this.dim = dim;
@@ -318,6 +345,8 @@ public class LuminaVectorGlobalIndexWriter implements GlobalIndexSingletonWriter
this.readBuf = ByteBuffer.allocateDirect(IO_BUFFER_SIZE);
this.readBuf.order(ByteOrder.nativeOrder());
this.readBuf.limit(0); // empty initially
+ this.phase = phase;
+ this.lastLoggedPercent = -1;
}
@Override
@@ -370,6 +399,15 @@ public class LuminaVectorGlobalIndexWriter implements GlobalIndexSingletonWriter
idBuf[i] = cursor + i;
}
cursor += batchSize;
+
+ int percent = (int) ((long) cursor * 100 / totalCount);
+ if (percent / 10 > lastLoggedPercent / 10) {
+ LOG.info(
+ "Lumina {} progress: {}/{} vectors ({}%)",
+ phase, cursor, totalCount, percent);
+ lastLoggedPercent = percent;
+ }
+
return batchSize;
}
(paimon) 04/37: [python] Add comment method for file_store_table (#7527)
Posted by ho...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hope pushed a commit to branch release-1.4
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit 86e2364a6407b5207f1d1acca79c7a6cd80849b3
Author: xuzifu666 <12...@qq.com>
AuthorDate: Wed Mar 25 17:02:01 2026 +0800
[python] Add comment method for file_store_table (#7527)
Currently Python API can not directly retrieve the `comment` from table;
a `comment` method provided to make it easier for users.
---
paimon-python/pypaimon/table/file_store_table.py | 4 ++
.../pypaimon/tests/table/file_store_table_test.py | 43 ++++++++++++++++++++++
2 files changed, 47 insertions(+)
diff --git a/paimon-python/pypaimon/table/file_store_table.py b/paimon-python/pypaimon/table/file_store_table.py
index 9db0fc3a48..d725d48021 100644
--- a/paimon-python/pypaimon/table/file_store_table.py
+++ b/paimon-python/pypaimon/table/file_store_table.py
@@ -86,6 +86,10 @@ class FileStoreTable(Table):
"""Get the current branch name from options."""
return self.options.branch()
+ def comment(self) -> Optional[str]:
+ """Get the table comment."""
+ return self.table_schema.comment
+
def consumer_manager(self):
"""Get the consumer manager for this table."""
from pypaimon.consumer.consumer_manager import ConsumerManager
diff --git a/paimon-python/pypaimon/tests/table/file_store_table_test.py b/paimon-python/pypaimon/tests/table/file_store_table_test.py
index d666f327f1..72c1ee7e2c 100644
--- a/paimon-python/pypaimon/tests/table/file_store_table_test.py
+++ b/paimon-python/pypaimon/tests/table/file_store_table_test.py
@@ -339,6 +339,49 @@ class FileStoreTableTest(unittest.TestCase):
self.assertEqual(copied_table.identifier, self.table.identifier)
self.assertEqual(copied_table.table_path, self.table.table_path)
+ def test_comment_none(self):
+ """Test that comment() returns None when table has no comment."""
+ # Default table created without comment should return None
+ self.assertIsNone(self.table.comment())
+
+ def test_comment_with_value(self):
+ """Test that comment() returns the correct comment when table has one."""
+ # Create a table with a comment
+ comment_text = "This is a test table comment"
+ schema = Schema.from_pyarrow_schema(
+ self.pa_schema,
+ partition_keys=['dt'],
+ options={CoreOptions.BUCKET.key(): "2"},
+ comment=comment_text
+ )
+ self.catalog.create_table('default.test_comment_table', schema, False)
+ comment_table = self.catalog.get_table('default.test_comment_table')
+
+ # Verify comment is returned correctly
+ self.assertEqual(comment_table.comment(), comment_text)
+
+ def test_comment_preserved_after_copy(self):
+ """Test that comment is preserved after copying the table."""
+ # Create a table with a comment
+ comment_text = "Test comment for copy"
+ schema = Schema.from_pyarrow_schema(
+ self.pa_schema,
+ partition_keys=['dt'],
+ options={CoreOptions.BUCKET.key(): "2"},
+ comment=comment_text
+ )
+ self.catalog.create_table('default.test_comment_copy_table', schema, False)
+ original_table = self.catalog.get_table('default.test_comment_copy_table')
+
+ # Verify original table has comment
+ self.assertEqual(original_table.comment(), comment_text)
+
+ # Copy the table
+ copied_table = original_table.copy({})
+
+ # Verify comment is preserved in copied table
+ self.assertEqual(copied_table.comment(), comment_text)
+
def test_truncate_table(self):
"""Test truncate_table functionality."""
# Create a write builder
(paimon) 26/37: [core] Support rename branch API and procedures (#7549)
Posted by ho...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hope pushed a commit to branch release-1.4
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit 74c5b493abd711adb71eda3015f59ac5c0eaf146
Author: xuzifu666 <12...@qq.com>
AuthorDate: Mon Mar 30 17:49:05 2026 +0800
[core] Support rename branch API and procedures (#7549)
Currently user can't change branch name directly, need to create a new
branch and then delete the old one, which is inconvenient. Therefore
provide an API for rename branch. This PR provided:
1. Experimental rename branch API in Table
2. RenameBranchProcedure in Flink/Spark
3. RestCatalog api for rename branch
---
docs/content/flink/procedures.md | 18 ++
docs/content/spark/procedures.md | 12 ++
docs/static/rest-catalog-open-api.yaml | 48 +++++
.../main/java/org/apache/paimon/rest/RESTApi.java | 22 +++
.../java/org/apache/paimon/rest/ResourcePaths.java | 13 ++
.../paimon/rest/requests/RenameBranchRequest.java | 46 +++++
.../org/apache/paimon/catalog/AbstractCatalog.java | 6 +
.../java/org/apache/paimon/catalog/Catalog.java | 15 ++
.../org/apache/paimon/catalog/DelegateCatalog.java | 6 +
.../java/org/apache/paimon/rest/RESTCatalog.java | 14 ++
.../paimon/table/AbstractFileStoreTable.java | 5 +
.../paimon/table/DelegatedFileStoreTable.java | 5 +
.../java/org/apache/paimon/table/FormatTable.java | 5 +
.../org/apache/paimon/table/ReadonlyTable.java | 8 +
.../main/java/org/apache/paimon/table/Table.java | 4 +
.../org/apache/paimon/utils/BranchManager.java | 2 +
.../apache/paimon/utils/CatalogBranchManager.java | 5 +
.../paimon/utils/FileSystemBranchManager.java | 19 ++
.../org/apache/paimon/rest/RESTCatalogServer.java | 46 ++++-
.../org/apache/paimon/rest/RESTCatalogTest.java | 21 ++-
.../paimon/utils/FileSystemBranchManagerTest.java | 196 +++++++++++++++++++++
.../flink/procedure/RenameBranchProcedure.java | 59 +++++++
.../services/org.apache.paimon.factories.Factory | 1 +
.../org/apache/paimon/flink/BranchSqlITCase.java | 54 ++++++
.../org/apache/paimon/spark/SparkProcedures.java | 2 +
.../spark/procedure/RenameBranchProcedure.java | 90 ++++++++++
.../spark/procedure/BranchProcedureTest.scala | 50 ++++++
27 files changed, 762 insertions(+), 10 deletions(-)
diff --git a/docs/content/flink/procedures.md b/docs/content/flink/procedures.md
index 739e3d4754..49e65d705b 100644
--- a/docs/content/flink/procedures.md
+++ b/docs/content/flink/procedures.md
@@ -793,6 +793,24 @@ All available procedures are listed below.
CALL sys.delete_branch(`table` => 'default.T', branch => 'branch1')
</td>
</tr>
+ <tr>
+ <td>rename_branch</td>
+ <td>
+ -- Use named argument<br/>
+ CALL [catalog.]sys.rename_branch(`table` => 'identifier', from_branch => 'branchName', to_branch => 'newBranchName')<br/><br/>
+ -- Use indexed argument<br/>
+ CALL [catalog.]sys.rename_branch('identifier', 'branchName', 'newBranchName')
+ </td>
+ <td>
+ To rename a branch. Arguments:
+ <li>table: the target table identifier. Cannot be empty.</li>
+ <li>from_branch: name of the branch to be renamed.</li>
+ <li>to_branch: new name of the branch.</li>
+ </td>
+ <td>
+ CALL sys.rename_branch(`table` => 'default.T', from_branch => 'branch1', to_branch => 'branch2')
+ </td>
+ </tr>
<tr>
<td>fast_forward</td>
<td>
diff --git a/docs/content/spark/procedures.md b/docs/content/spark/procedures.md
index 29ad120eab..ffc7005f42 100644
--- a/docs/content/spark/procedures.md
+++ b/docs/content/spark/procedures.md
@@ -317,6 +317,18 @@ This section introduce all available spark procedures about paimon.
CALL sys.delete_branch(table => 'test_db.T', branch => 'test_branch')
</td>
</tr>
+ <tr>
+ <td>rename_branch</td>
+ <td>
+ To rename a branch. Arguments:
+ <li>table: the target table identifier. Cannot be empty.</li>
+ <li>from_branch: name of the branch to be renamed.</li>
+ <li>to_branch: new name of the branch.</li>
+ </td>
+ <td>
+ CALL sys.rename_branch(table => 'test_db.T', from_branch => 'test_branch', to_branch => 'new_branch')
+ </td>
+ </tr>
<tr>
<td>fast_forward</td>
<td>
diff --git a/docs/static/rest-catalog-open-api.yaml b/docs/static/rest-catalog-open-api.yaml
index a4bda071f9..33f64524ec 100644
--- a/docs/static/rest-catalog-open-api.yaml
+++ b/docs/static/rest-catalog-open-api.yaml
@@ -1183,6 +1183,49 @@ paths:
$ref: '#/components/responses/BranchNotExistErrorResponse'
"500":
$ref: '#/components/responses/ServerErrorResponse'
+ /v1/{prefix}/databases/{database}/tables/{table}/branches/{branch}/rename:
+ post:
+ tags:
+ - branch
+ summary: rename branch
+ operationId: renameBranch
+ parameters:
+ - name: prefix
+ in: path
+ required: true
+ schema:
+ type: string
+ - name: database
+ in: path
+ required: true
+ schema:
+ type: string
+ - name: table
+ in: path
+ required: true
+ schema:
+ type: string
+ - name: branch
+ in: path
+ required: true
+ schema:
+ type: string
+ requestBody:
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/RenameBranchRequest'
+ responses:
+ "200":
+ description: Success, no content
+ "401":
+ $ref: '#/components/responses/UnauthorizedErrorResponse'
+ "404":
+ $ref: '#/components/responses/BranchNotExistErrorResponse'
+ "409":
+ description: Branch already exists
+ "500":
+ $ref: '#/components/responses/ServerErrorResponse'
/v1/{prefix}/databases/{database}/tables/{table}/branches/{branch}/forward:
post:
tags:
@@ -3251,6 +3294,11 @@ components:
fromTag:
nullable: true
type: string
+ RenameBranchRequest:
+ type: object
+ properties:
+ toBranch:
+ type: string
ForwardBranchRequest:
type: object
properties:
diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java b/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java
index a939c5ce16..88761ca5f0 100644
--- a/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java
+++ b/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java
@@ -49,6 +49,7 @@ import org.apache.paimon.rest.requests.ForwardBranchRequest;
import org.apache.paimon.rest.requests.ListPartitionsByNamesRequest;
import org.apache.paimon.rest.requests.MarkDonePartitionsRequest;
import org.apache.paimon.rest.requests.RegisterTableRequest;
+import org.apache.paimon.rest.requests.RenameBranchRequest;
import org.apache.paimon.rest.requests.RenameTableRequest;
import org.apache.paimon.rest.requests.ResetConsumerRequest;
import org.apache.paimon.rest.requests.RollbackSchemaRequest;
@@ -965,6 +966,27 @@ public class RESTApi {
restAuthFunction);
}
+ /**
+ * Rename branch for table.
+ *
+ * @param identifier database name and table name.
+ * @param fromBranch source branch name
+ * @param toBranch target branch name
+ * @throws NoSuchResourceException Exception thrown on HTTP 404 means the branch not exists
+ * @throws AlreadyExistsException Exception thrown on HTTP 409 means the target branch already
+ * exists
+ * @throws ForbiddenException Exception thrown on HTTP 403 means don't have the permission for
+ * this table
+ */
+ public void renameBranch(Identifier identifier, String fromBranch, String toBranch) {
+ RenameBranchRequest request = new RenameBranchRequest(toBranch);
+ client.post(
+ resourcePaths.renameBranch(
+ identifier.getDatabaseName(), identifier.getObjectName(), fromBranch),
+ request,
+ restAuthFunction);
+ }
+
/**
* Forward branch for table.
*
diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/ResourcePaths.java b/paimon-api/src/main/java/org/apache/paimon/rest/ResourcePaths.java
index 466ac975b4..6d1d16c82f 100644
--- a/paimon-api/src/main/java/org/apache/paimon/rest/ResourcePaths.java
+++ b/paimon-api/src/main/java/org/apache/paimon/rest/ResourcePaths.java
@@ -262,6 +262,19 @@ public class ResourcePaths {
"forward");
}
+ public String renameBranch(String databaseName, String tableName, String branch) {
+ return SLASH.join(
+ V1,
+ prefix,
+ DATABASES,
+ encodeString(databaseName),
+ TABLES,
+ encodeString(tableName),
+ BRANCHES,
+ encodeString(branch),
+ "rename");
+ }
+
public String tags(String databaseName, String objectName) {
return SLASH.join(
V1,
diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/requests/RenameBranchRequest.java b/paimon-api/src/main/java/org/apache/paimon/rest/requests/RenameBranchRequest.java
new file mode 100644
index 0000000000..63cf0011fa
--- /dev/null
+++ b/paimon-api/src/main/java/org/apache/paimon/rest/requests/RenameBranchRequest.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.rest.requests;
+
+import org.apache.paimon.rest.RESTRequest;
+
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator;
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter;
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty;
+
+/** Request for renaming branch. */
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class RenameBranchRequest implements RESTRequest {
+
+ private static final String FIELD_TO_BRANCH = "toBranch";
+
+ @JsonProperty(FIELD_TO_BRANCH)
+ private final String toBranch;
+
+ @JsonCreator
+ public RenameBranchRequest(@JsonProperty(FIELD_TO_BRANCH) String toBranch) {
+ this.toBranch = toBranch;
+ }
+
+ @JsonGetter(FIELD_TO_BRANCH)
+ public String toBranch() {
+ return toBranch;
+ }
+}
diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/AbstractCatalog.java b/paimon-core/src/main/java/org/apache/paimon/catalog/AbstractCatalog.java
index 709010d2ce..ad51ab1f4d 100644
--- a/paimon-core/src/main/java/org/apache/paimon/catalog/AbstractCatalog.java
+++ b/paimon-core/src/main/java/org/apache/paimon/catalog/AbstractCatalog.java
@@ -516,6 +516,12 @@ public abstract class AbstractCatalog implements Catalog {
throw new UnsupportedOperationException();
}
+ @Override
+ public void renameBranch(Identifier identifier, String fromBranch, String toBranch)
+ throws BranchNotExistException, BranchAlreadyExistException {
+ throw new UnsupportedOperationException();
+ }
+
@Override
public void fastForward(Identifier identifier, String branch) throws BranchNotExistException {
throw new UnsupportedOperationException();
diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/Catalog.java b/paimon-core/src/main/java/org/apache/paimon/catalog/Catalog.java
index 8924dc6ae1..6d337f3716 100644
--- a/paimon-core/src/main/java/org/apache/paimon/catalog/Catalog.java
+++ b/paimon-core/src/main/java/org/apache/paimon/catalog/Catalog.java
@@ -663,6 +663,7 @@ public interface Catalog extends AutoCloseable {
* <li>{@link #rollbackTo(Identifier, Instant)}.
* <li>{@link #createBranch(Identifier, String, String)}.
* <li>{@link #dropBranch(Identifier, String)}.
+ * <li>{@link #renameBranch(Identifier, String, String)}.
* <li>{@link #listBranches(Identifier)}.
* <li>{@link #getTag(Identifier, String)}.
* <li>{@link #createTag(Identifier, String, Long, String, boolean)}.
@@ -875,6 +876,20 @@ public interface Catalog extends AutoCloseable {
*/
void dropBranch(Identifier identifier, String branch) throws BranchNotExistException;
+ /**
+ * Rename a branch for this table.
+ *
+ * @param identifier path of the table, cannot be system or branch name.
+ * @param fromBranch the source branch name
+ * @param toBranch the target branch name
+ * @throws BranchNotExistException if the source branch doesn't exist
+ * @throws BranchAlreadyExistException if the target branch already exists
+ * @throws UnsupportedOperationException if the catalog does not {@link
+ * #supportsVersionManagement()}
+ */
+ void renameBranch(Identifier identifier, String fromBranch, String toBranch)
+ throws BranchNotExistException, BranchAlreadyExistException;
+
/**
* Fast-forward a branch to main branch.
*
diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/DelegateCatalog.java b/paimon-core/src/main/java/org/apache/paimon/catalog/DelegateCatalog.java
index 4a8eeabe50..ec5138a8cb 100644
--- a/paimon-core/src/main/java/org/apache/paimon/catalog/DelegateCatalog.java
+++ b/paimon-core/src/main/java/org/apache/paimon/catalog/DelegateCatalog.java
@@ -246,6 +246,12 @@ public abstract class DelegateCatalog implements Catalog {
wrapped.dropBranch(identifier, branch);
}
+ @Override
+ public void renameBranch(Identifier identifier, String fromBranch, String toBranch)
+ throws BranchNotExistException, BranchAlreadyExistException {
+ wrapped.renameBranch(identifier, fromBranch, toBranch);
+ }
+
@Override
public void fastForward(Identifier identifier, String branch) throws BranchNotExistException {
wrapped.fastForward(identifier, branch);
diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java
index dbb2a8fd48..364cdb3cd5 100644
--- a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java
+++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java
@@ -733,6 +733,20 @@ public class RESTCatalog implements Catalog {
}
}
+ @Override
+ public void renameBranch(Identifier identifier, String fromBranch, String toBranch)
+ throws BranchNotExistException, BranchAlreadyExistException {
+ try {
+ api.renameBranch(identifier, fromBranch, toBranch);
+ } catch (NoSuchResourceException e) {
+ throw new BranchNotExistException(identifier, fromBranch, e);
+ } catch (AlreadyExistsException e) {
+ throw new BranchAlreadyExistException(identifier, toBranch, e);
+ } catch (ForbiddenException e) {
+ throw new TableNoPermissionException(identifier, e);
+ }
+ }
+
@Override
public void fastForward(Identifier identifier, String branch) throws BranchNotExistException {
try {
diff --git a/paimon-core/src/main/java/org/apache/paimon/table/AbstractFileStoreTable.java b/paimon-core/src/main/java/org/apache/paimon/table/AbstractFileStoreTable.java
index 52b9d162a5..3583d17c08 100644
--- a/paimon-core/src/main/java/org/apache/paimon/table/AbstractFileStoreTable.java
+++ b/paimon-core/src/main/java/org/apache/paimon/table/AbstractFileStoreTable.java
@@ -718,6 +718,11 @@ abstract class AbstractFileStoreTable implements FileStoreTable {
branchManager().dropBranch(branchName);
}
+ @Override
+ public void renameBranch(String fromBranch, String toBranch) {
+ branchManager().renameBranch(fromBranch, toBranch);
+ }
+
@Override
public void fastForward(String branchName) {
branchManager().fastForward(branchName);
diff --git a/paimon-core/src/main/java/org/apache/paimon/table/DelegatedFileStoreTable.java b/paimon-core/src/main/java/org/apache/paimon/table/DelegatedFileStoreTable.java
index 9eaa4f8d2e..1f234d81ff 100644
--- a/paimon-core/src/main/java/org/apache/paimon/table/DelegatedFileStoreTable.java
+++ b/paimon-core/src/main/java/org/apache/paimon/table/DelegatedFileStoreTable.java
@@ -285,6 +285,11 @@ public abstract class DelegatedFileStoreTable implements FileStoreTable {
wrapped.deleteBranch(branchName);
}
+ @Override
+ public void renameBranch(String fromBranch, String toBranch) {
+ wrapped.renameBranch(fromBranch, toBranch);
+ }
+
@Override
public void fastForward(String branchName) {
wrapped.fastForward(branchName);
diff --git a/paimon-core/src/main/java/org/apache/paimon/table/FormatTable.java b/paimon-core/src/main/java/org/apache/paimon/table/FormatTable.java
index d7af8510a9..7f13faf753 100644
--- a/paimon-core/src/main/java/org/apache/paimon/table/FormatTable.java
+++ b/paimon-core/src/main/java/org/apache/paimon/table/FormatTable.java
@@ -417,6 +417,11 @@ public interface FormatTable extends Table {
throw new UnsupportedOperationException();
}
+ @Override
+ default void renameBranch(String fromBranch, String toBranch) {
+ throw new UnsupportedOperationException();
+ }
+
@Override
default void fastForward(String branchName) {
throw new UnsupportedOperationException();
diff --git a/paimon-core/src/main/java/org/apache/paimon/table/ReadonlyTable.java b/paimon-core/src/main/java/org/apache/paimon/table/ReadonlyTable.java
index 55f14e7025..b685f86e3e 100644
--- a/paimon-core/src/main/java/org/apache/paimon/table/ReadonlyTable.java
+++ b/paimon-core/src/main/java/org/apache/paimon/table/ReadonlyTable.java
@@ -265,6 +265,14 @@ public interface ReadonlyTable extends InnerTable {
this.getClass().getSimpleName()));
}
+ @Override
+ default void renameBranch(String fromBranch, String toBranch) {
+ throw new UnsupportedOperationException(
+ String.format(
+ "Readonly Table %s does not support renameBranch.",
+ this.getClass().getSimpleName()));
+ }
+
@Override
default void fastForward(String branchName) {
throw new UnsupportedOperationException(
diff --git a/paimon-core/src/main/java/org/apache/paimon/table/Table.java b/paimon-core/src/main/java/org/apache/paimon/table/Table.java
index 13d56d6849..98c00c0101 100644
--- a/paimon-core/src/main/java/org/apache/paimon/table/Table.java
+++ b/paimon-core/src/main/java/org/apache/paimon/table/Table.java
@@ -202,6 +202,10 @@ public interface Table extends Serializable {
}
}
+ /** Rename a branch. */
+ @Experimental
+ void renameBranch(String fromBranch, String toBranch);
+
/** Merge a branch to main branch. */
@Experimental
void fastForward(String branchName);
diff --git a/paimon-core/src/main/java/org/apache/paimon/utils/BranchManager.java b/paimon-core/src/main/java/org/apache/paimon/utils/BranchManager.java
index e055724efe..c830a799bf 100644
--- a/paimon-core/src/main/java/org/apache/paimon/utils/BranchManager.java
+++ b/paimon-core/src/main/java/org/apache/paimon/utils/BranchManager.java
@@ -59,6 +59,8 @@ public interface BranchManager {
void fastForward(String branchName);
+ void renameBranch(String fromBranch, String toBranch);
+
List<String> branches();
default boolean branchExists(String branchName) {
diff --git a/paimon-core/src/main/java/org/apache/paimon/utils/CatalogBranchManager.java b/paimon-core/src/main/java/org/apache/paimon/utils/CatalogBranchManager.java
index 8723135bd0..4ba9c39475 100644
--- a/paimon-core/src/main/java/org/apache/paimon/utils/CatalogBranchManager.java
+++ b/paimon-core/src/main/java/org/apache/paimon/utils/CatalogBranchManager.java
@@ -108,6 +108,11 @@ public class CatalogBranchManager implements BranchManager {
});
}
+ @Override
+ public void renameBranch(String fromBranch, String toBranch) {
+ executePost(catalog -> catalog.renameBranch(identifier, fromBranch, toBranch));
+ }
+
@Override
public List<String> branches() {
return executeGet(catalog -> catalog.listBranches(identifier));
diff --git a/paimon-core/src/main/java/org/apache/paimon/utils/FileSystemBranchManager.java b/paimon-core/src/main/java/org/apache/paimon/utils/FileSystemBranchManager.java
index ef3a04ec99..3257ea1060 100644
--- a/paimon-core/src/main/java/org/apache/paimon/utils/FileSystemBranchManager.java
+++ b/paimon-core/src/main/java/org/apache/paimon/utils/FileSystemBranchManager.java
@@ -220,6 +220,25 @@ public class FileSystemBranchManager implements BranchManager {
checkArgument(!branchExists(branchName), "Branch name '%s' already exists.", branchName);
}
+ @Override
+ public void renameBranch(String fromBranch, String toBranch) {
+ checkArgument(!BranchManager.isMainBranch(fromBranch), "Cannot rename the main branch.");
+ checkArgument(branchExists(fromBranch), "Branch name '%s' doesn't exist.", fromBranch);
+ checkArgument(!branchExists(toBranch), "Branch name '%s' already exists.", toBranch);
+ BranchManager.validateBranch(toBranch);
+
+ try {
+ // Use rename for atomic operation and better performance
+ fileIO.rename(branchPath(fromBranch), branchPath(toBranch));
+ } catch (IOException e) {
+ throw new RuntimeException(
+ String.format(
+ "Exception occurs when rename branch from '%s' to '%s'.",
+ fromBranch, toBranch),
+ e);
+ }
+ }
+
@Override
public List<String> branches() {
try {
diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java
index bd9036a4b0..5961101c3a 100644
--- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java
+++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java
@@ -61,6 +61,7 @@ import org.apache.paimon.rest.requests.CreateTagRequest;
import org.apache.paimon.rest.requests.CreateViewRequest;
import org.apache.paimon.rest.requests.ListPartitionsByNamesRequest;
import org.apache.paimon.rest.requests.MarkDonePartitionsRequest;
+import org.apache.paimon.rest.requests.RenameBranchRequest;
import org.apache.paimon.rest.requests.RenameTableRequest;
import org.apache.paimon.rest.requests.ResetConsumerRequest;
import org.apache.paimon.rest.requests.RollbackSchemaRequest;
@@ -1837,15 +1838,42 @@ public class RESTCatalogServer {
case "POST":
if (resources.length == 6) {
branch = RESTUtil.decodeString(resources[4]);
- branchManager.fastForward(branch);
- branchIdentifier =
- new Identifier(
- identifier.getDatabaseName(),
- identifier.getTableName(),
- branch);
- tableLatestSnapshotStore.put(
- identifier.getFullName(),
- tableLatestSnapshotStore.get(branchIdentifier.getFullName()));
+ if ("rename".equals(resources[5])) {
+ // Rename branch: /branches/{branch}/rename
+ RenameBranchRequest requestBody =
+ RESTApi.fromJson(data, RenameBranchRequest.class);
+ String toBranch = requestBody.toBranch();
+ table.renameBranch(branch, toBranch);
+ // Update store for renamed branch
+ Identifier fromBranchIdentifier =
+ new Identifier(
+ identifier.getDatabaseName(),
+ identifier.getTableName(),
+ branch);
+ Identifier toBranchIdentifier =
+ new Identifier(
+ identifier.getDatabaseName(),
+ identifier.getTableName(),
+ toBranch);
+ tableLatestSnapshotStore.put(
+ toBranchIdentifier.getFullName(),
+ tableLatestSnapshotStore.get(
+ fromBranchIdentifier.getFullName()));
+ tableMetadataStore.put(
+ toBranchIdentifier.getFullName(),
+ tableMetadataStore.get(fromBranchIdentifier.getFullName()));
+ } else if ("forward".equals(resources[5])) {
+ // Fast forward branch
+ branchManager.fastForward(branch);
+ branchIdentifier =
+ new Identifier(
+ identifier.getDatabaseName(),
+ identifier.getTableName(),
+ branch);
+ tableLatestSnapshotStore.put(
+ identifier.getFullName(),
+ tableLatestSnapshotStore.get(branchIdentifier.getFullName()));
+ }
} else {
CreateBranchRequest requestBody =
RESTApi.fromJson(data, CreateBranchRequest.class);
diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java
index 0c20dfdc17..e70a11321a 100644
--- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java
@@ -2132,7 +2132,26 @@ public abstract class RESTCatalogTest extends CatalogTestBase {
Catalog.BranchAlreadyExistException.class,
() -> restCatalog.createBranch(identifier, "my_branch", null));
assertThat(restCatalog.listBranches(identifier)).containsOnly("my_branch");
- restCatalog.dropBranch(identifier, "my_branch");
+
+ // Test rename branch
+ restCatalog.renameBranch(identifier, "my_branch", "renamed_branch");
+ assertThat(restCatalog.listBranches(identifier)).containsOnly("renamed_branch");
+ assertThat(restCatalog.getTable(new Identifier(databaseName, "table", "renamed_branch")))
+ .isNotNull();
+
+ // Test rename to existing branch should fail
+ restCatalog.createBranch(identifier, "another_branch", null);
+ assertThrows(
+ Catalog.BranchAlreadyExistException.class,
+ () -> restCatalog.renameBranch(identifier, "renamed_branch", "another_branch"));
+
+ // Test rename non-existent branch should fail
+ assertThrows(
+ Catalog.BranchNotExistException.class,
+ () -> restCatalog.renameBranch(identifier, "non_existent_branch", "new_branch"));
+
+ restCatalog.dropBranch(identifier, "renamed_branch");
+ restCatalog.dropBranch(identifier, "another_branch");
assertThrows(
Catalog.BranchNotExistException.class,
diff --git a/paimon-core/src/test/java/org/apache/paimon/utils/FileSystemBranchManagerTest.java b/paimon-core/src/test/java/org/apache/paimon/utils/FileSystemBranchManagerTest.java
new file mode 100644
index 0000000000..c74b53d3f3
--- /dev/null
+++ b/paimon-core/src/test/java/org/apache/paimon/utils/FileSystemBranchManagerTest.java
@@ -0,0 +1,196 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.utils;
+
+import org.apache.paimon.fs.FileIO;
+import org.apache.paimon.fs.FileIOFinder;
+import org.apache.paimon.fs.Path;
+import org.apache.paimon.schema.Schema;
+import org.apache.paimon.schema.SchemaManager;
+import org.apache.paimon.types.DataTypes;
+
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import java.util.List;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+/** Unit tests for {@link FileSystemBranchManager}. */
+class FileSystemBranchManagerTest {
+
+ @TempDir java.nio.file.Path tempDir;
+
+ private Path tablePath;
+ private FileIO fileIO;
+ private SnapshotManager snapshotManager;
+ private TagManager tagManager;
+ private SchemaManager schemaManager;
+ private FileSystemBranchManager branchManager;
+
+ @BeforeEach
+ void before() throws Exception {
+ tablePath = new Path(tempDir.toUri().toString());
+ fileIO = FileIOFinder.find(tablePath);
+
+ // Create schema
+ Schema schema =
+ Schema.newBuilder()
+ .column("product_id", DataTypes.INT())
+ .column("price", DataTypes.INT())
+ .column("sales", DataTypes.INT())
+ .primaryKey("product_id")
+ .build();
+
+ // Create schema manager and initialize schema
+ schemaManager = new SchemaManager(fileIO, tablePath);
+ schemaManager.createTable(schema);
+
+ // Create snapshot, tag, and branch managers
+ snapshotManager = new SnapshotManager(fileIO, tablePath, null, null, null);
+ tagManager = new TagManager(fileIO, tablePath);
+ branchManager =
+ new FileSystemBranchManager(
+ fileIO, tablePath, snapshotManager, tagManager, schemaManager);
+ }
+
+ @Test
+ void testRenameBranchBasic() {
+ // Create a branch
+ branchManager.createBranch("old_branch");
+ assertThat(branchManager.branchExists("old_branch")).isTrue();
+
+ // Rename the branch
+ branchManager.renameBranch("old_branch", "new_branch");
+
+ // Verify old branch no longer exists
+ assertThat(branchManager.branchExists("old_branch")).isFalse();
+
+ // Verify new branch exists
+ assertThat(branchManager.branchExists("new_branch")).isTrue();
+
+ // Verify branches list contains new branch
+ List<String> branches = branchManager.branches();
+ assertThat(branches).contains("new_branch");
+ assertThat(branches).doesNotContain("old_branch");
+ }
+
+ @Test
+ void testRenameNonExistentBranch() {
+ // Try to rename non-existent branch should throw exception
+ assertThatThrownBy(() -> branchManager.renameBranch("non_existent", "new_branch"))
+ .isInstanceOf(RuntimeException.class)
+ .hasMessageContaining("doesn't exist");
+ }
+
+ @Test
+ void testRenameToExistingBranch() {
+ // Create two branches
+ branchManager.createBranch("branch1");
+ branchManager.createBranch("branch2");
+
+ // Try to rename to existing branch should throw exception
+ assertThatThrownBy(() -> branchManager.renameBranch("branch1", "branch2"))
+ .isInstanceOf(RuntimeException.class)
+ .hasMessageContaining("already exists");
+ }
+
+ @Test
+ void testRenameMainBranchShouldFail() {
+ // Try to rename main branch should throw exception
+ assertThatThrownBy(() -> branchManager.renameBranch("main", "renamed_main"))
+ .isInstanceOf(RuntimeException.class)
+ .hasMessageContaining("main");
+ }
+
+ @Test
+ void testRenameBranchFromTag() {
+ // Create a tag
+ branchManager.createBranch("old_branch");
+
+ // Rename branch created from tag
+ branchManager.renameBranch("old_branch", "new_branch");
+
+ // Verify new branch exists
+ assertThat(branchManager.branchExists("new_branch")).isTrue();
+ assertThat(branchManager.branchExists("old_branch")).isFalse();
+ }
+
+ @Test
+ void testRenameBranchPreservesData() {
+ // Create a branch
+ branchManager.createBranch("test_branch");
+ assertThat(branchManager.branchExists("test_branch")).isTrue();
+
+ // Rename the branch
+ branchManager.renameBranch("test_branch", "renamed_branch");
+
+ // Verify the renamed branch exists and the original does not
+ assertThat(branchManager.branchExists("test_branch")).isFalse();
+ assertThat(branchManager.branchExists("renamed_branch")).isTrue();
+ }
+
+ @Test
+ void testRenameBranchMultipleTimes() {
+ // Create a branch
+ branchManager.createBranch("branch1");
+
+ // Rename multiple times
+ branchManager.renameBranch("branch1", "branch2");
+ branchManager.renameBranch("branch2", "branch3");
+
+ // Verify final state
+ assertThat(branchManager.branchExists("branch1")).isFalse();
+ assertThat(branchManager.branchExists("branch2")).isFalse();
+ assertThat(branchManager.branchExists("branch3")).isTrue();
+
+ List<String> branches = branchManager.branches();
+ assertThat(branches).contains("branch3");
+ assertThat(branches).doesNotContain("branch1", "branch2");
+ }
+
+ @Test
+ void testRenameBranchValidatesBranchNames() {
+ // Try to rename with invalid target branch name
+ branchManager.createBranch("valid_branch");
+
+ // Test numeric branch name
+ assertThatThrownBy(() -> branchManager.renameBranch("valid_branch", "123"))
+ .isInstanceOf(RuntimeException.class)
+ .hasMessageContaining("pure numeric");
+
+ // Test blank branch name
+ assertThatThrownBy(() -> branchManager.renameBranch("valid_branch", ""))
+ .isInstanceOf(RuntimeException.class)
+ .hasMessageContaining("blank");
+ }
+
+ @Test
+ void testRenameBranchToSameName() {
+ // Create a branch
+ branchManager.createBranch("same_name");
+
+ // Try to rename to the same name should throw exception
+ assertThatThrownBy(() -> branchManager.renameBranch("same_name", "same_name"))
+ .isInstanceOf(RuntimeException.class)
+ .hasMessageContaining("already exists");
+ }
+}
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/RenameBranchProcedure.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/RenameBranchProcedure.java
new file mode 100644
index 0000000000..8b60c7aa50
--- /dev/null
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/RenameBranchProcedure.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.flink.procedure;
+
+import org.apache.paimon.catalog.Catalog;
+import org.apache.paimon.catalog.Identifier;
+import org.apache.paimon.table.Table;
+
+import org.apache.flink.table.annotation.ArgumentHint;
+import org.apache.flink.table.annotation.DataTypeHint;
+import org.apache.flink.table.annotation.ProcedureHint;
+import org.apache.flink.table.procedure.ProcedureContext;
+
+/**
+ * Rename branch procedure. Usage:
+ *
+ * <pre><code>
+ * CALL sys.rename_branch('tableId', 'fromBranch', 'toBranch')
+ * </code></pre>
+ */
+public class RenameBranchProcedure extends ProcedureBase {
+
+ public static final String IDENTIFIER = "rename_branch";
+
+ @Override
+ public String identifier() {
+ return IDENTIFIER;
+ }
+
+ @ProcedureHint(
+ argument = {
+ @ArgumentHint(name = "table", type = @DataTypeHint("STRING")),
+ @ArgumentHint(name = "from_branch", type = @DataTypeHint("STRING")),
+ @ArgumentHint(name = "to_branch", type = @DataTypeHint("STRING"))
+ })
+ public String[] call(
+ ProcedureContext procedureContext, String tableId, String fromBranch, String toBranch)
+ throws Catalog.TableNotExistException {
+ Table table = catalog.getTable(Identifier.fromString(tableId));
+ table.renameBranch(fromBranch, toBranch);
+ return new String[] {"Success"};
+ }
+}
diff --git a/paimon-flink/paimon-flink-common/src/main/resources/META-INF/services/org.apache.paimon.factories.Factory b/paimon-flink/paimon-flink-common/src/main/resources/META-INF/services/org.apache.paimon.factories.Factory
index 092b470aa7..f87a250ec2 100644
--- a/paimon-flink/paimon-flink-common/src/main/resources/META-INF/services/org.apache.paimon.factories.Factory
+++ b/paimon-flink/paimon-flink-common/src/main/resources/META-INF/services/org.apache.paimon.factories.Factory
@@ -62,6 +62,7 @@ org.apache.paimon.flink.procedure.ExpireTagsProcedure
org.apache.paimon.flink.procedure.ReplaceTagProcedure
org.apache.paimon.flink.procedure.CreateBranchProcedure
org.apache.paimon.flink.procedure.DeleteBranchProcedure
+org.apache.paimon.flink.procedure.RenameBranchProcedure
org.apache.paimon.flink.procedure.DropPartitionProcedure
org.apache.paimon.flink.procedure.MergeIntoProcedure
org.apache.paimon.flink.procedure.ResetConsumerProcedure
diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/BranchSqlITCase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/BranchSqlITCase.java
index bcd3e0d964..1b7d50c852 100644
--- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/BranchSqlITCase.java
+++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/BranchSqlITCase.java
@@ -869,6 +869,60 @@ public class BranchSqlITCase extends CatalogITCaseBase {
.isEmpty();
}
+ @Test
+ public void testRenameBranch() throws Exception {
+ sql(
+ "CREATE TABLE T ("
+ + " pt INT"
+ + ", k INT"
+ + ", v STRING"
+ + ", PRIMARY KEY (pt, k) NOT ENFORCED"
+ + " ) PARTITIONED BY (pt) WITH ("
+ + " 'bucket' = '2'"
+ + " )");
+
+ // snapshot 1.
+ sql("INSERT INTO T VALUES (1, 10, 'apple'), (1, 20, 'banana')");
+ // snapshot 2.
+ sql("INSERT INTO T VALUES (2, 10, 'cat'), (2, 20, 'dog')");
+
+ // create tag
+ sql("CALL sys.create_tag('default.T', 'tag1', 1)");
+
+ // create branch from tag
+ sql("CALL sys.create_branch('default.T', 'branch1', 'tag1')");
+
+ // verify branch exists
+ FileStoreTable table = paimonTable("T");
+ assertThat(table.branchManager().branchExists("branch1")).isTrue();
+
+ // rename branch
+ sql("CALL sys.rename_branch('default.T', 'branch1', 'branch2')");
+
+ // verify old branch does not exist
+ table = paimonTable("T");
+ assertThat(table.branchManager().branchExists("branch1")).isFalse();
+
+ // verify new branch exists
+ assertThat(table.branchManager().branchExists("branch2")).isTrue();
+
+ // verify data in renamed branch
+ assertThat(collectResult("SELECT * FROM T$branch_branch2"))
+ .containsExactlyInAnyOrder("+I[1, 10, apple]", "+I[1, 20, banana]");
+
+ // rename non-existent branch should fail
+ assertThatThrownBy(
+ () ->
+ sql(
+ "CALL sys.rename_branch('default.T', 'nonexistent', 'new_branch')"))
+ .hasMessageContaining("Branch");
+
+ // rename to existing branch should fail
+ sql("CALL sys.create_branch('default.T', 'branch3')");
+ assertThatThrownBy(() -> sql("CALL sys.rename_branch('default.T', 'branch2', 'branch3')"))
+ .hasMessageContaining("Branch");
+ }
+
private List<String> collectResult(String sql) throws Exception {
List<String> result = new ArrayList<>();
try (CloseableIterator<Row> it = tEnv.executeSql(sql).collect()) {
diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkProcedures.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkProcedures.java
index 74bf04d68c..f27478bbbd 100644
--- a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkProcedures.java
+++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkProcedures.java
@@ -46,6 +46,7 @@ import org.apache.paimon.spark.procedure.ProcedureBuilder;
import org.apache.paimon.spark.procedure.PurgeFilesProcedure;
import org.apache.paimon.spark.procedure.RemoveOrphanFilesProcedure;
import org.apache.paimon.spark.procedure.RemoveUnexistingFilesProcedure;
+import org.apache.paimon.spark.procedure.RenameBranchProcedure;
import org.apache.paimon.spark.procedure.RenameTagProcedure;
import org.apache.paimon.spark.procedure.RepairProcedure;
import org.apache.paimon.spark.procedure.ReplaceTagProcedure;
@@ -98,6 +99,7 @@ public class SparkProcedures {
procedureBuilders.put("create_global_index", CreateGlobalIndexProcedure::builder);
procedureBuilders.put("drop_global_index", DropGlobalIndexProcedure::builder);
procedureBuilders.put("delete_branch", DeleteBranchProcedure::builder);
+ procedureBuilders.put("rename_branch", RenameBranchProcedure::builder);
procedureBuilders.put("compact", CompactProcedure::builder);
procedureBuilders.put("compact_database", CompactDatabaseProcedure::builder);
procedureBuilders.put("rescale", RescaleProcedure::builder);
diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/RenameBranchProcedure.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/RenameBranchProcedure.java
new file mode 100644
index 0000000000..f30209875e
--- /dev/null
+++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/RenameBranchProcedure.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.spark.procedure;
+
+import org.apache.spark.sql.catalyst.InternalRow;
+import org.apache.spark.sql.connector.catalog.Identifier;
+import org.apache.spark.sql.connector.catalog.TableCatalog;
+import org.apache.spark.sql.types.DataTypes;
+import org.apache.spark.sql.types.Metadata;
+import org.apache.spark.sql.types.StructField;
+import org.apache.spark.sql.types.StructType;
+
+import static org.apache.spark.sql.types.DataTypes.StringType;
+
+/** Spark procedure to rename a branch. */
+public class RenameBranchProcedure extends BaseProcedure {
+
+ private static final ProcedureParameter[] PARAMETERS =
+ new ProcedureParameter[] {
+ ProcedureParameter.required("table", StringType),
+ ProcedureParameter.required("from_branch", StringType),
+ ProcedureParameter.required("to_branch", StringType)
+ };
+
+ private static final StructType OUTPUT_TYPE =
+ new StructType(
+ new StructField[] {
+ new StructField("result", DataTypes.BooleanType, true, Metadata.empty())
+ });
+
+ protected RenameBranchProcedure(TableCatalog tableCatalog) {
+ super(tableCatalog);
+ }
+
+ @Override
+ public ProcedureParameter[] parameters() {
+ return PARAMETERS;
+ }
+
+ @Override
+ public StructType outputType() {
+ return OUTPUT_TYPE;
+ }
+
+ @Override
+ public InternalRow[] call(InternalRow args) {
+ Identifier tableIdent = toIdentifier(args.getString(0), PARAMETERS[0].name());
+ String fromBranch = args.getString(1);
+ String toBranch = args.getString(2);
+ InternalRow[] result =
+ modifyPaimonTable(
+ tableIdent,
+ table -> {
+ table.renameBranch(fromBranch, toBranch);
+ InternalRow outputRow = newInternalRow(true);
+ return new InternalRow[] {outputRow};
+ });
+ return result;
+ }
+
+ public static ProcedureBuilder builder() {
+ return new BaseProcedure.Builder<RenameBranchProcedure>() {
+ @Override
+ public RenameBranchProcedure doBuild() {
+ return new RenameBranchProcedure(tableCatalog());
+ }
+ };
+ }
+
+ @Override
+ public String description() {
+ return "RenameBranchProcedure";
+ }
+}
diff --git a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/BranchProcedureTest.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/BranchProcedureTest.scala
index 67786a47fe..0db7436019 100644
--- a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/BranchProcedureTest.scala
+++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/BranchProcedureTest.scala
@@ -181,4 +181,54 @@ class BranchProcedureTest extends PaimonSparkTestBase with StreamTest {
Seq(Row("20240725", "apple", 5), Row("20240725", "banana", 7)))
}
}
+
+ test("Paimon Procedure: rename branch") {
+ withTable("T") {
+ sql("CREATE TABLE T (a INT, b STRING) TBLPROPERTIES ('primary-key'='a', 'bucket'='3')")
+
+ sql("INSERT INTO T VALUES (1, 'a'), (2, 'b')")
+
+ // create tag
+ sql("CALL sys.create_tag(table => 'test.T', tag => 'tag1', snapshot => 1)")
+
+ // create branch from tag
+ sql("CALL sys.create_branch(table => 'test.T', branch => 'branch1', tag => 'tag1')")
+
+ // verify branch exists
+ val table = loadTable("T")
+ assert(table.branchManager().branchExists("branch1"))
+
+ // rename branch
+ checkAnswer(
+ sql(
+ "CALL sys.rename_branch(table => 'test.T', from_branch => 'branch1', to_branch => 'branch2')"),
+ Row(true) :: Nil
+ )
+
+ // verify old branch does not exist
+ assert(!table.branchManager().branchExists("branch1"))
+
+ // verify new branch exists
+ assert(table.branchManager().branchExists("branch2"))
+
+ // verify data in renamed branch
+ checkAnswer(
+ sql("SELECT * FROM `T$branch_branch2` ORDER BY a"),
+ Row(1, "a") :: Row(2, "b") :: Nil
+ )
+
+ // rename non-existent branch should fail
+ intercept[Exception] {
+ sql(
+ "CALL sys.rename_branch(table => 'test.T', from_branch => 'nonexistent', to_branch => 'new_branch')")
+ }
+
+ // rename to existing branch should fail
+ sql("CALL sys.create_branch(table => 'test.T', branch => 'branch3')")
+ intercept[Exception] {
+ sql(
+ "CALL sys.rename_branch(table => 'test.T', from_branch => 'branch2', to_branch => 'branch3')")
+ }
+ }
+ }
}
(paimon) 28/37: [python] Add FUSE support for REST Catalog (#7483)
Posted by ho...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hope pushed a commit to branch release-1.4
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit afb2effb1f240795c0199ac66c0a71f852573e99
Author: shyjsarah <44...@users.noreply.github.com>
AuthorDate: Mon Mar 30 03:15:30 2026 -0700
[python] Add FUSE support for REST Catalog (#7483)
---
docs/content/pypaimon/fuse-support.md | 94 ++++++
.../pypaimon/catalog/rest/fuse_support.py | 173 +++++++++++
.../pypaimon/catalog/rest/rest_catalog.py | 25 +-
paimon-python/pypaimon/common/options/config.py | 35 +++
paimon-python/pypaimon/filesystem/local_file_io.py | 24 ++
.../pypaimon/tests/rest/test_fuse_local_path.py | 346 +++++++++++++++++++++
6 files changed, 695 insertions(+), 2 deletions(-)
diff --git a/docs/content/pypaimon/fuse-support.md b/docs/content/pypaimon/fuse-support.md
new file mode 100644
index 0000000000..a9bea49f86
--- /dev/null
+++ b/docs/content/pypaimon/fuse-support.md
@@ -0,0 +1,94 @@
+---
+title: "FUSE Support"
+weight: 7
+type: docs
+aliases:
+ - /pypaimon/fuse-support.html
+---
+
+# FUSE Support
+
+When using PyPaimon REST Catalog to access remote object storage (such as OSS, S3, or HDFS), data access typically goes through remote storage SDKs. However, in scenarios where remote storage paths are mounted locally via FUSE (Filesystem in Userspace), users can access data directly through local filesystem paths for better performance.
+
+This feature enables PyPaimon to use local file access when FUSE mount is available, bypassing remote storage SDKs.
+
+## Configuration
+
+| Option | Type | Default | Description |
+|--------|------|---------|-------------|
+| `fuse.enabled` | Boolean | `false` | Whether to enable FUSE local path mapping |
+| `fuse.root` | String | (none) | FUSE mounted local root path, e.g., `/mnt/fuse/warehouse` |
+| `fuse.validation-mode` | String | `strict` | Validation mode: `strict`, `warn`, or `none` |
+
+## Usage
+
+```python
+from pypaimon import CatalogFactory
+
+catalog_options = {
+ 'metastore': 'rest',
+ 'uri': 'http://rest-server:8080',
+ 'warehouse': 'oss://my-catalog/',
+ 'token.provider': 'xxx',
+
+ # FUSE local path configuration
+ 'fuse.enabled': 'true',
+ 'fuse.root': '/mnt/fuse/warehouse',
+ 'fuse.validation-mode': 'strict'
+}
+
+catalog = CatalogFactory.create(catalog_options)
+```
+
+## Validation Modes
+
+Validation is performed on first data access to verify FUSE mount correctness. The `validation-mode` controls behavior when the local path does not exist:
+
+| Mode | Behavior | Use Case |
+|------|----------|----------|
+| `strict` | Throw exception, block operation | Production, safety first |
+| `warn` | Log warning, fallback to default FileIO | Testing, compatibility first |
+| `none` | Skip validation, use directly | Trusted environment, performance first |
+
+**Note**: Configuration errors (e.g., `fuse.enabled=true` but `fuse.root` not configured) will throw exceptions directly, regardless of validation mode.
+
+## How It Works
+
+1. When `fuse.enabled=true`, PyPaimon attempts to use local file access
+2. On first data access, validation is triggered (unless mode is `none`)
+3. Validation fetches the `default` database location and converts it to local path
+4. If local path exists, subsequent data access uses `FuseLocalFileIO`
+5. Path translation uses database/table logical names: remote path `oss://<catalog-id>/<db-id>/<table-id>` → local path `<root>/<db-name>/<table-name>`
+6. If validation fails, behavior depends on `validation-mode`
+
+## Example Scenario
+
+Assume you have:
+- Remote storage paths use UUIDs: `oss://clg-paimon-xxx/db-xxx/tbl-xxx`
+- FUSE mount: `/mnt/fuse/warehouse` (mounted to `pvfs://demo_catalog`)
+- FUSE exposes logical names: `/mnt/fuse/warehouse/my_db/my_table`
+
+```python
+from pypaimon import CatalogFactory
+
+catalog = CatalogFactory.create({
+ 'metastore': 'rest',
+ 'uri': 'http://rest-server:8080',
+ 'warehouse': 'oss://my-catalog/',
+ 'fuse.enabled': 'true',
+ 'fuse.root': '/mnt/fuse/warehouse',
+ 'fuse.validation-mode': 'none'
+})
+
+# When reading table 'my_db.my_table', PyPaimon will:
+# 1. Convert "oss://clg-paimon-xxx/db-xxx/tbl-xxx" to "/mnt/fuse/warehouse/my_db/my_table"
+# 2. Use FuseLocalFileIO to read from local path
+table = catalog.get_table('my_db.my_table')
+reader = table.new_read_builder().new_read()
+```
+
+## Limitations
+
+- Only catalog-level FUSE mount is supported (single `fuse.root` configuration)
+- Validation only checks if local path exists, not data consistency
+- If FUSE mount becomes unavailable after validation, file operations may fail
diff --git a/paimon-python/pypaimon/catalog/rest/fuse_support.py b/paimon-python/pypaimon/catalog/rest/fuse_support.py
new file mode 100644
index 0000000000..a56de0492b
--- /dev/null
+++ b/paimon-python/pypaimon/catalog/rest/fuse_support.py
@@ -0,0 +1,173 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+import logging
+from urllib.parse import urlparse
+
+from pypaimon.common.identifier import Identifier
+from pypaimon.common.options.config import FuseOptions
+from pypaimon.filesystem.local_file_io import LocalFileIO, FuseLocalFileIO
+
+logger = logging.getLogger(__name__)
+
+
+class FusePathResolver:
+ """Resolves FUSE local paths and validates FUSE mount availability."""
+
+ def __init__(self, options, rest_api):
+ self.fuse_root = options.get(FuseOptions.FUSE_ROOT)
+ self.validation_mode = options.get(FuseOptions.FUSE_VALIDATION_MODE, "strict")
+ self.fuse_mode = options.get(FuseOptions.FUSE_MODE, "pvfs")
+ self._validation_state = None # None=not validated, True=passed, False=failed
+ self._rest_api = rest_api
+ self._options = options
+
+ def resolve_local_path(self, original_path, identifier=None):
+ """
+ Resolve FUSE local path.
+
+ In 'pvfs' mode, use database/table logical names from identifier to build the path.
+ If identifier has no object name, returns database-level path (used for validation).
+ In 'raw' mode, use URI path segments directly.
+
+ Returns:
+ Local path
+
+ Raises:
+ ValueError: If fuse.root is not configured or pvfs mode missing identifier
+ """
+ if not self.fuse_root:
+ raise ValueError(
+ "FUSE local path is enabled but fuse.root is not configured"
+ )
+
+ root = self.fuse_root.rstrip('/')
+
+ if self.fuse_mode == "pvfs":
+ if identifier is None:
+ raise ValueError(
+ "FUSE path mode 'pvfs' requires an Identifier to resolve "
+ "the local path, but identifier is None."
+ )
+ db = identifier.get_database_name()
+ obj = identifier.get_object_name()
+ if obj:
+ return "{}/{}/{}".format(root, db, obj)
+ return "{}/{}".format(root, db)
+ elif self.fuse_mode == "raw":
+ # raw mode: use URI path segments directly
+ uri = urlparse(original_path)
+ path_part = uri.path.lstrip('/')
+ if not uri.scheme:
+ # No scheme means path like "catalog/db/table",
+ # skip the first segment (catalog name) to align with scheme-based paths
+ segments = path_part.split('/')
+ if len(segments) > 1:
+ path_part = '/'.join(segments[1:])
+ return "{}/{}".format(root, path_part)
+ else:
+ raise ValueError(
+ "Invalid fuse.mode: '{}'. "
+ "Supported modes are 'pvfs' and 'raw'.".format(self.fuse_mode)
+ )
+
+ def validate(self):
+ """
+ Validate FUSE local path is correctly mounted.
+
+ Get default database's location, convert to local path and check if it exists.
+ """
+ if self.validation_mode == "none":
+ self._validation_state = True
+ return
+
+ # Get default database details, API call failure raises exception directly
+ db = self._rest_api.get_database("default")
+ remote_location = db.location
+
+ if not remote_location:
+ logger.info("Default database has no location, skipping FUSE validation")
+ self._validation_state = True
+ return
+
+ expected_local = self.resolve_local_path(
+ remote_location, Identifier.create("default", None)
+ )
+ local_file_io = LocalFileIO(expected_local, self._options)
+
+ # Only validate if local path exists, handle based on validation mode
+ if not local_file_io.exists(expected_local):
+ error_msg = (
+ "FUSE local path validation failed: "
+ "local path '{}' does not exist "
+ "for default database location '{}'".format(expected_local, remote_location)
+ )
+ self._handle_validation_error(error_msg)
+ else:
+ self._validation_state = True
+ logger.info("FUSE local path validation passed")
+
+ def _handle_validation_error(self, error_msg):
+ """Handle validation error based on validation mode."""
+ if self.validation_mode == "strict":
+ raise ValueError(error_msg)
+ elif self.validation_mode == "warn":
+ logger.warning("%s. Falling back to default FileIO.", error_msg)
+ self._validation_state = False # Mark validation failed, fallback to default FileIO
+
+ def get_file_io(self, table_path, identifier, data_token_enabled,
+ rest_token_file_io_factory, default_file_io_factory):
+ """
+ Get FileIO for data access, supporting FUSE local path mapping.
+
+ Args:
+ table_path: The remote table path
+ identifier: Table identifier
+ data_token_enabled: Whether data token is enabled
+ rest_token_file_io_factory: Factory callable for RESTTokenFileIO
+ default_file_io_factory: Factory callable for default FileIO
+
+ Returns:
+ FileIO instance (FuseLocalFileIO or fallback)
+ """
+ # Configuration error raises exception directly
+ local_path = self.resolve_local_path(table_path, identifier)
+
+ # Perform validation (only once)
+ if self._validation_state is None:
+ self.validate()
+
+ # Validation passed, return FUSE-aware local FileIO
+ if self._validation_state:
+ return FuseLocalFileIO(
+ path=table_path.rstrip('/'),
+ fuse_path=local_path.rstrip('/'),
+ catalog_options=self._options,
+ )
+
+ # warn mode validation failed, fallback to default FileIO
+ if data_token_enabled:
+ return rest_token_file_io_factory()
+ return default_file_io_factory()
+
+ @property
+ def validation_state(self):
+ return self._validation_state
+
+ @validation_state.setter
+ def validation_state(self, value):
+ self._validation_state = value
diff --git a/paimon-python/pypaimon/catalog/rest/rest_catalog.py b/paimon-python/pypaimon/catalog/rest/rest_catalog.py
index a8f232a19c..fc2f516a8b 100644
--- a/paimon-python/pypaimon/catalog/rest/rest_catalog.py
+++ b/paimon-python/pypaimon/catalog/rest/rest_catalog.py
@@ -15,8 +15,8 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
+import logging
from typing import Any, Callable, Dict, List, Optional, Union
-
from pypaimon.api.api_response import GetTableResponse, PagedList, ErrorResponse
from pypaimon.api.rest_api import RESTApi
from pypaimon.api.rest_exception import NoSuchResourceException, AlreadyExistsException, ForbiddenException
@@ -32,7 +32,7 @@ from pypaimon.catalog.database import Database
from pypaimon.catalog.rest.property_change import PropertyChange
from pypaimon.catalog.rest.rest_token_file_io import RESTTokenFileIO
from pypaimon.catalog.rest.table_metadata import TableMetadata
-from pypaimon.common.options.config import CatalogOptions
+from pypaimon.common.options.config import CatalogOptions, FuseOptions
from pypaimon.common.options.core_options import CoreOptions
from pypaimon.common.file_io import FileIO
from pypaimon.common.identifier import Identifier
@@ -46,6 +46,8 @@ from pypaimon.table.format.format_table import FormatTable, Format
from pypaimon.table.iceberg.iceberg_table import IcebergTable
from pypaimon.table.object.object_table import ObjectTable
+logger = logging.getLogger(__name__)
+
FORMAT_TABLE_TYPE = "format-table"
ICEBERG_TABLE_TYPE = "iceberg-table"
OBJECT_TABLE_TYPE = "object-table"
@@ -59,6 +61,13 @@ class RESTCatalog(Catalog):
context.prefer_io_loader, context.fallback_io_loader)
self.data_token_enabled = self.rest_api.options.get(CatalogOptions.DATA_TOKEN_ENABLED)
+ # FUSE support (lazy import only when enabled)
+ self.fuse_enabled = self.context.options.get(FuseOptions.FUSE_ENABLED, False)
+ self._fuse_resolver = None
+ if self.fuse_enabled:
+ from pypaimon.catalog.rest.fuse_support import FusePathResolver
+ self._fuse_resolver = FusePathResolver(self.context.options, self.rest_api)
+
def catalog_loader(self):
"""
Create and return a RESTCatalogLoader for this catalog.
@@ -385,6 +394,18 @@ class RESTCatalog(Catalog):
return FileIO.get(table_path, self.context.options)
def file_io_for_data(self, table_path: str, identifier: Identifier):
+ """
+ Get FileIO for data access, supporting FUSE local path mapping.
+ """
+ if self._fuse_resolver is not None:
+ return self._fuse_resolver.get_file_io(
+ table_path, identifier, self.data_token_enabled,
+ rest_token_file_io_factory=lambda: RESTTokenFileIO(
+ identifier, table_path, self.context.options),
+ default_file_io_factory=lambda: self.file_io_from_options(table_path),
+ )
+
+ # Fallback to original logic
return RESTTokenFileIO(identifier, table_path, self.context.options) \
if self.data_token_enabled else self.file_io_from_options(table_path)
diff --git a/paimon-python/pypaimon/common/options/config.py b/paimon-python/pypaimon/common/options/config.py
index 8b0230da64..83d46c85bf 100644
--- a/paimon-python/pypaimon/common/options/config.py
+++ b/paimon-python/pypaimon/common/options/config.py
@@ -85,3 +85,38 @@ class CatalogOptions:
HTTP_USER_AGENT_HEADER = ConfigOptions.key(
"header.HTTP_USER_AGENT").string_type().no_default_value().with_description("HTTP User Agent header")
BLOB_FILE_IO_DEFAULT_CACHE_SIZE = 2 ** 31 - 1
+
+
+class FuseOptions:
+ """FUSE configuration options."""
+
+ FUSE_ENABLED = (
+ ConfigOptions.key("fuse.enabled")
+ .boolean_type()
+ .default_value(False)
+ .with_description("Whether to enable FUSE local path mapping")
+ )
+
+ FUSE_ROOT = (
+ ConfigOptions.key("fuse.root")
+ .string_type()
+ .no_default_value()
+ .with_description("FUSE mounted local root path, e.g., /mnt/fuse/warehouse")
+ )
+
+ FUSE_VALIDATION_MODE = (
+ ConfigOptions.key("fuse.validation-mode")
+ .string_type()
+ .default_value("strict")
+ .with_description("Validation mode: strict, warn, or none")
+ )
+
+ FUSE_MODE = (
+ ConfigOptions.key("fuse.mode")
+ .string_type()
+ .default_value("pvfs")
+ .with_description(
+ "FUSE path mode: 'pvfs' uses database/table logical names, "
+ "'raw' uses URI path segments directly"
+ )
+ )
diff --git a/paimon-python/pypaimon/filesystem/local_file_io.py b/paimon-python/pypaimon/filesystem/local_file_io.py
index cf9399309f..44dadaaf66 100644
--- a/paimon-python/pypaimon/filesystem/local_file_io.py
+++ b/paimon-python/pypaimon/filesystem/local_file_io.py
@@ -453,3 +453,27 @@ class LocalFileIO(FileIO):
except Exception as e:
self.delete_quietly(path)
raise RuntimeError(f"Failed to write blob file {path}: {e}") from e
+
+
+class FuseLocalFileIO(LocalFileIO):
+ """LocalFileIO that translates remote OSS paths to FUSE-mounted local paths.
+
+ All file operations receive paths like:
+ oss://clg-paimon-xxx/db-xxx/tbl-xxx/manifest/manifest-xxx
+ This class replaces the path prefix with fuse_path so the actual
+ I/O goes through the FUSE mount point.
+ """
+
+ def __init__(self, path: str, fuse_path: str,
+ catalog_options: Optional[Options] = None):
+ super().__init__(path=fuse_path, catalog_options=catalog_options)
+ self.path = path
+ self.fuse_path = fuse_path
+
+ def _to_file(self, path: str) -> Path:
+ return super()._to_file(self._translate(path))
+
+ def _translate(self, path: str) -> str:
+ if path == self.path or path.startswith(self.path + "/"):
+ return self.fuse_path + path[len(self.path):]
+ return path
diff --git a/paimon-python/pypaimon/tests/rest/test_fuse_local_path.py b/paimon-python/pypaimon/tests/rest/test_fuse_local_path.py
new file mode 100644
index 0000000000..13e56c7099
--- /dev/null
+++ b/paimon-python/pypaimon/tests/rest/test_fuse_local_path.py
@@ -0,0 +1,346 @@
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+import unittest
+from unittest.mock import MagicMock, patch
+
+from pypaimon.catalog.rest.fuse_support import FusePathResolver
+from pypaimon.catalog.rest.rest_catalog import RESTCatalog
+from pypaimon.common.options import Options
+from pypaimon.common.options.config import FuseOptions
+
+
+class TestFuseLocalPath(unittest.TestCase):
+ """Test cases for FUSE local path functionality."""
+
+ def _create_resolver(
+ self,
+ root="/mnt/fuse/warehouse",
+ validation_mode="strict",
+ mode="pvfs"
+ ):
+ """Helper to create a FusePathResolver with given config."""
+ options = Options({
+ "uri": "http://localhost:8080",
+ "warehouse": "oss://catalog/warehouse",
+ FuseOptions.FUSE_ENABLED.key(): "true",
+ FuseOptions.FUSE_ROOT.key(): root,
+ FuseOptions.FUSE_VALIDATION_MODE.key(): validation_mode,
+ FuseOptions.FUSE_MODE.key(): mode,
+ })
+ rest_api = MagicMock()
+ return FusePathResolver(options, rest_api), options, rest_api
+
+ def _create_catalog_with_fuse(
+ self,
+ enabled=True,
+ root="/mnt/fuse/warehouse",
+ validation_mode="strict",
+ mode="pvfs"
+ ):
+ """Helper to create a mock RESTCatalog with FUSE configuration."""
+ options = Options({
+ "uri": "http://localhost:8080",
+ "warehouse": "oss://catalog/warehouse",
+ FuseOptions.FUSE_ENABLED.key(): str(enabled).lower(),
+ FuseOptions.FUSE_ROOT.key(): root,
+ FuseOptions.FUSE_VALIDATION_MODE.key(): validation_mode,
+ FuseOptions.FUSE_MODE.key(): mode,
+ })
+
+ catalog = MagicMock(spec=RESTCatalog)
+ catalog.fuse_enabled = enabled
+ catalog.data_token_enabled = False
+ catalog.rest_api = MagicMock()
+ catalog.context = MagicMock()
+ catalog.context.options = options
+
+ if enabled:
+ resolver = FusePathResolver(options, catalog.rest_api)
+ catalog._fuse_resolver = resolver
+ else:
+ catalog._fuse_resolver = None
+
+ catalog.file_io_for_data = RESTCatalog.file_io_for_data.__get__(catalog)
+ catalog.file_io_from_options = MagicMock(return_value=MagicMock())
+
+ return catalog
+
+ # ========== _resolve_fuse_local_path Tests ==========
+
+ # --- pvfs mode tests ---
+
+ def test_resolve_pvfs_mode_with_identifier(self):
+ """Test pvfs mode uses identifier logical names."""
+ from pypaimon.common.identifier import Identifier
+ resolver, _, _ = self._create_resolver(mode="pvfs")
+ identifier = Identifier.create("my_db", "my_table")
+
+ result = resolver.resolve_local_path(
+ "oss://clg-paimon-xxx/db-xxx/tbl-xxx", identifier
+ )
+ self.assertEqual(result, "/mnt/fuse/warehouse/my_db/my_table")
+
+ def test_resolve_pvfs_mode_with_trailing_slash(self):
+ """Test pvfs mode with trailing slash on root."""
+ from pypaimon.common.identifier import Identifier
+ resolver, _, _ = self._create_resolver(mode="pvfs", root="/mnt/fuse/warehouse/")
+ identifier = Identifier.create("my_db", "my_table")
+
+ result = resolver.resolve_local_path(
+ "oss://clg-paimon-xxx/db-xxx/tbl-xxx", identifier
+ )
+ self.assertEqual(result, "/mnt/fuse/warehouse/my_db/my_table")
+
+ def test_resolve_pvfs_mode_without_identifier_raises(self):
+ """Test pvfs mode raises ValueError when identifier is None."""
+ resolver, _, _ = self._create_resolver(mode="pvfs")
+
+ with self.assertRaises(ValueError) as context:
+ resolver.resolve_local_path("oss://clg-paimon-xxx/db-xxx/tbl-xxx")
+
+ self.assertIn("identifier is None", str(context.exception))
+
+ # --- raw mode tests ---
+
+ def test_resolve_raw_mode_basic(self):
+ """Test raw mode basic path conversion."""
+ resolver, _, _ = self._create_resolver(mode="raw")
+
+ result = resolver.resolve_local_path("oss://catalog/db1/table1")
+ self.assertEqual(result, "/mnt/fuse/warehouse/db1/table1")
+
+ def test_resolve_raw_mode_with_trailing_slash(self):
+ """Test raw mode with trailing slash on root."""
+ resolver, _, _ = self._create_resolver(mode="raw", root="/mnt/fuse/warehouse/")
+
+ result = resolver.resolve_local_path("oss://catalog/db1/table1")
+ self.assertEqual(result, "/mnt/fuse/warehouse/db1/table1")
+
+ def test_resolve_raw_mode_deep_path(self):
+ """Test raw mode with deep path."""
+ resolver, _, _ = self._create_resolver(mode="raw")
+
+ result = resolver.resolve_local_path(
+ "oss://catalog/db1/table1/partition1/file.parquet"
+ )
+ self.assertEqual(
+ result,
+ "/mnt/fuse/warehouse/db1/table1/partition1/file.parquet"
+ )
+
+ def test_resolve_raw_mode_without_scheme(self):
+ """Test raw mode path without scheme skips first segment."""
+ resolver, _, _ = self._create_resolver(mode="raw")
+
+ result = resolver.resolve_local_path("catalog/db1/table1")
+ self.assertEqual(result, "/mnt/fuse/warehouse/db1/table1")
+
+ def test_resolve_raw_mode_ignores_identifier(self):
+ """Test raw mode uses URI path even when identifier is provided."""
+ from pypaimon.common.identifier import Identifier
+ resolver, _, _ = self._create_resolver(mode="raw")
+ identifier = Identifier.create("my_db", "my_table")
+
+ result = resolver.resolve_local_path(
+ "oss://catalog/db-uuid/tbl-uuid", identifier
+ )
+ self.assertEqual(result, "/mnt/fuse/warehouse/db-uuid/tbl-uuid")
+
+ # --- common tests ---
+
+ def test_resolve_fuse_local_path_missing_root(self):
+ """Test error when root is not configured."""
+ resolver, _, _ = self._create_resolver(root=None)
+
+ with self.assertRaises(ValueError) as context:
+ resolver.resolve_local_path("oss://catalog/db1/table1")
+
+ self.assertIn("fuse.root is not configured", str(context.exception))
+
+ # ========== Validation Tests ==========
+
+ def test_validation_mode_none_skips_validation(self):
+ """Test none mode skips validation."""
+ resolver, _, _ = self._create_resolver(validation_mode="none")
+
+ resolver.validate()
+
+ self.assertTrue(resolver.validation_state)
+
+ def test_validation_mode_strict_raises_on_failure(self):
+ """Test strict mode raises exception on validation failure."""
+ resolver, _, rest_api = self._create_resolver(validation_mode="strict")
+
+ # Mock default database with location
+ mock_db = MagicMock()
+ mock_db.location = "oss://catalog/default"
+ rest_api.get_database.return_value = mock_db
+
+ # Mock LocalFileIO to return False for exists
+ with patch('pypaimon.catalog.rest.fuse_support.LocalFileIO') as mock_local_io:
+ mock_instance = MagicMock()
+ mock_instance.exists.return_value = False
+ mock_local_io.return_value = mock_instance
+
+ with self.assertRaises(ValueError) as context:
+ resolver.validate()
+
+ self.assertIn("FUSE local path validation failed", str(context.exception))
+
+ def test_validation_mode_warn_fallback_on_failure(self):
+ """Test warn mode falls back to default FileIO on validation failure."""
+ resolver, _, rest_api = self._create_resolver(validation_mode="warn")
+
+ # Mock default database with location
+ mock_db = MagicMock()
+ mock_db.location = "oss://catalog/default"
+ rest_api.get_database.return_value = mock_db
+
+ # Mock LocalFileIO to return False for exists
+ with patch('pypaimon.catalog.rest.fuse_support.LocalFileIO') as mock_local_io:
+ mock_instance = MagicMock()
+ mock_instance.exists.return_value = False
+ mock_local_io.return_value = mock_instance
+
+ # Should not raise, just set state to False
+ resolver.validate()
+
+ self.assertFalse(resolver.validation_state)
+
+ def test_validation_passes_when_local_exists(self):
+ """Test validation passes when local path exists."""
+ resolver, _, rest_api = self._create_resolver(validation_mode="strict")
+
+ # Mock default database with location
+ mock_db = MagicMock()
+ mock_db.location = "oss://catalog/default"
+ rest_api.get_database.return_value = mock_db
+
+ # Mock LocalFileIO to return True for exists
+ with patch('pypaimon.catalog.rest.fuse_support.LocalFileIO') as mock_local_io:
+ mock_instance = MagicMock()
+ mock_instance.exists.return_value = True
+ mock_local_io.return_value = mock_instance
+
+ resolver.validate()
+
+ self.assertTrue(resolver.validation_state)
+
+ def test_validation_skips_when_no_location(self):
+ """Test validation skips when default database has no location."""
+ resolver, _, rest_api = self._create_resolver(validation_mode="strict")
+
+ # Mock default database without location
+ mock_db = MagicMock()
+ mock_db.location = None
+ rest_api.get_database.return_value = mock_db
+
+ resolver.validate()
+
+ self.assertTrue(resolver.validation_state)
+
+ # ========== file_io_for_data Tests ==========
+
+ def test_file_io_for_data_disabled_fuse(self):
+ """Test that disabled FUSE uses default FileIO."""
+ catalog = self._create_catalog_with_fuse(enabled=False)
+ catalog.data_token_enabled = False
+
+ from pypaimon.common.identifier import Identifier
+ identifier = Identifier.create("db1", "table1")
+
+ _ = catalog.file_io_for_data("oss://catalog/db1/table1", identifier)
+ catalog.file_io_from_options.assert_called_once()
+
+ def test_file_io_for_data_uses_local_when_validated(self):
+ """Test that validated FUSE uses FuseLocalFileIO."""
+ catalog = self._create_catalog_with_fuse(enabled=True, validation_mode="none")
+ catalog._fuse_resolver.validation_state = True # Already validated
+
+ from pypaimon.common.identifier import Identifier
+ identifier = Identifier.create("db1", "table1")
+
+ with patch('pypaimon.catalog.rest.fuse_support.FuseLocalFileIO') as mock_fuse_io:
+ mock_fuse_io.return_value = MagicMock()
+ _ = catalog.file_io_for_data("oss://catalog/db1/table1", identifier)
+ mock_fuse_io.assert_called_once()
+
+ def test_file_io_for_data_fallback_when_validation_failed(self):
+ """Test that failed validation falls back to default FileIO."""
+ catalog = self._create_catalog_with_fuse(enabled=True, validation_mode="warn")
+ catalog._fuse_resolver.validation_state = False # Validation failed
+ catalog.data_token_enabled = False
+
+ from pypaimon.common.identifier import Identifier
+ identifier = Identifier.create("db1", "table1")
+
+ _ = catalog.file_io_for_data("oss://catalog/db1/table1", identifier)
+ catalog.file_io_from_options.assert_called_once()
+
+ # ========== Invalid Mode Tests ==========
+
+ def test_resolve_invalid_mode_raises(self):
+ """Test that an invalid mode raises ValueError."""
+ resolver, _, _ = self._create_resolver(mode="invalid")
+
+ with self.assertRaises(ValueError) as context:
+ resolver.resolve_local_path("oss://catalog/db1/table1")
+
+ self.assertIn("Invalid fuse.mode", str(context.exception))
+ self.assertIn("invalid", str(context.exception))
+
+ # ========== Raw Mode Validation Tests ==========
+
+ def test_validation_raw_mode_strict_raises_on_failure(self):
+ """Test strict validation in raw mode raises exception on failure."""
+ resolver, _, rest_api = self._create_resolver(validation_mode="strict", mode="raw")
+
+ mock_db = MagicMock()
+ mock_db.location = "oss://catalog/default"
+ rest_api.get_database.return_value = mock_db
+
+ with patch('pypaimon.catalog.rest.fuse_support.LocalFileIO') as mock_local_io:
+ mock_instance = MagicMock()
+ mock_instance.exists.return_value = False
+ mock_local_io.return_value = mock_instance
+
+ with self.assertRaises(ValueError) as context:
+ resolver.validate()
+
+ self.assertIn("FUSE local path validation failed", str(context.exception))
+
+ def test_validation_raw_mode_passes_when_local_exists(self):
+ """Test validation passes in raw mode when local path exists."""
+ resolver, _, rest_api = self._create_resolver(validation_mode="strict", mode="raw")
+
+ mock_db = MagicMock()
+ mock_db.location = "oss://catalog/default"
+ rest_api.get_database.return_value = mock_db
+
+ with patch('pypaimon.catalog.rest.fuse_support.LocalFileIO') as mock_local_io:
+ mock_instance = MagicMock()
+ mock_instance.exists.return_value = True
+ mock_local_io.return_value = mock_instance
+
+ resolver.validate()
+
+ self.assertTrue(resolver.validation_state)
+
+
+if __name__ == '__main__':
+ unittest.main()
(paimon) 22/37: [flink] Sopport refreshing partition asynchronously for lookup join (#7402)
Posted by ho...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hope pushed a commit to branch release-1.4
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit ec02f8e6823b023faf1e30eb5e18c58d9976fad1
Author: umi <55...@users.noreply.github.com>
AuthorDate: Fri Mar 27 09:27:30 2026 +0800
[flink] Sopport refreshing partition asynchronously for lookup join (#7402)
---
.github/workflows/utitcase-flink-1.x-common.yml | 2 +-
.../generated/flink_connector_configuration.html | 8 +-
.../apache/paimon/flink/FlinkConnectorOptions.java | 12 +
.../flink/lookup/FileStoreLookupFunction.java | 56 +++-
.../paimon/flink/lookup/FullCacheLookupTable.java | 11 +
.../paimon/flink/lookup/PartitionRefresher.java | 223 +++++++++++++++
.../org/apache/paimon/flink/LookupJoinITCase.java | 313 +++++++++++++++++++++
7 files changed, 611 insertions(+), 14 deletions(-)
diff --git a/.github/workflows/utitcase-flink-1.x-common.yml b/.github/workflows/utitcase-flink-1.x-common.yml
index 6825d8e7a3..f22b56014f 100644
--- a/.github/workflows/utitcase-flink-1.x-common.yml
+++ b/.github/workflows/utitcase-flink-1.x-common.yml
@@ -38,7 +38,7 @@ concurrency:
jobs:
build_test:
runs-on: ubuntu-latest
- timeout-minutes: 60
+ timeout-minutes: 100
steps:
- name: Checkout code
diff --git a/docs/layouts/shortcodes/generated/flink_connector_configuration.html b/docs/layouts/shortcodes/generated/flink_connector_configuration.html
index 1003aff77c..d4c8b5a167 100644
--- a/docs/layouts/shortcodes/generated/flink_connector_configuration.html
+++ b/docs/layouts/shortcodes/generated/flink_connector_configuration.html
@@ -80,6 +80,12 @@ under the License.
<td>Duration</td>
<td>Specific dynamic partition refresh interval for lookup, scan all partitions and obtain corresponding partition.</td>
</tr>
+ <tr>
+ <td><h5>lookup.dynamic-partition.refresh.async</h5></td>
+ <td style="word-wrap: break-word;">false</td>
+ <td>Boolean</td>
+ <td>Whether to refresh dynamic partition lookup table asynchronously. This option only works for full cache dimension table. When enabled, partition changes will be loaded in a background thread while the old partition data continues serving queries. When disabled (default), partition refresh is synchronous and blocks queries until the new partition data is fully loaded.</td>
+ </tr>
<tr>
<td><h5>lookup.refresh.async</h5></td>
<td style="word-wrap: break-word;">false</td>
@@ -357,4 +363,4 @@ under the License.
<td>Defines a custom parallelism for the unaware-bucket table compaction job. By default, if this option is not defined, the planner will derive the parallelism for each statement individually by also considering the global configuration.</td>
</tr>
</tbody>
-</table>
+</table>
\ No newline at end of file
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/FlinkConnectorOptions.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/FlinkConnectorOptions.java
index 8febafc5e3..89d3e2174a 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/FlinkConnectorOptions.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/FlinkConnectorOptions.java
@@ -276,6 +276,18 @@ public class FlinkConnectorOptions {
.defaultValue(false)
.withDescription("Whether to refresh lookup table in an async thread.");
+ public static final ConfigOption<Boolean> LOOKUP_DYNAMIC_PARTITION_REFRESH_ASYNC =
+ ConfigOptions.key("lookup.dynamic-partition.refresh.async")
+ .booleanType()
+ .defaultValue(false)
+ .withDescription(
+ "Whether to refresh dynamic partition lookup table asynchronously. "
+ + "This option only works for full cache dimension table. "
+ + "When enabled, partition changes will be loaded in a background thread "
+ + "while the old partition data continues serving queries. "
+ + "When disabled (default), partition refresh is synchronous and blocks queries "
+ + "until the new partition data is fully loaded.");
+
public static final ConfigOption<Integer> LOOKUP_REFRESH_ASYNC_PENDING_SNAPSHOT_COUNT =
ConfigOptions.key("lookup.refresh.async.pending-snapshot-count")
.intType()
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/FileStoreLookupFunction.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/FileStoreLookupFunction.java
index 5b71664ca6..69d834615f 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/FileStoreLookupFunction.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/FileStoreLookupFunction.java
@@ -68,6 +68,7 @@ import java.util.stream.IntStream;
import static org.apache.paimon.CoreOptions.CONTINUOUS_DISCOVERY_INTERVAL;
import static org.apache.paimon.flink.FlinkConnectorOptions.LOOKUP_CACHE_MODE;
+import static org.apache.paimon.flink.FlinkConnectorOptions.LOOKUP_DYNAMIC_PARTITION_REFRESH_ASYNC;
import static org.apache.paimon.flink.FlinkConnectorOptions.LOOKUP_REFRESH_FULL_LOAD_THRESHOLD;
import static org.apache.paimon.flink.FlinkConnectorOptions.LOOKUP_REFRESH_TIME_PERIODS_BLACKLIST;
import static org.apache.paimon.flink.query.RemoteTableQuery.isRemoteServiceAvailable;
@@ -93,8 +94,12 @@ public class FileStoreLookupFunction implements Serializable, Closeable {
private final List<InternalRow.FieldGetter> projectFieldsGetters;
private transient File path;
+ private transient String tmpDirectory;
private transient LookupTable lookupTable;
+ // partition refresh
+ @Nullable private transient PartitionRefresher partitionRefresher;
+
// interval of refreshing lookup table
private transient Duration refreshInterval;
// timestamp when refreshing lookup table
@@ -161,7 +166,7 @@ public class FileStoreLookupFunction implements Serializable, Closeable {
public void open(FunctionContext context) throws Exception {
this.functionContext = context;
- String tmpDirectory = getTmpDirectory(context);
+ this.tmpDirectory = getTmpDirectory(context);
open(tmpDirectory);
}
@@ -236,6 +241,16 @@ public class FileStoreLookupFunction implements Serializable, Closeable {
lookupTable.specifyPartitions(
partitions, partitionLoader.createSpecificPartFilter());
}
+ if (partitionLoader instanceof DynamicPartitionLoader) {
+ // Initialize partition refresher
+ this.partitionRefresher =
+ new PartitionRefresher(
+ options.get(LOOKUP_DYNAMIC_PARTITION_REFRESH_ASYNC)
+ && lookupTable instanceof FullCacheLookupTable,
+ table.name(),
+ this.tmpDirectory,
+ partitionLoader.partitions());
+ }
}
if (cacheRowFilter != null) {
@@ -271,13 +286,16 @@ public class FileStoreLookupFunction implements Serializable, Closeable {
if (partitionLoader == null) {
return lookupInternal(key);
}
-
- if (partitionLoader.partitions().isEmpty()) {
+ List<BinaryRow> partitions =
+ partitionRefresher != null
+ ? partitionRefresher.currentPartitions()
+ : partitionLoader.partitions();
+ if (partitions.isEmpty()) {
return Collections.emptyList();
}
List<RowData> rows = new ArrayList<>();
- for (BinaryRow partition : partitionLoader.partitions()) {
+ for (BinaryRow partition : partitions) {
rows.addAll(lookupInternal(JoinedRow.join(key, partition)));
}
return rows;
@@ -324,7 +342,18 @@ public class FileStoreLookupFunction implements Serializable, Closeable {
return;
}
- // 2. refresh dynamic partition
+ // 2. check if async partition refresh has completed, and switch if so
+ if (partitionRefresher != null && partitionRefresher.isPartitionRefreshAsync()) {
+ LookupTable newLookupTable =
+ partitionRefresher.getNewLookupTable(partitionLoader.partitions());
+ if (newLookupTable != null) {
+ lookupTable.close();
+ lookupTable = newLookupTable;
+ path = partitionRefresher.path();
+ }
+ }
+
+ // 3. refresh dynamic partition
if (partitionLoader != null) {
boolean partitionChanged = partitionLoader.checkRefresh();
List<BinaryRow> partitions = partitionLoader.partitions();
@@ -334,18 +363,17 @@ public class FileStoreLookupFunction implements Serializable, Closeable {
}
if (partitionChanged) {
- // reopen with latest partition
- lookupTable.specifyPartitions(
- partitionLoader.partitions(), partitionLoader.createSpecificPartFilter());
- lookupTable.close();
- lookupTable.open();
+ partitionRefresher.startRefresh(
+ partitions,
+ partitionLoader.createSpecificPartFilter(),
+ lookupTable,
+ cacheRowFilter);
nextRefreshTime = System.currentTimeMillis() + refreshInterval.toMillis();
- // no need to refresh the lookup table because it is reopened
return;
}
}
- // 3. refresh lookup table
+ // 4. refresh lookup table
if (shouldRefreshLookupTable()) {
// Check if we should do full load (close and reopen table) instead of incremental
// refresh
@@ -415,6 +443,10 @@ public class FileStoreLookupFunction implements Serializable, Closeable {
@Override
public void close() throws IOException {
+ if (partitionRefresher != null) {
+ partitionRefresher.close();
+ }
+
if (lookupTable != null) {
lookupTable.close();
lookupTable = null;
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/FullCacheLookupTable.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/FullCacheLookupTable.java
index 81af38ea8f..6b02084d8a 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/FullCacheLookupTable.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/FullCacheLookupTable.java
@@ -414,5 +414,16 @@ public abstract class FullCacheLookupTable implements LookupTable {
joinKey,
requiredCachedBucketIds);
}
+
+ public Context copy(File newTempPath) {
+ return new Context(
+ table.wrapped(),
+ projection,
+ tablePredicate,
+ projectedPredicate,
+ newTempPath,
+ joinKey,
+ requiredCachedBucketIds);
+ }
}
}
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/PartitionRefresher.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/PartitionRefresher.java
new file mode 100644
index 0000000000..cbf6b025bb
--- /dev/null
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/PartitionRefresher.java
@@ -0,0 +1,223 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.flink.lookup;
+
+import org.apache.paimon.data.BinaryRow;
+import org.apache.paimon.data.InternalRow;
+import org.apache.paimon.options.Options;
+import org.apache.paimon.predicate.Predicate;
+import org.apache.paimon.utils.ExecutorThreadFactory;
+import org.apache.paimon.utils.ExecutorUtils;
+import org.apache.paimon.utils.FileIOUtils;
+import org.apache.paimon.utils.Filter;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.annotation.Nullable;
+
+import java.io.Closeable;
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+import java.util.UUID;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+
+import static org.apache.paimon.lookup.rocksdb.RocksDBOptions.LOOKUP_CACHE_ROWS;
+
+/** Manages partition refresh logic for {@link FullCacheLookupTable}. */
+public class PartitionRefresher implements Closeable {
+
+ private static final Logger LOG = LoggerFactory.getLogger(PartitionRefresher.class);
+
+ private final boolean partitionRefreshAsync;
+ private final String tableName;
+ private final String tmpDirectory;
+ private volatile File path;
+ private ExecutorService partitionRefreshExecutor;
+ private AtomicReference<LookupTable> pendingLookupTable;
+ private AtomicReference<Exception> partitionRefreshException;
+
+ /** Current partitions being used for lookup. Updated when partition refresh completes. */
+ private List<BinaryRow> currentPartitions;
+
+ public PartitionRefresher(
+ boolean partitionRefreshAsync,
+ String tableName,
+ String tmpDirectory,
+ List<BinaryRow> initialPartitions) {
+ this.partitionRefreshAsync = partitionRefreshAsync;
+ this.tableName = tableName;
+ this.tmpDirectory = tmpDirectory;
+ this.currentPartitions = initialPartitions;
+ if (!partitionRefreshAsync) {
+ return;
+ }
+ this.pendingLookupTable = new AtomicReference<>(null);
+ this.partitionRefreshException = new AtomicReference<>(null);
+ this.partitionRefreshExecutor =
+ Executors.newSingleThreadExecutor(
+ new ExecutorThreadFactory(
+ String.format(
+ "%s-lookup-refresh-partition",
+ Thread.currentThread().getName())));
+ }
+
+ /** Get the current partitions being used for lookup. */
+ public List<BinaryRow> currentPartitions() {
+ return currentPartitions;
+ }
+
+ /**
+ * Start partition refresh. Chooses sync or async mode based on configuration.
+ *
+ * @param newPartitions the new partitions to refresh to
+ * @param partitionFilter the partition filter for the new partitions
+ * @param lookupTable the current lookup table to refresh
+ * @param cacheRowFilter the cache row filter, may be null
+ */
+ public void startRefresh(
+ List<BinaryRow> newPartitions,
+ @Nullable Predicate partitionFilter,
+ LookupTable lookupTable,
+ @Nullable Filter<InternalRow> cacheRowFilter)
+ throws Exception {
+ if (partitionRefreshAsync) {
+ asyncPartitionRefresh(
+ newPartitions,
+ partitionFilter,
+ ((FullCacheLookupTable) lookupTable).context,
+ cacheRowFilter);
+ } else {
+ syncPartitionRefresh(newPartitions, partitionFilter, lookupTable);
+ }
+ }
+
+ private void syncPartitionRefresh(
+ List<BinaryRow> newPartitions,
+ @Nullable Predicate partitionFilter,
+ LookupTable lookupTable)
+ throws Exception {
+ LOG.info(
+ "Synchronously refreshing partition for table {}, new partitions detected.",
+ tableName);
+ lookupTable.close();
+ lookupTable.specifyPartitions(newPartitions, partitionFilter);
+ lookupTable.open();
+ this.currentPartitions = newPartitions;
+ LOG.info("Synchronous partition refresh completed for table {}.", tableName);
+ }
+
+ private void asyncPartitionRefresh(
+ List<BinaryRow> newPartitions,
+ @Nullable Predicate partitionFilter,
+ FullCacheLookupTable.Context context,
+ @Nullable Filter<InternalRow> cacheRowFilter) {
+
+ LOG.info(
+ "Starting async partition refresh for table {}, new partitions detected.",
+ tableName);
+
+ partitionRefreshExecutor.submit(
+ () -> {
+ try {
+ this.path = new File(tmpDirectory, "lookup-" + UUID.randomUUID());
+ if (!path.mkdirs()) {
+ throw new RuntimeException("Failed to create dir: " + path);
+ }
+ FullCacheLookupTable.Context newContext = context.copy(path);
+ Options options = Options.fromMap(context.table.options());
+ FullCacheLookupTable newTable =
+ FullCacheLookupTable.create(
+ newContext, options.get(LOOKUP_CACHE_ROWS));
+ if (cacheRowFilter != null) {
+ newTable.specifyCacheRowFilter(cacheRowFilter);
+ }
+ newTable.specifyPartitions(newPartitions, partitionFilter);
+ newTable.open();
+
+ pendingLookupTable.set(newTable);
+ LOG.info("Async partition refresh completed for table {}.", tableName);
+ } catch (Exception e) {
+ LOG.error("Async partition refresh failed for table {}.", tableName, e);
+ partitionRefreshException.set(e);
+ if (path != null) {
+ FileIOUtils.deleteDirectoryQuietly(path);
+ }
+ }
+ });
+ }
+
+ /**
+ * Check if an async partition refresh has completed.
+ *
+ * @param newPartitions the new partitions to update after refresh completes
+ * @return a Pair containing the new lookup table and its temp path if ready, or null if no
+ * switch is needed
+ */
+ @Nullable
+ public LookupTable getNewLookupTable(List<BinaryRow> newPartitions) throws Exception {
+ if (!partitionRefreshAsync) {
+ return null;
+ }
+
+ Exception asyncException = partitionRefreshException.getAndSet(null);
+ if (asyncException != null) {
+ LOG.error(
+ "Async partition refresh failed for table {}, will stop running.",
+ tableName,
+ asyncException);
+ throw asyncException;
+ }
+
+ LookupTable newTable = pendingLookupTable.getAndSet(null);
+ if (newTable == null) {
+ return null;
+ }
+
+ this.currentPartitions = newPartitions;
+ LOG.info("Switched to new lookup table for table {} with new partitions.", tableName);
+ return newTable;
+ }
+
+ /** Close partition refresh resources. */
+ @Override
+ public void close() throws IOException {
+ if (partitionRefreshExecutor != null) {
+ ExecutorUtils.gracefulShutdown(1L, TimeUnit.MINUTES, partitionRefreshExecutor);
+ }
+ if (pendingLookupTable != null) {
+ LookupTable pending = pendingLookupTable.getAndSet(null);
+ if (pending != null) {
+ pending.close();
+ }
+ }
+ }
+
+ public boolean isPartitionRefreshAsync() {
+ return partitionRefreshAsync;
+ }
+
+ public File path() {
+ return path;
+ }
+}
diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/LookupJoinITCase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/LookupJoinITCase.java
index 408d74ea0c..32a4948501 100644
--- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/LookupJoinITCase.java
+++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/LookupJoinITCase.java
@@ -1250,4 +1250,317 @@ public class LookupJoinITCase extends CatalogITCaseBase {
iterator.close();
}
+
+ @ParameterizedTest
+ @EnumSource(LookupCacheMode.class)
+ public void testAsyncPartitionRefresh(LookupCacheMode mode) throws Exception {
+ // This test verifies asynchronous partition refresh:
+ // when max_pt() changes, the lookup table is refreshed in a background thread,
+ // old partition data continues serving queries until the new partition is fully loaded.
+ sql(
+ "CREATE TABLE PARTITIONED_DIM (pt STRING, k INT, v INT, PRIMARY KEY (pt, k) NOT ENFORCED)"
+ + "PARTITIONED BY (`pt`) WITH ("
+ + "'bucket' = '1', "
+ + "'lookup.dynamic-partition' = 'max_pt()', "
+ + "'lookup.dynamic-partition.refresh-interval' = '1 ms', "
+ + "'lookup.dynamic-partition.refresh.async' = 'true', "
+ + "'lookup.cache' = '%s', "
+ + "'continuous.discovery-interval'='1 ms')",
+ mode);
+
+ // insert data into partition '1'
+ sql("INSERT INTO PARTITIONED_DIM VALUES ('1', 1, 100), ('1', 2, 200)");
+
+ String query =
+ "SELECT T.i, D.v FROM T LEFT JOIN PARTITIONED_DIM "
+ + "for system_time as of T.proctime AS D ON T.i = D.k";
+ BlockingIterator<Row, Row> iterator = BlockingIterator.of(sEnv.executeSql(query).collect());
+
+ // verify initial lookup returns partition '1' data
+ sql("INSERT INTO T VALUES (1), (2)");
+ List<Row> result = iterator.collect(2);
+ assertThat(result).containsExactlyInAnyOrder(Row.of(1, 100), Row.of(2, 200));
+
+ // insert data into a new partition '2', which will trigger async partition refresh
+ sql("INSERT INTO PARTITIONED_DIM VALUES ('2', 1, 1000), ('2', 2, 2000)");
+ Thread.sleep(500); // wait for async refresh to complete
+ // trigger a lookup to check async completion and switch to new partition
+ sql("INSERT INTO T VALUES (1), (2)");
+ iterator.collect(2);
+ Thread.sleep(500);
+ sql("INSERT INTO T VALUES (1), (2)");
+ result = iterator.collect(2);
+ assertThat(result).containsExactlyInAnyOrder(Row.of(1, 1000), Row.of(2, 2000));
+
+ // insert another new partition '3' and verify switch again
+ sql("INSERT INTO PARTITIONED_DIM VALUES ('3', 1, 10000), ('3', 2, 20000)");
+ Thread.sleep(500); // wait for async refresh to complete
+ sql("INSERT INTO T VALUES (1), (2)");
+ iterator.collect(2);
+ Thread.sleep(500);
+ sql("INSERT INTO T VALUES (1), (2)");
+ result = iterator.collect(2);
+ assertThat(result).containsExactlyInAnyOrder(Row.of(1, 10000), Row.of(2, 20000));
+
+ iterator.close();
+ }
+
+ @ParameterizedTest
+ @EnumSource(
+ value = LookupCacheMode.class,
+ names = {"FULL", "MEMORY"})
+ public void testAsyncPartitionRefreshServesOldDataDuringRefresh(LookupCacheMode mode)
+ throws Exception {
+ // Verify that during async refresh, queries still return old partition data
+ // until the new partition is fully loaded and switched.
+ sql(
+ "CREATE TABLE PARTITIONED_DIM (pt STRING, k INT, v INT, PRIMARY KEY (pt, k) NOT ENFORCED)"
+ + "PARTITIONED BY (`pt`) WITH ("
+ + "'bucket' = '1', "
+ + "'lookup.dynamic-partition' = 'max_pt()', "
+ + "'lookup.dynamic-partition.refresh-interval' = '1 ms', "
+ + "'lookup.dynamic-partition.refresh.async' = 'true', "
+ + "'lookup.cache' = '%s', "
+ + "'continuous.discovery-interval'='1 ms')",
+ mode);
+
+ sql("INSERT INTO PARTITIONED_DIM VALUES ('1', 1, 100), ('1', 2, 200)");
+
+ String query =
+ "SELECT T.i, D.v FROM T LEFT JOIN PARTITIONED_DIM "
+ + "for system_time as of T.proctime AS D ON T.i = D.k";
+ BlockingIterator<Row, Row> iterator = BlockingIterator.of(sEnv.executeSql(query).collect());
+
+ sql("INSERT INTO T VALUES (1), (2)");
+ List<Row> result = iterator.collect(2);
+ assertThat(result).containsExactlyInAnyOrder(Row.of(1, 100), Row.of(2, 200));
+
+ // insert new partition '2' to trigger async refresh
+ sql("INSERT INTO PARTITIONED_DIM VALUES ('2', 1, 1000), ('2', 2, 2000)");
+
+ // immediately query before async refresh completes — should still return old partition data
+ sql("INSERT INTO T VALUES (1), (2)");
+ result = iterator.collect(2);
+ // old partition data (100, 200) should still be served
+ assertThat(result).containsExactlyInAnyOrder(Row.of(1, 100), Row.of(2, 200));
+
+ // now wait for async refresh to complete and trigger switch
+ Thread.sleep(500);
+ sql("INSERT INTO T VALUES (1), (2)");
+ result = iterator.collect(2);
+ // after switch, new partition data should be returned
+ assertThat(result).containsExactlyInAnyOrder(Row.of(1, 1000), Row.of(2, 2000));
+
+ iterator.close();
+ }
+
+ @Test
+ public void testAsyncPartitionRefreshWithMultiPartitionKeys() throws Exception {
+ // Verify async partition refresh works correctly with multi-level partition keys.
+ sql(
+ "CREATE TABLE PARTITIONED_DIM (pt1 STRING, pt2 INT, k INT, v INT, PRIMARY KEY (pt1, pt2, k) NOT ENFORCED)"
+ + "PARTITIONED BY (`pt1`, `pt2`) WITH ("
+ + "'bucket' = '1', "
+ + "'scan.partitions' = 'pt1=max_pt()', "
+ + "'lookup.dynamic-partition.refresh-interval' = '1 ms', "
+ + "'lookup.dynamic-partition.refresh.async' = 'true', "
+ + "'lookup.cache' = '%s', "
+ + "'continuous.discovery-interval'='1 ms')",
+ LookupCacheMode.FULL);
+
+ sql(
+ "INSERT INTO PARTITIONED_DIM VALUES "
+ + "('2024', 1, 1, 100), ('2024', 1, 2, 200), "
+ + "('2024', 2, 1, 300), ('2024', 2, 2, 400)");
+
+ String query =
+ "SELECT D.pt1, D.pt2, T.i, D.v FROM T LEFT JOIN PARTITIONED_DIM "
+ + "for system_time as of T.proctime AS D ON T.i = D.k";
+ BlockingIterator<Row, Row> iterator = BlockingIterator.of(sEnv.executeSql(query).collect());
+
+ sql("INSERT INTO T VALUES (1), (2)");
+ List<Row> result = iterator.collect(4);
+ assertThat(result)
+ .containsExactlyInAnyOrder(
+ Row.of("2024", 1, 1, 100),
+ Row.of("2024", 1, 2, 200),
+ Row.of("2024", 2, 1, 300),
+ Row.of("2024", 2, 2, 400));
+
+ // insert new max partition '2025' with sub-partitions
+ sql(
+ "INSERT INTO PARTITIONED_DIM VALUES "
+ + "('2025', 1, 1, 1000), ('2025', 1, 2, 2000), "
+ + "('2025', 2, 1, 3000), ('2025', 2, 2, 4000)");
+ Thread.sleep(500);
+ sql("INSERT INTO T VALUES (1), (2)");
+ iterator.collect(4);
+ Thread.sleep(500);
+ sql("INSERT INTO T VALUES (1), (2)");
+ result = iterator.collect(4);
+ assertThat(result)
+ .containsExactlyInAnyOrder(
+ Row.of("2025", 1, 1, 1000),
+ Row.of("2025", 1, 2, 2000),
+ Row.of("2025", 2, 1, 3000),
+ Row.of("2025", 2, 2, 4000));
+
+ iterator.close();
+ }
+
+ @Test
+ public void testAsyncPartitionRefreshWithOverwrite() throws Exception {
+ // Verify async partition refresh works correctly when a new max partition
+ // is created via INSERT OVERWRITE.
+ sql(
+ "CREATE TABLE PARTITIONED_DIM (pt INT, k INT, v INT, PRIMARY KEY (pt, k) NOT ENFORCED)"
+ + "PARTITIONED BY (`pt`) WITH ("
+ + "'bucket' = '1', "
+ + "'lookup.dynamic-partition' = 'max_pt()', "
+ + "'lookup.dynamic-partition.refresh-interval' = '1 ms', "
+ + "'lookup.dynamic-partition.refresh.async' = 'true', "
+ + "'lookup.cache' = '%s', "
+ + "'continuous.discovery-interval'='1 ms')",
+ LookupCacheMode.FULL);
+
+ sql("INSERT INTO PARTITIONED_DIM VALUES (1, 1, 100), (1, 2, 200)");
+
+ String query =
+ "SELECT T.i, D.v FROM T LEFT JOIN PARTITIONED_DIM "
+ + "for system_time as of T.proctime AS D ON T.i = D.k";
+ BlockingIterator<Row, Row> iterator = BlockingIterator.of(sEnv.executeSql(query).collect());
+
+ sql("INSERT INTO T VALUES (1), (2)");
+ List<Row> result = iterator.collect(2);
+ assertThat(result).containsExactlyInAnyOrder(Row.of(1, 100), Row.of(2, 200));
+
+ // overwrite current max partition with new data
+ sql("INSERT OVERWRITE PARTITIONED_DIM PARTITION (pt = 1) VALUES (1, 150), (2, 250)");
+ Thread.sleep(500);
+ sql("INSERT INTO T VALUES (1), (2)");
+ result = iterator.collect(2);
+ assertThat(result).containsExactlyInAnyOrder(Row.of(1, 150), Row.of(2, 250));
+
+ // overwrite to create a new max partition
+ sql(
+ "INSERT OVERWRITE PARTITIONED_DIM PARTITION (pt = 2) VALUES (1, 1000), (2, 2000), (3, 3000)");
+ Thread.sleep(500);
+ sql("INSERT INTO T VALUES (1), (2), (3)");
+ iterator.collect(3);
+ Thread.sleep(500);
+ sql("INSERT INTO T VALUES (1), (2), (3)");
+ result = iterator.collect(3);
+ assertThat(result)
+ .containsExactlyInAnyOrder(Row.of(1, 1000), Row.of(2, 2000), Row.of(3, 3000));
+
+ iterator.close();
+ }
+
+ @Test
+ public void testAsyncPartitionRefreshWithMaxTwoPt() throws Exception {
+ // Verify async partition refresh works correctly with max_two_pt() strategy.
+ sql(
+ "CREATE TABLE TWO_PT_DIM (pt STRING, k INT, v INT, PRIMARY KEY (pt, k) NOT ENFORCED)"
+ + "PARTITIONED BY (`pt`) WITH ("
+ + "'bucket' = '1', "
+ + "'lookup.dynamic-partition' = 'max_two_pt()', "
+ + "'lookup.dynamic-partition.refresh-interval' = '1 ms', "
+ + "'lookup.dynamic-partition.refresh.async' = 'true', "
+ + "'lookup.cache' = '%s', "
+ + "'continuous.discovery-interval'='1 ms')",
+ LookupCacheMode.FULL);
+
+ // insert data into partitions '1' and '2'
+ sql(
+ "INSERT INTO TWO_PT_DIM VALUES "
+ + "('1', 1, 100), ('1', 2, 200), "
+ + "('2', 1, 300), ('2', 2, 400)");
+
+ String query =
+ "SELECT D.pt, T.i, D.v FROM T LEFT JOIN TWO_PT_DIM "
+ + "for system_time as of T.proctime AS D ON T.i = D.k";
+ BlockingIterator<Row, Row> iterator = BlockingIterator.of(sEnv.executeSql(query).collect());
+
+ sql("INSERT INTO T VALUES (1), (2)");
+ List<Row> result = iterator.collect(4);
+ assertThat(result)
+ .containsExactlyInAnyOrder(
+ Row.of("1", 1, 100),
+ Row.of("1", 2, 200),
+ Row.of("2", 1, 300),
+ Row.of("2", 2, 400));
+
+ // insert new partition '3', now max_two_pt should be '2' and '3'
+ sql("INSERT INTO TWO_PT_DIM VALUES " + "('3', 1, 1000), ('3', 2, 2000)");
+ sql("INSERT INTO T VALUES (1), (2)");
+ iterator.collect(4);
+ Thread.sleep(500);
+ sql("INSERT INTO T VALUES (1), (2)");
+ result = iterator.collect(4);
+ // should now see data from partitions '2' and '3'
+ assertThat(result)
+ .containsExactlyInAnyOrder(
+ Row.of("2", 1, 300),
+ Row.of("2", 2, 400),
+ Row.of("3", 1, 1000),
+ Row.of("3", 2, 2000));
+
+ // insert another partition '4', max_two_pt should be '3' and '4'
+ sql("INSERT INTO TWO_PT_DIM VALUES " + "('4', 1, 10000), ('4', 2, 20000)");
+ sql("INSERT INTO T VALUES (1), (2)");
+ iterator.collect(4);
+ Thread.sleep(500);
+ sql("INSERT INTO T VALUES (1), (2)");
+ result = iterator.collect(4);
+ assertThat(result)
+ .containsExactlyInAnyOrder(
+ Row.of("3", 1, 1000),
+ Row.of("3", 2, 2000),
+ Row.of("4", 1, 10000),
+ Row.of("4", 2, 20000));
+
+ iterator.close();
+ }
+
+ @ParameterizedTest
+ @EnumSource(
+ value = LookupCacheMode.class,
+ names = {"FULL", "MEMORY"})
+ public void testAsyncPartitionRefreshWithNonPkTable(LookupCacheMode mode) throws Exception {
+ // Verify async partition refresh works correctly with non-primary-key append tables.
+ sql(
+ "CREATE TABLE NON_PK_DIM (pt STRING, k INT, v INT)"
+ + "PARTITIONED BY (`pt`) WITH ("
+ + "'lookup.dynamic-partition' = 'max_pt()', "
+ + "'lookup.dynamic-partition.refresh-interval' = '1 ms', "
+ + "'lookup.dynamic-partition.refresh.async' = 'true', "
+ + "'lookup.cache' = '%s', "
+ + "'continuous.discovery-interval'='1 ms')",
+ mode);
+
+ sql("INSERT INTO NON_PK_DIM VALUES ('1', 1, 100), ('1', 1, 101), ('1', 2, 200)");
+
+ String query =
+ "SELECT T.i, D.v FROM T LEFT JOIN NON_PK_DIM "
+ + "for system_time as of T.proctime AS D ON T.i = D.k";
+ BlockingIterator<Row, Row> iterator = BlockingIterator.of(sEnv.executeSql(query).collect());
+
+ sql("INSERT INTO T VALUES (1), (2)");
+ List<Row> result = iterator.collect(3);
+ // non-pk table may return multiple matches
+ assertThat(result)
+ .containsExactlyInAnyOrder(Row.of(1, 100), Row.of(1, 101), Row.of(2, 200));
+
+ // insert new partition '2' to trigger async refresh
+ sql("INSERT INTO NON_PK_DIM VALUES ('2', 1, 1000), ('2', 1, 1001), ('2', 2, 2000)");
+ sql("INSERT INTO T VALUES (1), (2)");
+ iterator.collect(3);
+ Thread.sleep(500);
+ sql("INSERT INTO T VALUES (1), (2)");
+ result = iterator.collect(3);
+ assertThat(result)
+ .containsExactlyInAnyOrder(Row.of(1, 1000), Row.of(1, 1001), Row.of(2, 2000));
+
+ iterator.close();
+ }
}
(paimon) 35/37: [python] Support rename branch api (#7561)
Posted by ho...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hope pushed a commit to branch release-1.4
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit a6a47a89a5fdac39cbfca50221e5f40d227531c7
Author: xuzifu666 <12...@qq.com>
AuthorDate: Tue Mar 31 16:27:42 2026 +0800
[python] Support rename branch api (#7561)
---
docs/content/pypaimon/python-api.md | 13 ++++++
paimon-python/pypaimon/branch/branch_manager.py | 13 ++++++
.../pypaimon/branch/catalog_branch_manager.py | 16 +++++++
.../pypaimon/branch/filesystem_branch_manager.py | 54 ++++++++++++++++++++++
paimon-python/pypaimon/table/file_store_table.py | 15 ++++++
.../pypaimon/tests/table/file_store_table_test.py | 46 ++++++++++++++++++
6 files changed, 157 insertions(+)
diff --git a/docs/content/pypaimon/python-api.md b/docs/content/pypaimon/python-api.md
index 7cdd86d731..197a018ef1 100644
--- a/docs/content/pypaimon/python-api.md
+++ b/docs/content/pypaimon/python-api.md
@@ -915,6 +915,19 @@ Delete an existing branch:
table.branch_manager().drop_branch('feature_branch')
```
+### Rename Branch
+
+Rename an existing branch to a new name:
+
+```python
+# Rename a branch
+table.branch_manager().rename_branch('old_branch_name', 'new_branch_name')
+```
+
+{{< hint warning >}}
+The source branch must exist and cannot be the main branch. The target branch name must be valid and not already exist.
+{{< /hint >}}
+
### Fast Forward
Fast forward the main branch to a specific branch:
diff --git a/paimon-python/pypaimon/branch/branch_manager.py b/paimon-python/pypaimon/branch/branch_manager.py
index cb2b1e87ad..fbd5fede38 100644
--- a/paimon-python/pypaimon/branch/branch_manager.py
+++ b/paimon-python/pypaimon/branch/branch_manager.py
@@ -65,6 +65,19 @@ class BranchManager:
"""
raise NotImplementedError("Subclasses must implement drop_branch")
+ def rename_branch(self, from_branch: str, to_branch: str) -> None:
+ """
+ Rename a branch.
+
+ Args:
+ from_branch: Current name of the branch
+ to_branch: New name for the branch
+
+ Raises:
+ NotImplementedError: Subclasses must implement this method
+ """
+ raise NotImplementedError("Subclasses must implement rename_branch")
+
def fast_forward(self, branch_name: str) -> None:
"""
Fast forward the current branch to the specified branch.
diff --git a/paimon-python/pypaimon/branch/catalog_branch_manager.py b/paimon-python/pypaimon/branch/catalog_branch_manager.py
index a70c6ace99..1cffc60f4e 100644
--- a/paimon-python/pypaimon/branch/catalog_branch_manager.py
+++ b/paimon-python/pypaimon/branch/catalog_branch_manager.py
@@ -121,6 +121,22 @@ class CatalogBranchManager(BranchManager):
self._execute_post(_drop)
+ def rename_branch(self, from_branch: str, to_branch: str) -> None:
+ """
+ Rename a branch.
+
+ Args:
+ from_branch: Current name of the branch
+ to_branch: New name for the branch
+
+ Raises:
+ ValueError: If from_branch or to_branch is invalid
+ """
+ def _rename(catalog: Catalog):
+ catalog.rename_branch(self.identifier, from_branch, to_branch)
+
+ self._execute_post(_rename)
+
def fast_forward(self, branch_name: str) -> None:
"""
Fast forward the current branch to the specified branch.
diff --git a/paimon-python/pypaimon/branch/filesystem_branch_manager.py b/paimon-python/pypaimon/branch/filesystem_branch_manager.py
index e5c2982c39..ca8111b62f 100644
--- a/paimon-python/pypaimon/branch/filesystem_branch_manager.py
+++ b/paimon-python/pypaimon/branch/filesystem_branch_manager.py
@@ -199,6 +199,60 @@ class FileSystemBranchManager(BranchManager):
)
raise RuntimeError(f"Failed to delete branch '{branch_name}'") from e
+ def rename_branch(self, from_branch: str, to_branch: str) -> None:
+ """
+ Rename a branch.
+
+ Args:
+ from_branch: Current name of the branch
+ to_branch: New name for the branch
+
+ Raises:
+ ValueError: If from_branch or to_branch is blank, from_branch doesn't exist,
+ to_branch already exists, or trying to rename the main branch
+ RuntimeError: If the rename operation fails
+ """
+ # Check if from_branch is main branch
+ if self.is_main_branch(from_branch):
+ raise ValueError(f"Cannot rename the main branch '{from_branch}'.")
+
+ # Check if from_branch is blank
+ if not from_branch or from_branch.isspace():
+ raise ValueError("Source branch name shouldn't be blank.")
+
+ # Check if to_branch is blank
+ if not to_branch or to_branch.isspace():
+ raise ValueError("Target branch name shouldn't be blank.")
+
+ # Validate the new branch name
+ try:
+ self.validate_branch(to_branch)
+ except ValueError as e:
+ raise ValueError(f"Invalid target branch name: {e}")
+
+ # Check if from_branch exists
+ if not self.branch_exists(from_branch):
+ raise ValueError(f"Source branch '{from_branch}' doesn't exist.")
+
+ # Check if to_branch already exists
+ if self.branch_exists(to_branch):
+ raise ValueError(f"Target branch '{to_branch}' already exists.")
+
+ try:
+ # Rename the branch directory
+ from_path = self.branch_path(from_branch)
+ to_path = self.branch_path(to_branch)
+ self.file_io.rename(from_path, to_path)
+ logger.info(f"Successfully renamed branch from '{from_branch}' to '{to_branch}'")
+ except Exception as e:
+ logger.warning(
+ f"Renaming branch from '{from_branch}' to '{to_branch}' failed due to an exception. "
+ f"Please try again."
+ )
+ raise RuntimeError(
+ f"Failed to rename branch from '{from_branch}' to '{to_branch}'"
+ ) from e
+
def fast_forward(self, branch_name: str) -> None:
"""
Fast forward the current branch to the specified branch.
diff --git a/paimon-python/pypaimon/table/file_store_table.py b/paimon-python/pypaimon/table/file_store_table.py
index d725d48021..9115c8e885 100644
--- a/paimon-python/pypaimon/table/file_store_table.py
+++ b/paimon-python/pypaimon/table/file_store_table.py
@@ -132,6 +132,21 @@ class FileStoreTable(Table):
from pypaimon.changelog.changelog_manager import ChangelogManager
return ChangelogManager(self.file_io, self.table_path, self.current_branch())
+ def rename_branch(self, from_branch: str, to_branch: str) -> None:
+ """
+ Rename a branch.
+
+ Args:
+ from_branch: Current name of the branch
+ to_branch: New name for the branch
+
+ Raises:
+ ValueError: If from_branch or to_branch is blank, from_branch doesn't exist,
+ or to_branch already exists
+ """
+ branch_mgr = self.branch_manager()
+ branch_mgr.rename_branch(from_branch, to_branch)
+
def create_tag(
self,
tag_name: str,
diff --git a/paimon-python/pypaimon/tests/table/file_store_table_test.py b/paimon-python/pypaimon/tests/table/file_store_table_test.py
index 72c1ee7e2c..22068d3a01 100644
--- a/paimon-python/pypaimon/tests/table/file_store_table_test.py
+++ b/paimon-python/pypaimon/tests/table/file_store_table_test.py
@@ -339,6 +339,52 @@ class FileStoreTableTest(unittest.TestCase):
self.assertEqual(copied_table.identifier, self.table.identifier)
self.assertEqual(copied_table.table_path, self.table.table_path)
+ def test_rename_branch_basic(self):
+ """Test rename_branch method."""
+ # Get branch_manager
+ branch_manager = self.table.branch_manager()
+
+ # Create a branch first
+ branch_manager.create_branch("old-branch")
+ self.assertTrue(branch_manager.branch_exists("old-branch"))
+
+ # Rename the branch using table's rename_branch method
+ self.table.rename_branch("old-branch", "new-branch")
+
+ # Verify old branch doesn't exist
+ self.assertFalse(branch_manager.branch_exists("old-branch"))
+
+ # Verify new branch exists
+ self.assertTrue(branch_manager.branch_exists("new-branch"))
+
+ def test_rename_branch_from_nonexistent(self):
+ """Test renaming from non-existent branch raises error."""
+ with self.assertRaises(ValueError) as context:
+ self.table.rename_branch("nonexistent-branch", "new-branch")
+
+ self.assertIn("doesn't exist", str(context.exception))
+
+ def test_rename_branch_to_existing(self):
+ """Test renaming to existing branch raises error."""
+ # Get branch_manager
+ branch_manager = self.table.branch_manager()
+
+ # Create two branches
+ branch_manager.create_branch("branch1")
+ branch_manager.create_branch("branch2")
+
+ with self.assertRaises(ValueError) as context:
+ self.table.rename_branch("branch1", "branch2")
+
+ self.assertIn("already exists", str(context.exception))
+
+ def test_rename_main_branch_fails(self):
+ """Test renaming main branch raises error."""
+ with self.assertRaises(ValueError) as context:
+ self.table.rename_branch("main", "new-branch")
+
+ self.assertIn("main branch", str(context.exception))
+
def test_comment_none(self):
"""Test that comment() returns None when table has no comment."""
# Default table created without comment should return None
(paimon) 37/37: [spark] Fix view resolution for CTE and ORDER BY ordinal (#7552)
Posted by ho...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hope pushed a commit to branch release-1.4
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit f509a8fe15bc7e5b7db40923f3c1d283eb498a73
Author: Zouxxyy <zo...@alibaba-inc.com>
AuthorDate: Tue Mar 31 21:07:29 2026 +0800
[spark] Fix view resolution for CTE and ORDER BY ordinal (#7552)
Fix PaimonViewResolver to apply early analysis rules (CTESubstitution,
SubstituteUnresolvedOrdinals) to parsed view plans. These rules run in
Spark's earlyBatches before the Resolution batch, so they won't re-run
for plans injected by PaimonViewResolver during resolution.
Without this fix:
- Views with CTE (`WITH t AS ...`) fail with `TABLE_OR_VIEW_NOT_FOUND`
- Views with `ORDER BY 1` (ordinal) produce incorrect results
---
.../catalyst/analysis/PaimonViewResolver.scala | 8 +++--
.../paimon/spark/sql/PaimonViewTestBase.scala | 38 ++++++++++++++++++++++
2 files changed, 44 insertions(+), 2 deletions(-)
diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/analysis/PaimonViewResolver.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/analysis/PaimonViewResolver.scala
index 45de6a1c76..b45463e916 100644
--- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/analysis/PaimonViewResolver.scala
+++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/analysis/PaimonViewResolver.scala
@@ -24,7 +24,7 @@ import org.apache.paimon.spark.catalog.SupportView
import org.apache.paimon.view.View
import org.apache.spark.sql.SparkSession
-import org.apache.spark.sql.catalyst.analysis.{GetColumnByOrdinal, UnresolvedRelation, UnresolvedTableOrView}
+import org.apache.spark.sql.catalyst.analysis.{CTESubstitution, GetColumnByOrdinal, SubstituteUnresolvedOrdinals, UnresolvedRelation, UnresolvedTableOrView}
import org.apache.spark.sql.catalyst.expressions.{Alias, Attribute, UpCast}
import org.apache.spark.sql.catalyst.parser.ParseException
import org.apache.spark.sql.catalyst.parser.extensions.{CurrentOrigin, Origin}
@@ -62,6 +62,10 @@ case class PaimonViewResolver(spark: SparkSession)
val parsedPlan =
parseViewText(nameParts.toArray.mkString("."), view.query(SupportView.DIALECT))
+ // Apply early analysis rules that won't re-run for plans injected during Resolution batch.
+ val earlyRules = Seq(CTESubstitution, SubstituteUnresolvedOrdinals)
+ val rewritten = earlyRules.foldLeft(parsedPlan)((plan, rule) => rule.apply(plan))
+
val aliases = SparkTypeUtils.fromPaimonRowType(view.rowType()).fields.zipWithIndex.map {
case (expected, pos) =>
val attr = GetColumnByOrdinal(pos, expected.dataType)
@@ -69,7 +73,7 @@ case class PaimonViewResolver(spark: SparkSession)
Some(expected.metadata))
}
- SubqueryAlias(nameParts, Project(aliases, parsedPlan))
+ SubqueryAlias(nameParts, Project(aliases, rewritten))
}
private def parseViewText(name: String, viewText: String): LogicalPlan = {
diff --git a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/PaimonViewTestBase.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/PaimonViewTestBase.scala
index b276af89b5..7c503d98a5 100644
--- a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/PaimonViewTestBase.scala
+++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/PaimonViewTestBase.scala
@@ -204,4 +204,42 @@ abstract class PaimonViewTestBase extends PaimonHiveTestBase {
}
}
}
+
+ test("Paimon View: create view with CTE") {
+ Seq(sparkCatalogName, paimonHiveCatalogName).foreach {
+ catalogName =>
+ sql(s"USE $catalogName")
+ withDatabase("test_db") {
+ sql("CREATE DATABASE test_db")
+ sql("USE test_db")
+ withView("v1") {
+ sql("""
+ |CREATE VIEW v1 AS
+ | WITH t(a, b, c, d) AS (SELECT 1, 2, 3, 4)
+ | SELECT * FROM t
+ |""".stripMargin)
+ checkAnswer(sql("SELECT * FROM v1"), Seq(Row(1, 2, 3, 4)))
+ }
+ }
+ }
+ }
+
+ test("Paimon View: create view with ORDER BY ordinal") {
+ Seq(sparkCatalogName, paimonHiveCatalogName).foreach {
+ catalogName =>
+ sql(s"USE $catalogName")
+ withDatabase("test_db") {
+ sql("CREATE DATABASE test_db")
+ sql("USE test_db")
+ withTable("t") {
+ withView("v_ord") {
+ sql("CREATE TABLE t (id INT, name STRING) USING paimon")
+ sql("INSERT INTO t VALUES (2, 'b'), (1, 'a')")
+ sql("CREATE VIEW v_ord AS SELECT * FROM t ORDER BY 1")
+ checkAnswer(sql("SELECT * FROM v_ord"), Seq(Row(1, "a"), Row(2, "b")))
+ }
+ }
+ }
+ }
+ }
}