You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@paimon.apache.org by lz...@apache.org on 2025/11/11 05:41:02 UTC

(paimon) branch release-1.3 updated (26dfce04e3 -> 1390842f9b)

This is an automated email from the ASF dual-hosted git repository.

lzljs3620320 pushed a change to branch release-1.3
in repository https://gitbox.apache.org/repos/asf/paimon.git


    from 26dfce04e3 [doc] Fix error link in views page
     new 75b44862bb [core] Support non null column with write type (#6513)
     new 3d09deb93d [hotfix] Add more informat to check partition spec in InternalRowPartitionComputer
     new 81e24ab09a [hotfix] Print partition spec and type when error in InternalRowPartitionComputer
     new f97750d196 [Python] Keep the variable names of Identifier consistent with Java (#6520)
     new 134e1b6976 [Python] Suppport multi prepare commit in the same TableWrite  (#6526)
     new 47c521d3a7 [Python] Rename to BATCH_COMMIT_IDENTIFIER in snapshot.py
     new e6ea490bf9 [python] support custom source split target size and split open file cost (#6527)
     new e233be189e [core] Fix that cannot get partition info if all files are in level-0 when enable dv. (#6531)
     new 048c0d4d83 [python] add test case for reading blob by to_iterator (#6536)
     new e9f4b2f3c3 [doc] fix postpone.default-bucket-num default value (#6554)
     new e4ecfac05c [Python] Remove the use of reference types in DATA_FILE_META_SCHEMA (#6549)
     new 4b16ee2131 [common] Remove token expire time (#6544)
     new 787210fbea [python] Fix AtomicType.to_dict() inconsistency with java (#6548)
     new ae57450853 [python] Fix File Source type in data file meta (#6571)
     new ce4ebcdafe [Python] License added in test (#6572)
     new a47a28ec38 [core] support incremental clustering in dv mode (#6559)
     new 5360c0323f [core] Refactor deletion vectors support for incremental cluster
     new 1390842f9b [Python] Add Mixed read and write test between Java and Python. (#6579)

The 18 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 .github/workflows/paimon-python-checks.yml         |  28 ++-
 .../content/primary-key-table/data-distribution.md |   2 +-
 .../org/apache/paimon/rest/RESTTokenFileIO.java    |   2 -
 .../paimon/utils/InternalRowPartitionComputer.java |   6 +-
 .../append/cluster/HistoryPartitionCluster.java    |  10 +-
 .../append/cluster/IncrementalClusterManager.java  | 120 ++++++++++--
 .../org/apache/paimon/schema/SchemaValidation.java |   3 -
 .../apache/paimon/table/sink/TableWriteImpl.java   |  15 +-
 .../paimon/table/system/PartitionsTable.java       |   3 +-
 .../test/java/org/apache/paimon/JavaPyE2ETest.java | 159 +++++++++++++++
 .../cluster/IncrementalClusterManagerTest.java     |   7 +-
 .../paimon/table/DataEvolutionTableTest.java       |  41 ++++
 .../apache/paimon/flink/action/CompactAction.java  |  26 ++-
 .../cluster/IncrementalClusterSplitSource.java     |  18 +-
 .../cluster/RemoveClusterBeforeFilesOperator.java  |  23 ++-
 .../apache/paimon/flink/BatchFileStoreITCase.java  |  10 +
 .../action/IncrementalClusterActionITCase.java     | 142 ++++++++++++++
 paimon-python/dev/lint-python.sh                   |  49 ++++-
 paimon-python/dev/run_mixed_tests.sh               | 217 +++++++++++++++++++++
 paimon-python/pypaimon/api/rest_api.py             |  16 +-
 paimon-python/pypaimon/common/core_options.py      |  20 ++
 paimon-python/pypaimon/common/identifier.py        |  26 +--
 paimon-python/pypaimon/common/memory_size.py       | 201 +++++++++++++++++++
 .../pypaimon/manifest/schema/data_file_meta.py     |  10 +-
 .../pypaimon/manifest/schema/manifest_file_meta.py |   4 +-
 .../pypaimon/manifest/schema/simple_stats.py       |  44 +++--
 .../pypaimon/read/scanner/full_starting_scanner.py |   5 +-
 paimon-python/pypaimon/schema/data_types.py        |   8 +-
 .../pypaimon/snapshot/catalog_snapshot_commit.py   |   6 +-
 paimon-python/pypaimon/snapshot/snapshot.py        |   2 +
 paimon-python/pypaimon/table/file_store_table.py   |   5 +-
 paimon-python/pypaimon/table/table.py              |   6 +-
 paimon-python/pypaimon/tests/blob_table_test.py    |  80 ++++++++
 paimon-python/pypaimon/tests/data_types_test.py    |  67 +++++++
 .../pypaimon/{api => tests/e2e}/__init__.py        |   0
 .../pypaimon/tests/e2e/java_py_read_write_test.py  |  89 +++++++++
 .../{catalog => tests/manifest}/__init__.py        |   0
 .../tests/manifest/manifest_schema_test.py         | 151 ++++++++++++++
 .../pypaimon/tests/py36/ao_simple_test.py          |  56 ++++++
 paimon-python/pypaimon/tests/reader_base_test.py   | 132 +++++++++++++
 paimon-python/pypaimon/tests/rest/rest_server.py   |  18 +-
 .../pypaimon/tests/write/table_write_test.py       | 155 +++++++++++++++
 paimon-python/pypaimon/write/file_store_commit.py  |   2 +-
 paimon-python/pypaimon/write/file_store_write.py   |   5 +-
 .../{batch_table_commit.py => table_commit.py}     |  19 +-
 .../write/{batch_table_write.py => table_write.py} |  29 ++-
 .../{batch_write_builder.py => write_builder.py}   |  35 +++-
 .../pypaimon/write/writer/data_blob_writer.py      |   2 +-
 paimon-python/pypaimon/write/writer/data_writer.py |   2 +-
 .../paimon/spark/procedure/CompactProcedure.java   |  39 ++--
 .../spark/procedure/CompactProcedureTestBase.scala | 136 +++++++++++++
 51 files changed, 2081 insertions(+), 170 deletions(-)
 mode change 100644 => 100755 .github/workflows/paimon-python-checks.yml
 create mode 100644 paimon-core/src/test/java/org/apache/paimon/JavaPyE2ETest.java
 create mode 100755 paimon-python/dev/run_mixed_tests.sh
 mode change 100644 => 100755 paimon-python/pypaimon/api/rest_api.py
 mode change 100644 => 100755 paimon-python/pypaimon/common/identifier.py
 create mode 100644 paimon-python/pypaimon/common/memory_size.py
 mode change 100644 => 100755 paimon-python/pypaimon/schema/data_types.py
 mode change 100644 => 100755 paimon-python/pypaimon/snapshot/catalog_snapshot_commit.py
 create mode 100755 paimon-python/pypaimon/tests/data_types_test.py
 copy paimon-python/pypaimon/{api => tests/e2e}/__init__.py (100%)
 create mode 100644 paimon-python/pypaimon/tests/e2e/java_py_read_write_test.py
 copy paimon-python/pypaimon/{catalog => tests/manifest}/__init__.py (100%)
 create mode 100644 paimon-python/pypaimon/tests/manifest/manifest_schema_test.py
 mode change 100644 => 100755 paimon-python/pypaimon/tests/rest/rest_server.py
 create mode 100644 paimon-python/pypaimon/tests/write/table_write_test.py
 rename paimon-python/pypaimon/write/{batch_table_commit.py => table_commit.py} (86%)
 rename paimon-python/pypaimon/write/{batch_table_write.py => table_write.py} (89%)
 rename paimon-python/pypaimon/write/{batch_write_builder.py => write_builder.py} (73%)


(paimon) 17/18: [core] Refactor deletion vectors support for incremental cluster

Posted by lz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

lzljs3620320 pushed a commit to branch release-1.3
in repository https://gitbox.apache.org/repos/asf/paimon.git

commit 5360c0323f5aebcf18cf6aae62b1268f0a147132
Author: JingsongLi <ji...@gmail.com>
AuthorDate: Mon Nov 10 18:23:59 2025 +0800

    [core] Refactor deletion vectors support for incremental cluster
---
 .../append/cluster/IncrementalClusterManager.java  | 159 ++++++++++-----------
 .../apache/paimon/flink/action/CompactAction.java  |  40 ++----
 .../cluster/IncrementalClusterSplitSource.java     |  18 +--
 .../cluster/RemoveClusterBeforeFilesOperator.java  |  15 +-
 .../paimon/spark/procedure/CompactProcedure.java   |  54 ++-----
 5 files changed, 114 insertions(+), 172 deletions(-)

diff --git a/paimon-core/src/main/java/org/apache/paimon/append/cluster/IncrementalClusterManager.java b/paimon-core/src/main/java/org/apache/paimon/append/cluster/IncrementalClusterManager.java
index f8ab6f364b..20dc2a49d2 100644
--- a/paimon-core/src/main/java/org/apache/paimon/append/cluster/IncrementalClusterManager.java
+++ b/paimon-core/src/main/java/org/apache/paimon/append/cluster/IncrementalClusterManager.java
@@ -43,6 +43,7 @@ import org.apache.paimon.table.source.DeletionFile;
 import org.apache.paimon.table.source.SplitGenerator;
 import org.apache.paimon.table.source.snapshot.SnapshotReader;
 import org.apache.paimon.utils.InternalRowPartitionComputer;
+import org.apache.paimon.utils.Pair;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -56,7 +57,6 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
-import java.util.Set;
 import java.util.stream.Collectors;
 
 import static org.apache.paimon.CoreOptions.CLUSTERING_INCREMENTAL;
@@ -67,6 +67,7 @@ public class IncrementalClusterManager {
 
     private static final Logger LOG = LoggerFactory.getLogger(IncrementalClusterManager.class);
 
+    private final FileStoreTable table;
     private final InternalRowPartitionComputer partitionComputer;
     private final Snapshot snapshot;
     private final SnapshotReader snapshotReader;
@@ -85,6 +86,7 @@ public class IncrementalClusterManager {
         checkArgument(
                 table.bucketMode() == BucketMode.BUCKET_UNAWARE,
                 "only append unaware-bucket table support incremental clustering.");
+        this.table = table;
         CoreOptions options = table.coreOptions();
         checkArgument(
                 options.clusteringIncrementalEnabled(),
@@ -204,41 +206,82 @@ public class IncrementalClusterManager {
         return partitionLevels;
     }
 
-    public List<DataSplit> toSplits(
-            BinaryRow partition,
-            List<DataFileMeta> files,
-            @Nullable AppendDeleteFileMaintainer dvIndexFileMaintainer) {
-        List<DataSplit> splits = new ArrayList<>();
-
-        DataSplit.Builder builder =
-                DataSplit.builder()
-                        .withPartition(partition)
-                        .withBucket(0)
-                        .withTotalBuckets(1)
-                        .isStreaming(false);
-
-        SplitGenerator splitGenerator = snapshotReader.splitGenerator();
-        List<SplitGenerator.SplitGroup> splitGroups = splitGenerator.splitForBatch(files);
-
-        for (SplitGenerator.SplitGroup splitGroup : splitGroups) {
-            List<DataFileMeta> dataFiles = splitGroup.files;
-            String bucketPath = snapshotReader.pathFactory().bucketPath(partition, 0).toString();
-            builder.withDataFiles(dataFiles)
-                    .rawConvertible(splitGroup.rawConvertible)
-                    .withBucketPath(bucketPath);
-
-            if (dvIndexFileMaintainer != null) {
-                List<DeletionFile> dataDeletionFiles = new ArrayList<>();
-                for (DataFileMeta file : dataFiles) {
-                    dataDeletionFiles.add(dvIndexFileMaintainer.getDeletionFile(file.fileName()));
+    public Map<BinaryRow, Pair<List<DataSplit>, CommitMessage>> toSplitsAndRewriteDvFiles(
+            Map<BinaryRow, CompactUnit> compactUnits) {
+        Map<BinaryRow, Pair<List<DataSplit>, CommitMessage>> result = new HashMap<>();
+        boolean dvEnabled = table.coreOptions().deletionVectorsEnabled();
+        for (BinaryRow partition : compactUnits.keySet()) {
+            CompactUnit unit = compactUnits.get(partition);
+            AppendDeleteFileMaintainer dvMaintainer =
+                    dvEnabled
+                            ? BaseAppendDeleteFileMaintainer.forUnawareAppend(
+                                    table.store().newIndexFileHandler(), snapshot, partition)
+                            : null;
+            List<DataSplit> splits = new ArrayList<>();
+
+            DataSplit.Builder builder =
+                    DataSplit.builder()
+                            .withPartition(partition)
+                            .withBucket(0)
+                            .withTotalBuckets(1)
+                            .isStreaming(false);
+
+            SplitGenerator splitGenerator = snapshotReader.splitGenerator();
+            List<SplitGenerator.SplitGroup> splitGroups =
+                    splitGenerator.splitForBatch(unit.files());
+
+            for (SplitGenerator.SplitGroup splitGroup : splitGroups) {
+                List<DataFileMeta> dataFiles = splitGroup.files;
+
+                String bucketPath =
+                        snapshotReader.pathFactory().bucketPath(partition, 0).toString();
+                builder.withDataFiles(dataFiles)
+                        .rawConvertible(splitGroup.rawConvertible)
+                        .withBucketPath(bucketPath);
+
+                if (dvMaintainer != null) {
+                    List<DeletionFile> dataDeletionFiles = new ArrayList<>();
+                    for (DataFileMeta file : dataFiles) {
+                        DeletionFile deletionFile =
+                                dvMaintainer.notifyRemovedDeletionVector(file.fileName());
+                        dataDeletionFiles.add(deletionFile);
+                    }
+                    builder.withDataDeletionFiles(dataDeletionFiles);
                 }
-                builder.withDataDeletionFiles(dataDeletionFiles);
+                splits.add(builder.build());
             }
 
-            splits.add(builder.build());
+            // generate delete dv index meta
+            CommitMessage dvCommitMessage = null;
+            if (dvMaintainer != null) {
+                List<IndexFileMeta> newIndexFiles = new ArrayList<>();
+                List<IndexFileMeta> deletedIndexFiles = new ArrayList<>();
+                List<IndexManifestEntry> indexEntries = dvMaintainer.persist();
+                for (IndexManifestEntry entry : indexEntries) {
+                    if (entry.kind() == FileKind.ADD) {
+                        newIndexFiles.add(entry.indexFile());
+                    } else {
+                        deletedIndexFiles.add(entry.indexFile());
+                    }
+                }
+                dvCommitMessage =
+                        new CommitMessageImpl(
+                                dvMaintainer.getPartition(),
+                                0,
+                                table.coreOptions().bucket(),
+                                DataIncrement.emptyIncrement(),
+                                new CompactIncrement(
+                                        Collections.emptyList(),
+                                        Collections.emptyList(),
+                                        Collections.emptyList(),
+                                        newIndexFiles,
+                                        deletedIndexFiles));
+            }
+
+            result.put(partition, Pair.of(splits, dvCommitMessage));
         }
 
-        return splits;
+        return result;
     }
 
     public static List<DataFileMeta> upgrade(
@@ -248,60 +291,6 @@ public class IncrementalClusterManager {
                 .collect(Collectors.toList());
     }
 
-    public static Map<BinaryRow, AppendDeleteFileMaintainer> createAppendDvMaintainers(
-            FileStoreTable table, Set<BinaryRow> partitions, Snapshot snapshot) {
-        Map<BinaryRow, AppendDeleteFileMaintainer> dvMaintainers = new HashMap<>();
-        for (BinaryRow partition : partitions) {
-            AppendDeleteFileMaintainer appendDvMaintainer =
-                    BaseAppendDeleteFileMaintainer.forUnawareAppend(
-                            table.store().newIndexFileHandler(), snapshot, partition);
-            dvMaintainers.put(partition, appendDvMaintainer);
-        }
-        return dvMaintainers;
-    }
-
-    public static List<CommitMessage> producePartitionDvIndexCommitMessages(
-            FileStoreTable table,
-            List<DataSplit> splits,
-            AppendDeleteFileMaintainer appendDvMaintainer) {
-        checkArgument(appendDvMaintainer != null);
-        // remove deletion vector for files to be clustered
-        for (DataSplit dataSplit : splits) {
-            checkArgument(
-                    dataSplit.partition().equals(appendDvMaintainer.getPartition()),
-                    "partition of this dataSplit is not matched with the Dv Maintainer!");
-            dataSplit
-                    .dataFiles()
-                    .forEach(f -> appendDvMaintainer.notifyRemovedDeletionVector(f.fileName()));
-        }
-
-        // generate new dv index meta, handle by partition
-        List<IndexFileMeta> newIndexFiles = new ArrayList<>();
-        List<IndexFileMeta> deletedIndexFiles = new ArrayList<>();
-        List<IndexManifestEntry> indexEntries = appendDvMaintainer.persist();
-        for (IndexManifestEntry entry : indexEntries) {
-            if (entry.kind() == FileKind.ADD) {
-                newIndexFiles.add(entry.indexFile());
-            } else {
-                deletedIndexFiles.add(entry.indexFile());
-            }
-        }
-        CommitMessageImpl commitMessage =
-                new CommitMessageImpl(
-                        appendDvMaintainer.getPartition(),
-                        0,
-                        table.coreOptions().bucket(),
-                        DataIncrement.emptyIncrement(),
-                        new CompactIncrement(
-                                Collections.emptyList(),
-                                Collections.emptyList(),
-                                Collections.emptyList(),
-                                newIndexFiles,
-                                deletedIndexFiles));
-
-        return Collections.singletonList(commitMessage);
-    }
-
     public static void logForPartitionLevel(
             Map<BinaryRow, List<LevelSortedRun>> partitionLevels,
             InternalRowPartitionComputer partitionComputer) {
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/CompactAction.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/CompactAction.java
index 4a914e8ac9..42e20f5aab 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/CompactAction.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/CompactAction.java
@@ -23,7 +23,6 @@ import org.apache.paimon.append.cluster.IncrementalClusterManager;
 import org.apache.paimon.compact.CompactUnit;
 import org.apache.paimon.data.BinaryRow;
 import org.apache.paimon.data.InternalRow;
-import org.apache.paimon.deletionvectors.append.AppendDeleteFileMaintainer;
 import org.apache.paimon.flink.FlinkConnectorOptions;
 import org.apache.paimon.flink.cluster.IncrementalClusterSplitSource;
 import org.apache.paimon.flink.cluster.RewriteIncrementalClusterCommittableOperator;
@@ -71,7 +70,6 @@ import javax.annotation.Nullable;
 
 import java.time.Duration;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Iterator;
@@ -254,48 +252,28 @@ public class CompactAction extends TableActionBase {
             return false;
         }
 
-        Map<BinaryRow, AppendDeleteFileMaintainer> appendDvMaintainers =
-                table.coreOptions().deletionVectorsEnabled()
-                        ? IncrementalClusterManager.createAppendDvMaintainers(
-                                table, compactUnits.keySet(), incrementalClusterManager.snapshot())
-                        : Collections.emptyMap();
-        Map<BinaryRow, DataSplit[]> partitionSplits =
-                compactUnits.entrySet().stream()
-                        .collect(
-                                Collectors.toMap(
-                                        Map.Entry::getKey,
-                                        entry ->
-                                                incrementalClusterManager
-                                                        .toSplits(
-                                                                entry.getKey(),
-                                                                entry.getValue().files(),
-                                                                appendDvMaintainers.get(
-                                                                        entry.getKey()))
-                                                        .toArray(new DataSplit[0])));
+        Map<BinaryRow, Pair<List<DataSplit>, CommitMessage>> partitionSplits =
+                incrementalClusterManager.toSplitsAndRewriteDvFiles(compactUnits);
 
         // 2. read,sort and write in partition
         List<DataStream<Committable>> dataStreams = new ArrayList<>();
 
-        for (Map.Entry<BinaryRow, DataSplit[]> entry : partitionSplits.entrySet()) {
-            DataSplit[] splits = entry.getValue();
+        for (Map.Entry<BinaryRow, Pair<List<DataSplit>, CommitMessage>> entry :
+                partitionSplits.entrySet()) {
+            BinaryRow partition = entry.getKey();
+            List<DataSplit> splits = entry.getValue().getKey();
+            CommitMessage dvCommitMessage = entry.getValue().getRight();
             LinkedHashMap<String, String> partitionSpec =
-                    partitionComputer.generatePartValues(entry.getKey());
+                    partitionComputer.generatePartValues(partition);
 
             // 2.1 generate source for current partition
-            List<CommitMessage> partitionDvIndexCommitMessages =
-                    appendDvMaintainers.get(entry.getKey()) == null
-                            ? Collections.emptyList()
-                            : IncrementalClusterManager.producePartitionDvIndexCommitMessages(
-                                    table,
-                                    Arrays.asList(splits),
-                                    appendDvMaintainers.get(entry.getKey()));
             Pair<DataStream<RowData>, DataStream<Committable>> sourcePair =
                     IncrementalClusterSplitSource.buildSource(
                             env,
                             table,
                             partitionSpec,
                             splits,
-                            partitionDvIndexCommitMessages,
+                            dvCommitMessage,
                             options.get(FlinkConnectorOptions.SCAN_PARALLELISM));
 
             // 2.2 cluster in partition
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/cluster/IncrementalClusterSplitSource.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/cluster/IncrementalClusterSplitSource.java
index 957c70fe60..132dab660e 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/cluster/IncrementalClusterSplitSource.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/cluster/IncrementalClusterSplitSource.java
@@ -52,11 +52,12 @@ import java.util.Map;
 
 /** Source for Incremental Clustering. */
 public class IncrementalClusterSplitSource extends AbstractNonCoordinatedSource<Split> {
-    private static final long serialVersionUID = 1L;
 
-    private final Split[] splits;
+    private static final long serialVersionUID = 2L;
 
-    public IncrementalClusterSplitSource(Split[] splits) {
+    private final List<Split> splits;
+
+    public IncrementalClusterSplitSource(List<Split> splits) {
         this.splits = splits;
     }
 
@@ -74,7 +75,7 @@ public class IncrementalClusterSplitSource extends AbstractNonCoordinatedSource<
     private class Reader extends AbstractNonCoordinatedSourceReader<Split> {
 
         @Override
-        public InputStatus pollNext(ReaderOutput<Split> output) throws Exception {
+        public InputStatus pollNext(ReaderOutput<Split> output) {
             for (Split split : splits) {
                 DataSplit dataSplit = (DataSplit) split;
                 output.collect(dataSplit);
@@ -87,12 +88,12 @@ public class IncrementalClusterSplitSource extends AbstractNonCoordinatedSource<
             StreamExecutionEnvironment env,
             FileStoreTable table,
             Map<String, String> partitionSpec,
-            DataSplit[] splits,
-            List<CommitMessage> partitionDvIndexCommitMessages,
+            List<DataSplit> splits,
+            @Nullable CommitMessage dvCommitMessage,
             @Nullable Integer parallelism) {
         DataStream<Split> source =
                 env.fromSource(
-                                new IncrementalClusterSplitSource(splits),
+                                new IncrementalClusterSplitSource((List) splits),
                                 WatermarkStrategy.noWatermarks(),
                                 String.format(
                                         "Incremental-cluster split generator: %s - %s",
@@ -120,8 +121,7 @@ public class IncrementalClusterSplitSource extends AbstractNonCoordinatedSource<
                         .transform(
                                 "Remove files to be clustered",
                                 new CommittableTypeInfo(),
-                                new RemoveClusterBeforeFilesOperator(
-                                        partitionDvIndexCommitMessages))
+                                new RemoveClusterBeforeFilesOperator(dvCommitMessage))
                         .forceNonParallel());
     }
 }
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/cluster/RemoveClusterBeforeFilesOperator.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/cluster/RemoveClusterBeforeFilesOperator.java
index 4c0d9f9941..83b41940bc 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/cluster/RemoveClusterBeforeFilesOperator.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/cluster/RemoveClusterBeforeFilesOperator.java
@@ -29,18 +29,19 @@ import org.apache.paimon.table.source.Split;
 
 import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
 
+import javax.annotation.Nullable;
+
 import java.util.Collections;
-import java.util.List;
 
 /** Operator used with {@link IncrementalClusterSplitSource}, to remove files to be clustered. */
 public class RemoveClusterBeforeFilesOperator extends BoundedOneInputOperator<Split, Committable> {
 
-    private static final long serialVersionUID = 1L;
+    private static final long serialVersionUID = 2L;
 
-    private final List<CommitMessage> partitionDvIndexCommitMessages;
+    private final @Nullable CommitMessage dvCommitMessage;
 
-    public RemoveClusterBeforeFilesOperator(List<CommitMessage> partitionDvIndexCommitMessages) {
-        this.partitionDvIndexCommitMessages = partitionDvIndexCommitMessages;
+    public RemoveClusterBeforeFilesOperator(@Nullable CommitMessage dvCommitMessage) {
+        this.dvCommitMessage = dvCommitMessage;
     }
 
     @Override
@@ -67,10 +68,10 @@ public class RemoveClusterBeforeFilesOperator extends BoundedOneInputOperator<Sp
     }
 
     private void emitDvIndexCommitMessages(long checkpointId) {
-        for (CommitMessage commitMessage : partitionDvIndexCommitMessages) {
+        if (dvCommitMessage != null) {
             output.collect(
                     new StreamRecord<>(
-                            new Committable(checkpointId, Committable.Kind.FILE, commitMessage)));
+                            new Committable(checkpointId, Committable.Kind.FILE, dvCommitMessage)));
         }
     }
 }
diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/CompactProcedure.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/CompactProcedure.java
index 45da3d2ce0..e66f1ae7d5 100644
--- a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/CompactProcedure.java
+++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/CompactProcedure.java
@@ -26,7 +26,6 @@ import org.apache.paimon.append.AppendCompactTask;
 import org.apache.paimon.append.cluster.IncrementalClusterManager;
 import org.apache.paimon.compact.CompactUnit;
 import org.apache.paimon.data.BinaryRow;
-import org.apache.paimon.deletionvectors.append.AppendDeleteFileMaintainer;
 import org.apache.paimon.disk.IOManager;
 import org.apache.paimon.index.IndexFileMeta;
 import org.apache.paimon.io.CompactIncrement;
@@ -558,26 +557,8 @@ public class CompactProcedure extends BaseProcedure {
         Map<BinaryRow, CompactUnit> compactUnits =
                 incrementalClusterManager.prepareForCluster(fullCompaction);
 
-        Map<BinaryRow, AppendDeleteFileMaintainer> appendDvMaintainers =
-                table.coreOptions().deletionVectorsEnabled()
-                        ? IncrementalClusterManager.createAppendDvMaintainers(
-                                table, compactUnits.keySet(), incrementalClusterManager.snapshot())
-                        : Collections.emptyMap();
-
-        // generate splits for each partition
-        Map<BinaryRow, DataSplit[]> partitionSplits =
-                compactUnits.entrySet().stream()
-                        .collect(
-                                Collectors.toMap(
-                                        Map.Entry::getKey,
-                                        entry ->
-                                                incrementalClusterManager
-                                                        .toSplits(
-                                                                entry.getKey(),
-                                                                entry.getValue().files(),
-                                                                appendDvMaintainers.get(
-                                                                        entry.getKey()))
-                                                        .toArray(new DataSplit[0])));
+        Map<BinaryRow, Pair<List<DataSplit>, CommitMessage>> partitionSplits =
+                incrementalClusterManager.toSplitsAndRewriteDvFiles(compactUnits);
 
         // sort in partition
         TableSorter sorter =
@@ -592,13 +573,15 @@ public class CompactProcedure extends BaseProcedure {
 
         Dataset<Row> datasetForWrite =
                 partitionSplits.values().stream()
+                        .map(Pair::getKey)
                         .map(
-                                split -> {
+                                splits -> {
                                     Dataset<Row> dataset =
                                             PaimonUtils.createDataset(
                                                     spark(),
                                                     ScanPlanHelper$.MODULE$.createNewScanPlan(
-                                                            split, relation));
+                                                            splits.toArray(new DataSplit[0]),
+                                                            relation));
                                     return sorter.sort(dataset);
                                 })
                         .reduce(Dataset::union)
@@ -624,6 +607,8 @@ public class CompactProcedure extends BaseProcedure {
             List<CommitMessage> clusterMessages = new ArrayList<>();
             for (Map.Entry<BinaryRow, List<DataFileMeta>> entry : partitionClustered.entrySet()) {
                 BinaryRow partition = entry.getKey();
+                CommitMessageImpl dvCommitMessage =
+                        (CommitMessageImpl) partitionSplits.get(partition).getValue();
                 List<DataFileMeta> clusterBefore = compactUnits.get(partition).files();
                 // upgrade the clustered file to outputLevel
                 List<DataFileMeta> clusterAfter =
@@ -633,26 +618,15 @@ public class CompactProcedure extends BaseProcedure {
                         "Partition {}: upgrade file level to {}",
                         partition,
                         compactUnits.get(partition).outputLevel());
-                // get the dv index messages
-                List<CommitMessage> partitionDvIndexCommitMessages =
-                        appendDvMaintainers.get(entry.getKey()) == null
-                                ? Collections.emptyList()
-                                : IncrementalClusterManager.producePartitionDvIndexCommitMessages(
-                                        table,
-                                        Arrays.asList(partitionSplits.get(partition)),
-                                        appendDvMaintainers.get(entry.getKey()));
+
                 List<IndexFileMeta> newIndexFiles = new ArrayList<>();
                 List<IndexFileMeta> deletedIndexFiles = new ArrayList<>();
-                for (CommitMessage dvCommitMessage : partitionDvIndexCommitMessages) {
-                    newIndexFiles.addAll(
-                            ((CommitMessageImpl) dvCommitMessage)
-                                    .compactIncrement()
-                                    .newIndexFiles());
-                    deletedIndexFiles.addAll(
-                            ((CommitMessageImpl) dvCommitMessage)
-                                    .compactIncrement()
-                                    .deletedIndexFiles());
+                if (dvCommitMessage != null) {
+                    newIndexFiles = dvCommitMessage.compactIncrement().newIndexFiles();
+                    deletedIndexFiles = dvCommitMessage.compactIncrement().deletedIndexFiles();
                 }
+
+                // get the dv index messages
                 CompactIncrement compactIncrement =
                         new CompactIncrement(
                                 clusterBefore,


(paimon) 12/18: [common] Remove token expire time (#6544)

Posted by lz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

lzljs3620320 pushed a commit to branch release-1.3
in repository https://gitbox.apache.org/repos/asf/paimon.git

commit 4b16ee213124f3773e41be35d9e4bc6b03f75a9b
Author: YeJunHao <41...@users.noreply.github.com>
AuthorDate: Fri Nov 7 15:17:40 2025 +0800

    [common] Remove token expire time (#6544)
---
 paimon-common/src/main/java/org/apache/paimon/rest/RESTTokenFileIO.java | 2 --
 1 file changed, 2 deletions(-)

diff --git a/paimon-common/src/main/java/org/apache/paimon/rest/RESTTokenFileIO.java b/paimon-common/src/main/java/org/apache/paimon/rest/RESTTokenFileIO.java
index f5de31afb8..e3b321ff2e 100644
--- a/paimon-common/src/main/java/org/apache/paimon/rest/RESTTokenFileIO.java
+++ b/paimon-common/src/main/java/org/apache/paimon/rest/RESTTokenFileIO.java
@@ -45,7 +45,6 @@ import java.io.IOException;
 import java.io.UncheckedIOException;
 import java.util.Map;
 import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
 
 import static org.apache.paimon.options.CatalogOptions.FILE_IO_ALLOW_CACHE;
 import static org.apache.paimon.rest.RESTApi.TOKEN_EXPIRATION_SAFE_TIME_MILLIS;
@@ -64,7 +63,6 @@ public class RESTTokenFileIO implements FileIO {
 
     private static final Cache<RESTToken, FileIO> FILE_IO_CACHE =
             Caffeine.newBuilder()
-                    .expireAfterAccess(30, TimeUnit.MINUTES)
                     .maximumSize(1000)
                     .removalListener(
                             (ignored, value, cause) -> IOUtils.closeQuietly((FileIO) value))


(paimon) 05/18: [Python] Suppport multi prepare commit in the same TableWrite (#6526)

Posted by lz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

lzljs3620320 pushed a commit to branch release-1.3
in repository https://gitbox.apache.org/repos/asf/paimon.git

commit 134e1b69769b858b153780367f2b0be56f7fe8a9
Author: umi <55...@users.noreply.github.com>
AuthorDate: Wed Nov 5 16:00:04 2025 +0800

    [Python] Suppport multi prepare commit in the same TableWrite  (#6526)
---
 paimon-python/pypaimon/snapshot/snapshot.py        |   2 +
 paimon-python/pypaimon/table/file_store_table.py   |   5 +-
 paimon-python/pypaimon/table/table.py              |   6 +-
 .../pypaimon/tests/py36/ao_simple_test.py          |  56 ++++++++
 .../pypaimon/tests/write/table_write_test.py       | 155 +++++++++++++++++++++
 paimon-python/pypaimon/write/file_store_write.py   |   5 +-
 .../{batch_table_commit.py => table_commit.py}     |  19 ++-
 .../write/{batch_table_write.py => table_write.py} |  29 ++--
 .../{batch_write_builder.py => write_builder.py}   |  35 +++--
 9 files changed, 287 insertions(+), 25 deletions(-)

diff --git a/paimon-python/pypaimon/snapshot/snapshot.py b/paimon-python/pypaimon/snapshot/snapshot.py
index 96b287ab55..f164f5eb61 100644
--- a/paimon-python/pypaimon/snapshot/snapshot.py
+++ b/paimon-python/pypaimon/snapshot/snapshot.py
@@ -21,6 +21,8 @@ from typing import Dict, Optional
 
 from pypaimon.common.json_util import json_field
 
+COMMIT_IDENTIFIER = 0x7fffffffffffffff
+
 
 @dataclass
 class Snapshot:
diff --git a/paimon-python/pypaimon/table/file_store_table.py b/paimon-python/pypaimon/table/file_store_table.py
index cde282eff0..6132c9bd69 100644
--- a/paimon-python/pypaimon/table/file_store_table.py
+++ b/paimon-python/pypaimon/table/file_store_table.py
@@ -28,7 +28,7 @@ from pypaimon.schema.schema_manager import SchemaManager
 from pypaimon.schema.table_schema import TableSchema
 from pypaimon.table.bucket_mode import BucketMode
 from pypaimon.table.table import Table
-from pypaimon.write.batch_write_builder import BatchWriteBuilder
+from pypaimon.write.write_builder import BatchWriteBuilder, StreamWriteBuilder
 from pypaimon.write.row_key_extractor import (DynamicBucketRowKeyExtractor,
                                               FixedBucketRowKeyExtractor,
                                               PostponeBucketRowKeyExtractor,
@@ -98,6 +98,9 @@ class FileStoreTable(Table):
     def new_batch_write_builder(self) -> BatchWriteBuilder:
         return BatchWriteBuilder(self)
 
+    def new_stream_write_builder(self) -> StreamWriteBuilder:
+        return StreamWriteBuilder(self)
+
     def create_row_key_extractor(self) -> RowKeyExtractor:
         bucket_mode = self.bucket_mode()
         if bucket_mode == BucketMode.HASH_FIXED:
diff --git a/paimon-python/pypaimon/table/table.py b/paimon-python/pypaimon/table/table.py
index 3a1fe2e622..e20784f1fc 100644
--- a/paimon-python/pypaimon/table/table.py
+++ b/paimon-python/pypaimon/table/table.py
@@ -19,7 +19,7 @@
 from abc import ABC, abstractmethod
 
 from pypaimon.read.read_builder import ReadBuilder
-from pypaimon.write.batch_write_builder import BatchWriteBuilder
+from pypaimon.write.write_builder import BatchWriteBuilder, StreamWriteBuilder
 
 
 class Table(ABC):
@@ -32,3 +32,7 @@ class Table(ABC):
     @abstractmethod
     def new_batch_write_builder(self) -> BatchWriteBuilder:
         """Returns a builder for building batch table write and table commit."""
+
+    @abstractmethod
+    def new_stream_write_builder(self) -> StreamWriteBuilder:
+        """Returns a builder for building stream table write and table commit."""
diff --git a/paimon-python/pypaimon/tests/py36/ao_simple_test.py b/paimon-python/pypaimon/tests/py36/ao_simple_test.py
index efb3189e06..b14fbcf8db 100644
--- a/paimon-python/pypaimon/tests/py36/ao_simple_test.py
+++ b/paimon-python/pypaimon/tests/py36/ao_simple_test.py
@@ -421,3 +421,59 @@ class AOSimpleTest(RESTBaseTest):
                                          session_token="TOKEN",
                                          region="cn-hangzhou",
                                          endpoint_override="oss-bucket." + props[OssOptions.OSS_ENDPOINT])
+
+    def test_multi_prepare_commit_ao(self):
+        schema = Schema.from_pyarrow_schema(self.pa_schema, partition_keys=['dt'])
+        self.rest_catalog.create_table('default.test_append_only_parquet', schema, False)
+        table = self.rest_catalog.get_table('default.test_append_only_parquet')
+        write_builder = table.new_stream_write_builder()
+
+        table_write = write_builder.new_write()
+        table_commit = write_builder.new_commit()
+        # write 1
+        data1 = {
+            'user_id': [1, 2, 3, 4],
+            'item_id': [1001, 1002, 1003, 1004],
+            'behavior': ['a', 'b', 'c', None],
+            'dt': ['p1', 'p1', 'p2', 'p1'],
+        }
+        pa_table = pa.Table.from_pydict(data1, schema=self.pa_schema)
+        table_write.write_arrow(pa_table)
+        table_write.prepare_commit(0)
+        # write 2
+        data2 = {
+            'user_id': [5, 6, 7, 8],
+            'item_id': [1005, 1006, 1007, 1008],
+            'behavior': ['e', 'f', 'g', 'h'],
+            'dt': ['p2', 'p1', 'p2', 'p2'],
+        }
+        pa_table = pa.Table.from_pydict(data2, schema=self.pa_schema)
+        table_write.write_arrow(pa_table)
+        table_write.prepare_commit(1)
+        # write 3
+        data3 = {
+            'user_id': [9, 10],
+            'item_id': [1009, 1010],
+            'behavior': ['i', 'j'],
+            'dt': ['p2', 'p1'],
+        }
+        pa_table = pa.Table.from_pydict(data3, schema=self.pa_schema)
+        table_write.write_arrow(pa_table)
+        cm = table_write.prepare_commit(2)
+        # commit
+        table_commit.commit(cm, 2)
+        table_write.close()
+        table_commit.close()
+        self.assertEqual(2, table_write.file_store_write.commit_identifier)
+
+        read_builder = table.new_read_builder()
+        table_read = read_builder.new_read()
+        splits = read_builder.new_scan().plan().splits()
+        actual = table_sort_by(table_read.to_arrow(splits), 'user_id')
+        expected = pa.Table.from_pydict({
+            'user_id': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10],
+            'item_id': [1001, 1002, 1003, 1004, 1005, 1006, 1007, 1008, 1009, 1010],
+            'behavior': ['a', 'b', 'c', None, 'e', 'f', 'g', 'h', 'i', 'j'],
+            'dt': ['p1', 'p1', 'p2', 'p1', 'p2', 'p1', 'p2', 'p2', 'p2', 'p1']
+        }, schema=self.pa_schema)
+        self.assertEqual(expected, actual)
diff --git a/paimon-python/pypaimon/tests/write/table_write_test.py b/paimon-python/pypaimon/tests/write/table_write_test.py
new file mode 100644
index 0000000000..21b76731ac
--- /dev/null
+++ b/paimon-python/pypaimon/tests/write/table_write_test.py
@@ -0,0 +1,155 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+import os
+import shutil
+
+import tempfile
+import unittest
+
+from pypaimon import CatalogFactory, Schema
+import pyarrow as pa
+
+
+class TableWriteTest(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.tempdir = tempfile.mkdtemp()
+        cls.warehouse = os.path.join(cls.tempdir, 'warehouse')
+        cls.catalog = CatalogFactory.create({
+            'warehouse': cls.warehouse
+        })
+        cls.catalog.create_database('default', True)
+
+        cls.pa_schema = pa.schema([
+            ('user_id', pa.int32()),
+            ('item_id', pa.int64()),
+            ('behavior', pa.string()),
+            ('dt', pa.string())
+        ])
+        cls.expected = pa.Table.from_pydict({
+            'user_id': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10],
+            'item_id': [1001, 1002, 1003, 1004, 1005, 1006, 1007, 1008, 1009, 1010],
+            'behavior': ['a', 'b', 'c', None, 'e', 'f', 'g', 'h', 'i', 'j'],
+            'dt': ['p1', 'p1', 'p2', 'p1', 'p2', 'p1', 'p2', 'p2', 'p2', 'p1']
+        }, schema=cls.pa_schema)
+
+    @classmethod
+    def tearDownClass(cls):
+        shutil.rmtree(cls.tempdir, ignore_errors=True)
+
+    def test_multi_prepare_commit_ao(self):
+        schema = Schema.from_pyarrow_schema(self.pa_schema, partition_keys=['dt'])
+        self.catalog.create_table('default.test_append_only_parquet', schema, False)
+        table = self.catalog.get_table('default.test_append_only_parquet')
+        write_builder = table.new_stream_write_builder()
+
+        table_write = write_builder.new_write()
+        table_commit = write_builder.new_commit()
+        # write 1
+        data1 = {
+            'user_id': [1, 2, 3, 4],
+            'item_id': [1001, 1002, 1003, 1004],
+            'behavior': ['a', 'b', 'c', None],
+            'dt': ['p1', 'p1', 'p2', 'p1'],
+        }
+        pa_table = pa.Table.from_pydict(data1, schema=self.pa_schema)
+        table_write.write_arrow(pa_table)
+        table_write.prepare_commit(0)
+        # write 2
+        data2 = {
+            'user_id': [5, 6, 7, 8],
+            'item_id': [1005, 1006, 1007, 1008],
+            'behavior': ['e', 'f', 'g', 'h'],
+            'dt': ['p2', 'p1', 'p2', 'p2'],
+        }
+        pa_table = pa.Table.from_pydict(data2, schema=self.pa_schema)
+        table_write.write_arrow(pa_table)
+        table_write.prepare_commit(1)
+        # write 3
+        data3 = {
+            'user_id': [9, 10],
+            'item_id': [1009, 1010],
+            'behavior': ['i', 'j'],
+            'dt': ['p2', 'p1'],
+        }
+        pa_table = pa.Table.from_pydict(data3, schema=self.pa_schema)
+        table_write.write_arrow(pa_table)
+        cm = table_write.prepare_commit(2)
+        # commit
+        table_commit.commit(cm, 2)
+        table_write.close()
+        table_commit.close()
+        self.assertEqual(2, table_write.file_store_write.commit_identifier)
+
+        read_builder = table.new_read_builder()
+        table_read = read_builder.new_read()
+        splits = read_builder.new_scan().plan().splits()
+        actual = table_read.to_arrow(splits).sort_by('user_id')
+        self.assertEqual(self.expected, actual)
+
+    def test_multi_prepare_commit_pk(self):
+        schema = Schema.from_pyarrow_schema(self.pa_schema, partition_keys=['dt'], primary_keys=['user_id', 'dt'],
+                                            options={'bucket': '2'})
+        self.catalog.create_table('default.test_primary_key_parquet', schema, False)
+        table = self.catalog.get_table('default.test_primary_key_parquet')
+        write_builder = table.new_stream_write_builder()
+
+        table_write = write_builder.new_write()
+        table_commit = write_builder.new_commit()
+        # write 1
+        data1 = {
+            'user_id': [1, 2, 3, 4],
+            'item_id': [1001, 1002, 1003, 1004],
+            'behavior': ['a', 'b', 'c', None],
+            'dt': ['p1', 'p1', 'p2', 'p1'],
+        }
+        pa_table = pa.Table.from_pydict(data1, schema=self.pa_schema)
+        table_write.write_arrow(pa_table)
+        table_write.prepare_commit(0)
+        # write 2
+        data2 = {
+            'user_id': [5, 6, 7, 8],
+            'item_id': [1005, 1006, 1007, 1008],
+            'behavior': ['e', 'f', 'g', 'h'],
+            'dt': ['p2', 'p1', 'p2', 'p2'],
+        }
+        pa_table = pa.Table.from_pydict(data2, schema=self.pa_schema)
+        table_write.write_arrow(pa_table)
+        table_write.prepare_commit(1)
+        # write 3
+        data3 = {
+            'user_id': [9, 10],
+            'item_id': [1009, 1010],
+            'behavior': ['i', 'j'],
+            'dt': ['p2', 'p1'],
+        }
+        pa_table = pa.Table.from_pydict(data3, schema=self.pa_schema)
+        table_write.write_arrow(pa_table)
+        cm = table_write.prepare_commit(2)
+        # commit
+        table_commit.commit(cm, 2)
+        table_write.close()
+        table_commit.close()
+        self.assertEqual(2, table_write.file_store_write.commit_identifier)
+
+        read_builder = table.new_read_builder()
+        table_read = read_builder.new_read()
+        splits = read_builder.new_scan().plan().splits()
+        actual = table_read.to_arrow(splits).sort_by('user_id')
+        self.assertEqual(self.expected, actual)
diff --git a/paimon-python/pypaimon/write/file_store_write.py b/paimon-python/pypaimon/write/file_store_write.py
index 9ebabf1103..c100b64966 100644
--- a/paimon-python/pypaimon/write/file_store_write.py
+++ b/paimon-python/pypaimon/write/file_store_write.py
@@ -37,6 +37,7 @@ class FileStoreWrite:
         self.data_writers: Dict[Tuple, DataWriter] = {}
         self.max_seq_numbers: dict = {}
         self.write_cols = None
+        self.commit_identifier = 0
 
     def write(self, partition: Tuple, bucket: int, data: pa.RecordBatch):
         key = (partition, bucket)
@@ -48,6 +49,7 @@ class FileStoreWrite:
     def _create_data_writer(self, partition: Tuple, bucket: int) -> DataWriter:
         def max_seq_number():
             return self._seq_number_stats(partition).get(bucket, 1)
+
         # Check if table has blob columns
         if self._has_blob_columns():
             return DataBlobWriter(
@@ -83,7 +85,8 @@ class FileStoreWrite:
                 return True
         return False
 
-    def prepare_commit(self) -> List[CommitMessage]:
+    def prepare_commit(self, commit_identifier) -> List[CommitMessage]:
+        self.commit_identifier = commit_identifier
         commit_messages = []
         for (partition, bucket), writer in self.data_writers.items():
             committed_files = writer.prepare_commit()
diff --git a/paimon-python/pypaimon/write/batch_table_commit.py b/paimon-python/pypaimon/write/table_commit.py
similarity index 86%
rename from paimon-python/pypaimon/write/batch_table_commit.py
rename to paimon-python/pypaimon/write/table_commit.py
index 7f42e1cef1..0dcfabf99e 100644
--- a/paimon-python/pypaimon/write/batch_table_commit.py
+++ b/paimon-python/pypaimon/write/table_commit.py
@@ -16,14 +16,14 @@
 # limitations under the License.
 ################################################################################
 
-import time
 from typing import List, Optional
 
+from pypaimon.snapshot.snapshot import COMMIT_IDENTIFIER
 from pypaimon.write.commit_message import CommitMessage
 from pypaimon.write.file_store_commit import FileStoreCommit
 
 
-class BatchTableCommit:
+class TableCommit:
     """Python implementation of BatchTableCommit for batch writing scenarios."""
 
     def __init__(self, table, commit_user: str, static_partition: Optional[dict]):
@@ -41,15 +41,13 @@ class BatchTableCommit:
         self.file_store_commit = FileStoreCommit(snapshot_commit, table, commit_user)
         self.batch_committed = False
 
-    def commit(self, commit_messages: List[CommitMessage]):
+    def _commit(self, commit_messages: List[CommitMessage], commit_identifier: int = COMMIT_IDENTIFIER):
         self._check_committed()
 
         non_empty_messages = [msg for msg in commit_messages if not msg.is_empty()]
         if not non_empty_messages:
             return
 
-        commit_identifier = int(time.time() * 1000)
-
         try:
             if self.overwrite_partition is not None:
                 self.file_store_commit.overwrite(
@@ -76,3 +74,14 @@ class BatchTableCommit:
         if self.batch_committed:
             raise RuntimeError("BatchTableCommit only supports one-time committing.")
         self.batch_committed = True
+
+
+class BatchTableCommit(TableCommit):
+    def commit(self, commit_messages: List[CommitMessage]):
+        self._commit(commit_messages, COMMIT_IDENTIFIER)
+
+
+class StreamTableCommit(TableCommit):
+
+    def commit(self, commit_messages: List[CommitMessage], commit_identifier: int = COMMIT_IDENTIFIER):
+        self._commit(commit_messages, commit_identifier)
diff --git a/paimon-python/pypaimon/write/batch_table_write.py b/paimon-python/pypaimon/write/table_write.py
similarity index 89%
rename from paimon-python/pypaimon/write/batch_table_write.py
rename to paimon-python/pypaimon/write/table_write.py
index f8d0660bfa..3ccc078223 100644
--- a/paimon-python/pypaimon/write/batch_table_write.py
+++ b/paimon-python/pypaimon/write/table_write.py
@@ -15,18 +15,18 @@
 #  See the License for the specific language governing permissions and
 # limitations under the License.
 ################################################################################
-
 from collections import defaultdict
 from typing import List
 
 import pyarrow as pa
 
 from pypaimon.schema.data_types import PyarrowFieldParser
+from pypaimon.snapshot.snapshot import COMMIT_IDENTIFIER
 from pypaimon.write.commit_message import CommitMessage
 from pypaimon.write.file_store_write import FileStoreWrite
 
 
-class BatchTableWrite:
+class TableWrite:
     def __init__(self, table):
         from pypaimon.table.file_store_table import FileStoreTable
 
@@ -34,7 +34,6 @@ class BatchTableWrite:
         self.table_pyarrow_schema = PyarrowFieldParser.from_paimon_schema(self.table.table_schema.fields)
         self.file_store_write = FileStoreWrite(self.table)
         self.row_key_extractor = self.table.create_row_key_extractor()
-        self.batch_committed = False
 
     def write_arrow(self, table: pa.Table):
         batches_iterator = table.to_batches()
@@ -59,12 +58,6 @@ class BatchTableWrite:
         record_batch = pa.RecordBatch.from_pandas(dataframe, schema=pa_schema)
         return self.write_arrow_batch(record_batch)
 
-    def prepare_commit(self) -> List[CommitMessage]:
-        if self.batch_committed:
-            raise RuntimeError("BatchTableWrite only supports one-time committing.")
-        self.batch_committed = True
-        return self.file_store_write.prepare_commit()
-
     def with_write_type(self, write_cols: List[str]):
         for col in write_cols:
             if col not in self.table_pyarrow_schema.names:
@@ -83,3 +76,21 @@ class BatchTableWrite:
                              f"Input schema is: {data_schema} "
                              f"Table schema is: {self.table_pyarrow_schema} "
                              f"Write cols is: {self.file_store_write.write_cols}")
+
+
+class BatchTableWrite(TableWrite):
+    def __init__(self, table):
+        super().__init__(table)
+        self.batch_committed = False
+
+    def prepare_commit(self) -> List[CommitMessage]:
+        if self.batch_committed:
+            raise RuntimeError("BatchTableWrite only supports one-time committing.")
+        self.batch_committed = True
+        return self.file_store_write.prepare_commit(COMMIT_IDENTIFIER)
+
+
+class StreamTableWrite(TableWrite):
+
+    def prepare_commit(self, commit_identifier: int = COMMIT_IDENTIFIER) -> List[CommitMessage]:
+        return self.file_store_write.prepare_commit(commit_identifier)
diff --git a/paimon-python/pypaimon/write/batch_write_builder.py b/paimon-python/pypaimon/write/write_builder.py
similarity index 73%
rename from paimon-python/pypaimon/write/batch_write_builder.py
rename to paimon-python/pypaimon/write/write_builder.py
index 2380530fbc..8c9ed725f5 100644
--- a/paimon-python/pypaimon/write/batch_write_builder.py
+++ b/paimon-python/pypaimon/write/write_builder.py
@@ -17,14 +17,15 @@
 ################################################################################
 
 import uuid
+from abc import ABC
 from typing import Optional
 
 from pypaimon.common.core_options import CoreOptions
-from pypaimon.write.batch_table_commit import BatchTableCommit
-from pypaimon.write.batch_table_write import BatchTableWrite
+from pypaimon.write.table_commit import BatchTableCommit, StreamTableCommit, TableCommit
+from pypaimon.write.table_write import BatchTableWrite, StreamTableWrite, TableWrite
 
 
-class BatchWriteBuilder:
+class WriteBuilder(ABC):
     def __init__(self, table):
         from pypaimon.table.file_store_table import FileStoreTable
 
@@ -36,15 +37,33 @@ class BatchWriteBuilder:
         self.static_partition = static_partition if static_partition is not None else {}
         return self
 
-    def new_write(self) -> BatchTableWrite:
-        return BatchTableWrite(self.table)
+    def new_write(self) -> TableWrite:
+        """Returns a table write."""
 
-    def new_commit(self) -> BatchTableCommit:
-        commit = BatchTableCommit(self.table, self.commit_user, self.static_partition)
-        return commit
+    def new_commit(self) -> TableCommit:
+        """Returns a table commit."""
 
     def _create_commit_user(self):
         if CoreOptions.COMMIT_USER_PREFIX in self.table.options:
             return f"{self.table.options.get(CoreOptions.COMMIT_USER_PREFIX)}_{uuid.uuid4()}"
         else:
             return str(uuid.uuid4())
+
+
+class BatchWriteBuilder(WriteBuilder):
+
+    def new_write(self) -> BatchTableWrite:
+        return BatchTableWrite(self.table)
+
+    def new_commit(self) -> BatchTableCommit:
+        commit = BatchTableCommit(self.table, self.commit_user, self.static_partition)
+        return commit
+
+
+class StreamWriteBuilder(WriteBuilder):
+    def new_write(self) -> StreamTableWrite:
+        return StreamTableWrite(self.table)
+
+    def new_commit(self) -> StreamTableCommit:
+        commit = StreamTableCommit(self.table, self.commit_user, self.static_partition)
+        return commit


(paimon) 14/18: [python] Fix File Source type in data file meta (#6571)

Posted by lz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

lzljs3620320 pushed a commit to branch release-1.3
in repository https://gitbox.apache.org/repos/asf/paimon.git

commit ae574508537d1d7f98c819e44a0ade6d38c1b342
Author: umi <55...@users.noreply.github.com>
AuthorDate: Mon Nov 10 11:20:32 2025 +0800

    [python] Fix File Source type in data file meta (#6571)
---
 .../pypaimon/manifest/schema/data_file_meta.py     |  4 +--
 paimon-python/pypaimon/tests/manifest/__init__.py  | 17 +++++++++++
 .../tests/{ => manifest}/manifest_schema_test.py   | 35 +++++++---------------
 paimon-python/pypaimon/write/file_store_commit.py  |  2 +-
 .../pypaimon/write/writer/data_blob_writer.py      |  2 +-
 paimon-python/pypaimon/write/writer/data_writer.py |  2 +-
 6 files changed, 33 insertions(+), 29 deletions(-)

diff --git a/paimon-python/pypaimon/manifest/schema/data_file_meta.py b/paimon-python/pypaimon/manifest/schema/data_file_meta.py
index a414644e27..405c2e3483 100644
--- a/paimon-python/pypaimon/manifest/schema/data_file_meta.py
+++ b/paimon-python/pypaimon/manifest/schema/data_file_meta.py
@@ -44,7 +44,7 @@ class DataFileMeta:
     creation_time: Optional[datetime] = None
     delete_row_count: Optional[int] = None
     embedded_index: Optional[bytes] = None
-    file_source: Optional[str] = None
+    file_source: Optional[int] = None
     value_stats_cols: Optional[List[str]] = None
     external_path: Optional[str] = None
     first_row_id: Optional[int] = None
@@ -163,7 +163,7 @@ DATA_FILE_META_SCHEMA = {
          "default": None},
         {"name": "_DELETE_ROW_COUNT", "type": ["null", "long"], "default": None},
         {"name": "_EMBEDDED_FILE_INDEX", "type": ["null", "bytes"], "default": None},
-        {"name": "_FILE_SOURCE", "type": ["null", "string"], "default": None},
+        {"name": "_FILE_SOURCE", "type": ["null", "int"], "default": None},
         {"name": "_VALUE_STATS_COLS",
          "type": ["null", {"type": "array", "items": "string"}],
          "default": None},
diff --git a/paimon-python/pypaimon/tests/manifest/__init__.py b/paimon-python/pypaimon/tests/manifest/__init__.py
new file mode 100644
index 0000000000..53ed4d36c2
--- /dev/null
+++ b/paimon-python/pypaimon/tests/manifest/__init__.py
@@ -0,0 +1,17 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
diff --git a/paimon-python/pypaimon/tests/manifest_schema_test.py b/paimon-python/pypaimon/tests/manifest/manifest_schema_test.py
similarity index 87%
rename from paimon-python/pypaimon/tests/manifest_schema_test.py
rename to paimon-python/pypaimon/tests/manifest/manifest_schema_test.py
index 3d82ededa6..b4b60ffd43 100644
--- a/paimon-python/pypaimon/tests/manifest_schema_test.py
+++ b/paimon-python/pypaimon/tests/manifest/manifest_schema_test.py
@@ -1,23 +1,7 @@
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements.  See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership.  The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License.  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-"""
-
 import unittest
 
+from pypaimon.manifest.schema import data_file_meta
+
 from pypaimon.manifest.schema.data_file_meta import DATA_FILE_META_SCHEMA
 from pypaimon.manifest.schema.manifest_file_meta import MANIFEST_FILE_META_SCHEMA
 from pypaimon.manifest.schema.simple_stats import (
@@ -28,7 +12,14 @@ from pypaimon.manifest.schema.simple_stats import (
 
 
 class ManifestSchemaTest(unittest.TestCase):
-    """Test cases for the manifest schema definitions."""
+    def test_file_source_field_type_and_default(self):
+        schema = data_file_meta.DATA_FILE_META_SCHEMA
+        fields = schema.get("fields", [])
+        file_source_field = next((f for f in fields if f.get("name") == "_FILE_SOURCE"), None)
+
+        self.assertIsNotNone(file_source_field, "_FILE_SOURCE field not found in DATA_FILE_META_SCHEMA")
+        self.assertEqual(file_source_field.get("type"), ["null", "int"])
+        self.assertIsNone(file_source_field.get("default"))
 
     def test_data_file_meta_schema_structure(self):
         """Test that DATA_FILE_META_SCHEMA has the correct structure."""
@@ -73,7 +64,7 @@ class ManifestSchemaTest(unittest.TestCase):
                          ["null", {"type": "long", "logicalType": "timestamp-millis"}])
         self.assertEqual(field_map["_DELETE_ROW_COUNT"]["type"], ["null", "long"])
         self.assertEqual(field_map["_EMBEDDED_FILE_INDEX"]["type"], ["null", "bytes"])
-        self.assertEqual(field_map["_FILE_SOURCE"]["type"], ["null", "string"])
+        self.assertEqual(field_map["_FILE_SOURCE"]["type"], ["null", "int"])
         self.assertEqual(field_map["_VALUE_STATS_COLS"]["type"], ["null", {"type": "array", "items": "string"}])
         self.assertEqual(field_map["_EXTERNAL_PATH"]["type"], ["null", "string"])
         self.assertEqual(field_map["_FIRST_ROW_ID"]["type"], ["null", "long"])
@@ -141,7 +132,3 @@ class ManifestSchemaTest(unittest.TestCase):
             PARTITION_STATS_SCHEMA["name"]
         ]
         self.assertEqual(len(names), len(set(names)), "Schema names should be unique")
-
-
-if __name__ == "__main__":
-    unittest.main()
diff --git a/paimon-python/pypaimon/write/file_store_commit.py b/paimon-python/pypaimon/write/file_store_commit.py
index afeba52b3c..224e4a82c9 100644
--- a/paimon-python/pypaimon/write/file_store_commit.py
+++ b/paimon-python/pypaimon/write/file_store_commit.py
@@ -361,7 +361,7 @@ class FileStoreCommit:
         for entry in commit_entries:
             # Check if this is an append file that needs row ID assignment
             if (entry.kind == 0 and  # ADD kind
-                    entry.file.file_source == "APPEND" and  # APPEND file source
+                    entry.file.file_source == 0 and  # APPEND file source
                     entry.file.first_row_id is None):  # No existing first_row_id
 
                 if self._is_blob_file(entry.file.file_name):
diff --git a/paimon-python/pypaimon/write/writer/data_blob_writer.py b/paimon-python/pypaimon/write/writer/data_blob_writer.py
index b711d2e695..9d2e0982a4 100644
--- a/paimon-python/pypaimon/write/writer/data_blob_writer.py
+++ b/paimon-python/pypaimon/write/writer/data_blob_writer.py
@@ -301,7 +301,7 @@ class DataBlobWriter(DataWriter):
             extra_files=[],
             creation_time=datetime.now(),
             delete_row_count=0,
-            file_source="APPEND",
+            file_source=0,
             value_stats_cols=self.normal_column_names,
             file_path=str(file_path),
             write_cols=self.write_cols)
diff --git a/paimon-python/pypaimon/write/writer/data_writer.py b/paimon-python/pypaimon/write/writer/data_writer.py
index 24e3b0ca48..351ff32979 100644
--- a/paimon-python/pypaimon/write/writer/data_writer.py
+++ b/paimon-python/pypaimon/write/writer/data_writer.py
@@ -210,7 +210,7 @@ class DataWriter(ABC):
             extra_files=[],
             creation_time=datetime.now(),
             delete_row_count=0,
-            file_source="APPEND",
+            file_source=0,
             value_stats_cols=None,  # None means all columns in the data have statistics
             external_path=None,
             first_row_id=None,


(paimon) 13/18: [python] Fix AtomicType.to_dict() inconsistency with java (#6548)

Posted by lz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

lzljs3620320 pushed a commit to branch release-1.3
in repository https://gitbox.apache.org/repos/asf/paimon.git

commit 787210fbeadc346a906bcab37ca1ecbed4868b59
Author: universe-hcy <47...@users.noreply.github.com>
AuthorDate: Mon Nov 10 10:30:56 2025 +0800

    [python] Fix AtomicType.to_dict() inconsistency with java (#6548)
---
 .github/workflows/paimon-python-checks.yml      |  4 +-
 paimon-python/pypaimon/schema/data_types.py     | 14 ++++--
 paimon-python/pypaimon/tests/data_types_test.py | 67 +++++++++++++++++++++++++
 3 files changed, 79 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/paimon-python-checks.yml b/.github/workflows/paimon-python-checks.yml
old mode 100644
new mode 100755
index 789e3dd9d6..c4c7e32a84
--- a/.github/workflows/paimon-python-checks.yml
+++ b/.github/workflows/paimon-python-checks.yml
@@ -67,10 +67,10 @@ jobs:
           if [[ "${{ matrix.python-version }}" == "3.6.15" ]]; then
             python -m pip install --upgrade pip==21.3.1
             python --version
-            python -m pip install -q readerwriterlock==1.0.9 'fsspec==2021.10.1' 'cachetools==4.2.4' 'ossfs==2021.8.0' pyarrow==6.0.1 pandas==1.1.5 'polars==0.9.12' 'fastavro==1.4.7' zstandard==0.19.0 dataclasses==0.8.0 flake8 pytest py4j==0.10.9.9 requests 2>&1 >/dev/null
+            python -m pip install -q readerwriterlock==1.0.9 'fsspec==2021.10.1' 'cachetools==4.2.4' 'ossfs==2021.8.0' pyarrow==6.0.1 pandas==1.1.5 'polars==0.9.12' 'fastavro==1.4.7' zstandard==0.19.0 dataclasses==0.8.0 flake8 pytest py4j==0.10.9.9 requests parameterized==0.8.1 2>&1 >/dev/null
           else
             python -m pip install --upgrade pip
-            python -m pip install -q readerwriterlock==1.0.9 fsspec==2024.3.1 cachetools==5.3.3 ossfs==2023.12.0 ray==2.48.0 fastavro==1.11.1 pyarrow==16.0.0 zstandard==0.24.0 polars==1.32.0 duckdb==1.3.2 numpy==1.24.3 pandas==2.0.3 flake8==4.0.1 pytest~=7.0 py4j==0.10.9.9 requests 2>&1 >/dev/null
+            python -m pip install -q readerwriterlock==1.0.9 fsspec==2024.3.1 cachetools==5.3.3 ossfs==2023.12.0 ray==2.48.0 fastavro==1.11.1 pyarrow==16.0.0 zstandard==0.24.0 polars==1.32.0 duckdb==1.3.2 numpy==1.24.3 pandas==2.0.3 flake8==4.0.1 pytest~=7.0 py4j==0.10.9.9 requests parameterized==0.9.0 2>&1 >/dev/null
           fi
       - name: Run lint-python.sh
         shell: bash
diff --git a/paimon-python/pypaimon/schema/data_types.py b/paimon-python/pypaimon/schema/data_types.py
index 3f7b30c291..91404cb193 100755
--- a/paimon-python/pypaimon/schema/data_types.py
+++ b/paimon-python/pypaimon/schema/data_types.py
@@ -73,8 +73,10 @@ class AtomicType(DataType):
         super().__init__(nullable)
         self.type = type
 
-    def to_dict(self) -> Dict[str, Any]:
-        return {"type": self.type if self.nullable else self.type + " NOT NULL"}
+    def to_dict(self) -> str:
+        if not self.nullable:
+            return self.type + " NOT NULL"
+        return self.type
 
     @classmethod
     def from_dict(cls, data: str) -> "AtomicType":
@@ -119,7 +121,8 @@ class MultisetType(DataType):
 
     def to_dict(self) -> Dict[str, Any]:
         return {
-            "type": "MULTISET{}".format('<' + str(self.element) + '>' if self.element else ''),
+            "type": "MULTISET{}{}".format('<' + str(self.element) + '>' if self.element else '',
+                                          " NOT NULL" if not self.nullable else ""),
             "element": self.element.to_dict() if self.element else None,
             "nullable": self.nullable,
         }
@@ -232,7 +235,10 @@ class RowType(DataType):
         return DataTypeParser.parse_data_type(data)
 
     def __str__(self) -> str:
-        field_strs = ["{}: {}".format(field.name, field.type) for field in self.fields]
+        field_strs = []
+        for field in self.fields:
+            description = " COMMENT {}".format(field.description) if field.description else ""
+            field_strs.append("{}: {}{}".format(field.name, field.type, description))
         null_suffix = "" if self.nullable else " NOT NULL"
         return "ROW<{}>{}".format(', '.join(field_strs), null_suffix)
 
diff --git a/paimon-python/pypaimon/tests/data_types_test.py b/paimon-python/pypaimon/tests/data_types_test.py
new file mode 100755
index 0000000000..53644e24c5
--- /dev/null
+++ b/paimon-python/pypaimon/tests/data_types_test.py
@@ -0,0 +1,67 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+import unittest
+from parameterized import parameterized
+
+from pypaimon.schema.data_types import DataField, AtomicType, ArrayType, MultisetType, MapType, RowType
+
+
+class DataTypesTest(unittest.TestCase):
+    def test_atomic_type(self):
+        self.assertEqual(str(AtomicType("BLOB")), "BLOB")
+        self.assertEqual(str(AtomicType("TINYINT", nullable=False)), "TINYINT NOT NULL")
+        self.assertEqual(str(AtomicType("BIGINT", nullable=False)), "BIGINT NOT NULL")
+        self.assertEqual(str(AtomicType("BOOLEAN", nullable=False)), "BOOLEAN NOT NULL")
+        self.assertEqual(str(AtomicType("DOUBLE")), "DOUBLE")
+        self.assertEqual(str(AtomicType("STRING")), "STRING")
+        self.assertEqual(str(AtomicType("BINARY(12)")), "BINARY(12)")
+        self.assertEqual(str(AtomicType("DECIMAL(10, 6)")), "DECIMAL(10, 6)")
+        self.assertEqual(str(AtomicType("BYTES")), "BYTES")
+        self.assertEqual(str(AtomicType("DATE")), "DATE")
+        self.assertEqual(str(AtomicType("TIME(0)")), "TIME(0)")
+        self.assertEqual(str(AtomicType("TIMESTAMP(0)")), "TIMESTAMP(0)")
+        self.assertEqual(str(AtomicType("SMALLINT", nullable=False)),
+                         str(AtomicType.from_dict(AtomicType("SMALLINT", nullable=False).to_dict())))
+        self.assertEqual(str(AtomicType("INT")),
+                         str(AtomicType.from_dict(AtomicType("INT").to_dict())))
+
+    @parameterized.expand([
+        (ArrayType, AtomicType("TIMESTAMP(6)"), "ARRAY<TIMESTAMP(6)>", "ARRAY<ARRAY<TIMESTAMP(6)>>"),
+        (MultisetType, AtomicType("TIMESTAMP(6)"), "MULTISET<TIMESTAMP(6)>", "MULTISET<MULTISET<TIMESTAMP(6)>>")
+    ])
+    def test_complex_types(self, data_type_class, element_type, expected1, expected2):
+        self.assertEqual(str(data_type_class(True, element_type)), expected1)
+        self.assertEqual(str(data_type_class(True, data_type_class(True, element_type))), expected2)
+        self.assertEqual(str(data_type_class(False, element_type)), expected1 + " NOT NULL")
+        self.assertEqual(str(data_type_class(False, element_type)),
+                         str(data_type_class.from_dict(data_type_class(False, element_type).to_dict())))
+        self.assertEqual(str(data_type_class(True, element_type)),
+                         str(data_type_class.from_dict(data_type_class(True, element_type).to_dict())))
+
+    def test_map_type(self):
+        self.assertEqual(str(MapType(True, AtomicType("STRING"), AtomicType("TIMESTAMP(6)"))),
+                         "MAP<STRING, TIMESTAMP(6)>")
+
+    def test_row_type(self):
+        self.assertEqual(str(RowType(True, [DataField(0, "a", AtomicType("STRING"), "Someone's desc."),
+                                            DataField(1, "b", AtomicType("TIMESTAMP(6)"),)])),
+                         "ROW<a: STRING COMMENT Someone's desc., b: TIMESTAMP(6)>")
+        row_data = RowType(True, [DataField(0, "a", AtomicType("STRING"), "Someone's desc."),
+                                  DataField(1, "b", AtomicType("TIMESTAMP(6)"),)])
+        self.assertEqual(str(row_data),
+                         str(RowType.from_dict(row_data.to_dict())))


(paimon) 07/18: [python] support custom source split target size and split open file cost (#6527)

Posted by lz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

lzljs3620320 pushed a commit to branch release-1.3
in repository https://gitbox.apache.org/repos/asf/paimon.git

commit e6ea490bf9bec519c65e2ff4578dc9a180ddc7fa
Author: XiaoHongbo <13...@qq.com>
AuthorDate: Wed Nov 5 17:54:46 2025 +0800

    [python] support custom source split target size and split open file cost (#6527)
---
 paimon-python/pypaimon/common/core_options.py      |  20 ++
 paimon-python/pypaimon/common/memory_size.py       | 201 +++++++++++++++++++++
 .../pypaimon/read/scanner/full_starting_scanner.py |   5 +-
 paimon-python/pypaimon/tests/reader_base_test.py   | 132 ++++++++++++++
 4 files changed, 356 insertions(+), 2 deletions(-)

diff --git a/paimon-python/pypaimon/common/core_options.py b/paimon-python/pypaimon/common/core_options.py
index aab339e269..d6643399bb 100644
--- a/paimon-python/pypaimon/common/core_options.py
+++ b/paimon-python/pypaimon/common/core_options.py
@@ -18,6 +18,8 @@
 
 from enum import Enum
 
+from pypaimon.common.memory_size import MemorySize
+
 
 class CoreOptions(str, Enum):
     """Core options for paimon."""
@@ -48,6 +50,8 @@ class CoreOptions(str, Enum):
     # Scan options
     SCAN_FALLBACK_BRANCH = "scan.fallback-branch"
     INCREMENTAL_BETWEEN_TIMESTAMP = "incremental-between-timestamp"
+    SOURCE_SPLIT_TARGET_SIZE = "source.split.target-size"
+    SOURCE_SPLIT_OPEN_FILE_COST = "source.split.open-file-cost"
     # Commit options
     COMMIT_USER_PREFIX = "commit.user-prefix"
     ROW_TRACKING_ENABLED = "row-tracking.enabled"
@@ -56,3 +60,19 @@ class CoreOptions(str, Enum):
     @staticmethod
     def get_blob_as_descriptor(options: dict) -> bool:
         return options.get(CoreOptions.FILE_BLOB_AS_DESCRIPTOR, "false").lower() == 'true'
+
+    @staticmethod
+    def get_split_target_size(options: dict) -> int:
+        """Get split target size from options, default to 128MB."""
+        if CoreOptions.SOURCE_SPLIT_TARGET_SIZE in options:
+            size_str = options[CoreOptions.SOURCE_SPLIT_TARGET_SIZE]
+            return MemorySize.parse(size_str).get_bytes()
+        return MemorySize.of_mebi_bytes(128).get_bytes()
+
+    @staticmethod
+    def get_split_open_file_cost(options: dict) -> int:
+        """Get split open file cost from options, default to 4MB."""
+        if CoreOptions.SOURCE_SPLIT_OPEN_FILE_COST in options:
+            cost_str = options[CoreOptions.SOURCE_SPLIT_OPEN_FILE_COST]
+            return MemorySize.parse(cost_str).get_bytes()
+        return MemorySize.of_mebi_bytes(4).get_bytes()
diff --git a/paimon-python/pypaimon/common/memory_size.py b/paimon-python/pypaimon/common/memory_size.py
new file mode 100644
index 0000000000..b68ccfc210
--- /dev/null
+++ b/paimon-python/pypaimon/common/memory_size.py
@@ -0,0 +1,201 @@
+################################################################################
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+from typing import Optional
+
+
+class MemorySize:
+    """MemorySize is a representation of a number of bytes, viewable in different units."""
+
+    ZERO = None
+    MAX_VALUE = None
+
+    def __init__(self, bytes: int):
+        """Constructs a new MemorySize."""
+        if bytes < 0:
+            raise ValueError("bytes must be >= 0")
+        self.bytes = bytes
+
+    @staticmethod
+    def of_mebi_bytes(mebi_bytes: int) -> 'MemorySize':
+        return MemorySize(mebi_bytes << 20)
+
+    @staticmethod
+    def of_kibi_bytes(kibi_bytes: int) -> 'MemorySize':
+        return MemorySize(kibi_bytes << 10)
+
+    @staticmethod
+    def of_bytes(bytes: int) -> 'MemorySize':
+        return MemorySize(bytes)
+
+    def get_bytes(self) -> int:
+        return self.bytes
+
+    def get_kibi_bytes(self) -> int:
+        return self.bytes >> 10
+
+    def get_mebi_bytes(self) -> int:
+        return self.bytes >> 20
+
+    def get_gibi_bytes(self) -> int:
+        return self.bytes >> 30
+
+    def get_tebi_bytes(self) -> int:
+        return self.bytes >> 40
+
+    def __eq__(self, other) -> bool:
+        return isinstance(other, MemorySize) and self.bytes == other.bytes
+
+    def __hash__(self) -> int:
+        return hash(self.bytes)
+
+    def __str__(self) -> str:
+        return self.format_to_string()
+
+    def format_to_string(self) -> str:
+        ORDERED_UNITS = [MemoryUnit.BYTES, MemoryUnit.KILO_BYTES, MemoryUnit.MEGA_BYTES,
+                         MemoryUnit.GIGA_BYTES, MemoryUnit.TERA_BYTES]
+
+        highest_integer_unit = MemoryUnit.BYTES
+        for idx, unit in enumerate(ORDERED_UNITS):
+            if self.bytes % unit.multiplier != 0:
+                if idx == 0:
+                    highest_integer_unit = ORDERED_UNITS[0]
+                else:
+                    highest_integer_unit = ORDERED_UNITS[idx - 1]
+                break
+        else:
+            highest_integer_unit = MemoryUnit.BYTES
+
+        return f"{self.bytes // highest_integer_unit.multiplier} {highest_integer_unit.units[1]}"
+
+    def __repr__(self) -> str:
+        return f"MemorySize({self.bytes})"
+
+    def __lt__(self, other: 'MemorySize') -> bool:
+        return self.bytes < other.bytes
+
+    def __le__(self, other: 'MemorySize') -> bool:
+        return self.bytes <= other.bytes
+
+    def __gt__(self, other: 'MemorySize') -> bool:
+        return self.bytes > other.bytes
+
+    def __ge__(self, other: 'MemorySize') -> bool:
+        return self.bytes >= other.bytes
+
+    @staticmethod
+    def parse(text: str) -> 'MemorySize':
+        return MemorySize(MemorySize.parse_bytes(text))
+
+    @staticmethod
+    def parse_bytes(text: str) -> int:
+        if text is None:
+            raise ValueError("text cannot be None")
+
+        trimmed = text.strip()
+        if not trimmed:
+            raise ValueError("argument is an empty- or whitespace-only string")
+
+        pos = 0
+        while pos < len(trimmed) and trimmed[pos].isdigit():
+            pos += 1
+
+        number_str = trimmed[:pos]
+        unit_str = trimmed[pos:].strip().lower()
+
+        if not number_str:
+            raise ValueError("text does not start with a number")
+
+        try:
+            value = int(number_str)
+        except ValueError:
+            raise ValueError(
+                f"The value '{number_str}' cannot be represented as 64bit number (numeric overflow).")
+
+        unit = MemorySize._parse_unit(unit_str)
+        multiplier = unit.multiplier if unit else 1
+        result = value * multiplier
+
+        if result // multiplier != value:
+            raise ValueError(
+                f"The value '{text}' cannot be represented as 64bit number of bytes (numeric overflow).")
+
+        return result
+
+    @staticmethod
+    def _parse_unit(unit_str: str) -> Optional['MemoryUnit']:
+        if not unit_str:
+            return None
+
+        for unit in [MemoryUnit.BYTES, MemoryUnit.KILO_BYTES, MemoryUnit.MEGA_BYTES,
+                     MemoryUnit.GIGA_BYTES, MemoryUnit.TERA_BYTES]:
+            if unit_str in unit.units:
+                return unit
+
+        raise ValueError(
+            f"Memory size unit '{unit_str}' does not match any of the recognized units: "
+            f"{MemoryUnit.get_all_units()}")
+
+
+class MemoryUnit:
+    """Enum which defines memory unit, mostly used to parse value from configuration file."""
+
+    def __init__(self, units: list, multiplier: int):
+        self.units = units
+        self.multiplier = multiplier
+
+    BYTES = None
+    KILO_BYTES = None
+    MEGA_BYTES = None
+    GIGA_BYTES = None
+    TERA_BYTES = None
+
+    @staticmethod
+    def get_all_units() -> str:
+        all_units = []
+        for unit in [MemoryUnit.BYTES, MemoryUnit.KILO_BYTES, MemoryUnit.MEGA_BYTES,
+                     MemoryUnit.GIGA_BYTES, MemoryUnit.TERA_BYTES]:
+            all_units.append("(" + " | ".join(unit.units) + ")")
+        return " / ".join(all_units)
+
+    @staticmethod
+    def has_unit(text: str) -> bool:
+        if text is None:
+            raise ValueError("text cannot be None")
+
+        trimmed = text.strip()
+        if not trimmed:
+            raise ValueError("argument is an empty- or whitespace-only string")
+
+        pos = 0
+        while pos < len(trimmed) and trimmed[pos].isdigit():
+            pos += 1
+
+        unit = trimmed[pos:].strip().lower()
+        return len(unit) > 0
+
+
+MemoryUnit.BYTES = MemoryUnit(["b", "bytes"], 1)
+MemoryUnit.KILO_BYTES = MemoryUnit(["k", "kb", "kibibytes"], 1024)
+MemoryUnit.MEGA_BYTES = MemoryUnit(["m", "mb", "mebibytes"], 1024 * 1024)
+MemoryUnit.GIGA_BYTES = MemoryUnit(["g", "gb", "gibibytes"], 1024 * 1024 * 1024)
+MemoryUnit.TERA_BYTES = MemoryUnit(["t", "tb", "tebibytes"], 1024 * 1024 * 1024 * 1024)
+
+MemorySize.ZERO = MemorySize(0)
+MemorySize.MAX_VALUE = MemorySize(2**63 - 1)
diff --git a/paimon-python/pypaimon/read/scanner/full_starting_scanner.py b/paimon-python/pypaimon/read/scanner/full_starting_scanner.py
index b3c18a7bb6..dfbe329f43 100644
--- a/paimon-python/pypaimon/read/scanner/full_starting_scanner.py
+++ b/paimon-python/pypaimon/read/scanner/full_starting_scanner.py
@@ -54,8 +54,9 @@ class FullStartingScanner(StartingScanner):
         self.partition_key_predicate = trim_and_transform_predicate(
             self.predicate, self.table.field_names, self.table.partition_keys)
 
-        self.target_split_size = 128 * 1024 * 1024
-        self.open_file_cost = 4 * 1024 * 1024
+        # Get split target size and open file cost from table options
+        self.target_split_size = CoreOptions.get_split_target_size(self.table.options)
+        self.open_file_cost = CoreOptions.get_split_open_file_cost(self.table.options)
 
         self.idx_of_this_subtask = None
         self.number_of_para_subtasks = None
diff --git a/paimon-python/pypaimon/tests/reader_base_test.py b/paimon-python/pypaimon/tests/reader_base_test.py
index ebe98e4fc5..fdcb9f1ea2 100644
--- a/paimon-python/pypaimon/tests/reader_base_test.py
+++ b/paimon-python/pypaimon/tests/reader_base_test.py
@@ -675,3 +675,135 @@ class ReaderBasicTest(unittest.TestCase):
                 max_values)
 
         self.assertEqual(read_entry.file.value_stats.null_counts, null_counts)
+
+    def test_split_target_size(self):
+        """Test source.split.target-size configuration effect on split generation."""
+        from pypaimon.common.core_options import CoreOptions
+
+        pa_schema = pa.schema([
+            ('f0', pa.int64()),
+            ('f1', pa.string())
+        ])
+
+        # Test with small target_split_size (512B) - should generate more splits
+        schema_small = Schema.from_pyarrow_schema(
+            pa_schema,
+            options={CoreOptions.SOURCE_SPLIT_TARGET_SIZE: '512b'}
+        )
+        self.catalog.create_table('default.test_split_target_size_small', schema_small, False)
+        table_small = self.catalog.get_table('default.test_split_target_size_small')
+
+        for i in range(10):
+            write_builder = table_small.new_batch_write_builder()
+            table_write = write_builder.new_write()
+            table_commit = write_builder.new_commit()
+            data = pa.Table.from_pydict({
+                'f0': list(range(i * 100, (i + 1) * 100)),
+                'f1': [f'value_{j}' for j in range(i * 100, (i + 1) * 100)]
+            }, schema=pa_schema)
+            table_write.write_arrow(data)
+            table_commit.commit(table_write.prepare_commit())
+            table_write.close()
+            table_commit.close()
+
+        read_builder = table_small.new_read_builder()
+        splits_small = read_builder.new_scan().plan().splits()
+
+        schema_default = Schema.from_pyarrow_schema(pa_schema)
+        self.catalog.create_table('default.test_split_target_size_default', schema_default, False)
+        table_default = self.catalog.get_table('default.test_split_target_size_default')
+
+        for i in range(10):
+            write_builder = table_default.new_batch_write_builder()
+            table_write = write_builder.new_write()
+            table_commit = write_builder.new_commit()
+            data = pa.Table.from_pydict({
+                'f0': list(range(i * 100, (i + 1) * 100)),
+                'f1': [f'value_{j}' for j in range(i * 100, (i + 1) * 100)]
+            }, schema=pa_schema)
+            table_write.write_arrow(data)
+            table_commit.commit(table_write.prepare_commit())
+            table_write.close()
+            table_commit.close()
+
+        # Generate splits with default target_split_size
+        read_builder = table_default.new_read_builder()
+        splits_default = read_builder.new_scan().plan().splits()
+
+        self.assertGreater(
+            len(splits_small), len(splits_default),
+            f"Small target_split_size should generate more splits. "
+            f"Got {len(splits_small)} splits with 512B vs "
+            f"{len(splits_default)} splits with default")
+
+    def test_split_open_file_cost(self):
+        """Test source.split.open-file-cost configuration effect on split generation."""
+        from pypaimon.common.core_options import CoreOptions
+
+        pa_schema = pa.schema([
+            ('f0', pa.int64()),
+            ('f1', pa.string())
+        ])
+
+        # Test with large open_file_cost (64MB) - should generate more splits
+        schema_large_cost = Schema.from_pyarrow_schema(
+            pa_schema,
+            options={
+                CoreOptions.SOURCE_SPLIT_TARGET_SIZE: '128mb',
+                CoreOptions.SOURCE_SPLIT_OPEN_FILE_COST: '64mb'
+            }
+        )
+        self.catalog.create_table('default.test_split_open_file_cost_large', schema_large_cost, False)
+        table_large_cost = self.catalog.get_table('default.test_split_open_file_cost_large')
+
+        # Write multiple batches to create multiple files
+        # Write 10 batches, each with 100 rows
+        for i in range(10):
+            write_builder = table_large_cost.new_batch_write_builder()
+            table_write = write_builder.new_write()
+            table_commit = write_builder.new_commit()
+            data = pa.Table.from_pydict({
+                'f0': list(range(i * 100, (i + 1) * 100)),
+                'f1': [f'value_{j}' for j in range(i * 100, (i + 1) * 100)]
+            }, schema=pa_schema)
+            table_write.write_arrow(data)
+            table_commit.commit(table_write.prepare_commit())
+            table_write.close()
+            table_commit.close()
+
+        # Generate splits with large open_file_cost
+        read_builder = table_large_cost.new_read_builder()
+        splits_large_cost = read_builder.new_scan().plan().splits()
+
+        # Test with default open_file_cost (4MB) - should generate fewer splits
+        schema_default = Schema.from_pyarrow_schema(
+            pa_schema,
+            options={CoreOptions.SOURCE_SPLIT_TARGET_SIZE: '128mb'}
+        )
+        self.catalog.create_table('default.test_split_open_file_cost_default', schema_default, False)
+        table_default = self.catalog.get_table('default.test_split_open_file_cost_default')
+
+        # Write same amount of data
+        for i in range(10):
+            write_builder = table_default.new_batch_write_builder()
+            table_write = write_builder.new_write()
+            table_commit = write_builder.new_commit()
+            data = pa.Table.from_pydict({
+                'f0': list(range(i * 100, (i + 1) * 100)),
+                'f1': [f'value_{j}' for j in range(i * 100, (i + 1) * 100)]
+            }, schema=pa_schema)
+            table_write.write_arrow(data)
+            table_commit.commit(table_write.prepare_commit())
+            table_write.close()
+            table_commit.close()
+
+        # Generate splits with default open_file_cost
+        read_builder = table_default.new_read_builder()
+        splits_default = read_builder.new_scan().plan().splits()
+
+        # With default open_file_cost (4MB), more files can be packed into each split
+        self.assertGreater(
+            len(splits_large_cost), len(splits_default),
+            f"Large open_file_cost should generate more splits. "
+            f"Got {len(splits_large_cost)} splits with 64MB cost vs "
+            f"{len(splits_default)} splits with default")


(paimon) 04/18: [Python] Keep the variable names of Identifier consistent with Java (#6520)

Posted by lz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

lzljs3620320 pushed a commit to branch release-1.3
in repository https://gitbox.apache.org/repos/asf/paimon.git

commit f97750d1962b71dd94c8a9a2c85ee3334972af31
Author: universe-hcy <47...@users.noreply.github.com>
AuthorDate: Tue Nov 4 15:39:14 2025 +0800

    [Python] Keep the variable names of Identifier consistent with Java (#6520)
---
 paimon-python/pypaimon/api/rest_api.py             | 16 ++++++-------
 paimon-python/pypaimon/common/identifier.py        | 26 +++++++++++-----------
 paimon-python/pypaimon/schema/data_types.py        |  6 ++---
 .../pypaimon/snapshot/catalog_snapshot_commit.py   |  6 ++---
 paimon-python/pypaimon/tests/rest/rest_server.py   | 18 +++++++++------
 5 files changed, 37 insertions(+), 35 deletions(-)

diff --git a/paimon-python/pypaimon/api/rest_api.py b/paimon-python/pypaimon/api/rest_api.py
old mode 100644
new mode 100755
index 037a531e58..d715c8a4aa
--- a/paimon-python/pypaimon/api/rest_api.py
+++ b/paimon-python/pypaimon/api/rest_api.py
@@ -219,15 +219,15 @@ class RESTApi:
     def create_table(self, identifier: Identifier, schema: Schema) -> None:
         request = CreateTableRequest(identifier, schema)
         return self.client.post(
-            self.resource_paths.tables(identifier.database_name),
+            self.resource_paths.tables(identifier.get_database_name()),
             request,
             self.rest_auth_function)
 
     def get_table(self, identifier: Identifier) -> GetTableResponse:
         return self.client.get(
             self.resource_paths.table(
-                identifier.database_name,
-                identifier.object_name),
+                identifier.get_database_name(),
+                identifier.get_object_name()),
             GetTableResponse,
             self.rest_auth_function,
         )
@@ -235,8 +235,8 @@ class RESTApi:
     def drop_table(self, identifier: Identifier) -> GetTableResponse:
         return self.client.delete(
             self.resource_paths.table(
-                identifier.database_name,
-                identifier.object_name),
+                identifier.get_database_name(),
+                identifier.get_object_name()),
             self.rest_auth_function,
         )
 
@@ -250,8 +250,8 @@ class RESTApi:
     def load_table_token(self, identifier: Identifier) -> GetTableTokenResponse:
         return self.client.get(
             self.resource_paths.table_token(
-                identifier.database_name,
-                identifier.object_name),
+                identifier.get_database_name(),
+                identifier.get_object_name()),
             GetTableTokenResponse,
             self.rest_auth_function,
         )
@@ -282,7 +282,7 @@ class RESTApi:
         request = CommitTableRequest(table_uuid, snapshot, statistics)
         response = self.client.post_with_response_type(
             self.resource_paths.commit_table(
-                identifier.database_name, identifier.object_name),
+                identifier.get_database_name(), identifier.get_object_name()),
             request,
             CommitTableResponse,
             self.rest_auth_function
diff --git a/paimon-python/pypaimon/common/identifier.py b/paimon-python/pypaimon/common/identifier.py
old mode 100644
new mode 100755
index 0731db50db..6851a18074
--- a/paimon-python/pypaimon/common/identifier.py
+++ b/paimon-python/pypaimon/common/identifier.py
@@ -27,13 +27,13 @@ SYSTEM_BRANCH_PREFIX = 'branch-'
 @dataclass
 class Identifier:
 
-    database_name: str = json_field("database", default=None)
-    object_name: str = json_field("object", default=None)
-    branch_name: Optional[str] = json_field("branch", default=None)
+    database: str = json_field("database", default=None)
+    object: str = json_field("object", default=None)
+    branch: Optional[str] = json_field("branch", default=None)
 
     @classmethod
-    def create(cls, database_name: str, object_name: str) -> "Identifier":
-        return cls(database_name, object_name)
+    def create(cls, database: str, object: str) -> "Identifier":
+        return cls(database, object)
 
     @classmethod
     def from_string(cls, full_name: str) -> "Identifier":
@@ -46,21 +46,21 @@ class Identifier:
             raise ValueError("Invalid identifier format: {}".format(full_name))
 
     def get_full_name(self) -> str:
-        if self.branch_name:
-            return "{}.{}.{}".format(self.database_name, self.object_name, self.branch_name)
-        return "{}.{}".format(self.database_name, self.object_name)
+        if self.branch:
+            return "{}.{}.{}".format(self.database, self.object, self.branch)
+        return "{}.{}".format(self.database, self.object)
 
     def get_database_name(self) -> str:
-        return self.database_name
+        return self.database
 
     def get_table_name(self) -> str:
-        return self.object_name
+        return self.object
 
     def get_object_name(self) -> str:
-        return self.object_name
+        return self.object
 
     def get_branch_name(self) -> Optional[str]:
-        return self.branch_name
+        return self.branch
 
     def is_system_table(self) -> bool:
-        return self.object_name.startswith('$')
+        return self.object.startswith('$')
diff --git a/paimon-python/pypaimon/schema/data_types.py b/paimon-python/pypaimon/schema/data_types.py
old mode 100644
new mode 100755
index c65771d7f2..3f7b30c291
--- a/paimon-python/pypaimon/schema/data_types.py
+++ b/paimon-python/pypaimon/schema/data_types.py
@@ -73,10 +73,8 @@ class AtomicType(DataType):
         super().__init__(nullable)
         self.type = type
 
-    def to_dict(self) -> str:
-        if not self.nullable:
-            return self.type + " NOT NULL"
-        return self.type
+    def to_dict(self) -> Dict[str, Any]:
+        return {"type": self.type if self.nullable else self.type + " NOT NULL"}
 
     @classmethod
     def from_dict(cls, data: str) -> "AtomicType":
diff --git a/paimon-python/pypaimon/snapshot/catalog_snapshot_commit.py b/paimon-python/pypaimon/snapshot/catalog_snapshot_commit.py
old mode 100644
new mode 100755
index 26796f7766..1ffc80af57
--- a/paimon-python/pypaimon/snapshot/catalog_snapshot_commit.py
+++ b/paimon-python/pypaimon/snapshot/catalog_snapshot_commit.py
@@ -57,9 +57,9 @@ class CatalogSnapshotCommit(SnapshotCommit):
             Exception: If commit fails
         """
         new_identifier = Identifier(
-            database_name=self.identifier.get_database_name(),
-            object_name=self.identifier.get_table_name(),
-            branch_name=branch
+            database=self.identifier.get_database_name(),
+            object=self.identifier.get_table_name(),
+            branch=branch
         )
 
         # Call catalog's commit_snapshot method
diff --git a/paimon-python/pypaimon/tests/rest/rest_server.py b/paimon-python/pypaimon/tests/rest/rest_server.py
old mode 100644
new mode 100755
index d2908e59be..1970b79dd0
--- a/paimon-python/pypaimon/tests/rest/rest_server.py
+++ b/paimon-python/pypaimon/tests/rest/rest_server.py
@@ -253,9 +253,9 @@ class RESTCatalogServer:
                 source = self.table_metadata_store.get(source_table.get_full_name())
                 self.table_metadata_store.update({destination_table.get_full_name(): source})
                 source_table_dir = (Path(self.data_path) / self.warehouse
-                                    / source_table.database_name / source_table.object_name)
+                                    / source_table.get_database_name() / source_table.get_object_name())
                 destination_table_dir = (Path(self.data_path) / self.warehouse
-                                         / destination_table.database_name / destination_table.object_name)
+                                         / destination_table.get_database_name() / destination_table.get_object_name())
                 if not source_table_dir.exists():
                     destination_table_dir.mkdir(parents=True)
                 else:
@@ -352,7 +352,7 @@ class RESTCatalogServer:
                 table_name_part = table_parts[0]
                 branch_part = table_parts[1]
                 # Recreate identifier without branch for lookup
-                lookup_identifier = Identifier.create(identifier.database_name, table_name_part)
+                lookup_identifier = Identifier.create(identifier.get_database_name(), table_name_part)
             else:
                 lookup_identifier = identifier
                 branch_part = None
@@ -431,8 +431,10 @@ class RESTCatalogServer:
                     create_table.identifier, 0, create_table.schema, str(uuid.uuid4()), False
                 )
                 self.table_metadata_store.update({create_table.identifier.get_full_name(): table_metadata})
-                table_dir = Path(
-                    self.data_path) / self.warehouse / database_name / create_table.identifier.object_name / 'schema'
+                table_dir = (
+                    Path(self.data_path) / self.warehouse / database_name /
+                    create_table.identifier.get_object_name() / 'schema'
+                )
                 if not table_dir.exists():
                     table_dir.mkdir(parents=True)
                 with open(table_dir / "schema-0", "w") as f:
@@ -446,7 +448,8 @@ class RESTCatalogServer:
             if identifier.get_full_name() not in self.table_metadata_store:
                 raise TableNotExistException(identifier)
             table_metadata = self.table_metadata_store[identifier.get_full_name()]
-            table_path = f'file://{self.data_path}/{self.warehouse}/{identifier.database_name}/{identifier.object_name}'
+            table_path = (f'file://{self.data_path}/{self.warehouse}/'
+                          f'{identifier.get_database_name()}/{identifier.get_object_name()}')
             schema = table_metadata.schema.to_schema()
             response = self.mock_table(identifier, table_metadata, table_path, schema)
             return self._mock_response(response, 200)
@@ -522,7 +525,8 @@ class RESTCatalogServer:
         import uuid
 
         # Construct table path: {warehouse}/{database}/{table}
-        table_path = os.path.join(self.data_path, self.warehouse, identifier.database_name, identifier.object_name)
+        table_path = os.path.join(self.data_path, self.warehouse, identifier.get_database_name(),
+                                  identifier.get_object_name())
 
         # Create directory structure
         snapshot_dir = os.path.join(table_path, "snapshot")


(paimon) 02/18: [hotfix] Add more informat to check partition spec in InternalRowPartitionComputer

Posted by lz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

lzljs3620320 pushed a commit to branch release-1.3
in repository https://gitbox.apache.org/repos/asf/paimon.git

commit 3d09deb93d45e3b6cdd01d41accf9ddd2bc8afa0
Author: JingsongLi <ji...@gmail.com>
AuthorDate: Mon Nov 3 22:09:00 2025 +0800

    [hotfix] Add more informat to check partition spec in InternalRowPartitionComputer
---
 .../java/org/apache/paimon/utils/InternalRowPartitionComputer.java  | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/paimon-common/src/main/java/org/apache/paimon/utils/InternalRowPartitionComputer.java b/paimon-common/src/main/java/org/apache/paimon/utils/InternalRowPartitionComputer.java
index edefce3189..0f207aaa64 100644
--- a/paimon-common/src/main/java/org/apache/paimon/utils/InternalRowPartitionComputer.java
+++ b/paimon-common/src/main/java/org/apache/paimon/utils/InternalRowPartitionComputer.java
@@ -107,7 +107,11 @@ public class InternalRowPartitionComputer {
 
     public static GenericRow convertSpecToInternalRow(
             Map<String, String> spec, RowType partType, String defaultPartValue) {
-        checkArgument(spec.size() == partType.getFieldCount());
+        checkArgument(
+                spec.size() == partType.getFieldCount(),
+                "Partition spec size %s not match partition field count %s",
+                spec.size(),
+                partType.getFieldCount());
         GenericRow partRow = new GenericRow(spec.size());
         List<String> fieldNames = partType.getFieldNames();
         for (Map.Entry<String, String> entry : spec.entrySet()) {


(paimon) 16/18: [core] support incremental clustering in dv mode (#6559)

Posted by lz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

lzljs3620320 pushed a commit to branch release-1.3
in repository https://gitbox.apache.org/repos/asf/paimon.git

commit a47a28ec38550f319aee467206f0c5e1b4cf028d
Author: LsomeYeah <94...@users.noreply.github.com>
AuthorDate: Mon Nov 10 17:46:30 2025 +0800

    [core] support incremental clustering in dv mode (#6559)
---
 .../append/cluster/HistoryPartitionCluster.java    |  10 +-
 .../append/cluster/IncrementalClusterManager.java  |  97 +++++++++++++-
 .../org/apache/paimon/schema/SchemaValidation.java |   3 -
 .../cluster/IncrementalClusterManagerTest.java     |   7 +-
 .../apache/paimon/flink/action/CompactAction.java  |  22 +++-
 .../cluster/IncrementalClusterSplitSource.java     |   6 +-
 .../cluster/RemoveClusterBeforeFilesOperator.java  |  20 ++-
 .../action/IncrementalClusterActionITCase.java     | 142 +++++++++++++++++++++
 .../paimon/spark/procedure/CompactProcedure.java   |  39 +++++-
 .../spark/procedure/CompactProcedureTestBase.scala | 136 ++++++++++++++++++++
 10 files changed, 464 insertions(+), 18 deletions(-)

diff --git a/paimon-core/src/main/java/org/apache/paimon/append/cluster/HistoryPartitionCluster.java b/paimon-core/src/main/java/org/apache/paimon/append/cluster/HistoryPartitionCluster.java
index 83f92aa9c7..8acbb8e778 100644
--- a/paimon-core/src/main/java/org/apache/paimon/append/cluster/HistoryPartitionCluster.java
+++ b/paimon-core/src/main/java/org/apache/paimon/append/cluster/HistoryPartitionCluster.java
@@ -18,6 +18,7 @@
 
 package org.apache.paimon.append.cluster;
 
+import org.apache.paimon.Snapshot;
 import org.apache.paimon.annotation.VisibleForTesting;
 import org.apache.paimon.compact.CompactUnit;
 import org.apache.paimon.data.BinaryRow;
@@ -54,6 +55,7 @@ public class HistoryPartitionCluster {
     private static final Logger LOG = LoggerFactory.getLogger(HistoryPartitionCluster.class);
 
     private final FileStoreTable table;
+    private final Snapshot snapshot;
     private final IncrementalClusterStrategy incrementalClusterStrategy;
     private final InternalRowPartitionComputer partitionComputer;
     private final PartitionPredicate specifiedPartitions;
@@ -63,12 +65,14 @@ public class HistoryPartitionCluster {
 
     public HistoryPartitionCluster(
             FileStoreTable table,
+            Snapshot snapshot,
             IncrementalClusterStrategy incrementalClusterStrategy,
             InternalRowPartitionComputer partitionComputer,
             PartitionPredicate specifiedPartitions,
             Duration historyPartitionIdleTime,
             int historyPartitionLimit) {
         this.table = table;
+        this.snapshot = snapshot;
         this.incrementalClusterStrategy = incrementalClusterStrategy;
         this.partitionComputer = partitionComputer;
         this.specifiedPartitions = specifiedPartitions;
@@ -80,6 +84,7 @@ public class HistoryPartitionCluster {
     @Nullable
     public static HistoryPartitionCluster create(
             FileStoreTable table,
+            Snapshot snapshot,
             IncrementalClusterStrategy incrementalClusterStrategy,
             InternalRowPartitionComputer partitionComputer,
             @Nullable PartitionPredicate specifiedPartitions) {
@@ -98,6 +103,7 @@ public class HistoryPartitionCluster {
         int limit = table.coreOptions().clusteringHistoryPartitionLimit();
         return new HistoryPartitionCluster(
                 table,
+                snapshot,
                 incrementalClusterStrategy,
                 partitionComputer,
                 specifiedPartitions,
@@ -130,7 +136,8 @@ public class HistoryPartitionCluster {
                         .toEpochMilli();
 
         List<BinaryRow> historyPartitions =
-                table.newSnapshotReader().withLevelMinMaxFilter((min, max) -> min < maxLevel)
+                table.newSnapshotReader().withSnapshot(snapshot)
+                        .withLevelMinMaxFilter((min, max) -> min < maxLevel)
                         .withLevelFilter(level -> level < maxLevel).partitionEntries().stream()
                         .filter(entry -> entry.lastFileCreationTime() < historyMilli)
                         .sorted(Comparator.comparingLong(PartitionEntry::lastFileCreationTime))
@@ -140,6 +147,7 @@ public class HistoryPartitionCluster {
         // read dataFileMeta for history partitions
         List<DataSplit> historyDataSplits =
                 table.newSnapshotReader()
+                        .withSnapshot(snapshot)
                         .withPartitionFilter(historyPartitions)
                         .read()
                         .dataSplits();
diff --git a/paimon-core/src/main/java/org/apache/paimon/append/cluster/IncrementalClusterManager.java b/paimon-core/src/main/java/org/apache/paimon/append/cluster/IncrementalClusterManager.java
index 6865b4b82e..f8ab6f364b 100644
--- a/paimon-core/src/main/java/org/apache/paimon/append/cluster/IncrementalClusterManager.java
+++ b/paimon-core/src/main/java/org/apache/paimon/append/cluster/IncrementalClusterManager.java
@@ -19,16 +19,27 @@
 package org.apache.paimon.append.cluster;
 
 import org.apache.paimon.CoreOptions;
+import org.apache.paimon.Snapshot;
 import org.apache.paimon.annotation.VisibleForTesting;
 import org.apache.paimon.compact.CompactUnit;
 import org.apache.paimon.data.BinaryRow;
+import org.apache.paimon.deletionvectors.append.AppendDeleteFileMaintainer;
+import org.apache.paimon.deletionvectors.append.BaseAppendDeleteFileMaintainer;
+import org.apache.paimon.index.IndexFileMeta;
+import org.apache.paimon.io.CompactIncrement;
 import org.apache.paimon.io.DataFileMeta;
+import org.apache.paimon.io.DataIncrement;
+import org.apache.paimon.manifest.FileKind;
+import org.apache.paimon.manifest.IndexManifestEntry;
 import org.apache.paimon.mergetree.LevelSortedRun;
 import org.apache.paimon.mergetree.SortedRun;
 import org.apache.paimon.partition.PartitionPredicate;
 import org.apache.paimon.table.BucketMode;
 import org.apache.paimon.table.FileStoreTable;
+import org.apache.paimon.table.sink.CommitMessage;
+import org.apache.paimon.table.sink.CommitMessageImpl;
 import org.apache.paimon.table.source.DataSplit;
+import org.apache.paimon.table.source.DeletionFile;
 import org.apache.paimon.table.source.SplitGenerator;
 import org.apache.paimon.table.source.snapshot.SnapshotReader;
 import org.apache.paimon.utils.InternalRowPartitionComputer;
@@ -39,11 +50,13 @@ import org.slf4j.LoggerFactory;
 import javax.annotation.Nullable;
 
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
+import java.util.Set;
 import java.util.stream.Collectors;
 
 import static org.apache.paimon.CoreOptions.CLUSTERING_INCREMENTAL;
@@ -55,6 +68,7 @@ public class IncrementalClusterManager {
     private static final Logger LOG = LoggerFactory.getLogger(IncrementalClusterManager.class);
 
     private final InternalRowPartitionComputer partitionComputer;
+    private final Snapshot snapshot;
     private final SnapshotReader snapshotReader;
     private final IncrementalClusterStrategy incrementalClusterStrategy;
     private final CoreOptions.OrderType clusterCurve;
@@ -83,8 +97,12 @@ public class IncrementalClusterManager {
                         table.store().partitionType(),
                         table.partitionKeys().toArray(new String[0]),
                         table.coreOptions().legacyPartitionName());
+        this.snapshot = table.snapshotManager().latestSnapshot();
         this.snapshotReader =
-                table.newSnapshotReader().dropStats().withPartitionFilter(specifiedPartitions);
+                table.newSnapshotReader()
+                        .dropStats()
+                        .withPartitionFilter(specifiedPartitions)
+                        .withSnapshot(snapshot);
         this.incrementalClusterStrategy =
                 new IncrementalClusterStrategy(
                         table.schemaManager(),
@@ -96,7 +114,11 @@ public class IncrementalClusterManager {
         this.clusterKeys = options.clusteringColumns();
         this.historyPartitionCluster =
                 HistoryPartitionCluster.create(
-                        table, incrementalClusterStrategy, partitionComputer, specifiedPartitions);
+                        table,
+                        snapshot,
+                        incrementalClusterStrategy,
+                        partitionComputer,
+                        specifiedPartitions);
     }
 
     public Map<BinaryRow, CompactUnit> prepareForCluster(boolean fullCompaction) {
@@ -182,7 +204,10 @@ public class IncrementalClusterManager {
         return partitionLevels;
     }
 
-    public List<DataSplit> toSplits(BinaryRow partition, List<DataFileMeta> files) {
+    public List<DataSplit> toSplits(
+            BinaryRow partition,
+            List<DataFileMeta> files,
+            @Nullable AppendDeleteFileMaintainer dvIndexFileMaintainer) {
         List<DataSplit> splits = new ArrayList<>();
 
         DataSplit.Builder builder =
@@ -202,6 +227,14 @@ public class IncrementalClusterManager {
                     .rawConvertible(splitGroup.rawConvertible)
                     .withBucketPath(bucketPath);
 
+            if (dvIndexFileMaintainer != null) {
+                List<DeletionFile> dataDeletionFiles = new ArrayList<>();
+                for (DataFileMeta file : dataFiles) {
+                    dataDeletionFiles.add(dvIndexFileMaintainer.getDeletionFile(file.fileName()));
+                }
+                builder.withDataDeletionFiles(dataDeletionFiles);
+            }
+
             splits.add(builder.build());
         }
 
@@ -215,6 +248,60 @@ public class IncrementalClusterManager {
                 .collect(Collectors.toList());
     }
 
+    public static Map<BinaryRow, AppendDeleteFileMaintainer> createAppendDvMaintainers(
+            FileStoreTable table, Set<BinaryRow> partitions, Snapshot snapshot) {
+        Map<BinaryRow, AppendDeleteFileMaintainer> dvMaintainers = new HashMap<>();
+        for (BinaryRow partition : partitions) {
+            AppendDeleteFileMaintainer appendDvMaintainer =
+                    BaseAppendDeleteFileMaintainer.forUnawareAppend(
+                            table.store().newIndexFileHandler(), snapshot, partition);
+            dvMaintainers.put(partition, appendDvMaintainer);
+        }
+        return dvMaintainers;
+    }
+
+    public static List<CommitMessage> producePartitionDvIndexCommitMessages(
+            FileStoreTable table,
+            List<DataSplit> splits,
+            AppendDeleteFileMaintainer appendDvMaintainer) {
+        checkArgument(appendDvMaintainer != null);
+        // remove deletion vector for files to be clustered
+        for (DataSplit dataSplit : splits) {
+            checkArgument(
+                    dataSplit.partition().equals(appendDvMaintainer.getPartition()),
+                    "partition of this dataSplit is not matched with the Dv Maintainer!");
+            dataSplit
+                    .dataFiles()
+                    .forEach(f -> appendDvMaintainer.notifyRemovedDeletionVector(f.fileName()));
+        }
+
+        // generate new dv index meta, handle by partition
+        List<IndexFileMeta> newIndexFiles = new ArrayList<>();
+        List<IndexFileMeta> deletedIndexFiles = new ArrayList<>();
+        List<IndexManifestEntry> indexEntries = appendDvMaintainer.persist();
+        for (IndexManifestEntry entry : indexEntries) {
+            if (entry.kind() == FileKind.ADD) {
+                newIndexFiles.add(entry.indexFile());
+            } else {
+                deletedIndexFiles.add(entry.indexFile());
+            }
+        }
+        CommitMessageImpl commitMessage =
+                new CommitMessageImpl(
+                        appendDvMaintainer.getPartition(),
+                        0,
+                        table.coreOptions().bucket(),
+                        DataIncrement.emptyIncrement(),
+                        new CompactIncrement(
+                                Collections.emptyList(),
+                                Collections.emptyList(),
+                                Collections.emptyList(),
+                                newIndexFiles,
+                                deletedIndexFiles));
+
+        return Collections.singletonList(commitMessage);
+    }
+
     public static void logForPartitionLevel(
             Map<BinaryRow, List<LevelSortedRun>> partitionLevels,
             InternalRowPartitionComputer partitionComputer) {
@@ -247,6 +334,10 @@ public class IncrementalClusterManager {
         return clusterKeys;
     }
 
+    public Snapshot snapshot() {
+        return snapshot;
+    }
+
     @VisibleForTesting
     HistoryPartitionCluster historyPartitionCluster() {
         return historyPartitionCluster;
diff --git a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java
index 572e8e17ee..09de891a37 100644
--- a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java
+++ b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java
@@ -661,9 +661,6 @@ public class SchemaValidation {
                     schema.primaryKeys().isEmpty(),
                     "Cannot define %s for incremental clustering table.",
                     PRIMARY_KEY.key());
-            checkArgument(
-                    !options.deletionVectorsEnabled(),
-                    "Cannot enable deletion vectors mode for incremental clustering table.");
         }
     }
 }
diff --git a/paimon-core/src/test/java/org/apache/paimon/append/cluster/IncrementalClusterManagerTest.java b/paimon-core/src/test/java/org/apache/paimon/append/cluster/IncrementalClusterManagerTest.java
index 4b62caab88..a4d1071092 100644
--- a/paimon-core/src/test/java/org/apache/paimon/append/cluster/IncrementalClusterManagerTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/append/cluster/IncrementalClusterManagerTest.java
@@ -134,12 +134,7 @@ public class IncrementalClusterManagerTest {
     }
 
     @Test
-    public void testUpgrade() throws Exception {
-        // Create a valid table for IncrementalClusterManager
-        Map<String, String> options = new HashMap<>();
-        FileStoreTable table = createTable(options, Collections.emptyList());
-        IncrementalClusterManager incrementalClusterManager = new IncrementalClusterManager(table);
-
+    public void testUpgrade() {
         // Create test files with different levels
         List<DataFileMeta> filesAfterCluster = new ArrayList<>();
         DataFileMeta file1 = createFile(100, 1, 0);
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/CompactAction.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/CompactAction.java
index 6fdd790651..4a914e8ac9 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/CompactAction.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/CompactAction.java
@@ -23,6 +23,7 @@ import org.apache.paimon.append.cluster.IncrementalClusterManager;
 import org.apache.paimon.compact.CompactUnit;
 import org.apache.paimon.data.BinaryRow;
 import org.apache.paimon.data.InternalRow;
+import org.apache.paimon.deletionvectors.append.AppendDeleteFileMaintainer;
 import org.apache.paimon.flink.FlinkConnectorOptions;
 import org.apache.paimon.flink.cluster.IncrementalClusterSplitSource;
 import org.apache.paimon.flink.cluster.RewriteIncrementalClusterCommittableOperator;
@@ -50,6 +51,7 @@ import org.apache.paimon.predicate.PredicateBuilder;
 import org.apache.paimon.predicate.PredicateProjectionConverter;
 import org.apache.paimon.table.BucketMode;
 import org.apache.paimon.table.FileStoreTable;
+import org.apache.paimon.table.sink.CommitMessage;
 import org.apache.paimon.table.source.DataSplit;
 import org.apache.paimon.types.RowType;
 import org.apache.paimon.utils.InternalRowPartitionComputer;
@@ -69,6 +71,7 @@ import javax.annotation.Nullable;
 
 import java.time.Duration;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Iterator;
@@ -250,6 +253,12 @@ public class CompactAction extends TableActionBase {
                             + "Please set '--force_start_flink_job true' if you need forcibly start a flink job.");
             return false;
         }
+
+        Map<BinaryRow, AppendDeleteFileMaintainer> appendDvMaintainers =
+                table.coreOptions().deletionVectorsEnabled()
+                        ? IncrementalClusterManager.createAppendDvMaintainers(
+                                table, compactUnits.keySet(), incrementalClusterManager.snapshot())
+                        : Collections.emptyMap();
         Map<BinaryRow, DataSplit[]> partitionSplits =
                 compactUnits.entrySet().stream()
                         .collect(
@@ -259,7 +268,9 @@ public class CompactAction extends TableActionBase {
                                                 incrementalClusterManager
                                                         .toSplits(
                                                                 entry.getKey(),
-                                                                entry.getValue().files())
+                                                                entry.getValue().files(),
+                                                                appendDvMaintainers.get(
+                                                                        entry.getKey()))
                                                         .toArray(new DataSplit[0])));
 
         // 2. read,sort and write in partition
@@ -269,13 +280,22 @@ public class CompactAction extends TableActionBase {
             DataSplit[] splits = entry.getValue();
             LinkedHashMap<String, String> partitionSpec =
                     partitionComputer.generatePartValues(entry.getKey());
+
             // 2.1 generate source for current partition
+            List<CommitMessage> partitionDvIndexCommitMessages =
+                    appendDvMaintainers.get(entry.getKey()) == null
+                            ? Collections.emptyList()
+                            : IncrementalClusterManager.producePartitionDvIndexCommitMessages(
+                                    table,
+                                    Arrays.asList(splits),
+                                    appendDvMaintainers.get(entry.getKey()));
             Pair<DataStream<RowData>, DataStream<Committable>> sourcePair =
                     IncrementalClusterSplitSource.buildSource(
                             env,
                             table,
                             partitionSpec,
                             splits,
+                            partitionDvIndexCommitMessages,
                             options.get(FlinkConnectorOptions.SCAN_PARALLELISM));
 
             // 2.2 cluster in partition
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/cluster/IncrementalClusterSplitSource.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/cluster/IncrementalClusterSplitSource.java
index 6d18181242..957c70fe60 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/cluster/IncrementalClusterSplitSource.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/cluster/IncrementalClusterSplitSource.java
@@ -27,6 +27,7 @@ import org.apache.paimon.flink.source.SimpleSourceSplit;
 import org.apache.paimon.flink.source.operator.ReadOperator;
 import org.apache.paimon.flink.utils.JavaTypeInfo;
 import org.apache.paimon.table.FileStoreTable;
+import org.apache.paimon.table.sink.CommitMessage;
 import org.apache.paimon.table.source.DataSplit;
 import org.apache.paimon.table.source.Split;
 import org.apache.paimon.utils.Pair;
@@ -46,6 +47,7 @@ import org.apache.flink.table.runtime.typeutils.InternalTypeInfo;
 
 import javax.annotation.Nullable;
 
+import java.util.List;
 import java.util.Map;
 
 /** Source for Incremental Clustering. */
@@ -86,6 +88,7 @@ public class IncrementalClusterSplitSource extends AbstractNonCoordinatedSource<
             FileStoreTable table,
             Map<String, String> partitionSpec,
             DataSplit[] splits,
+            List<CommitMessage> partitionDvIndexCommitMessages,
             @Nullable Integer parallelism) {
         DataStream<Split> source =
                 env.fromSource(
@@ -117,7 +120,8 @@ public class IncrementalClusterSplitSource extends AbstractNonCoordinatedSource<
                         .transform(
                                 "Remove files to be clustered",
                                 new CommittableTypeInfo(),
-                                new RemoveClusterBeforeFilesOperator())
+                                new RemoveClusterBeforeFilesOperator(
+                                        partitionDvIndexCommitMessages))
                         .forceNonParallel());
     }
 }
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/cluster/RemoveClusterBeforeFilesOperator.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/cluster/RemoveClusterBeforeFilesOperator.java
index 907a69ba12..4c0d9f9941 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/cluster/RemoveClusterBeforeFilesOperator.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/cluster/RemoveClusterBeforeFilesOperator.java
@@ -22,6 +22,7 @@ import org.apache.paimon.flink.sink.Committable;
 import org.apache.paimon.flink.utils.BoundedOneInputOperator;
 import org.apache.paimon.io.CompactIncrement;
 import org.apache.paimon.io.DataIncrement;
+import org.apache.paimon.table.sink.CommitMessage;
 import org.apache.paimon.table.sink.CommitMessageImpl;
 import org.apache.paimon.table.source.DataSplit;
 import org.apache.paimon.table.source.Split;
@@ -29,12 +30,19 @@ import org.apache.paimon.table.source.Split;
 import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
 
 import java.util.Collections;
+import java.util.List;
 
 /** Operator used with {@link IncrementalClusterSplitSource}, to remove files to be clustered. */
 public class RemoveClusterBeforeFilesOperator extends BoundedOneInputOperator<Split, Committable> {
 
     private static final long serialVersionUID = 1L;
 
+    private final List<CommitMessage> partitionDvIndexCommitMessages;
+
+    public RemoveClusterBeforeFilesOperator(List<CommitMessage> partitionDvIndexCommitMessages) {
+        this.partitionDvIndexCommitMessages = partitionDvIndexCommitMessages;
+    }
+
     @Override
     public void processElement(StreamRecord<Split> element) throws Exception {
         DataSplit dataSplit = (DataSplit) element.getValue();
@@ -54,5 +62,15 @@ public class RemoveClusterBeforeFilesOperator extends BoundedOneInputOperator<Sp
     }
 
     @Override
-    public void endInput() throws Exception {}
+    public void endInput() throws Exception {
+        emitDvIndexCommitMessages(Long.MAX_VALUE);
+    }
+
+    private void emitDvIndexCommitMessages(long checkpointId) {
+        for (CommitMessage commitMessage : partitionDvIndexCommitMessages) {
+            output.collect(
+                    new StreamRecord<>(
+                            new Committable(checkpointId, Committable.Kind.FILE, commitMessage)));
+        }
+    }
 }
diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/IncrementalClusterActionITCase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/IncrementalClusterActionITCase.java
index 48e0f1fa30..bf7087e77e 100644
--- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/IncrementalClusterActionITCase.java
+++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/IncrementalClusterActionITCase.java
@@ -21,15 +21,25 @@ package org.apache.paimon.flink.action;
 import org.apache.paimon.CoreOptions;
 import org.apache.paimon.Snapshot;
 import org.apache.paimon.catalog.Identifier;
+import org.apache.paimon.data.BinaryRow;
 import org.apache.paimon.data.BinaryString;
 import org.apache.paimon.data.GenericRow;
+import org.apache.paimon.deletionvectors.BitmapDeletionVector;
+import org.apache.paimon.deletionvectors.append.AppendDeleteFileMaintainer;
+import org.apache.paimon.deletionvectors.append.BaseAppendDeleteFileMaintainer;
+import org.apache.paimon.index.IndexFileMeta;
+import org.apache.paimon.io.CompactIncrement;
 import org.apache.paimon.io.DataFileMeta;
+import org.apache.paimon.io.DataIncrement;
+import org.apache.paimon.manifest.FileKind;
+import org.apache.paimon.manifest.IndexManifestEntry;
 import org.apache.paimon.schema.Schema;
 import org.apache.paimon.table.FileStoreTable;
 import org.apache.paimon.table.sink.BatchTableCommit;
 import org.apache.paimon.table.sink.BatchTableWrite;
 import org.apache.paimon.table.sink.BatchWriteBuilder;
 import org.apache.paimon.table.sink.CommitMessage;
+import org.apache.paimon.table.sink.CommitMessageImpl;
 import org.apache.paimon.table.source.DataSplit;
 import org.apache.paimon.table.source.ReadBuilder;
 import org.apache.paimon.table.source.Split;
@@ -565,6 +575,112 @@ public class IncrementalClusterActionITCase extends ActionITCaseBase {
         assertThat(((DataSplit) splits.get(0)).dataFiles().get(0).level()).isEqualTo(5);
     }
 
+    @Test
+    public void testClusterWithDeletionVector() throws Exception {
+        Map<String, String> dynamicOptions = new HashMap<>();
+        dynamicOptions.put(CoreOptions.DELETION_VECTORS_ENABLED.key(), "true");
+        FileStoreTable table = createTable(null, 1, dynamicOptions);
+
+        BinaryString randomStr = BinaryString.fromString(randomString(150));
+        List<CommitMessage> messages = new ArrayList<>();
+        // first write
+        for (int i = 0; i < 3; i++) {
+            for (int j = 0; j < 3; j++) {
+                messages.addAll(write(GenericRow.of(i, j, randomStr, 0)));
+            }
+        }
+        commit(messages);
+        ReadBuilder readBuilder = table.newReadBuilder().withProjection(new int[] {0, 1});
+
+        // first cluster
+        runAction(Collections.emptyList());
+
+        // second write
+        messages.clear();
+        messages.addAll(
+                write(
+                        GenericRow.of(0, 3, null, 0),
+                        GenericRow.of(1, 3, null, 0),
+                        GenericRow.of(2, 3, null, 0)));
+        messages.addAll(
+                write(
+                        GenericRow.of(3, 0, null, 0),
+                        GenericRow.of(3, 1, null, 0),
+                        GenericRow.of(3, 2, null, 0),
+                        GenericRow.of(3, 3, null, 0)));
+        commit(messages);
+
+        // write deletion vector for the table
+        AppendDeleteFileMaintainer maintainer =
+                BaseAppendDeleteFileMaintainer.forUnawareAppend(
+                        table.store().newIndexFileHandler(),
+                        table.latestSnapshot().get(),
+                        BinaryRow.EMPTY_ROW);
+        List<DataFileMeta> files =
+                readBuilder.newScan().plan().splits().stream()
+                        .map(s -> ((DataSplit) s).dataFiles())
+                        .flatMap(List::stream)
+                        .collect(Collectors.toList());
+        // delete (0,0) and (0,3)
+        for (DataFileMeta file : files) {
+            if (file.rowCount() == 9 || file.rowCount() == 3) {
+                BitmapDeletionVector dv = new BitmapDeletionVector();
+                dv.delete(0);
+                maintainer.notifyNewDeletionVector(file.fileName(), dv);
+            }
+        }
+        commit(produceDvIndexMessages(table, maintainer));
+        List<String> result1 =
+                getResult(
+                        readBuilder.newRead(),
+                        readBuilder.newScan().plan().splits(),
+                        readBuilder.readType());
+        List<String> expected1 =
+                Lists.newArrayList(
+                        "+I[0, 1]",
+                        "+I[1, 0]",
+                        "+I[1, 1]",
+                        "+I[0, 2]",
+                        "+I[1, 2]",
+                        "+I[2, 0]",
+                        "+I[2, 1]",
+                        "+I[2, 2]",
+                        "+I[1, 3]",
+                        "+I[2, 3]",
+                        "+I[3, 0]",
+                        "+I[3, 1]",
+                        "+I[3, 2]",
+                        "+I[3, 3]");
+        assertThat(result1).containsExactlyElementsOf(expected1);
+
+        // second cluster
+        runAction(Collections.emptyList());
+        checkSnapshot(table);
+        List<Split> splits = readBuilder.newScan().plan().splits();
+        List<String> result2 =
+                getResult(
+                        readBuilder.newRead(),
+                        readBuilder.newScan().plan().splits(),
+                        readBuilder.readType());
+        assertThat(result2.size()).isEqualTo(expected1.size());
+        assertThat(splits.size()).isEqualTo(1);
+        assertThat(((DataSplit) splits.get(0)).dataFiles().size()).isEqualTo(2);
+        assertThat(((DataSplit) splits.get(0)).dataFiles().get(0).level()).isEqualTo(5);
+        // dv index for level-5 file should be retained
+        assertThat(splits.get(0).deletionFiles().get().get(0)).isNotNull();
+        assertThat(((DataSplit) splits.get(0)).dataFiles().get(1).level()).isEqualTo(4);
+        assertThat((splits.get(0).deletionFiles().get().get(1))).isNull();
+
+        // full cluster
+        runAction(Lists.newArrayList("--compact_strategy", "full"));
+        checkSnapshot(table);
+        splits = readBuilder.newScan().plan().splits();
+        assertThat(splits.size()).isEqualTo(1);
+        assertThat(((DataSplit) splits.get(0)).dataFiles().size()).isEqualTo(1);
+        assertThat(((DataSplit) splits.get(0)).dataFiles().get(0).level()).isEqualTo(5);
+        assertThat(splits.get(0).deletionFiles().get().get(0)).isNull();
+    }
+
     protected FileStoreTable createTable(String partitionKeys, int sinkParallelism)
             throws Exception {
         return createTable(partitionKeys, sinkParallelism, Collections.emptyMap());
@@ -646,6 +762,32 @@ public class IncrementalClusterActionITCase extends ActionITCaseBase {
                 .isEqualTo(Snapshot.CommitKind.COMPACT);
     }
 
+    private List<CommitMessage> produceDvIndexMessages(
+            FileStoreTable table, AppendDeleteFileMaintainer maintainer) {
+        List<IndexFileMeta> newIndexFiles = new ArrayList<>();
+        List<IndexFileMeta> deletedIndexFiles = new ArrayList<>();
+        List<IndexManifestEntry> indexEntries = maintainer.persist();
+        for (IndexManifestEntry entry : indexEntries) {
+            if (entry.kind() == FileKind.ADD) {
+                newIndexFiles.add(entry.indexFile());
+            } else {
+                deletedIndexFiles.add(entry.indexFile());
+            }
+        }
+        return Collections.singletonList(
+                new CommitMessageImpl(
+                        maintainer.getPartition(),
+                        0,
+                        table.coreOptions().bucket(),
+                        DataIncrement.emptyIncrement(),
+                        new CompactIncrement(
+                                Collections.emptyList(),
+                                Collections.emptyList(),
+                                Collections.emptyList(),
+                                newIndexFiles,
+                                deletedIndexFiles)));
+    }
+
     private void runAction(List<String> extra) throws Exception {
         StreamExecutionEnvironment env = streamExecutionEnvironmentBuilder().batchMode().build();
         ArrayList<String> baseArgs =
diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/CompactProcedure.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/CompactProcedure.java
index 7cd20be759..45da3d2ce0 100644
--- a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/CompactProcedure.java
+++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/CompactProcedure.java
@@ -26,7 +26,9 @@ import org.apache.paimon.append.AppendCompactTask;
 import org.apache.paimon.append.cluster.IncrementalClusterManager;
 import org.apache.paimon.compact.CompactUnit;
 import org.apache.paimon.data.BinaryRow;
+import org.apache.paimon.deletionvectors.append.AppendDeleteFileMaintainer;
 import org.apache.paimon.disk.IOManager;
+import org.apache.paimon.index.IndexFileMeta;
 import org.apache.paimon.io.CompactIncrement;
 import org.apache.paimon.io.DataFileMeta;
 import org.apache.paimon.io.DataIncrement;
@@ -556,6 +558,12 @@ public class CompactProcedure extends BaseProcedure {
         Map<BinaryRow, CompactUnit> compactUnits =
                 incrementalClusterManager.prepareForCluster(fullCompaction);
 
+        Map<BinaryRow, AppendDeleteFileMaintainer> appendDvMaintainers =
+                table.coreOptions().deletionVectorsEnabled()
+                        ? IncrementalClusterManager.createAppendDvMaintainers(
+                                table, compactUnits.keySet(), incrementalClusterManager.snapshot())
+                        : Collections.emptyMap();
+
         // generate splits for each partition
         Map<BinaryRow, DataSplit[]> partitionSplits =
                 compactUnits.entrySet().stream()
@@ -566,7 +574,9 @@ public class CompactProcedure extends BaseProcedure {
                                                 incrementalClusterManager
                                                         .toSplits(
                                                                 entry.getKey(),
-                                                                entry.getValue().files())
+                                                                entry.getValue().files(),
+                                                                appendDvMaintainers.get(
+                                                                        entry.getKey()))
                                                         .toArray(new DataSplit[0])));
 
         // sort in partition
@@ -623,8 +633,33 @@ public class CompactProcedure extends BaseProcedure {
                         "Partition {}: upgrade file level to {}",
                         partition,
                         compactUnits.get(partition).outputLevel());
+                // get the dv index messages
+                List<CommitMessage> partitionDvIndexCommitMessages =
+                        appendDvMaintainers.get(entry.getKey()) == null
+                                ? Collections.emptyList()
+                                : IncrementalClusterManager.producePartitionDvIndexCommitMessages(
+                                        table,
+                                        Arrays.asList(partitionSplits.get(partition)),
+                                        appendDvMaintainers.get(entry.getKey()));
+                List<IndexFileMeta> newIndexFiles = new ArrayList<>();
+                List<IndexFileMeta> deletedIndexFiles = new ArrayList<>();
+                for (CommitMessage dvCommitMessage : partitionDvIndexCommitMessages) {
+                    newIndexFiles.addAll(
+                            ((CommitMessageImpl) dvCommitMessage)
+                                    .compactIncrement()
+                                    .newIndexFiles());
+                    deletedIndexFiles.addAll(
+                            ((CommitMessageImpl) dvCommitMessage)
+                                    .compactIncrement()
+                                    .deletedIndexFiles());
+                }
                 CompactIncrement compactIncrement =
-                        new CompactIncrement(clusterBefore, clusterAfter, Collections.emptyList());
+                        new CompactIncrement(
+                                clusterBefore,
+                                clusterAfter,
+                                Collections.emptyList(),
+                                newIndexFiles,
+                                deletedIndexFiles);
                 clusterMessages.add(
                         new CommitMessageImpl(
                                 partition,
diff --git a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/CompactProcedureTestBase.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/CompactProcedureTestBase.scala
index 5f653695d0..74f80befed 100644
--- a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/CompactProcedureTestBase.scala
+++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/CompactProcedureTestBase.scala
@@ -29,6 +29,7 @@ import org.apache.spark.sql.{Dataset, Row}
 import org.apache.spark.sql.execution.streaming.MemoryStream
 import org.apache.spark.sql.streaming.StreamTest
 import org.assertj.core.api.Assertions
+import org.scalatest.time.Span
 
 import java.util
 
@@ -1137,6 +1138,141 @@ abstract class CompactProcedureTestBase extends PaimonSparkTestBase with StreamT
     }
   }
 
+  test("Paimon Procedure: cluster with deletion vectors") {
+    failAfter(Span(5, org.scalatest.time.Minutes)) {
+      withTempDir {
+        checkpointDir =>
+          spark.sql(
+            s"""
+               |CREATE TABLE T (a INT, b INT, c STRING)
+               |TBLPROPERTIES ('bucket'='-1', 'deletion-vectors.enabled'='true','num-levels'='6', 'num-sorted-run.compaction-trigger'='2', 'clustering.columns'='a,b', 'clustering.strategy'='zorder', 'clustering.incremental' = 'true')
+               |""".stripMargin)
+          val location = loadTable("T").location().toString
+
+          val inputData = MemoryStream[(Int, Int, String)]
+          val stream = inputData
+            .toDS()
+            .toDF("a", "b", "c")
+            .writeStream
+            .option("checkpointLocation", checkpointDir.getCanonicalPath)
+            .foreachBatch {
+              (batch: Dataset[Row], _: Long) =>
+                batch.write.format("paimon").mode("append").save(location)
+            }
+            .start()
+
+          val query = () => spark.sql("SELECT * FROM T")
+
+          try {
+            val random = new Random()
+            val randomStr = random.nextString(40)
+            // first write
+            inputData.addData((0, 0, randomStr))
+            inputData.addData((0, 1, randomStr))
+            inputData.addData((0, 2, randomStr))
+            inputData.addData((1, 0, randomStr))
+            inputData.addData((1, 1, randomStr))
+            inputData.addData((1, 2, randomStr))
+            inputData.addData((2, 0, randomStr))
+            inputData.addData((2, 1, randomStr))
+            inputData.addData((2, 2, randomStr))
+            stream.processAllAvailable()
+
+            val result = new util.ArrayList[Row]()
+            for (a <- 0 until 3) {
+              for (b <- 0 until 3) {
+                result.add(Row(a, b, randomStr))
+              }
+            }
+            Assertions.assertThat(query().collect()).containsExactlyElementsOf(result)
+
+            // first cluster, the outputLevel should be 5
+            checkAnswer(spark.sql("CALL paimon.sys.compact(table => 'T')"), Row(true) :: Nil)
+
+            // first cluster result
+            val result2 = new util.ArrayList[Row]()
+            result2.add(0, Row(0, 0, randomStr))
+            result2.add(1, Row(0, 1, randomStr))
+            result2.add(2, Row(1, 0, randomStr))
+            result2.add(3, Row(1, 1, randomStr))
+            result2.add(4, Row(0, 2, randomStr))
+            result2.add(5, Row(1, 2, randomStr))
+            result2.add(6, Row(2, 0, randomStr))
+            result2.add(7, Row(2, 1, randomStr))
+            result2.add(8, Row(2, 2, randomStr))
+
+            Assertions.assertThat(query().collect()).containsExactlyElementsOf(result2)
+
+            var clusteredTable = loadTable("T")
+            checkSnapshot(clusteredTable)
+            var dataSplits = clusteredTable.newSnapshotReader().read().dataSplits()
+            Assertions.assertThat(dataSplits.size()).isEqualTo(1)
+            Assertions.assertThat(dataSplits.get(0).dataFiles().size()).isEqualTo(1)
+            Assertions.assertThat(dataSplits.get(0).dataFiles().get(0).level()).isEqualTo(5)
+
+            // second write
+            inputData.addData((0, 3, null), (1, 3, null), (2, 3, null))
+            inputData.addData((3, 0, null), (3, 1, null), (3, 2, null), (3, 3, null))
+            stream.processAllAvailable()
+
+            // delete (0,0), which is in level-5 file
+            spark.sql("DELETE FROM T WHERE a=0 and b=0;").collect()
+            // delete (0,3), which is in level-0 file
+            spark.sql("DELETE FROM T WHERE a=0 and b=3;").collect()
+
+            val result3 = new util.ArrayList[Row]()
+            result3.addAll(result2.subList(1, result2.size()))
+            for (a <- 1 until 3) {
+              result3.add(Row(a, 3, null))
+            }
+            for (b <- 0 until 4) {
+              result3.add(Row(3, b, null))
+            }
+
+            Assertions.assertThat(query().collect()).containsExactlyElementsOf(result3)
+
+            // second cluster, the outputLevel should be 4. dv index for level-0 will be updated
+            // and dv index for level-5 will be retained
+            checkAnswer(spark.sql("CALL paimon.sys.compact(table => 'T')"), Row(true) :: Nil)
+            // second cluster result, level-5 and level-4 are individually ordered
+            val result4 = new util.ArrayList[Row]()
+            result4.addAll(result2.subList(1, result2.size()))
+            result4.add(Row(1, 3, null))
+            result4.add(Row(3, 0, null))
+            result4.add(Row(3, 1, null))
+            result4.add(Row(2, 3, null))
+            result4.add(Row(3, 2, null))
+            result4.add(Row(3, 3, null))
+            Assertions.assertThat(query().collect()).containsExactlyElementsOf(result4)
+
+            clusteredTable = loadTable("T")
+            checkSnapshot(clusteredTable)
+            dataSplits = clusteredTable.newSnapshotReader().read().dataSplits()
+            Assertions.assertThat(dataSplits.size()).isEqualTo(1)
+            Assertions.assertThat(dataSplits.get(0).dataFiles().size()).isEqualTo(2)
+            Assertions.assertThat(dataSplits.get(0).dataFiles().get(0).level()).isEqualTo(5)
+            Assertions.assertThat(dataSplits.get(0).deletionFiles().get().get(0)).isNotNull
+            Assertions.assertThat(dataSplits.get(0).dataFiles().get(1).level()).isEqualTo(4)
+            Assertions.assertThat(dataSplits.get(0).deletionFiles().get().get(1)).isNull()
+
+            // full cluster
+            checkAnswer(
+              spark.sql("CALL paimon.sys.compact(table => 'T', compact_strategy => 'full')"),
+              Row(true) :: Nil)
+            clusteredTable = loadTable("T")
+            checkSnapshot(clusteredTable)
+            dataSplits = clusteredTable.newSnapshotReader().read().dataSplits()
+            Assertions.assertThat(dataSplits.size()).isEqualTo(1)
+            Assertions.assertThat(dataSplits.get(0).dataFiles().size()).isEqualTo(1)
+            Assertions.assertThat(dataSplits.get(0).deletionFiles().get().get(0)).isNull()
+
+          } finally {
+            stream.stop()
+          }
+      }
+    }
+  }
+
   def checkSnapshot(table: FileStoreTable): Unit = {
     Assertions
       .assertThat(table.latestSnapshot().get().commitKind().toString)


(paimon) 10/18: [doc] fix postpone.default-bucket-num default value (#6554)

Posted by lz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

lzljs3620320 pushed a commit to branch release-1.3
in repository https://gitbox.apache.org/repos/asf/paimon.git

commit e9f4b2f3c3f230c66390bf6f680072caff247729
Author: Vova Kolmakov <wo...@gmail.com>
AuthorDate: Fri Nov 7 12:36:58 2025 +0700

    [doc] fix postpone.default-bucket-num default value (#6554)
---
 docs/content/primary-key-table/data-distribution.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/content/primary-key-table/data-distribution.md b/docs/content/primary-key-table/data-distribution.md
index 4a54afa7aa..6dfae0fe9a 100644
--- a/docs/content/primary-key-table/data-distribution.md
+++ b/docs/content/primary-key-table/data-distribution.md
@@ -78,7 +78,7 @@ To move the records into the correct bucket and make them readable,
 you need to run a compaction job.
 See `compact` [procedure]({{< ref "flink/procedures" >}}).
 The bucket number for the partitions compacted for the first time
-is configured by the option `postpone.default-bucket-num`, whose default value is `4`.
+is configured by the option `postpone.default-bucket-num`, whose default value is `1`.
 
 Finally, when you feel that the bucket number of some partition is too small,
 you can also run a rescale job.


(paimon) 06/18: [Python] Rename to BATCH_COMMIT_IDENTIFIER in snapshot.py

Posted by lz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

lzljs3620320 pushed a commit to branch release-1.3
in repository https://gitbox.apache.org/repos/asf/paimon.git

commit 47c521d3a7df941c5d0bcc1ee374a3e3d539af4e
Author: JingsongLi <ji...@gmail.com>
AuthorDate: Wed Nov 5 16:03:42 2025 +0800

    [Python] Rename to BATCH_COMMIT_IDENTIFIER in snapshot.py
---
 paimon-python/pypaimon/snapshot/snapshot.py  | 2 +-
 paimon-python/pypaimon/write/table_commit.py | 8 ++++----
 paimon-python/pypaimon/write/table_write.py  | 6 +++---
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/paimon-python/pypaimon/snapshot/snapshot.py b/paimon-python/pypaimon/snapshot/snapshot.py
index f164f5eb61..d5f72dedcf 100644
--- a/paimon-python/pypaimon/snapshot/snapshot.py
+++ b/paimon-python/pypaimon/snapshot/snapshot.py
@@ -21,7 +21,7 @@ from typing import Dict, Optional
 
 from pypaimon.common.json_util import json_field
 
-COMMIT_IDENTIFIER = 0x7fffffffffffffff
+BATCH_COMMIT_IDENTIFIER = 0x7fffffffffffffff
 
 
 @dataclass
diff --git a/paimon-python/pypaimon/write/table_commit.py b/paimon-python/pypaimon/write/table_commit.py
index 0dcfabf99e..eab7606128 100644
--- a/paimon-python/pypaimon/write/table_commit.py
+++ b/paimon-python/pypaimon/write/table_commit.py
@@ -18,7 +18,7 @@
 
 from typing import List, Optional
 
-from pypaimon.snapshot.snapshot import COMMIT_IDENTIFIER
+from pypaimon.snapshot.snapshot import BATCH_COMMIT_IDENTIFIER
 from pypaimon.write.commit_message import CommitMessage
 from pypaimon.write.file_store_commit import FileStoreCommit
 
@@ -41,7 +41,7 @@ class TableCommit:
         self.file_store_commit = FileStoreCommit(snapshot_commit, table, commit_user)
         self.batch_committed = False
 
-    def _commit(self, commit_messages: List[CommitMessage], commit_identifier: int = COMMIT_IDENTIFIER):
+    def _commit(self, commit_messages: List[CommitMessage], commit_identifier: int = BATCH_COMMIT_IDENTIFIER):
         self._check_committed()
 
         non_empty_messages = [msg for msg in commit_messages if not msg.is_empty()]
@@ -78,10 +78,10 @@ class TableCommit:
 
 class BatchTableCommit(TableCommit):
     def commit(self, commit_messages: List[CommitMessage]):
-        self._commit(commit_messages, COMMIT_IDENTIFIER)
+        self._commit(commit_messages, BATCH_COMMIT_IDENTIFIER)
 
 
 class StreamTableCommit(TableCommit):
 
-    def commit(self, commit_messages: List[CommitMessage], commit_identifier: int = COMMIT_IDENTIFIER):
+    def commit(self, commit_messages: List[CommitMessage], commit_identifier: int = BATCH_COMMIT_IDENTIFIER):
         self._commit(commit_messages, commit_identifier)
diff --git a/paimon-python/pypaimon/write/table_write.py b/paimon-python/pypaimon/write/table_write.py
index 3ccc078223..7f415a0ba7 100644
--- a/paimon-python/pypaimon/write/table_write.py
+++ b/paimon-python/pypaimon/write/table_write.py
@@ -21,7 +21,7 @@ from typing import List
 import pyarrow as pa
 
 from pypaimon.schema.data_types import PyarrowFieldParser
-from pypaimon.snapshot.snapshot import COMMIT_IDENTIFIER
+from pypaimon.snapshot.snapshot import BATCH_COMMIT_IDENTIFIER
 from pypaimon.write.commit_message import CommitMessage
 from pypaimon.write.file_store_write import FileStoreWrite
 
@@ -87,10 +87,10 @@ class BatchTableWrite(TableWrite):
         if self.batch_committed:
             raise RuntimeError("BatchTableWrite only supports one-time committing.")
         self.batch_committed = True
-        return self.file_store_write.prepare_commit(COMMIT_IDENTIFIER)
+        return self.file_store_write.prepare_commit(BATCH_COMMIT_IDENTIFIER)
 
 
 class StreamTableWrite(TableWrite):
 
-    def prepare_commit(self, commit_identifier: int = COMMIT_IDENTIFIER) -> List[CommitMessage]:
+    def prepare_commit(self, commit_identifier) -> List[CommitMessage]:
         return self.file_store_write.prepare_commit(commit_identifier)


(paimon) 15/18: [Python] License added in test (#6572)

Posted by lz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

lzljs3620320 pushed a commit to branch release-1.3
in repository https://gitbox.apache.org/repos/asf/paimon.git

commit ce4ebcdafe99489163f1c9e690d46169675f2311
Author: umi <55...@users.noreply.github.com>
AuthorDate: Mon Nov 10 11:47:08 2025 +0800

    [Python] License added in test (#6572)
---
 .../pypaimon/tests/manifest/manifest_schema_test.py     | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/paimon-python/pypaimon/tests/manifest/manifest_schema_test.py b/paimon-python/pypaimon/tests/manifest/manifest_schema_test.py
index b4b60ffd43..13890f783c 100644
--- a/paimon-python/pypaimon/tests/manifest/manifest_schema_test.py
+++ b/paimon-python/pypaimon/tests/manifest/manifest_schema_test.py
@@ -1,3 +1,20 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
 import unittest
 
 from pypaimon.manifest.schema import data_file_meta


(paimon) 01/18: [core] Support non null column with write type (#6513)

Posted by lz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

lzljs3620320 pushed a commit to branch release-1.3
in repository https://gitbox.apache.org/repos/asf/paimon.git

commit 75b44862bbee640a39088ddf2d0b1cbc399ff38d
Author: YeJunHao <41...@users.noreply.github.com>
AuthorDate: Mon Nov 3 20:00:19 2025 +0800

    [core] Support non null column with write type (#6513)
---
 .../apache/paimon/table/sink/TableWriteImpl.java   | 15 +++++---
 .../paimon/table/DataEvolutionTableTest.java       | 41 ++++++++++++++++++++++
 2 files changed, 52 insertions(+), 4 deletions(-)

diff --git a/paimon-core/src/main/java/org/apache/paimon/table/sink/TableWriteImpl.java b/paimon-core/src/main/java/org/apache/paimon/table/sink/TableWriteImpl.java
index 325911b0ea..493a2b4ffe 100644
--- a/paimon-core/src/main/java/org/apache/paimon/table/sink/TableWriteImpl.java
+++ b/paimon-core/src/main/java/org/apache/paimon/table/sink/TableWriteImpl.java
@@ -54,7 +54,6 @@ import static org.apache.paimon.utils.Preconditions.checkState;
  */
 public class TableWriteImpl<T> implements InnerTableWrite, Restorable<List<State<T>>> {
 
-    private final RowType rowType;
     private final FileStoreWrite<T> write;
     private final KeyAndBucketExtractor<InternalRow> keyAndBucketExtractor;
     private final RecordExtractor<T> recordExtractor;
@@ -63,8 +62,9 @@ public class TableWriteImpl<T> implements InnerTableWrite, Restorable<List<State
 
     private boolean batchCommitted = false;
     private BucketMode bucketMode;
+    private RowType writeType;
+    private int[] notNullFieldIndex;
 
-    private final int[] notNullFieldIndex;
     private final @Nullable DefaultValueRow defaultValueRow;
 
     public TableWriteImpl(
@@ -74,7 +74,7 @@ public class TableWriteImpl<T> implements InnerTableWrite, Restorable<List<State
             RecordExtractor<T> recordExtractor,
             @Nullable RowKindGenerator rowKindGenerator,
             @Nullable RowKindFilter rowKindFilter) {
-        this.rowType = rowType;
+        this.writeType = rowType;
         this.write = write;
         this.keyAndBucketExtractor = keyAndBucketExtractor;
         this.recordExtractor = recordExtractor;
@@ -115,6 +115,13 @@ public class TableWriteImpl<T> implements InnerTableWrite, Restorable<List<State
     @Override
     public TableWriteImpl<T> withWriteType(RowType writeType) {
         write.withWriteType(writeType);
+        this.writeType = writeType;
+        List<String> notNullColumnNames =
+                writeType.getFields().stream()
+                        .filter(field -> !field.type().isNullable())
+                        .map(DataField::name)
+                        .collect(Collectors.toList());
+        this.notNullFieldIndex = writeType.getFieldIndices(notNullColumnNames);
         return this;
     }
 
@@ -189,7 +196,7 @@ public class TableWriteImpl<T> implements InnerTableWrite, Restorable<List<State
     private void checkNullability(InternalRow row) {
         for (int idx : notNullFieldIndex) {
             if (row.isNullAt(idx)) {
-                String columnName = rowType.getFields().get(idx).name();
+                String columnName = writeType.getFields().get(idx).name();
                 throw new RuntimeException(
                         String.format("Cannot write null to non-null column(%s)", columnName));
             }
diff --git a/paimon-core/src/test/java/org/apache/paimon/table/DataEvolutionTableTest.java b/paimon-core/src/test/java/org/apache/paimon/table/DataEvolutionTableTest.java
index dc4db3da5b..0cd2cf4608 100644
--- a/paimon-core/src/test/java/org/apache/paimon/table/DataEvolutionTableTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/table/DataEvolutionTableTest.java
@@ -419,6 +419,47 @@ public class DataEvolutionTableTest extends TableTestBase {
                 .isEqualTo(2);
     }
 
+    @Test
+    public void testNonNullColumn() throws Exception {
+        Schema.Builder schemaBuilder = Schema.newBuilder();
+        schemaBuilder.column("f0", DataTypes.INT());
+        schemaBuilder.column("f1", DataTypes.STRING());
+        schemaBuilder.column("f2", DataTypes.STRING().copy(false));
+        schemaBuilder.option(CoreOptions.ROW_TRACKING_ENABLED.key(), "true");
+        schemaBuilder.option(CoreOptions.DATA_EVOLUTION_ENABLED.key(), "true");
+
+        Schema schema = schemaBuilder.build();
+
+        catalog.createTable(identifier(), schema, true);
+        Table table = catalog.getTable(identifier());
+        BatchWriteBuilder builder = table.newBatchWriteBuilder();
+        BatchTableWrite write = builder.newWrite();
+        write.write(GenericRow.of(1, BinaryString.fromString("a"), BinaryString.fromString("b")));
+        BatchTableCommit commit = builder.newCommit();
+        List<CommitMessage> commitables = write.prepareCommit();
+        commit.commit(commitables);
+
+        write =
+                builder.newWrite()
+                        .withWriteType(schema.rowType().project(Collections.singletonList("f2")));
+        write.write(GenericRow.of(BinaryString.fromString("c")));
+        commit = builder.newCommit();
+        commitables = write.prepareCommit();
+        setFirstRowId(commitables, 0L);
+        commit.commit(commitables);
+
+        ReadBuilder readBuilder = table.newReadBuilder();
+        RecordReader<InternalRow> reader =
+                readBuilder.newRead().createReader(readBuilder.newScan().plan());
+        assertThat(reader).isInstanceOf(DataEvolutionFileReader.class);
+        reader.forEachRemaining(
+                r -> {
+                    assertThat(r.getInt(0)).isEqualTo(1);
+                    assertThat(r.getString(1).toString()).isEqualTo("a");
+                    assertThat(r.getString(2).toString()).isEqualTo("c");
+                });
+    }
+
     protected Schema schemaDefault() {
         Schema.Builder schemaBuilder = Schema.newBuilder();
         schemaBuilder.column("f0", DataTypes.INT());


(paimon) 08/18: [core] Fix that cannot get partition info if all files are in level-0 when enable dv. (#6531)

Posted by lz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

lzljs3620320 pushed a commit to branch release-1.3
in repository https://gitbox.apache.org/repos/asf/paimon.git

commit e233be189eb22e51fcb869f8b4a2d08def7363ef
Author: zhoulii <zh...@foxmail.com>
AuthorDate: Wed Nov 5 17:43:50 2025 +0800

    [core] Fix that cannot get partition info if all files are in level-0 when enable dv. (#6531)
---
 .../java/org/apache/paimon/table/system/PartitionsTable.java   |  3 ++-
 .../java/org/apache/paimon/flink/BatchFileStoreITCase.java     | 10 ++++++++++
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/paimon-core/src/main/java/org/apache/paimon/table/system/PartitionsTable.java b/paimon-core/src/main/java/org/apache/paimon/table/system/PartitionsTable.java
index 07efaf85a5..04d3490df8 100644
--- a/paimon-core/src/main/java/org/apache/paimon/table/system/PartitionsTable.java
+++ b/paimon-core/src/main/java/org/apache/paimon/table/system/PartitionsTable.java
@@ -182,7 +182,8 @@ public class PartitionsTable implements ReadonlyTable {
                 throw new IllegalArgumentException("Unsupported split: " + split.getClass());
             }
 
-            List<PartitionEntry> partitions = fileStoreTable.newScan().listPartitionEntries();
+            List<PartitionEntry> partitions =
+                    fileStoreTable.newScan().withLevelFilter(level -> true).listPartitionEntries();
 
             @SuppressWarnings("unchecked")
             CastExecutor<InternalRow, BinaryString> partitionCastExecutor =
diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/BatchFileStoreITCase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/BatchFileStoreITCase.java
index 30b2e6d0fd..c5e394ca8b 100644
--- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/BatchFileStoreITCase.java
+++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/BatchFileStoreITCase.java
@@ -1025,4 +1025,14 @@ public class BatchFileStoreITCase extends CatalogITCaseBase {
         sql("INSERT INTO test_table VALUES (1, 'A')");
         assertThat(sql("SELECT * FROM `test_table$files`")).isNotEmpty();
     }
+
+    @Test
+    public void testLevel0FileCanBeReadForPartitionsTable() {
+        sql(
+                "CREATE TABLE test_table (a int PRIMARY KEY NOT ENFORCED, b string, dt string) "
+                        + "PARTITIONED BY (dt)"
+                        + "WITH ('deletion-vectors.enabled' = 'true', 'write-only' = 'true');");
+        sql("INSERT INTO test_table VALUES (1, 'A', '2024-12-01')");
+        assertThat(sql("SELECT * FROM `test_table$partitions`")).isNotEmpty();
+    }
 }


(paimon) 11/18: [Python] Remove the use of reference types in DATA_FILE_META_SCHEMA (#6549)

Posted by lz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

lzljs3620320 pushed a commit to branch release-1.3
in repository https://gitbox.apache.org/repos/asf/paimon.git

commit e4ecfac05ccd1a952ce7fb6ea5cf5431c804c099
Author: Zhang Jiawei <30...@users.noreply.github.com>
AuthorDate: Fri Nov 7 13:37:29 2025 +0800

    [Python] Remove the use of reference types in DATA_FILE_META_SCHEMA (#6549)
---
 .../pypaimon/manifest/schema/data_file_meta.py     |   6 +-
 .../pypaimon/manifest/schema/manifest_file_meta.py |   4 +-
 .../pypaimon/manifest/schema/simple_stats.py       |  44 +++---
 .../pypaimon/tests/manifest_schema_test.py         | 147 +++++++++++++++++++++
 4 files changed, 181 insertions(+), 20 deletions(-)

diff --git a/paimon-python/pypaimon/manifest/schema/data_file_meta.py b/paimon-python/pypaimon/manifest/schema/data_file_meta.py
index 8206061c84..a414644e27 100644
--- a/paimon-python/pypaimon/manifest/schema/data_file_meta.py
+++ b/paimon-python/pypaimon/manifest/schema/data_file_meta.py
@@ -21,7 +21,7 @@ from datetime import datetime
 from pathlib import Path
 from typing import List, Optional
 
-from pypaimon.manifest.schema.simple_stats import (SIMPLE_STATS_SCHEMA,
+from pypaimon.manifest.schema.simple_stats import (KEY_STATS_SCHEMA, VALUE_STATS_SCHEMA,
                                                    SimpleStats)
 from pypaimon.table.row.generic_row import GenericRow
 
@@ -149,8 +149,8 @@ DATA_FILE_META_SCHEMA = {
         {"name": "_ROW_COUNT", "type": "long"},
         {"name": "_MIN_KEY", "type": "bytes"},
         {"name": "_MAX_KEY", "type": "bytes"},
-        {"name": "_KEY_STATS", "type": SIMPLE_STATS_SCHEMA},
-        {"name": "_VALUE_STATS", "type": "SimpleStats"},
+        {"name": "_KEY_STATS", "type": KEY_STATS_SCHEMA},
+        {"name": "_VALUE_STATS", "type": VALUE_STATS_SCHEMA},
         {"name": "_MIN_SEQUENCE_NUMBER", "type": "long"},
         {"name": "_MAX_SEQUENCE_NUMBER", "type": "long"},
         {"name": "_SCHEMA_ID", "type": "long"},
diff --git a/paimon-python/pypaimon/manifest/schema/manifest_file_meta.py b/paimon-python/pypaimon/manifest/schema/manifest_file_meta.py
index 443c6a0944..a830be6cc2 100644
--- a/paimon-python/pypaimon/manifest/schema/manifest_file_meta.py
+++ b/paimon-python/pypaimon/manifest/schema/manifest_file_meta.py
@@ -18,7 +18,7 @@
 
 from dataclasses import dataclass
 
-from pypaimon.manifest.schema.simple_stats import (SIMPLE_STATS_SCHEMA,
+from pypaimon.manifest.schema.simple_stats import (PARTITION_STATS_SCHEMA,
                                                    SimpleStats)
 
 
@@ -41,7 +41,7 @@ MANIFEST_FILE_META_SCHEMA = {
         {"name": "_FILE_SIZE", "type": "long"},
         {"name": "_NUM_ADDED_FILES", "type": "long"},
         {"name": "_NUM_DELETED_FILES", "type": "long"},
-        {"name": "_PARTITION_STATS", "type": SIMPLE_STATS_SCHEMA},
+        {"name": "_PARTITION_STATS", "type": PARTITION_STATS_SCHEMA},
         {"name": "_SCHEMA_ID", "type": "long"},
     ]
 }
diff --git a/paimon-python/pypaimon/manifest/schema/simple_stats.py b/paimon-python/pypaimon/manifest/schema/simple_stats.py
index 7519f881c3..10d9e62420 100644
--- a/paimon-python/pypaimon/manifest/schema/simple_stats.py
+++ b/paimon-python/pypaimon/manifest/schema/simple_stats.py
@@ -41,20 +41,34 @@ class SimpleStats:
         return cls._empty_stats
 
 
-SIMPLE_STATS_SCHEMA = {
+SIMPLE_STATS_FIELDS = [
+    {"name": "_MIN_VALUES", "type": "bytes"},
+    {"name": "_MAX_VALUES", "type": "bytes"},
+    {"name": "_NULL_COUNTS",
+     "type": [
+         "null",
+         {
+             "type": "array",
+             "items": ["null", "long"]
+         }
+     ],
+     "default": None},
+]
+
+KEY_STATS_SCHEMA = {
+    "type": "record",
+    "name": "record_KEY_STATS",
+    "fields": SIMPLE_STATS_FIELDS
+}
+
+VALUE_STATS_SCHEMA = {
+    "type": "record",
+    "name": "record_VALUE_STATS",
+    "fields": SIMPLE_STATS_FIELDS
+}
+
+PARTITION_STATS_SCHEMA = {
     "type": "record",
-    "name": "SimpleStats",
-    "fields": [
-        {"name": "_MIN_VALUES", "type": "bytes"},
-        {"name": "_MAX_VALUES", "type": "bytes"},
-        {"name": "_NULL_COUNTS",
-         "type": [
-             "null",
-             {
-                 "type": "array",
-                 "items": ["null", "long"]
-             }
-         ],
-         "default": None},
-    ]
+    "name": "record_PARTITION_STATS",
+    "fields": SIMPLE_STATS_FIELDS
 }
diff --git a/paimon-python/pypaimon/tests/manifest_schema_test.py b/paimon-python/pypaimon/tests/manifest_schema_test.py
new file mode 100644
index 0000000000..3d82ededa6
--- /dev/null
+++ b/paimon-python/pypaimon/tests/manifest_schema_test.py
@@ -0,0 +1,147 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+import unittest
+
+from pypaimon.manifest.schema.data_file_meta import DATA_FILE_META_SCHEMA
+from pypaimon.manifest.schema.manifest_file_meta import MANIFEST_FILE_META_SCHEMA
+from pypaimon.manifest.schema.simple_stats import (
+    KEY_STATS_SCHEMA,
+    VALUE_STATS_SCHEMA,
+    PARTITION_STATS_SCHEMA
+)
+
+
+class ManifestSchemaTest(unittest.TestCase):
+    """Test cases for the manifest schema definitions."""
+
+    def test_data_file_meta_schema_structure(self):
+        """Test that DATA_FILE_META_SCHEMA has the correct structure."""
+        self.assertIsInstance(DATA_FILE_META_SCHEMA, dict)
+        self.assertEqual(DATA_FILE_META_SCHEMA["type"], "record")
+        self.assertEqual(DATA_FILE_META_SCHEMA["name"], "DataFileMeta")
+        self.assertIn("fields", DATA_FILE_META_SCHEMA)
+
+        fields = DATA_FILE_META_SCHEMA["fields"]
+        self.assertIsInstance(fields, list)
+
+        # Create a mapping of field names to field definitions for easier testing
+        field_map = {field["name"]: field for field in fields}
+
+        # Check that all expected fields are present
+        expected_fields = [
+            "_FILE_NAME", "_FILE_SIZE", "_ROW_COUNT", "_MIN_KEY", "_MAX_KEY",
+            "_KEY_STATS", "_VALUE_STATS", "_MIN_SEQUENCE_NUMBER",
+            "_MAX_SEQUENCE_NUMBER", "_SCHEMA_ID", "_LEVEL", "_EXTRA_FILES",
+            "_CREATION_TIME", "_DELETE_ROW_COUNT", "_EMBEDDED_FILE_INDEX",
+            "_FILE_SOURCE", "_VALUE_STATS_COLS", "_EXTERNAL_PATH",
+            "_FIRST_ROW_ID", "_WRITE_COLS"
+        ]
+
+        for field_name in expected_fields:
+            self.assertIn(field_name, field_map, f"Field {field_name} is missing")
+
+        # Check specific field types
+        self.assertEqual(field_map["_FILE_NAME"]["type"], "string")
+        self.assertEqual(field_map["_FILE_SIZE"]["type"], "long")
+        self.assertEqual(field_map["_ROW_COUNT"]["type"], "long")
+        self.assertEqual(field_map["_MIN_KEY"]["type"], "bytes")
+        self.assertEqual(field_map["_MAX_KEY"]["type"], "bytes")
+        self.assertEqual(field_map["_KEY_STATS"]["type"], KEY_STATS_SCHEMA)
+        self.assertEqual(field_map["_VALUE_STATS"]["type"], VALUE_STATS_SCHEMA)
+        self.assertEqual(field_map["_MIN_SEQUENCE_NUMBER"]["type"], "long")
+        self.assertEqual(field_map["_MAX_SEQUENCE_NUMBER"]["type"], "long")
+        self.assertEqual(field_map["_SCHEMA_ID"]["type"], "long")
+        self.assertEqual(field_map["_LEVEL"]["type"], "int")
+        self.assertEqual(field_map["_EXTRA_FILES"]["type"], {"type": "array", "items": "string"})
+        self.assertEqual(field_map["_CREATION_TIME"]["type"],
+                         ["null", {"type": "long", "logicalType": "timestamp-millis"}])
+        self.assertEqual(field_map["_DELETE_ROW_COUNT"]["type"], ["null", "long"])
+        self.assertEqual(field_map["_EMBEDDED_FILE_INDEX"]["type"], ["null", "bytes"])
+        self.assertEqual(field_map["_FILE_SOURCE"]["type"], ["null", "string"])
+        self.assertEqual(field_map["_VALUE_STATS_COLS"]["type"], ["null", {"type": "array", "items": "string"}])
+        self.assertEqual(field_map["_EXTERNAL_PATH"]["type"], ["null", "string"])
+        self.assertEqual(field_map["_FIRST_ROW_ID"]["type"], ["null", "long"])
+        self.assertEqual(field_map["_WRITE_COLS"]["type"], ["null", {"type": "array", "items": "string"}])
+
+    def test_manifest_file_meta_schema_structure(self):
+        """Test that MANIFEST_FILE_META_SCHEMA has the correct structure."""
+        self.assertIsInstance(MANIFEST_FILE_META_SCHEMA, dict)
+        self.assertEqual(MANIFEST_FILE_META_SCHEMA["type"], "record")
+        self.assertEqual(MANIFEST_FILE_META_SCHEMA["name"], "ManifestFileMeta")
+        self.assertIn("fields", MANIFEST_FILE_META_SCHEMA)
+
+        fields = MANIFEST_FILE_META_SCHEMA["fields"]
+        self.assertIsInstance(fields, list)
+
+        # Create a mapping of field names to field definitions for easier testing
+        field_map = {field["name"]: field for field in fields}
+
+        # Check that all expected fields are present
+        expected_fields = [
+            "_VERSION", "_FILE_NAME", "_FILE_SIZE", "_NUM_ADDED_FILES",
+            "_NUM_DELETED_FILES", "_PARTITION_STATS", "_SCHEMA_ID"
+        ]
+
+        for field_name in expected_fields:
+            self.assertIn(field_name, field_map, f"Field {field_name} is missing")
+
+        # Check specific field types
+        self.assertEqual(field_map["_VERSION"]["type"], "int")
+        self.assertEqual(field_map["_FILE_NAME"]["type"], "string")
+        self.assertEqual(field_map["_FILE_SIZE"]["type"], "long")
+        self.assertEqual(field_map["_NUM_ADDED_FILES"]["type"], "long")
+        self.assertEqual(field_map["_NUM_DELETED_FILES"]["type"], "long")
+        self.assertEqual(field_map["_PARTITION_STATS"]["type"], PARTITION_STATS_SCHEMA)
+        self.assertEqual(field_map["_SCHEMA_ID"]["type"], "long")
+
+    def test_schema_references(self):
+        """Test that schema references are correctly used."""
+        data_file_fields = {field["name"]: field for field in DATA_FILE_META_SCHEMA["fields"]}
+        manifest_file_fields = {field["name"]: field for field in MANIFEST_FILE_META_SCHEMA["fields"]}
+
+        # Check that _KEY_STATS references KEY_STATS_SCHEMA
+        key_stats_field = data_file_fields["_KEY_STATS"]
+        self.assertEqual(key_stats_field["type"], KEY_STATS_SCHEMA)
+
+        # Check that _VALUE_STATS references VALUE_STATS_SCHEMA
+        value_stats_field = data_file_fields["_VALUE_STATS"]
+        self.assertEqual(value_stats_field["type"], VALUE_STATS_SCHEMA)
+
+        # Check that _PARTITION_STATS references PARTITION_STATS_SCHEMA
+        partition_stats_field = manifest_file_fields["_PARTITION_STATS"]
+        self.assertEqual(partition_stats_field["type"], PARTITION_STATS_SCHEMA)
+
+    def test_schema_consistency(self):
+        """Test that schema definitions are consistent."""
+        # Verify that all stats schemas have the same structure
+        self.assertEqual(KEY_STATS_SCHEMA["type"], "record")
+        self.assertEqual(VALUE_STATS_SCHEMA["type"], "record")
+        self.assertEqual(PARTITION_STATS_SCHEMA["type"], "record")
+
+        # Verify that all stats schemas have different names
+        names = [
+            KEY_STATS_SCHEMA["name"],
+            VALUE_STATS_SCHEMA["name"],
+            PARTITION_STATS_SCHEMA["name"]
+        ]
+        self.assertEqual(len(names), len(set(names)), "Schema names should be unique")
+
+
+if __name__ == "__main__":
+    unittest.main()


(paimon) 09/18: [python] add test case for reading blob by to_iterator (#6536)

Posted by lz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

lzljs3620320 pushed a commit to branch release-1.3
in repository https://gitbox.apache.org/repos/asf/paimon.git

commit 048c0d4d83a1818abeb130e94f7819dff629b18f
Author: XiaoHongbo <13...@qq.com>
AuthorDate: Thu Nov 6 11:25:52 2025 +0800

    [python] add test case for reading blob by to_iterator (#6536)
---
 paimon-python/pypaimon/tests/blob_table_test.py | 80 +++++++++++++++++++++++++
 1 file changed, 80 insertions(+)

diff --git a/paimon-python/pypaimon/tests/blob_table_test.py b/paimon-python/pypaimon/tests/blob_table_test.py
index fe987ea0a6..b9e2760e3e 100644
--- a/paimon-python/pypaimon/tests/blob_table_test.py
+++ b/paimon-python/pypaimon/tests/blob_table_test.py
@@ -1334,6 +1334,86 @@ class DataBlobWriterTest(unittest.TestCase):
         self.assertEqual(result.num_rows, 3, "Should have 5 rows")
         self.assertEqual(result.num_columns, 3, "Should have 3 columns")
 
+    def test_blob_read_row_by_row_iterator(self):
+        """Test reading blob data row by row using to_iterator()."""
+        from pypaimon import Schema
+        from pypaimon.table.row.blob import Blob
+        from pypaimon.table.row.internal_row import RowKind
+
+        pa_schema = pa.schema([
+            ('id', pa.int32()),
+            ('name', pa.string()),
+            ('blob_data', pa.large_binary()),
+        ])
+
+        schema = Schema.from_pyarrow_schema(
+            pa_schema,
+            options={
+                'row-tracking.enabled': 'true',
+                'data-evolution.enabled': 'true'
+            }
+        )
+        self.catalog.create_table('test_db.blob_iterator_test', schema, False)
+        table = self.catalog.get_table('test_db.blob_iterator_test')
+
+        expected_data = {
+            1: {'name': 'Alice', 'blob': b'blob_1'},
+            2: {'name': 'Bob', 'blob': b'blob_2_data'},
+            3: {'name': 'Charlie', 'blob': b'blob_3_content'},
+            4: {'name': 'David', 'blob': b'blob_4_large_content'},
+            5: {'name': 'Eve', 'blob': b'blob_5_very_large_content_data'}
+        }
+
+        test_data = pa.Table.from_pydict({
+            'id': list(expected_data.keys()),
+            'name': [expected_data[i]['name'] for i in expected_data.keys()],
+            'blob_data': [expected_data[i]['blob'] for i in expected_data.keys()]
+        }, schema=pa_schema)
+
+        write_builder = table.new_batch_write_builder()
+        writer = write_builder.new_write()
+        writer.write_arrow(test_data)
+        commit_messages = writer.prepare_commit()
+        commit = write_builder.new_commit()
+        commit.commit(commit_messages)
+        writer.close()
+
+        # Verify blob files were created
+        file_names = [f.file_name for f in commit_messages[0].new_files]
+        self.assertGreater(
+            len([f for f in file_names if f.endswith('.blob')]), 0,
+            "Should have at least one blob file")
+
+        # Read using to_iterator
+        iterator = table.new_read_builder().new_read().to_iterator(
+            table.new_read_builder().new_scan().plan().splits())
+
+        rows = []
+        value = next(iterator, None)
+        while value is not None:
+            rows.append(value)
+            value = next(iterator, None)
+
+        self.assertEqual(len(rows), 5, "Should have 5 rows")
+
+        for row in rows:
+            row_id = row.get_field(0)
+            self.assertIn(row_id, expected_data, f"ID {row_id} should be in expected data")
+
+            expected = expected_data[row_id]
+            self.assertEqual(row.get_field(1), expected['name'], f"Row {row_id}: name should match")
+
+            row_blob = row.get_field(2)
+            blob_bytes = row_blob.to_data() if isinstance(row_blob, Blob) else row_blob
+            self.assertIsInstance(blob_bytes, bytes, f"Row {row_id}: blob should be bytes")
+            self.assertEqual(blob_bytes, expected['blob'], f"Row {row_id}: blob data should match")
+            self.assertEqual(len(blob_bytes), len(expected['blob']), f"Row {row_id}: blob size should match")
+
+            self.assertIn(
+                row.get_row_kind(),
+                [RowKind.INSERT, RowKind.UPDATE_BEFORE, RowKind.UPDATE_AFTER, RowKind.DELETE],
+                f"Row {row_id}: RowKind should be valid")
+
 
 if __name__ == '__main__':
     unittest.main()


(paimon) 03/18: [hotfix] Print partition spec and type when error in InternalRowPartitionComputer

Posted by lz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

lzljs3620320 pushed a commit to branch release-1.3
in repository https://gitbox.apache.org/repos/asf/paimon.git

commit 81e24ab09a7b35febef0d5379349340415471bfb
Author: JingsongLi <ji...@gmail.com>
AuthorDate: Mon Nov 3 22:25:45 2025 +0800

    [hotfix] Print partition spec and type when error in InternalRowPartitionComputer
---
 .../java/org/apache/paimon/utils/InternalRowPartitionComputer.java  | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/paimon-common/src/main/java/org/apache/paimon/utils/InternalRowPartitionComputer.java b/paimon-common/src/main/java/org/apache/paimon/utils/InternalRowPartitionComputer.java
index 0f207aaa64..2382647aa9 100644
--- a/paimon-common/src/main/java/org/apache/paimon/utils/InternalRowPartitionComputer.java
+++ b/paimon-common/src/main/java/org/apache/paimon/utils/InternalRowPartitionComputer.java
@@ -109,9 +109,9 @@ public class InternalRowPartitionComputer {
             Map<String, String> spec, RowType partType, String defaultPartValue) {
         checkArgument(
                 spec.size() == partType.getFieldCount(),
-                "Partition spec size %s not match partition field count %s",
-                spec.size(),
-                partType.getFieldCount());
+                "Partition spec %s size not match partition type %s",
+                spec,
+                partType);
         GenericRow partRow = new GenericRow(spec.size());
         List<String> fieldNames = partType.getFieldNames();
         for (Map.Entry<String, String> entry : spec.entrySet()) {


(paimon) 18/18: [Python] Add Mixed read and write test between Java and Python. (#6579)

Posted by lz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

lzljs3620320 pushed a commit to branch release-1.3
in repository https://gitbox.apache.org/repos/asf/paimon.git

commit 1390842f9b0ebcc4f48b52c41135f918b5839dce
Author: umi <55...@users.noreply.github.com>
AuthorDate: Tue Nov 11 10:53:37 2025 +0800

    [Python] Add Mixed read and write test between Java and Python. (#6579)
---
 .github/workflows/paimon-python-checks.yml         |  24 +++
 .../test/java/org/apache/paimon/JavaPyE2ETest.java | 159 +++++++++++++++
 paimon-python/dev/lint-python.sh                   |  49 ++++-
 paimon-python/dev/run_mixed_tests.sh               | 217 +++++++++++++++++++++
 paimon-python/pypaimon/tests/e2e/__init__.py       |  16 ++
 .../pypaimon/tests/e2e/java_py_read_write_test.py  |  89 +++++++++
 6 files changed, 548 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/paimon-python-checks.yml b/.github/workflows/paimon-python-checks.yml
index c4c7e32a84..80277e701c 100755
--- a/.github/workflows/paimon-python-checks.yml
+++ b/.github/workflows/paimon-python-checks.yml
@@ -32,6 +32,9 @@ on:
 
 env:
   PYTHON_VERSIONS: "['3.6.15', '3.10']"
+  JDK_VERSION: 8
+  MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=30 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true
+
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.number || github.run_id }}
@@ -49,6 +52,17 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v2
 
+      - name: Set up JDK ${{ env.JDK_VERSION }}
+        uses: actions/setup-java@v4
+        with:
+          java-version: ${{ env.JDK_VERSION }}
+          distribution: 'temurin'
+
+      - name: Set up Maven
+        uses: stCarolas/setup-maven@v4.5
+        with:
+          maven-version: 3.8.8
+
       - name: Install system dependencies
         shell: bash
         run: |
@@ -58,9 +72,19 @@ jobs:
             curl \
             && rm -rf /var/lib/apt/lists/*
 
+      - name: Verify Java and Maven installation
+        run: |
+          java -version
+          mvn -version
+
       - name: Verify Python version
         run: python --version
 
+      - name: Build Java
+        run: |
+          echo "Start compiling modules"
+          mvn -T 2C -B clean install -DskipTests
+
       - name: Install Python dependencies
         shell: bash
         run: |
diff --git a/paimon-core/src/test/java/org/apache/paimon/JavaPyE2ETest.java b/paimon-core/src/test/java/org/apache/paimon/JavaPyE2ETest.java
new file mode 100644
index 0000000000..3759bc8757
--- /dev/null
+++ b/paimon-core/src/test/java/org/apache/paimon/JavaPyE2ETest.java
@@ -0,0 +1,159 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon;
+
+import org.apache.paimon.catalog.Catalog;
+import org.apache.paimon.catalog.CatalogContext;
+import org.apache.paimon.catalog.CatalogFactory;
+import org.apache.paimon.catalog.Identifier;
+import org.apache.paimon.data.BinaryString;
+import org.apache.paimon.data.DataFormatTestUtil;
+import org.apache.paimon.data.GenericRow;
+import org.apache.paimon.data.InternalRow;
+import org.apache.paimon.fs.Path;
+import org.apache.paimon.schema.Schema;
+import org.apache.paimon.table.FileStoreTable;
+import org.apache.paimon.table.Table;
+import org.apache.paimon.table.sink.InnerTableCommit;
+import org.apache.paimon.table.sink.StreamTableWrite;
+import org.apache.paimon.table.source.Split;
+import org.apache.paimon.table.source.TableRead;
+import org.apache.paimon.types.DataTypes;
+import org.apache.paimon.utils.TraceableFileIO;
+
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.condition.EnabledIfSystemProperty;
+
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.UUID;
+
+import static org.apache.paimon.table.SimpleTableTestBase.getResult;
+import static org.assertj.core.api.Assertions.assertThat;
+
+/** Mixed language overwrite test for Java and Python interoperability. */
+public class JavaPyE2ETest {
+
+    java.nio.file.Path tempDir = Paths.get("../paimon-python/pypaimon/tests/e2e").toAbsolutePath();
+
+    // Fields from TableTestBase that we need
+    protected final String commitUser = UUID.randomUUID().toString();
+    protected Path warehouse;
+    protected Catalog catalog;
+    protected String database;
+
+    @BeforeEach
+    public void before() throws Exception {
+        database = "default";
+
+        // Create warehouse directory if it doesn't exist
+        if (!Files.exists(tempDir.resolve("warehouse"))) {
+            Files.createDirectories(tempDir.resolve("warehouse"));
+        }
+
+        warehouse = new Path(TraceableFileIO.SCHEME + "://" + tempDir.resolve("warehouse"));
+        catalog = CatalogFactory.createCatalog(CatalogContext.create(warehouse));
+
+        // Create database if it doesn't exist
+        try {
+            catalog.createDatabase(database, false);
+        } catch (Catalog.DatabaseAlreadyExistException e) {
+            // Database already exists, ignore
+        }
+    }
+
+    @Test
+    @EnabledIfSystemProperty(named = "run.e2e.tests", matches = "true")
+    public void testJavaWriteRead() throws Exception {
+        Identifier identifier = identifier("mixed_test_tablej");
+        Schema schema =
+                Schema.newBuilder()
+                        .column("id", DataTypes.INT())
+                        .column("name", DataTypes.STRING())
+                        .column("category", DataTypes.STRING())
+                        .column("value", DataTypes.DOUBLE())
+                        .partitionKeys("category")
+                        .option("dynamic-partition-overwrite", "false")
+                        .build();
+
+        catalog.createTable(identifier, schema, true);
+        Table table = catalog.getTable(identifier);
+        FileStoreTable fileStoreTable = (FileStoreTable) table;
+
+        try (StreamTableWrite write = fileStoreTable.newWrite(commitUser);
+                InnerTableCommit commit = fileStoreTable.newCommit(commitUser)) {
+
+            write.write(createRow(1, "Apple", "Fruit", 1.5));
+            write.write(createRow(2, "Banana", "Fruit", 0.8));
+            write.write(createRow(3, "Carrot", "Vegetable", 0.6));
+            write.write(createRow(4, "Broccoli", "Vegetable", 1.2));
+            write.write(createRow(5, "Chicken", "Meat", 5.0));
+            write.write(createRow(6, "Beef", "Meat", 8.0));
+
+            commit.commit(0, write.prepareCommit(true, 0));
+        }
+
+        List<Split> splits =
+                new ArrayList<>(fileStoreTable.newSnapshotReader().read().dataSplits());
+        TableRead read = fileStoreTable.newRead();
+        List<String> res =
+                getResult(
+                        read,
+                        splits,
+                        row -> DataFormatTestUtil.toStringNoRowKind(row, table.rowType()));
+        assertThat(res)
+                .containsExactlyInAnyOrder(
+                        "1, Apple, Fruit, 1.5",
+                        "2, Banana, Fruit, 0.8",
+                        "3, Carrot, Vegetable, 0.6",
+                        "4, Broccoli, Vegetable, 1.2",
+                        "5, Chicken, Meat, 5.0",
+                        "6, Beef, Meat, 8.0");
+    }
+
+    @Test
+    @EnabledIfSystemProperty(named = "run.e2e.tests", matches = "true")
+    public void testRead() throws Exception {
+        Identifier identifier = identifier("mixed_test_tablep");
+        Table table = catalog.getTable(identifier);
+        FileStoreTable fileStoreTable = (FileStoreTable) table;
+        List<Split> splits =
+                new ArrayList<>(fileStoreTable.newSnapshotReader().read().dataSplits());
+        TableRead read = fileStoreTable.newRead();
+        List<String> res =
+                getResult(
+                        read,
+                        splits,
+                        row -> DataFormatTestUtil.toStringNoRowKind(row, table.rowType()));
+        System.out.println(res);
+    }
+
+    // Helper method from TableTestBase
+    protected Identifier identifier(String tableName) {
+        return new Identifier(database, tableName);
+    }
+
+    private static InternalRow createRow(int id, String name, String category, double value) {
+        return GenericRow.of(
+                id, BinaryString.fromString(name), BinaryString.fromString(category), value);
+    }
+}
diff --git a/paimon-python/dev/lint-python.sh b/paimon-python/dev/lint-python.sh
index 9cbb612137..fb57bfdd41 100755
--- a/paimon-python/dev/lint-python.sh
+++ b/paimon-python/dev/lint-python.sh
@@ -119,13 +119,12 @@ function get_all_supported_checks() {
 # exec all selected check stages
 function check_stage() {
     print_function "STAGE" "checks starting"
-    for fun in ${SUPPORT_CHECKS[@]}; do
+    for fun in "${SUPPORT_CHECKS[@]}"; do
         $fun
     done
     echo "All the checks are finished, the detailed information can be found in: $LOG_FILE"
 }
 
-
 ###############################################################All Checks Definitions###############################################################
 #########################
 # This part defines all check functions such as tox_check and flake8_check
@@ -176,7 +175,7 @@ function pytest_check() {
         TEST_DIR="pypaimon/tests/py36"
         echo "Running tests for Python 3.6: $TEST_DIR"
     else
-        TEST_DIR="pypaimon/tests --ignore=pypaimon/tests/py36"
+        TEST_DIR="pypaimon/tests --ignore=pypaimon/tests/py36 --ignore=pypaimon/tests/e2e"
         echo "Running tests for Python $PYTHON_VERSION (excluding py36): pypaimon/tests --ignore=pypaimon/tests/py36"
     fi
 
@@ -194,6 +193,43 @@ function pytest_check() {
         print_function "STAGE" "pytest checks... [SUCCESS]"
     fi
 }
+
+# Mixed tests check - runs Java-Python interoperability tests
+function mixed_check() {
+    # Get Python version
+    PYTHON_VERSION=$(python -c "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')")
+    echo "Detected Python version: $PYTHON_VERSION"
+    if [ "$PYTHON_VERSION" = "3.6" ]; then
+        print_function "STAGE" "mixed tests checks... [SKIPPED]"
+        return
+    fi
+    print_function "STAGE" "mixed tests checks"
+
+    # Path to the mixed tests script
+    MIXED_TESTS_SCRIPT="$CURRENT_DIR/dev/run_mixed_tests.sh"
+
+    if [ ! -f "$MIXED_TESTS_SCRIPT" ]; then
+        echo "Mixed tests script not found at: $MIXED_TESTS_SCRIPT"
+        print_function "STAGE" "mixed tests checks... [FAILED]"
+        exit 1
+    fi
+
+    # Make sure the script is executable
+    chmod +x "$MIXED_TESTS_SCRIPT"
+
+    # Run the mixed tests script
+    set -o pipefail
+    ($MIXED_TESTS_SCRIPT) 2>&1 | tee -a $LOG_FILE
+
+    MIXED_TESTS_STATUS=$?
+    if [ $MIXED_TESTS_STATUS -ne 0 ]; then
+        print_function "STAGE" "mixed tests checks... [FAILED]"
+        # Stop the running script.
+        exit 1;
+    else
+        print_function "STAGE" "mixed tests checks... [SUCCESS]"
+    fi
+}
 ###############################################################All Checks Definitions###############################################################
 # CURRENT_DIR is "paimon-python/"
 SCRIPT_PATH="$(readlink -f "$0")"
@@ -236,15 +272,16 @@ INCLUDE_CHECKS=""
 USAGE="
 usage: $0 [options]
 -h          print this help message and exit
--e [tox,flake8,sphinx,mypy]
+-e [tox,flake8,sphinx,mypy,mixed]
             exclude checks which split by comma(,)
--i [tox,flake8,sphinx,mypy]
+-i [tox,flake8,sphinx,mypy,mixed]
             include checks which split by comma(,)
 -l          list all checks supported.
 Examples:
   ./lint-python.sh                 =>  exec all checks.
   ./lint-python.sh -e tox,flake8   =>  exclude checks tox,flake8.
   ./lint-python.sh -i flake8       =>  include checks flake8.
+  ./lint-python.sh -i mixed        =>  include checks mixed.
   ./lint-python.sh -l              =>  list all checks supported.
 "
 while getopts "hfs:i:e:lr" arg; do
@@ -261,7 +298,7 @@ while getopts "hfs:i:e:lr" arg; do
             ;;
         l)
             printf "current supported checks includes:\n"
-            for fun in ${SUPPORT_CHECKS[@]}; do
+            for fun in "${SUPPORT_CHECKS[@]}"; do
                 echo ${fun%%_check*}
             done
             exit 2
diff --git a/paimon-python/dev/run_mixed_tests.sh b/paimon-python/dev/run_mixed_tests.sh
new file mode 100755
index 0000000000..8387e79eb9
--- /dev/null
+++ b/paimon-python/dev/run_mixed_tests.sh
@@ -0,0 +1,217 @@
+#!/bin/bash
+
+################################################################################
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+# Mixed Java and Python test runner
+# This script runs Java test first, then Python test to verify interoperability
+
+set -e  # Exit on any error
+
+# Colors for output
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+NC='\033[0m' # No Color
+
+# Get script directory
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+PAIMON_PYTHON_DIR="$PROJECT_ROOT/paimon-python/pypaimon/tests/e2e"
+PAIMON_CORE_DIR="$PROJECT_ROOT/paimon-core"
+
+echo -e "${YELLOW}=== Mixed Java-Python Read Write Test Runner ===${NC}"
+echo "Project root: $PROJECT_ROOT"
+echo "Paimon Python dir: $PAIMON_PYTHON_DIR"
+echo "Paimon Core dir: $PAIMON_CORE_DIR"
+echo ""
+
+# Function to clean up warehouse directory
+cleanup_warehouse() {
+    echo -e "${YELLOW}=== Cleaning up warehouse directory ===${NC}"
+
+    local warehouse_dir="$PAIMON_PYTHON_DIR/warehouse"
+
+    if [[ -d "$warehouse_dir" ]]; then
+        echo "Removing warehouse directory: $warehouse_dir"
+        rm -rf "$warehouse_dir"
+        echo -e "${GREEN}✓ Warehouse directory cleaned up successfully${NC}"
+    else
+        echo "Warehouse directory does not exist, no cleanup needed"
+    fi
+
+    echo ""
+}
+
+# Function to run Java test
+run_java_write_test() {
+    echo -e "${YELLOW}=== Step 1: Running Java Test (JavaPyE2ETest.testJavaWriteRead) ===${NC}"
+
+    cd "$PROJECT_ROOT"
+
+    # Run the specific Java test method
+    echo "Running Maven test for JavaPyE2ETest.testJavaWriteRead..."
+    if mvn test -Dtest=org.apache.paimon.JavaPyE2ETest#testJavaWriteRead -pl paimon-core -q -Drun.e2e.tests=true; then
+        echo -e "${GREEN}✓ Java test completed successfully${NC}"
+        return 0
+    else
+        echo -e "${RED}✗ Java test failed${NC}"
+        return 1
+    fi
+}
+
+# Function to run Python test
+run_python_read_test() {
+    echo -e "${YELLOW}=== Step 2: Running Python Test (JavaPyReadWriteTest.testRead) ===${NC}"
+
+    cd "$PAIMON_PYTHON_DIR"
+
+    # Run the specific Python test method
+    echo "Running Python test for JavaPyReadWriteTest.testRead..."
+    if python -m pytest java_py_read_write_test.py::JavaPyReadWriteTest::test_read -v; then
+        echo -e "${GREEN}✓ Python test completed successfully${NC}"
+#        source deactivate
+        return 0
+    else
+        echo -e "${RED}✗ Python test failed${NC}"
+#        source deactivate
+        return 1
+    fi
+}
+
+# Function to run Python Write test for Python-Write-Java-Read scenario
+run_python_write_test() {
+    echo -e "${YELLOW}=== Step 3: Running Python Write Test (JavaPyReadWriteTest.test_py_write_read) ===${NC}"
+
+    cd "$PAIMON_PYTHON_DIR"
+
+    # Run the specific Python test method for writing data
+    echo "Running Python test for JavaPyReadWriteTest.test_py_write_read (Python Write)..."
+    if python -m pytest java_py_read_write_test.py::JavaPyReadWriteTest::test_py_write_read -v; then
+        echo -e "${GREEN}✓ Python write test completed successfully${NC}"
+        return 0
+    else
+        echo -e "${RED}✗ Python write test failed${NC}"
+        return 1
+    fi
+}
+
+# Function to run Java Read test for Python-Write-Java-Read scenario
+run_java_read_test() {
+    echo -e "${YELLOW}=== Step 4: Running Java Read Test (JavaPyE2ETest.testRead) ===${NC}"
+
+    cd "$PROJECT_ROOT"
+
+    # Run the specific Java test method for reading Python-written data
+    echo "Running Maven test for JavaPyE2ETest.testRead (Java Read)..."
+    if mvn test -Dtest=org.apache.paimon.JavaPyE2ETest#testRead -pl paimon-core -q -Drun.e2e.tests=true; then
+        echo -e "${GREEN}✓ Java read test completed successfully${NC}"
+        return 0
+    else
+        echo -e "${RED}✗ Java read test failed${NC}"
+        return 1
+    fi
+}
+
+# Main execution
+main() {
+    local java_write_result=0
+    local python_read_result=0
+    local python_write_result=0
+    local java_read_result=0
+
+    echo -e "${YELLOW}Starting mixed language test execution...${NC}"
+    echo ""
+
+    # Run Java write test
+    if ! run_java_write_test; then
+        java_write_result=1
+        echo -e "${RED}Java test failed, but continuing with Python test...${NC}"
+        echo ""
+    else
+        echo ""
+    fi
+
+    # Run Python read test
+    if ! run_python_read_test; then
+        python_read_result=1
+    fi
+
+    echo ""
+
+    # Run Python Write - Java Read test sequence
+    echo -e "${YELLOW}Starting Python Write - Java Read test sequence...${NC}"
+    echo ""
+
+    # Run Python write test
+    if ! run_python_write_test; then
+        python_write_result=1
+        echo -e "${RED}Python write test failed, but continuing with Java read test...${NC}"
+        echo ""
+    else
+        echo ""
+    fi
+
+    # Run Java read test
+    if ! run_java_read_test; then
+        java_read_result=1
+    fi
+
+    echo ""
+    echo -e "${YELLOW}=== Test Results Summary ===${NC}"
+
+    if [[ $java_write_result -eq 0 ]]; then
+        echo -e "${GREEN}✓ Java Write Test (JavaPyE2ETest.testJavaWriteRead): PASSED${NC}"
+    else
+        echo -e "${RED}✗ Java Write Test (JavaPyE2ETest.testJavaWriteRead): FAILED${NC}"
+    fi
+
+    if [[ $python_read_result -eq 0 ]]; then
+        echo -e "${GREEN}✓ Python Read Test (JavaPyReadWriteTest.testRead): PASSED${NC}"
+    else
+        echo -e "${RED}✗ Python Read Test (JavaPyReadWriteTest.testRead): FAILED${NC}"
+    fi
+
+    if [[ $python_write_result -eq 0 ]]; then
+        echo -e "${GREEN}✓ Python Write Test (JavaPyReadWriteTest.test_py_write_read): PASSED${NC}"
+    else
+        echo -e "${RED}✗ Python Write Test (JavaPyReadWriteTest.test_py_write_read): FAILED${NC}"
+    fi
+
+    if [[ $java_read_result -eq 0 ]]; then
+        echo -e "${GREEN}✓ Java Read Test (JavaPyE2ETest.testRead): PASSED${NC}"
+    else
+        echo -e "${RED}✗ Java Read Test (JavaPyE2ETest.testRead): FAILED${NC}"
+    fi
+
+    echo ""
+
+    # Clean up warehouse directory after all tests
+    cleanup_warehouse
+
+    if [[ $java_write_result -eq 0 && $python_read_result -eq 0 && $python_write_result -eq 0 && $java_read_result -eq 0 ]]; then
+        echo -e "${GREEN}🎉 All tests passed! Java-Python interoperability verified.${NC}"
+        return 0
+    else
+        echo -e "${RED}❌ Some tests failed. Please check the output above.${NC}"
+        return 1
+    fi
+}
+
+# Run main function
+main "$@"
\ No newline at end of file
diff --git a/paimon-python/pypaimon/tests/e2e/__init__.py b/paimon-python/pypaimon/tests/e2e/__init__.py
new file mode 100644
index 0000000000..a67d5ea255
--- /dev/null
+++ b/paimon-python/pypaimon/tests/e2e/__init__.py
@@ -0,0 +1,16 @@
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing,
+#  software distributed under the License is distributed on an
+#  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+#  KIND, either express or implied.  See the License for the
+#  specific language governing permissions and limitations
+#  under the License.
diff --git a/paimon-python/pypaimon/tests/e2e/java_py_read_write_test.py b/paimon-python/pypaimon/tests/e2e/java_py_read_write_test.py
new file mode 100644
index 0000000000..405484d505
--- /dev/null
+++ b/paimon-python/pypaimon/tests/e2e/java_py_read_write_test.py
@@ -0,0 +1,89 @@
+################################################################################
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+import os
+import unittest
+
+import pandas as pd
+import pyarrow as pa
+from pypaimon.catalog.catalog_factory import CatalogFactory
+from pypaimon.schema.schema import Schema
+
+
+class JavaPyReadWriteTest(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.tempdir = os.path.abspath(".")
+        cls.warehouse = os.path.join(cls.tempdir, 'warehouse')
+        cls.catalog = CatalogFactory.create({
+            'warehouse': cls.warehouse
+        })
+        cls.catalog.create_database('default', True)
+
+    def test_py_write_read(self):
+        pa_schema = pa.schema([
+            ('id', pa.int32()),
+            ('name', pa.string()),
+            ('category', pa.string()),
+            ('value', pa.float64())
+        ])
+
+        schema = Schema.from_pyarrow_schema(
+            pa_schema,
+            partition_keys=['category'],
+            options={'dynamic-partition-overwrite': 'false'}
+        )
+
+        self.catalog.create_table('default.mixed_test_tablep', schema, False)
+        table = self.catalog.get_table('default.mixed_test_tablep')
+
+        initial_data = pd.DataFrame({
+            'id': [1, 2, 3, 4, 5, 6],
+            'name': ['Apple', 'Banana', 'Carrot', 'Broccoli', 'Chicken', 'Beef'],
+            'category': ['Fruit', 'Fruit', 'Vegetable', 'Vegetable', 'Meat', 'Meat'],
+            'value': [1.5, 0.8, 0.6, 1.2, 5.0, 8.0]
+        })
+        # Write initial data
+        write_builder = table.new_batch_write_builder()
+        table_write = write_builder.new_write()
+        table_commit = write_builder.new_commit()
+
+        table_write.write_pandas(initial_data)
+        table_commit.commit(table_write.prepare_commit())
+        table_write.close()
+        table_commit.close()
+
+        # Verify initial data
+        read_builder = table.new_read_builder()
+        table_scan = read_builder.new_scan()
+        table_read = read_builder.new_read()
+        initial_result = table_read.to_pandas(table_scan.plan().splits())
+        print(initial_result)
+        self.assertEqual(len(initial_result), 6)
+        self.assertListEqual(
+            initial_result['name'].tolist(),
+            ['Apple', 'Banana', 'Carrot', 'Broccoli', 'Chicken', 'Beef']
+        )
+
+    def test_read(self):
+        table = self.catalog.get_table('default.mixed_test_tablej')
+        read_builder = table.new_read_builder()
+        table_scan = read_builder.new_scan()
+        table_read = read_builder.new_read()
+        res = table_read.to_pandas(table_scan.plan().splits())
+        print(res)