You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@paimon.apache.org by lz...@apache.org on 2026/04/27 06:34:25 UTC
(paimon) branch master updated: [iceberg] Fix compaction commits emitting 'overwrite' instead of 'replace' in Iceberg snapshot operation (#7702)
This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 4a26749126 [iceberg] Fix compaction commits emitting 'overwrite' instead of 'replace' in Iceberg snapshot operation (#7702)
4a26749126 is described below
commit 4a267491269ad587bc7bba2fc6104385aa7ac1ab
Author: 0dunay0 <bi...@yelp.com>
AuthorDate: Mon Apr 27 01:34:19 2026 -0500
[iceberg] Fix compaction commits emitting 'overwrite' instead of 'replace' in Iceberg snapshot operation (#7702)
When Paimon compacts an LSM table with the Iceberg metadata committer
enabled, the resulting Iceberg snapshot has its `operation` field set to
`"overwrite"`. According to the [Iceberg
spec](https://iceberg.apache.org/spec/#snapshots), compaction and file
rewrite operations should use `"replace"` instead. Using `"overwrite"`
makes downstream consumers like Polaris REST catalogs and query engines
think the compaction was a data mutation, which is incorrect.
The problem was in
`IcebergCommitCallback.createWithDeleteManifestFileMetas()`, which
hardcoded `IcebergSnapshotSummary.OVERWRITE` whenever files were
removed, without checking whether the commit was actually a compaction.
The fix:
- Add an `IcebergSnapshotSummary.REPLACE` constant for the `"replace"`
operation
- Pass `Snapshot.CommitKind` into `createWithDeleteManifestFileMetas()`
and use `REPLACE` when `commitKind == COMPACT`, `OVERWRITE` otherwise
- Add an assertion in `IcebergCompatibilityTest.testDeleteImpl()` to
verify that compaction commits produce `operation: "replace"` in the
Iceberg snapshot metadata
---
.../java/org/apache/paimon/iceberg/IcebergCommitCallback.java | 11 ++++++++---
.../paimon/iceberg/metadata/IcebergSnapshotSummary.java | 1 +
.../org/apache/paimon/iceberg/IcebergCompatibilityTest.java | 10 ++++++++++
3 files changed, 19 insertions(+), 3 deletions(-)
diff --git a/paimon-core/src/main/java/org/apache/paimon/iceberg/IcebergCommitCallback.java b/paimon-core/src/main/java/org/apache/paimon/iceberg/IcebergCommitCallback.java
index 3852c7fb6e..4378bab3c9 100644
--- a/paimon-core/src/main/java/org/apache/paimon/iceberg/IcebergCommitCallback.java
+++ b/paimon-core/src/main/java/org/apache/paimon/iceberg/IcebergCommitCallback.java
@@ -548,7 +548,8 @@ public class IcebergCommitCallback implements CommitCallback, TagCallback {
addedFiles,
modifiedPartitions,
baseDataManifestFileMetas,
- snapshotId);
+ snapshotId,
+ snapshot.commitKind());
newDataManifestFileMetas = result.getLeft();
snapshotSummary = result.getRight();
}
@@ -778,7 +779,8 @@ public class IcebergCommitCallback implements CommitCallback, TagCallback {
Map<String, Pair<BinaryRow, DataFileMeta>> addedFiles,
List<BinaryRow> modifiedPartitions,
List<IcebergManifestFileMeta> baseManifestFileMetas,
- long currentSnapshotId)
+ long currentSnapshotId,
+ Snapshot.CommitKind commitKind)
throws IOException {
IcebergSnapshotSummary snapshotSummary = IcebergSnapshotSummary.APPEND;
List<IcebergManifestFileMeta> newManifestFileMetas = new ArrayList<>();
@@ -833,7 +835,10 @@ public class IcebergCommitCallback implements CommitCallback, TagCallback {
newManifestFileMetas.add(fileMeta);
} else {
// some file is removed, rewrite this file meta
- snapshotSummary = IcebergSnapshotSummary.OVERWRITE;
+ snapshotSummary =
+ commitKind == Snapshot.CommitKind.COMPACT
+ ? IcebergSnapshotSummary.REPLACE
+ : IcebergSnapshotSummary.OVERWRITE;
List<IcebergManifestEntry> newEntries = new ArrayList<>();
for (IcebergManifestEntry entry : entries) {
if (entry.isLive()) {
diff --git a/paimon-core/src/main/java/org/apache/paimon/iceberg/metadata/IcebergSnapshotSummary.java b/paimon-core/src/main/java/org/apache/paimon/iceberg/metadata/IcebergSnapshotSummary.java
index c8f64d4107..43a3e41536 100644
--- a/paimon-core/src/main/java/org/apache/paimon/iceberg/metadata/IcebergSnapshotSummary.java
+++ b/paimon-core/src/main/java/org/apache/paimon/iceberg/metadata/IcebergSnapshotSummary.java
@@ -38,6 +38,7 @@ public class IcebergSnapshotSummary {
public static final IcebergSnapshotSummary APPEND = new IcebergSnapshotSummary("append");
public static final IcebergSnapshotSummary OVERWRITE = new IcebergSnapshotSummary("overwrite");
+ public static final IcebergSnapshotSummary REPLACE = new IcebergSnapshotSummary("replace");
private final Map<String, String> summary;
diff --git a/paimon-core/src/test/java/org/apache/paimon/iceberg/IcebergCompatibilityTest.java b/paimon-core/src/test/java/org/apache/paimon/iceberg/IcebergCompatibilityTest.java
index a3f28f7ab4..2b5e794a57 100644
--- a/paimon-core/src/test/java/org/apache/paimon/iceberg/IcebergCompatibilityTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/iceberg/IcebergCompatibilityTest.java
@@ -185,6 +185,16 @@ public class IcebergCompatibilityTest {
// In dv mode, full compaction will remove all dv index and rewrite data files
assertThat(getIcebergResult()).containsExactlyInAnyOrder("Record(2, 21)");
+ FileIO fileIO = table.fileIO();
+ long latestSnapshotId = table.snapshotManager().latestSnapshotId();
+ IcebergMetadata metadata =
+ IcebergMetadata.fromPath(
+ fileIO,
+ new Path(
+ table.location(),
+ "metadata/v" + latestSnapshotId + ".metadata.json"));
+ assertThat(metadata.currentSnapshot().summary().operation()).isEqualTo("replace");
+
write.close();
commit.close();
}