You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by hu...@apache.org on 2026/06/20 18:19:32 UTC
(iceberg) branch main updated: Flink: Backport: Add equality delete conversion committer (#16874) (#16888)
This is an automated email from the ASF dual-hosted git repository.
huaxingao pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/main by this push:
new 2a2d5c0a62 Flink: Backport: Add equality delete conversion committer (#16874) (#16888)
2a2d5c0a62 is described below
commit 2a2d5c0a6230a1df08bbca8dc5c93d53122081fd
Author: Maximilian Michels <mx...@apache.org>
AuthorDate: Sat Jun 20 20:19:25 2026 +0200
Flink: Backport: Add equality delete conversion committer (#16874) (#16888)
---
.../operator/EqualityConvertCommitter.java | 363 ++++++++++++++
.../operator/TestEqualityConvertCommitter.java | 524 +++++++++++++++++++++
.../operator/EqualityConvertCommitter.java | 363 ++++++++++++++
.../operator/TestEqualityConvertCommitter.java | 524 +++++++++++++++++++++
4 files changed, 1774 insertions(+)
diff --git a/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertCommitter.java b/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertCommitter.java
new file mode 100644
index 0000000000..c83b06a55a
--- /dev/null
+++ b/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertCommitter.java
@@ -0,0 +1,363 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.flink.maintenance.operator;
+
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.metrics.Counter;
+import org.apache.flink.metrics.MetricGroup;
+import org.apache.flink.streaming.api.operators.AbstractStreamOperator;
+import org.apache.flink.streaming.api.operators.TwoInputStreamOperator;
+import org.apache.flink.streaming.api.watermark.Watermark;
+import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
+import org.apache.iceberg.DataFile;
+import org.apache.iceberg.DeleteFile;
+import org.apache.iceberg.RowDelta;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.exceptions.CommitStateUnknownException;
+import org.apache.iceberg.flink.TableLoader;
+import org.apache.iceberg.flink.maintenance.api.Trigger;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.relocated.com.google.common.collect.Sets;
+import org.apache.iceberg.util.ContentFileUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Commits data files and DVs to the target branch. Receives {@link DVWriteResult}s from parallel
+ * {@link EqualityConvertDVWriter} instances (input 1) and an {@link EqualityConvertPlan} from the
+ * planner (input 2). Assembles the final file lists and commits using a {@link RowDelta} operation
+ * once the plan result and done-timestamp watermark have both arrived.
+ *
+ * <p>The commit is gated on the plan's done-timestamp watermark.
+ *
+ * <p>Emits a {@link Trigger} after each cycle (commit, no-op, or error) so the downstream {@link
+ * TaskResultAggregator} can track task completion. This is the sole source of Trigger records for
+ * the Aggregator.
+ *
+ * <p>No-op vs error: a no-op cycle (empty plan result from {@code
+ * EqualityConvertPlanner.emitNoOpResult}) returns early in {@code commitIfNeeded} without writing
+ * anything; the Trigger emit in {@code processWatermark} still happens, so the maintenance task
+ * completes cleanly. Errors are reported via {@link TaskResultAggregator#ERROR_STREAM} side output;
+ * the Aggregator collects them and surfaces failure on its own watermark.
+ *
+ * <p>The committer is intentionally stateless: {@code bufferedResults} and {@code planResult} are
+ * not checkpointed. The maintenance framework ensures mutual exclusive tasks, and on restart the
+ * planner re-derives its position from {@link #COMMITTED_STAGING_SNAPSHOT_PROPERTY} on main, so the
+ * committer never receives a plan for an already-committed staging snapshot.
+ */
+@Internal
+public class EqualityConvertCommitter extends AbstractStreamOperator<Trigger>
+ implements TwoInputStreamOperator<DVWriteResult, EqualityConvertPlan, Trigger> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(EqualityConvertCommitter.class);
+
+ static final String COMMITTED_STAGING_SNAPSHOT_PROPERTY = "equality-convert-staging-snapshot";
+
+ private static final String ADDED_DV_NUM_METRIC = "addedDvNum";
+ private static final String COMMIT_DURATION_MS_METRIC = "commitDurationMs";
+
+ private final String tableName;
+ private final String taskName;
+ private final TableLoader tableLoader;
+ private final String targetBranch;
+ private final boolean stagingOnTargetBranch;
+
+ private transient Table table;
+ private transient List<DVWriteResult> bufferedResults;
+ private transient EqualityConvertPlan planResult;
+
+ private transient Counter errorCounter;
+ private transient Counter addedDataFileNumCounter;
+ private transient Counter addedDataFileSizeCounter;
+ private transient Counter addedDvNumCounter;
+ private transient Counter commitDurationMsCounter;
+
+ public EqualityConvertCommitter(
+ String tableName,
+ String taskName,
+ TableLoader tableLoader,
+ String stagingBranch,
+ String targetBranch) {
+ this.tableName = tableName;
+ this.taskName = taskName;
+ this.tableLoader = tableLoader;
+ this.targetBranch = targetBranch;
+ this.stagingOnTargetBranch = stagingBranch.equals(targetBranch);
+ }
+
+ @Override
+ public void open() throws Exception {
+ super.open();
+ if (!tableLoader.isOpen()) {
+ tableLoader.open();
+ }
+
+ this.table = tableLoader.loadTable();
+ this.bufferedResults = Lists.newArrayList();
+
+ MetricGroup taskMetricGroup =
+ TableMaintenanceMetrics.groupFor(getRuntimeContext(), tableName, taskName, 0);
+ this.errorCounter = taskMetricGroup.counter(TableMaintenanceMetrics.ERROR_COUNTER);
+ this.addedDataFileNumCounter =
+ taskMetricGroup.counter(TableMaintenanceMetrics.ADDED_DATA_FILE_NUM_METRIC);
+ this.addedDataFileSizeCounter =
+ taskMetricGroup.counter(TableMaintenanceMetrics.ADDED_DATA_FILE_SIZE_METRIC);
+ this.addedDvNumCounter = taskMetricGroup.counter(ADDED_DV_NUM_METRIC);
+ this.commitDurationMsCounter = taskMetricGroup.counter(COMMIT_DURATION_MS_METRIC);
+ }
+
+ @Override
+ public void processElement1(StreamRecord<DVWriteResult> record) {
+ bufferedResults.add(record.getValue());
+ }
+
+ @Override
+ public void processElement2(StreamRecord<EqualityConvertPlan> record) {
+ planResult = record.getValue();
+ }
+
+ @Override
+ public void processWatermark(Watermark mark) throws Exception {
+ if (planResult != null && mark.getTimestamp() >= planResult.doneTimestamp()) {
+ try {
+ commitIfNeeded();
+ } catch (Exception e) {
+ LOG.error(
+ "Failed to commit equality convert result for table {} task {}",
+ tableName,
+ taskName,
+ e);
+ output.collect(TaskResultAggregator.ERROR_STREAM, new StreamRecord<>(e));
+ errorCounter.inc();
+ }
+
+ // Emit Trigger for the Aggregator (even on error or no-op).
+ output.collect(new StreamRecord<>(Trigger.create(planResult.triggerTimestamp(), 0)));
+
+ bufferedResults.clear();
+ planResult = null;
+ }
+
+ // Always forward watermarks to prevent stalling downstream.
+ super.processWatermark(mark);
+ }
+
+ @Override
+ public void close() throws Exception {
+ super.close();
+ tableLoader.close();
+ }
+
+ private void commitIfNeeded() {
+ for (DVWriteResult result : bufferedResults) {
+ if (result.isAbort()) {
+ LOG.warn(
+ "Skipping commit for table {} task {}: a DV writer reported an error.",
+ tableName,
+ taskName);
+ deleteUncommittedDVs();
+ return;
+ }
+ }
+
+ // No-op cycle: the planner emitted an empty plan result (see
+ // EqualityConvertPlanner.emitNoOpResult) because the next staging snapshot was filtered by
+ // shouldSkip or there was nothing new on staging. processWatermark still forwards a Trigger to
+ // TaskResultAggregator, so the maintenance task completes cleanly.
+ if (planResult.noOp()) {
+ return;
+ }
+
+ table.refresh();
+
+ List<DeleteFile> allDvFiles = Lists.newArrayList();
+ List<DeleteFile> allRewrittenDvFiles = Lists.newArrayList();
+ for (DVWriteResult result : bufferedResults) {
+ allDvFiles.addAll(result.dvFiles());
+ allRewrittenDvFiles.addAll(result.rewrittenDvFiles());
+ }
+
+ RowDelta rowDelta = buildRowDelta(planResult.dataFiles(), allDvFiles, allRewrittenDvFiles);
+
+ long startNano = System.nanoTime();
+ try {
+ commit(rowDelta);
+ } catch (CommitStateUnknownException e) {
+ // Commit outcome unknown: the DVs may already be referenced by a committed snapshot. Leave
+ // them for Remove Orphan Files rather than risk deleting live data.
+ throw e;
+ } catch (Exception e) {
+ // Commit definitively failed: the DVs this cycle wrote are unreferenced. Delete them so a
+ // failed cycle does not leak Puffin files. Rewritten DVs stay (still live on the target).
+ deleteUncommittedDVs();
+ throw e;
+ }
+
+ long durationMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNano);
+ commitDurationMsCounter.inc(durationMs);
+
+ LOG.info(
+ "Committed {} data files and {} DV files to branch '{}' for table {} in {} ms. "
+ + "Processed staging snapshot {}.",
+ planResult.dataFiles().size(),
+ allDvFiles.size(),
+ targetBranch,
+ tableName,
+ durationMs,
+ planResult.stagingSnapshotId());
+
+ // Only count files actually added by this commit. When sameBranch, the writer already
+ // committed the data files to target and buildRowDelta does not re-add them.
+ if (!stagingOnTargetBranch) {
+ for (DataFile dataFile : planResult.dataFiles()) {
+ addedDataFileNumCounter.inc();
+ addedDataFileSizeCounter.inc(dataFile.fileSizeInBytes());
+ }
+ }
+
+ addedDvNumCounter.inc(allDvFiles.size());
+ }
+
+ @VisibleForTesting
+ void commit(RowDelta rowDelta) {
+ rowDelta.commit();
+ }
+
+ /**
+ * Deletes the DVs this cycle wrote but did not commit (abort or definite commit failure). Only
+ * the newly written DVs are removed; rewritten DVs remain referenced on the target branch. Best
+ * effort: a delete failure is logged, not propagated, so it never masks the original error.
+ */
+ private void deleteUncommittedDVs() {
+ for (DVWriteResult result : bufferedResults) {
+ if (result.isAbort()) {
+ continue;
+ }
+
+ for (DeleteFile dvFile : result.dvFiles()) {
+ try {
+ table.io().deleteFile(dvFile.location());
+ } catch (RuntimeException e) {
+ LOG.warn(
+ "Failed to delete uncommitted DV {} for table {} task {}",
+ dvFile.location(),
+ tableName,
+ taskName,
+ e);
+ }
+ }
+ }
+ }
+
+ private RowDelta buildRowDelta(
+ List<DataFile> dataFiles, List<DeleteFile> allDvFiles, List<DeleteFile> allRewrittenDvFiles) {
+ RowDelta rowDelta = table.newRowDelta();
+
+ // Fail the commit on external target-branch activity since the planner's snapshot. The next
+ // trigger detects the change and reindexes.
+ if (planResult.mainSnapshotId() != null) {
+ rowDelta.validateFromSnapshot(planResult.mainSnapshotId());
+ }
+
+ rowDelta.validateNoConflictingDataFiles();
+ rowDelta.validateNoConflictingDeleteFiles();
+
+ Set<String> referencedDataFiles = Sets.newHashSet();
+ for (DeleteFile dvFile : allDvFiles) {
+ if (dvFile.referencedDataFile() != null) {
+ referencedDataFiles.add(dvFile.referencedDataFile());
+ }
+ }
+
+ if (!referencedDataFiles.isEmpty()) {
+ rowDelta.validateDataFilesExist(referencedDataFiles);
+ }
+
+ // When stagingBranch == targetBranch, the writer already committed data files and DVs to the
+ // target branch. Re-adding them here would produce duplicate manifest entries. Skip those
+ // paths; only the new DVs from the writer need to be added.
+ if (!stagingOnTargetBranch) {
+ for (DataFile dataFile : dataFiles) {
+ rowDelta.addRows(dataFile);
+ }
+ }
+
+ for (DeleteFile dvFile : allDvFiles) {
+ rowDelta.addDeletes(dvFile);
+ }
+
+ if (!stagingOnTargetBranch) {
+ addStagingDeletes(rowDelta, referencedDataFiles, planResult.stagingDVFiles());
+ }
+
+ removeRewrittenDVs(
+ rowDelta, allRewrittenDvFiles, planResult.stagingDVFiles(), stagingOnTargetBranch);
+
+ rowDelta.toBranch(targetBranch);
+ rowDelta.set(
+ COMMITTED_STAGING_SNAPSHOT_PROPERTY, String.valueOf(planResult.stagingSnapshotId()));
+ return rowDelta;
+ }
+
+ /** Adds staging delete files, skipping DVs that overlap with conversion DVs (V3 rule). */
+ private static void addStagingDeletes(
+ RowDelta rowDelta, Set<String> dvCoveredDataFiles, List<DeleteFile> stagingDVFiles) {
+ for (DeleteFile stagingDelete : stagingDVFiles) {
+ // V3 allows one DV per data file. When a staging snapshot contains both a writer-committed
+ // DV and an eq-delete that resolves to additional positions in the same data file, the
+ // writer folds the staging DV into a new merged DV (via
+ // EqualityConvertDVWriter.collectExistingDVs). Skip the superseded staging DV; adding both
+ // would commit two DVs for the same data file.
+ if (ContentFileUtil.isDV(stagingDelete)
+ && stagingDelete.referencedDataFile() != null
+ && dvCoveredDataFiles.contains(stagingDelete.referencedDataFile())) {
+ continue;
+ }
+
+ rowDelta.addDeletes(stagingDelete);
+ }
+ }
+
+ /**
+ * Removes rewritten DVs. On a separate target branch, staging DVs are not yet on target, so they
+ * are skipped: removeDeletes would fail. On a shared branch they are already on target and must
+ * be removed like any other rewritten DV; the merged DV that supersedes them would otherwise be a
+ * second DV for the same data file (V3 allows one).
+ */
+ private static void removeRewrittenDVs(
+ RowDelta rowDelta,
+ List<DeleteFile> allRewrittenDvFiles,
+ List<DeleteFile> stagingDVFiles,
+ boolean stagingOnTargetBranch) {
+ Set<String> stagingDeleteLocations = Sets.newHashSet();
+ for (DeleteFile sd : stagingDVFiles) {
+ stagingDeleteLocations.add(sd.location());
+ }
+
+ for (DeleteFile rewrittenDv : allRewrittenDvFiles) {
+ if (stagingOnTargetBranch || !stagingDeleteLocations.contains(rewrittenDv.location())) {
+ rowDelta.removeDeletes(rewrittenDv);
+ }
+ }
+ }
+}
diff --git a/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/TestEqualityConvertCommitter.java b/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/TestEqualityConvertCommitter.java
new file mode 100644
index 0000000000..f74570c878
--- /dev/null
+++ b/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/TestEqualityConvertCommitter.java
@@ -0,0 +1,524 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.flink.maintenance.operator;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.io.IOException;
+import java.util.List;
+import org.apache.flink.streaming.api.watermark.Watermark;
+import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
+import org.apache.flink.streaming.util.TwoInputStreamOperatorTestHarness;
+import org.apache.iceberg.DataFile;
+import org.apache.iceberg.DeleteFile;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.ManifestFile;
+import org.apache.iceberg.ManifestFiles;
+import org.apache.iceberg.ManifestReader;
+import org.apache.iceberg.RowDelta;
+import org.apache.iceberg.SnapshotRef;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.data.FileHelpers;
+import org.apache.iceberg.data.GenericRecord;
+import org.apache.iceberg.deletes.PositionDelete;
+import org.apache.iceberg.exceptions.CommitStateUnknownException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.flink.maintenance.api.Trigger;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.junit.jupiter.api.Test;
+
+class TestEqualityConvertCommitter extends OperatorTestBase {
+
+ @Test
+ void commitsDataFilesToMainBranch() throws Exception {
+ Table table = createTable(3, FileFormat.PARQUET);
+ insert(table, 1, "a");
+
+ DataFile stagingDataFile = getFirstDataFile(table);
+ long snapshotIdBefore = table.currentSnapshot().snapshotId();
+ long stagingSnapshotId = 42L;
+
+ try (TwoInputStreamOperatorTestHarness<DVWriteResult, EqualityConvertPlan, Trigger> harness =
+ createHarness()) {
+ harness.open();
+
+ long doneTs = System.currentTimeMillis();
+ EqualityConvertPlan planResult =
+ new EqualityConvertPlan(
+ Lists.newArrayList(stagingDataFile),
+ Lists.newArrayList(),
+ stagingSnapshotId,
+ snapshotIdBefore,
+ doneTs - 1,
+ doneTs);
+
+ harness.processElement2(new StreamRecord<>(planResult, doneTs - 1));
+ harness.processBothWatermarks(new Watermark(doneTs));
+
+ assertThat(harness.extractOutputValues()).hasSize(1);
+ table.refresh();
+ assertThat(table.currentSnapshot().snapshotId()).isNotEqualTo(snapshotIdBefore);
+ assertThat(
+ table
+ .currentSnapshot()
+ .summary()
+ .get(EqualityConvertCommitter.COMMITTED_STAGING_SNAPSHOT_PROPERTY))
+ .isEqualTo(String.valueOf(stagingSnapshotId));
+ }
+ }
+
+ @Test
+ void skipsCommitForEmptyCycle() throws Exception {
+ Table table = createTable(3, FileFormat.PARQUET);
+
+ try (TwoInputStreamOperatorTestHarness<DVWriteResult, EqualityConvertPlan, Trigger> harness =
+ createHarness()) {
+ harness.open();
+
+ long doneTs = System.currentTimeMillis();
+ EqualityConvertPlan planResult = EqualityConvertPlan.noOp(null, doneTs - 1, doneTs);
+
+ harness.processElement2(new StreamRecord<>(planResult, doneTs - 1));
+ harness.processBothWatermarks(new Watermark(doneTs));
+
+ assertThat(harness.extractOutputValues()).hasSize(1);
+ table.refresh();
+ assertThat(table.currentSnapshot()).isNull();
+ }
+ }
+
+ @Test
+ void abortsCommitWhenDVWriterFailed() throws Exception {
+ Table table = createTable(3, FileFormat.PARQUET);
+ insert(table, 1, "a");
+
+ DataFile stagingDataFile = getFirstDataFile(table);
+ long snapshotIdBefore = table.currentSnapshot().snapshotId();
+
+ try (TwoInputStreamOperatorTestHarness<DVWriteResult, EqualityConvertPlan, Trigger> harness =
+ createHarness()) {
+ harness.open();
+
+ long doneTs = System.currentTimeMillis();
+ EqualityConvertPlan planResult =
+ new EqualityConvertPlan(
+ Lists.newArrayList(stagingDataFile),
+ Lists.newArrayList(),
+ 42L,
+ snapshotIdBefore,
+ doneTs - 1,
+ doneTs);
+
+ // Receive an abort signal from the DVWriter followed by the planning planResult.
+ harness.processElement1(new StreamRecord<>(DVWriteResult.ABORT, doneTs));
+ harness.processElement2(new StreamRecord<>(planResult, doneTs - 1));
+
+ harness.processBothWatermarks(new Watermark(doneTs));
+
+ // The cycle must complete (Trigger emitted) but nothing must be committed to the table.
+ assertThat(harness.extractOutputValues()).hasSize(1);
+ table.refresh();
+ assertThat(table.currentSnapshot().snapshotId()).isEqualTo(snapshotIdBefore);
+ }
+ }
+
+ @Test
+ void failsCommitWhenExternalCommitLandsAfterPlan() throws Exception {
+ Table table = createTable(3, FileFormat.PARQUET);
+ insert(table, 1, "a");
+
+ DataFile stagingDataFile = getFirstDataFile(table);
+ long mainSnapshotIdAtPlan = table.currentSnapshot().snapshotId();
+ long stagingSnapshotId = 555L;
+
+ try (TwoInputStreamOperatorTestHarness<DVWriteResult, EqualityConvertPlan, Trigger> harness =
+ createHarness()) {
+ harness.open();
+
+ long doneTs = System.currentTimeMillis();
+ EqualityConvertPlan planResult =
+ new EqualityConvertPlan(
+ Lists.newArrayList(stagingDataFile),
+ Lists.newArrayList(),
+ stagingSnapshotId,
+ mainSnapshotIdAtPlan,
+ doneTs - 1,
+ doneTs);
+
+ harness.processElement2(new StreamRecord<>(planResult, doneTs - 1));
+
+ // External writer appends to main between plan and commit.
+ insert(table, 2, "b");
+ table.refresh();
+ long mainSnapshotIdAfterExternal = table.currentSnapshot().snapshotId();
+ assertThat(mainSnapshotIdAfterExternal).isNotEqualTo(mainSnapshotIdAtPlan);
+
+ harness.processBothWatermarks(new Watermark(doneTs));
+
+ // Trigger still fires (so the aggregator records the cycle), but the commit
+ // must have been rejected by validateNoConflictingDataFiles.
+ assertThat(harness.extractOutputValues()).hasSize(1);
+
+ List<StreamRecord<Exception>> errors =
+ Lists.newArrayList(harness.getSideOutput(TaskResultAggregator.ERROR_STREAM));
+ assertThat(errors).hasSize(1);
+ Exception failure = errors.get(0).getValue();
+ assertThat(failure).isInstanceOf(ValidationException.class);
+ assertThat(failure)
+ .hasMessageContaining("Found conflicting files that can contain records matching");
+
+ // Target head is still the external commit, not our cycle's commit.
+ table.refresh();
+ assertThat(table.currentSnapshot().snapshotId()).isEqualTo(mainSnapshotIdAfterExternal);
+ assertThat(
+ table
+ .currentSnapshot()
+ .summary()
+ .get(EqualityConvertCommitter.COMMITTED_STAGING_SNAPSHOT_PROPERTY))
+ .isNull();
+ }
+ }
+
+ @Test
+ void failsReplayOfCommittedPlanFromOlderState() throws Exception {
+ Table table = createTable(3, FileFormat.PARQUET);
+ insert(table, 1, "a");
+
+ DataFile stagingDataFile = getFirstDataFile(table);
+ long mainSnapshotIdAtPlan = table.currentSnapshot().snapshotId();
+ long stagingSnapshotId = 909L;
+
+ try (TwoInputStreamOperatorTestHarness<DVWriteResult, EqualityConvertPlan, Trigger> harness =
+ createHarness()) {
+ harness.open();
+
+ long doneTs = System.currentTimeMillis();
+ EqualityConvertPlan planResult =
+ new EqualityConvertPlan(
+ Lists.newArrayList(stagingDataFile),
+ Lists.newArrayList(),
+ stagingSnapshotId,
+ mainSnapshotIdAtPlan,
+ doneTs - 1,
+ doneTs);
+
+ // Cycle 1 commits, advancing the target past mainSnapshotIdAtPlan and writing the marker.
+ harness.processElement2(new StreamRecord<>(planResult, doneTs - 1));
+ harness.processBothWatermarks(new Watermark(doneTs));
+ assertThat(harness.extractOutputValues()).hasSize(1);
+
+ table.refresh();
+ long committedSnapshotId = table.currentSnapshot().snapshotId();
+ assertThat(committedSnapshotId).isNotEqualTo(mainSnapshotIdAtPlan);
+
+ // Restart from older state: the identical plan with its stale mainSnapshotId is replayed.
+ // The committer is stateless, so validateFromSnapshot is the only guard against a duplicate
+ // commit. It must reject the replay.
+ long doneTs2 = doneTs + 1;
+ EqualityConvertPlan replay =
+ new EqualityConvertPlan(
+ Lists.newArrayList(stagingDataFile),
+ Lists.newArrayList(),
+ stagingSnapshotId,
+ mainSnapshotIdAtPlan,
+ doneTs2 - 1,
+ doneTs2);
+
+ harness.processElement2(new StreamRecord<>(replay, doneTs2 - 1));
+ harness.processBothWatermarks(new Watermark(doneTs2));
+
+ // Trigger still fires (so the aggregator records the cycle), but the commit was rejected.
+ assertThat(harness.extractOutputValues()).hasSize(2);
+
+ List<StreamRecord<Exception>> errors =
+ Lists.newArrayList(harness.getSideOutput(TaskResultAggregator.ERROR_STREAM));
+ assertThat(errors).hasSize(1);
+ Exception failure = errors.get(0).getValue();
+ assertThat(failure).isInstanceOf(ValidationException.class);
+ assertThat(failure)
+ .hasMessageContaining("Found conflicting files that can contain records matching");
+
+ // No duplicate commit: target head is still cycle 1's commit.
+ table.refresh();
+ assertThat(table.currentSnapshot().snapshotId()).isEqualTo(committedSnapshotId);
+ }
+ }
+
+ @Test
+ void deletesWrittenDvsWhenCommitFails() throws Exception {
+ Table table = createTable(2, FileFormat.PARQUET);
+ insert(table, 1, "a");
+
+ DataFile stagingDataFile = getFirstDataFile(table);
+ long mainSnapshotIdAtPlan = table.currentSnapshot().snapshotId();
+
+ DeleteFile writtenDv = writePosDeleteFile(table, stagingDataFile.location(), 0L);
+ assertThat(table.io().newInputFile(writtenDv.location()).exists()).isTrue();
+
+ try (TwoInputStreamOperatorTestHarness<DVWriteResult, EqualityConvertPlan, Trigger> harness =
+ createHarness()) {
+ harness.open();
+
+ long doneTs = System.currentTimeMillis();
+ EqualityConvertPlan planResult =
+ new EqualityConvertPlan(
+ Lists.newArrayList(stagingDataFile),
+ Lists.newArrayList(),
+ 777L,
+ mainSnapshotIdAtPlan,
+ doneTs - 1,
+ doneTs);
+
+ harness.processElement1(
+ new StreamRecord<>(
+ new DVWriteResult(Lists.newArrayList(writtenDv), Lists.newArrayList()), doneTs));
+ harness.processElement2(new StreamRecord<>(planResult, doneTs - 1));
+
+ // External writer appends to main between plan and commit, so the commit is rejected.
+ insert(table, 2, "b");
+ table.refresh();
+
+ harness.processBothWatermarks(new Watermark(doneTs));
+
+ assertThat(harness.extractOutputValues()).hasSize(1);
+
+ List<StreamRecord<Exception>> errors =
+ Lists.newArrayList(harness.getSideOutput(TaskResultAggregator.ERROR_STREAM));
+ assertThat(errors).hasSize(1);
+ assertThat(errors.get(0).getValue()).isInstanceOf(ValidationException.class);
+
+ // The DV written this cycle is unreferenced after the failed commit and must be deleted.
+ assertThat(table.io().newInputFile(writtenDv.location()).exists()).isFalse();
+ }
+ }
+
+ @Test
+ void deletesSiblingDvsOnAbortButKeepsRewrittenDvs() throws Exception {
+ Table table = createTable(2, FileFormat.PARQUET);
+ insert(table, 1, "a");
+
+ DataFile stagingDataFile = getFirstDataFile(table);
+ long snapshotIdBefore = table.currentSnapshot().snapshotId();
+
+ DeleteFile writtenDv = writePosDeleteFile(table, stagingDataFile.location(), 0L);
+ DeleteFile rewrittenDv = writePosDeleteFile(table, stagingDataFile.location(), 1L);
+ assertThat(table.io().newInputFile(writtenDv.location()).exists()).isTrue();
+ assertThat(table.io().newInputFile(rewrittenDv.location()).exists()).isTrue();
+
+ try (TwoInputStreamOperatorTestHarness<DVWriteResult, EqualityConvertPlan, Trigger> harness =
+ createHarness()) {
+ harness.open();
+
+ long doneTs = System.currentTimeMillis();
+ EqualityConvertPlan planResult =
+ new EqualityConvertPlan(
+ Lists.newArrayList(stagingDataFile),
+ Lists.newArrayList(),
+ 42L,
+ snapshotIdBefore,
+ doneTs - 1,
+ doneTs);
+
+ // One writer task wrote a DV (rewriting an existing one); a sibling task aborted.
+ harness.processElement1(
+ new StreamRecord<>(
+ new DVWriteResult(Lists.newArrayList(writtenDv), Lists.newArrayList(rewrittenDv)),
+ doneTs));
+ harness.processElement1(new StreamRecord<>(DVWriteResult.ABORT, doneTs));
+ harness.processElement2(new StreamRecord<>(planResult, doneTs - 1));
+ harness.processBothWatermarks(new Watermark(doneTs));
+
+ assertThat(harness.extractOutputValues()).hasSize(1);
+ table.refresh();
+ assertThat(table.currentSnapshot().snapshotId()).isEqualTo(snapshotIdBefore);
+
+ // The freshly written DV is removed; the rewritten DV is still live on target and stays.
+ assertThat(table.io().newInputFile(writtenDv.location()).exists()).isFalse();
+ assertThat(table.io().newInputFile(rewrittenDv.location()).exists()).isTrue();
+ }
+ }
+
+ @Test
+ void retainsWrittenDvsWhenCommitStateUnknown() throws Exception {
+ Table table = createTable(2, FileFormat.PARQUET);
+ insert(table, 1, "a");
+
+ DataFile stagingDataFile = getFirstDataFile(table);
+ long mainSnapshotIdAtPlan = table.currentSnapshot().snapshotId();
+
+ DeleteFile writtenDv = writePosDeleteFile(table, stagingDataFile.location(), 0L);
+ assertThat(table.io().newInputFile(writtenDv.location()).exists()).isTrue();
+
+ EqualityConvertCommitter committer =
+ new EqualityConvertCommitter(
+ DUMMY_TABLE_NAME, DUMMY_TASK_NAME, tableLoader(), "staging", SnapshotRef.MAIN_BRANCH) {
+ @Override
+ void commit(RowDelta rowDelta) {
+ throw new CommitStateUnknownException(
+ new RuntimeException("simulated unknown commit state"));
+ }
+ };
+
+ try (TwoInputStreamOperatorTestHarness<DVWriteResult, EqualityConvertPlan, Trigger> harness =
+ new TwoInputStreamOperatorTestHarness<>(committer)) {
+ harness.open();
+
+ long doneTs = System.currentTimeMillis();
+ EqualityConvertPlan planResult =
+ new EqualityConvertPlan(
+ Lists.newArrayList(stagingDataFile),
+ Lists.newArrayList(),
+ 888L,
+ mainSnapshotIdAtPlan,
+ doneTs - 1,
+ doneTs);
+
+ harness.processElement1(
+ new StreamRecord<>(
+ new DVWriteResult(Lists.newArrayList(writtenDv), Lists.newArrayList()), doneTs));
+ harness.processElement2(new StreamRecord<>(planResult, doneTs - 1));
+ harness.processBothWatermarks(new Watermark(doneTs));
+
+ assertThat(harness.extractOutputValues()).hasSize(1);
+
+ List<StreamRecord<Exception>> errors =
+ Lists.newArrayList(harness.getSideOutput(TaskResultAggregator.ERROR_STREAM));
+ assertThat(errors).hasSize(1);
+ assertThat(errors.get(0).getValue()).isInstanceOf(CommitStateUnknownException.class);
+
+ // Commit outcome unknown: the DV may be live, so it must not be deleted.
+ assertThat(table.io().newInputFile(writtenDv.location()).exists()).isTrue();
+ }
+ }
+
+ @Test
+ void removesRewrittenStagingDvOnSharedBranch() throws Exception {
+ // Shared branch: stagingBranch == targetBranch == main, so the writer's DVs are already on
+ // target. A staging DV folded into a merged conversion DV must be removed, else the data file
+ // would carry two DVs (V3 allows one per data file).
+ Table table = createTable(3, FileFormat.PARQUET);
+ insert(table, 1, "a");
+
+ DataFile dataFile = getFirstDataFile(table);
+
+ // Staging DV already committed on the shared target branch; the writer rewrites it by folding
+ // it into a merged DV for the same data file.
+ DeleteFile stagingDv = writeDV(table, dataFile.location(), 0L);
+ table.newRowDelta().addDeletes(stagingDv).commit();
+ table.refresh();
+ long mainSnapshotId = table.currentSnapshot().snapshotId();
+
+ DeleteFile mergedDv = writeDV(table, dataFile.location(), 0L);
+
+ try (TwoInputStreamOperatorTestHarness<DVWriteResult, EqualityConvertPlan, Trigger> harness =
+ sharedBranchHarness()) {
+ harness.open();
+
+ long doneTs = System.currentTimeMillis();
+ EqualityConvertPlan planResult =
+ new EqualityConvertPlan(
+ Lists.newArrayList(),
+ Lists.newArrayList(stagingDv),
+ 123L,
+ mainSnapshotId,
+ doneTs - 1,
+ doneTs);
+
+ harness.processElement1(
+ new StreamRecord<>(
+ new DVWriteResult(Lists.newArrayList(mergedDv), Lists.newArrayList(stagingDv)),
+ doneTs));
+ harness.processElement2(new StreamRecord<>(planResult, doneTs - 1));
+ harness.processBothWatermarks(new Watermark(doneTs));
+
+ assertThat(harness.extractOutputValues()).hasSize(1);
+
+ // Exactly one DV references the data file: the merged DV, with the rewritten staging DV
+ // removed rather than left as a second DV on the same data file.
+ table.refresh();
+ List<DeleteFile> dvs = deletesForDataFile(table, dataFile.location());
+ assertThat(dvs).hasSize(1);
+ assertThat(dvs.get(0).location()).isEqualTo(mergedDv.location());
+ }
+ }
+
+ private TwoInputStreamOperatorTestHarness<DVWriteResult, EqualityConvertPlan, Trigger>
+ createHarness() throws Exception {
+ return new TwoInputStreamOperatorTestHarness<>(
+ new EqualityConvertCommitter(
+ DUMMY_TABLE_NAME, DUMMY_TASK_NAME, tableLoader(), "staging", SnapshotRef.MAIN_BRANCH));
+ }
+
+ private TwoInputStreamOperatorTestHarness<DVWriteResult, EqualityConvertPlan, Trigger>
+ sharedBranchHarness() throws Exception {
+ return new TwoInputStreamOperatorTestHarness<>(
+ new EqualityConvertCommitter(
+ DUMMY_TABLE_NAME,
+ DUMMY_TASK_NAME,
+ tableLoader(),
+ SnapshotRef.MAIN_BRANCH,
+ SnapshotRef.MAIN_BRANCH));
+ }
+
+ private static List<DeleteFile> deletesForDataFile(Table table, String dataFilePath) {
+ List<DeleteFile> deletes = Lists.newArrayList();
+ for (ManifestFile manifest : table.currentSnapshot().deleteManifests(table.io())) {
+ try (ManifestReader<DeleteFile> reader =
+ ManifestFiles.readDeleteManifest(manifest, table.io(), table.specs())) {
+ for (DeleteFile file : reader) {
+ if (dataFilePath.equals(file.referencedDataFile())) {
+ deletes.add(file.copy());
+ }
+ }
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ return deletes;
+ }
+
+ private static DeleteFile writeDV(Table table, String dataFilePath, long... positions)
+ throws IOException {
+ List<PositionDelete<?>> deletes = Lists.newArrayList();
+ GenericRecord nested = GenericRecord.create(table.schema());
+ for (long pos : positions) {
+ PositionDelete<GenericRecord> delete = PositionDelete.create();
+ delete.set(dataFilePath, pos, nested);
+ deletes.add(delete);
+ }
+
+ return FileHelpers.writePosDeleteFile(table, null, null, deletes, 3);
+ }
+
+ private static DataFile getFirstDataFile(Table table) {
+ for (ManifestFile manifest : table.currentSnapshot().dataManifests(table.io())) {
+ try (ManifestReader<DataFile> reader =
+ ManifestFiles.read(manifest, table.io(), table.specs())) {
+ for (DataFile file : reader) {
+ return file.copy();
+ }
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ throw new IllegalStateException("No data files found");
+ }
+}
diff --git a/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertCommitter.java b/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertCommitter.java
new file mode 100644
index 0000000000..c83b06a55a
--- /dev/null
+++ b/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertCommitter.java
@@ -0,0 +1,363 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.flink.maintenance.operator;
+
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.metrics.Counter;
+import org.apache.flink.metrics.MetricGroup;
+import org.apache.flink.streaming.api.operators.AbstractStreamOperator;
+import org.apache.flink.streaming.api.operators.TwoInputStreamOperator;
+import org.apache.flink.streaming.api.watermark.Watermark;
+import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
+import org.apache.iceberg.DataFile;
+import org.apache.iceberg.DeleteFile;
+import org.apache.iceberg.RowDelta;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.exceptions.CommitStateUnknownException;
+import org.apache.iceberg.flink.TableLoader;
+import org.apache.iceberg.flink.maintenance.api.Trigger;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.relocated.com.google.common.collect.Sets;
+import org.apache.iceberg.util.ContentFileUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Commits data files and DVs to the target branch. Receives {@link DVWriteResult}s from parallel
+ * {@link EqualityConvertDVWriter} instances (input 1) and an {@link EqualityConvertPlan} from the
+ * planner (input 2). Assembles the final file lists and commits using a {@link RowDelta} operation
+ * once the plan result and done-timestamp watermark have both arrived.
+ *
+ * <p>The commit is gated on the plan's done-timestamp watermark.
+ *
+ * <p>Emits a {@link Trigger} after each cycle (commit, no-op, or error) so the downstream {@link
+ * TaskResultAggregator} can track task completion. This is the sole source of Trigger records for
+ * the Aggregator.
+ *
+ * <p>No-op vs error: a no-op cycle (empty plan result from {@code
+ * EqualityConvertPlanner.emitNoOpResult}) returns early in {@code commitIfNeeded} without writing
+ * anything; the Trigger emit in {@code processWatermark} still happens, so the maintenance task
+ * completes cleanly. Errors are reported via {@link TaskResultAggregator#ERROR_STREAM} side output;
+ * the Aggregator collects them and surfaces failure on its own watermark.
+ *
+ * <p>The committer is intentionally stateless: {@code bufferedResults} and {@code planResult} are
+ * not checkpointed. The maintenance framework ensures mutual exclusive tasks, and on restart the
+ * planner re-derives its position from {@link #COMMITTED_STAGING_SNAPSHOT_PROPERTY} on main, so the
+ * committer never receives a plan for an already-committed staging snapshot.
+ */
+@Internal
+public class EqualityConvertCommitter extends AbstractStreamOperator<Trigger>
+ implements TwoInputStreamOperator<DVWriteResult, EqualityConvertPlan, Trigger> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(EqualityConvertCommitter.class);
+
+ static final String COMMITTED_STAGING_SNAPSHOT_PROPERTY = "equality-convert-staging-snapshot";
+
+ private static final String ADDED_DV_NUM_METRIC = "addedDvNum";
+ private static final String COMMIT_DURATION_MS_METRIC = "commitDurationMs";
+
+ private final String tableName;
+ private final String taskName;
+ private final TableLoader tableLoader;
+ private final String targetBranch;
+ private final boolean stagingOnTargetBranch;
+
+ private transient Table table;
+ private transient List<DVWriteResult> bufferedResults;
+ private transient EqualityConvertPlan planResult;
+
+ private transient Counter errorCounter;
+ private transient Counter addedDataFileNumCounter;
+ private transient Counter addedDataFileSizeCounter;
+ private transient Counter addedDvNumCounter;
+ private transient Counter commitDurationMsCounter;
+
+ public EqualityConvertCommitter(
+ String tableName,
+ String taskName,
+ TableLoader tableLoader,
+ String stagingBranch,
+ String targetBranch) {
+ this.tableName = tableName;
+ this.taskName = taskName;
+ this.tableLoader = tableLoader;
+ this.targetBranch = targetBranch;
+ this.stagingOnTargetBranch = stagingBranch.equals(targetBranch);
+ }
+
+ @Override
+ public void open() throws Exception {
+ super.open();
+ if (!tableLoader.isOpen()) {
+ tableLoader.open();
+ }
+
+ this.table = tableLoader.loadTable();
+ this.bufferedResults = Lists.newArrayList();
+
+ MetricGroup taskMetricGroup =
+ TableMaintenanceMetrics.groupFor(getRuntimeContext(), tableName, taskName, 0);
+ this.errorCounter = taskMetricGroup.counter(TableMaintenanceMetrics.ERROR_COUNTER);
+ this.addedDataFileNumCounter =
+ taskMetricGroup.counter(TableMaintenanceMetrics.ADDED_DATA_FILE_NUM_METRIC);
+ this.addedDataFileSizeCounter =
+ taskMetricGroup.counter(TableMaintenanceMetrics.ADDED_DATA_FILE_SIZE_METRIC);
+ this.addedDvNumCounter = taskMetricGroup.counter(ADDED_DV_NUM_METRIC);
+ this.commitDurationMsCounter = taskMetricGroup.counter(COMMIT_DURATION_MS_METRIC);
+ }
+
+ @Override
+ public void processElement1(StreamRecord<DVWriteResult> record) {
+ bufferedResults.add(record.getValue());
+ }
+
+ @Override
+ public void processElement2(StreamRecord<EqualityConvertPlan> record) {
+ planResult = record.getValue();
+ }
+
+ @Override
+ public void processWatermark(Watermark mark) throws Exception {
+ if (planResult != null && mark.getTimestamp() >= planResult.doneTimestamp()) {
+ try {
+ commitIfNeeded();
+ } catch (Exception e) {
+ LOG.error(
+ "Failed to commit equality convert result for table {} task {}",
+ tableName,
+ taskName,
+ e);
+ output.collect(TaskResultAggregator.ERROR_STREAM, new StreamRecord<>(e));
+ errorCounter.inc();
+ }
+
+ // Emit Trigger for the Aggregator (even on error or no-op).
+ output.collect(new StreamRecord<>(Trigger.create(planResult.triggerTimestamp(), 0)));
+
+ bufferedResults.clear();
+ planResult = null;
+ }
+
+ // Always forward watermarks to prevent stalling downstream.
+ super.processWatermark(mark);
+ }
+
+ @Override
+ public void close() throws Exception {
+ super.close();
+ tableLoader.close();
+ }
+
+ private void commitIfNeeded() {
+ for (DVWriteResult result : bufferedResults) {
+ if (result.isAbort()) {
+ LOG.warn(
+ "Skipping commit for table {} task {}: a DV writer reported an error.",
+ tableName,
+ taskName);
+ deleteUncommittedDVs();
+ return;
+ }
+ }
+
+ // No-op cycle: the planner emitted an empty plan result (see
+ // EqualityConvertPlanner.emitNoOpResult) because the next staging snapshot was filtered by
+ // shouldSkip or there was nothing new on staging. processWatermark still forwards a Trigger to
+ // TaskResultAggregator, so the maintenance task completes cleanly.
+ if (planResult.noOp()) {
+ return;
+ }
+
+ table.refresh();
+
+ List<DeleteFile> allDvFiles = Lists.newArrayList();
+ List<DeleteFile> allRewrittenDvFiles = Lists.newArrayList();
+ for (DVWriteResult result : bufferedResults) {
+ allDvFiles.addAll(result.dvFiles());
+ allRewrittenDvFiles.addAll(result.rewrittenDvFiles());
+ }
+
+ RowDelta rowDelta = buildRowDelta(planResult.dataFiles(), allDvFiles, allRewrittenDvFiles);
+
+ long startNano = System.nanoTime();
+ try {
+ commit(rowDelta);
+ } catch (CommitStateUnknownException e) {
+ // Commit outcome unknown: the DVs may already be referenced by a committed snapshot. Leave
+ // them for Remove Orphan Files rather than risk deleting live data.
+ throw e;
+ } catch (Exception e) {
+ // Commit definitively failed: the DVs this cycle wrote are unreferenced. Delete them so a
+ // failed cycle does not leak Puffin files. Rewritten DVs stay (still live on the target).
+ deleteUncommittedDVs();
+ throw e;
+ }
+
+ long durationMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNano);
+ commitDurationMsCounter.inc(durationMs);
+
+ LOG.info(
+ "Committed {} data files and {} DV files to branch '{}' for table {} in {} ms. "
+ + "Processed staging snapshot {}.",
+ planResult.dataFiles().size(),
+ allDvFiles.size(),
+ targetBranch,
+ tableName,
+ durationMs,
+ planResult.stagingSnapshotId());
+
+ // Only count files actually added by this commit. When sameBranch, the writer already
+ // committed the data files to target and buildRowDelta does not re-add them.
+ if (!stagingOnTargetBranch) {
+ for (DataFile dataFile : planResult.dataFiles()) {
+ addedDataFileNumCounter.inc();
+ addedDataFileSizeCounter.inc(dataFile.fileSizeInBytes());
+ }
+ }
+
+ addedDvNumCounter.inc(allDvFiles.size());
+ }
+
+ @VisibleForTesting
+ void commit(RowDelta rowDelta) {
+ rowDelta.commit();
+ }
+
+ /**
+ * Deletes the DVs this cycle wrote but did not commit (abort or definite commit failure). Only
+ * the newly written DVs are removed; rewritten DVs remain referenced on the target branch. Best
+ * effort: a delete failure is logged, not propagated, so it never masks the original error.
+ */
+ private void deleteUncommittedDVs() {
+ for (DVWriteResult result : bufferedResults) {
+ if (result.isAbort()) {
+ continue;
+ }
+
+ for (DeleteFile dvFile : result.dvFiles()) {
+ try {
+ table.io().deleteFile(dvFile.location());
+ } catch (RuntimeException e) {
+ LOG.warn(
+ "Failed to delete uncommitted DV {} for table {} task {}",
+ dvFile.location(),
+ tableName,
+ taskName,
+ e);
+ }
+ }
+ }
+ }
+
+ private RowDelta buildRowDelta(
+ List<DataFile> dataFiles, List<DeleteFile> allDvFiles, List<DeleteFile> allRewrittenDvFiles) {
+ RowDelta rowDelta = table.newRowDelta();
+
+ // Fail the commit on external target-branch activity since the planner's snapshot. The next
+ // trigger detects the change and reindexes.
+ if (planResult.mainSnapshotId() != null) {
+ rowDelta.validateFromSnapshot(planResult.mainSnapshotId());
+ }
+
+ rowDelta.validateNoConflictingDataFiles();
+ rowDelta.validateNoConflictingDeleteFiles();
+
+ Set<String> referencedDataFiles = Sets.newHashSet();
+ for (DeleteFile dvFile : allDvFiles) {
+ if (dvFile.referencedDataFile() != null) {
+ referencedDataFiles.add(dvFile.referencedDataFile());
+ }
+ }
+
+ if (!referencedDataFiles.isEmpty()) {
+ rowDelta.validateDataFilesExist(referencedDataFiles);
+ }
+
+ // When stagingBranch == targetBranch, the writer already committed data files and DVs to the
+ // target branch. Re-adding them here would produce duplicate manifest entries. Skip those
+ // paths; only the new DVs from the writer need to be added.
+ if (!stagingOnTargetBranch) {
+ for (DataFile dataFile : dataFiles) {
+ rowDelta.addRows(dataFile);
+ }
+ }
+
+ for (DeleteFile dvFile : allDvFiles) {
+ rowDelta.addDeletes(dvFile);
+ }
+
+ if (!stagingOnTargetBranch) {
+ addStagingDeletes(rowDelta, referencedDataFiles, planResult.stagingDVFiles());
+ }
+
+ removeRewrittenDVs(
+ rowDelta, allRewrittenDvFiles, planResult.stagingDVFiles(), stagingOnTargetBranch);
+
+ rowDelta.toBranch(targetBranch);
+ rowDelta.set(
+ COMMITTED_STAGING_SNAPSHOT_PROPERTY, String.valueOf(planResult.stagingSnapshotId()));
+ return rowDelta;
+ }
+
+ /** Adds staging delete files, skipping DVs that overlap with conversion DVs (V3 rule). */
+ private static void addStagingDeletes(
+ RowDelta rowDelta, Set<String> dvCoveredDataFiles, List<DeleteFile> stagingDVFiles) {
+ for (DeleteFile stagingDelete : stagingDVFiles) {
+ // V3 allows one DV per data file. When a staging snapshot contains both a writer-committed
+ // DV and an eq-delete that resolves to additional positions in the same data file, the
+ // writer folds the staging DV into a new merged DV (via
+ // EqualityConvertDVWriter.collectExistingDVs). Skip the superseded staging DV; adding both
+ // would commit two DVs for the same data file.
+ if (ContentFileUtil.isDV(stagingDelete)
+ && stagingDelete.referencedDataFile() != null
+ && dvCoveredDataFiles.contains(stagingDelete.referencedDataFile())) {
+ continue;
+ }
+
+ rowDelta.addDeletes(stagingDelete);
+ }
+ }
+
+ /**
+ * Removes rewritten DVs. On a separate target branch, staging DVs are not yet on target, so they
+ * are skipped: removeDeletes would fail. On a shared branch they are already on target and must
+ * be removed like any other rewritten DV; the merged DV that supersedes them would otherwise be a
+ * second DV for the same data file (V3 allows one).
+ */
+ private static void removeRewrittenDVs(
+ RowDelta rowDelta,
+ List<DeleteFile> allRewrittenDvFiles,
+ List<DeleteFile> stagingDVFiles,
+ boolean stagingOnTargetBranch) {
+ Set<String> stagingDeleteLocations = Sets.newHashSet();
+ for (DeleteFile sd : stagingDVFiles) {
+ stagingDeleteLocations.add(sd.location());
+ }
+
+ for (DeleteFile rewrittenDv : allRewrittenDvFiles) {
+ if (stagingOnTargetBranch || !stagingDeleteLocations.contains(rewrittenDv.location())) {
+ rowDelta.removeDeletes(rewrittenDv);
+ }
+ }
+ }
+}
diff --git a/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/TestEqualityConvertCommitter.java b/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/TestEqualityConvertCommitter.java
new file mode 100644
index 0000000000..f74570c878
--- /dev/null
+++ b/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/TestEqualityConvertCommitter.java
@@ -0,0 +1,524 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.flink.maintenance.operator;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.io.IOException;
+import java.util.List;
+import org.apache.flink.streaming.api.watermark.Watermark;
+import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
+import org.apache.flink.streaming.util.TwoInputStreamOperatorTestHarness;
+import org.apache.iceberg.DataFile;
+import org.apache.iceberg.DeleteFile;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.ManifestFile;
+import org.apache.iceberg.ManifestFiles;
+import org.apache.iceberg.ManifestReader;
+import org.apache.iceberg.RowDelta;
+import org.apache.iceberg.SnapshotRef;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.data.FileHelpers;
+import org.apache.iceberg.data.GenericRecord;
+import org.apache.iceberg.deletes.PositionDelete;
+import org.apache.iceberg.exceptions.CommitStateUnknownException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.flink.maintenance.api.Trigger;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.junit.jupiter.api.Test;
+
+class TestEqualityConvertCommitter extends OperatorTestBase {
+
+ @Test
+ void commitsDataFilesToMainBranch() throws Exception {
+ Table table = createTable(3, FileFormat.PARQUET);
+ insert(table, 1, "a");
+
+ DataFile stagingDataFile = getFirstDataFile(table);
+ long snapshotIdBefore = table.currentSnapshot().snapshotId();
+ long stagingSnapshotId = 42L;
+
+ try (TwoInputStreamOperatorTestHarness<DVWriteResult, EqualityConvertPlan, Trigger> harness =
+ createHarness()) {
+ harness.open();
+
+ long doneTs = System.currentTimeMillis();
+ EqualityConvertPlan planResult =
+ new EqualityConvertPlan(
+ Lists.newArrayList(stagingDataFile),
+ Lists.newArrayList(),
+ stagingSnapshotId,
+ snapshotIdBefore,
+ doneTs - 1,
+ doneTs);
+
+ harness.processElement2(new StreamRecord<>(planResult, doneTs - 1));
+ harness.processBothWatermarks(new Watermark(doneTs));
+
+ assertThat(harness.extractOutputValues()).hasSize(1);
+ table.refresh();
+ assertThat(table.currentSnapshot().snapshotId()).isNotEqualTo(snapshotIdBefore);
+ assertThat(
+ table
+ .currentSnapshot()
+ .summary()
+ .get(EqualityConvertCommitter.COMMITTED_STAGING_SNAPSHOT_PROPERTY))
+ .isEqualTo(String.valueOf(stagingSnapshotId));
+ }
+ }
+
+ @Test
+ void skipsCommitForEmptyCycle() throws Exception {
+ Table table = createTable(3, FileFormat.PARQUET);
+
+ try (TwoInputStreamOperatorTestHarness<DVWriteResult, EqualityConvertPlan, Trigger> harness =
+ createHarness()) {
+ harness.open();
+
+ long doneTs = System.currentTimeMillis();
+ EqualityConvertPlan planResult = EqualityConvertPlan.noOp(null, doneTs - 1, doneTs);
+
+ harness.processElement2(new StreamRecord<>(planResult, doneTs - 1));
+ harness.processBothWatermarks(new Watermark(doneTs));
+
+ assertThat(harness.extractOutputValues()).hasSize(1);
+ table.refresh();
+ assertThat(table.currentSnapshot()).isNull();
+ }
+ }
+
+ @Test
+ void abortsCommitWhenDVWriterFailed() throws Exception {
+ Table table = createTable(3, FileFormat.PARQUET);
+ insert(table, 1, "a");
+
+ DataFile stagingDataFile = getFirstDataFile(table);
+ long snapshotIdBefore = table.currentSnapshot().snapshotId();
+
+ try (TwoInputStreamOperatorTestHarness<DVWriteResult, EqualityConvertPlan, Trigger> harness =
+ createHarness()) {
+ harness.open();
+
+ long doneTs = System.currentTimeMillis();
+ EqualityConvertPlan planResult =
+ new EqualityConvertPlan(
+ Lists.newArrayList(stagingDataFile),
+ Lists.newArrayList(),
+ 42L,
+ snapshotIdBefore,
+ doneTs - 1,
+ doneTs);
+
+ // Receive an abort signal from the DVWriter followed by the planning planResult.
+ harness.processElement1(new StreamRecord<>(DVWriteResult.ABORT, doneTs));
+ harness.processElement2(new StreamRecord<>(planResult, doneTs - 1));
+
+ harness.processBothWatermarks(new Watermark(doneTs));
+
+ // The cycle must complete (Trigger emitted) but nothing must be committed to the table.
+ assertThat(harness.extractOutputValues()).hasSize(1);
+ table.refresh();
+ assertThat(table.currentSnapshot().snapshotId()).isEqualTo(snapshotIdBefore);
+ }
+ }
+
+ @Test
+ void failsCommitWhenExternalCommitLandsAfterPlan() throws Exception {
+ Table table = createTable(3, FileFormat.PARQUET);
+ insert(table, 1, "a");
+
+ DataFile stagingDataFile = getFirstDataFile(table);
+ long mainSnapshotIdAtPlan = table.currentSnapshot().snapshotId();
+ long stagingSnapshotId = 555L;
+
+ try (TwoInputStreamOperatorTestHarness<DVWriteResult, EqualityConvertPlan, Trigger> harness =
+ createHarness()) {
+ harness.open();
+
+ long doneTs = System.currentTimeMillis();
+ EqualityConvertPlan planResult =
+ new EqualityConvertPlan(
+ Lists.newArrayList(stagingDataFile),
+ Lists.newArrayList(),
+ stagingSnapshotId,
+ mainSnapshotIdAtPlan,
+ doneTs - 1,
+ doneTs);
+
+ harness.processElement2(new StreamRecord<>(planResult, doneTs - 1));
+
+ // External writer appends to main between plan and commit.
+ insert(table, 2, "b");
+ table.refresh();
+ long mainSnapshotIdAfterExternal = table.currentSnapshot().snapshotId();
+ assertThat(mainSnapshotIdAfterExternal).isNotEqualTo(mainSnapshotIdAtPlan);
+
+ harness.processBothWatermarks(new Watermark(doneTs));
+
+ // Trigger still fires (so the aggregator records the cycle), but the commit
+ // must have been rejected by validateNoConflictingDataFiles.
+ assertThat(harness.extractOutputValues()).hasSize(1);
+
+ List<StreamRecord<Exception>> errors =
+ Lists.newArrayList(harness.getSideOutput(TaskResultAggregator.ERROR_STREAM));
+ assertThat(errors).hasSize(1);
+ Exception failure = errors.get(0).getValue();
+ assertThat(failure).isInstanceOf(ValidationException.class);
+ assertThat(failure)
+ .hasMessageContaining("Found conflicting files that can contain records matching");
+
+ // Target head is still the external commit, not our cycle's commit.
+ table.refresh();
+ assertThat(table.currentSnapshot().snapshotId()).isEqualTo(mainSnapshotIdAfterExternal);
+ assertThat(
+ table
+ .currentSnapshot()
+ .summary()
+ .get(EqualityConvertCommitter.COMMITTED_STAGING_SNAPSHOT_PROPERTY))
+ .isNull();
+ }
+ }
+
+ @Test
+ void failsReplayOfCommittedPlanFromOlderState() throws Exception {
+ Table table = createTable(3, FileFormat.PARQUET);
+ insert(table, 1, "a");
+
+ DataFile stagingDataFile = getFirstDataFile(table);
+ long mainSnapshotIdAtPlan = table.currentSnapshot().snapshotId();
+ long stagingSnapshotId = 909L;
+
+ try (TwoInputStreamOperatorTestHarness<DVWriteResult, EqualityConvertPlan, Trigger> harness =
+ createHarness()) {
+ harness.open();
+
+ long doneTs = System.currentTimeMillis();
+ EqualityConvertPlan planResult =
+ new EqualityConvertPlan(
+ Lists.newArrayList(stagingDataFile),
+ Lists.newArrayList(),
+ stagingSnapshotId,
+ mainSnapshotIdAtPlan,
+ doneTs - 1,
+ doneTs);
+
+ // Cycle 1 commits, advancing the target past mainSnapshotIdAtPlan and writing the marker.
+ harness.processElement2(new StreamRecord<>(planResult, doneTs - 1));
+ harness.processBothWatermarks(new Watermark(doneTs));
+ assertThat(harness.extractOutputValues()).hasSize(1);
+
+ table.refresh();
+ long committedSnapshotId = table.currentSnapshot().snapshotId();
+ assertThat(committedSnapshotId).isNotEqualTo(mainSnapshotIdAtPlan);
+
+ // Restart from older state: the identical plan with its stale mainSnapshotId is replayed.
+ // The committer is stateless, so validateFromSnapshot is the only guard against a duplicate
+ // commit. It must reject the replay.
+ long doneTs2 = doneTs + 1;
+ EqualityConvertPlan replay =
+ new EqualityConvertPlan(
+ Lists.newArrayList(stagingDataFile),
+ Lists.newArrayList(),
+ stagingSnapshotId,
+ mainSnapshotIdAtPlan,
+ doneTs2 - 1,
+ doneTs2);
+
+ harness.processElement2(new StreamRecord<>(replay, doneTs2 - 1));
+ harness.processBothWatermarks(new Watermark(doneTs2));
+
+ // Trigger still fires (so the aggregator records the cycle), but the commit was rejected.
+ assertThat(harness.extractOutputValues()).hasSize(2);
+
+ List<StreamRecord<Exception>> errors =
+ Lists.newArrayList(harness.getSideOutput(TaskResultAggregator.ERROR_STREAM));
+ assertThat(errors).hasSize(1);
+ Exception failure = errors.get(0).getValue();
+ assertThat(failure).isInstanceOf(ValidationException.class);
+ assertThat(failure)
+ .hasMessageContaining("Found conflicting files that can contain records matching");
+
+ // No duplicate commit: target head is still cycle 1's commit.
+ table.refresh();
+ assertThat(table.currentSnapshot().snapshotId()).isEqualTo(committedSnapshotId);
+ }
+ }
+
+ @Test
+ void deletesWrittenDvsWhenCommitFails() throws Exception {
+ Table table = createTable(2, FileFormat.PARQUET);
+ insert(table, 1, "a");
+
+ DataFile stagingDataFile = getFirstDataFile(table);
+ long mainSnapshotIdAtPlan = table.currentSnapshot().snapshotId();
+
+ DeleteFile writtenDv = writePosDeleteFile(table, stagingDataFile.location(), 0L);
+ assertThat(table.io().newInputFile(writtenDv.location()).exists()).isTrue();
+
+ try (TwoInputStreamOperatorTestHarness<DVWriteResult, EqualityConvertPlan, Trigger> harness =
+ createHarness()) {
+ harness.open();
+
+ long doneTs = System.currentTimeMillis();
+ EqualityConvertPlan planResult =
+ new EqualityConvertPlan(
+ Lists.newArrayList(stagingDataFile),
+ Lists.newArrayList(),
+ 777L,
+ mainSnapshotIdAtPlan,
+ doneTs - 1,
+ doneTs);
+
+ harness.processElement1(
+ new StreamRecord<>(
+ new DVWriteResult(Lists.newArrayList(writtenDv), Lists.newArrayList()), doneTs));
+ harness.processElement2(new StreamRecord<>(planResult, doneTs - 1));
+
+ // External writer appends to main between plan and commit, so the commit is rejected.
+ insert(table, 2, "b");
+ table.refresh();
+
+ harness.processBothWatermarks(new Watermark(doneTs));
+
+ assertThat(harness.extractOutputValues()).hasSize(1);
+
+ List<StreamRecord<Exception>> errors =
+ Lists.newArrayList(harness.getSideOutput(TaskResultAggregator.ERROR_STREAM));
+ assertThat(errors).hasSize(1);
+ assertThat(errors.get(0).getValue()).isInstanceOf(ValidationException.class);
+
+ // The DV written this cycle is unreferenced after the failed commit and must be deleted.
+ assertThat(table.io().newInputFile(writtenDv.location()).exists()).isFalse();
+ }
+ }
+
+ @Test
+ void deletesSiblingDvsOnAbortButKeepsRewrittenDvs() throws Exception {
+ Table table = createTable(2, FileFormat.PARQUET);
+ insert(table, 1, "a");
+
+ DataFile stagingDataFile = getFirstDataFile(table);
+ long snapshotIdBefore = table.currentSnapshot().snapshotId();
+
+ DeleteFile writtenDv = writePosDeleteFile(table, stagingDataFile.location(), 0L);
+ DeleteFile rewrittenDv = writePosDeleteFile(table, stagingDataFile.location(), 1L);
+ assertThat(table.io().newInputFile(writtenDv.location()).exists()).isTrue();
+ assertThat(table.io().newInputFile(rewrittenDv.location()).exists()).isTrue();
+
+ try (TwoInputStreamOperatorTestHarness<DVWriteResult, EqualityConvertPlan, Trigger> harness =
+ createHarness()) {
+ harness.open();
+
+ long doneTs = System.currentTimeMillis();
+ EqualityConvertPlan planResult =
+ new EqualityConvertPlan(
+ Lists.newArrayList(stagingDataFile),
+ Lists.newArrayList(),
+ 42L,
+ snapshotIdBefore,
+ doneTs - 1,
+ doneTs);
+
+ // One writer task wrote a DV (rewriting an existing one); a sibling task aborted.
+ harness.processElement1(
+ new StreamRecord<>(
+ new DVWriteResult(Lists.newArrayList(writtenDv), Lists.newArrayList(rewrittenDv)),
+ doneTs));
+ harness.processElement1(new StreamRecord<>(DVWriteResult.ABORT, doneTs));
+ harness.processElement2(new StreamRecord<>(planResult, doneTs - 1));
+ harness.processBothWatermarks(new Watermark(doneTs));
+
+ assertThat(harness.extractOutputValues()).hasSize(1);
+ table.refresh();
+ assertThat(table.currentSnapshot().snapshotId()).isEqualTo(snapshotIdBefore);
+
+ // The freshly written DV is removed; the rewritten DV is still live on target and stays.
+ assertThat(table.io().newInputFile(writtenDv.location()).exists()).isFalse();
+ assertThat(table.io().newInputFile(rewrittenDv.location()).exists()).isTrue();
+ }
+ }
+
+ @Test
+ void retainsWrittenDvsWhenCommitStateUnknown() throws Exception {
+ Table table = createTable(2, FileFormat.PARQUET);
+ insert(table, 1, "a");
+
+ DataFile stagingDataFile = getFirstDataFile(table);
+ long mainSnapshotIdAtPlan = table.currentSnapshot().snapshotId();
+
+ DeleteFile writtenDv = writePosDeleteFile(table, stagingDataFile.location(), 0L);
+ assertThat(table.io().newInputFile(writtenDv.location()).exists()).isTrue();
+
+ EqualityConvertCommitter committer =
+ new EqualityConvertCommitter(
+ DUMMY_TABLE_NAME, DUMMY_TASK_NAME, tableLoader(), "staging", SnapshotRef.MAIN_BRANCH) {
+ @Override
+ void commit(RowDelta rowDelta) {
+ throw new CommitStateUnknownException(
+ new RuntimeException("simulated unknown commit state"));
+ }
+ };
+
+ try (TwoInputStreamOperatorTestHarness<DVWriteResult, EqualityConvertPlan, Trigger> harness =
+ new TwoInputStreamOperatorTestHarness<>(committer)) {
+ harness.open();
+
+ long doneTs = System.currentTimeMillis();
+ EqualityConvertPlan planResult =
+ new EqualityConvertPlan(
+ Lists.newArrayList(stagingDataFile),
+ Lists.newArrayList(),
+ 888L,
+ mainSnapshotIdAtPlan,
+ doneTs - 1,
+ doneTs);
+
+ harness.processElement1(
+ new StreamRecord<>(
+ new DVWriteResult(Lists.newArrayList(writtenDv), Lists.newArrayList()), doneTs));
+ harness.processElement2(new StreamRecord<>(planResult, doneTs - 1));
+ harness.processBothWatermarks(new Watermark(doneTs));
+
+ assertThat(harness.extractOutputValues()).hasSize(1);
+
+ List<StreamRecord<Exception>> errors =
+ Lists.newArrayList(harness.getSideOutput(TaskResultAggregator.ERROR_STREAM));
+ assertThat(errors).hasSize(1);
+ assertThat(errors.get(0).getValue()).isInstanceOf(CommitStateUnknownException.class);
+
+ // Commit outcome unknown: the DV may be live, so it must not be deleted.
+ assertThat(table.io().newInputFile(writtenDv.location()).exists()).isTrue();
+ }
+ }
+
+ @Test
+ void removesRewrittenStagingDvOnSharedBranch() throws Exception {
+ // Shared branch: stagingBranch == targetBranch == main, so the writer's DVs are already on
+ // target. A staging DV folded into a merged conversion DV must be removed, else the data file
+ // would carry two DVs (V3 allows one per data file).
+ Table table = createTable(3, FileFormat.PARQUET);
+ insert(table, 1, "a");
+
+ DataFile dataFile = getFirstDataFile(table);
+
+ // Staging DV already committed on the shared target branch; the writer rewrites it by folding
+ // it into a merged DV for the same data file.
+ DeleteFile stagingDv = writeDV(table, dataFile.location(), 0L);
+ table.newRowDelta().addDeletes(stagingDv).commit();
+ table.refresh();
+ long mainSnapshotId = table.currentSnapshot().snapshotId();
+
+ DeleteFile mergedDv = writeDV(table, dataFile.location(), 0L);
+
+ try (TwoInputStreamOperatorTestHarness<DVWriteResult, EqualityConvertPlan, Trigger> harness =
+ sharedBranchHarness()) {
+ harness.open();
+
+ long doneTs = System.currentTimeMillis();
+ EqualityConvertPlan planResult =
+ new EqualityConvertPlan(
+ Lists.newArrayList(),
+ Lists.newArrayList(stagingDv),
+ 123L,
+ mainSnapshotId,
+ doneTs - 1,
+ doneTs);
+
+ harness.processElement1(
+ new StreamRecord<>(
+ new DVWriteResult(Lists.newArrayList(mergedDv), Lists.newArrayList(stagingDv)),
+ doneTs));
+ harness.processElement2(new StreamRecord<>(planResult, doneTs - 1));
+ harness.processBothWatermarks(new Watermark(doneTs));
+
+ assertThat(harness.extractOutputValues()).hasSize(1);
+
+ // Exactly one DV references the data file: the merged DV, with the rewritten staging DV
+ // removed rather than left as a second DV on the same data file.
+ table.refresh();
+ List<DeleteFile> dvs = deletesForDataFile(table, dataFile.location());
+ assertThat(dvs).hasSize(1);
+ assertThat(dvs.get(0).location()).isEqualTo(mergedDv.location());
+ }
+ }
+
+ private TwoInputStreamOperatorTestHarness<DVWriteResult, EqualityConvertPlan, Trigger>
+ createHarness() throws Exception {
+ return new TwoInputStreamOperatorTestHarness<>(
+ new EqualityConvertCommitter(
+ DUMMY_TABLE_NAME, DUMMY_TASK_NAME, tableLoader(), "staging", SnapshotRef.MAIN_BRANCH));
+ }
+
+ private TwoInputStreamOperatorTestHarness<DVWriteResult, EqualityConvertPlan, Trigger>
+ sharedBranchHarness() throws Exception {
+ return new TwoInputStreamOperatorTestHarness<>(
+ new EqualityConvertCommitter(
+ DUMMY_TABLE_NAME,
+ DUMMY_TASK_NAME,
+ tableLoader(),
+ SnapshotRef.MAIN_BRANCH,
+ SnapshotRef.MAIN_BRANCH));
+ }
+
+ private static List<DeleteFile> deletesForDataFile(Table table, String dataFilePath) {
+ List<DeleteFile> deletes = Lists.newArrayList();
+ for (ManifestFile manifest : table.currentSnapshot().deleteManifests(table.io())) {
+ try (ManifestReader<DeleteFile> reader =
+ ManifestFiles.readDeleteManifest(manifest, table.io(), table.specs())) {
+ for (DeleteFile file : reader) {
+ if (dataFilePath.equals(file.referencedDataFile())) {
+ deletes.add(file.copy());
+ }
+ }
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ return deletes;
+ }
+
+ private static DeleteFile writeDV(Table table, String dataFilePath, long... positions)
+ throws IOException {
+ List<PositionDelete<?>> deletes = Lists.newArrayList();
+ GenericRecord nested = GenericRecord.create(table.schema());
+ for (long pos : positions) {
+ PositionDelete<GenericRecord> delete = PositionDelete.create();
+ delete.set(dataFilePath, pos, nested);
+ deletes.add(delete);
+ }
+
+ return FileHelpers.writePosDeleteFile(table, null, null, deletes, 3);
+ }
+
+ private static DataFile getFirstDataFile(Table table) {
+ for (ManifestFile manifest : table.currentSnapshot().dataManifests(table.io())) {
+ try (ManifestReader<DataFile> reader =
+ ManifestFiles.read(manifest, table.io(), table.specs())) {
+ for (DataFile file : reader) {
+ return file.copy();
+ }
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ throw new IllegalStateException("No data files found");
+ }
+}