You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by hu...@apache.org on 2026/06/19 02:52:50 UTC

(iceberg) branch main updated: Flink: Add equality delete conversion DV resolution and writing (#16858)

This is an automated email from the ASF dual-hosted git repository.

huaxingao pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg.git


The following commit(s) were added to refs/heads/main by this push:
     new a3a755897f Flink: Add equality delete conversion DV resolution and writing (#16858)
a3a755897f is described below

commit a3a755897fc43029adeea92f1bdb645903852ac1
Author: Maximilian Michels <mx...@apache.org>
AuthorDate: Fri Jun 19 04:52:43 2026 +0200

    Flink: Add equality delete conversion DV resolution and writing (#16858)
    
    * Flink: Add equality delete conversion DV resolution and writing
    
    Adds EqualityConvertDVWriter, a two input operator keyed by data file path for
    the equality delete conversion task. It consumes the DVPositions emitted by
    EqualityConvertPKIndex and produces a DVWriteResult for the committer. Keying by
    data file path collects every position for a data file on one task, so the
    operator can write exactly one DV per file.
    
    Positions are buffered until the watermark reaches the cycle's plan timestamp,
    broadcast on the second input, at which point the operator resolves and writes
    the DVs.
    
    Existing DVs, including staging DVs for newly added data files, are folded into
    the rewrite because V3 allows only one DV per data file. Delete manifests are
    first pruned by partition summary to bound reads to the cycle's affected
    partitions rather than the table's full DV history.
    
    The operator fails fast if the main branch snapshot changed since planning. On
    an upstream abort signal or a write failure it emits DVWriteResult.ABORT to
    ensure the committer doesn't commit partial results.
    
    * fixup! Add test for changed main snapshot abort and staging-DV merge
---
 flink/v2.1/build.gradle                            |   1 +
 .../operator/EqualityConvertDVWriter.java          | 354 ++++++++++++++++
 .../maintenance/operator/OperatorTestBase.java     |  12 +
 .../operator/TestEqualityConvertDVWriter.java      | 472 +++++++++++++++++++++
 4 files changed, 839 insertions(+)

diff --git a/flink/v2.1/build.gradle b/flink/v2.1/build.gradle
index f93b61646e..b77c60e9cc 100644
--- a/flink/v2.1/build.gradle
+++ b/flink/v2.1/build.gradle
@@ -68,6 +68,7 @@ project(":iceberg-flink:iceberg-flink-${flinkMajorVersion}") {
     }
 
     implementation libs.datasketches
+    implementation libs.roaringbitmap
 
     testImplementation libs.flink21.connector.test.utils
     testImplementation libs.flink21.core
diff --git a/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertDVWriter.java b/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertDVWriter.java
new file mode 100644
index 0000000000..486b933ee6
--- /dev/null
+++ b/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertDVWriter.java
@@ -0,0 +1,354 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.flink.maintenance.operator;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.streaming.api.operators.AbstractStreamOperator;
+import org.apache.flink.streaming.api.operators.TwoInputStreamOperator;
+import org.apache.flink.streaming.api.watermark.Watermark;
+import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
+import org.apache.iceberg.DeleteFile;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.ManifestFile;
+import org.apache.iceberg.ManifestFiles;
+import org.apache.iceberg.ManifestReader;
+import org.apache.iceberg.PartitionField;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.data.BaseDeleteLoader;
+import org.apache.iceberg.data.DeleteLoader;
+import org.apache.iceberg.deletes.BaseDVFileWriter;
+import org.apache.iceberg.deletes.PositionDeleteIndex;
+import org.apache.iceberg.expressions.Expression;
+import org.apache.iceberg.expressions.Expressions;
+import org.apache.iceberg.expressions.ManifestEvaluator;
+import org.apache.iceberg.flink.TableLoader;
+import org.apache.iceberg.io.DeleteWriteResult;
+import org.apache.iceberg.io.OutputFileFactory;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.relocated.com.google.common.collect.Sets;
+import org.apache.iceberg.types.Types;
+import org.apache.iceberg.util.ContentFileUtil;
+import org.apache.iceberg.util.StructLikeWrapper;
+import org.roaringbitmap.longlong.Roaring64Bitmap;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Keyed parallel resolver that buffers {@link DVPosition}s per data-file path, then writes Puffin
+ * DV files directly via {@link BaseDVFileWriter}. Plan metadata arrives broadcast on input 2, so
+ * every parallel task sees the cycle's metadata and can validate against the main snapshot.
+ *
+ * <p>Each buffered {@link DVPosition} carries the data file's {@code specId} + encoded partition,
+ * so writing DVs needs no data-manifest scan. Existing DVs are folded into the rewrite (V3 allows
+ * one DV per data file): delete manifests are pruned by partition summary to the cycle's affected
+ * partitions, then filtered to entries referencing the affected data files. No cross-cycle state is
+ * kept; reads are bounded by the pruned manifest set, not the table's full DV history.
+ *
+ * <p>Buffered positions are transient per-task. On failure recovery, upstream replay rebuilds them.
+ */
+@Internal
+public class EqualityConvertDVWriter extends AbstractStreamOperator<DVWriteResult>
+    implements TwoInputStreamOperator<DVPosition, EqualityConvertPlan, DVWriteResult> {
+
+  private static final Logger LOG = LoggerFactory.getLogger(EqualityConvertDVWriter.class);
+
+  private final String tableName;
+  private final String taskName;
+  private final TableLoader tableLoader;
+  private final String targetBranch;
+
+  private transient Table table;
+  private transient OutputFileFactory fileFactory;
+  private transient DeleteLoader deleteLoader;
+  private transient Map<String, FilePositions> positionsByFile;
+  private transient EqualityConvertPlan planResult;
+  private transient boolean hasUpstreamError;
+  private transient int manifestsRead;
+
+  public EqualityConvertDVWriter(
+      String tableName, String taskName, TableLoader tableLoader, String targetBranch) {
+    this.tableName = tableName;
+    this.taskName = taskName;
+    this.tableLoader = tableLoader;
+    this.targetBranch = targetBranch;
+  }
+
+  @Override
+  public void open() throws Exception {
+    super.open();
+    if (!tableLoader.isOpen()) {
+      tableLoader.open();
+    }
+
+    table = tableLoader.loadTable();
+    int subtaskIndex = getRuntimeContext().getTaskInfo().getIndexOfThisSubtask();
+    fileFactory =
+        OutputFileFactory.builderFor(table, subtaskIndex, 0L).format(FileFormat.PUFFIN).build();
+    deleteLoader = new BaseDeleteLoader(deleteFile -> table.io().newInputFile(deleteFile));
+    positionsByFile = Maps.newHashMap();
+  }
+
+  @Override
+  public void processElement1(StreamRecord<DVPosition> record) {
+    DVPosition pos = record.getValue();
+    if (pos.isAbort()) {
+      hasUpstreamError = true;
+    }
+
+    if (!hasUpstreamError) {
+      positionsByFile
+          .computeIfAbsent(
+              pos.dataFilePath(), k -> new FilePositions(pos.specId(), pos.partition()))
+          .positions
+          .addLong(pos.position());
+    }
+  }
+
+  @Override
+  public void processElement2(StreamRecord<EqualityConvertPlan> record) {
+    planResult = record.getValue();
+  }
+
+  @Override
+  public void processWatermark(Watermark mark) throws Exception {
+    if (planResult != null && mark.getTimestamp() >= planResult.doneTimestamp()) {
+      if (hasUpstreamError) {
+        output.collect(new StreamRecord<>(DVWriteResult.ABORT));
+      } else {
+        try {
+          resolveAndWrite();
+        } catch (Exception e) {
+          LOG.error("Error writing DVs for table {} task {}", tableName, taskName, e);
+          output.collect(TaskResultAggregator.ERROR_STREAM, new StreamRecord<>(e));
+          output.collect(new StreamRecord<>(DVWriteResult.ABORT));
+        }
+      }
+
+      positionsByFile.clear();
+      hasUpstreamError = false;
+      planResult = null;
+    }
+
+    super.processWatermark(mark);
+  }
+
+  private void resolveAndWrite() throws IOException {
+    if (positionsByFile.isEmpty()) {
+      return;
+    }
+
+    table.refresh();
+
+    Snapshot mainSnapshot = table.snapshot(targetBranch);
+
+    // Fail fast if the main branch changed since planning, to avoid writing DV files that the
+    // committer would reject via validateFromSnapshot. The next cycle will reindex.
+    if (mainSnapshot != null
+        && planResult.mainSnapshotId() != null
+        && mainSnapshot.snapshotId() != planResult.mainSnapshotId()) {
+      throw new IllegalStateException(
+          "Main branch snapshot changed since planning: expected "
+              + planResult.mainSnapshotId()
+              + " but found: "
+              + mainSnapshot.snapshotId());
+    }
+
+    Map<String, DeleteFile> dvs = collectExistingDVs(mainSnapshot, positionsByFile.keySet());
+
+    // Fold staging DVs into the rewrite so the writer emits one DV per data file (V3 rule). Flink
+    // writes a staging DV only for a newly added data file, so it never collides with a distinct
+    // existing DV: on a separate target branch collectExistingDVs has not seen it yet; on a shared
+    // branch it IS that existing DV, so the put is idempotent.
+    for (DeleteFile sd : planResult.stagingDVFiles()) {
+      if (ContentFileUtil.isDV(sd) && sd.referencedDataFile() != null) {
+        dvs.put(sd.referencedDataFile(), sd);
+      }
+    }
+
+    BaseDVFileWriter dvWriter =
+        new BaseDVFileWriter(fileFactory, path -> loadPreviousDV(path, dvs));
+    try (dvWriter) {
+      for (Map.Entry<String, FilePositions> entry : positionsByFile.entrySet()) {
+        String dataFilePath = entry.getKey();
+        FilePositions filePositions = entry.getValue();
+        PartitionSpec spec = table.specs().get(filePositions.specId);
+        StructLike partition = filePositions.partition(spec.partitionType());
+
+        filePositions.positions.forEach(
+            (long pos) -> dvWriter.delete(dataFilePath, pos, spec, partition));
+      }
+    }
+
+    DeleteWriteResult result = dvWriter.result();
+    LOG.info(
+        "Wrote {} DV files (rewriting {}) for {} data files in table {} task {}.",
+        result.deleteFiles().size(),
+        result.rewrittenDeleteFiles().size(),
+        positionsByFile.size(),
+        tableName,
+        taskName);
+
+    output.collect(
+        new StreamRecord<>(
+            new DVWriteResult(
+                Lists.newArrayList(result.deleteFiles()),
+                Lists.newArrayList(result.rewrittenDeleteFiles()))));
+  }
+
+  @Override
+  public void close() throws Exception {
+    super.close();
+    tableLoader.close();
+  }
+
+  private Map<String, DeleteFile> collectExistingDVs(
+      Snapshot mainSnapshot, Set<String> affectedPaths) {
+    manifestsRead = 0;
+    Map<String, DeleteFile> dvs = Maps.newHashMap();
+    if (mainSnapshot == null) {
+      return dvs;
+    }
+
+    // Prune delete manifests whose partition summaries cannot cover the cycle's affected
+    // partitions. A DV inherits its referenced data file's spec and partition, so partition pruning
+    // works for DV manifests.
+    Map<Integer, ManifestEvaluator> evaluators = partitionEvaluators(positionsByFile);
+    for (ManifestFile manifest : mainSnapshot.deleteManifests(table.io())) {
+      ManifestEvaluator evaluator = evaluators.get(manifest.partitionSpecId());
+      if (evaluator == null || !evaluator.eval(manifest)) {
+        continue;
+      }
+
+      readDVEntries(manifest, affectedPaths, dvs);
+    }
+
+    return dvs;
+  }
+
+  private Map<Integer, ManifestEvaluator> partitionEvaluators(
+      Map<String, FilePositions> positions) {
+    Map<Integer, StructLikeWrapper> templatesBySpec = Maps.newHashMap();
+    Map<Integer, Set<StructLikeWrapper>> partitionsBySpec = Maps.newHashMap();
+    for (FilePositions filePositions : positions.values()) {
+      PartitionSpec spec = table.specs().get(filePositions.specId);
+      StructLikeWrapper template =
+          templatesBySpec.computeIfAbsent(
+              filePositions.specId, id -> StructLikeWrapper.forType(spec.partitionType()));
+      partitionsBySpec
+          .computeIfAbsent(filePositions.specId, k -> Sets.newHashSet())
+          .add(template.copyFor(filePositions.partition(spec.partitionType())));
+    }
+
+    Map<Integer, ManifestEvaluator> evaluators = Maps.newHashMap();
+    for (Map.Entry<Integer, Set<StructLikeWrapper>> entry : partitionsBySpec.entrySet()) {
+      PartitionSpec spec = table.specs().get(entry.getKey());
+      Expression filter = partitionFilter(spec, entry.getValue());
+      evaluators.put(entry.getKey(), ManifestEvaluator.forPartitionFilter(filter, spec, false));
+    }
+
+    return evaluators;
+  }
+
+  private static Expression partitionFilter(PartitionSpec spec, Set<StructLikeWrapper> partitions) {
+    List<PartitionField> fields = spec.fields();
+    Expression anyPartition = Expressions.alwaysFalse();
+    for (StructLikeWrapper wrapper : partitions) {
+      StructLike partition = wrapper.get();
+      Expression onePartition = Expressions.alwaysTrue();
+      for (int i = 0; i < fields.size(); i++) {
+        String name = fields.get(i).name();
+        Object value = partition.get(i, Object.class);
+        Expression predicate =
+            value == null ? Expressions.isNull(name) : Expressions.equal(name, value);
+        onePartition = Expressions.and(onePartition, predicate);
+      }
+
+      anyPartition = Expressions.or(anyPartition, onePartition);
+    }
+
+    return anyPartition;
+  }
+
+  private void readDVEntries(
+      ManifestFile manifest, Set<String> filterPaths, Map<String, DeleteFile> out) {
+    manifestsRead++;
+    try (ManifestReader<DeleteFile> reader =
+        ManifestFiles.readDeleteManifest(manifest, table.io(), table.specs())) {
+      for (DeleteFile deleteFile : reader) {
+        if (ContentFileUtil.isDV(deleteFile)
+            && deleteFile.referencedDataFile() != null
+            && filterPaths.contains(deleteFile.referencedDataFile())) {
+          out.put(deleteFile.referencedDataFile(), deleteFile);
+        }
+      }
+    } catch (IOException e) {
+      throw new UncheckedIOException("Failed to read manifest: " + manifest.path(), e);
+    }
+  }
+
+  @VisibleForTesting
+  int manifestsReadLastCycle() {
+    return manifestsRead;
+  }
+
+  @VisibleForTesting
+  int retainedStateSize() {
+    return positionsByFile.size();
+  }
+
+  private PositionDeleteIndex loadPreviousDV(String dataFilePath, Map<String, DeleteFile> dvs) {
+    DeleteFile existingDV = dvs.get(dataFilePath);
+    if (existingDV == null) {
+      return null;
+    }
+
+    return deleteLoader.loadPositionDeletes(ImmutableList.of(existingDV), dataFilePath);
+  }
+
+  private static final class FilePositions {
+    private final int specId;
+    private final byte[] encodedPartition;
+    private final Roaring64Bitmap positions = new Roaring64Bitmap();
+    private StructLike decodedPartition;
+
+    FilePositions(int specId, byte[] encodedPartition) {
+      this.specId = specId;
+      this.encodedPartition = encodedPartition;
+    }
+
+    StructLike partition(Types.StructType partitionType) {
+      if (decodedPartition == null) {
+        decodedPartition = StructLikeSerializer.decodePartition(encodedPartition, partitionType);
+      }
+
+      return decodedPartition;
+    }
+  }
+}
diff --git a/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/OperatorTestBase.java b/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/OperatorTestBase.java
index b4f6ef95fc..a80e5c1bb2 100644
--- a/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/OperatorTestBase.java
+++ b/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/OperatorTestBase.java
@@ -188,6 +188,18 @@ public class OperatorTestBase {
                 "format-version", String.valueOf(formatVersion), "write.upsert.enabled", "true"));
   }
 
+  protected static Table createPartitionedTableWithDelete(int formatVersion) {
+    return CATALOG_EXTENSION
+        .catalog()
+        .createTable(
+            TestFixtures.TABLE_IDENTIFIER,
+            SCHEMA_WITH_PRIMARY_KEY,
+            PartitionSpec.builderFor(SCHEMA_WITH_PRIMARY_KEY).identity("data").build(),
+            null,
+            ImmutableMap.of(
+                "format-version", String.valueOf(formatVersion), "write.upsert.enabled", "true"));
+  }
+
   protected static Table createPartitionedTable(int formatVersion, FileFormat fileFormat) {
     return CATALOG_EXTENSION
         .catalog()
diff --git a/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/TestEqualityConvertDVWriter.java b/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/TestEqualityConvertDVWriter.java
new file mode 100644
index 0000000000..a0c2f3568b
--- /dev/null
+++ b/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/TestEqualityConvertDVWriter.java
@@ -0,0 +1,472 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.flink.maintenance.operator;
+
+import static org.apache.iceberg.flink.SimpleDataUtil.createRecord;
+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.util.List;
+import org.apache.flink.streaming.api.watermark.Watermark;
+import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
+import org.apache.flink.streaming.util.TwoInputStreamOperatorTestHarness;
+import org.apache.iceberg.DataFile;
+import org.apache.iceberg.DeleteFile;
+import org.apache.iceberg.ManifestFile;
+import org.apache.iceberg.ManifestFiles;
+import org.apache.iceberg.ManifestReader;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.RowDelta;
+import org.apache.iceberg.SnapshotRef;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.junit.jupiter.api.Test;
+
+class TestEqualityConvertDVWriter extends OperatorTestBase {
+
+  private static final byte[] EMPTY_PARTITION = new byte[0];
+
+  @Test
+  void writesDVFileForSinglePosition() throws Exception {
+    Table table = createTableWithDelete(3);
+    insert(table, 1, "a");
+
+    String dataFilePath = getFirstDataFilePath(table);
+
+    try (TwoInputStreamOperatorTestHarness<DVPosition, EqualityConvertPlan, DVWriteResult> harness =
+        createHarness()) {
+      harness.open();
+
+      long time = System.currentTimeMillis();
+      harness.processElement1(
+          new StreamRecord<>(new DVPosition(dataFilePath, 0, 0, EMPTY_PARTITION, 0L), time));
+      harness.processElement2(new StreamRecord<>(emptyEqualityConvertPlan(), time));
+
+      harness.processBothWatermarks(new Watermark(time));
+
+      List<DVWriteResult> output = harness.extractOutputValues();
+      assertThat(output).hasSize(1);
+      assertThat(output.get(0).hasError()).isFalse();
+      assertThat(output.get(0).dvFiles()).hasSize(1);
+    }
+  }
+
+  @Test
+  void writesSingleDVFileForMultiplePositionsOnSameDataFile() throws Exception {
+    Table table = createTableWithDelete(3);
+    insert(table, 1, "a");
+    insert(table, 2, "b");
+    insert(table, 3, "c");
+
+    String dataFilePath = getFirstDataFilePath(table);
+
+    try (TwoInputStreamOperatorTestHarness<DVPosition, EqualityConvertPlan, DVWriteResult> harness =
+        createHarness()) {
+      harness.open();
+
+      long time = System.currentTimeMillis();
+      harness.processElement1(
+          new StreamRecord<>(new DVPosition(dataFilePath, 0, 0, EMPTY_PARTITION, 0L), time));
+      harness.processElement1(
+          new StreamRecord<>(new DVPosition(dataFilePath, 1, 0, EMPTY_PARTITION, 0L), time));
+      harness.processElement1(
+          new StreamRecord<>(new DVPosition(dataFilePath, 2, 0, EMPTY_PARTITION, 0L), time));
+      harness.processElement2(new StreamRecord<>(emptyEqualityConvertPlan(), time));
+
+      harness.processBothWatermarks(new Watermark(time));
+
+      List<DVWriteResult> output = harness.extractOutputValues();
+      assertThat(output).hasSize(1);
+      assertThat(output.get(0).hasError()).isFalse();
+      assertThat(output.get(0).dvFiles()).hasSize(1);
+      assertThat(output.get(0).dvFiles().get(0).recordCount()).isEqualTo(3);
+      assertThat(output.get(0).rewrittenDvFiles()).isEmpty();
+    }
+  }
+
+  @Test
+  void emptyPositionsProducesNoOutput() throws Exception {
+    createTableWithDelete(3);
+
+    try (TwoInputStreamOperatorTestHarness<DVPosition, EqualityConvertPlan, DVWriteResult> harness =
+        createHarness()) {
+      harness.open();
+
+      long time = System.currentTimeMillis();
+      harness.processElement2(new StreamRecord<>(emptyEqualityConvertPlan(), time));
+
+      harness.processBothWatermarks(new Watermark(time));
+
+      assertThat(harness.extractOutputValues()).isEmpty();
+    }
+  }
+
+  @Test
+  void noOutputWithoutPlanResult() throws Exception {
+    Table table = createTableWithDelete(3);
+    insert(table, 1, "a");
+
+    String dataFilePath = getFirstDataFilePath(table);
+
+    try (TwoInputStreamOperatorTestHarness<DVPosition, EqualityConvertPlan, DVWriteResult> harness =
+        createHarness()) {
+      harness.open();
+
+      long time = System.currentTimeMillis();
+      harness.processElement1(
+          new StreamRecord<>(new DVPosition(dataFilePath, 0, 0, EMPTY_PARTITION, 0L), time));
+
+      harness.processBothWatermarks(new Watermark(time));
+
+      assertThat(harness.extractOutputValues()).isEmpty();
+    }
+  }
+
+  @Test
+  void writesDVFilesForMultipleDataFiles() throws Exception {
+    Table table = createTableWithDelete(3);
+    insert(table, 1, "a");
+    insert(table, 2, "b");
+
+    List<String> dataFilePaths = getDataFilePaths(table);
+    assertThat(dataFilePaths).hasSize(2);
+
+    try (TwoInputStreamOperatorTestHarness<DVPosition, EqualityConvertPlan, DVWriteResult> harness =
+        createHarness()) {
+      harness.open();
+
+      long time = System.currentTimeMillis();
+      harness.processElement1(
+          new StreamRecord<>(
+              new DVPosition(dataFilePaths.get(0), 0, 0, EMPTY_PARTITION, 0L), time));
+      harness.processElement1(
+          new StreamRecord<>(
+              new DVPosition(dataFilePaths.get(1), 0, 0, EMPTY_PARTITION, 0L), time));
+      harness.processElement2(new StreamRecord<>(emptyEqualityConvertPlan(), time));
+
+      harness.processBothWatermarks(new Watermark(time));
+
+      List<DVWriteResult> output = harness.extractOutputValues();
+      assertThat(output).hasSize(1);
+      assertThat(output.get(0).dvFiles()).hasSize(2);
+    }
+  }
+
+  @Test
+  void routesErrorToErrorStream() throws Exception {
+    Table table = createTableWithDelete(3);
+    insert(table, 1, "a");
+
+    String dataFilePath = getFirstDataFilePath(table);
+
+    try (TwoInputStreamOperatorTestHarness<DVPosition, EqualityConvertPlan, DVWriteResult> harness =
+        createHarness()) {
+      harness.open();
+
+      long time = System.currentTimeMillis();
+      harness.processElement1(
+          new StreamRecord<>(new DVPosition(dataFilePath, 0, 0, EMPTY_PARTITION, 0L), time));
+      harness.processElement2(new StreamRecord<>(emptyEqualityConvertPlan(), time));
+
+      dropTable();
+
+      harness.processBothWatermarks(new Watermark(time));
+
+      assertThat(harness.extractOutputValues()).hasSize(1);
+      assertThat(harness.extractOutputValues().get(0).hasError()).isTrue();
+      assertThat(harness.getSideOutput(TaskResultAggregator.ERROR_STREAM)).hasSize(1);
+    }
+  }
+
+  @Test
+  void mergesStagingDVIntoWrittenDV() throws Exception {
+    Table table = createTableWithDelete(3);
+    insert(table, 1, "a");
+
+    String dataFilePath = getFirstDataFilePath(table);
+
+    // Staging DV for the data file at position 0, produced but not committed to the target branch.
+    DeleteFile stagingDV = writeStagingDV(dataFilePath, 0);
+
+    try (TwoInputStreamOperatorTestHarness<DVPosition, EqualityConvertPlan, DVWriteResult> harness =
+        createHarness()) {
+      harness.open();
+
+      long time = System.currentTimeMillis();
+      EqualityConvertPlan planResult =
+          new EqualityConvertPlan(
+              Lists.newArrayList(), Lists.newArrayList(stagingDV), 1L, null, 0L, 0L);
+
+      // New position 1 in the same file. Must merge it with the staged position 0.
+      harness.processElement1(
+          new StreamRecord<>(new DVPosition(dataFilePath, 1, 0, EMPTY_PARTITION, 0L), time));
+      harness.processElement2(new StreamRecord<>(planResult, time));
+
+      harness.processBothWatermarks(new Watermark(time));
+
+      List<DVWriteResult> output = harness.extractOutputValues();
+      assertThat(output).hasSize(1);
+      assertThat(output.get(0).hasError()).isFalse();
+      assertThat(output.get(0).dvFiles()).hasSize(1);
+      assertThat(output.get(0).dvFiles().get(0).recordCount()).isEqualTo(2);
+      assertThat(output.get(0).rewrittenDvFiles()).hasSize(1);
+      assertThat(output.get(0).rewrittenDvFiles().get(0).location())
+          .isEqualTo(stagingDV.location());
+    }
+  }
+
+  @Test
+  void abortsWhenMainSnapshotChangedSincePlanning() throws Exception {
+    Table table = createTableWithDelete(3);
+    insert(table, 1, "a");
+
+    String dataFilePath = getFirstDataFilePath(table);
+    long plannedSnapshotId = table.currentSnapshot().snapshotId();
+
+    try (TwoInputStreamOperatorTestHarness<DVPosition, EqualityConvertPlan, DVWriteResult> harness =
+        createHarness()) {
+      harness.open();
+
+      long time = System.currentTimeMillis();
+      EqualityConvertPlan planResult =
+          new EqualityConvertPlan(
+              Lists.newArrayList(), Lists.newArrayList(), 1L, plannedSnapshotId, 0L, 0L);
+
+      harness.processElement1(
+          new StreamRecord<>(new DVPosition(dataFilePath, 0, 0, EMPTY_PARTITION, 0L), time));
+      harness.processElement2(new StreamRecord<>(planResult, time));
+
+      // Advance the target branch after planning. The writer must refuse to write stale DVs.
+      insert(table, 2, "b");
+
+      harness.processBothWatermarks(new Watermark(time));
+
+      List<DVWriteResult> output = harness.extractOutputValues();
+      assertThat(output).hasSize(1);
+      assertThat(output.get(0).hasError()).isTrue();
+      assertThat(harness.getSideOutput(TaskResultAggregator.ERROR_STREAM)).hasSize(1);
+    }
+  }
+
+  @Test
+  void stateResetBetweenBatches() throws Exception {
+    Table table = createTableWithDelete(3);
+    insert(table, 1, "a");
+
+    String dataFilePath = getFirstDataFilePath(table);
+
+    try (TwoInputStreamOperatorTestHarness<DVPosition, EqualityConvertPlan, DVWriteResult> harness =
+        createHarness()) {
+      harness.open();
+
+      long time1 = System.currentTimeMillis();
+      harness.processElement1(
+          new StreamRecord<>(new DVPosition(dataFilePath, 0, 0, EMPTY_PARTITION, 0L), time1));
+      harness.processElement2(new StreamRecord<>(emptyEqualityConvertPlan(), time1));
+      harness.processBothWatermarks(new Watermark(time1));
+
+      assertThat(harness.extractOutputValues()).hasSize(1);
+
+      // Second batch with no positions, plan only.
+      long time2 = time1 + 1;
+      harness.processElement2(new StreamRecord<>(emptyEqualityConvertPlan(), time2));
+      harness.processBothWatermarks(new Watermark(time2));
+
+      // Cumulative output: still hasSize(1) since the empty batch produced nothing.
+      assertThat(harness.extractOutputValues()).hasSize(1);
+    }
+  }
+
+  @Test
+  void abortsOnUpstreamAbortPosition() throws Exception {
+    createTableWithDelete(3);
+
+    try (TwoInputStreamOperatorTestHarness<DVPosition, EqualityConvertPlan, DVWriteResult> harness =
+        createHarness()) {
+      harness.open();
+
+      long time = System.currentTimeMillis();
+      harness.processElement1(new StreamRecord<>(DVPosition.ABORT, time));
+      harness.processElement2(new StreamRecord<>(emptyEqualityConvertPlan(), time));
+      harness.processBothWatermarks(new Watermark(time));
+
+      List<DVWriteResult> output = harness.extractOutputValues();
+      assertThat(output).hasSize(1);
+      assertThat(output.get(0).hasError()).isTrue();
+      assertThat(harness.getSideOutput(TaskResultAggregator.ERROR_STREAM)).isNull();
+    }
+  }
+
+  @Test
+  void doesNotRetainStateAcrossCycles() throws Exception {
+    Table table = createTableWithDelete(3);
+    int cycles = 5;
+    for (int i = 0; i < cycles; i++) {
+      insert(table, i, "v" + i);
+    }
+
+    List<String> dataFilePaths = getDataFilePaths(table);
+    assertThat(dataFilePaths).hasSize(cycles);
+
+    EqualityConvertDVWriter resolver = newResolver();
+    try (TwoInputStreamOperatorTestHarness<DVPosition, EqualityConvertPlan, DVWriteResult> harness =
+        new TwoInputStreamOperatorTestHarness<>(resolver)) {
+      harness.open();
+
+      long time = System.currentTimeMillis();
+      for (int i = 0; i < cycles; i++) {
+        // Each cycle resolves a position in a distinct data file, growing the set of files seen.
+        harness.processElement1(
+            new StreamRecord<>(
+                new DVPosition(dataFilePaths.get(i), 0, 0, EMPTY_PARTITION, 0L), time + i));
+        harness.processElement2(new StreamRecord<>(emptyEqualityConvertPlan(), time + i));
+        harness.processBothWatermarks(new Watermark(time + i));
+
+        // No cross-cycle state: the buffer clears every cycle regardless of how many distinct data
+        // files have been processed.
+        assertThat(resolver.retainedStateSize()).isZero();
+        assertThat(harness.extractOutputValues()).hasSize(i + 1);
+      }
+    }
+  }
+
+  @Test
+  void prunesDeleteManifestsByPartition() throws Exception {
+    Table table = createPartitionedTableWithDelete(3);
+    insertPartitioned(table, Lists.newArrayList(createRecord(1, "a")), "a");
+    DataFile fileA = dataFile(table, "a");
+    insertPartitioned(table, Lists.newArrayList(createRecord(2, "b")), "b");
+    DataFile fileB = dataFile(table, "b");
+
+    // Commit one DV per partition: two separate delete manifests, summaries {a} and {b}.
+    commitDV(table, fileA);
+    commitDV(table, fileB);
+    assertThat(table.currentSnapshot().deleteManifests(table.io())).hasSize(2);
+
+    // A new cycle resolves another position in partition a's file. Only partition a's delete
+    // manifest overlaps; partition b's manifest must be pruned.
+    EqualityConvertDVWriter resolver = newResolver();
+    try (TwoInputStreamOperatorTestHarness<DVPosition, EqualityConvertPlan, DVWriteResult> harness =
+        new TwoInputStreamOperatorTestHarness<>(resolver)) {
+      harness.open();
+
+      long time = System.currentTimeMillis();
+      harness.processElement1(new StreamRecord<>(dvPosition(table, fileA, 0), time));
+      harness.processElement2(new StreamRecord<>(emptyEqualityConvertPlan(), time));
+      harness.processBothWatermarks(new Watermark(time));
+
+      List<DVWriteResult> output = harness.extractOutputValues();
+      assertThat(output).hasSize(1);
+      // Folded partition a's existing DV: the matching manifest was read.
+      assertThat(output.get(0).rewrittenDvFiles()).hasSize(1);
+      assertThat(output.get(0).dvFiles()).hasSize(1);
+      // Partition b's manifest was skipped: only one manifest read this cycle.
+      assertThat(resolver.manifestsReadLastCycle()).isEqualTo(1);
+    }
+  }
+
+  private void commitDV(Table table, DataFile dataFile) throws Exception {
+    try (TwoInputStreamOperatorTestHarness<DVPosition, EqualityConvertPlan, DVWriteResult> harness =
+        createHarness()) {
+      harness.open();
+
+      long time = System.currentTimeMillis();
+      harness.processElement1(new StreamRecord<>(dvPosition(table, dataFile, 0), time));
+      harness.processElement2(new StreamRecord<>(emptyEqualityConvertPlan(), time));
+      harness.processBothWatermarks(new Watermark(time));
+
+      RowDelta rowDelta = table.newRowDelta();
+      harness.extractOutputValues().get(0).dvFiles().forEach(rowDelta::addDeletes);
+      rowDelta.commit();
+      table.refresh();
+    }
+  }
+
+  private static DVPosition dvPosition(Table table, DataFile dataFile, long position) {
+    PartitionSpec spec = table.specs().get(dataFile.specId());
+    byte[] partition =
+        new StructLikeSerializer().encodePartition(dataFile.partition(), spec.partitionType());
+    return new DVPosition(dataFile.location(), position, dataFile.specId(), partition, 0L);
+  }
+
+  private static DataFile dataFile(Table table, String partitionValue) {
+    for (ManifestFile manifest : table.currentSnapshot().dataManifests(table.io())) {
+      try (ManifestReader<DataFile> reader =
+          ManifestFiles.read(manifest, table.io(), table.specs())) {
+        for (DataFile file : reader) {
+          if (partitionValue.equals(String.valueOf(file.partition().get(0, Object.class)))) {
+            return file;
+          }
+        }
+      } catch (Exception e) {
+        throw new RuntimeException(e);
+      }
+    }
+
+    throw new IllegalStateException("No data file for partition: " + partitionValue);
+  }
+
+  private DeleteFile writeStagingDV(String dataFilePath, long position) throws Exception {
+    try (TwoInputStreamOperatorTestHarness<DVPosition, EqualityConvertPlan, DVWriteResult> harness =
+        createHarness()) {
+      harness.open();
+
+      long time = System.currentTimeMillis();
+      harness.processElement1(
+          new StreamRecord<>(new DVPosition(dataFilePath, position, 0, EMPTY_PARTITION, 0L), time));
+      harness.processElement2(new StreamRecord<>(emptyEqualityConvertPlan(), time));
+      harness.processBothWatermarks(new Watermark(time));
+
+      return harness.extractOutputValues().get(0).dvFiles().get(0);
+    }
+  }
+
+  private EqualityConvertDVWriter newResolver() {
+    return new EqualityConvertDVWriter(
+        DUMMY_TABLE_NAME, DUMMY_TASK_NAME, tableLoader(), SnapshotRef.MAIN_BRANCH);
+  }
+
+  private TwoInputStreamOperatorTestHarness<DVPosition, EqualityConvertPlan, DVWriteResult>
+      createHarness() throws Exception {
+    return new TwoInputStreamOperatorTestHarness<>(newResolver());
+  }
+
+  private static EqualityConvertPlan emptyEqualityConvertPlan() {
+    return new EqualityConvertPlan(Lists.newArrayList(), Lists.newArrayList(), 1L, null, 0L, 0L);
+  }
+
+  private static String getFirstDataFilePath(Table table) {
+    return getDataFilePaths(table).get(0);
+  }
+
+  private static List<String> getDataFilePaths(Table table) {
+    List<String> paths = Lists.newArrayList();
+    for (ManifestFile manifest : table.currentSnapshot().dataManifests(table.io())) {
+      try (ManifestReader<DataFile> reader =
+          ManifestFiles.read(manifest, table.io(), table.specs())) {
+        for (DataFile file : reader) {
+          paths.add(file.location());
+        }
+      } catch (Exception e) {
+        throw new RuntimeException(e);
+      }
+    }
+
+    return paths;
+  }
+}