You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by am...@apache.org on 2025/03/22 16:53:41 UTC

(iceberg) 02/02: Spark 3.4: Detect dangling DVs properly

This is an automated email from the ASF dual-hosted git repository.

amoghj pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg.git

commit c2a7d9f0ad1406a74d972ed1dc2b7bee1606944a
Author: Eduard Tudenhoefner <et...@gmail.com>
AuthorDate: Fri Mar 21 18:49:19 2025 +0100

    Spark 3.4: Detect dangling DVs properly
    
    this backports #12270 to Spark 3.4
---
 .../actions/RemoveDanglingDeletesSparkAction.java  | 38 ++++++++-
 .../TestRewritePositionDeleteFilesAction.java      | 91 ++++++++++++++++++++++
 .../spark/source/TestPositionDeletesTable.java     | 11 +--
 3 files changed, 131 insertions(+), 9 deletions(-)

diff --git a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/actions/RemoveDanglingDeletesSparkAction.java b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/actions/RemoveDanglingDeletesSparkAction.java
index c8c883e553..af8114fc23 100644
--- a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/actions/RemoveDanglingDeletesSparkAction.java
+++ b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/actions/RemoveDanglingDeletesSparkAction.java
@@ -26,6 +26,7 @@ import java.util.List;
 import java.util.stream.Collectors;
 import org.apache.iceberg.DataFile;
 import org.apache.iceberg.DeleteFile;
+import org.apache.iceberg.FileFormat;
 import org.apache.iceberg.MetadataTableType;
 import org.apache.iceberg.Partitioning;
 import org.apache.iceberg.RewriteFiles;
@@ -35,6 +36,7 @@ import org.apache.iceberg.actions.RemoveDanglingDeleteFiles;
 import org.apache.iceberg.spark.JobGroupInfo;
 import org.apache.iceberg.spark.SparkDeleteFile;
 import org.apache.iceberg.types.Types;
+import org.apache.iceberg.util.DeleteFileSet;
 import org.apache.spark.sql.Column;
 import org.apache.spark.sql.Dataset;
 import org.apache.spark.sql.Row;
@@ -59,6 +61,7 @@ import org.slf4j.LoggerFactory;
 class RemoveDanglingDeletesSparkAction
     extends BaseSnapshotUpdateSparkAction<RemoveDanglingDeletesSparkAction>
     implements RemoveDanglingDeleteFiles {
+
   private static final Logger LOG = LoggerFactory.getLogger(RemoveDanglingDeletesSparkAction.class);
   private final Table table;
 
@@ -79,6 +82,7 @@ class RemoveDanglingDeletesSparkAction
           .removedDeleteFiles(Collections.emptyList())
           .build();
     }
+
     String desc = String.format("Removing dangling delete files in %s", table.name());
     JobGroupInfo info = newJobGroupInfo("REMOVE-DELETES", desc);
     return withJobGroupInfo(info, this::doExecute);
@@ -86,14 +90,19 @@ class RemoveDanglingDeletesSparkAction
 
   Result doExecute() {
     RewriteFiles rewriteFiles = table.newRewrite();
-    List<DeleteFile> danglingDeletes = findDanglingDeletes();
+    DeleteFileSet danglingDeletes = DeleteFileSet.create();
+    danglingDeletes.addAll(findDanglingDeletes());
+    danglingDeletes.addAll(findDanglingDvs());
+
     for (DeleteFile deleteFile : danglingDeletes) {
       LOG.debug("Removing dangling delete file {}", deleteFile.location());
       rewriteFiles.deleteFile(deleteFile);
     }
+
     if (!danglingDeletes.isEmpty()) {
       commit(rewriteFiles);
     }
+
     return ImmutableRemoveDanglingDeleteFiles.Result.builder()
         .removedDeleteFiles(danglingDeletes)
         .build();
@@ -124,10 +133,12 @@ class RemoveDanglingDeletesSparkAction
             .groupBy("partition", "spec_id")
             .agg(min("sequence_number"))
             .toDF("grouped_partition", "grouped_spec_id", "min_data_sequence_number");
+
     Dataset<Row> deleteEntries =
         loadMetadataTable(table, MetadataTableType.ENTRIES)
             // find live delete files
             .filter("data_file.content != 0 AND status < 2");
+
     Column joinOnPartition =
         deleteEntries
             .col("data_file.spec_id")
@@ -136,9 +147,10 @@ class RemoveDanglingDeletesSparkAction
                 deleteEntries
                     .col("data_file.partition")
                     .equalTo(minSequenceNumberByPartition.col("grouped_partition")));
+
     Column filterOnDanglingDeletes =
         col("min_data_sequence_number")
-            // delete fies without any data files in partition
+            // delete files without any data files in partition
             .isNull()
             // position delete files without any applicable data files in partition
             .or(
@@ -150,6 +162,7 @@ class RemoveDanglingDeletesSparkAction
                 col("data_file.content")
                     .equalTo("2")
                     .and(col("sequence_number").$less$eq(col("min_data_sequence_number"))));
+
     Dataset<Row> danglingDeletes =
         deleteEntries
             .join(minSequenceNumberByPartition, joinOnPartition, "left")
@@ -161,6 +174,27 @@ class RemoveDanglingDeletesSparkAction
         .collect(Collectors.toList());
   }
 
+  private List<DeleteFile> findDanglingDvs() {
+    Dataset<Row> dvs =
+        loadMetadataTable(table, MetadataTableType.DELETE_FILES)
+            .where(col("file_format").equalTo(FileFormat.PUFFIN.name()));
+    Dataset<Row> dataFiles = loadMetadataTable(table, MetadataTableType.DATA_FILES);
+
+    // a DV not pointing to a valid data file path is implicitly a dangling delete
+    List<Row> danglingDvs =
+        dvs.join(
+                dataFiles,
+                dvs.col("referenced_data_file").equalTo(dataFiles.col("file_path")),
+                "leftouter")
+            .filter(dataFiles.col("file_path").isNull())
+            .select(dvs.col("*"))
+            .collectAsList();
+    return danglingDvs.stream()
+        // map on driver because SparkDeleteFile is not serializable
+        .map(row -> deleteFileWrapper(dvs.schema(), row))
+        .collect(Collectors.toList());
+  }
+
   private DeleteFile deleteFileWrapper(StructType sparkFileType, Row row) {
     int specId = row.getInt(row.fieldIndex("spec_id"));
     Types.StructType combinedFileType = DataFile.getType(Partitioning.partitionType(table));
diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewritePositionDeleteFilesAction.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewritePositionDeleteFilesAction.java
index 8afce14c14..fe21f510da 100644
--- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewritePositionDeleteFilesAction.java
+++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewritePositionDeleteFilesAction.java
@@ -55,8 +55,10 @@ import org.apache.iceberg.Schema;
 import org.apache.iceberg.StructLike;
 import org.apache.iceberg.Table;
 import org.apache.iceberg.TableProperties;
+import org.apache.iceberg.actions.RewriteDataFiles;
 import org.apache.iceberg.actions.RewritePositionDeleteFiles.FileGroupRewriteResult;
 import org.apache.iceberg.actions.RewritePositionDeleteFiles.Result;
+import org.apache.iceberg.actions.SizeBasedDataRewriter;
 import org.apache.iceberg.actions.SizeBasedFileRewriter;
 import org.apache.iceberg.catalog.TableIdentifier;
 import org.apache.iceberg.data.FileHelpers;
@@ -383,6 +385,95 @@ public class TestRewritePositionDeleteFilesAction extends CatalogTestBase {
     assertThat(actualDeletes).as("New position deletes").isEmpty();
   }
 
+  @TestTemplate
+  public void testRemoveDanglingDVsAfterCompaction() {
+    sql(
+        "create table %s (s string, id string) PARTITIONED BY (bucket(8, id)) "
+            + "tblproperties ('format-version'='3',"
+            + "'write.update.mode'='merge-on-read',"
+            + "'write.delete.mode'='merge-on-read',"
+            + "'write.merge.mode'='merge-on-read')",
+        tableName);
+    sql("insert into %s select * from (values ('foo', '1'), ('bar', '1')) order by 1", tableName);
+    sql("insert into %s select * from (values ('foo', '1'), ('bat', '1')) order by 1", tableName);
+    sql("insert into %s select * from (values ('bar', '1'), ('bat', '1')) order by 1", tableName);
+
+    List<Object[]> objects = sql("select * from %s.files", tableName);
+    assertThat(objects).hasSize(3);
+
+    sql("delete from %s where s = 'foo'", tableName);
+    assertThat(sql("select * from %s.files", tableName)).hasSize(5);
+
+    sql("delete from %s where s = 'bar'", tableName);
+    assertThat(sql("select * from %s.files", tableName)).hasSize(6);
+
+    assertThat(sql("select * from %s.data_files", tableName)).hasSize(3);
+    assertThat(sql("select * from %s.delete_files", tableName)).hasSize(3);
+
+    Set<DeleteFile> deleteFilesBefore =
+        TestHelpers.deleteFiles(validationCatalog.loadTable(tableIdent));
+    assertThat(deleteFilesBefore).hasSize(3);
+
+    RewriteDataFiles.Result result =
+        SparkActions.get(spark)
+            .rewriteDataFiles(validationCatalog.loadTable(tableIdent))
+            .option(SizeBasedFileRewriter.REWRITE_ALL, "true")
+            .option(RewriteDataFiles.REMOVE_DANGLING_DELETES, "true")
+            .execute();
+
+    Set<DeleteFile> deleteFilesAfter =
+        TestHelpers.deleteFiles(validationCatalog.loadTable(tableIdent));
+    assertThat(deleteFilesAfter).isEmpty();
+    assertThat(result.addedDataFilesCount()).isEqualTo(1);
+    assertThat(result.rewrittenDataFilesCount()).isEqualTo(3);
+    assertThat(sql("select * from %s.delete_files", tableName)).hasSameSizeAs(deleteFilesAfter);
+    assertThat(result.removedDeleteFilesCount()).isEqualTo(deleteFilesBefore.size());
+  }
+
+  @TestTemplate
+  public void testValidDVsAreNotRemovedDuringDanglingDeletesRemoval() {
+    sql(
+        "create table %s (s string, id string) PARTITIONED BY (bucket(8, id)) "
+            + "tblproperties ('format-version'='3',"
+            + "'write.update.mode'='merge-on-read',"
+            + "'write.delete.mode'='merge-on-read',"
+            + "'write.merge.mode'='merge-on-read')",
+        tableName);
+    sql("insert into %s select * from (values ('foo', '1'), ('bar', '1')) order by 1", tableName);
+    sql("insert into %s select * from (values ('foo', '1'), ('bat', '1')) order by 1", tableName);
+    sql("insert into %s select * from (values ('bar', '1'), ('bat', '1')) order by 1", tableName);
+
+    List<Object[]> objects = sql("select * from %s.files", tableName);
+    assertThat(objects).hasSize(3);
+
+    sql("delete from %s where s = 'foo'", tableName);
+    assertThat(sql("select * from %s.files", tableName)).hasSize(5);
+
+    assertThat(sql("select * from %s.data_files", tableName)).hasSize(3);
+    assertThat(sql("select * from %s.delete_files", tableName)).hasSize(2);
+
+    Set<DeleteFile> deleteFilesBefore =
+        TestHelpers.deleteFiles(validationCatalog.loadTable(tableIdent));
+    assertThat(deleteFilesBefore).hasSize(2);
+
+    // data files are not compacted and removing dangling deletes should not remove valid DVs
+    RewriteDataFiles.Result result =
+        SparkActions.get(spark)
+            .rewriteDataFiles(validationCatalog.loadTable(tableIdent))
+            .option(RewriteDataFiles.REMOVE_DANGLING_DELETES, "true")
+            .option(SizeBasedFileRewriter.MIN_FILE_SIZE_BYTES, "0")
+            .option(SizeBasedDataRewriter.DELETE_RATIO_THRESHOLD, "1.0")
+            .execute();
+
+    Set<DeleteFile> deleteFilesAfter =
+        TestHelpers.deleteFiles(validationCatalog.loadTable(tableIdent));
+    assertThat(deleteFilesAfter).isEqualTo(deleteFilesBefore);
+    assertThat(result.addedDataFilesCount()).isEqualTo(0);
+    assertThat(result.rewrittenDataFilesCount()).isEqualTo(0);
+    assertThat(sql("select * from %s.delete_files", tableName)).hasSameSizeAs(deleteFilesAfter);
+    assertThat(result.removedDeleteFilesCount()).isEqualTo(0);
+  }
+
   @TestTemplate
   public void testSomePartitionsDanglingDeletes() throws Exception {
     Table table = createTablePartitioned(4, 2, SCALE);
diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPositionDeletesTable.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPositionDeletesTable.java
index 87cbbe3cea..c30a730917 100644
--- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPositionDeletesTable.java
+++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPositionDeletesTable.java
@@ -1203,13 +1203,10 @@ public class TestPositionDeletesTable extends CatalogTestBase {
                     .option(SparkWriteOptions.REWRITTEN_FILE_SCAN_TASK_SET_ID, fileSetID)
                     .append())
         .isInstanceOf(AnalysisException.class)
-        .hasMessage(
-            "[INCOMPATIBLE_DATA_FOR_TABLE.CANNOT_FIND_DATA] Cannot write incompatible data for the table `"
-                + catalogName
-                + "`.`default`.`"
-                + tableName
-                + "`.`position_deletes`"
-                + ": Cannot find data for the output column `partition`.");
+        .hasMessageContaining(
+            "Cannot write incompatible data to table '%s.default.%s.position_deletes'",
+            catalogName, tableName)
+        .hasMessageContaining("Cannot find data for output column 'partition'.");
 
     dropTable(tableName);
   }