You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by am...@apache.org on 2025/03/22 16:53:41 UTC
(iceberg) 02/02: Spark 3.4: Detect dangling DVs properly
This is an automated email from the ASF dual-hosted git repository.
amoghj pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg.git
commit c2a7d9f0ad1406a74d972ed1dc2b7bee1606944a
Author: Eduard Tudenhoefner <et...@gmail.com>
AuthorDate: Fri Mar 21 18:49:19 2025 +0100
Spark 3.4: Detect dangling DVs properly
this backports #12270 to Spark 3.4
---
.../actions/RemoveDanglingDeletesSparkAction.java | 38 ++++++++-
.../TestRewritePositionDeleteFilesAction.java | 91 ++++++++++++++++++++++
.../spark/source/TestPositionDeletesTable.java | 11 +--
3 files changed, 131 insertions(+), 9 deletions(-)
diff --git a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/actions/RemoveDanglingDeletesSparkAction.java b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/actions/RemoveDanglingDeletesSparkAction.java
index c8c883e553..af8114fc23 100644
--- a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/actions/RemoveDanglingDeletesSparkAction.java
+++ b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/actions/RemoveDanglingDeletesSparkAction.java
@@ -26,6 +26,7 @@ import java.util.List;
import java.util.stream.Collectors;
import org.apache.iceberg.DataFile;
import org.apache.iceberg.DeleteFile;
+import org.apache.iceberg.FileFormat;
import org.apache.iceberg.MetadataTableType;
import org.apache.iceberg.Partitioning;
import org.apache.iceberg.RewriteFiles;
@@ -35,6 +36,7 @@ import org.apache.iceberg.actions.RemoveDanglingDeleteFiles;
import org.apache.iceberg.spark.JobGroupInfo;
import org.apache.iceberg.spark.SparkDeleteFile;
import org.apache.iceberg.types.Types;
+import org.apache.iceberg.util.DeleteFileSet;
import org.apache.spark.sql.Column;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
@@ -59,6 +61,7 @@ import org.slf4j.LoggerFactory;
class RemoveDanglingDeletesSparkAction
extends BaseSnapshotUpdateSparkAction<RemoveDanglingDeletesSparkAction>
implements RemoveDanglingDeleteFiles {
+
private static final Logger LOG = LoggerFactory.getLogger(RemoveDanglingDeletesSparkAction.class);
private final Table table;
@@ -79,6 +82,7 @@ class RemoveDanglingDeletesSparkAction
.removedDeleteFiles(Collections.emptyList())
.build();
}
+
String desc = String.format("Removing dangling delete files in %s", table.name());
JobGroupInfo info = newJobGroupInfo("REMOVE-DELETES", desc);
return withJobGroupInfo(info, this::doExecute);
@@ -86,14 +90,19 @@ class RemoveDanglingDeletesSparkAction
Result doExecute() {
RewriteFiles rewriteFiles = table.newRewrite();
- List<DeleteFile> danglingDeletes = findDanglingDeletes();
+ DeleteFileSet danglingDeletes = DeleteFileSet.create();
+ danglingDeletes.addAll(findDanglingDeletes());
+ danglingDeletes.addAll(findDanglingDvs());
+
for (DeleteFile deleteFile : danglingDeletes) {
LOG.debug("Removing dangling delete file {}", deleteFile.location());
rewriteFiles.deleteFile(deleteFile);
}
+
if (!danglingDeletes.isEmpty()) {
commit(rewriteFiles);
}
+
return ImmutableRemoveDanglingDeleteFiles.Result.builder()
.removedDeleteFiles(danglingDeletes)
.build();
@@ -124,10 +133,12 @@ class RemoveDanglingDeletesSparkAction
.groupBy("partition", "spec_id")
.agg(min("sequence_number"))
.toDF("grouped_partition", "grouped_spec_id", "min_data_sequence_number");
+
Dataset<Row> deleteEntries =
loadMetadataTable(table, MetadataTableType.ENTRIES)
// find live delete files
.filter("data_file.content != 0 AND status < 2");
+
Column joinOnPartition =
deleteEntries
.col("data_file.spec_id")
@@ -136,9 +147,10 @@ class RemoveDanglingDeletesSparkAction
deleteEntries
.col("data_file.partition")
.equalTo(minSequenceNumberByPartition.col("grouped_partition")));
+
Column filterOnDanglingDeletes =
col("min_data_sequence_number")
- // delete fies without any data files in partition
+ // delete files without any data files in partition
.isNull()
// position delete files without any applicable data files in partition
.or(
@@ -150,6 +162,7 @@ class RemoveDanglingDeletesSparkAction
col("data_file.content")
.equalTo("2")
.and(col("sequence_number").$less$eq(col("min_data_sequence_number"))));
+
Dataset<Row> danglingDeletes =
deleteEntries
.join(minSequenceNumberByPartition, joinOnPartition, "left")
@@ -161,6 +174,27 @@ class RemoveDanglingDeletesSparkAction
.collect(Collectors.toList());
}
+ private List<DeleteFile> findDanglingDvs() {
+ Dataset<Row> dvs =
+ loadMetadataTable(table, MetadataTableType.DELETE_FILES)
+ .where(col("file_format").equalTo(FileFormat.PUFFIN.name()));
+ Dataset<Row> dataFiles = loadMetadataTable(table, MetadataTableType.DATA_FILES);
+
+ // a DV not pointing to a valid data file path is implicitly a dangling delete
+ List<Row> danglingDvs =
+ dvs.join(
+ dataFiles,
+ dvs.col("referenced_data_file").equalTo(dataFiles.col("file_path")),
+ "leftouter")
+ .filter(dataFiles.col("file_path").isNull())
+ .select(dvs.col("*"))
+ .collectAsList();
+ return danglingDvs.stream()
+ // map on driver because SparkDeleteFile is not serializable
+ .map(row -> deleteFileWrapper(dvs.schema(), row))
+ .collect(Collectors.toList());
+ }
+
private DeleteFile deleteFileWrapper(StructType sparkFileType, Row row) {
int specId = row.getInt(row.fieldIndex("spec_id"));
Types.StructType combinedFileType = DataFile.getType(Partitioning.partitionType(table));
diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewritePositionDeleteFilesAction.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewritePositionDeleteFilesAction.java
index 8afce14c14..fe21f510da 100644
--- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewritePositionDeleteFilesAction.java
+++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewritePositionDeleteFilesAction.java
@@ -55,8 +55,10 @@ import org.apache.iceberg.Schema;
import org.apache.iceberg.StructLike;
import org.apache.iceberg.Table;
import org.apache.iceberg.TableProperties;
+import org.apache.iceberg.actions.RewriteDataFiles;
import org.apache.iceberg.actions.RewritePositionDeleteFiles.FileGroupRewriteResult;
import org.apache.iceberg.actions.RewritePositionDeleteFiles.Result;
+import org.apache.iceberg.actions.SizeBasedDataRewriter;
import org.apache.iceberg.actions.SizeBasedFileRewriter;
import org.apache.iceberg.catalog.TableIdentifier;
import org.apache.iceberg.data.FileHelpers;
@@ -383,6 +385,95 @@ public class TestRewritePositionDeleteFilesAction extends CatalogTestBase {
assertThat(actualDeletes).as("New position deletes").isEmpty();
}
+ @TestTemplate
+ public void testRemoveDanglingDVsAfterCompaction() {
+ sql(
+ "create table %s (s string, id string) PARTITIONED BY (bucket(8, id)) "
+ + "tblproperties ('format-version'='3',"
+ + "'write.update.mode'='merge-on-read',"
+ + "'write.delete.mode'='merge-on-read',"
+ + "'write.merge.mode'='merge-on-read')",
+ tableName);
+ sql("insert into %s select * from (values ('foo', '1'), ('bar', '1')) order by 1", tableName);
+ sql("insert into %s select * from (values ('foo', '1'), ('bat', '1')) order by 1", tableName);
+ sql("insert into %s select * from (values ('bar', '1'), ('bat', '1')) order by 1", tableName);
+
+ List<Object[]> objects = sql("select * from %s.files", tableName);
+ assertThat(objects).hasSize(3);
+
+ sql("delete from %s where s = 'foo'", tableName);
+ assertThat(sql("select * from %s.files", tableName)).hasSize(5);
+
+ sql("delete from %s where s = 'bar'", tableName);
+ assertThat(sql("select * from %s.files", tableName)).hasSize(6);
+
+ assertThat(sql("select * from %s.data_files", tableName)).hasSize(3);
+ assertThat(sql("select * from %s.delete_files", tableName)).hasSize(3);
+
+ Set<DeleteFile> deleteFilesBefore =
+ TestHelpers.deleteFiles(validationCatalog.loadTable(tableIdent));
+ assertThat(deleteFilesBefore).hasSize(3);
+
+ RewriteDataFiles.Result result =
+ SparkActions.get(spark)
+ .rewriteDataFiles(validationCatalog.loadTable(tableIdent))
+ .option(SizeBasedFileRewriter.REWRITE_ALL, "true")
+ .option(RewriteDataFiles.REMOVE_DANGLING_DELETES, "true")
+ .execute();
+
+ Set<DeleteFile> deleteFilesAfter =
+ TestHelpers.deleteFiles(validationCatalog.loadTable(tableIdent));
+ assertThat(deleteFilesAfter).isEmpty();
+ assertThat(result.addedDataFilesCount()).isEqualTo(1);
+ assertThat(result.rewrittenDataFilesCount()).isEqualTo(3);
+ assertThat(sql("select * from %s.delete_files", tableName)).hasSameSizeAs(deleteFilesAfter);
+ assertThat(result.removedDeleteFilesCount()).isEqualTo(deleteFilesBefore.size());
+ }
+
+ @TestTemplate
+ public void testValidDVsAreNotRemovedDuringDanglingDeletesRemoval() {
+ sql(
+ "create table %s (s string, id string) PARTITIONED BY (bucket(8, id)) "
+ + "tblproperties ('format-version'='3',"
+ + "'write.update.mode'='merge-on-read',"
+ + "'write.delete.mode'='merge-on-read',"
+ + "'write.merge.mode'='merge-on-read')",
+ tableName);
+ sql("insert into %s select * from (values ('foo', '1'), ('bar', '1')) order by 1", tableName);
+ sql("insert into %s select * from (values ('foo', '1'), ('bat', '1')) order by 1", tableName);
+ sql("insert into %s select * from (values ('bar', '1'), ('bat', '1')) order by 1", tableName);
+
+ List<Object[]> objects = sql("select * from %s.files", tableName);
+ assertThat(objects).hasSize(3);
+
+ sql("delete from %s where s = 'foo'", tableName);
+ assertThat(sql("select * from %s.files", tableName)).hasSize(5);
+
+ assertThat(sql("select * from %s.data_files", tableName)).hasSize(3);
+ assertThat(sql("select * from %s.delete_files", tableName)).hasSize(2);
+
+ Set<DeleteFile> deleteFilesBefore =
+ TestHelpers.deleteFiles(validationCatalog.loadTable(tableIdent));
+ assertThat(deleteFilesBefore).hasSize(2);
+
+ // data files are not compacted and removing dangling deletes should not remove valid DVs
+ RewriteDataFiles.Result result =
+ SparkActions.get(spark)
+ .rewriteDataFiles(validationCatalog.loadTable(tableIdent))
+ .option(RewriteDataFiles.REMOVE_DANGLING_DELETES, "true")
+ .option(SizeBasedFileRewriter.MIN_FILE_SIZE_BYTES, "0")
+ .option(SizeBasedDataRewriter.DELETE_RATIO_THRESHOLD, "1.0")
+ .execute();
+
+ Set<DeleteFile> deleteFilesAfter =
+ TestHelpers.deleteFiles(validationCatalog.loadTable(tableIdent));
+ assertThat(deleteFilesAfter).isEqualTo(deleteFilesBefore);
+ assertThat(result.addedDataFilesCount()).isEqualTo(0);
+ assertThat(result.rewrittenDataFilesCount()).isEqualTo(0);
+ assertThat(sql("select * from %s.delete_files", tableName)).hasSameSizeAs(deleteFilesAfter);
+ assertThat(result.removedDeleteFilesCount()).isEqualTo(0);
+ }
+
@TestTemplate
public void testSomePartitionsDanglingDeletes() throws Exception {
Table table = createTablePartitioned(4, 2, SCALE);
diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPositionDeletesTable.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPositionDeletesTable.java
index 87cbbe3cea..c30a730917 100644
--- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPositionDeletesTable.java
+++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPositionDeletesTable.java
@@ -1203,13 +1203,10 @@ public class TestPositionDeletesTable extends CatalogTestBase {
.option(SparkWriteOptions.REWRITTEN_FILE_SCAN_TASK_SET_ID, fileSetID)
.append())
.isInstanceOf(AnalysisException.class)
- .hasMessage(
- "[INCOMPATIBLE_DATA_FOR_TABLE.CANNOT_FIND_DATA] Cannot write incompatible data for the table `"
- + catalogName
- + "`.`default`.`"
- + tableName
- + "`.`position_deletes`"
- + ": Cannot find data for the output column `partition`.");
+ .hasMessageContaining(
+ "Cannot write incompatible data to table '%s.default.%s.position_deletes'",
+ catalogName, tableName)
+ .hasMessageContaining("Cannot find data for output column 'partition'.");
dropTable(tableName);
}