You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by am...@apache.org on 2026/06/10 18:27:18 UTC
(iceberg) branch 1.10.x updated: Spark: Backport #16699 rewrite manifest row lineage fix to 1.10.x (#16760)
This is an automated email from the ASF dual-hosted git repository.
amogh-jahagirdar pushed a commit to branch 1.10.x
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/1.10.x by this push:
new a609798eb4 Spark: Backport #16699 rewrite manifest row lineage fix to 1.10.x (#16760)
a609798eb4 is described below
commit a609798eb4c2c69233b02c76206877355eae017a
Author: Amogh Jahagirdar <am...@apache.org>
AuthorDate: Wed Jun 10 12:27:11 2026 -0600
Spark: Backport #16699 rewrite manifest row lineage fix to 1.10.x (#16760)
---
.../org/apache/iceberg/spark/SparkContentFile.java | 11 ++
.../spark/actions/TestRewriteManifestsAction.java | 125 +++++++++++++++++++++
.../org/apache/iceberg/spark/SparkContentFile.java | 11 ++
.../spark/actions/TestRewriteManifestsAction.java | 125 +++++++++++++++++++++
.../org/apache/iceberg/spark/SparkContentFile.java | 11 ++
.../spark/actions/TestRewriteManifestsAction.java | 125 +++++++++++++++++++++
6 files changed, 408 insertions(+)
diff --git a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/SparkContentFile.java b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/SparkContentFile.java
index bad31d8d85..616be45d67 100644
--- a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/SparkContentFile.java
+++ b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/SparkContentFile.java
@@ -54,6 +54,7 @@ public abstract class SparkContentFile<F> implements ContentFile<F> {
private final int sortOrderIdPosition;
private final int fileSpecIdPosition;
private final int equalityIdsPosition;
+ private final int firstRowIdPosition;
private final int referencedDataFilePosition;
private final int contentOffsetPosition;
private final int contentSizePosition;
@@ -106,6 +107,7 @@ public abstract class SparkContentFile<F> implements ContentFile<F> {
this.sortOrderIdPosition = positions.get(DataFile.SORT_ORDER_ID.name());
this.fileSpecIdPosition = positions.get(DataFile.SPEC_ID.name());
this.equalityIdsPosition = positions.get(DataFile.EQUALITY_IDS.name());
+ this.firstRowIdPosition = positions.get(DataFile.FIRST_ROW_ID.name());
this.referencedDataFilePosition = positions.get(DataFile.REFERENCED_DATA_FILE.name());
this.contentOffsetPosition = positions.get(DataFile.CONTENT_OFFSET.name());
this.contentSizePosition = positions.get(DataFile.CONTENT_SIZE.name());
@@ -258,6 +260,15 @@ public abstract class SparkContentFile<F> implements ContentFile<F> {
return wrapped.getLong(contentSizePosition);
}
+ @Override
+ public Long firstRowId() {
+ if (wrapped.isNullAt(firstRowIdPosition)) {
+ return null;
+ }
+
+ return wrapped.getLong(firstRowIdPosition);
+ }
+
private int fieldPosition(String name, StructType sparkType) {
try {
return sparkType.fieldIndex(name);
diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteManifestsAction.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteManifestsAction.java
index 4f95416878..f66a914563 100644
--- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteManifestsAction.java
+++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteManifestsAction.java
@@ -51,6 +51,7 @@ import org.apache.iceberg.ManifestContent;
import org.apache.iceberg.ManifestFile;
import org.apache.iceberg.ManifestFiles;
import org.apache.iceberg.ManifestWriter;
+import org.apache.iceberg.MetadataColumns;
import org.apache.iceberg.Parameter;
import org.apache.iceberg.ParameterizedTestExtension;
import org.apache.iceberg.Parameters;
@@ -195,6 +196,119 @@ public class TestRewriteManifestsAction extends TestBase {
}
}
+ @TestTemplate
+ public void testRewriteV3ManifestsPreservesFirstRowId() {
+ assumeThat(formatVersion).isGreaterThanOrEqualTo(3);
+
+ PartitionSpec spec = PartitionSpec.unpartitioned();
+ Table table =
+ TABLES.create(
+ SCHEMA,
+ spec,
+ ImmutableMap.of(TableProperties.FORMAT_VERSION, String.valueOf(formatVersion)),
+ tableLocation);
+
+ writeRecords(Lists.newArrayList(new ThreeColumnRecord(1, null, "AAAA")));
+ writeRecords(Lists.newArrayList(new ThreeColumnRecord(2, "CCCC", "CCCC")));
+ table.refresh();
+
+ assertThat(table.currentSnapshot().dataManifests(table.io())).hasSize(2);
+
+ List<Row> rowsBefore = recordsWithLineage();
+ assertThat(rowsBefore)
+ .extracting(r -> r.<Long>getAs(MetadataColumns.ROW_ID.name()))
+ .doesNotContainNull();
+
+ SparkActions.get()
+ .rewriteManifests(table)
+ .rewriteIf(manifest -> true)
+ .option(RewriteManifestsSparkAction.USE_CACHING, useCaching)
+ .execute();
+
+ assertThat(table.currentSnapshot().dataManifests(table.io())).hasSize(1);
+
+ List<Row> rowsAfter = recordsWithLineage();
+
+ assertThat(rowsAfter).containsExactlyElementsOf(rowsBefore);
+ }
+
+ @TestTemplate
+ public void testRewriteV3PartitionedManifestsPreservesFirstRowId() {
+ assumeThat(formatVersion).isGreaterThanOrEqualTo(3);
+
+ PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).identity("c1").build();
+ Table table =
+ TABLES.create(
+ SCHEMA,
+ spec,
+ ImmutableMap.of(TableProperties.FORMAT_VERSION, String.valueOf(formatVersion)),
+ tableLocation);
+
+ writeRecords(Lists.newArrayList(new ThreeColumnRecord(1, "AAAA", "AAAA")));
+ writeRecords(Lists.newArrayList(new ThreeColumnRecord(2, "BBBB", "BBBB")));
+ table.refresh();
+
+ assertThat(table.currentSnapshot().dataManifests(table.io())).hasSize(2);
+
+ List<Row> rowsBefore = recordsWithLineage();
+ assertThat(rowsBefore)
+ .extracting(r -> r.<Long>getAs(MetadataColumns.ROW_ID.name()))
+ .doesNotContainNull();
+
+ SparkActions.get()
+ .rewriteManifests(table)
+ .rewriteIf(manifest -> true)
+ .option(RewriteManifestsSparkAction.USE_CACHING, useCaching)
+ .execute();
+
+ assertThat(table.currentSnapshot().dataManifests(table.io())).hasSize(1);
+
+ List<Row> rowsAfter = recordsWithLineage();
+
+ assertThat(rowsAfter).containsExactlyElementsOf(rowsBefore);
+ }
+
+ @TestTemplate
+ public void testRewriteManifestsAfterV2ToV3Upgrade() {
+ assumeThat(formatVersion).isGreaterThanOrEqualTo(3);
+
+ PartitionSpec spec = PartitionSpec.unpartitioned();
+ Table table =
+ TABLES.create(
+ SCHEMA, spec, ImmutableMap.of(TableProperties.FORMAT_VERSION, "2"), tableLocation);
+
+ ThreeColumnRecord record1 = new ThreeColumnRecord(1, null, "AAAA");
+ ThreeColumnRecord record2 = new ThreeColumnRecord(2, "CCCC", "CCCC");
+ writeRecords(Lists.newArrayList(record1));
+ writeRecords(Lists.newArrayList(record2));
+ table.refresh();
+
+ assertThat(table.currentSnapshot().dataManifests(table.io())).hasSize(2);
+
+ table
+ .updateProperties()
+ .set(TableProperties.FORMAT_VERSION, String.valueOf(formatVersion))
+ .commit();
+
+ SparkActions.get()
+ .rewriteManifests(table)
+ .rewriteIf(manifest -> true)
+ .option(RewriteManifestsSparkAction.USE_CACHING, useCaching)
+ .execute();
+
+ assertThat(table.currentSnapshot().dataManifests(table.io())).hasSize(1);
+
+ List<Row> rowsAfter = recordsWithLineage();
+
+ assertThat(rowsAfter)
+ .extracting(r -> r.<Long>getAs(MetadataColumns.ROW_ID.name()))
+ .doesNotContainNull()
+ .doesNotHaveDuplicates();
+ assertThat(rowsAfter)
+ .extracting(r -> new ThreeColumnRecord(r.getAs("c1"), r.getAs("c2"), r.getAs("c3")))
+ .containsExactlyInAnyOrder(record1, record2);
+ }
+
@TestTemplate
public void testRewriteManifestsEmptyTable() throws IOException {
PartitionSpec spec = PartitionSpec.unpartitioned();
@@ -1134,6 +1248,17 @@ public class TestRewriteManifestsAction extends TestBase {
.collectAsList();
}
+ private List<Row> recordsWithLineage() {
+ return spark
+ .read()
+ .format("iceberg")
+ .load(tableLocation)
+ .selectExpr(
+ MetadataColumns.ROW_ID.name(), MetadataColumns.LAST_UPDATED_SEQUENCE_NUMBER.name(), "*")
+ .orderBy(MetadataColumns.ROW_ID.name())
+ .collectAsList();
+ }
+
private void writeRecords(List<ThreeColumnRecord> records) {
Dataset<Row> df = spark.createDataFrame(records, ThreeColumnRecord.class);
writeDF(df);
diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkContentFile.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkContentFile.java
index bad31d8d85..616be45d67 100644
--- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkContentFile.java
+++ b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkContentFile.java
@@ -54,6 +54,7 @@ public abstract class SparkContentFile<F> implements ContentFile<F> {
private final int sortOrderIdPosition;
private final int fileSpecIdPosition;
private final int equalityIdsPosition;
+ private final int firstRowIdPosition;
private final int referencedDataFilePosition;
private final int contentOffsetPosition;
private final int contentSizePosition;
@@ -106,6 +107,7 @@ public abstract class SparkContentFile<F> implements ContentFile<F> {
this.sortOrderIdPosition = positions.get(DataFile.SORT_ORDER_ID.name());
this.fileSpecIdPosition = positions.get(DataFile.SPEC_ID.name());
this.equalityIdsPosition = positions.get(DataFile.EQUALITY_IDS.name());
+ this.firstRowIdPosition = positions.get(DataFile.FIRST_ROW_ID.name());
this.referencedDataFilePosition = positions.get(DataFile.REFERENCED_DATA_FILE.name());
this.contentOffsetPosition = positions.get(DataFile.CONTENT_OFFSET.name());
this.contentSizePosition = positions.get(DataFile.CONTENT_SIZE.name());
@@ -258,6 +260,15 @@ public abstract class SparkContentFile<F> implements ContentFile<F> {
return wrapped.getLong(contentSizePosition);
}
+ @Override
+ public Long firstRowId() {
+ if (wrapped.isNullAt(firstRowIdPosition)) {
+ return null;
+ }
+
+ return wrapped.getLong(firstRowIdPosition);
+ }
+
private int fieldPosition(String name, StructType sparkType) {
try {
return sparkType.fieldIndex(name);
diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteManifestsAction.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteManifestsAction.java
index 4f95416878..f66a914563 100644
--- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteManifestsAction.java
+++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteManifestsAction.java
@@ -51,6 +51,7 @@ import org.apache.iceberg.ManifestContent;
import org.apache.iceberg.ManifestFile;
import org.apache.iceberg.ManifestFiles;
import org.apache.iceberg.ManifestWriter;
+import org.apache.iceberg.MetadataColumns;
import org.apache.iceberg.Parameter;
import org.apache.iceberg.ParameterizedTestExtension;
import org.apache.iceberg.Parameters;
@@ -195,6 +196,119 @@ public class TestRewriteManifestsAction extends TestBase {
}
}
+ @TestTemplate
+ public void testRewriteV3ManifestsPreservesFirstRowId() {
+ assumeThat(formatVersion).isGreaterThanOrEqualTo(3);
+
+ PartitionSpec spec = PartitionSpec.unpartitioned();
+ Table table =
+ TABLES.create(
+ SCHEMA,
+ spec,
+ ImmutableMap.of(TableProperties.FORMAT_VERSION, String.valueOf(formatVersion)),
+ tableLocation);
+
+ writeRecords(Lists.newArrayList(new ThreeColumnRecord(1, null, "AAAA")));
+ writeRecords(Lists.newArrayList(new ThreeColumnRecord(2, "CCCC", "CCCC")));
+ table.refresh();
+
+ assertThat(table.currentSnapshot().dataManifests(table.io())).hasSize(2);
+
+ List<Row> rowsBefore = recordsWithLineage();
+ assertThat(rowsBefore)
+ .extracting(r -> r.<Long>getAs(MetadataColumns.ROW_ID.name()))
+ .doesNotContainNull();
+
+ SparkActions.get()
+ .rewriteManifests(table)
+ .rewriteIf(manifest -> true)
+ .option(RewriteManifestsSparkAction.USE_CACHING, useCaching)
+ .execute();
+
+ assertThat(table.currentSnapshot().dataManifests(table.io())).hasSize(1);
+
+ List<Row> rowsAfter = recordsWithLineage();
+
+ assertThat(rowsAfter).containsExactlyElementsOf(rowsBefore);
+ }
+
+ @TestTemplate
+ public void testRewriteV3PartitionedManifestsPreservesFirstRowId() {
+ assumeThat(formatVersion).isGreaterThanOrEqualTo(3);
+
+ PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).identity("c1").build();
+ Table table =
+ TABLES.create(
+ SCHEMA,
+ spec,
+ ImmutableMap.of(TableProperties.FORMAT_VERSION, String.valueOf(formatVersion)),
+ tableLocation);
+
+ writeRecords(Lists.newArrayList(new ThreeColumnRecord(1, "AAAA", "AAAA")));
+ writeRecords(Lists.newArrayList(new ThreeColumnRecord(2, "BBBB", "BBBB")));
+ table.refresh();
+
+ assertThat(table.currentSnapshot().dataManifests(table.io())).hasSize(2);
+
+ List<Row> rowsBefore = recordsWithLineage();
+ assertThat(rowsBefore)
+ .extracting(r -> r.<Long>getAs(MetadataColumns.ROW_ID.name()))
+ .doesNotContainNull();
+
+ SparkActions.get()
+ .rewriteManifests(table)
+ .rewriteIf(manifest -> true)
+ .option(RewriteManifestsSparkAction.USE_CACHING, useCaching)
+ .execute();
+
+ assertThat(table.currentSnapshot().dataManifests(table.io())).hasSize(1);
+
+ List<Row> rowsAfter = recordsWithLineage();
+
+ assertThat(rowsAfter).containsExactlyElementsOf(rowsBefore);
+ }
+
+ @TestTemplate
+ public void testRewriteManifestsAfterV2ToV3Upgrade() {
+ assumeThat(formatVersion).isGreaterThanOrEqualTo(3);
+
+ PartitionSpec spec = PartitionSpec.unpartitioned();
+ Table table =
+ TABLES.create(
+ SCHEMA, spec, ImmutableMap.of(TableProperties.FORMAT_VERSION, "2"), tableLocation);
+
+ ThreeColumnRecord record1 = new ThreeColumnRecord(1, null, "AAAA");
+ ThreeColumnRecord record2 = new ThreeColumnRecord(2, "CCCC", "CCCC");
+ writeRecords(Lists.newArrayList(record1));
+ writeRecords(Lists.newArrayList(record2));
+ table.refresh();
+
+ assertThat(table.currentSnapshot().dataManifests(table.io())).hasSize(2);
+
+ table
+ .updateProperties()
+ .set(TableProperties.FORMAT_VERSION, String.valueOf(formatVersion))
+ .commit();
+
+ SparkActions.get()
+ .rewriteManifests(table)
+ .rewriteIf(manifest -> true)
+ .option(RewriteManifestsSparkAction.USE_CACHING, useCaching)
+ .execute();
+
+ assertThat(table.currentSnapshot().dataManifests(table.io())).hasSize(1);
+
+ List<Row> rowsAfter = recordsWithLineage();
+
+ assertThat(rowsAfter)
+ .extracting(r -> r.<Long>getAs(MetadataColumns.ROW_ID.name()))
+ .doesNotContainNull()
+ .doesNotHaveDuplicates();
+ assertThat(rowsAfter)
+ .extracting(r -> new ThreeColumnRecord(r.getAs("c1"), r.getAs("c2"), r.getAs("c3")))
+ .containsExactlyInAnyOrder(record1, record2);
+ }
+
@TestTemplate
public void testRewriteManifestsEmptyTable() throws IOException {
PartitionSpec spec = PartitionSpec.unpartitioned();
@@ -1134,6 +1248,17 @@ public class TestRewriteManifestsAction extends TestBase {
.collectAsList();
}
+ private List<Row> recordsWithLineage() {
+ return spark
+ .read()
+ .format("iceberg")
+ .load(tableLocation)
+ .selectExpr(
+ MetadataColumns.ROW_ID.name(), MetadataColumns.LAST_UPDATED_SEQUENCE_NUMBER.name(), "*")
+ .orderBy(MetadataColumns.ROW_ID.name())
+ .collectAsList();
+ }
+
private void writeRecords(List<ThreeColumnRecord> records) {
Dataset<Row> df = spark.createDataFrame(records, ThreeColumnRecord.class);
writeDF(df);
diff --git a/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/SparkContentFile.java b/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/SparkContentFile.java
index bad31d8d85..616be45d67 100644
--- a/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/SparkContentFile.java
+++ b/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/SparkContentFile.java
@@ -54,6 +54,7 @@ public abstract class SparkContentFile<F> implements ContentFile<F> {
private final int sortOrderIdPosition;
private final int fileSpecIdPosition;
private final int equalityIdsPosition;
+ private final int firstRowIdPosition;
private final int referencedDataFilePosition;
private final int contentOffsetPosition;
private final int contentSizePosition;
@@ -106,6 +107,7 @@ public abstract class SparkContentFile<F> implements ContentFile<F> {
this.sortOrderIdPosition = positions.get(DataFile.SORT_ORDER_ID.name());
this.fileSpecIdPosition = positions.get(DataFile.SPEC_ID.name());
this.equalityIdsPosition = positions.get(DataFile.EQUALITY_IDS.name());
+ this.firstRowIdPosition = positions.get(DataFile.FIRST_ROW_ID.name());
this.referencedDataFilePosition = positions.get(DataFile.REFERENCED_DATA_FILE.name());
this.contentOffsetPosition = positions.get(DataFile.CONTENT_OFFSET.name());
this.contentSizePosition = positions.get(DataFile.CONTENT_SIZE.name());
@@ -258,6 +260,15 @@ public abstract class SparkContentFile<F> implements ContentFile<F> {
return wrapped.getLong(contentSizePosition);
}
+ @Override
+ public Long firstRowId() {
+ if (wrapped.isNullAt(firstRowIdPosition)) {
+ return null;
+ }
+
+ return wrapped.getLong(firstRowIdPosition);
+ }
+
private int fieldPosition(String name, StructType sparkType) {
try {
return sparkType.fieldIndex(name);
diff --git a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteManifestsAction.java b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteManifestsAction.java
index 4f95416878..f66a914563 100644
--- a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteManifestsAction.java
+++ b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteManifestsAction.java
@@ -51,6 +51,7 @@ import org.apache.iceberg.ManifestContent;
import org.apache.iceberg.ManifestFile;
import org.apache.iceberg.ManifestFiles;
import org.apache.iceberg.ManifestWriter;
+import org.apache.iceberg.MetadataColumns;
import org.apache.iceberg.Parameter;
import org.apache.iceberg.ParameterizedTestExtension;
import org.apache.iceberg.Parameters;
@@ -195,6 +196,119 @@ public class TestRewriteManifestsAction extends TestBase {
}
}
+ @TestTemplate
+ public void testRewriteV3ManifestsPreservesFirstRowId() {
+ assumeThat(formatVersion).isGreaterThanOrEqualTo(3);
+
+ PartitionSpec spec = PartitionSpec.unpartitioned();
+ Table table =
+ TABLES.create(
+ SCHEMA,
+ spec,
+ ImmutableMap.of(TableProperties.FORMAT_VERSION, String.valueOf(formatVersion)),
+ tableLocation);
+
+ writeRecords(Lists.newArrayList(new ThreeColumnRecord(1, null, "AAAA")));
+ writeRecords(Lists.newArrayList(new ThreeColumnRecord(2, "CCCC", "CCCC")));
+ table.refresh();
+
+ assertThat(table.currentSnapshot().dataManifests(table.io())).hasSize(2);
+
+ List<Row> rowsBefore = recordsWithLineage();
+ assertThat(rowsBefore)
+ .extracting(r -> r.<Long>getAs(MetadataColumns.ROW_ID.name()))
+ .doesNotContainNull();
+
+ SparkActions.get()
+ .rewriteManifests(table)
+ .rewriteIf(manifest -> true)
+ .option(RewriteManifestsSparkAction.USE_CACHING, useCaching)
+ .execute();
+
+ assertThat(table.currentSnapshot().dataManifests(table.io())).hasSize(1);
+
+ List<Row> rowsAfter = recordsWithLineage();
+
+ assertThat(rowsAfter).containsExactlyElementsOf(rowsBefore);
+ }
+
+ @TestTemplate
+ public void testRewriteV3PartitionedManifestsPreservesFirstRowId() {
+ assumeThat(formatVersion).isGreaterThanOrEqualTo(3);
+
+ PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).identity("c1").build();
+ Table table =
+ TABLES.create(
+ SCHEMA,
+ spec,
+ ImmutableMap.of(TableProperties.FORMAT_VERSION, String.valueOf(formatVersion)),
+ tableLocation);
+
+ writeRecords(Lists.newArrayList(new ThreeColumnRecord(1, "AAAA", "AAAA")));
+ writeRecords(Lists.newArrayList(new ThreeColumnRecord(2, "BBBB", "BBBB")));
+ table.refresh();
+
+ assertThat(table.currentSnapshot().dataManifests(table.io())).hasSize(2);
+
+ List<Row> rowsBefore = recordsWithLineage();
+ assertThat(rowsBefore)
+ .extracting(r -> r.<Long>getAs(MetadataColumns.ROW_ID.name()))
+ .doesNotContainNull();
+
+ SparkActions.get()
+ .rewriteManifests(table)
+ .rewriteIf(manifest -> true)
+ .option(RewriteManifestsSparkAction.USE_CACHING, useCaching)
+ .execute();
+
+ assertThat(table.currentSnapshot().dataManifests(table.io())).hasSize(1);
+
+ List<Row> rowsAfter = recordsWithLineage();
+
+ assertThat(rowsAfter).containsExactlyElementsOf(rowsBefore);
+ }
+
+ @TestTemplate
+ public void testRewriteManifestsAfterV2ToV3Upgrade() {
+ assumeThat(formatVersion).isGreaterThanOrEqualTo(3);
+
+ PartitionSpec spec = PartitionSpec.unpartitioned();
+ Table table =
+ TABLES.create(
+ SCHEMA, spec, ImmutableMap.of(TableProperties.FORMAT_VERSION, "2"), tableLocation);
+
+ ThreeColumnRecord record1 = new ThreeColumnRecord(1, null, "AAAA");
+ ThreeColumnRecord record2 = new ThreeColumnRecord(2, "CCCC", "CCCC");
+ writeRecords(Lists.newArrayList(record1));
+ writeRecords(Lists.newArrayList(record2));
+ table.refresh();
+
+ assertThat(table.currentSnapshot().dataManifests(table.io())).hasSize(2);
+
+ table
+ .updateProperties()
+ .set(TableProperties.FORMAT_VERSION, String.valueOf(formatVersion))
+ .commit();
+
+ SparkActions.get()
+ .rewriteManifests(table)
+ .rewriteIf(manifest -> true)
+ .option(RewriteManifestsSparkAction.USE_CACHING, useCaching)
+ .execute();
+
+ assertThat(table.currentSnapshot().dataManifests(table.io())).hasSize(1);
+
+ List<Row> rowsAfter = recordsWithLineage();
+
+ assertThat(rowsAfter)
+ .extracting(r -> r.<Long>getAs(MetadataColumns.ROW_ID.name()))
+ .doesNotContainNull()
+ .doesNotHaveDuplicates();
+ assertThat(rowsAfter)
+ .extracting(r -> new ThreeColumnRecord(r.getAs("c1"), r.getAs("c2"), r.getAs("c3")))
+ .containsExactlyInAnyOrder(record1, record2);
+ }
+
@TestTemplate
public void testRewriteManifestsEmptyTable() throws IOException {
PartitionSpec spec = PartitionSpec.unpartitioned();
@@ -1134,6 +1248,17 @@ public class TestRewriteManifestsAction extends TestBase {
.collectAsList();
}
+ private List<Row> recordsWithLineage() {
+ return spark
+ .read()
+ .format("iceberg")
+ .load(tableLocation)
+ .selectExpr(
+ MetadataColumns.ROW_ID.name(), MetadataColumns.LAST_UPDATED_SEQUENCE_NUMBER.name(), "*")
+ .orderBy(MetadataColumns.ROW_ID.name())
+ .collectAsList();
+ }
+
private void writeRecords(List<ThreeColumnRecord> records) {
Dataset<Row> df = spark.createDataFrame(records, ThreeColumnRecord.class);
writeDF(df);