You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by ru...@apache.org on 2026/03/11 20:54:28 UTC
(iceberg) branch main updated: Core, Flink, Spark: Deprecate Manifest read methods which rely manifest Metadata (#15575)
This is an automated email from the ASF dual-hosted git repository.
russellspitzer pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/main by this push:
new ace2d03750 Core, Flink, Spark: Deprecate Manifest read methods which rely manifest Metadata (#15575)
ace2d03750 is described below
commit ace2d0375069fec0c6284328de460ba4530af87e
Author: Russell Spitzer <ru...@GMAIL.COM>
AuthorDate: Wed Mar 11 15:54:20 2026 -0500
Core, Flink, Spark: Deprecate Manifest read methods which rely manifest Metadata (#15575)
---
.../apache/iceberg/RewriteDataFilesBenchmark.java | 3 +-
.../java/org/apache/iceberg/ManifestFiles.java | 29 ++++++++++++++++++--
.../java/org/apache/iceberg/ManifestReader.java | 8 ++++++
.../src/test/java/org/apache/iceberg/TestBase.java | 17 +++++++-----
.../org/apache/iceberg/TestManifestReader.java | 32 +++++++++++++++-------
.../apache/iceberg/TestManifestReaderStats.java | 23 +++++++++-------
.../org/apache/iceberg/TestManifestWriter.java | 3 +-
.../apache/iceberg/TestManifestWriterVersions.java | 8 ++++--
.../java/org/apache/iceberg/TestMergeAppend.java | 6 ++--
.../java/org/apache/iceberg/TestRewriteFiles.java | 2 +-
.../org/apache/iceberg/TestRewriteManifests.java | 21 +++++++++-----
.../iceberg/TestSequenceNumberForV2Table.java | 3 +-
.../flink/TestFlinkTableSinkCompaction.java | 3 +-
.../iceberg/flink/sink/TestIcebergSinkCompact.java | 3 +-
.../flink/TestFlinkTableSinkCompaction.java | 3 +-
.../iceberg/flink/sink/TestIcebergSinkCompact.java | 3 +-
.../flink/TestFlinkTableSinkCompaction.java | 3 +-
.../iceberg/flink/sink/TestIcebergSinkCompact.java | 3 +-
.../java/org/apache/iceberg/ValidationHelpers.java | 3 +-
.../spark/source/TestCompressionSettings.java | 3 +-
.../iceberg/spark/source/TestSparkDataFile.java | 3 +-
.../iceberg/spark/source/TestSparkDataWrite.java | 6 ++--
.../java/org/apache/iceberg/ValidationHelpers.java | 3 +-
.../spark/source/TestCompressionSettings.java | 3 +-
.../iceberg/spark/source/TestSparkDataFile.java | 3 +-
.../iceberg/spark/source/TestSparkDataWrite.java | 6 ++--
.../java/org/apache/iceberg/ValidationHelpers.java | 3 +-
.../spark/source/TestCompressionSettings.java | 3 +-
.../iceberg/spark/source/TestSparkDataFile.java | 3 +-
.../iceberg/spark/source/TestSparkDataWrite.java | 6 ++--
.../java/org/apache/iceberg/ValidationHelpers.java | 3 +-
.../spark/source/TestCompressionSettings.java | 3 +-
.../iceberg/spark/source/TestSparkDataFile.java | 3 +-
.../iceberg/spark/source/TestSparkDataWrite.java | 6 ++--
34 files changed, 159 insertions(+), 74 deletions(-)
diff --git a/core/src/jmh/java/org/apache/iceberg/RewriteDataFilesBenchmark.java b/core/src/jmh/java/org/apache/iceberg/RewriteDataFilesBenchmark.java
index b56521548b..34b5b58599 100644
--- a/core/src/jmh/java/org/apache/iceberg/RewriteDataFilesBenchmark.java
+++ b/core/src/jmh/java/org/apache/iceberg/RewriteDataFilesBenchmark.java
@@ -146,7 +146,8 @@ public class RewriteDataFilesBenchmark {
List<DataFile> dataFilesReadFromManifests = Lists.newArrayList();
for (ManifestFile dataManifest : table.currentSnapshot().dataManifests(table.io())) {
- try (ManifestReader<DataFile> manifestReader = ManifestFiles.read(dataManifest, table.io())) {
+ try (ManifestReader<DataFile> manifestReader =
+ ManifestFiles.read(dataManifest, table.io(), table.specs())) {
manifestReader
.iterator()
.forEachRemaining(
diff --git a/core/src/main/java/org/apache/iceberg/ManifestFiles.java b/core/src/main/java/org/apache/iceberg/ManifestFiles.java
index 86c4e68067..63cee1e856 100644
--- a/core/src/main/java/org/apache/iceberg/ManifestFiles.java
+++ b/core/src/main/java/org/apache/iceberg/ManifestFiles.java
@@ -97,14 +97,30 @@ public class ManifestFiles {
*
* @param manifest a ManifestFile
* @param io a FileIO
+ * @param specsById a Map from spec ID to partition spec
* @return a manifest reader
*/
- public static CloseableIterable<String> readPaths(ManifestFile manifest, FileIO io) {
+ public static CloseableIterable<String> readPaths(
+ ManifestFile manifest, FileIO io, Map<Integer, PartitionSpec> specsById) {
return CloseableIterable.transform(
- read(manifest, io, null).select(ImmutableList.of("file_path")).liveEntries(),
+ read(manifest, io, specsById).select(ImmutableList.of("file_path")).liveEntries(),
entry -> entry.file().location());
}
+ /**
+ * Returns a {@link CloseableIterable} of file paths in the {@link ManifestFile}.
+ *
+ * @param manifest a ManifestFile
+ * @param io a FileIO
+ * @return a manifest reader
+ * @deprecated since 1.11.0, will be removed in 1.12.0; use {@link #readPaths(ManifestFile,
+ * FileIO, Map)} instead.
+ */
+ @Deprecated
+ public static CloseableIterable<String> readPaths(ManifestFile manifest, FileIO io) {
+ return readPaths(manifest, io, null);
+ }
+
/**
* Returns a new {@link ManifestReader} for a {@link ManifestFile}.
*
@@ -115,7 +131,11 @@ public class ManifestFiles {
* @param manifest a ManifestFile
* @param io a FileIO
* @return a manifest reader
+ * @deprecated since 1.11.0, will be removed in 1.12.0; use {@link #read(ManifestFile, FileIO,
+ * Map)} instead. Reading partition specs from manifest file metadata will not be supported
+ * for non-Avro manifest formats.
*/
+ @Deprecated
public static ManifestReader<DataFile> read(ManifestFile manifest, FileIO io) {
return read(manifest, io, null);
}
@@ -306,6 +326,11 @@ public class ManifestFiles {
return AvroEncoderUtil.decode(manifestData);
}
+ /**
+ * @deprecated since 1.11.0, will be removed in 1.12.0; use {@link #open(ManifestFile, FileIO,
+ * Map)} instead.
+ */
+ @Deprecated
static ManifestReader<?> open(ManifestFile manifest, FileIO io) {
return open(manifest, io, null);
}
diff --git a/core/src/main/java/org/apache/iceberg/ManifestReader.java b/core/src/main/java/org/apache/iceberg/ManifestReader.java
index 40ea17aaa5..668a3764de 100644
--- a/core/src/main/java/org/apache/iceberg/ManifestReader.java
+++ b/core/src/main/java/org/apache/iceberg/ManifestReader.java
@@ -45,6 +45,8 @@ import org.apache.iceberg.relocated.com.google.common.collect.Lists;
import org.apache.iceberg.relocated.com.google.common.collect.Sets;
import org.apache.iceberg.types.Types;
import org.apache.iceberg.util.PartitionSet;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Base reader for data and delete manifest files.
@@ -53,6 +55,8 @@ import org.apache.iceberg.util.PartitionSet;
*/
public class ManifestReader<F extends ContentFile<F>> extends CloseableGroup
implements CloseableIterable<F> {
+ private static final Logger LOG = LoggerFactory.getLogger(ManifestReader.class);
+
static final ImmutableList<String> ALL_COLUMNS = ImmutableList.of("*");
private static final Set<String> STATS_COLUMNS =
@@ -126,6 +130,10 @@ public class ManifestReader<F extends ContentFile<F>> extends CloseableGroup
if (specsById != null) {
this.spec = specsById.get(specId);
} else {
+ LOG.warn(
+ "Reading partition spec from manifest file metadata is deprecated and will be "
+ + "removed in the 1.12.0 release. Pass specsById to avoid reading from file metadata: {}",
+ file.location());
this.spec = readPartitionSpec(file);
}
diff --git a/core/src/test/java/org/apache/iceberg/TestBase.java b/core/src/test/java/org/apache/iceberg/TestBase.java
index ddac370944..e6e6583f9f 100644
--- a/core/src/test/java/org/apache/iceberg/TestBase.java
+++ b/core/src/test/java/org/apache/iceberg/TestBase.java
@@ -491,7 +491,8 @@ public class TestBase {
long id = snap.snapshotId();
Iterator<String> newPaths = paths(newFiles).iterator();
- for (ManifestEntry<DataFile> entry : ManifestFiles.read(manifest, FILE_IO).entries()) {
+ for (ManifestEntry<DataFile> entry :
+ ManifestFiles.read(manifest, FILE_IO, table.specs()).entries()) {
DataFile file = entry.file();
if (sequenceNumber != null) {
V1Assert.assertEquals(
@@ -603,7 +604,8 @@ public class TestBase {
Iterator<Long> ids,
Iterator<DataFile> expectedFiles,
Iterator<ManifestEntry.Status> statuses) {
- for (ManifestEntry<DataFile> entry : ManifestFiles.read(manifest, FILE_IO).entries()) {
+ for (ManifestEntry<DataFile> entry :
+ ManifestFiles.read(manifest, FILE_IO, table.specs()).entries()) {
DataFile file = entry.file();
DataFile expected = expectedFiles.next();
@@ -629,7 +631,7 @@ public class TestBase {
Iterator<DeleteFile> expectedFiles,
Iterator<ManifestEntry.Status> statuses) {
for (ManifestEntry<DeleteFile> entry :
- ManifestFiles.readDeleteManifest(manifest, FILE_IO, null).entries()) {
+ ManifestFiles.readDeleteManifest(manifest, FILE_IO, table.specs()).entries()) {
DeleteFile file = entry.file();
DeleteFile expected = expectedFiles.next();
@@ -787,12 +789,13 @@ public class TestBase {
}
}
- static void validateManifestEntries(
+ void validateManifestEntries(
ManifestFile manifest,
Iterator<Long> ids,
Iterator<DataFile> expectedFiles,
Iterator<ManifestEntry.Status> expectedStatuses) {
- for (ManifestEntry<DataFile> entry : ManifestFiles.read(manifest, FILE_IO).entries()) {
+ for (ManifestEntry<DataFile> entry :
+ ManifestFiles.read(manifest, FILE_IO, table.specs()).entries()) {
DataFile file = entry.file();
DataFile expected = expectedFiles.next();
final ManifestEntry.Status expectedStatus = expectedStatuses.next();
@@ -846,8 +849,8 @@ public class TestBase {
return Iterators.forArray(files);
}
- static Iterator<DataFile> files(ManifestFile manifest) {
- return ManifestFiles.read(manifest, FILE_IO).iterator();
+ Iterator<DataFile> files(ManifestFile manifest) {
+ return ManifestFiles.read(manifest, FILE_IO, table.specs()).iterator();
}
static long recordCount(ContentFile<?>... files) {
diff --git a/core/src/test/java/org/apache/iceberg/TestManifestReader.java b/core/src/test/java/org/apache/iceberg/TestManifestReader.java
index fe4e4a74d1..0af0c87d35 100644
--- a/core/src/test/java/org/apache/iceberg/TestManifestReader.java
+++ b/core/src/test/java/org/apache/iceberg/TestManifestReader.java
@@ -51,7 +51,7 @@ public class TestManifestReader extends TestBase {
@TestTemplate
public void testManifestReaderWithEmptyInheritableMetadata() throws IOException {
ManifestFile manifest = writeManifest(1000L, manifestEntry(Status.EXISTING, 1000L, FILE_A));
- try (ManifestReader<DataFile> reader = ManifestFiles.read(manifest, FILE_IO)) {
+ try (ManifestReader<DataFile> reader = ManifestFiles.read(manifest, FILE_IO, table.specs())) {
ManifestEntry<DataFile> entry = Iterables.getOnlyElement(reader.entries());
assertThat(entry.status()).isEqualTo(Status.EXISTING);
assertThat(entry.file().location()).isEqualTo(FILE_A.location());
@@ -63,7 +63,8 @@ public class TestManifestReader extends TestBase {
public void testReaderWithFilterWithoutSelect() throws IOException {
ManifestFile manifest = writeManifest(1000L, FILE_A, FILE_B, FILE_C);
try (ManifestReader<DataFile> reader =
- ManifestFiles.read(manifest, FILE_IO).filterRows(Expressions.equal("id", 0))) {
+ ManifestFiles.read(manifest, FILE_IO, table.specs())
+ .filterRows(Expressions.equal("id", 0))) {
List<DataFile> files = Streams.stream(reader).collect(Collectors.toList());
// note that all files are returned because the reader returns data files that may match, and
@@ -78,7 +79,7 @@ public class TestManifestReader extends TestBase {
@TestTemplate
public void testInvalidUsage() throws IOException {
ManifestFile manifest = writeManifest(FILE_A, FILE_B);
- assertThatThrownBy(() -> ManifestFiles.read(manifest, FILE_IO))
+ assertThatThrownBy(() -> ManifestFiles.read(manifest, FILE_IO, table.specs()))
.isInstanceOf(IllegalArgumentException.class)
.hasMessage("Cannot read from ManifestFile with null (unassigned) snapshot ID");
}
@@ -86,7 +87,7 @@ public class TestManifestReader extends TestBase {
@TestTemplate
public void testManifestReaderWithPartitionMetadata() throws IOException {
ManifestFile manifest = writeManifest(1000L, manifestEntry(Status.EXISTING, 123L, FILE_A));
- try (ManifestReader<DataFile> reader = ManifestFiles.read(manifest, FILE_IO)) {
+ try (ManifestReader<DataFile> reader = ManifestFiles.read(manifest, FILE_IO, table.specs())) {
ManifestEntry<DataFile> entry = Iterables.getOnlyElement(reader.entries());
assertThat(entry.snapshotId()).isEqualTo(123L);
@@ -106,7 +107,7 @@ public class TestManifestReader extends TestBase {
table.ops().commit(table.ops().current(), table.ops().current().updatePartitionSpec(spec));
ManifestFile manifest = writeManifest(1000L, manifestEntry(Status.EXISTING, 123L, FILE_A));
- try (ManifestReader<DataFile> reader = ManifestFiles.read(manifest, FILE_IO)) {
+ try (ManifestReader<DataFile> reader = ManifestFiles.read(manifest, FILE_IO, table.specs())) {
ManifestEntry<DataFile> entry = Iterables.getOnlyElement(reader.entries());
assertThat(entry.snapshotId()).isEqualTo(123L);
@@ -126,7 +127,7 @@ public class TestManifestReader extends TestBase {
@TestTemplate
public void testDataFilePositions() throws IOException {
ManifestFile manifest = writeManifest(1000L, FILE_A, FILE_B, FILE_C);
- try (ManifestReader<DataFile> reader = ManifestFiles.read(manifest, FILE_IO)) {
+ try (ManifestReader<DataFile> reader = ManifestFiles.read(manifest, FILE_IO, table.specs())) {
long expectedPos = 0L;
for (DataFile file : reader) {
assertThat(file.pos()).as("Position should match").isEqualTo(expectedPos);
@@ -138,7 +139,7 @@ public class TestManifestReader extends TestBase {
@TestTemplate
public void testDataFileManifestPaths() throws IOException {
ManifestFile manifest = writeManifest(1000L, FILE_A, FILE_B, FILE_C);
- try (ManifestReader<DataFile> reader = ManifestFiles.read(manifest, FILE_IO)) {
+ try (ManifestReader<DataFile> reader = ManifestFiles.read(manifest, FILE_IO, table.specs())) {
for (DataFile file : reader) {
assertThat(file.manifestLocation()).isEqualTo(manifest.path());
}
@@ -151,7 +152,7 @@ public class TestManifestReader extends TestBase {
ManifestFile manifest =
writeDeleteManifest(formatVersion, 1000L, FILE_A_DELETES, FILE_B_DELETES);
try (ManifestReader<DeleteFile> reader =
- ManifestFiles.readDeleteManifest(manifest, FILE_IO, null)) {
+ ManifestFiles.readDeleteManifest(manifest, FILE_IO, table.specs())) {
long expectedPos = 0L;
for (DeleteFile file : reader) {
assertThat(file.pos()).as("Position should match").isEqualTo(expectedPos);
@@ -168,7 +169,7 @@ public class TestManifestReader extends TestBase {
ManifestFile manifest =
writeDeleteManifest(formatVersion, 1000L, FILE_A_DELETES, FILE_B_DELETES);
try (ManifestReader<DeleteFile> reader =
- ManifestFiles.readDeleteManifest(manifest, FILE_IO, null)) {
+ ManifestFiles.readDeleteManifest(manifest, FILE_IO, table.specs())) {
for (DeleteFile file : reader) {
assertThat(file.manifestLocation()).isEqualTo(manifest.path());
}
@@ -227,9 +228,20 @@ public class TestManifestReader extends TestBase {
.withSplitOffsets(ImmutableList.of(2L, 1000L)) // Offset 1000 is out of bounds
.build();
ManifestFile manifest = writeManifest(1000L, invalidOffset);
- try (ManifestReader<DataFile> reader = ManifestFiles.read(manifest, FILE_IO)) {
+ try (ManifestReader<DataFile> reader = ManifestFiles.read(manifest, FILE_IO, table.specs())) {
DataFile file = Iterables.getOnlyElement(reader);
assertThat(file.splitOffsets()).isNull();
}
}
+
+ @SuppressWarnings("deprecation")
+ @TestTemplate
+ public void testDeprecatedReadWithoutSpecsById() throws IOException {
+ ManifestFile manifest = writeManifest(1000L, manifestEntry(Status.EXISTING, 1000L, FILE_A));
+ try (ManifestReader<DataFile> reader = ManifestFiles.read(manifest, FILE_IO)) {
+ ManifestEntry<DataFile> entry = Iterables.getOnlyElement(reader.entries());
+ assertThat(entry.status()).isEqualTo(Status.EXISTING);
+ assertThat(entry.file().location()).isEqualTo(FILE_A.location());
+ }
+ }
}
diff --git a/core/src/test/java/org/apache/iceberg/TestManifestReaderStats.java b/core/src/test/java/org/apache/iceberg/TestManifestReaderStats.java
index 0e2f4c0ebe..483910d300 100644
--- a/core/src/test/java/org/apache/iceberg/TestManifestReaderStats.java
+++ b/core/src/test/java/org/apache/iceberg/TestManifestReaderStats.java
@@ -59,7 +59,7 @@ public class TestManifestReaderStats extends TestBase {
@TestTemplate
public void testReadIncludesFullStats() throws IOException {
ManifestFile manifest = writeManifest(1000L, FILE);
- try (ManifestReader<DataFile> reader = ManifestFiles.read(manifest, FILE_IO)) {
+ try (ManifestReader<DataFile> reader = ManifestFiles.read(manifest, FILE_IO, table.specs())) {
CloseableIterable<ManifestEntry<DataFile>> entries = reader.entries();
ManifestEntry<DataFile> entry = entries.iterator().next();
assertFullStats(entry.file());
@@ -70,7 +70,8 @@ public class TestManifestReaderStats extends TestBase {
public void testReadEntriesWithFilterIncludesFullStats() throws IOException {
ManifestFile manifest = writeManifest(1000L, FILE);
try (ManifestReader<DataFile> reader =
- ManifestFiles.read(manifest, FILE_IO).filterRows(Expressions.equal("id", 3))) {
+ ManifestFiles.read(manifest, FILE_IO, table.specs())
+ .filterRows(Expressions.equal("id", 3))) {
CloseableIterable<ManifestEntry<DataFile>> entries = reader.entries();
ManifestEntry<DataFile> entry = entries.iterator().next();
assertFullStats(entry.file());
@@ -81,7 +82,8 @@ public class TestManifestReaderStats extends TestBase {
public void testReadIteratorWithFilterIncludesFullStats() throws IOException {
ManifestFile manifest = writeManifest(1000L, FILE);
try (ManifestReader<DataFile> reader =
- ManifestFiles.read(manifest, FILE_IO).filterRows(Expressions.equal("id", 3))) {
+ ManifestFiles.read(manifest, FILE_IO, table.specs())
+ .filterRows(Expressions.equal("id", 3))) {
DataFile entry = reader.iterator().next();
assertFullStats(entry);
}
@@ -91,7 +93,7 @@ public class TestManifestReaderStats extends TestBase {
public void testReadEntriesWithFilterAndSelectIncludesFullStats() throws IOException {
ManifestFile manifest = writeManifest(1000L, FILE);
try (ManifestReader<DataFile> reader =
- ManifestFiles.read(manifest, FILE_IO)
+ ManifestFiles.read(manifest, FILE_IO, table.specs())
.select(ImmutableList.of("file_path"))
.filterRows(Expressions.equal("id", 3))) {
CloseableIterable<ManifestEntry<DataFile>> entries = reader.entries();
@@ -104,7 +106,7 @@ public class TestManifestReaderStats extends TestBase {
public void testReadIteratorWithFilterAndSelectDropsStats() throws IOException {
ManifestFile manifest = writeManifest(1000L, FILE);
try (ManifestReader<DataFile> reader =
- ManifestFiles.read(manifest, FILE_IO)
+ ManifestFiles.read(manifest, FILE_IO, table.specs())
.select(ImmutableList.of("file_path"))
.filterRows(Expressions.equal("id", 3))) {
DataFile entry = reader.iterator().next();
@@ -116,7 +118,7 @@ public class TestManifestReaderStats extends TestBase {
public void testReadIteratorWithFilterAndSelectRecordCountDropsStats() throws IOException {
ManifestFile manifest = writeManifest(1000L, FILE);
try (ManifestReader<DataFile> reader =
- ManifestFiles.read(manifest, FILE_IO)
+ ManifestFiles.read(manifest, FILE_IO, table.specs())
.select(ImmutableList.of("file_path", "record_count"))
.filterRows(Expressions.equal("id", 3))) {
DataFile entry = reader.iterator().next();
@@ -128,7 +130,7 @@ public class TestManifestReaderStats extends TestBase {
public void testReadIteratorWithFilterAndSelectStatsIncludesFullStats() throws IOException {
ManifestFile manifest = writeManifest(1000L, FILE);
try (ManifestReader<DataFile> reader =
- ManifestFiles.read(manifest, FILE_IO)
+ ManifestFiles.read(manifest, FILE_IO, table.specs())
.select(ImmutableList.of("file_path", "value_counts"))
.filterRows(Expressions.equal("id", 3))) {
DataFile entry = reader.iterator().next();
@@ -143,7 +145,7 @@ public class TestManifestReaderStats extends TestBase {
public void testReadIteratorWithProjectStats() throws IOException {
ManifestFile manifest = writeManifest(1000L, FILE);
try (ManifestReader<DataFile> reader =
- ManifestFiles.read(manifest, FILE_IO)
+ ManifestFiles.read(manifest, FILE_IO, table.specs())
.project(new Schema(ImmutableList.of(DataFile.FILE_PATH, DataFile.VALUE_COUNTS)))) {
DataFile entry = reader.iterator().next();
@@ -162,7 +164,8 @@ public class TestManifestReaderStats extends TestBase {
public void testReadEntriesWithSelectNotProjectStats() throws IOException {
ManifestFile manifest = writeManifest(1000L, FILE);
try (ManifestReader<DataFile> reader =
- ManifestFiles.read(manifest, FILE_IO).select(ImmutableList.of("file_path"))) {
+ ManifestFiles.read(manifest, FILE_IO, table.specs())
+ .select(ImmutableList.of("file_path"))) {
CloseableIterable<ManifestEntry<DataFile>> entries = reader.entries();
ManifestEntry<DataFile> entry = entries.iterator().next();
DataFile dataFile = entry.file();
@@ -185,7 +188,7 @@ public class TestManifestReaderStats extends TestBase {
public void testReadEntriesWithSelectCertainStatNotProjectStats() throws IOException {
ManifestFile manifest = writeManifest(1000L, FILE);
try (ManifestReader<DataFile> reader =
- ManifestFiles.read(manifest, FILE_IO)
+ ManifestFiles.read(manifest, FILE_IO, table.specs())
.select(ImmutableList.of("file_path", "value_counts"))) {
DataFile dataFile = reader.iterator().next();
diff --git a/core/src/test/java/org/apache/iceberg/TestManifestWriter.java b/core/src/test/java/org/apache/iceberg/TestManifestWriter.java
index 5d682421d5..00e66bdd7d 100644
--- a/core/src/test/java/org/apache/iceberg/TestManifestWriter.java
+++ b/core/src/test/java/org/apache/iceberg/TestManifestWriter.java
@@ -100,7 +100,8 @@ public class TestManifestWriter extends TestBase {
writer.close();
ManifestFile manifest = writer.toManifestFile();
assertThat(manifest.sequenceNumber()).isEqualTo(-1);
- ManifestReader<DataFile> manifestReader = ManifestFiles.read(manifest, table.io());
+ ManifestReader<DataFile> manifestReader =
+ ManifestFiles.read(manifest, table.io(), table.specs());
for (ManifestEntry<DataFile> entry : manifestReader.entries()) {
assertThat(entry.dataSequenceNumber()).isEqualTo(1000);
assertThat(entry.fileSequenceNumber()).isEqualTo(ManifestWriter.UNASSIGNED_SEQ);
diff --git a/core/src/test/java/org/apache/iceberg/TestManifestWriterVersions.java b/core/src/test/java/org/apache/iceberg/TestManifestWriterVersions.java
index b0e1c6939b..fdd9cc33a4 100644
--- a/core/src/test/java/org/apache/iceberg/TestManifestWriterVersions.java
+++ b/core/src/test/java/org/apache/iceberg/TestManifestWriterVersions.java
@@ -26,6 +26,7 @@ import static org.assertj.core.api.Assumptions.assumeThat;
import java.io.IOException;
import java.nio.file.Path;
import java.util.List;
+import java.util.Map;
import org.apache.iceberg.encryption.EncryptedOutputFile;
import org.apache.iceberg.encryption.EncryptingFileIO;
import org.apache.iceberg.encryption.EncryptionManager;
@@ -65,6 +66,9 @@ public class TestManifestWriterVersions {
.bucket("id", 16)
.build();
+ private static final Map<Integer, PartitionSpec> SPECS_BY_ID =
+ ImmutableMap.of(SPEC.specId(), SPEC);
+
private static final long SEQUENCE_NUMBER = 34L;
private static final long SNAPSHOT_ID = 987134631982734L;
private static final String PATH =
@@ -465,7 +469,7 @@ public class TestManifestWriterVersions {
private List<ManifestEntry<DataFile>> readManifestAsList(ManifestFile manifest)
throws IOException {
try (CloseableIterable<ManifestEntry<DataFile>> reader =
- ManifestFiles.read(manifest, io).entries()) {
+ ManifestFiles.read(manifest, io, SPECS_BY_ID).entries()) {
return Lists.newArrayList(Iterables.transform(reader, ManifestEntry::copy));
}
}
@@ -491,7 +495,7 @@ public class TestManifestWriterVersions {
private ManifestEntry<DeleteFile> readDeleteManifest(ManifestFile manifest) throws IOException {
try (CloseableIterable<ManifestEntry<DeleteFile>> reader =
- ManifestFiles.readDeleteManifest(manifest, io, null).entries()) {
+ ManifestFiles.readDeleteManifest(manifest, io, SPECS_BY_ID).entries()) {
List<ManifestEntry<DeleteFile>> entries = Lists.newArrayList(reader);
assertThat(entries).hasSize(1);
return entries.get(0);
diff --git a/core/src/test/java/org/apache/iceberg/TestMergeAppend.java b/core/src/test/java/org/apache/iceberg/TestMergeAppend.java
index 4863aa37be..3947f16fe1 100644
--- a/core/src/test/java/org/apache/iceberg/TestMergeAppend.java
+++ b/core/src/test/java/org/apache/iceberg/TestMergeAppend.java
@@ -1473,7 +1473,8 @@ public class TestMergeAppend extends TestBase {
// field ids of manifest entries in two manifests with different specs of the same source field
// should be different
ManifestEntry<DataFile> entry =
- ManifestFiles.read(committedSnapshot.allManifests(table.io()).get(0), FILE_IO)
+ ManifestFiles.read(
+ committedSnapshot.allManifests(table.io()).get(0), FILE_IO, table.specs())
.entries()
.iterator()
.next();
@@ -1486,7 +1487,8 @@ public class TestMergeAppend extends TestBase {
assertThat(field.name()).isEqualTo("data_bucket");
entry =
- ManifestFiles.read(committedSnapshot.allManifests(table.io()).get(1), FILE_IO)
+ ManifestFiles.read(
+ committedSnapshot.allManifests(table.io()).get(1), FILE_IO, table.specs())
.entries()
.iterator()
.next();
diff --git a/core/src/test/java/org/apache/iceberg/TestRewriteFiles.java b/core/src/test/java/org/apache/iceberg/TestRewriteFiles.java
index 16c4a0c20e..72a3c89b74 100644
--- a/core/src/test/java/org/apache/iceberg/TestRewriteFiles.java
+++ b/core/src/test/java/org/apache/iceberg/TestRewriteFiles.java
@@ -365,7 +365,7 @@ public class TestRewriteFiles extends TestBase {
long pendingId = pending.snapshotId();
ManifestFile newManifest = pending.allManifests(table.io()).get(0);
validateManifestEntries(newManifest, ids(pendingId), files(FILE_D), statuses(ADDED));
- assertThat(ManifestFiles.read(newManifest, FILE_IO).entries())
+ assertThat(ManifestFiles.read(newManifest, FILE_IO, table.specs()).entries())
.allSatisfy(entry -> assertThat(entry.dataSequenceNumber()).isEqualTo(oldSequenceNumber));
assertThat(newManifest.sequenceNumber()).isEqualTo(oldSequenceNumber + 1);
diff --git a/core/src/test/java/org/apache/iceberg/TestRewriteManifests.java b/core/src/test/java/org/apache/iceberg/TestRewriteManifests.java
index 0d5414eaf0..19ee156c9e 100644
--- a/core/src/test/java/org/apache/iceberg/TestRewriteManifests.java
+++ b/core/src/test/java/org/apache/iceberg/TestRewriteManifests.java
@@ -133,7 +133,8 @@ public class TestRewriteManifests extends TestBase {
// get the correct file order
List<DataFile> files;
List<Long> ids;
- try (ManifestReader<DataFile> reader = ManifestFiles.read(manifests.get(0), table.io())) {
+ try (ManifestReader<DataFile> reader =
+ ManifestFiles.read(manifests.get(0), table.io(), table.specs())) {
if (reader.iterator().next().location().equals(FILE_A.location())) {
files = Arrays.asList(FILE_A, FILE_B);
ids = Arrays.asList(manifestAppendId, fileAppendId);
@@ -193,7 +194,8 @@ public class TestRewriteManifests extends TestBase {
// get the file order correct
List<DataFile> files;
List<Long> ids;
- try (ManifestReader<DataFile> reader = ManifestFiles.read(manifests.get(0), table.io())) {
+ try (ManifestReader<DataFile> reader =
+ ManifestFiles.read(manifests.get(0), table.io(), table.specs())) {
if (reader.iterator().next().location().equals(FILE_A.location())) {
files = Arrays.asList(FILE_A, FILE_B);
ids = Arrays.asList(appendIdA, appendIdB);
@@ -232,7 +234,8 @@ public class TestRewriteManifests extends TestBase {
.clusterBy(file -> "file")
.rewriteIf(
manifest -> {
- try (ManifestReader<DataFile> reader = ManifestFiles.read(manifest, table.io())) {
+ try (ManifestReader<DataFile> reader =
+ ManifestFiles.read(manifest, table.io(), table.specs())) {
return !reader.iterator().next().location().equals(FILE_A.location());
} catch (IOException x) {
throw new RuntimeIOException(x);
@@ -246,7 +249,8 @@ public class TestRewriteManifests extends TestBase {
// get the file order correct
List<DataFile> files;
List<Long> ids;
- try (ManifestReader<DataFile> reader = ManifestFiles.read(manifests.get(0), table.io())) {
+ try (ManifestReader<DataFile> reader =
+ ManifestFiles.read(manifests.get(0), table.io(), table.specs())) {
if (reader.iterator().next().location().equals(FILE_B.location())) {
files = Arrays.asList(FILE_B, FILE_C);
ids = Arrays.asList(appendIdB, appendIdC);
@@ -307,7 +311,8 @@ public class TestRewriteManifests extends TestBase {
.clusterBy(file -> "file")
.rewriteIf(
manifest -> {
- try (ManifestReader<DataFile> reader = ManifestFiles.read(manifest, table.io())) {
+ try (ManifestReader<DataFile> reader =
+ ManifestFiles.read(manifest, table.io(), table.specs())) {
return !reader.iterator().next().location().equals(FILE_A.location());
} catch (IOException x) {
throw new RuntimeIOException(x);
@@ -327,7 +332,8 @@ public class TestRewriteManifests extends TestBase {
// get the file order correct
List<DataFile> files;
List<Long> ids;
- try (ManifestReader<DataFile> reader = ManifestFiles.read(manifests.get(0), table.io())) {
+ try (ManifestReader<DataFile> reader =
+ ManifestFiles.read(manifests.get(0), table.io(), table.specs())) {
if (reader.iterator().next().location().equals(FILE_A.location())) {
files = Arrays.asList(FILE_A, FILE_B);
ids = Arrays.asList(appendIdA, appendIdB);
@@ -845,7 +851,8 @@ public class TestRewriteManifests extends TestBase {
.clusterBy(dataFile -> "const-value")
.rewriteIf(
manifest -> {
- try (ManifestReader<DataFile> reader = ManifestFiles.read(manifest, table.io())) {
+ try (ManifestReader<DataFile> reader =
+ ManifestFiles.read(manifest, table.io(), table.specs())) {
return !reader.iterator().next().location().equals(FILE_B.location());
} catch (IOException x) {
throw new RuntimeIOException(x);
diff --git a/core/src/test/java/org/apache/iceberg/TestSequenceNumberForV2Table.java b/core/src/test/java/org/apache/iceberg/TestSequenceNumberForV2Table.java
index d416a25d1c..76178d4ea3 100644
--- a/core/src/test/java/org/apache/iceberg/TestSequenceNumberForV2Table.java
+++ b/core/src/test/java/org/apache/iceberg/TestSequenceNumberForV2Table.java
@@ -72,7 +72,8 @@ public class TestSequenceNumberForV2Table extends TestBase {
"Last sequence number should be 3", 3, readMetadata().lastSequenceNumber());
// FILE_A and FILE_B in manifest may reorder
- for (ManifestEntry<DataFile> entry : ManifestFiles.read(newManifest, FILE_IO).entries()) {
+ for (ManifestEntry<DataFile> entry :
+ ManifestFiles.read(newManifest, FILE_IO, table.specs()).entries()) {
if (entry.file().location().equals(FILE_A.location())) {
V2Assert.assertEquals(
"FILE_A sequence number should be 1", 1, entry.dataSequenceNumber().longValue());
diff --git a/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/TestFlinkTableSinkCompaction.java b/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/TestFlinkTableSinkCompaction.java
index b0c4f595aa..e5e478fe4a 100644
--- a/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/TestFlinkTableSinkCompaction.java
+++ b/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/TestFlinkTableSinkCompaction.java
@@ -206,7 +206,8 @@ public class TestFlinkTableSinkCompaction extends CatalogTestBase {
private List<DataFile> getDataFiles(Snapshot snapshot, Table table) throws IOException {
List<DataFile> dataFiles = Lists.newArrayList();
for (ManifestFile dataManifest : snapshot.dataManifests(table.io())) {
- try (ManifestReader<DataFile> reader = ManifestFiles.read(dataManifest, table.io())) {
+ try (ManifestReader<DataFile> reader =
+ ManifestFiles.read(dataManifest, table.io(), table.specs())) {
reader.iterator().forEachRemaining(dataFiles::add);
}
}
diff --git a/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/sink/TestIcebergSinkCompact.java b/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/sink/TestIcebergSinkCompact.java
index 48d5f9382a..8042fe6e0f 100644
--- a/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/sink/TestIcebergSinkCompact.java
+++ b/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/sink/TestIcebergSinkCompact.java
@@ -111,7 +111,8 @@ class TestIcebergSinkCompact extends TestFlinkIcebergSinkBase {
private List<DataFile> getDataFiles(Snapshot snapshot, Table table) throws IOException {
List<DataFile> dataFiles = Lists.newArrayList();
for (ManifestFile dataManifest : snapshot.dataManifests(table.io())) {
- try (ManifestReader<DataFile> reader = ManifestFiles.read(dataManifest, table.io())) {
+ try (ManifestReader<DataFile> reader =
+ ManifestFiles.read(dataManifest, table.io(), table.specs())) {
reader.iterator().forEachRemaining(dataFiles::add);
}
}
diff --git a/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/TestFlinkTableSinkCompaction.java b/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/TestFlinkTableSinkCompaction.java
index b0c4f595aa..e5e478fe4a 100644
--- a/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/TestFlinkTableSinkCompaction.java
+++ b/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/TestFlinkTableSinkCompaction.java
@@ -206,7 +206,8 @@ public class TestFlinkTableSinkCompaction extends CatalogTestBase {
private List<DataFile> getDataFiles(Snapshot snapshot, Table table) throws IOException {
List<DataFile> dataFiles = Lists.newArrayList();
for (ManifestFile dataManifest : snapshot.dataManifests(table.io())) {
- try (ManifestReader<DataFile> reader = ManifestFiles.read(dataManifest, table.io())) {
+ try (ManifestReader<DataFile> reader =
+ ManifestFiles.read(dataManifest, table.io(), table.specs())) {
reader.iterator().forEachRemaining(dataFiles::add);
}
}
diff --git a/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/sink/TestIcebergSinkCompact.java b/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/sink/TestIcebergSinkCompact.java
index 48d5f9382a..8042fe6e0f 100644
--- a/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/sink/TestIcebergSinkCompact.java
+++ b/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/sink/TestIcebergSinkCompact.java
@@ -111,7 +111,8 @@ class TestIcebergSinkCompact extends TestFlinkIcebergSinkBase {
private List<DataFile> getDataFiles(Snapshot snapshot, Table table) throws IOException {
List<DataFile> dataFiles = Lists.newArrayList();
for (ManifestFile dataManifest : snapshot.dataManifests(table.io())) {
- try (ManifestReader<DataFile> reader = ManifestFiles.read(dataManifest, table.io())) {
+ try (ManifestReader<DataFile> reader =
+ ManifestFiles.read(dataManifest, table.io(), table.specs())) {
reader.iterator().forEachRemaining(dataFiles::add);
}
}
diff --git a/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/TestFlinkTableSinkCompaction.java b/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/TestFlinkTableSinkCompaction.java
index b0c4f595aa..e5e478fe4a 100644
--- a/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/TestFlinkTableSinkCompaction.java
+++ b/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/TestFlinkTableSinkCompaction.java
@@ -206,7 +206,8 @@ public class TestFlinkTableSinkCompaction extends CatalogTestBase {
private List<DataFile> getDataFiles(Snapshot snapshot, Table table) throws IOException {
List<DataFile> dataFiles = Lists.newArrayList();
for (ManifestFile dataManifest : snapshot.dataManifests(table.io())) {
- try (ManifestReader<DataFile> reader = ManifestFiles.read(dataManifest, table.io())) {
+ try (ManifestReader<DataFile> reader =
+ ManifestFiles.read(dataManifest, table.io(), table.specs())) {
reader.iterator().forEachRemaining(dataFiles::add);
}
}
diff --git a/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/sink/TestIcebergSinkCompact.java b/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/sink/TestIcebergSinkCompact.java
index 48d5f9382a..8042fe6e0f 100644
--- a/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/sink/TestIcebergSinkCompact.java
+++ b/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/sink/TestIcebergSinkCompact.java
@@ -111,7 +111,8 @@ class TestIcebergSinkCompact extends TestFlinkIcebergSinkBase {
private List<DataFile> getDataFiles(Snapshot snapshot, Table table) throws IOException {
List<DataFile> dataFiles = Lists.newArrayList();
for (ManifestFile dataManifest : snapshot.dataManifests(table.io())) {
- try (ManifestReader<DataFile> reader = ManifestFiles.read(dataManifest, table.io())) {
+ try (ManifestReader<DataFile> reader =
+ ManifestFiles.read(dataManifest, table.io(), table.specs())) {
reader.iterator().forEachRemaining(dataFiles::add);
}
}
diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/ValidationHelpers.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/ValidationHelpers.java
index dac09fea75..1090245326 100644
--- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/ValidationHelpers.java
+++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/ValidationHelpers.java
@@ -58,7 +58,8 @@ public class ValidationHelpers {
List<Long> actualSnapshotIds = Lists.newArrayList();
List<String> actualFiles = Lists.newArrayList();
- for (ManifestEntry<DataFile> entry : ManifestFiles.read(manifest, table.io()).entries()) {
+ for (ManifestEntry<DataFile> entry :
+ ManifestFiles.read(manifest, table.io(), table.specs()).entries()) {
actualDataSeqs.add(entry.dataSequenceNumber());
actualFileSeqs.add(entry.fileSequenceNumber());
actualSnapshotIds.add(entry.snapshotId());
diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java
index 24a14bb64d..678d23de85 100644
--- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java
+++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java
@@ -207,7 +207,8 @@ public class TestCompressionSettings extends CatalogTestBase {
.append();
Table table = catalog.loadTable(TableIdentifier.of("default", TABLE_NAME));
List<ManifestFile> manifestFiles = table.currentSnapshot().dataManifests(table.io());
- try (ManifestReader<DataFile> reader = ManifestFiles.read(manifestFiles.get(0), table.io())) {
+ try (ManifestReader<DataFile> reader =
+ ManifestFiles.read(manifestFiles.get(0), table.io(), table.specs())) {
DataFile file = reader.iterator().next();
InputFile inputFile = table.io().newInputFile(file.location());
assertThat(getCompressionType(inputFile))
diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java
index 833a0b6b5e..3c00835da3 100644
--- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java
+++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java
@@ -183,7 +183,8 @@ public class TestSparkDataFile {
assertThat(manifests).hasSize(1);
List<DataFile> dataFiles = Lists.newArrayList();
- try (ManifestReader<DataFile> reader = ManifestFiles.read(manifests.get(0), table.io())) {
+ try (ManifestReader<DataFile> reader =
+ ManifestFiles.read(manifests.get(0), table.io(), table.specs())) {
for (DataFile dataFile : reader) {
checkDataFile(dataFile.copy(), DataFiles.builder(dataFilesSpec).copy(dataFile).build());
dataFiles.add(dataFile.copy());
diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java
index 4f4c0c7f7a..765ed9aaa0 100644
--- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java
+++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java
@@ -148,7 +148,7 @@ public class TestSparkDataWrite {
assertThat(actual).hasSameSizeAs(expected).isEqualTo(expected);
for (ManifestFile manifest :
SnapshotUtil.latestSnapshot(table, branch).allManifests(table.io())) {
- for (DataFile file : ManifestFiles.read(manifest, table.io())) {
+ for (DataFile file : ManifestFiles.read(manifest, table.io(), table.specs())) {
// TODO: avro not support split
if (!format.equals(FileFormat.AVRO)) {
assertThat(file.splitOffsets()).as("Split offsets not present").isNotNull();
@@ -397,7 +397,7 @@ public class TestSparkDataWrite {
List<DataFile> files = Lists.newArrayList();
for (ManifestFile manifest :
SnapshotUtil.latestSnapshot(table, branch).allManifests(table.io())) {
- for (DataFile file : ManifestFiles.read(manifest, table.io())) {
+ for (DataFile file : ManifestFiles.read(manifest, table.io(), table.specs())) {
files.add(file);
}
}
@@ -624,7 +624,7 @@ public class TestSparkDataWrite {
List<DataFile> files = Lists.newArrayList();
for (ManifestFile manifest :
SnapshotUtil.latestSnapshot(table, branch).allManifests(table.io())) {
- for (DataFile file : ManifestFiles.read(manifest, table.io())) {
+ for (DataFile file : ManifestFiles.read(manifest, table.io(), table.specs())) {
files.add(file);
}
}
diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/ValidationHelpers.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/ValidationHelpers.java
index dac09fea75..1090245326 100644
--- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/ValidationHelpers.java
+++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/ValidationHelpers.java
@@ -58,7 +58,8 @@ public class ValidationHelpers {
List<Long> actualSnapshotIds = Lists.newArrayList();
List<String> actualFiles = Lists.newArrayList();
- for (ManifestEntry<DataFile> entry : ManifestFiles.read(manifest, table.io()).entries()) {
+ for (ManifestEntry<DataFile> entry :
+ ManifestFiles.read(manifest, table.io(), table.specs()).entries()) {
actualDataSeqs.add(entry.dataSequenceNumber());
actualFileSeqs.add(entry.fileSequenceNumber());
actualSnapshotIds.add(entry.snapshotId());
diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java
index 19ec6d13dd..d381822483 100644
--- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java
+++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java
@@ -207,7 +207,8 @@ public class TestCompressionSettings extends CatalogTestBase {
.append();
Table table = catalog.loadTable(TableIdentifier.of("default", TABLE_NAME));
List<ManifestFile> manifestFiles = table.currentSnapshot().dataManifests(table.io());
- try (ManifestReader<DataFile> reader = ManifestFiles.read(manifestFiles.get(0), table.io())) {
+ try (ManifestReader<DataFile> reader =
+ ManifestFiles.read(manifestFiles.get(0), table.io(), table.specs())) {
DataFile file = reader.iterator().next();
InputFile inputFile = table.io().newInputFile(file.location());
assertThat(getCompressionType(inputFile))
diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java
index 3051e27d72..600b3eab1d 100644
--- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java
+++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java
@@ -183,7 +183,8 @@ public class TestSparkDataFile {
assertThat(manifests).hasSize(1);
List<DataFile> dataFiles = Lists.newArrayList();
- try (ManifestReader<DataFile> reader = ManifestFiles.read(manifests.get(0), table.io())) {
+ try (ManifestReader<DataFile> reader =
+ ManifestFiles.read(manifests.get(0), table.io(), table.specs())) {
for (DataFile dataFile : reader) {
checkDataFile(dataFile.copy(), DataFiles.builder(dataFilesSpec).copy(dataFile).build());
dataFiles.add(dataFile.copy());
diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java
index 94547c2cf8..0cabcdac2b 100644
--- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java
+++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java
@@ -148,7 +148,7 @@ public class TestSparkDataWrite {
assertThat(actual).hasSameSizeAs(expected).isEqualTo(expected);
for (ManifestFile manifest :
SnapshotUtil.latestSnapshot(table, branch).allManifests(table.io())) {
- for (DataFile file : ManifestFiles.read(manifest, table.io())) {
+ for (DataFile file : ManifestFiles.read(manifest, table.io(), table.specs())) {
// TODO: avro not support split
if (!format.equals(FileFormat.AVRO)) {
assertThat(file.splitOffsets()).as("Split offsets not present").isNotNull();
@@ -397,7 +397,7 @@ public class TestSparkDataWrite {
List<DataFile> files = Lists.newArrayList();
for (ManifestFile manifest :
SnapshotUtil.latestSnapshot(table, branch).allManifests(table.io())) {
- for (DataFile file : ManifestFiles.read(manifest, table.io())) {
+ for (DataFile file : ManifestFiles.read(manifest, table.io(), table.specs())) {
files.add(file);
}
}
@@ -624,7 +624,7 @@ public class TestSparkDataWrite {
List<DataFile> files = Lists.newArrayList();
for (ManifestFile manifest :
SnapshotUtil.latestSnapshot(table, branch).allManifests(table.io())) {
- for (DataFile file : ManifestFiles.read(manifest, table.io())) {
+ for (DataFile file : ManifestFiles.read(manifest, table.io(), table.specs())) {
files.add(file);
}
}
diff --git a/spark/v4.0/spark/src/test/java/org/apache/iceberg/ValidationHelpers.java b/spark/v4.0/spark/src/test/java/org/apache/iceberg/ValidationHelpers.java
index dac09fea75..1090245326 100644
--- a/spark/v4.0/spark/src/test/java/org/apache/iceberg/ValidationHelpers.java
+++ b/spark/v4.0/spark/src/test/java/org/apache/iceberg/ValidationHelpers.java
@@ -58,7 +58,8 @@ public class ValidationHelpers {
List<Long> actualSnapshotIds = Lists.newArrayList();
List<String> actualFiles = Lists.newArrayList();
- for (ManifestEntry<DataFile> entry : ManifestFiles.read(manifest, table.io()).entries()) {
+ for (ManifestEntry<DataFile> entry :
+ ManifestFiles.read(manifest, table.io(), table.specs()).entries()) {
actualDataSeqs.add(entry.dataSequenceNumber());
actualFileSeqs.add(entry.fileSequenceNumber());
actualSnapshotIds.add(entry.snapshotId());
diff --git a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java
index 19ec6d13dd..d381822483 100644
--- a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java
+++ b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java
@@ -207,7 +207,8 @@ public class TestCompressionSettings extends CatalogTestBase {
.append();
Table table = catalog.loadTable(TableIdentifier.of("default", TABLE_NAME));
List<ManifestFile> manifestFiles = table.currentSnapshot().dataManifests(table.io());
- try (ManifestReader<DataFile> reader = ManifestFiles.read(manifestFiles.get(0), table.io())) {
+ try (ManifestReader<DataFile> reader =
+ ManifestFiles.read(manifestFiles.get(0), table.io(), table.specs())) {
DataFile file = reader.iterator().next();
InputFile inputFile = table.io().newInputFile(file.location());
assertThat(getCompressionType(inputFile))
diff --git a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java
index 35a837b6b8..ef87e8f296 100644
--- a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java
+++ b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java
@@ -190,7 +190,8 @@ public class TestSparkDataFile {
assertThat(manifests).hasSize(1);
List<DataFile> dataFiles = Lists.newArrayList();
- try (ManifestReader<DataFile> reader = ManifestFiles.read(manifests.get(0), table.io())) {
+ try (ManifestReader<DataFile> reader =
+ ManifestFiles.read(manifests.get(0), table.io(), table.specs())) {
for (DataFile dataFile : reader) {
checkDataFile(dataFile.copy(), DataFiles.builder(dataFilesSpec).copy(dataFile).build());
dataFiles.add(dataFile.copy());
diff --git a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java
index 94547c2cf8..0cabcdac2b 100644
--- a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java
+++ b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java
@@ -148,7 +148,7 @@ public class TestSparkDataWrite {
assertThat(actual).hasSameSizeAs(expected).isEqualTo(expected);
for (ManifestFile manifest :
SnapshotUtil.latestSnapshot(table, branch).allManifests(table.io())) {
- for (DataFile file : ManifestFiles.read(manifest, table.io())) {
+ for (DataFile file : ManifestFiles.read(manifest, table.io(), table.specs())) {
// TODO: avro not support split
if (!format.equals(FileFormat.AVRO)) {
assertThat(file.splitOffsets()).as("Split offsets not present").isNotNull();
@@ -397,7 +397,7 @@ public class TestSparkDataWrite {
List<DataFile> files = Lists.newArrayList();
for (ManifestFile manifest :
SnapshotUtil.latestSnapshot(table, branch).allManifests(table.io())) {
- for (DataFile file : ManifestFiles.read(manifest, table.io())) {
+ for (DataFile file : ManifestFiles.read(manifest, table.io(), table.specs())) {
files.add(file);
}
}
@@ -624,7 +624,7 @@ public class TestSparkDataWrite {
List<DataFile> files = Lists.newArrayList();
for (ManifestFile manifest :
SnapshotUtil.latestSnapshot(table, branch).allManifests(table.io())) {
- for (DataFile file : ManifestFiles.read(manifest, table.io())) {
+ for (DataFile file : ManifestFiles.read(manifest, table.io(), table.specs())) {
files.add(file);
}
}
diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/ValidationHelpers.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/ValidationHelpers.java
index dac09fea75..1090245326 100644
--- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/ValidationHelpers.java
+++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/ValidationHelpers.java
@@ -58,7 +58,8 @@ public class ValidationHelpers {
List<Long> actualSnapshotIds = Lists.newArrayList();
List<String> actualFiles = Lists.newArrayList();
- for (ManifestEntry<DataFile> entry : ManifestFiles.read(manifest, table.io()).entries()) {
+ for (ManifestEntry<DataFile> entry :
+ ManifestFiles.read(manifest, table.io(), table.specs()).entries()) {
actualDataSeqs.add(entry.dataSequenceNumber());
actualFileSeqs.add(entry.fileSequenceNumber());
actualSnapshotIds.add(entry.snapshotId());
diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java
index 19ec6d13dd..d381822483 100644
--- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java
+++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java
@@ -207,7 +207,8 @@ public class TestCompressionSettings extends CatalogTestBase {
.append();
Table table = catalog.loadTable(TableIdentifier.of("default", TABLE_NAME));
List<ManifestFile> manifestFiles = table.currentSnapshot().dataManifests(table.io());
- try (ManifestReader<DataFile> reader = ManifestFiles.read(manifestFiles.get(0), table.io())) {
+ try (ManifestReader<DataFile> reader =
+ ManifestFiles.read(manifestFiles.get(0), table.io(), table.specs())) {
DataFile file = reader.iterator().next();
InputFile inputFile = table.io().newInputFile(file.location());
assertThat(getCompressionType(inputFile))
diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java
index 35a837b6b8..ef87e8f296 100644
--- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java
+++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java
@@ -190,7 +190,8 @@ public class TestSparkDataFile {
assertThat(manifests).hasSize(1);
List<DataFile> dataFiles = Lists.newArrayList();
- try (ManifestReader<DataFile> reader = ManifestFiles.read(manifests.get(0), table.io())) {
+ try (ManifestReader<DataFile> reader =
+ ManifestFiles.read(manifests.get(0), table.io(), table.specs())) {
for (DataFile dataFile : reader) {
checkDataFile(dataFile.copy(), DataFiles.builder(dataFilesSpec).copy(dataFile).build());
dataFiles.add(dataFile.copy());
diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java
index 465e6a5abf..63e9a638e5 100644
--- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java
+++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java
@@ -148,7 +148,7 @@ public class TestSparkDataWrite {
assertThat(actual).hasSameSizeAs(expected).isEqualTo(expected);
for (ManifestFile manifest :
SnapshotUtil.latestSnapshot(table, branch).allManifests(table.io())) {
- for (DataFile file : ManifestFiles.read(manifest, table.io())) {
+ for (DataFile file : ManifestFiles.read(manifest, table.io(), table.specs())) {
// TODO: avro not support split
if (!format.equals(FileFormat.AVRO)) {
assertThat(file.splitOffsets()).as("Split offsets not present").isNotNull();
@@ -397,7 +397,7 @@ public class TestSparkDataWrite {
List<DataFile> files = Lists.newArrayList();
for (ManifestFile manifest :
SnapshotUtil.latestSnapshot(table, branch).allManifests(table.io())) {
- for (DataFile file : ManifestFiles.read(manifest, table.io())) {
+ for (DataFile file : ManifestFiles.read(manifest, table.io(), table.specs())) {
files.add(file);
}
}
@@ -624,7 +624,7 @@ public class TestSparkDataWrite {
List<DataFile> files = Lists.newArrayList();
for (ManifestFile manifest :
SnapshotUtil.latestSnapshot(table, branch).allManifests(table.io())) {
- for (DataFile file : ManifestFiles.read(manifest, table.io())) {
+ for (DataFile file : ManifestFiles.read(manifest, table.io(), table.specs())) {
files.add(file);
}
}