You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by am...@apache.org on 2026/07/16 14:13:40 UTC
(iceberg) branch main updated: Core: Clean up TestTrackedFileAdapters and drop TrackedFileBuilder (#17144)
This is an automated email from the ASF dual-hosted git repository.
amogh-jahagirdar pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/main by this push:
new 2660ac0fe2 Core: Clean up TestTrackedFileAdapters and drop TrackedFileBuilder (#17144)
2660ac0fe2 is described below
commit 2660ac0fe2330aebc4393401279aad94ceac7f0b
Author: gaborkaszab <ga...@gmail.com>
AuthorDate: Thu Jul 16 16:13:32 2026 +0200
Core: Clean up TestTrackedFileAdapters and drop TrackedFileBuilder (#17144)
- Remove the last usage of TrackedFileBuilder
- Removed usage of setters with ordinals in TestTrackedFileAdapters
- Dropped TrackedFileBuilder and its test suite
---
.../org/apache/iceberg/TrackedFileBuilder.java | 361 ---------
.../apache/iceberg/TestTrackedFileAdapters.java | 248 ++++--
.../org/apache/iceberg/TestTrackedFileBuilder.java | 863 ---------------------
3 files changed, 163 insertions(+), 1309 deletions(-)
diff --git a/core/src/main/java/org/apache/iceberg/TrackedFileBuilder.java b/core/src/main/java/org/apache/iceberg/TrackedFileBuilder.java
deleted file mode 100644
index d7aa28c329..0000000000
--- a/core/src/main/java/org/apache/iceberg/TrackedFileBuilder.java
+++ /dev/null
@@ -1,361 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iceberg;
-
-import java.nio.ByteBuffer;
-import java.util.List;
-import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
-
-class TrackedFileBuilder {
- private final long snapshotId;
- private final FileContent contentType;
-
- // Required fields
- private Integer formatVersion = null;
- private String location = null;
- private FileFormat fileFormat = null;
- private Long recordCount = null;
- private Long fileSizeInBytes = null;
- private PartitionData partitionData = null;
-
- // optional fields
- private Integer specId = null;
- private ContentStats contentStats = null;
- private Integer sortOrderId = null;
- private DeletionVector deletionVector = null;
- private ManifestInfo manifestInfo = null;
- private ByteBuffer keyMetadata = null;
- private List<Long> splitOffsets = null;
- private List<Integer> equalityIds = null;
-
- // tracking-related fields
- private Tracking sourceTracking = null;
- private boolean dvUpdated = false;
- private ByteBuffer deletedPositions = null;
- private ByteBuffer replacedPositions = null;
-
- /**
- * Creates a builder for a newly added data file entry.
- *
- * @param newSnapshotId the snapshot ID in which the new tracked file will be committed
- */
- static TrackedFileBuilder data(long newSnapshotId) {
- return new TrackedFileBuilder(FileContent.DATA, newSnapshotId);
- }
-
- /**
- * Creates a builder for a newly added equality delete file entry.
- *
- * @param newSnapshotId the snapshot ID in which the new tracked file will be committed
- */
- static TrackedFileBuilder equalityDelete(long newSnapshotId) {
- return new TrackedFileBuilder(FileContent.EQUALITY_DELETES, newSnapshotId);
- }
-
- /**
- * Creates a builder for a newly added data manifest entry.
- *
- * @param newSnapshotId the snapshot ID in which the new tracked file will be committed
- */
- static TrackedFileBuilder dataManifest(long newSnapshotId) {
- return new TrackedFileBuilder(FileContent.DATA_MANIFEST, newSnapshotId);
- }
-
- /**
- * Creates a builder for a newly added delete manifest entry.
- *
- * @param newSnapshotId the snapshot ID in which the new tracked file will be committed
- */
- static TrackedFileBuilder deleteManifest(long newSnapshotId) {
- return new TrackedFileBuilder(FileContent.DELETE_MANIFEST, newSnapshotId);
- }
-
- /**
- * Creates a builder for a tracked file derived from {@code source}.
- *
- * @param source source tracked file to copy fields from
- * @param newSnapshotId the snapshot ID in which the new tracked file will be committed
- */
- static TrackedFileBuilder from(TrackedFile source, long newSnapshotId) {
- Preconditions.checkArgument(source != null, "Invalid source: null");
- return new TrackedFileBuilder(source, newSnapshotId);
- }
-
- /**
- * Returns a DELETED tracked file derived from {@code source}.
- *
- * @param source source tracked file
- * @param newSnapshotId the snapshot ID in which the new tracked file will be committed
- */
- static TrackedFile deleted(TrackedFile source, long newSnapshotId) {
- Preconditions.checkArgument(source != null, "Invalid source: null");
- return terminal(source, TrackingBuilder.deleted(source.tracking(), newSnapshotId));
- }
-
- /**
- * Returns a REPLACED tracked file derived from {@code source}.
- *
- * <p>Manifest entries cannot transition to REPLACED.
- *
- * @param source source tracked file
- * @param newSnapshotId the snapshot ID in which the new tracked file will be committed
- */
- static TrackedFile replaced(TrackedFile source, long newSnapshotId) {
- Preconditions.checkArgument(source != null, "Invalid source: null");
- Preconditions.checkArgument(
- !isLeafManifest(source.contentType()),
- "Manifest entries cannot transition to REPLACED, but entry type is: %s",
- source.contentType());
- return terminal(source, TrackingBuilder.replaced(source.tracking(), newSnapshotId));
- }
-
- private static TrackedFile terminal(TrackedFile source, Tracking tracking) {
- return new TrackedFileStruct(
- tracking,
- source.contentType(),
- source.formatVersion(),
- source.location(),
- source.fileFormat(),
- (PartitionData) source.partition(),
- source.recordCount(),
- source.fileSizeInBytes(),
- source.specId(),
- source.contentStats(),
- source.sortOrderId(),
- source.deletionVector(),
- source.manifestInfo(),
- source.keyMetadata(),
- source.splitOffsets(),
- source.equalityIds());
- }
-
- private TrackedFileBuilder(FileContent contentType, long snapshotId) {
- this.contentType = contentType;
- this.snapshotId = snapshotId;
- }
-
- private TrackedFileBuilder(TrackedFile source, long snapshotId) {
- this.contentType = source.contentType();
- this.snapshotId = snapshotId;
- this.formatVersion = source.formatVersion();
- this.location = source.location();
- this.fileFormat = source.fileFormat();
- this.recordCount = source.recordCount();
- this.fileSizeInBytes = source.fileSizeInBytes();
- this.partitionData = (PartitionData) source.partition();
- this.specId = source.specId();
- this.contentStats = source.contentStats();
- this.sortOrderId = source.sortOrderId();
- this.deletionVector = source.deletionVector();
- this.manifestInfo = source.manifestInfo();
- this.keyMetadata = source.keyMetadata();
- this.splitOffsets = source.splitOffsets();
- this.equalityIds = source.equalityIds();
- this.sourceTracking = source.tracking();
- }
-
- TrackedFileBuilder formatVersion(int newFormatVersion) {
- Preconditions.checkArgument(
- newFormatVersion >= 0, "Invalid format version: %s (must be >= 0)", newFormatVersion);
- this.formatVersion = newFormatVersion;
- return this;
- }
-
- TrackedFileBuilder location(String newLocation) {
- Preconditions.checkArgument(newLocation != null, "Invalid location: null");
- this.location = newLocation;
- return this;
- }
-
- TrackedFileBuilder fileFormat(FileFormat newFileFormat) {
- Preconditions.checkArgument(newFileFormat != null, "Invalid file format: null");
- this.fileFormat = newFileFormat;
- return this;
- }
-
- TrackedFileBuilder recordCount(long newRecordCount) {
- Preconditions.checkArgument(
- newRecordCount >= 0, "Invalid record count: %s (must be >= 0)", newRecordCount);
- this.recordCount = newRecordCount;
- return this;
- }
-
- TrackedFileBuilder fileSizeInBytes(long newFileSizeInBytes) {
- Preconditions.checkArgument(
- newFileSizeInBytes >= 0,
- "Invalid file size in bytes: %s (must be >= 0)",
- newFileSizeInBytes);
- this.fileSizeInBytes = newFileSizeInBytes;
- return this;
- }
-
- TrackedFileBuilder specId(int newSpecId) {
- Preconditions.checkArgument(newSpecId >= 0, "Invalid spec ID: %s (must be >= 0)", newSpecId);
- this.specId = newSpecId;
- return this;
- }
-
- TrackedFileBuilder partition(PartitionData newPartitionData) {
- Preconditions.checkArgument(newPartitionData != null, "Invalid partition: null");
- this.partitionData = newPartitionData;
- return this;
- }
-
- TrackedFileBuilder contentStats(ContentStats newContentStats) {
- Preconditions.checkArgument(newContentStats != null, "Invalid content stats: null");
- this.contentStats = newContentStats;
- return this;
- }
-
- TrackedFileBuilder sortOrderId(int newSortOrderId) {
- Preconditions.checkArgument(
- !isLeafManifest(contentType),
- "Sort order ID cannot be added to manifest entries, but entry type is: %s",
- contentType);
- Preconditions.checkArgument(
- newSortOrderId >= 0, "Invalid sort order ID: %s (must be >= 0)", newSortOrderId);
- this.sortOrderId = newSortOrderId;
- return this;
- }
-
- TrackedFileBuilder deletionVector(DeletionVector newDeletionVector) {
- Preconditions.checkArgument(newDeletionVector != null, "Invalid deletion vector: null");
- Preconditions.checkArgument(
- contentType == FileContent.DATA,
- "Deletion vector can only be added to DATA entries, but entry type is: %s",
- contentType);
- Preconditions.checkArgument(
- this.deletionVector == null || !this.deletionVector.equals(newDeletionVector),
- "The same deletion vector already added");
- this.deletionVector = newDeletionVector;
- this.dvUpdated = true;
- return this;
- }
-
- TrackedFileBuilder manifestInfo(ManifestInfo newManifestInfo) {
- Preconditions.checkArgument(newManifestInfo != null, "Invalid manifest info: null");
- Preconditions.checkArgument(
- isLeafManifest(contentType),
- "Manifest info can only be added to manifests, but entry type is: %s",
- contentType);
- this.manifestInfo = newManifestInfo;
- return this;
- }
-
- TrackedFileBuilder keyMetadata(ByteBuffer newKeyMetadata) {
- Preconditions.checkArgument(newKeyMetadata != null, "Invalid key metadata: null");
- this.keyMetadata = newKeyMetadata;
- return this;
- }
-
- TrackedFileBuilder splitOffsets(List<Long> newSplitOffsets) {
- Preconditions.checkArgument(newSplitOffsets != null, "Invalid split offsets: null");
- Preconditions.checkArgument(
- !isLeafManifest(contentType),
- "Split offsets cannot be added to manifest entries, but entry type is: %s",
- contentType);
- this.splitOffsets = newSplitOffsets;
- return this;
- }
-
- TrackedFileBuilder equalityIds(List<Integer> newEqualityIds) {
- Preconditions.checkArgument(newEqualityIds != null, "Invalid equality IDs: null");
- Preconditions.checkArgument(
- contentType == FileContent.EQUALITY_DELETES,
- "Equality IDs can only be added to EQUALITY_DELETES entries, but entry type is: %s",
- contentType);
- this.equalityIds = newEqualityIds;
- return this;
- }
-
- TrackedFileBuilder deletedPositions(ByteBuffer newDeletedPositions) {
- Preconditions.checkArgument(newDeletedPositions != null, "Invalid deleted positions: null");
- Preconditions.checkArgument(
- isLeafManifest(contentType),
- "Deleted positions can only be added to manifest entries, but entry type is: %s",
- contentType);
- this.deletedPositions = newDeletedPositions;
- return this;
- }
-
- TrackedFileBuilder replacedPositions(ByteBuffer newReplacedPositions) {
- Preconditions.checkArgument(newReplacedPositions != null, "Invalid replaced positions: null");
- Preconditions.checkArgument(
- isLeafManifest(contentType),
- "Replaced positions can only be added to manifest entries, but entry type is: %s",
- contentType);
- this.replacedPositions = newReplacedPositions;
- return this;
- }
-
- private static boolean isLeafManifest(FileContent contentType) {
- return contentType == FileContent.DATA_MANIFEST || contentType == FileContent.DELETE_MANIFEST;
- }
-
- TrackedFile build() {
- Preconditions.checkArgument(formatVersion != null, "Missing required field: format version");
- Preconditions.checkArgument(location != null, "Missing required field: location");
- Preconditions.checkArgument(fileFormat != null, "Missing required field: file format");
- Preconditions.checkArgument(recordCount != null, "Missing required field: record count");
- Preconditions.checkArgument(
- fileSizeInBytes != null, "Missing required field: file size in bytes");
- Preconditions.checkArgument(partitionData != null, "Missing required field: partition data");
- Preconditions.checkArgument(
- !isLeafManifest(contentType) || manifestInfo != null,
- "Missing required field: manifest info");
- Preconditions.checkArgument(
- contentType != FileContent.EQUALITY_DELETES || equalityIds != null,
- "Missing required field: equality IDs");
-
- TrackingBuilder trackingBuilder =
- sourceTracking == null
- ? TrackingBuilder.added(snapshotId)
- : TrackingBuilder.from(sourceTracking, snapshotId);
-
- if (dvUpdated) {
- trackingBuilder.dvUpdated();
- }
-
- if (deletedPositions != null) {
- trackingBuilder.deletedPositions(deletedPositions);
- }
-
- if (replacedPositions != null) {
- trackingBuilder.replacedPositions(replacedPositions);
- }
-
- return new TrackedFileStruct(
- trackingBuilder.build(),
- contentType,
- formatVersion,
- location,
- fileFormat,
- partitionData,
- recordCount,
- fileSizeInBytes,
- specId,
- contentStats,
- sortOrderId,
- deletionVector,
- manifestInfo,
- keyMetadata,
- splitOffsets,
- equalityIds);
- }
-}
diff --git a/core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java b/core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java
index f9b04f3de4..6223dc0d16 100644
--- a/core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java
+++ b/core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java
@@ -59,40 +59,42 @@ class TestTrackedFileAdapters {
.build();
private static final PartitionData PARTITION = partition("books");
- // Tracking field ordinals, looked up from the schema so the tests do not hard-code offsets.
- private static final int DATA_SEQUENCE_NUMBER_ORDINAL =
- ordinalOf(Tracking.schema(), "sequence_number");
- private static final int FILE_SEQUENCE_NUMBER_ORDINAL =
- ordinalOf(Tracking.schema(), "file_sequence_number");
- private static final int FIRST_ROW_ID_ORDINAL = ordinalOf(Tracking.schema(), "first_row_id");
- // manifestPos is appended after the tracking schema fields by the manifest reader.
+ // manifestPos is populated by readers using the setter with the position of the field.
private static final int MANIFEST_POS_ORDINAL = Tracking.schema().fields().size();
- // TrackedFile optional field ordinals, looked up from the schema.
- private static final Types.StructType TRACKED_FILE_SCHEMA =
- TrackedFile.schemaWithContentStats(Types.StructType.of(), Types.StructType.of());
- private static final int CONTENT_TYPE_ORDINAL = ordinalOf(TRACKED_FILE_SCHEMA, "content_type");
- private static final int SPEC_ID_ORDINAL = ordinalOf(TRACKED_FILE_SCHEMA, "spec_id");
- private static final int DELETION_VECTOR_ORDINAL =
- ordinalOf(TRACKED_FILE_SCHEMA, "deletion_vector");
-
@Test
void testDataFileAdapterDelegation() {
+ TrackingStruct tracking =
+ new TrackingStruct(
+ EntryStatus.ADDED,
+ 42L,
+ DATA_SEQUENCE_NUMBER,
+ FILE_SEQUENCE_NUMBER,
+ null,
+ FIRST_ROW_ID,
+ null,
+ null);
+ tracking.setManifestLocation(MANIFEST_LOCATION);
+ tracking.set(MANIFEST_POS_ORDINAL, MANIFEST_POS);
+
TrackedFile file =
- TrackedFileBuilder.data(42L)
- .formatVersion(FORMAT_VERSION_V4)
- .location(DATA_FILE_LOCATION)
- .fileFormat(FileFormat.PARQUET)
- .partition(PARTITION)
- .recordCount(100L)
- .fileSizeInBytes(1024L)
- .specId(PARTITIONED_SPEC_ID)
- .contentStats(createContentStats())
- .sortOrderId(3)
- .keyMetadata(ByteBuffer.wrap(new byte[] {1, 2, 3}))
- .splitOffsets(ImmutableList.of(50L, 100L))
- .build();
- populateTrackingFields(file);
+ new TrackedFileStruct(
+ tracking,
+ FileContent.DATA,
+ FORMAT_VERSION_V4,
+ DATA_FILE_LOCATION,
+ FileFormat.PARQUET,
+ PARTITION,
+ 100L,
+ 1024L,
+ PARTITIONED_SPEC_ID,
+ createContentStats(),
+ 3,
+ null,
+ null,
+ ByteBuffer.wrap(new byte[] {1, 2, 3}),
+ ImmutableList.of(50L, 100L),
+ null);
DataFile dataFile = TrackedFileAdapters.asDataFile(file, specsById(PARTITIONED_SPEC));
@@ -138,22 +140,37 @@ class TestTrackedFileAdapters {
@Test
void testEqualityDeleteFileAdapterDelegation() {
+ TrackingStruct tracking =
+ new TrackingStruct(
+ EntryStatus.ADDED,
+ 42L,
+ DATA_SEQUENCE_NUMBER,
+ FILE_SEQUENCE_NUMBER,
+ null,
+ FIRST_ROW_ID,
+ null,
+ null);
+ tracking.setManifestLocation(MANIFEST_LOCATION);
+ tracking.set(MANIFEST_POS_ORDINAL, MANIFEST_POS);
+
TrackedFile file =
- TrackedFileBuilder.equalityDelete(42L)
- .formatVersion(FORMAT_VERSION_V4)
- .location("s3://bucket/eq-delete.avro")
- .fileFormat(FileFormat.AVRO)
- .partition(PARTITION)
- .recordCount(50L)
- .fileSizeInBytes(512L)
- .specId(PARTITIONED_SPEC_ID)
- .contentStats(createContentStats())
- .sortOrderId(5)
- .keyMetadata(ByteBuffer.wrap(new byte[] {4, 5}))
- .splitOffsets(ImmutableList.of(200L))
- .equalityIds(ImmutableList.of(1, 2, 3))
- .build();
- populateTrackingFields(file);
+ new TrackedFileStruct(
+ tracking,
+ FileContent.EQUALITY_DELETES,
+ FORMAT_VERSION_V4,
+ "s3://bucket/eq-delete.avro",
+ FileFormat.AVRO,
+ PARTITION,
+ 50L,
+ 512L,
+ PARTITIONED_SPEC_ID,
+ createContentStats(),
+ 5,
+ null,
+ null,
+ ByteBuffer.wrap(new byte[] {4, 5}),
+ ImmutableList.of(200L),
+ ImmutableList.of(1, 2, 3));
DeleteFile deleteFile =
TrackedFileAdapters.asEqualityDeleteFile(file, specsById(PARTITIONED_SPEC));
@@ -208,18 +225,37 @@ class TestTrackedFileAdapters {
.cardinality(10L)
.build();
+ TrackingStruct tracking =
+ new TrackingStruct(
+ EntryStatus.ADDED,
+ 42L,
+ DATA_SEQUENCE_NUMBER,
+ FILE_SEQUENCE_NUMBER,
+ 42L,
+ FIRST_ROW_ID,
+ null,
+ null);
+ tracking.setManifestLocation(MANIFEST_LOCATION);
+ tracking.set(MANIFEST_POS_ORDINAL, MANIFEST_POS);
+
TrackedFile file =
- TrackedFileBuilder.data(42L)
- .formatVersion(FORMAT_VERSION_V4)
- .location(DATA_FILE_LOCATION)
- .fileFormat(FileFormat.PARQUET)
- .partition(PARTITION)
- .recordCount(100L)
- .fileSizeInBytes(1024L)
- .specId(PARTITIONED_SPEC_ID)
- .deletionVector(dv)
- .build();
- populateTrackingFields(file);
+ new TrackedFileStruct(
+ tracking,
+ FileContent.DATA,
+ FORMAT_VERSION_V4,
+ DATA_FILE_LOCATION,
+ FileFormat.PARQUET,
+ PARTITION,
+ 100L,
+ 1024L,
+ PARTITIONED_SPEC_ID,
+ null,
+ null,
+ dv,
+ null,
+ null,
+ null,
+ null);
DeleteFile dvFile = TrackedFileAdapters.asDVDeleteFile(file, specsById(PARTITIONED_SPEC));
@@ -299,9 +335,25 @@ class TestTrackedFileAdapters {
TrackedFileAdapters.asEqualityDeleteFile(
dummyTrackedFile(FileContent.EQUALITY_DELETES), UNPARTITIONED));
- TrackedFileStruct dvFile = dummyTrackedFile(FileContent.DATA);
- dvFile.set(DELETION_VECTOR_ORDINAL, deletionVector());
- assertNullTrackingFields(TrackedFileAdapters.asDVDeleteFile(dvFile, UNPARTITIONED));
+ TrackedFileStruct fileWithDV =
+ new TrackedFileStruct(
+ null,
+ FileContent.DATA,
+ 0,
+ null,
+ null,
+ null,
+ 0L,
+ 0L,
+ null,
+ null,
+ null,
+ deletionVector(),
+ null,
+ null,
+ null,
+ null);
+ assertNullTrackingFields(TrackedFileAdapters.asDVDeleteFile(fileWithDV, UNPARTITIONED));
}
@Test
@@ -337,8 +389,24 @@ class TestTrackedFileAdapters {
@Test
void testUnknownSpecIdThrows() {
- TrackedFileStruct file = dummyTrackedFile(FileContent.DATA);
- file.set(SPEC_ID_ORDINAL, 99);
+ TrackedFileStruct file =
+ new TrackedFileStruct(
+ null,
+ FileContent.DATA,
+ 0,
+ null,
+ null,
+ null,
+ 0L,
+ 0L,
+ 99,
+ null,
+ null,
+ null,
+ null,
+ null,
+ null,
+ null);
assertThatThrownBy(() -> TrackedFileAdapters.asDataFile(file, ImmutableMap.of()))
.isInstanceOf(IllegalArgumentException.class)
@@ -347,9 +415,25 @@ class TestTrackedFileAdapters {
@Test
void testSpecIdMismatchThrows() {
+ TrackedFileStruct file =
+ new TrackedFileStruct(
+ null,
+ FileContent.DATA,
+ 0,
+ null,
+ null,
+ null,
+ 0L,
+ 0L,
+ PARTITIONED_SPEC_ID,
+ null,
+ null,
+ null,
+ null,
+ null,
+ null,
+ null);
int mismatchedSpecId = PARTITIONED_SPEC_ID + 1;
- TrackedFileStruct file = dummyTrackedFile(FileContent.DATA);
- file.set(SPEC_ID_ORDINAL, PARTITIONED_SPEC_ID);
PartitionSpec mismatched =
PartitionSpec.builderFor(PARTITION_SCHEMA)
.identity("category")
@@ -387,18 +471,23 @@ class TestTrackedFileAdapters {
/** Minimal file with no tracking, used by the rejection and null-tracking tests. */
private static TrackedFileStruct dummyTrackedFile(FileContent contentType) {
- TrackedFileStruct file = new TrackedFileStruct();
- file.set(CONTENT_TYPE_ORDINAL, contentType.id());
- return file;
- }
-
- private static void populateTrackingFields(TrackedFile file) {
- TrackingStruct tracking = (TrackingStruct) file.tracking();
- tracking.set(DATA_SEQUENCE_NUMBER_ORDINAL, DATA_SEQUENCE_NUMBER);
- tracking.set(FILE_SEQUENCE_NUMBER_ORDINAL, FILE_SEQUENCE_NUMBER);
- tracking.set(FIRST_ROW_ID_ORDINAL, FIRST_ROW_ID);
- tracking.setManifestLocation(MANIFEST_LOCATION);
- tracking.set(MANIFEST_POS_ORDINAL, MANIFEST_POS);
+ return new TrackedFileStruct(
+ null,
+ contentType,
+ FORMAT_VERSION_V4,
+ DATA_FILE_LOCATION,
+ FileFormat.PARQUET,
+ null,
+ 1L,
+ 1L,
+ null,
+ null,
+ null,
+ null,
+ null,
+ null,
+ null,
+ null);
}
private static DeletionVector deletionVector() {
@@ -457,15 +546,4 @@ class TestTrackedFileAdapters {
.withFieldStats(fieldStatsList)
.build();
}
-
- private static int ordinalOf(Types.StructType schema, String fieldName) {
- List<Types.NestedField> fields = schema.fields();
- for (int i = 0; i < fields.size(); i += 1) {
- if (fields.get(i).name().equals(fieldName)) {
- return i;
- }
- }
-
- throw new IllegalArgumentException("No such field: " + fieldName);
- }
}
diff --git a/core/src/test/java/org/apache/iceberg/TestTrackedFileBuilder.java b/core/src/test/java/org/apache/iceberg/TestTrackedFileBuilder.java
deleted file mode 100644
index d6b0701fdf..0000000000
--- a/core/src/test/java/org/apache/iceberg/TestTrackedFileBuilder.java
+++ /dev/null
@@ -1,863 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iceberg;
-
-import static org.apache.iceberg.types.Types.NestedField.optional;
-import static org.assertj.core.api.Assertions.assertThat;
-import static org.assertj.core.api.Assertions.assertThatThrownBy;
-
-import java.nio.ByteBuffer;
-import java.util.stream.Stream;
-import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
-import org.apache.iceberg.types.Types;
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.params.ParameterizedTest;
-import org.junit.jupiter.params.provider.Arguments;
-import org.junit.jupiter.params.provider.MethodSource;
-
-public class TestTrackedFileBuilder {
- private static final int FORMAT_VERSION_V4 = 4;
- private static final Schema TABLE_SCHEMA =
- new Schema(
- optional(1, "id", Types.IntegerType.get()), optional(2, "data", Types.StringType.get()));
- private static final Types.StructType PARTITION_TYPE =
- PartitionSpec.builderFor(TABLE_SCHEMA).identity("id").build().partitionType();
- private static final PartitionData PARTITION_DATA = new PartitionData(PARTITION_TYPE);
- private static final ManifestInfo MANIFEST_INFO =
- ManifestInfoStruct.builder()
- .addedFilesCount(10)
- .existingFilesCount(20)
- .deletedFilesCount(3)
- .replacedFilesCount(2)
- .addedRowsCount(1000L)
- .existingRowsCount(2000L)
- .deletedRowsCount(300L)
- .replacedRowsCount(200L)
- .minSequenceNumber(5L)
- .build();
- private static final DeletionVector DELETION_VECTOR =
- DeletionVectorStruct.builder()
- .location("s3://bucket/data/dv.puffin")
- .offset(0L)
- .sizeInBytes(128L)
- .cardinality(10L)
- .build();
- private static final ContentStats CONTENT_STATS =
- BaseContentStats.builder()
- .withTableSchema(TABLE_SCHEMA)
- .withFieldStats(
- BaseFieldStats.<Integer>builder()
- .fieldId(1)
- .type(Types.IntegerType.get())
- .valueCount(2000L)
- .nullValueCount(0L)
- .lowerBound(1)
- .upperBound(1000)
- .build())
- .withFieldStats(
- BaseFieldStats.<String>builder()
- .fieldId(2)
- .type(Types.StringType.get())
- .valueCount(2000L)
- .nullValueCount(5L)
- .lowerBound("a")
- .upperBound("z")
- .build())
- .build();
- private static final ByteBuffer KEY_METADATA = ByteBuffer.wrap(new byte[] {1, 2, 3});
- private static final ImmutableList<Long> SPLIT_OFFSETS = ImmutableList.of(0L, 4096L, 8192L);
- private static final ByteBuffer DELETED_POSITIONS = ByteBuffer.wrap(new byte[] {10, 11, 12});
- private static final ByteBuffer REPLACED_POSITIONS = ByteBuffer.wrap(new byte[] {20, 21, 22});
-
- private static Stream<Arguments> missingRequiredFieldCases() {
- return Stream.of(
- Arguments.of("formatVersion", "Missing required field: format version"),
- Arguments.of("location", "Missing required field: location"),
- Arguments.of("fileFormat", "Missing required field: file format"),
- Arguments.of("recordCount", "Missing required field: record count"),
- Arguments.of("fileSizeInBytes", "Missing required field: file size in bytes"),
- Arguments.of("partition", "Missing required field: partition data"));
- }
-
- @ParameterizedTest
- @MethodSource("missingRequiredFieldCases")
- public void missingRequiredFields(String missingField, String expectedMessage) {
- TrackedFileBuilder dataBuilder =
- builderWithMissingRequiredField(TrackedFileBuilder.data(50L), missingField);
- TrackedFileBuilder equalityDeleteBuilder =
- builderWithMissingRequiredField(TrackedFileBuilder.equalityDelete(50L), missingField);
- TrackedFileBuilder dataManifestBuilder =
- builderWithMissingRequiredField(TrackedFileBuilder.dataManifest(50L), missingField);
- TrackedFileBuilder deleteManifestBuilder =
- builderWithMissingRequiredField(TrackedFileBuilder.deleteManifest(50L), missingField);
-
- assertThatThrownBy(dataBuilder::build)
- .isInstanceOf(IllegalArgumentException.class)
- .hasMessage(expectedMessage);
- assertThatThrownBy(equalityDeleteBuilder::build)
- .isInstanceOf(IllegalArgumentException.class)
- .hasMessage(expectedMessage);
- assertThatThrownBy(dataManifestBuilder::build)
- .isInstanceOf(IllegalArgumentException.class)
- .hasMessage(expectedMessage);
- assertThatThrownBy(deleteManifestBuilder::build)
- .isInstanceOf(IllegalArgumentException.class)
- .hasMessage(expectedMessage);
- }
-
- private TrackedFileBuilder builderWithMissingRequiredField(
- TrackedFileBuilder builder, String missingField) {
- if (!"formatVersion".equals(missingField)) {
- builder.formatVersion(FORMAT_VERSION_V4);
- }
- if (!"location".equals(missingField)) {
- builder.location("s3://bucket/data/file");
- }
- if (!"fileFormat".equals(missingField)) {
- builder.fileFormat(FileFormat.PARQUET);
- }
- if (!"recordCount".equals(missingField)) {
- builder.recordCount(2000L);
- }
- if (!"fileSizeInBytes".equals(missingField)) {
- builder.fileSizeInBytes(12345L);
- }
- if (!"partition".equals(missingField)) {
- builder.partition(PARTITION_DATA);
- }
- return builder;
- }
-
- @ParameterizedTest
- @MethodSource("manifestBuilders")
- public void missingFieldsForManifests(TrackedFileBuilder builder, FileContent contentType) {
- assertThatThrownBy(
- () ->
- builder
- .formatVersion(FORMAT_VERSION_V4)
- .location("s3://bucket/data/manifest.avro")
- .fileFormat(FileFormat.AVRO)
- .recordCount(420L)
- .fileSizeInBytes(556L)
- .partition(PARTITION_DATA)
- .build())
- .isInstanceOf(IllegalArgumentException.class)
- .hasMessage("Missing required field: manifest info");
- }
-
- @Test
- public void missingEqualityIdsForEqualityDeletes() {
- assertThatThrownBy(
- () ->
- TrackedFileBuilder.equalityDelete(50L)
- .formatVersion(FORMAT_VERSION_V4)
- .location("s3://bucket/data/eq_delete.parquet")
- .fileFormat(FileFormat.PARQUET)
- .recordCount(2000L)
- .fileSizeInBytes(12345L)
- .partition(PARTITION_DATA)
- .build())
- .isInstanceOf(IllegalArgumentException.class)
- .hasMessage("Missing required field: equality IDs");
- }
-
- private static Stream<Arguments> nonEqualityDeleteBuilders() {
- return Stream.of(
- Arguments.of(TrackedFileBuilder.data(10L), FileContent.DATA),
- Arguments.of(TrackedFileBuilder.dataManifest(10L), FileContent.DATA_MANIFEST),
- Arguments.of(TrackedFileBuilder.deleteManifest(10L), FileContent.DELETE_MANIFEST),
- Arguments.of(TrackedFileBuilder.from(sourceData(12L), 20L), FileContent.DATA),
- Arguments.of(
- TrackedFileBuilder.from(sourceDataManifest(21L), 25L), FileContent.DATA_MANIFEST),
- Arguments.of(
- TrackedFileBuilder.from(sourceDeleteManifest(12L), 20L), FileContent.DELETE_MANIFEST));
- }
-
- @ParameterizedTest
- @MethodSource("nonEqualityDeleteBuilders")
- public void invalidEqualityIdsForContentType(
- TrackedFileBuilder builder, FileContent contentType) {
- assertThatThrownBy(() -> builder.equalityIds(ImmutableList.of(1)))
- .isInstanceOf(IllegalArgumentException.class)
- .hasMessage(
- "Equality IDs can only be added to EQUALITY_DELETES entries, but entry type is: "
- + contentType);
- }
-
- private static Stream<Arguments> nonDataBuilders() {
- return Stream.of(
- Arguments.of(TrackedFileBuilder.equalityDelete(10L), FileContent.EQUALITY_DELETES),
- Arguments.of(TrackedFileBuilder.dataManifest(10L), FileContent.DATA_MANIFEST),
- Arguments.of(TrackedFileBuilder.deleteManifest(10L), FileContent.DELETE_MANIFEST),
- Arguments.of(
- TrackedFileBuilder.from(sourceEqualityDelete(12L), 20L), FileContent.EQUALITY_DELETES),
- Arguments.of(
- TrackedFileBuilder.from(sourceDataManifest(21L), 25L), FileContent.DATA_MANIFEST),
- Arguments.of(
- TrackedFileBuilder.from(sourceDeleteManifest(12L), 20L), FileContent.DELETE_MANIFEST));
- }
-
- @ParameterizedTest
- @MethodSource("nonDataBuilders")
- public void invalidDeletionVectorForContentType(
- TrackedFileBuilder builder, FileContent contentType) {
- assertThatThrownBy(() -> builder.deletionVector(DELETION_VECTOR))
- .isInstanceOf(IllegalArgumentException.class)
- .hasMessage(
- "Deletion vector can only be added to DATA entries, but entry type is: " + contentType);
- }
-
- @ParameterizedTest
- @MethodSource("manifestBuilders")
- public void invalidSortOrderIdForContentType(
- TrackedFileBuilder builder, FileContent contentType) {
- assertThatThrownBy(() -> builder.sortOrderId(1))
- .isInstanceOf(IllegalArgumentException.class)
- .hasMessage(
- "Sort order ID cannot be added to manifest entries, but entry type is: " + contentType);
- }
-
- @ParameterizedTest
- @MethodSource("manifestBuilders")
- public void invalidSplitOffsetsForContentType(
- TrackedFileBuilder builder, FileContent contentType) {
- assertThatThrownBy(() -> builder.splitOffsets(SPLIT_OFFSETS))
- .isInstanceOf(IllegalArgumentException.class)
- .hasMessage(
- "Split offsets cannot be added to manifest entries, but entry type is: " + contentType);
- }
-
- private static Stream<Arguments> nonManifestBuilders() {
- return Stream.of(
- Arguments.of(TrackedFileBuilder.data(10L), FileContent.DATA),
- Arguments.of(TrackedFileBuilder.equalityDelete(10L), FileContent.EQUALITY_DELETES),
- Arguments.of(TrackedFileBuilder.from(sourceData(12L), 20L), FileContent.DATA),
- Arguments.of(
- TrackedFileBuilder.from(sourceEqualityDelete(12L), 20L), FileContent.EQUALITY_DELETES));
- }
-
- @ParameterizedTest
- @MethodSource("nonManifestBuilders")
- public void invalidManifestInfoForContentType(
- TrackedFileBuilder builder, FileContent contentType) {
- assertThatThrownBy(() -> builder.manifestInfo(MANIFEST_INFO))
- .isInstanceOf(IllegalArgumentException.class)
- .hasMessage(
- "Manifest info can only be added to manifests, but entry type is: " + contentType);
- }
-
- @ParameterizedTest
- @MethodSource("nonManifestBuilders")
- public void invalidDeletedPositionsForContentType(
- TrackedFileBuilder builder, FileContent contentType) {
- assertThatThrownBy(() -> builder.deletedPositions(DELETED_POSITIONS))
- .isInstanceOf(IllegalArgumentException.class)
- .hasMessage(
- "Deleted positions can only be added to manifest entries, but entry type is: "
- + contentType);
- }
-
- @ParameterizedTest
- @MethodSource("nonManifestBuilders")
- public void invalidReplacedPositionsForContentType(
- TrackedFileBuilder builder, FileContent contentType) {
- assertThatThrownBy(() -> builder.replacedPositions(REPLACED_POSITIONS))
- .isInstanceOf(IllegalArgumentException.class)
- .hasMessage(
- "Replaced positions can only be added to manifest entries, but entry type is: "
- + contentType);
- }
-
- @Test
- public void invalidNullInputs() {
- assertThatThrownBy(() -> TrackedFileBuilder.data(30L).location(null))
- .isInstanceOf(IllegalArgumentException.class)
- .hasMessage("Invalid location: null");
-
- assertThatThrownBy(() -> TrackedFileBuilder.data(30L).fileFormat(null))
- .isInstanceOf(IllegalArgumentException.class)
- .hasMessage("Invalid file format: null");
-
- assertThatThrownBy(() -> TrackedFileBuilder.data(30L).partition(null))
- .isInstanceOf(IllegalArgumentException.class)
- .hasMessage("Invalid partition: null");
-
- assertThatThrownBy(() -> TrackedFileBuilder.data(30L).contentStats(null))
- .isInstanceOf(IllegalArgumentException.class)
- .hasMessage("Invalid content stats: null");
-
- assertThatThrownBy(() -> TrackedFileBuilder.data(30L).deletionVector(null))
- .isInstanceOf(IllegalArgumentException.class)
- .hasMessage("Invalid deletion vector: null");
-
- assertThatThrownBy(() -> TrackedFileBuilder.dataManifest(30L).manifestInfo(null))
- .isInstanceOf(IllegalArgumentException.class)
- .hasMessage("Invalid manifest info: null");
-
- assertThatThrownBy(() -> TrackedFileBuilder.data(30L).keyMetadata(null))
- .isInstanceOf(IllegalArgumentException.class)
- .hasMessage("Invalid key metadata: null");
-
- assertThatThrownBy(() -> TrackedFileBuilder.data(30L).splitOffsets(null))
- .isInstanceOf(IllegalArgumentException.class)
- .hasMessage("Invalid split offsets: null");
-
- assertThatThrownBy(() -> TrackedFileBuilder.equalityDelete(30L).equalityIds(null))
- .isInstanceOf(IllegalArgumentException.class)
- .hasMessage("Invalid equality IDs: null");
-
- assertThatThrownBy(() -> TrackedFileBuilder.from(null, 20L))
- .isInstanceOf(IllegalArgumentException.class)
- .hasMessage("Invalid source: null");
-
- assertThatThrownBy(
- () ->
- TrackedFileBuilder.from(
- entryWithInheritedSeqNums(sourceDataManifest(10L), 15L), 20L)
- .deletedPositions(null))
- .isInstanceOf(IllegalArgumentException.class)
- .hasMessage("Invalid deleted positions: null");
-
- assertThatThrownBy(
- () ->
- TrackedFileBuilder.from(
- entryWithInheritedSeqNums(sourceDeleteManifest(100L), 150L), 200L)
- .replacedPositions(null))
- .isInstanceOf(IllegalArgumentException.class)
- .hasMessage("Invalid replaced positions: null");
-
- assertThatThrownBy(() -> TrackedFileBuilder.deleted(null, 20L))
- .isInstanceOf(IllegalArgumentException.class)
- .hasMessage("Invalid source: null");
-
- assertThatThrownBy(() -> TrackedFileBuilder.replaced(null, 20L))
- .isInstanceOf(IllegalArgumentException.class)
- .hasMessage("Invalid source: null");
- }
-
- @Test
- public void invalidNegativeInputs() {
- assertThatThrownBy(() -> TrackedFileBuilder.dataManifest(40L).formatVersion(-1))
- .isInstanceOf(IllegalArgumentException.class)
- .hasMessage("Invalid format version: -1 (must be >= 0)");
-
- assertThatThrownBy(() -> TrackedFileBuilder.dataManifest(40L).recordCount(-1))
- .isInstanceOf(IllegalArgumentException.class)
- .hasMessage("Invalid record count: -1 (must be >= 0)");
-
- assertThatThrownBy(() -> TrackedFileBuilder.dataManifest(40L).fileSizeInBytes(-1))
- .isInstanceOf(IllegalArgumentException.class)
- .hasMessage("Invalid file size in bytes: -1 (must be >= 0)");
-
- assertThatThrownBy(() -> TrackedFileBuilder.dataManifest(40L).specId(-1))
- .isInstanceOf(IllegalArgumentException.class)
- .hasMessage("Invalid spec ID: -1 (must be >= 0)");
-
- assertThatThrownBy(() -> TrackedFileBuilder.data(40L).sortOrderId(-1))
- .isInstanceOf(IllegalArgumentException.class)
- .hasMessage("Invalid sort order ID: -1 (must be >= 0)");
- }
-
- @Test
- public void buildDataFileWithRequiredFieldsOnly() {
- TrackedFile trackedFile =
- TrackedFileBuilder.data(50L)
- .formatVersion(FORMAT_VERSION_V4)
- .location("s3://bucket/data/file.parquet")
- .fileFormat(FileFormat.PARQUET)
- .recordCount(2000L)
- .fileSizeInBytes(12345L)
- .partition(PARTITION_DATA)
- .build();
-
- assertThat(trackedFile.formatVersion()).isEqualTo(FORMAT_VERSION_V4);
- assertThat(trackedFile.contentType()).isEqualTo(FileContent.DATA);
- assertThat(trackedFile.location()).isEqualTo("s3://bucket/data/file.parquet");
- assertThat(trackedFile.fileFormat()).isEqualTo(FileFormat.PARQUET);
- assertThat(trackedFile.recordCount()).isEqualTo(2000L);
- assertThat(trackedFile.fileSizeInBytes()).isEqualTo(12345L);
- assertThat(trackedFile.partition()).isSameAs(PARTITION_DATA);
-
- assertThat(trackedFile.tracking().status()).isEqualTo(EntryStatus.ADDED);
- assertThat(trackedFile.tracking().snapshotId()).isEqualTo(50L);
- assertThat(trackedFile.tracking().dvSnapshotId()).isNull();
-
- assertThat(trackedFile.specId()).isNull();
- assertThat(trackedFile.contentStats()).isNull();
- assertThat(trackedFile.sortOrderId()).isNull();
- assertThat(trackedFile.deletionVector()).isNull();
- assertThat(trackedFile.manifestInfo()).isNull();
- assertThat(trackedFile.keyMetadata()).isNull();
- assertThat(trackedFile.splitOffsets()).isNull();
- assertThat(trackedFile.equalityIds()).isNull();
- }
-
- @Test
- public void buildDataFileWithAllFields() {
- TrackedFile trackedFile =
- TrackedFileBuilder.data(50L)
- .formatVersion(FORMAT_VERSION_V4)
- .location("s3://bucket/data/file.parquet")
- .fileFormat(FileFormat.PARQUET)
- .recordCount(2000L)
- .fileSizeInBytes(12345L)
- .specId(7)
- .partition(PARTITION_DATA)
- .contentStats(CONTENT_STATS)
- .sortOrderId(3)
- .deletionVector(DELETION_VECTOR)
- .keyMetadata(KEY_METADATA)
- .splitOffsets(SPLIT_OFFSETS)
- .build();
-
- assertThat(trackedFile.formatVersion()).isEqualTo(FORMAT_VERSION_V4);
- assertThat(trackedFile.contentType()).isEqualTo(FileContent.DATA);
- assertThat(trackedFile.location()).isEqualTo("s3://bucket/data/file.parquet");
- assertThat(trackedFile.fileFormat()).isEqualTo(FileFormat.PARQUET);
- assertThat(trackedFile.recordCount()).isEqualTo(2000L);
- assertThat(trackedFile.fileSizeInBytes()).isEqualTo(12345L);
- assertThat(trackedFile.specId()).isEqualTo(7);
- assertThat(trackedFile.partition()).isSameAs(PARTITION_DATA);
- assertThat(trackedFile.contentStats()).isSameAs(CONTENT_STATS);
- assertThat(trackedFile.sortOrderId()).isEqualTo(3);
- assertThat(trackedFile.deletionVector()).isSameAs(DELETION_VECTOR);
- assertThat(trackedFile.keyMetadata()).isEqualTo(KEY_METADATA);
- assertThat(trackedFile.splitOffsets()).isEqualTo(SPLIT_OFFSETS);
-
- assertThat(trackedFile.tracking().status()).isEqualTo(EntryStatus.ADDED);
- assertThat(trackedFile.tracking().snapshotId()).isEqualTo(50L);
- assertThat(trackedFile.tracking().dvSnapshotId()).isEqualTo(50L);
-
- // Unsupported fields for data files
- assertThat(trackedFile.manifestInfo()).isNull();
- assertThat(trackedFile.equalityIds()).isNull();
- }
-
- @Test
- public void buildEqualityDeleteFileWithRequiredFieldsOnly() {
- TrackedFile trackedFile =
- TrackedFileBuilder.equalityDelete(50L)
- .formatVersion(FORMAT_VERSION_V4)
- .location("s3://bucket/data/eq_delete.parquet")
- .fileFormat(FileFormat.PARQUET)
- .recordCount(2000L)
- .fileSizeInBytes(12345L)
- .partition(PARTITION_DATA)
- .equalityIds(ImmutableList.of(1))
- .build();
-
- assertThat(trackedFile.formatVersion()).isEqualTo(FORMAT_VERSION_V4);
- assertThat(trackedFile.contentType()).isEqualTo(FileContent.EQUALITY_DELETES);
- assertThat(trackedFile.location()).isEqualTo("s3://bucket/data/eq_delete.parquet");
- assertThat(trackedFile.fileFormat()).isEqualTo(FileFormat.PARQUET);
- assertThat(trackedFile.recordCount()).isEqualTo(2000L);
- assertThat(trackedFile.fileSizeInBytes()).isEqualTo(12345L);
- assertThat(trackedFile.partition()).isSameAs(PARTITION_DATA);
- assertThat(trackedFile.equalityIds()).containsExactly(1);
-
- assertThat(trackedFile.tracking().status()).isEqualTo(EntryStatus.ADDED);
- assertThat(trackedFile.tracking().snapshotId()).isEqualTo(50L);
-
- assertThat(trackedFile.specId()).isNull();
- assertThat(trackedFile.contentStats()).isNull();
- assertThat(trackedFile.sortOrderId()).isNull();
- assertThat(trackedFile.deletionVector()).isNull();
- assertThat(trackedFile.manifestInfo()).isNull();
- assertThat(trackedFile.keyMetadata()).isNull();
- assertThat(trackedFile.splitOffsets()).isNull();
- }
-
- @Test
- public void buildEqualityDeleteFileWithAllFields() {
- TrackedFile trackedFile =
- TrackedFileBuilder.equalityDelete(50L)
- .formatVersion(FORMAT_VERSION_V4)
- .location("s3://bucket/data/eq_delete.parquet")
- .fileFormat(FileFormat.PARQUET)
- .recordCount(2000L)
- .fileSizeInBytes(12345L)
- .specId(7)
- .partition(PARTITION_DATA)
- .contentStats(CONTENT_STATS)
- .keyMetadata(KEY_METADATA)
- .splitOffsets(SPLIT_OFFSETS)
- .sortOrderId(3)
- .equalityIds(ImmutableList.of(1, 2))
- .build();
-
- assertThat(trackedFile.formatVersion()).isEqualTo(FORMAT_VERSION_V4);
- assertThat(trackedFile.contentType()).isEqualTo(FileContent.EQUALITY_DELETES);
- assertThat(trackedFile.location()).isEqualTo("s3://bucket/data/eq_delete.parquet");
- assertThat(trackedFile.fileFormat()).isEqualTo(FileFormat.PARQUET);
- assertThat(trackedFile.recordCount()).isEqualTo(2000L);
- assertThat(trackedFile.fileSizeInBytes()).isEqualTo(12345L);
- assertThat(trackedFile.specId()).isEqualTo(7);
- assertThat(trackedFile.partition()).isSameAs(PARTITION_DATA);
- assertThat(trackedFile.contentStats()).isSameAs(CONTENT_STATS);
- assertThat(trackedFile.keyMetadata()).isEqualTo(KEY_METADATA);
- assertThat(trackedFile.splitOffsets()).isEqualTo(SPLIT_OFFSETS);
- assertThat(trackedFile.sortOrderId()).isEqualTo(3);
- assertThat(trackedFile.equalityIds()).containsExactly(1, 2);
-
- assertThat(trackedFile.tracking().status()).isEqualTo(EntryStatus.ADDED);
- assertThat(trackedFile.tracking().snapshotId()).isEqualTo(50L);
-
- // Unsupported fields for equality delete files
- assertThat(trackedFile.deletionVector()).isNull();
- assertThat(trackedFile.manifestInfo()).isNull();
- }
-
- private static Stream<Arguments> manifestBuilders() {
- return Stream.of(
- Arguments.of(TrackedFileBuilder.dataManifest(50L), FileContent.DATA_MANIFEST),
- Arguments.of(TrackedFileBuilder.deleteManifest(50L), FileContent.DELETE_MANIFEST));
- }
-
- @ParameterizedTest
- @MethodSource("manifestBuilders")
- public void buildManifestWithRequiredFieldsOnly(
- TrackedFileBuilder builder, FileContent contentType) {
- TrackedFile trackedFile =
- builder
- .formatVersion(FORMAT_VERSION_V4)
- .location("s3://bucket/data/manifest.avro")
- .fileFormat(FileFormat.AVRO)
- .recordCount(420L)
- .fileSizeInBytes(556L)
- .partition(PARTITION_DATA)
- .manifestInfo(MANIFEST_INFO)
- .build();
-
- assertThat(trackedFile.formatVersion()).isEqualTo(FORMAT_VERSION_V4);
- assertThat(trackedFile.contentType()).isEqualTo(contentType);
- assertThat(trackedFile.location()).isEqualTo("s3://bucket/data/manifest.avro");
- assertThat(trackedFile.fileFormat()).isEqualTo(FileFormat.AVRO);
- assertThat(trackedFile.recordCount()).isEqualTo(420L);
- assertThat(trackedFile.fileSizeInBytes()).isEqualTo(556L);
- assertThat(trackedFile.partition()).isSameAs(PARTITION_DATA);
- assertThat(trackedFile.manifestInfo()).isSameAs(MANIFEST_INFO);
-
- assertThat(trackedFile.tracking().status()).isEqualTo(EntryStatus.ADDED);
- assertThat(trackedFile.tracking().snapshotId()).isEqualTo(50L);
-
- assertThat(trackedFile.specId()).isNull();
- assertThat(trackedFile.contentStats()).isNull();
- assertThat(trackedFile.sortOrderId()).isNull();
- assertThat(trackedFile.deletionVector()).isNull();
- assertThat(trackedFile.keyMetadata()).isNull();
- assertThat(trackedFile.splitOffsets()).isNull();
- assertThat(trackedFile.equalityIds()).isNull();
- }
-
- @ParameterizedTest
- @MethodSource("manifestBuilders")
- public void buildManifestWithAllFields(TrackedFileBuilder builder, FileContent contentType) {
- TrackedFile trackedFile =
- builder
- .formatVersion(FORMAT_VERSION_V4)
- .location("s3://bucket/data/manifest.avro")
- .fileFormat(FileFormat.AVRO)
- .recordCount(420L)
- .fileSizeInBytes(556L)
- .specId(7)
- .partition(PARTITION_DATA)
- .contentStats(CONTENT_STATS)
- .keyMetadata(KEY_METADATA)
- .manifestInfo(MANIFEST_INFO)
- .build();
-
- assertThat(trackedFile.formatVersion()).isEqualTo(FORMAT_VERSION_V4);
- assertThat(trackedFile.contentType()).isEqualTo(contentType);
- assertThat(trackedFile.location()).isEqualTo("s3://bucket/data/manifest.avro");
- assertThat(trackedFile.fileFormat()).isEqualTo(FileFormat.AVRO);
- assertThat(trackedFile.recordCount()).isEqualTo(420L);
- assertThat(trackedFile.fileSizeInBytes()).isEqualTo(556L);
- assertThat(trackedFile.specId()).isEqualTo(7);
- assertThat(trackedFile.partition()).isSameAs(PARTITION_DATA);
- assertThat(trackedFile.contentStats()).isSameAs(CONTENT_STATS);
- assertThat(trackedFile.keyMetadata()).isEqualTo(KEY_METADATA);
- assertThat(trackedFile.manifestInfo()).isSameAs(MANIFEST_INFO);
-
- assertThat(trackedFile.tracking().status()).isEqualTo(EntryStatus.ADDED);
- assertThat(trackedFile.tracking().snapshotId()).isEqualTo(50L);
-
- // Unsupported fields for manifests
- assertThat(trackedFile.sortOrderId()).isNull();
- assertThat(trackedFile.deletionVector()).isNull();
- assertThat(trackedFile.splitOffsets()).isNull();
- assertThat(trackedFile.equalityIds()).isNull();
- }
-
- private static Stream<Arguments> manifestSources() {
- return Stream.of(
- Arguments.of(sourceDataManifest(10L), FileContent.DATA_MANIFEST),
- Arguments.of(sourceDeleteManifest(10L), FileContent.DELETE_MANIFEST));
- }
-
- @ParameterizedTest
- @MethodSource("manifestSources")
- public void buildManifestFromSourceWithDeletedPositions(
- TrackedFile source, FileContent contentType) {
- entryWithInheritedSeqNums(source, 7L);
-
- TrackedFile trackedFile =
- TrackedFileBuilder.from(source, 20L).deletedPositions(DELETED_POSITIONS).build();
-
- assertThat(trackedFile.contentType()).isEqualTo(contentType);
- assertThat(trackedFile.tracking().status()).isEqualTo(EntryStatus.MODIFIED);
- assertThat(trackedFile.tracking().dvSnapshotId()).isEqualTo(20L);
- assertThat(trackedFile.tracking().deletedPositions()).isEqualTo(DELETED_POSITIONS);
- assertThat(trackedFile.tracking().replacedPositions()).isNull();
- }
-
- @ParameterizedTest
- @MethodSource("manifestSources")
- public void buildManifestFromSourceWithReplacedPositions(
- TrackedFile source, FileContent contentType) {
- entryWithInheritedSeqNums(source, 7L);
-
- TrackedFile trackedFile =
- TrackedFileBuilder.from(source, 20L).replacedPositions(REPLACED_POSITIONS).build();
-
- assertThat(trackedFile.contentType()).isEqualTo(contentType);
- assertThat(trackedFile.tracking().status()).isEqualTo(EntryStatus.MODIFIED);
- assertThat(trackedFile.tracking().dvSnapshotId()).isEqualTo(20L);
- assertThat(trackedFile.tracking().deletedPositions()).isNull();
- assertThat(trackedFile.tracking().replacedPositions()).isEqualTo(REPLACED_POSITIONS);
- }
-
- @ParameterizedTest
- @MethodSource("manifestSources")
- public void buildManifestFromSourceClearsPositions(TrackedFile source, FileContent contentType) {
- entryWithInheritedSeqNums(source, 7L);
-
- TrackedFile sourceWithPositions =
- TrackedFileBuilder.from(source, 15L)
- .deletedPositions(DELETED_POSITIONS)
- .replacedPositions(REPLACED_POSITIONS)
- .build();
-
- TrackedFile newEntry = TrackedFileBuilder.from(sourceWithPositions, 20L).build();
-
- // Building a new entry from this source should not carry the positions over
- assertThat(newEntry.contentType()).isEqualTo(contentType);
- assertThat(newEntry.tracking().status()).isEqualTo(EntryStatus.EXISTING);
- assertThat(newEntry.tracking().deletedPositions()).isNull();
- assertThat(newEntry.tracking().replacedPositions()).isNull();
- assertThat(newEntry.tracking().dvSnapshotId()).isEqualTo(15L);
- }
-
- @Test
- public void buildDataFileFromSource() {
- TrackedFile source = entryWithInheritedSeqNums(sourceData(10L), 45L);
-
- TrackedFile trackedFile = TrackedFileBuilder.from(source, 20L).build();
-
- assertThat(trackedFile.contentType()).isEqualTo(FileContent.DATA);
- assertThat(trackedFile.tracking().status()).isEqualTo(EntryStatus.EXISTING);
- assertThat(trackedFile.tracking().snapshotId()).isEqualTo(source.tracking().snapshotId());
- verifyFieldsAreFromSource(trackedFile, source);
- }
-
- @Test
- public void updateDVWhenBuildingDataFileFromSource() {
- TrackedFile source = entryWithInheritedSeqNums(sourceData(10L), 45L);
-
- DeletionVector dv =
- DeletionVectorStruct.builder()
- .location("s3://bucket/data/new_dv.puffin")
- .offset(5L)
- .sizeInBytes(256L)
- .cardinality(40L)
- .build();
-
- TrackedFile trackedFile = TrackedFileBuilder.from(source, 20L).deletionVector(dv).build();
-
- assertThat(trackedFile.deletionVector()).isNotSameAs(source.deletionVector()).isSameAs(dv);
- assertThat(trackedFile.tracking().status()).isEqualTo(EntryStatus.MODIFIED);
- assertThat(trackedFile.tracking().snapshotId()).isEqualTo(10L);
- assertThat(trackedFile.tracking().dataSequenceNumber()).isEqualTo(45L);
- assertThat(trackedFile.tracking().fileSequenceNumber()).isEqualTo(45L);
- assertThat(trackedFile.tracking().dvSnapshotId()).isEqualTo(20L);
- }
-
- @Test
- public void addingSameDeletionVectorFails() {
- TrackedFile source = entryWithInheritedSeqNums(sourceData(10L), 45L);
-
- DeletionVector dv =
- DeletionVectorStruct.builder()
- .location("s3://bucket/data/new_dv.puffin")
- .offset(5L)
- .sizeInBytes(256L)
- .cardinality(40L)
- .build();
-
- DeletionVector dvCopy = dv.copy();
-
- TrackedFile trackedFile = TrackedFileBuilder.from(source, 20L).deletionVector(dv).build();
-
- assertThatThrownBy(() -> TrackedFileBuilder.from(trackedFile, 30L).deletionVector(dv))
- .isInstanceOf(IllegalArgumentException.class)
- .hasMessage("The same deletion vector already added");
- assertThatThrownBy(() -> TrackedFileBuilder.from(trackedFile, 30L).deletionVector(dvCopy))
- .isInstanceOf(IllegalArgumentException.class)
- .hasMessage("The same deletion vector already added");
- }
-
- private static Stream<Arguments> nonManifestSources() {
- return Stream.of(
- Arguments.of(sourceData(10L), FileContent.DATA),
- Arguments.of(sourceEqualityDelete(10L), FileContent.EQUALITY_DELETES));
- }
-
- private static Stream<Arguments> allSources() {
- return Stream.concat(nonManifestSources(), manifestSources());
- }
-
- @ParameterizedTest
- @MethodSource("allSources")
- public void deletedFromSource(TrackedFile source, FileContent contentType) {
- entryWithInheritedSeqNums(source, 15L);
-
- TrackedFile deleted = TrackedFileBuilder.deleted(source, 20L);
-
- assertThat(deleted.contentType()).isEqualTo(contentType);
- assertThat(deleted.tracking().status()).isEqualTo(EntryStatus.DELETED);
- assertThat(deleted.tracking().snapshotId()).isEqualTo(20L);
- verifyFieldsAreFromSource(deleted, source);
- }
-
- @ParameterizedTest
- @MethodSource("nonManifestSources")
- public void replacedFromNonManifestSource(TrackedFile source, FileContent contentType) {
- entryWithInheritedSeqNums(source, 15L);
-
- TrackedFile replaced = TrackedFileBuilder.replaced(source, 20L);
-
- assertThat(replaced.contentType()).isEqualTo(contentType);
- assertThat(replaced.tracking().status()).isEqualTo(EntryStatus.REPLACED);
- assertThat(replaced.tracking().snapshotId()).isEqualTo(20L);
- verifyFieldsAreFromSource(replaced, source);
- }
-
- @ParameterizedTest
- @MethodSource("manifestSources")
- public void replacedFromManifestSourceFails(TrackedFile source, FileContent contentType) {
- entryWithInheritedSeqNums(source, 15L);
-
- assertThatThrownBy(() -> TrackedFileBuilder.replaced(source, 20L))
- .isInstanceOf(IllegalArgumentException.class)
- .hasMessage(
- "Manifest entries cannot transition to REPLACED, but entry type is: " + contentType);
- }
-
- private static TrackedFile sourceData(long snapshotId) {
- return TrackedFileBuilder.data(snapshotId)
- .formatVersion(FORMAT_VERSION_V4)
- .location("s3://bucket/data/file.parquet")
- .fileFormat(FileFormat.PARQUET)
- .recordCount(2000L)
- .fileSizeInBytes(12345L)
- .partition(PARTITION_DATA)
- .specId(7)
- .contentStats(CONTENT_STATS)
- .sortOrderId(3)
- .deletionVector(DELETION_VECTOR)
- .keyMetadata(KEY_METADATA)
- .splitOffsets(SPLIT_OFFSETS)
- .build();
- }
-
- private static TrackedFile sourceEqualityDelete(long snapshotId) {
- return TrackedFileBuilder.equalityDelete(snapshotId)
- .formatVersion(FORMAT_VERSION_V4)
- .location("s3://bucket/data/eq_delete.parquet")
- .fileFormat(FileFormat.PARQUET)
- .recordCount(2000L)
- .fileSizeInBytes(12345L)
- .partition(PARTITION_DATA)
- .equalityIds(ImmutableList.of(1))
- .build();
- }
-
- private static TrackedFile sourceDataManifest(long snapshotId) {
- return TrackedFileBuilder.dataManifest(snapshotId)
- .formatVersion(FORMAT_VERSION_V4)
- .location("s3://bucket/data/data_manifest.parquet")
- .fileFormat(FileFormat.PARQUET)
- .recordCount(420L)
- .fileSizeInBytes(556L)
- .partition(PARTITION_DATA)
- .manifestInfo(MANIFEST_INFO)
- .build();
- }
-
- private static TrackedFile sourceDeleteManifest(long snapshotId) {
- return TrackedFileBuilder.deleteManifest(snapshotId)
- .formatVersion(FORMAT_VERSION_V4)
- .location("s3://bucket/data/delete_manifest.parquet")
- .fileFormat(FileFormat.PARQUET)
- .recordCount(100L)
- .fileSizeInBytes(543L)
- .partition(PARTITION_DATA)
- .manifestInfo(MANIFEST_INFO)
- .build();
- }
-
- private static TrackedFile entryWithInheritedSeqNums(TrackedFile entry, long sequenceNumber) {
- Tracking manifestTrackingToInheritFrom =
- new TrackingStruct(
- EntryStatus.EXISTING, 123L, sequenceNumber, sequenceNumber, null, null, null, null);
-
- ((TrackingStruct) entry.tracking()).inheritFrom(manifestTrackingToInheritFrom);
- return entry;
- }
-
- /**
- * Verifies that fields in entry are the same as in source. Note, snapshot ID can't be verified
- * here, because based on the entry's status it is either carried over or not.
- */
- private static void verifyFieldsAreFromSource(TrackedFile entry, TrackedFile source) {
- assertThat(entry.formatVersion()).isEqualTo(source.formatVersion());
- assertThat(entry.location()).isEqualTo(source.location());
- assertThat(entry.fileFormat()).isEqualTo(source.fileFormat());
- assertThat(entry.recordCount()).isEqualTo(source.recordCount());
- assertThat(entry.fileSizeInBytes()).isEqualTo(source.fileSizeInBytes());
- assertThat(entry.specId()).isEqualTo(source.specId());
- assertThat(entry.partition()).isSameAs(source.partition());
- assertThat(entry.contentStats()).isSameAs(source.contentStats());
- assertThat(entry.sortOrderId()).isEqualTo(source.sortOrderId());
- assertThat(entry.deletionVector()).isSameAs(source.deletionVector());
- assertThat(entry.keyMetadata()).isEqualTo(source.keyMetadata());
- assertThat(entry.splitOffsets()).isEqualTo(source.splitOffsets());
- assertThat(entry.manifestInfo()).isSameAs(source.manifestInfo());
- assertThat(entry.equalityIds()).isEqualTo(source.equalityIds());
-
- assertThat(entry.tracking().dataSequenceNumber())
- .isEqualTo(source.tracking().dataSequenceNumber());
- assertThat(entry.tracking().fileSequenceNumber())
- .isEqualTo(source.tracking().fileSequenceNumber());
- assertThat(entry.tracking().dvSnapshotId()).isEqualTo(source.tracking().dvSnapshotId());
- assertThat(entry.tracking().firstRowId()).isEqualTo(source.tracking().firstRowId());
- assertThat(entry.tracking().deletedPositions()).isEqualTo(source.tracking().deletedPositions());
- assertThat(entry.tracking().replacedPositions())
- .isEqualTo(source.tracking().replacedPositions());
- }
-}