You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by ze...@apache.org on 2026/07/24 18:08:11 UTC
(iceberg-go) branch main updated: fix(table): cross-check deletion vector cardinality against manifest record_count (#1510)
This is an automated email from the ASF dual-hosted git repository.
zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-go.git
The following commit(s) were added to refs/heads/main by this push:
new 6466d3611 fix(table): cross-check deletion vector cardinality against manifest record_count (#1510)
6466d3611 is described below
commit 6466d3611513daf599ad682f916efe0d5d68daa3
Author: Tanmay Rauth <t_...@apple.com>
AuthorDate: Fri Jul 24 11:08:05 2026 -0700
fix(table): cross-check deletion vector cardinality against manifest record_count (#1510)
ReadDV now validates the decoded bitmap against the manifest entry's
record_count (field 103) and cross-checks it with the puffin blob's
cardinality property when present. record_count is a required
non-nullable long, so it always participates: a manifest zero against a
nonzero puffin property is a real disagreement, not an absent value, and
fails fast.
Blobs missing the puffin property are still validated against
record_count and accepted with a warning. Adds a hand-built fixture to
exercise the missing-property branch, which the puffin writer otherwise
forbids.
Closes #1058
---
table/dv/deletion_vector.go | 67 ++++++++++++-------
table/dv/deletion_vector_test.go | 125 +++++++++++++++++++++++++++++++++---
table/dv_scanner_end_to_end_test.go | 32 ++++-----
table/dv_scanner_read_test.go | 40 ++++++------
4 files changed, 197 insertions(+), 67 deletions(-)
diff --git a/table/dv/deletion_vector.go b/table/dv/deletion_vector.go
index 115caa59d..7eb378a7b 100644
--- a/table/dv/deletion_vector.go
+++ b/table/dv/deletion_vector.go
@@ -132,22 +132,24 @@ func SerializeDV(bitmap *RoaringPositionBitmap) ([]byte, error) {
// ReadDV reads a deletion vector from a puffin file using the manifest entry metadata.
// ContentOffset and ContentSizeInBytes must be set on the DataFile (required by v3 spec).
//
-// When the puffin blob carries a `cardinality` property (spec-mandated for
-// deletion-vector-v1), its value is parsed and used to validate the decoded
-// bitmap — this catches truncated or partially-overwritten blobs whose CRC
-// still validates over the bytes that are present.
+// The decoded bitmap's cardinality is cross-validated against two independent
+// sources, so a truncated or partially-overwritten blob whose CRC still
+// validates over the bytes that are present is rejected:
//
-// Java's BitmapPositionDeleteIndex validates against the manifest entry's
-// `record_count` field rather than the puffin blob property; the two are
-// always set to the same value by Java's writer, so this PR's behavior agrees
-// with Java for spec-conformant tables. A future change should cross-validate
-// both sources when they're independently available (tracked separately).
+// - The manifest entry's record_count (dvFile.Count()). This is field 103, a
+// required non-nullable long, so it is always available — zero means an
+// empty deletion vector, not "unknown". Java's BitmapPositionDeleteIndex
+// validates against this value, so it is our primary expected cardinality.
+// - The puffin blob's spec-mandated `cardinality` property, when present.
//
-// Blobs missing the spec-required cardinality property are accepted with a
-// slog warning rather than rejected — strict enforcement is deferred until
-// the Go DV writer guarantees the property is always present. The per-byte
-// CRC check in DeserializeDV still applies, so missing-property is degraded
-// integrity, not absent integrity.
+// When both sources are available they must agree; a disagreement (e.g. a
+// stale manifest record_count against a freshly written blob) is a writer bug
+// and fails fast. The bitmap is then validated against the manifest count.
+//
+// Blobs missing the spec-required cardinality property are still validated
+// against the manifest record_count and accepted with a slog warning rather
+// than rejected — the Go writer always emits the property, but third-party
+// writers may not, and the per-byte CRC check in DeserializeDV still applies.
func ReadDV(fs iceio.IO, dvFile iceberg.DataFile) (*RoaringPositionBitmap, error) {
if dvFile.FileFormat() != iceberg.PuffinFile {
return nil, fmt.Errorf("expected PUFFIN format for deletion vector, got %s", dvFile.FileFormat())
@@ -179,21 +181,40 @@ func ReadDV(fs iceio.IO, dvFile iceberg.DataFile) (*RoaringPositionBitmap, error
return nil, fmt.Errorf("read DV blob at offset %d: %w", offset, err)
}
- cardinality, ok, err := blobCardinality(reader.Blobs(), offset, size)
+ // Manifest record_count (field 103) is a required, non-nullable long, so it
+ // is always present for a DV data file: zero means an empty deletion
+ // vector, not "unknown". This is Java's primary expected cardinality.
+ manifestCardinality := dvFile.Count()
+
+ // The puffin blob independently declares its cardinality via the spec-
+ // mandated property; when present it is a second source to cross-check.
+ puffinCardinality, hasPuffinCardinality, err := blobCardinality(reader.Blobs(), offset, size)
if err != nil {
return nil, fmt.Errorf("DV file %s: %w", dvFile.FilePath(), err)
}
- if !ok {
- // Spec deviation: deletion-vector-v1 MUST carry a cardinality
- // property. We tolerate the absence to keep reading from third-
- // party writers that emit non-conformant files; flag so operators
- // can identify affected tables.
- slog.Warn("DV blob missing spec-required cardinality property; skipping cardinality validation",
+
+ // When both sources are available they must agree. A disagreement means a
+ // writer set the manifest record_count and the blob property inconsistently
+ // (e.g. a stale record_count after an incremental merge against a freshly
+ // written blob) — fail fast rather than silently trusting one over the other.
+ if hasPuffinCardinality && manifestCardinality != puffinCardinality {
+ return nil, fmt.Errorf("DV file %s: manifest record_count %d disagrees with puffin cardinality property %d",
+ dvFile.FilePath(), manifestCardinality, puffinCardinality)
+ }
+
+ if !hasPuffinCardinality {
+ // Spec deviation: deletion-vector-v1 MUST carry a cardinality property.
+ // The manifest record_count still bounds the decoded bitmap below, so
+ // this is degraded (not absent) validation; flag it so operators can
+ // spot non-conformant third-party writers.
+ slog.Warn("DV blob missing spec-required cardinality property; validating against manifest record_count only",
"dv_file", dvFile.FilePath(), "offset", offset)
- cardinality = -1
}
- return DeserializeDV(blobData, cardinality)
+ // Validate the decoded bitmap against the manifest record_count (always
+ // present, including zero). When the puffin property is present it has
+ // already been confirmed to agree with this value above.
+ return DeserializeDV(blobData, manifestCardinality)
}
// blobCardinality returns the cardinality declared by the puffin blob at the
diff --git a/table/dv/deletion_vector_test.go b/table/dv/deletion_vector_test.go
index 34d3a12c6..945447fb4 100644
--- a/table/dv/deletion_vector_test.go
+++ b/table/dv/deletion_vector_test.go
@@ -18,7 +18,9 @@
package dv
import (
+ "bytes"
"encoding/binary"
+ "encoding/json"
"errors"
"hash/crc32"
"os"
@@ -130,6 +132,50 @@ func writePuffinWithDVBlobAndProps(t *testing.T, dir string, dvBlobBytes []byte,
return path, meta
}
+// writeRawPuffinWithDVBlobNoCardinality hand-builds a Puffin file whose DV blob
+// deliberately omits the spec-required `cardinality` property. The in-tree
+// puffin.Writer rejects such blobs at AddBlob time, so the only way to exercise
+// ReadDV's missing-property branch (reachable via a non-conformant third-party
+// writer) is to assemble the bytes directly. The reader's validateBlobs does
+// not require the property, so this file loads cleanly.
+func writeRawPuffinWithDVBlobNoCardinality(t *testing.T, dir string, dvBlobBytes []byte) (string, puffin.BlobMetadata) {
+ t.Helper()
+
+ const puffinMagic = "PFA1"
+ meta := puffin.BlobMetadata{
+ Type: puffin.BlobTypeDeletionVector,
+ Fields: []int32{2147483546},
+ SnapshotID: -1,
+ SequenceNumber: -1,
+ Offset: int64(puffin.MagicSize),
+ Length: int64(len(dvBlobBytes)),
+ Properties: map[string]string{
+ // No "cardinality" — that is the whole point of this fixture.
+ "referenced-data-file": "s3://bucket/data/data-001.parquet",
+ },
+ }
+
+ payload, err := json.Marshal(puffin.Footer{Blobs: []puffin.BlobMetadata{meta}})
+ require.NoError(t, err)
+
+ var buf bytes.Buffer
+ buf.WriteString(puffinMagic) // header magic
+ buf.Write(dvBlobBytes) // blob at offset MagicSize
+ buf.WriteString(puffinMagic) // footer start magic
+ buf.Write(payload) // footer JSON payload
+
+ var trailer [12]byte // PayloadSize(4) + Flags(4) + Magic(4)
+ binary.LittleEndian.PutUint32(trailer[0:4], uint32(len(payload)))
+ binary.LittleEndian.PutUint32(trailer[4:8], 0) // flags = 0 (uncompressed)
+ copy(trailer[8:12], puffinMagic)
+ buf.Write(trailer[:])
+
+ path := filepath.Join(dir, "raw-dv-no-cardinality.puffin")
+ require.NoError(t, os.WriteFile(path, buf.Bytes(), 0o644))
+
+ return path, meta
+}
+
func newDVTestFile(path string, count int64, offset, size *int64) *mockDVFile {
return &mockDVFile{
path: path,
@@ -413,35 +459,94 @@ func TestReadDVCardinalityValidation(t *testing.T) {
assert.Equal(t, int64(5), bm.Cardinality())
})
- t.Run("mismatched cardinality is rejected", func(t *testing.T) {
+ t.Run("empty deletion vector with zero cardinality passes", func(t *testing.T) {
+ // record_count is a required non-nullable long, so an empty DV
+ // carries a legitimate zero. Manifest 0, puffin 0, and an empty
+ // bitmap all agree and must be accepted, not treated as "absent".
+ emptyBlob := readDVTestData(t, "empty-position-index.bin")
+ dir := t.TempDir()
+ path, meta := writePuffinWithDVBlobAndProps(t, dir, emptyBlob, map[string]string{
+ "referenced-data-file": "s3://bucket/data/data-001.parquet",
+ "cardinality": "0",
+ })
+ offset, size := meta.Offset, meta.Length
+ bm, err := ReadDV(iceio.LocalFS{}, newDVTestFile(path, 0, &offset, &size))
+ require.NoError(t, err)
+ assert.Equal(t, int64(0), bm.Cardinality())
+ })
+
+ t.Run("manifest disagreeing with puffin property is rejected", func(t *testing.T) {
dir := t.TempDir()
path, meta := writePuffinWithDVBlobAndProps(t, dir, dvBlobBytes, map[string]string{
"referenced-data-file": "s3://bucket/data/data-001.parquet",
- // Bitmap actually has 5 positions; claim 99.
+ // Puffin says 99; the manifest entry below says 5.
"cardinality": "99",
})
offset, size := meta.Offset, meta.Length
_, err := ReadDV(iceio.LocalFS{}, newDVTestFile(path, 5, &offset, &size))
require.Error(t, err)
- // Pin the specific error path: DeserializeDV's "cardinality
- // mismatch", not the helper's "invalid cardinality" parse error.
+ // Cross-check fires before DeserializeDV: the two metadata sources
+ // disagree, so we never reach the bitmap "cardinality mismatch" path.
+ assert.Contains(t, err.Error(), "manifest record_count 5 disagrees with puffin cardinality property 99")
+ })
+
+ t.Run("manifest zero disagreeing with nonzero puffin property is rejected", func(t *testing.T) {
+ // Regression for the record_count==0 trap: record_count is a required
+ // non-nullable long, so a manifest zero against a nonzero puffin
+ // property is a real disagreement, not "manifest cardinality absent".
+ // Must error rather than fall back to trusting the blob property.
+ dir := t.TempDir()
+ path, meta := writePuffinWithDVBlobAndProps(t, dir, dvBlobBytes, map[string]string{
+ "referenced-data-file": "s3://bucket/data/data-001.parquet",
+ "cardinality": "5",
+ })
+ offset, size := meta.Offset, meta.Length
+ _, err := ReadDV(iceio.LocalFS{}, newDVTestFile(path, 0, &offset, &size))
+ require.Error(t, err)
+ assert.Contains(t, err.Error(), "manifest record_count 0 disagrees with puffin cardinality property 5")
+ })
+
+ t.Run("agreeing metadata but wrong bitmap is rejected", func(t *testing.T) {
+ // Both sources agree on 99, but the decoded bitmap holds 5 positions.
+ // The cross-check passes, then DeserializeDV catches the bitmap
+ // against the agreed-upon expected cardinality.
+ dir := t.TempDir()
+ path, meta := writePuffinWithDVBlobAndProps(t, dir, dvBlobBytes, map[string]string{
+ "referenced-data-file": "s3://bucket/data/data-001.parquet",
+ "cardinality": "99",
+ })
+ offset, size := meta.Offset, meta.Length
+ _, err := ReadDV(iceio.LocalFS{}, newDVTestFile(path, 99, &offset, &size))
+ require.Error(t, err)
assert.Contains(t, err.Error(), "cardinality mismatch")
})
- t.Run("missing property is tolerated with a warning", func(t *testing.T) {
+ t.Run("missing property falls back to manifest record_count", func(t *testing.T) {
// No `cardinality` property — spec deviation tolerated for read
- // compatibility with third-party writers that omit it. ReadDV logs
- // a warning (see slog.Warn) and falls back to CRC-only validation
- // inside DeserializeDV. Strict enforcement is deferred until the
- // Go writer-side coverage lands.
+ // compatibility with third-party writers that omit it. ReadDV logs a
+ // warning and validates the bitmap against the manifest record_count
+ // (5), which matches, so the read succeeds.
dir := t.TempDir()
- path, meta := writePuffinWithDVBlob(t, dir, dvBlobBytes)
+ path, meta := writeRawPuffinWithDVBlobNoCardinality(t, dir, dvBlobBytes)
offset, size := meta.Offset, meta.Length
bm, err := ReadDV(iceio.LocalFS{}, newDVTestFile(path, 5, &offset, &size))
require.NoError(t, err)
assert.Equal(t, int64(5), bm.Cardinality())
})
+ t.Run("missing property still validates against manifest record_count", func(t *testing.T) {
+ // Property absent and the manifest record_count (3) disagrees with the
+ // decoded bitmap (5). With no puffin property to cross-check, the
+ // manifest count is the sole expected cardinality and must still be
+ // enforced by DeserializeDV.
+ dir := t.TempDir()
+ path, meta := writeRawPuffinWithDVBlobNoCardinality(t, dir, dvBlobBytes)
+ offset, size := meta.Offset, meta.Length
+ _, err := ReadDV(iceio.LocalFS{}, newDVTestFile(path, 3, &offset, &size))
+ require.Error(t, err)
+ assert.Contains(t, err.Error(), "cardinality mismatch")
+ })
+
// Note: the previous "malformed property is rejected" and "negative
// cardinality rejected on read" subtests were removed once the puffin
// writer started rejecting non-numeric and negative cardinality values
diff --git a/table/dv_scanner_end_to_end_test.go b/table/dv_scanner_end_to_end_test.go
index cba4d7938..07338fb7e 100644
--- a/table/dv_scanner_end_to_end_test.go
+++ b/table/dv_scanner_end_to_end_test.go
@@ -189,8 +189,8 @@ func TestDVScanEndToEnd(t *testing.T) {
dataPath := filepath.Join(tmp, "data-2.parquet")
df := writeIntParquetWithFieldID(t, fs, dataPath, 0, 10)
- puffinPath, offset, length := writeDVPuffinFixture(t, []uint64{1, 3, 5, 7, 9}, dataPath)
- dvFile := newDVMockDataFile(puffinPath, dataPath, offset, length)
+ puffinPath, offset, length, card := writeDVPuffinFixture(t, []uint64{1, 3, 5, 7, 9}, dataPath)
+ dvFile := newDVMockDataFile(puffinPath, dataPath, offset, length, card)
got := collectDVScanRows(t, tbl.Scan(), []FileScanTask{{
File: df,
@@ -205,8 +205,8 @@ func TestDVScanEndToEnd(t *testing.T) {
dataPath := filepath.Join(tmp, "data-3.parquet")
df := writeIntParquetWithFieldID(t, fs, dataPath, 0, 5)
- puffinPath, offset, length := writeDVPuffinFixture(t, []uint64{0, 4}, dataPath)
- dvFile := newDVMockDataFile(puffinPath, dataPath, offset, length)
+ puffinPath, offset, length, card := writeDVPuffinFixture(t, []uint64{0, 4}, dataPath)
+ dvFile := newDVMockDataFile(puffinPath, dataPath, offset, length, card)
got := collectDVScanRows(t, tbl.Scan(), []FileScanTask{{
File: df,
@@ -221,8 +221,8 @@ func TestDVScanEndToEnd(t *testing.T) {
dataPath := filepath.Join(tmp, "data-4.parquet")
df := writeIntParquetWithFieldID(t, fs, dataPath, 0, 5)
- puffinPath, offset, length := writeDVPuffinFixture(t, []uint64{0, 1, 2, 3, 4}, dataPath)
- dvFile := newDVMockDataFile(puffinPath, dataPath, offset, length)
+ puffinPath, offset, length, card := writeDVPuffinFixture(t, []uint64{0, 1, 2, 3, 4}, dataPath)
+ dvFile := newDVMockDataFile(puffinPath, dataPath, offset, length, card)
got := collectDVScanRows(t, tbl.Scan(), []FileScanTask{{
File: df,
@@ -242,8 +242,8 @@ func TestDVScanEndToEnd(t *testing.T) {
dataPath := filepath.Join(tmp, "data-5.parquet")
df := writeIntParquetWithFieldID(t, fs, dataPath, 0, 5)
- puffinPath, offset, length := writeDVPuffinFixture(t, nil, dataPath)
- dvFile := newDVMockDataFile(puffinPath, dataPath, offset, length)
+ puffinPath, offset, length, card := writeDVPuffinFixture(t, nil, dataPath)
+ dvFile := newDVMockDataFile(puffinPath, dataPath, offset, length, card)
got := collectDVScanRows(t, tbl.Scan(), []FileScanTask{{
File: df,
@@ -272,15 +272,15 @@ func TestDVScanEndToEnd(t *testing.T) {
dataPathB := filepath.Join(tmp, "data-6b.parquet")
dfA := writeIntParquetWithFieldID(t, fs, dataPathA, 0, 5)
dfB := writeIntParquetWithFieldID(t, fs, dataPathB, 10, 5)
- puffinA, offA, lenA := writeDVPuffinFixture(t, []uint64{1, 3}, dataPathA)
- puffinB, offB, lenB := writeDVPuffinFixture(t, []uint64{0, 2}, dataPathB)
+ puffinA, offA, lenA, cardA := writeDVPuffinFixture(t, []uint64{1, 3}, dataPathA)
+ puffinB, offB, lenB, cardB := writeDVPuffinFixture(t, []uint64{0, 2}, dataPathB)
tasks := []FileScanTask{
{File: dfA, DeletionVectorFiles: []iceberg.DataFile{
- newDVMockDataFile(puffinA, dataPathA, offA, lenA),
+ newDVMockDataFile(puffinA, dataPathA, offA, lenA, cardA),
}},
{File: dfB, DeletionVectorFiles: []iceberg.DataFile{
- newDVMockDataFile(puffinB, dataPathB, offB, lenB),
+ newDVMockDataFile(puffinB, dataPathB, offB, lenB, cardB),
}},
}
got := collectDVScanRows(t, tbl.Scan(), tasks)
@@ -302,8 +302,8 @@ func TestDVScanEndToEnd(t *testing.T) {
dataPath := filepath.Join(tmp, "data-7.parquet")
df := writeIntParquetWithFieldID(t, fs, dataPath, 0, 5)
- puffinPath, offset, length := writeDVPuffinFixture(t, []uint64{100}, dataPath)
- dvFile := newDVMockDataFile(puffinPath, dataPath, offset, length)
+ puffinPath, offset, length, card := writeDVPuffinFixture(t, []uint64{100}, dataPath)
+ dvFile := newDVMockDataFile(puffinPath, dataPath, offset, length, card)
got := collectDVScanRows(t, tbl.Scan(), []FileScanTask{{
File: df,
@@ -323,8 +323,8 @@ func TestDVScanEndToEnd(t *testing.T) {
dataPath := filepath.Join(tmp, "data-8.parquet")
df := writeIntParquetWithFieldID(t, fs, dataPath, 0, 5)
- puffinPath, offset, length := writeDVPuffinFixture(t, []uint64{1, 100}, dataPath)
- dvFile := newDVMockDataFile(puffinPath, dataPath, offset, length)
+ puffinPath, offset, length, card := writeDVPuffinFixture(t, []uint64{1, 100}, dataPath)
+ dvFile := newDVMockDataFile(puffinPath, dataPath, offset, length, card)
got := collectDVScanRows(t, tbl.Scan(), []FileScanTask{{
File: df,
diff --git a/table/dv_scanner_read_test.go b/table/dv_scanner_read_test.go
index 00c8c0ec1..499acbb50 100644
--- a/table/dv_scanner_read_test.go
+++ b/table/dv_scanner_read_test.go
@@ -51,7 +51,7 @@ import (
//
// TODO(#1041): replace the hand-built envelope with a dv.SerializeDV helper
// once that PR exports one. Today the dv package only exports the read side.
-func writeDVPuffinFixture(t *testing.T, positions []uint64, referencedDataFile string) (path string, offset, length int64) {
+func writeDVPuffinFixture(t *testing.T, positions []uint64, referencedDataFile string) (path string, offset, length, cardinality int64) {
t.Helper()
bitmap := dv.NewRoaringPositionBitmap()
@@ -91,20 +91,24 @@ func writeDVPuffinFixture(t *testing.T, positions []uint64, referencedDataFile s
path = filepath.Join(t.TempDir(), "dv.puffin")
require.NoError(t, os.WriteFile(path, puffinBuf.Bytes(), 0o644))
- return path, blobMeta.Offset, blobMeta.Length
+ return path, blobMeta.Offset, blobMeta.Length, bitmap.Cardinality()
}
// newDVMockDataFile builds a manifest-entry-shaped mock for a DV puffin blob.
-// FileSizeBytes is intentionally left at zero (mockDataFile's default): the
-// scanner read path consults ContentOffset / ContentSizeInBytes only, and
-// setting filesize to the blob length (rather than the puffin file size)
-// invites a misleading test fixture.
-func newDVMockDataFile(puffinPath, referencedDataFile string, offset, contentSize int64) *dvMockDataFile {
+// recordCount is the manifest entry's record_count (field 103): for a DV it
+// equals the blob cardinality, and ReadDV cross-checks the two, so callers
+// pass the cardinality returned by writeDVPuffinFixture. FileSizeBytes is
+// intentionally left at zero (mockDataFile's default): the scanner read path
+// consults ContentOffset / ContentSizeInBytes only, and setting filesize to
+// the blob length (rather than the puffin file size) invites a misleading
+// test fixture.
+func newDVMockDataFile(puffinPath, referencedDataFile string, offset, contentSize, recordCount int64) *dvMockDataFile {
return &dvMockDataFile{
mockDataFile: mockDataFile{
path: puffinPath,
contentType: iceberg.EntryContentPosDeletes,
format: iceberg.PuffinFile,
+ count: recordCount,
},
referencedDataFile: strPtr(referencedDataFile),
contentOffset: int64Ptr(offset),
@@ -137,10 +141,10 @@ func TestReadAllDeletionVectors(t *testing.T) {
t.Run("decodes positions and keys by referenced data file", func(t *testing.T) {
const dataFilePath = "file:///table/data/data-001.parquet"
- puffinPath, offset, length := writeDVPuffinFixture(t, []uint64{1, 3, 5, 7, 9}, dataFilePath)
+ puffinPath, offset, length, card := writeDVPuffinFixture(t, []uint64{1, 3, 5, 7, 9}, dataFilePath)
tasks := []FileScanTask{{DeletionVectorFiles: []iceberg.DataFile{
- newDVMockDataFile(puffinPath, dataFilePath, offset, length),
+ newDVMockDataFile(puffinPath, dataFilePath, offset, length, card),
}}}
got, err := readAllDeletionVectors(ctx, fs, tasks, 1)
@@ -162,8 +166,8 @@ func TestReadAllDeletionVectors(t *testing.T) {
t.Run("dedups identical DV referenced by two tasks", func(t *testing.T) {
const dataFilePath = "file:///table/data/data-002.parquet"
- puffinPath, offset, length := writeDVPuffinFixture(t, []uint64{2, 4}, dataFilePath)
- dvFile := newDVMockDataFile(puffinPath, dataFilePath, offset, length)
+ puffinPath, offset, length, card := writeDVPuffinFixture(t, []uint64{2, 4}, dataFilePath)
+ dvFile := newDVMockDataFile(puffinPath, dataFilePath, offset, length, card)
tasks := []FileScanTask{
{DeletionVectorFiles: []iceberg.DataFile{dvFile}},
@@ -186,7 +190,7 @@ func TestReadAllDeletionVectors(t *testing.T) {
})
t.Run("DV missing referenced_data_file is rejected", func(t *testing.T) {
- puffinPath, offset, length := writeDVPuffinFixture(t, []uint64{0}, "file:///placeholder.parquet")
+ puffinPath, offset, length, _ := writeDVPuffinFixture(t, []uint64{0}, "file:///placeholder.parquet")
dvFile := &dvMockDataFile{
mockDataFile: mockDataFile{
@@ -232,12 +236,12 @@ func TestReadAllDeletionVectors(t *testing.T) {
// right place to catch the case where two blobs share a puffin file
// but live at different content offsets — sameDVBlob compares both.
const dataFilePath = "file:///table/data/data-003.parquet"
- puffinA, offsetA, lengthA := writeDVPuffinFixture(t, []uint64{1, 2}, dataFilePath)
- puffinB, offsetB, lengthB := writeDVPuffinFixture(t, []uint64{3, 4}, dataFilePath)
+ puffinA, offsetA, lengthA, cardA := writeDVPuffinFixture(t, []uint64{1, 2}, dataFilePath)
+ puffinB, offsetB, lengthB, cardB := writeDVPuffinFixture(t, []uint64{3, 4}, dataFilePath)
tasks := []FileScanTask{{DeletionVectorFiles: []iceberg.DataFile{
- newDVMockDataFile(puffinA, dataFilePath, offsetA, lengthA),
- newDVMockDataFile(puffinB, dataFilePath, offsetB, lengthB),
+ newDVMockDataFile(puffinA, dataFilePath, offsetA, lengthA, cardA),
+ newDVMockDataFile(puffinB, dataFilePath, offsetB, lengthB, cardB),
}}}
_, err := readAllDeletionVectors(ctx, fs, tasks, 1)
@@ -256,8 +260,8 @@ func TestReadAllDeletionVectors(t *testing.T) {
// nil-ref entry in the input, the call returns a clean error and
// does not panic, regardless of how concurrency is configured.
const dataFilePath = "file:///table/data/data-004.parquet"
- puffinPath, offset, length := writeDVPuffinFixture(t, []uint64{42}, dataFilePath)
- valid := newDVMockDataFile(puffinPath, dataFilePath, offset, length)
+ puffinPath, offset, length, card := writeDVPuffinFixture(t, []uint64{42}, dataFilePath)
+ valid := newDVMockDataFile(puffinPath, dataFilePath, offset, length, card)
broken := &dvMockDataFile{
mockDataFile: mockDataFile{
path: "/nonexistent.puffin",