You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by at...@apache.org on 2026/06/24 12:57:20 UTC
(iceberg-go) branch main updated: fix(v3): re-enable row-group pruning with correct row lineage tracking (#1250)
This is an automated email from the ASF dual-hosted git repository.
laskoviymishka pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-go.git
The following commit(s) were added to refs/heads/main by this push:
new 98415485 fix(v3): re-enable row-group pruning with correct row lineage tracking (#1250)
98415485 is described below
commit 9841548519315ddc9689bc2412359d389264094e
Author: Tobias Pütz <to...@minio.io>
AuthorDate: Wed Jun 24 14:57:14 2026 +0200
fix(v3): re-enable row-group pruning with correct row lineage tracking (#1250)
Follow-up to #1246 and #1237
Re-enable row-group pruning while correctly tracking row lineage & add
adequate tests.
---------
Signed-off-by: Tobias Pütz <to...@minio.io>
---
table/arrow_scanner.go | 266 ++++++++++++++++-------
table/arrow_scanner_posdelete_regression_test.go | 2 +-
table/arrow_scanner_test.go | 2 +-
table/dv_scanner_read_test.go | 33 ++-
table/internal/parquet_files.go | 49 ++++-
table/internal/parquet_files_test.go | 11 +
table/mor_delete_pruning_test.go | 6 +-
table/row_lineage_prune_delete_test.go | 260 ++++++++++++++++++++++
table/scanner_internal_test.go | 70 +++++-
table/variant_shredded_posdelete_test.go | 2 +-
10 files changed, 597 insertions(+), 104 deletions(-)
diff --git a/table/arrow_scanner.go b/table/arrow_scanner.go
index 146c7f22..3643de66 100644
--- a/table/arrow_scanner.go
+++ b/table/arrow_scanner.go
@@ -432,19 +432,19 @@ func readDeletes(ctx context.Context, fs iceio.IO, dataFile iceberg.DataFile) (_
type set[T comparable] map[T]struct{}
// combinePositionalDeletes builds the surviving-row index list for a single record
-// batch. The deletes set holds file-relative (global) positions, and [start, end) is
-// the global position span the batch covers, so the indices are rebased to batch-local
-// coordinates (i-start): they index into the batch handed to compute.Take, not into the
-// whole file. Without the rebase, the second and later batches of a file would pass
-// indices >= the batch length and compute.Take would fail with "index error: N out of
-// bounds".
-func combinePositionalDeletes(mem memory.Allocator, deletes set[int64], start, end int64) arrow.Array {
+// batch. The deletes set holds file-relative (global) positions; cursor maps each
+// batch row to its original file position (jumping across pruned row groups), and
+// the appended indices are batch-local (the loop counter): they index into the
+// batch handed to compute.Take, not into the whole file. Without batch-local
+// indices, the second and later batches of a file would pass indices >= the batch
+// length and compute.Take would fail with "index error: N out of bounds".
+func combinePositionalDeletes(mem memory.Allocator, deletes set[int64], cursor *rowPositionCursor, nrows int64) arrow.Array {
bldr := array.NewInt64Builder(mem)
defer bldr.Release()
- for i := start; i < end; i++ {
- if _, ok := deletes[i]; !ok {
- bldr.Append(i - start)
+ for i := range nrows {
+ if _, ok := deletes[cursor.next()]; !ok {
+ bldr.Append(i)
}
}
@@ -453,16 +453,13 @@ func combinePositionalDeletes(mem memory.Allocator, deletes set[int64], start, e
type recProcessFn func(arrow.RecordBatch) (arrow.RecordBatch, error)
-func processPositionalDeletes(ctx context.Context, deletes set[int64]) recProcessFn {
- nextIdx, mem := int64(0), compute.GetAllocator(ctx)
+func processPositionalDeletes(ctx context.Context, deletes set[int64], cursor *rowPositionCursor) recProcessFn {
+ mem := compute.GetAllocator(ctx)
return func(r arrow.RecordBatch) (arrow.RecordBatch, error) {
defer r.Release()
- currentIdx := nextIdx
- nextIdx += r.NumRows()
-
- indices := combinePositionalDeletes(mem, deletes, currentIdx, nextIdx)
+ indices := combinePositionalDeletes(mem, deletes, cursor, r.NumRows())
defer indices.Release()
out, err := compute.Take(ctx, *compute.DefaultTakeOptions(),
@@ -485,38 +482,67 @@ func processPositionalDeletes(ctx context.Context, deletes set[int64]) recProces
// (GC-friendly) and shared across every per-batch Boolean array; each
// array.NewBoolean / array.NewSlice pair is released after the batch.
//
-// rowCount bounds the mask to the data file's row count. The closure-captured
-// nextIdx tracks absolute position across batches, mirroring
-// processPositionalDeletes.
-func filterByDeletionVector(ctx context.Context, bitmap *dv.RoaringPositionBitmap, rowCount int64) recProcessFn {
+// rowCount bounds the mask to the data file's row count. cursor maps each batch
+// row to its original file position: when no row group was pruned the positions
+// are contiguous and the mask is sliced zero-copy; when pruning skipped groups
+// the emitted rows are non-contiguous, so the keep bits are gathered per row at
+// the cursor's positions.
+func filterByDeletionVector(ctx context.Context, bitmap *dv.RoaringPositionBitmap, rowCount int64, cursor *rowPositionCursor) recProcessFn {
nextIdx := int64(0)
keepBits := bitmap.KeepMaskBytes(rowCount)
buf := memory.NewBufferBytes(keepBits)
+ mem := compute.GetAllocator(ctx)
return func(r arrow.RecordBatch) (arrow.RecordBatch, error) {
defer r.Release()
- currentIdx := nextIdx
- nextIdx += r.NumRows()
+ nrows := r.NumRows()
+
+ if !cursor.src.pruned() {
+ currentIdx := nextIdx
+ nextIdx += nrows
+
+ // Wrap (and slice) the shared keep-mask buffer for this batch.
+ // array.NewSlice on a Boolean array tracks the bit-level offset,
+ // so we don't need byte-aligned slicing — currentIdx can land
+ // anywhere within a byte.
+ full := array.NewBoolean(int(rowCount), buf, nil, 0)
+ defer full.Release()
+ sliced := array.NewSlice(full, currentIdx, nextIdx).(*array.Boolean)
+ defer sliced.Release()
+
+ return compute.FilterRecordBatch(ctx, r, sliced, compute.DefaultFilterOptions())
+ }
- // Wrap (and slice) the shared keep-mask buffer for this batch.
- // array.NewSlice on a Boolean array tracks the bit-level offset,
- // so we don't need byte-aligned slicing — currentIdx can land
- // anywhere within a byte.
- full := array.NewBoolean(int(rowCount), buf, nil, 0)
- defer full.Release()
- sliced := array.NewSlice(full, currentIdx, nextIdx).(*array.Boolean)
- defer sliced.Release()
+ bldr := array.NewBooleanBuilder(mem)
+ defer bldr.Release()
+ bldr.Reserve(int(nrows))
+ for range nrows {
+ pos := cursor.next()
+ // keepBits is sized for rowCount; a pos past it means row-group
+ // metadata and the manifest's File.Count() disagree. Keep the row
+ // rather than indexing out of bounds.
+ if pos >= rowCount {
+ bldr.Append(true)
- return compute.FilterRecordBatch(ctx, r, sliced, compute.DefaultFilterOptions())
+ continue
+ }
+ // Test keep bit at absolute position pos: byte pos/8, bit pos%8,
+ // LSB-first (the layout the fast path's array.NewBoolean reads).
+ bldr.Append(keepBits[pos>>3]&(1<<(uint(pos)&7)) != 0)
+ }
+ mask := bldr.NewBooleanArray()
+ defer mask.Release()
+
+ return compute.FilterRecordBatch(ctx, r, mask, compute.DefaultFilterOptions())
}
}
// enrichRecordsWithPosDeleteFields enriches a RecordBatch with the columns declared in the PositionalDeleteArrowSchema
// so that during the pipeline filtering stages that sheds filtered out records, we still have a way to
// preserve the original position of those records.
-func enrichRecordsWithPosDeleteFields(ctx context.Context, filePath iceberg.DataFile) recProcessFn {
- nextIdx, mem := int64(0), compute.GetAllocator(ctx)
+func enrichRecordsWithPosDeleteFields(ctx context.Context, filePath iceberg.DataFile, cursor *rowPositionCursor) recProcessFn {
+ mem := compute.GetAllocator(ctx)
return func(inData arrow.RecordBatch) (outData arrow.RecordBatch, err error) {
defer inData.Release()
@@ -537,12 +563,9 @@ func enrichRecordsWithPosDeleteFields(ctx context.Context, filePath iceberg.Data
filePathBldr, posBldr := rb.Field(0).(*array.StringBuilder), rb.Field(1).(*array.Int64Builder)
- startPos := nextIdx
- nextIdx += inData.NumRows()
-
- for i := startPos; i < nextIdx; i++ {
+ for range inData.NumRows() {
filePathBldr.Append(filePath.FilePath())
- posBldr.Append(i)
+ posBldr.Append(cursor.next())
}
newCols := rb.NewRecordBatch()
@@ -721,18 +744,78 @@ func fieldIndexByID(schema *arrow.Schema, fieldID int) int {
return -1
}
+// rowPositionSource holds the surviving row groups for a single file read.
+// processRecords populates spans (via the row-group tester) before any record is
+// read, then every position-keyed pipeline step takes its own cursor over the
+// shared, immutable spans. One source, many cursors: each step advances
+// independently but resolves the same original file positions.
+//
+// With no spans recorded (non-Parquet files, or a read with no pruning pass) the
+// cursors fall back to a contiguous counter from zero.
+type rowPositionSource struct {
+ spans []internal.RowGroupSpan
+}
+
+func (s *rowPositionSource) cursor() *rowPositionCursor {
+ return &rowPositionCursor{src: s}
+}
+
+// pruned reports whether any row group was skipped, i.e. whether emitted rows
+// are non-contiguous. Read only after spans are populated.
+func (s *rowPositionSource) pruned() bool {
+ return len(s.spans) > 0
+}
+
+// rowPositionCursor maps each emitted row back to its original position in the
+// data file. Surviving row groups are visited in file order, so positions jump
+// across the gaps left by pruned groups, keeping _row_id = first_row_id +
+// original position correct even while row-group pruning is active.
+type rowPositionCursor struct {
+ src *rowPositionSource
+ spanIdx int
+ consumed int64
+}
+
+// next returns the original file position of the next row. It is meant to be
+// called once per physical row read, in order. spanIdx is bounded to the last
+// span, so reading past the total row count degrades to positions beyond the
+// final span rather than panicking here. Callers that index a buffer sized to
+// the file's row count (e.g. filterByDeletionVector's keep-mask) must still
+// guard against the returned position exceeding that count.
+func (c *rowPositionCursor) next() int64 {
+ spans := c.src.spans
+ if len(spans) == 0 {
+ pos := c.consumed
+ c.consumed++
+
+ return pos
+ }
+
+ for c.spanIdx < len(spans)-1 && c.consumed >= spans[c.spanIdx].NumRows {
+ c.spanIdx++
+ c.consumed = 0
+ }
+
+ span := spans[c.spanIdx]
+ pos := span.FirstRowPos + c.consumed
+ c.consumed++
+
+ return pos
+}
+
// synthesizeRowLineageColumns fills the requested row-lineage columns from task
// constants: per the v3 spec a null value inherits first_row_id + position /
// data_sequence_number, a non-null value is kept. A column absent from the batch
// is appended. The caller sets synthesizeRowID/synthesizeSeq only when the
// matching task constant is present.
//
-// MUST run before any row-dropping step: _row_id is first_row_id + the row's ORIGINAL
-// position, so rowOffset (advanced by the full batch) is only correct before rows are
-// dropped. ToRequestedSchema then resolves the columns by reserved field id.
+// MUST run before any row-dropping step: _row_id is first_row_id + the row's
+// ORIGINAL position, drawn from cursor (advanced once per row in the full batch),
+// so it is only correct before rows are dropped. ToRequestedSchema then resolves
+// the columns by reserved field id.
func synthesizeRowLineageColumns(
ctx context.Context,
- rowOffset *int64,
+ cursor *rowPositionCursor,
task FileScanTask,
batch arrow.RecordBatch,
synthesizeRowID, synthesizeSeq bool,
@@ -750,7 +833,10 @@ func synthesizeRowLineageColumns(
}
}()
- synth := func(name string, fieldID int, value func(k int64) int64) error {
+ // perRow, when set, runs once for every physical row in order — including
+ // rows whose explicit value wins — so a cursor stays in lockstep with the
+ // batch even when value is never called.
+ synth := func(name string, fieldID int, perRow func(k int64), value func(k int64) int64) error {
idx := fieldIndexByID(schema, fieldID)
var existing *array.Int64
if idx >= 0 {
@@ -764,6 +850,9 @@ func synthesizeRowLineageColumns(
defer bldr.Release()
bldr.Reserve(int(nrows))
for k := range nrows {
+ if perRow != nil {
+ perRow(k)
+ }
if existing != nil && !existing.IsNull(int(k)) {
bldr.Append(existing.Value(int(k)))
} else {
@@ -791,24 +880,24 @@ func synthesizeRowLineageColumns(
if synthesizeRowID {
first := *task.FirstRowID
- if err := synth(iceberg.RowIDColumnName, iceberg.RowIDFieldID, func(k int64) int64 {
- return first + *rowOffset + k
- }); err != nil {
+ var pos int64
+ if err := synth(iceberg.RowIDColumnName, iceberg.RowIDFieldID,
+ func(int64) { pos = cursor.next() },
+ func(int64) int64 { return first + pos },
+ ); err != nil {
return nil, err
}
}
if synthesizeSeq {
seq := *task.DataSequenceNumber
- if err := synth(iceberg.LastUpdatedSequenceNumberColumnName, iceberg.LastUpdatedSequenceNumberFieldID, func(int64) int64 {
- return seq
- }); err != nil {
+ if err := synth(iceberg.LastUpdatedSequenceNumberColumnName, iceberg.LastUpdatedSequenceNumberFieldID, nil,
+ func(int64) int64 {
+ return seq
+ }); err != nil {
return nil, err
}
}
- // Advance so the next batch from this file uses the correct row position for _row_id.
- *rowOffset += nrows
-
meta := schema.Metadata()
return array.NewRecordBatch(arrow.NewSchema(fields, &meta), newCols, nrows), nil
@@ -821,7 +910,7 @@ func (as *arrowScan) processRecords(
rdr internal.FileReader,
columns []int,
pipeline []recProcessFn,
- skipRowGroupPruning bool,
+ posSource *rowPositionSource,
out chan<- enumeratedRecord,
) (err error) {
var (
@@ -831,11 +920,12 @@ func (as *arrowScan) processRecords(
// Row-group stats/bloom pruning skips whole groups, so emitted batches no
// longer cover contiguous file positions. Steps that key on the original
- // position (row-lineage _row_id, generated position deletes) need the full
- // sequence, so the per-row filter alone enforces the predicate while every
- // group is still read.
+ // position (row-lineage _row_id, positional/DV deletes, generated position
+ // deletes) recover it from posSource, which the tester seeds with each
+ // surviving group's file position before any record is read, so pruning
+ // stays enabled.
switch {
- case task.Value.File.FileFormat() == iceberg.ParquetFile && !skipRowGroupPruning:
+ case task.Value.File.FileFormat() == iceberg.ParquetFile:
statsFn, err := newParquetRowGroupStatsEvaluator(fileSchema, as.boundRowFilter, false)
if err != nil {
return err
@@ -846,10 +936,14 @@ func (as *arrowScan) processRecords(
return err
}
- testRowGroups = &internal.ParquetRowGroupTester{
+ tester := &internal.ParquetRowGroupTester{
StatsFn: statsFn,
BloomPreds: bloomPreds,
}
+ if posSource != nil {
+ tester.Survivors = &posSource.spans
+ }
+ testRowGroups = tester
}
recRdr, err = rdr.GetRecords(ctx, columns, testRowGroups)
@@ -930,22 +1024,42 @@ func (as *arrowScan) recordsFromTask(ctx context.Context, task internal.Enumerat
_, wantSeqNum := as.projectedSchema.FindFieldByID(iceberg.LastUpdatedSequenceNumberFieldID)
synthesizeRowID := rowLineageEnabled && wantRowID && task.Value.FirstRowID != nil
synthesizeSeq := rowLineageEnabled && wantSeqNum && task.Value.DataSequenceNumber != nil
- // Only _row_id depends on position, so only it forces contiguous reads.
- needsOriginalPositions := synthesizeRowID
+
+ // Every position-keyed step (_row_id synthesis, positional/DV deletes) needs
+ // each row's original file position; under row-group pruning the emitted rows
+ // are non-contiguous. processRecords seeds one shared source with the
+ // surviving groups and each step takes its own cursor over it.
+ hasDV := dvBitmap != nil && !dvBitmap.IsEmpty()
+ // Per the v3 spec a DV supersedes positional delete files for a data file, so
+ // when both are attached (only possible on a hand-built task; PlanFiles never
+ // produces it) the DV wins and positional deletes are ignored. This mirrors
+ // the planner (scanner.go) and Java's DeleteFileIndex, and keeps at most one
+ // row-dropping position step in the pipeline so cursors never run over a
+ // sequence an earlier step already shortened.
+ applyPosDeletes := len(positionalDeletes) > 0 && !hasDV
+ var posSource *rowPositionSource
+ if synthesizeRowID || applyPosDeletes || hasDV {
+ posSource = &rowPositionSource{}
+ }
+
if synthesizeRowID || synthesizeSeq {
// Mirror the columns synthesize will append so ToRequestedSchema never
// resolves a lineage field missing from the batch.
readSchema = iceberg.SchemaWithRowLineageColumns(iceSchema, synthesizeRowID, synthesizeSeq)
- var rowOffset int64
taskVal := task.Value
+ // Only _row_id depends on position, so a cursor is taken only then.
+ var cursor *rowPositionCursor
+ if synthesizeRowID {
+ cursor = posSource.cursor()
+ }
pipeline = append(pipeline, func(r arrow.RecordBatch) (arrow.RecordBatch, error) {
defer r.Release()
- return synthesizeRowLineageColumns(ctx, &rowOffset, taskVal, r, synthesizeRowID, synthesizeSeq)
+ return synthesizeRowLineageColumns(ctx, cursor, taskVal, r, synthesizeRowID, synthesizeSeq)
})
}
- if len(positionalDeletes) > 0 {
+ if applyPosDeletes {
deletes := set[int64]{}
for _, chunk := range positionalDeletes {
for _, a := range chunk.Chunks() {
@@ -955,15 +1069,11 @@ func (as *arrowScan) recordsFromTask(ctx context.Context, task internal.Enumerat
}
}
- pipeline = append(pipeline, processPositionalDeletes(ctx, deletes))
+ pipeline = append(pipeline, processPositionalDeletes(ctx, deletes, posSource.cursor()))
}
- // PlanFiles skips Parquet pos-delete matching for any data file that
- // has a DV (per spec), so in practice the two are mutually exclusive
- // per task. Append after the pos-delete step anyway so a manually-
- // constructed task with both sources still gets both filters applied.
- if dvBitmap != nil && !dvBitmap.IsEmpty() {
- pipeline = append(pipeline, filterByDeletionVector(ctx, dvBitmap, task.Value.File.Count()))
+ if hasDV {
+ pipeline = append(pipeline, filterByDeletionVector(ctx, dvBitmap, task.Value.File.Count(), posSource.cursor()))
}
if len(eqDeleteSets) > 0 {
@@ -1003,7 +1113,7 @@ func (as *arrowScan) recordsFromTask(ctx context.Context, task internal.Enumerat
return ToRequestedSchema(ctx, as.projectedSchema, readSchema, r, SchemaOptions{UseLargeTypes: as.useLargeTypes})
})
- err = as.processRecords(ctx, task, iceSchema, rdr, colIndices, pipeline, needsOriginalPositions, out)
+ err = as.processRecords(ctx, task, iceSchema, rdr, colIndices, pipeline, posSource, out)
return err
}
@@ -1032,8 +1142,14 @@ func (as *arrowScan) producePosDeletesFromTask(ctx context.Context, task interna
fields := append(iceSchema.Fields(), iceberg.PositionalDeleteSchema.Fields()...)
enrichedIcebergSchema := iceberg.NewSchema(iceSchema.ID+1, fields...)
+ // Each row's original file position is stamped into the generated delete
+ // records and matched against any pre-existing positional deletes. Under the
+ // delete filter, pruned row groups hold no matching rows, so skipping them
+ // drops no generated deletes; cursors over the shared source keep the
+ // emitted positions correct across the gaps.
+ posSource := &rowPositionSource{}
pipeline := make([]recProcessFn, 0, 2)
- pipeline = append(pipeline, enrichRecordsWithPosDeleteFields(ctx, task.Value.File))
+ pipeline = append(pipeline, enrichRecordsWithPosDeleteFields(ctx, task.Value.File, posSource.cursor()))
if len(positionalDeletes) > 0 {
deletes := set[int64]{}
for _, chunk := range positionalDeletes {
@@ -1044,7 +1160,7 @@ func (as *arrowScan) producePosDeletesFromTask(ctx context.Context, task interna
}
}
- pipeline = append(pipeline, processPositionalDeletes(ctx, deletes))
+ pipeline = append(pipeline, processPositionalDeletes(ctx, deletes, posSource.cursor()))
}
filterFunc, dropFile, err = as.getRecordFilter(ctx, iceSchema)
@@ -1075,9 +1191,7 @@ func (as *arrowScan) producePosDeletesFromTask(ctx context.Context, task interna
return ToRequestedSchema(ctx, iceberg.PositionalDeleteSchema, enrichedIcebergSchema, r, SchemaOptions{IncludeFieldIDs: true, UseLargeTypes: as.useLargeTypes})
})
- // enrichRecordsWithPosDeleteFields stamps positions by counting emitted
- // rows, so the reader must not prune row groups out from under it.
- err = as.processRecords(ctx, task, iceSchema, rdr, colIndices, pipeline, true, out)
+ err = as.processRecords(ctx, task, iceSchema, rdr, colIndices, pipeline, posSource, out)
return err
}
diff --git a/table/arrow_scanner_posdelete_regression_test.go b/table/arrow_scanner_posdelete_regression_test.go
index 42ec6f68..e9bca3e9 100644
--- a/table/arrow_scanner_posdelete_regression_test.go
+++ b/table/arrow_scanner_posdelete_regression_test.go
@@ -283,7 +283,7 @@ func TestProcessPositionalDeletesAcrossBatches(t *testing.T) {
// processPositionalDeletes owns (releases) each input batch it is handed.
deletes := set[int64]{3: {}}
- processFn := processPositionalDeletes(ctx, deletes)
+ processFn := processPositionalDeletes(ctx, deletes, (&rowPositionSource{}).cursor())
for i, b := range batches {
out, err := processFn(b)
diff --git a/table/arrow_scanner_test.go b/table/arrow_scanner_test.go
index 23c70fb8..b527d896 100644
--- a/table/arrow_scanner_test.go
+++ b/table/arrow_scanner_test.go
@@ -157,7 +157,7 @@ func TestEnrichRecordsWithPosDeleteFields(t *testing.T) {
}
}()
- enrichFn := enrichRecordsWithPosDeleteFields(ctx, &mockDataFile{path: "file://test_path.parquet"})
+ enrichFn := enrichRecordsWithPosDeleteFields(ctx, &mockDataFile{path: "file://test_path.parquet"}, (&rowPositionSource{}).cursor())
for i, b := range tc.inputBatches {
out, err := enrichFn(b)
require.NoError(t, err)
diff --git a/table/dv_scanner_read_test.go b/table/dv_scanner_read_test.go
index 1d038cd2..00c8c0ec 100644
--- a/table/dv_scanner_read_test.go
+++ b/table/dv_scanner_read_test.go
@@ -34,6 +34,7 @@ import (
iceio "github.com/apache/iceberg-go/io"
"github.com/apache/iceberg-go/puffin"
"github.com/apache/iceberg-go/table/dv"
+ "github.com/apache/iceberg-go/table/internal"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@@ -296,7 +297,7 @@ func TestFilterByDeletionVector(t *testing.T) {
for _, p := range []uint64{1, 3, 7, 9} {
bitmap.Set(p)
}
- filter := filterByDeletionVector(ctx, bitmap, 10)
+ filter := filterByDeletionVector(ctx, bitmap, 10, (&rowPositionSource{}).cursor())
mkBatch := func(start, end int64) arrow.RecordBatch {
bldr := array.NewInt64Builder(mem)
@@ -327,3 +328,33 @@ func TestFilterByDeletionVector(t *testing.T) {
out2.Column(0).(*array.Int64).Int64Values())
})
}
+
+// TestFilterByDeletionVectorOutOfBoundsPosition guards the slow-path bound: when
+// a surviving span's cursor positions exceed rowCount (row-group metadata and the
+// manifest's File.Count() disagree — a writer bug, hand-built file, or corruption)
+// the keep-mask index must not run off keepBits. The guard keeps such rows rather
+// than panicking, so the batch survives intact.
+func TestFilterByDeletionVectorOutOfBoundsPosition(t *testing.T) {
+ ctx := context.Background()
+ mem := memory.NewGoAllocator()
+
+ const rowCount = 4
+ bitmap := dv.NewRoaringPositionBitmap()
+ src := &rowPositionSource{spans: []internal.RowGroupSpan{{FirstRowPos: 100, NumRows: 3}}}
+ filter := filterByDeletionVector(ctx, bitmap, rowCount, src.cursor())
+
+ bldr := array.NewInt64Builder(mem)
+ defer bldr.Release()
+ for i := int64(0); i < 3; i++ {
+ bldr.Append(i)
+ }
+ col := bldr.NewArray()
+ defer col.Release()
+ schema := arrow.NewSchema([]arrow.Field{{Name: "pos", Type: arrow.PrimitiveTypes.Int64}}, nil)
+ batch := array.NewRecordBatch(schema, []arrow.Array{col}, 3)
+
+ out, err := filter(batch)
+ require.NoError(t, err)
+ defer out.Release()
+ assert.Equal(t, []int64{0, 1, 2}, out.Column(0).(*array.Int64).Int64Values())
+}
diff --git a/table/internal/parquet_files.go b/table/internal/parquet_files.go
index 0d3659b7..56daf40c 100644
--- a/table/internal/parquet_files.go
+++ b/table/internal/parquet_files.go
@@ -749,6 +749,18 @@ type RowGroupBloomPred struct {
type ParquetRowGroupTester struct {
StatsFn func(*metadata.RowGroupMetaData, []int) (bool, error)
BloomPreds []RowGroupBloomPred // nil = no bloom filter pass
+ // Survivors, if non-nil, is reset and then filled with one span per row group
+ // that survives pruning, in file order, with positions relative to the full
+ // file. It lets callers reconstruct each emitted row's original position even
+ // when pruning skips groups. Set to nil when no group is skipped.
+ Survivors *[]RowGroupSpan
+}
+
+// RowGroupSpan locates a surviving row group within the data file: the absolute
+// position of its first row and the number of rows it holds.
+type RowGroupSpan struct {
+ FirstRowPos int64
+ NumRows int64
}
type wrapPqArrowReader struct {
@@ -797,13 +809,20 @@ func (w wrapPqArrowReader) GetRecords(ctx context.Context, cols []int, tester an
bfReader = w.ParquetReader().GetBloomFilterReader()
}
+ if rowGroupTester.Survivors != nil {
+ *rowGroupTester.Survivors = (*rowGroupTester.Survivors)[:0]
+ }
+
rgList = make([]int, 0)
- for rg := 0; rg < numRg; rg++ {
- use := true
+ var firstRowPos int64
+ for rg := range numRg {
+ rgMeta := fileMeta.RowGroup(rg)
+ numRows := rgMeta.NumRows()
+ pos := firstRowPos
+ firstRowPos += numRows
+ use := true
if rowGroupTester.StatsFn != nil {
- rgMeta := fileMeta.RowGroup(rg)
-
var err error
use, err = rowGroupTester.StatsFn(rgMeta, cols)
if err != nil {
@@ -811,11 +830,7 @@ func (w wrapPqArrowReader) GetRecords(ctx context.Context, cols []int, tester an
}
}
- if !use {
- continue
- }
-
- if bfReader != nil {
+ if use && bfReader != nil {
var err error
use, err = checkRowGroupBloomFilters(bfReader, rg, fieldIDToColIdx, rowGroupTester.BloomPreds)
if err != nil {
@@ -823,10 +838,22 @@ func (w wrapPqArrowReader) GetRecords(ctx context.Context, cols []int, tester an
}
}
- if use {
- rgList = append(rgList, rg)
+ if !use {
+ continue
+ }
+
+ rgList = append(rgList, rg)
+ if rowGroupTester.Survivors != nil {
+ *rowGroupTester.Survivors = append(*rowGroupTester.Survivors,
+ RowGroupSpan{FirstRowPos: pos, NumRows: numRows})
}
}
+
+ // No group was skipped: emitted rows are contiguous from zero, so drop
+ // the spans and let position cursors take their contiguous fast path.
+ if rowGroupTester.Survivors != nil && len(rgList) == numRg {
+ *rowGroupTester.Survivors = nil
+ }
}
return w.GetRecordReader(ctx, cols, rgList)
diff --git a/table/internal/parquet_files_test.go b/table/internal/parquet_files_test.go
index fad1231b..4932358f 100644
--- a/table/internal/parquet_files_test.go
+++ b/table/internal/parquet_files_test.go
@@ -1101,44 +1101,55 @@ func TestBloomFilterRowGroupPruning(t *testing.T) {
t.Run("value in RG0 prunes RG1", func(t *testing.T) {
rdr := openBloomTestReader(t, data)
+ var survivors []internal.RowGroupSpan
tester := &internal.ParquetRowGroupTester{
StatsFn: alwaysKeep,
BloomPreds: []internal.RowGroupBloomPred{
{FieldID: 1, PhysBytes: [][]byte{int32PhysBytes(50)}}, // 50 is in RG0
},
+ Survivors: &survivors,
}
rr, err := rdr.GetRecords(ctx, cols, tester)
require.NoError(t, err)
assert.Equal(t, int64(rgSize), countRecords(t, rr), "only RG0 rows expected")
+ assert.Equal(t, []internal.RowGroupSpan{{FirstRowPos: 0, NumRows: rgSize}}, survivors,
+ "surviving RG0 keeps file position 0")
})
t.Run("value in RG1 prunes RG0", func(t *testing.T) {
rdr := openBloomTestReader(t, data)
+ var survivors []internal.RowGroupSpan
tester := &internal.ParquetRowGroupTester{
StatsFn: alwaysKeep,
BloomPreds: []internal.RowGroupBloomPred{
{FieldID: 1, PhysBytes: [][]byte{int32PhysBytes(150)}}, // 150 is in RG1
},
+ Survivors: &survivors,
}
rr, err := rdr.GetRecords(ctx, cols, tester)
require.NoError(t, err)
assert.Equal(t, int64(rgSize), countRecords(t, rr), "only RG1 rows expected")
+ assert.Equal(t, []internal.RowGroupSpan{{FirstRowPos: rgSize, NumRows: rgSize}}, survivors,
+ "surviving RG1 keeps its file position past the bloom-pruned RG0")
})
t.Run("value absent from all row groups skips all", func(t *testing.T) {
rdr := openBloomTestReader(t, data)
+ var survivors []internal.RowGroupSpan
tester := &internal.ParquetRowGroupTester{
StatsFn: alwaysKeep,
BloomPreds: []internal.RowGroupBloomPred{
{FieldID: 1, PhysBytes: [][]byte{int32PhysBytes(999)}}, // 999 not in either RG
},
+ Survivors: &survivors,
}
rr, err := rdr.GetRecords(ctx, cols, tester)
require.NoError(t, err)
assert.Equal(t, int64(0), countRecords(t, rr), "no rows expected when value absent")
+ assert.Empty(t, survivors, "no surviving row groups when value is absent everywhere")
})
t.Run("In predicate with a value in each RG keeps both", func(t *testing.T) {
diff --git a/table/mor_delete_pruning_test.go b/table/mor_delete_pruning_test.go
index c70a4c26..5e345301 100644
--- a/table/mor_delete_pruning_test.go
+++ b/table/mor_delete_pruning_test.go
@@ -36,9 +36,9 @@ import (
// TestMergeOnReadDeleteAcrossPrunedRowGroups guards the row-group pruning hazard
// in generated position deletes: a merge-on-read DELETE whose filter prunes a
// leading row group must still delete the right physical rows. enrichRecords-
-// WithPosDeleteFields stamps each surviving row's position by counting emitted
-// rows, so if the reader skips a pruned row group the positions are off by the
-// pruned group's size and the wrong rows get deleted.
+// WithPosDeleteFields stamps each surviving row's position from a rowPositionCursor
+// seeded with the surviving row groups, so a pruned group's rows are still counted
+// and the survivors keep their original file positions.
//
// The file has two row groups (ids 1..5, then 6..10). Deleting id == 7 matches
// only the second group, so the first is pruned by stats; a dense position
diff --git a/table/row_lineage_prune_delete_test.go b/table/row_lineage_prune_delete_test.go
new file mode 100644
index 00000000..482e297d
--- /dev/null
+++ b/table/row_lineage_prune_delete_test.go
@@ -0,0 +1,260 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package table_test
+
+import (
+ "context"
+ "fmt"
+ "path/filepath"
+ "slices"
+ "testing"
+
+ "github.com/apache/arrow-go/v18/arrow"
+ "github.com/apache/arrow-go/v18/arrow/array"
+ "github.com/apache/arrow-go/v18/arrow/memory"
+ "github.com/apache/iceberg-go"
+ iceio "github.com/apache/iceberg-go/io"
+ "github.com/apache/iceberg-go/table"
+ "github.com/apache/iceberg-go/table/dv"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+// These tests guard the interaction between row-group pruning and the
+// position-keyed scan steps. Re-enabling pruning while _row_id is projected
+// (or for any scan with positional/DV deletes) means processPositionalDeletes
+// and filterByDeletionVector see non-contiguous emitted rows; if they kept
+// counting positions densely they would target the wrong physical rows. Each
+// test prunes a leading row group and deletes a row that lives only in the
+// surviving group.
+
+// TestScanPruningWithPositionalDeletes covers a Parquet positional delete whose
+// target (pos 6 = id=7) is in the second row group while the filter prunes the
+// first. The deleted row must vanish and survivors must keep their _row_id.
+func TestScanPruningWithPositionalDeletes(t *testing.T) {
+ ctx := context.Background()
+ tbl := buildTwoRowGroupV3Table(t)
+
+ tasks, err := tbl.Scan().PlanFiles(ctx)
+ require.NoError(t, err)
+ require.Len(t, tasks, 1)
+ dataPath := tasks[0].File.FilePath()
+
+ posDelPath := tbl.Location() + "/data/pos-del.parquet"
+ posSc, err := table.SchemaToArrowSchema(iceberg.PositionalDeleteSchema, nil, true, false)
+ require.NoError(t, err)
+ writeParquetFile(t, posDelPath, posSc, fmt.Sprintf(`[{"file_path":%q,"pos":6}]`, dataPath))
+
+ b, err := iceberg.NewDataFileBuilder(
+ *iceberg.UnpartitionedSpec, iceberg.EntryContentPosDeletes,
+ posDelPath, iceberg.ParquetFile, nil, nil, nil, 1, 256)
+ require.NoError(t, err)
+ tx := tbl.NewTransaction()
+ require.NoError(t, tx.NewRowDelta(nil).AddDeletes(b.Build()).Commit(ctx))
+ tbl, err = tx.Commit(ctx)
+ require.NoError(t, err)
+
+ filter := table.WithRowFilter(iceberg.GreaterThan(iceberg.Reference("id"), int64(5)))
+
+ got := rowIDByID(t, tbl, filter)
+ assert.Equal(t, map[int64]int64{6: 5, 8: 7, 9: 8, 10: 9}, got,
+ "pos-6 (id=7) must be deleted and survivors keep their _row_id under pruning")
+
+ // Same scan without _row_id projected: pruning was already active here before
+ // the cursor change, so this guards a pre-existing correctness gap too.
+ assert.Equal(t, []int64{6, 8, 9, 10}, idsMatchingFilter(t, tbl, filter),
+ "pos-6 (id=7) must be deleted even when _row_id is not projected")
+}
+
+// TestScanPruningWithDeletionVector is the v3 deletion-vector analogue: the DV
+// drops pos 6 (id=7) in the surviving second row group while the first is pruned.
+func TestScanPruningWithDeletionVector(t *testing.T) {
+ ctx := context.Background()
+ tbl := buildTwoRowGroupV3Table(t)
+
+ tasks, err := tbl.Scan().PlanFiles(ctx)
+ require.NoError(t, err)
+ require.Len(t, tasks, 1)
+ dataPath := tasks[0].File.FilePath()
+
+ w := dv.NewDVWriter(iceio.LocalFS{}, unpartitionedSpecByID)
+ w.Add(dataPath, []int64{6}, 0, nil)
+ dvFiles, err := w.Flush(ctx, filepath.Join(filepath.Dir(dataPath), "dv-0001.puffin"))
+ require.NoError(t, err)
+
+ tx := tbl.NewTransaction()
+ require.NoError(t, tx.NewRowDelta(nil).AddDeletes(dvFiles...).Commit(ctx))
+ tbl, err = tx.Commit(ctx)
+ require.NoError(t, err)
+
+ filter := table.WithRowFilter(iceberg.GreaterThan(iceberg.Reference("id"), int64(5)))
+
+ got := rowIDByID(t, tbl, filter)
+ assert.Equal(t, map[int64]int64{6: 5, 8: 7, 9: 8, 10: 9}, got,
+ "DV pos-6 (id=7) must be deleted and survivors keep their _row_id under pruning")
+
+ // Without _row_id pruning was already active here, so this guards the
+ // pre-existing DV-under-pruning gap as well.
+ assert.Equal(t, []int64{6, 8, 9, 10}, idsMatchingFilter(t, tbl, filter),
+ "DV pos-6 (id=7) must be deleted even when _row_id is not projected")
+}
+
+// TestScanPruningSurvivingGroupSpansBatches forces a tiny read batch size so the
+// single surviving row group is emitted across several Arrow batches. The cursor
+// must keep advancing across batch boundaries, not reset per batch.
+func TestScanPruningSurvivingGroupSpansBatches(t *testing.T) {
+ ctx := context.Background()
+ tbl := buildTwoRowGroupV3Table(t)
+
+ tx := tbl.NewTransaction()
+ require.NoError(t, tx.SetProperties(iceberg.Properties{table.ParquetBatchSizeKey: "2"}))
+ tbl, err := tx.Commit(ctx)
+ require.NoError(t, err)
+
+ got := rowIDByID(t, tbl, table.WithRowFilter(iceberg.GreaterThan(iceberg.Reference("id"), int64(5))))
+ assert.Equal(t, map[int64]int64{6: 5, 7: 6, 8: 7, 9: 8, 10: 9}, got,
+ "_row_id must stay contiguous as one surviving row group spans multiple batches")
+}
+
+// TestReadTaskDeletionVectorSupersedesPositionalDeletes locks the spec contract
+// PlanFiles enforces (scanner.go) for hand-built tasks too: when a data file has
+// both a DV and a positional delete attached, the DV wins and positional deletes
+// are ignored. Applying both would also run two row-dropping position steps in
+// sequence, so the second cursor would map over an already-shortened batch.
+func TestReadTaskDeletionVectorSupersedesPositionalDeletes(t *testing.T) {
+ ctx := context.Background()
+ tbl := newV3RowLineageTestTable(t)
+
+ arrowSchema := arrow.NewSchema([]arrow.Field{
+ {Name: "id", Type: arrow.PrimitiveTypes.Int64, Nullable: false},
+ {Name: "data", Type: arrow.BinaryTypes.String, Nullable: true},
+ }, nil)
+ data, err := array.TableFromJSON(memory.DefaultAllocator, arrowSchema, []string{
+ `[{"id":1,"data":"a"},{"id":2,"data":"b"},{"id":3,"data":"c"},{"id":4,"data":"d"},{"id":5,"data":"e"}]`,
+ })
+ require.NoError(t, err)
+ defer data.Release()
+ tbl, err = tbl.Append(ctx, array.NewTableReader(data, -1), nil)
+ require.NoError(t, err)
+
+ tasks, err := tbl.Scan().PlanFiles(ctx)
+ require.NoError(t, err)
+ require.Len(t, tasks, 1)
+ dataFile := tasks[0].File
+ dataPath := dataFile.FilePath()
+
+ // DV deletes pos 0 (id=1); the positional delete targets pos 4 (id=5).
+ w := dv.NewDVWriter(iceio.LocalFS{}, unpartitionedSpecByID)
+ w.Add(dataPath, []int64{0}, 0, nil)
+ dvFiles, err := w.Flush(ctx, filepath.Join(filepath.Dir(dataPath), "dv-0001.puffin"))
+ require.NoError(t, err)
+
+ posDelPath := tbl.Location() + "/data/pos-del.parquet"
+ posSc, err := table.SchemaToArrowSchema(iceberg.PositionalDeleteSchema, nil, true, false)
+ require.NoError(t, err)
+ writeParquetFile(t, posDelPath, posSc, fmt.Sprintf(`[{"file_path":%q,"pos":4}]`, dataPath))
+ posDel, err := iceberg.NewDataFileBuilder(
+ *iceberg.UnpartitionedSpec, iceberg.EntryContentPosDeletes,
+ posDelPath, iceberg.ParquetFile, nil, nil, nil, 1, 256)
+ require.NoError(t, err)
+
+ task := table.FileScanTask{
+ File: dataFile,
+ DeleteFiles: []iceberg.DataFile{posDel.Build()},
+ DeletionVectorFiles: dvFiles,
+ Length: dataFile.FileSizeBytes(),
+ }
+
+ _, itr, err := tbl.Scan().ReadTasks(ctx, []table.FileScanTask{task})
+ require.NoError(t, err)
+
+ var ids []int64
+ for rec, err := range itr {
+ require.NoError(t, err)
+ col := rec.Column(rec.Schema().FieldIndices("id")[0]).(*array.Int64)
+ for i := range int(rec.NumRows()) {
+ ids = append(ids, col.Value(i))
+ }
+ rec.Release()
+ }
+ slices.Sort(ids)
+
+ assert.Equal(t, []int64{2, 3, 4, 5}, ids,
+ "DV (pos 0 / id=1) must win and the positional delete (pos 4 / id=5) must be ignored")
+}
+
+// Bloom-filter row-group pruning and its span accounting are covered at the
+// reader layer in TestBloomFilterRowGroupPruning (table/internal); an end-to-end
+// scan cannot prove bloom skipped a group, since a correct scan emits the same
+// rows whether or not the empty group was physically read.
+
+// buildTwoRowGroupV3Table writes a v3 row-lineage table with ids 1..10 across two
+// 5-row row groups, so an id>5 filter prunes the first group via stats.
+func buildTwoRowGroupV3Table(t *testing.T) *table.Table {
+ t.Helper()
+ ctx := context.Background()
+ tbl := newV3RowLineageTestTable(t)
+
+ tx := tbl.NewTransaction()
+ require.NoError(t, tx.SetProperties(iceberg.Properties{table.ParquetRowGroupLimitKey: "5"}))
+ tbl, err := tx.Commit(ctx)
+ require.NoError(t, err)
+
+ arrowSchema := arrow.NewSchema([]arrow.Field{
+ {Name: "id", Type: arrow.PrimitiveTypes.Int64, Nullable: false},
+ {Name: "data", Type: arrow.BinaryTypes.String, Nullable: true},
+ }, nil)
+ data, err := array.TableFromJSON(memory.DefaultAllocator, arrowSchema, []string{
+ `[{"id":1,"data":"a"},{"id":2,"data":"b"},{"id":3,"data":"c"},{"id":4,"data":"d"},` +
+ `{"id":5,"data":"e"},{"id":6,"data":"f"},{"id":7,"data":"g"},{"id":8,"data":"h"},` +
+ `{"id":9,"data":"i"},{"id":10,"data":"j"}]`,
+ })
+ require.NoError(t, err)
+ defer data.Release()
+
+ tbl, err = tbl.Append(ctx, array.NewTableReader(data, -1), nil)
+ require.NoError(t, err)
+
+ require.Equal(t, 2, parquetRowGroupCount(t, tbl),
+ "file must have multiple row groups for the id>5 / id==7 filter to prune one")
+
+ return tbl
+}
+
+// idsMatchingFilter scans without row lineage and returns the surviving ids, sorted.
+func idsMatchingFilter(t *testing.T, tbl *table.Table, opts ...table.ScanOption) []int64 {
+ t.Helper()
+
+ _, itr, err := tbl.Scan(opts...).ToArrowRecords(context.Background())
+ require.NoError(t, err)
+
+ var ids []int64
+ for rec, err := range itr {
+ require.NoError(t, err)
+ idIdx := rec.Schema().FieldIndices("id")
+ require.NotEmpty(t, idIdx)
+ col := rec.Column(idIdx[0]).(*array.Int64)
+ for i := range int(rec.NumRows()) {
+ ids = append(ids, col.Value(i))
+ }
+ rec.Release()
+ }
+ slices.Sort(ids)
+
+ return ids
+}
diff --git a/table/scanner_internal_test.go b/table/scanner_internal_test.go
index 73b2bc59..b1e45208 100644
--- a/table/scanner_internal_test.go
+++ b/table/scanner_internal_test.go
@@ -32,6 +32,7 @@ import (
"github.com/apache/arrow-go/v18/arrow/memory"
"github.com/apache/iceberg-go"
iceio "github.com/apache/iceberg-go/io"
+ "github.com/apache/iceberg-go/table/internal"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@@ -335,7 +336,7 @@ func TestSynthesizeRowLineageColumns(t *testing.T) {
firstRowID := int64(1000)
dataSeqNum := int64(5)
task := FileScanTask{FirstRowID: &firstRowID, DataSequenceNumber: &dataSeqNum}
- rowOffset := int64(0)
+ cursor := (&rowPositionSource{}).cursor()
// Build a batch with a data column plus _row_id and _last_updated_sequence_number (all nulls).
schema := arrow.NewSchema(
@@ -366,7 +367,7 @@ func TestSynthesizeRowLineageColumns(t *testing.T) {
seqArr.Release()
defer batch.Release()
- out, err := synthesizeRowLineageColumns(ctx, &rowOffset, task, batch, true, true)
+ out, err := synthesizeRowLineageColumns(ctx, cursor, task, batch, true, true)
require.NoError(t, err)
defer out.Release()
@@ -383,7 +384,7 @@ func TestSynthesizeRowLineageColumns(t *testing.T) {
assert.False(t, seqCol.IsNull(i), "row %d", i)
assert.EqualValues(t, 5, seqCol.Value(i), "row %d", i)
}
- assert.EqualValues(t, 3, rowOffset)
+ assert.EqualValues(t, 3, cursor.consumed)
}
// TestSynthesizeRowLineageColumnsPreservesExplicit covers the spec's null-
@@ -399,7 +400,7 @@ func TestSynthesizeRowLineageColumnsPreservesExplicit(t *testing.T) {
firstRowID := int64(1000)
dataSeqNum := int64(5)
task := FileScanTask{FirstRowID: &firstRowID, DataSequenceNumber: &dataSeqNum}
- rowOffset := int64(0)
+ cursor := (&rowPositionSource{}).cursor()
schema := arrow.NewSchema(
[]arrow.Field{
@@ -436,7 +437,7 @@ func TestSynthesizeRowLineageColumnsPreservesExplicit(t *testing.T) {
seqArr.Release()
defer batch.Release()
- out, err := synthesizeRowLineageColumns(ctx, &rowOffset, task, batch, true, true)
+ out, err := synthesizeRowLineageColumns(ctx, cursor, task, batch, true, true)
require.NoError(t, err)
defer out.Release()
@@ -458,7 +459,7 @@ func TestSynthesizeRowLineageColumnsPreservesExplicit(t *testing.T) {
assert.False(t, seqCol.IsNull(2))
assert.EqualValues(t, 9, seqCol.Value(2), "explicit seq must survive even with a different value")
- assert.EqualValues(t, 3, rowOffset)
+ assert.EqualValues(t, 3, cursor.consumed)
}
// TestSynthesizeRowLineageColumnsAppendsMissing covers the path where the
@@ -473,7 +474,7 @@ func TestSynthesizeRowLineageColumnsAppendsMissing(t *testing.T) {
firstRowID := int64(1000)
dataSeqNum := int64(5)
task := FileScanTask{FirstRowID: &firstRowID, DataSequenceNumber: &dataSeqNum}
- rowOffset := int64(0)
+ cursor := (&rowPositionSource{}).cursor()
md := arrow.NewMetadata([]string{"k"}, []string{"v"})
schema := arrow.NewSchema(
@@ -487,7 +488,7 @@ func TestSynthesizeRowLineageColumnsAppendsMissing(t *testing.T) {
xArr.Release()
defer batch.Release()
- out, err := synthesizeRowLineageColumns(ctx, &rowOffset, task, batch, true, true)
+ out, err := synthesizeRowLineageColumns(ctx, cursor, task, batch, true, true)
require.NoError(t, err)
defer out.Release()
@@ -517,7 +518,7 @@ func TestSynthesizeRowLineageColumnsRejectsWrongType(t *testing.T) {
firstRowID := int64(1000)
task := FileScanTask{FirstRowID: &firstRowID}
- rowOffset := int64(0)
+ cursor := (&rowPositionSource{}).cursor()
wrongField := lineageArrowField(iceberg.RowIDColumnName, iceberg.RowIDFieldID)
wrongField.Type = arrow.BinaryTypes.String
@@ -531,11 +532,60 @@ func TestSynthesizeRowLineageColumnsRejectsWrongType(t *testing.T) {
arr.Release()
defer batch.Release()
- _, err := synthesizeRowLineageColumns(ctx, &rowOffset, task, batch, true, false)
+ _, err := synthesizeRowLineageColumns(ctx, cursor, task, batch, true, false)
require.Error(t, err)
assert.Contains(t, err.Error(), "want int64")
}
+// TestRowPositionCursor covers the position reconstruction that lets row-group
+// pruning stay enabled: surviving spans map emitted rows back to their original
+// file positions, jumping across the gaps left by pruned groups even when a
+// single sequence of calls crosses a span seam.
+func TestRowPositionCursor(t *testing.T) {
+ tests := []struct {
+ name string
+ spans []internal.RowGroupSpan
+ calls int
+ want []int64
+ }{
+ {
+ name: "no spans falls back to contiguous from zero",
+ spans: nil,
+ calls: 4,
+ want: []int64{0, 1, 2, 3},
+ },
+ {
+ name: "pruned leading group starts at its file position",
+ spans: []internal.RowGroupSpan{{FirstRowPos: 5, NumRows: 5}},
+ calls: 5,
+ want: []int64{5, 6, 7, 8, 9},
+ },
+ {
+ name: "gap between surviving groups is skipped across the seam",
+ spans: []internal.RowGroupSpan{{FirstRowPos: 0, NumRows: 5}, {FirstRowPos: 10, NumRows: 5}},
+ calls: 10,
+ want: []int64{0, 1, 2, 3, 4, 10, 11, 12, 13, 14},
+ },
+ {
+ name: "empty leading span is stepped over",
+ spans: []internal.RowGroupSpan{{FirstRowPos: 0, NumRows: 0}, {FirstRowPos: 7, NumRows: 3}},
+ calls: 3,
+ want: []int64{7, 8, 9},
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ cursor := (&rowPositionSource{spans: tt.spans}).cursor()
+ got := make([]int64, tt.calls)
+ for i := range got {
+ got[i] = cursor.next()
+ }
+ assert.Equal(t, tt.want, got)
+ })
+ }
+}
+
// lineageArrowField builds an Int64 Arrow field tagged with the reserved
// row-lineage field id, mirroring how the Parquet reader tags columns.
func lineageArrowField(name string, fieldID int) arrow.Field {
diff --git a/table/variant_shredded_posdelete_test.go b/table/variant_shredded_posdelete_test.go
index 9fb71779..7e864be8 100644
--- a/table/variant_shredded_posdelete_test.go
+++ b/table/variant_shredded_posdelete_test.go
@@ -60,7 +60,7 @@ func TestShreddedVariantSurvivesPositionalDelete(t *testing.T) {
// Delete positions 1 and 3; survivors are 0, 2, 4. processPositionalDeletes
// owns (releases) the batch it is handed.
- survivors, err := processPositionalDeletes(ctx, set[int64]{1: {}, 3: {}})(batch)
+ survivors, err := processPositionalDeletes(ctx, set[int64]{1: {}, 3: {}}, (&rowPositionSource{}).cursor())(batch)
require.NoError(t, err)
ice := iceberg.NewSchema(0, iceberg.NestedField{ID: 2, Name: "var", Type: iceberg.VariantType{}})