You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by at...@apache.org on 2026/08/11 11:13:36 UTC
(iceberg-go) branch main updated: fix(table): reject ambiguous equality-delete columns (#1617)
This is an automated email from the ASF dual-hosted git repository.
laskoviymishka pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-go.git
The following commit(s) were added to refs/heads/main by this push:
new acd95e4fa fix(table): reject ambiguous equality-delete columns (#1617)
acd95e4fa is described below
commit acd95e4fa9423f587ddb09d4abafba0ad4b71ca6
Author: Minh Vu <vu...@gmail.com>
AuthorDate: Tue Aug 11 13:13:31 2026 +0200
fix(table): reject ambiguous equality-delete columns (#1617)
* fix(table): reject ambiguous equality-delete columns
Signed-off-by: Minh Vu <vu...@gmail.com>
* fix(table): reject ambiguous equality columns in data records
* fix(table): resolve equality deletes by field ID
* fix(table): reject ambiguous equality-delete fields
* style(table): satisfy lint in equality delete test
* fix(table): preserve nested equality-delete nulls
* fix(table): cache equality delete field resolution
* fix(table): resolve equality delete paths structurally
* test(table): close equality delete file
---------
Signed-off-by: Minh Vu <vu...@gmail.com>
---
table/arrow_scanner.go | 4 +-
table/equality_delete_reader.go | 233 ++++++++++++++++++++---
table/equality_delete_reader_bench_test.go | 7 +-
table/equality_delete_reader_internal_test.go | 170 +++++++++++++++++
table/equality_delete_reader_test.go | 260 +++++++++++++++++++++++++-
5 files changed, 639 insertions(+), 35 deletions(-)
diff --git a/table/arrow_scanner.go b/table/arrow_scanner.go
index 49819ea3e..28a7d3ebf 100644
--- a/table/arrow_scanner.go
+++ b/table/arrow_scanner.go
@@ -1167,7 +1167,7 @@ func (as *arrowScan) recordsFromTask(ctx context.Context, task tblutils.Enumerat
}
if len(eqDeleteSets) > 0 {
- eqFn, eqErr := processEqualityDeletes(ctx, eqDeleteSets)
+ eqFn, eqErr := processEqualityDeletesColumnarForFile(ctx, eqDeleteSets, iceSchema, task.Value.File.FilePath())
if eqErr != nil {
return eqErr
}
@@ -1481,7 +1481,7 @@ func (as *arrowScan) GetRecords(ctx context.Context, tasks []FileScanTask) (*arr
}
eqDeleteSets, err := readAllEqualityDeleteFiles(ctx, as.fs,
- as.metadata.CurrentSchema(), tasks, as.concurrency)
+ as.metadata.CurrentSchema(), as.metadata.NameMapping(), tasks, as.concurrency)
if err != nil {
// Positional deletes were fully loaded; release them before aborting.
releasePerFilePosDeletes(deletesPerFile)
diff --git a/table/equality_delete_reader.go b/table/equality_delete_reader.go
index f7b912acf..b861338e9 100644
--- a/table/equality_delete_reader.go
+++ b/table/equality_delete_reader.go
@@ -20,6 +20,7 @@ package table
import (
"bytes"
"context"
+ "errors"
"fmt"
"math"
"unsafe"
@@ -36,6 +37,8 @@ import (
"golang.org/x/sync/errgroup"
)
+var ErrAmbiguousEqualityColumn = errors.New("equality delete column is ambiguous")
+
// equalityDeleteSet holds the set of delete keys and the column names
// used to look them up in data records. Each set corresponds to one
// group of equality field IDs — delete files with different field IDs
@@ -46,11 +49,140 @@ type equalityDeleteSet struct {
colNames []string
}
+type arrowFieldRef struct {
+ path []int
+}
+
+type arrowFieldRefsByID map[int][]arrowFieldRef
+
+func equalityFieldLocation(filePath string) string {
+ location := filePath
+ if location == "" {
+ location = "data record"
+ }
+
+ return location
+}
+
+// indexArrowFields derives Arrow child paths from the structurally aligned,
+// ID-resolved Iceberg file schema. It deliberately ignores names: dots in an
+// Iceberg name are literal and must not be interpreted as a path.
+func indexArrowFields(schema *iceberg.Schema) arrowFieldRefsByID {
+ refs := make(arrowFieldRefsByID)
+ if schema == nil {
+ return refs
+ }
+
+ var visit func([]iceberg.NestedField, []int)
+ visit = func(fields []iceberg.NestedField, parentPath []int) {
+ for i, field := range fields {
+ path := append(append([]int(nil), parentPath...), i)
+ refs[field.ID] = append(refs[field.ID], arrowFieldRef{path: path})
+
+ if nested, ok := field.Type.(*iceberg.StructType); ok {
+ visit(nested.Fields(), path)
+ }
+ }
+ }
+ visit(schema.Fields(), nil)
+
+ return refs
+}
+
+// indexArrowFieldsByMetadata is used for delete files whose Arrow fields carry
+// IDs directly, before any name mapping is needed.
+func indexArrowFieldsByMetadata(schema *arrow.Schema) arrowFieldRefsByID {
+ refs := make(arrowFieldRefsByID)
+ var visit func([]arrow.Field, []int)
+ visit = func(fields []arrow.Field, parentPath []int) {
+ for i, field := range fields {
+ path := append(append([]int(nil), parentPath...), i)
+ if id := getFieldID(field); id != nil {
+ refs[*id] = append(refs[*id], arrowFieldRef{path: path})
+ }
+
+ if nested, ok := field.Type.(*arrow.StructType); ok {
+ visit(nested.Fields(), path)
+ }
+ }
+ }
+ visit(schema.Fields(), nil)
+
+ return refs
+}
+
+func resolveArrowField(refs arrowFieldRefsByID, fieldID int, fieldName, filePath string) (arrowFieldRef, error) {
+ matches := refs[fieldID]
+ location := equalityFieldLocation(filePath)
+ if len(matches) == 1 {
+ return matches[0], nil
+ }
+ if len(matches) > 1 {
+ return arrowFieldRef{}, fmt.Errorf("%w: equality field ID %d (%s) in %s: found %d fields",
+ ErrAmbiguousEqualityColumn, fieldID, fieldName, location, len(matches))
+ }
+
+ return arrowFieldRef{}, fmt.Errorf("equality field ID %d (%s) not found in %s", fieldID, fieldName, location)
+}
+
+func arrowArrayAtField(record arrow.RecordBatch, ref arrowFieldRef, fieldID int, fieldName, filePath string) (arrow.Array, error) {
+ result, _, err := arrowArraysAtField(record, ref, fieldID, fieldName, filePath)
+
+ return result, err
+}
+
+func arrowArraysAtField(record arrow.RecordBatch, ref arrowFieldRef, fieldID int, fieldName, filePath string) (arrow.Array, []arrow.Array, error) {
+ location := filePath
+ if location == "" {
+ location = "data record"
+ }
+ if len(ref.path) == 0 || ref.path[0] >= int(record.NumCols()) {
+ return nil, nil, fmt.Errorf("equality field ID %d (%s) not found in %s", fieldID, fieldName, location)
+ }
+
+ result := record.Column(ref.path[0])
+ parents := make([]arrow.Array, 0, len(ref.path)-1)
+ for _, index := range ref.path[1:] {
+ structArray, ok := result.(*array.Struct)
+ if !ok || index >= structArray.NumField() {
+ return nil, nil, fmt.Errorf("equality field ID %d (%s) has unsupported nested path in %s", fieldID, fieldName, location)
+ }
+ parents = append(parents, result)
+ result = structArray.Field(index)
+ }
+
+ return result, parents, nil
+}
+
+func makeArrowFieldEncoder(record arrow.RecordBatch, ref arrowFieldRef, fieldID int, fieldName, filePath string) (colEncoder, error) {
+ column, parents, err := arrowArraysAtField(record, ref, fieldID, fieldName, filePath)
+ if err != nil {
+ return nil, err
+ }
+
+ encoder := makeColEncoder(column)
+ if len(parents) == 0 {
+ return encoder, nil
+ }
+
+ return func(buf *bytes.Buffer, row int) {
+ for _, parent := range parents {
+ if parent.IsNull(row) {
+ buf.WriteByte(0)
+
+ return
+ }
+ }
+
+ encoder(buf, row)
+ }, nil
+}
+
// readAllEqualityDeleteFiles reads all unique equality delete files from
// the tasks and builds per-task delete key sets. Returns nil if there are
// no equality deletes. Delete files with different equality field IDs are
// kept as separate sets (not merged).
-func readAllEqualityDeleteFiles(ctx context.Context, fs iceio.IO, schema *iceberg.Schema, tasks []FileScanTask, concurrency int) (map[int][]*equalityDeleteSet, error) {
+func readAllEqualityDeleteFiles(ctx context.Context, fs iceio.IO, schema *iceberg.Schema, nameMapping iceberg.NameMapping, tasks []FileScanTask, concurrency int) (map[int][]*equalityDeleteSet, error) {
type deleteFileInfo struct {
file iceberg.DataFile
fieldIDs []int
@@ -100,7 +232,7 @@ func readAllEqualityDeleteFiles(ctx context.Context, fs iceio.IO, schema *iceber
for _, info := range uniqueDeletes {
g.Go(func() error {
- keys, colNames, err := readEqualityDeleteFile(ctx, fs, schema, info.file, info.fieldIDs)
+ keys, colNames, err := readEqualityDeleteFile(ctx, fs, schema, nameMapping, info.file, info.fieldIDs)
if err != nil {
return err
}
@@ -187,7 +319,7 @@ func readAllEqualityDeleteFiles(ctx context.Context, fs iceio.IO, schema *iceber
// readEqualityDeleteFile reads a single equality delete file and returns
// the set of encoded delete keys and the column names used.
-func readEqualityDeleteFile(ctx context.Context, fs iceio.IO, tableSchema *iceberg.Schema, dataFile iceberg.DataFile, fieldIDs []int) (set[string], []string, error) {
+func readEqualityDeleteFile(ctx context.Context, fs iceio.IO, tableSchema *iceberg.Schema, nameMapping iceberg.NameMapping, dataFile iceberg.DataFile, fieldIDs []int) (set[string], []string, error) {
src, err := internal.GetFile(ctx, fs, dataFile, true)
if err != nil {
return nil, nil, err
@@ -205,23 +337,50 @@ func readEqualityDeleteFile(ctx context.Context, fs iceio.IO, tableSchema *icebe
}
defer tbl.Release()
+ hasFieldIDs, err := VisitArrowSchema(tbl.Schema(), hasIDs{})
+ if err != nil {
+ return nil, nil, err
+ }
+
+ var fileSchema *iceberg.Schema
+ if !hasFieldIDs {
+ if nameMapping == nil {
+ nameMapping = tableSchema.NameMapping()
+ }
+
+ fileSchema, err = ArrowSchemaToIcebergWithOptions(tbl.Schema(), ArrowToIcebergOptions{
+ NameMapping: nameMapping,
+ TableSchema: tableSchema,
+ })
+ if err != nil {
+ return nil, nil, err
+ }
+ }
+
+ var fieldRefsByID arrowFieldRefsByID
+ if hasFieldIDs {
+ fieldRefsByID = indexArrowFieldsByMetadata(tbl.Schema())
+ } else {
+ fieldRefsByID = indexArrowFields(fileSchema)
+ }
+
// Resolve column names from field IDs.
colNames := make([]string, len(fieldIDs))
- colIndices := make([]int, len(fieldIDs))
+ fieldRefs := make([]arrowFieldRef, len(fieldIDs))
for i, fid := range fieldIDs {
name, ok := tableSchema.FindColumnName(fid)
if !ok {
- return nil, nil, fmt.Errorf("equality delete field ID %d not found in table schema", fid)
+ return nil, nil, fmt.Errorf("equality delete field ID %d not found in table schema for %s", fid, dataFile.FilePath())
}
- colNames[i] = name
- indices := tbl.Schema().FieldIndices(name)
- if len(indices) == 0 {
- return nil, nil, fmt.Errorf("equality delete column %q not found in delete file %s", name, dataFile.FilePath())
+ ref, err := resolveArrowField(fieldRefsByID, fid, name, dataFile.FilePath())
+ if err != nil {
+ return nil, nil, err
}
- colIndices[i] = indices[0]
+ colNames[i] = name
+ fieldRefs[i] = ref
}
// Build the set of encoded delete keys by iterating aligned batches.
@@ -234,9 +393,12 @@ func readEqualityDeleteFile(ctx context.Context, fs iceio.IO, tableSchema *icebe
for tr.Next() {
rec := tr.RecordBatch()
- encoders := make([]colEncoder, len(colIndices))
- for i, idx := range colIndices {
- encoders[i] = makeColEncoder(rec.Column(idx))
+ encoders := make([]colEncoder, len(fieldRefs))
+ for i, ref := range fieldRefs {
+ encoders[i], err = makeArrowFieldEncoder(rec, ref, fieldIDs[i], colNames[i], dataFile.FilePath())
+ if err != nil {
+ return nil, nil, err
+ }
}
numRows := int(rec.NumRows())
@@ -346,13 +508,6 @@ func encodeArrowValue(buf *bytes.Buffer, arr arrow.Array, idx int) {
}
}
-// processEqualityDeletes returns a pipeline function that filters out
-// rows whose equality key columns match any entry in the delete sets.
-// Each set is applied independently (they may have different field IDs).
-func processEqualityDeletes(ctx context.Context, eqDeleteSets []*equalityDeleteSet) (recProcessFn, error) {
- return processEqualityDeletesColumnar(ctx, eqDeleteSets)
-}
-
// colEncoder writes the value at row idx to buf. Resolved once per column
// to avoid per-row type switches.
type colEncoder func(buf *bytes.Buffer, row int)
@@ -524,9 +679,30 @@ func makeColEncoder(arr arrow.Array) colEncoder {
}
}
-// processEqualityDeletesColumnar resolves typed column encoders once per
-// batch, then iterates rows without per-row type switches.
-func processEqualityDeletesColumnar(ctx context.Context, eqDeleteSets []*equalityDeleteSet) (recProcessFn, error) {
+// processEqualityDeletesColumnarForFile resolves field paths once per file and
+// typed column encoders once per batch, then iterates rows without per-row type
+// switches. Each delete set is applied independently because sets may have
+// different field IDs.
+func processEqualityDeletesColumnarForFile(ctx context.Context, eqDeleteSets []*equalityDeleteSet, fileSchema *iceberg.Schema, dataFilePath string) (recProcessFn, error) {
+ fieldRefsByID := indexArrowFields(fileSchema)
+ fieldRefs := make([][]arrowFieldRef, len(eqDeleteSets))
+ for i, eqDel := range eqDeleteSets {
+ if len(eqDel.fieldIDs) != len(eqDel.colNames) {
+ return nil, fmt.Errorf("%w: equality delete set has %d field IDs and %d column names",
+ iceberg.ErrInvalidArgument, len(eqDel.fieldIDs), len(eqDel.colNames))
+ }
+
+ fieldRefs[i] = make([]arrowFieldRef, len(eqDel.fieldIDs))
+ for fieldIdx, fieldID := range eqDel.fieldIDs {
+ ref, err := resolveArrowField(fieldRefsByID, fieldID, eqDel.colNames[fieldIdx], dataFilePath)
+ if err != nil {
+ return nil, err
+ }
+
+ fieldRefs[i][fieldIdx] = ref
+ }
+ }
+
return func(r arrow.RecordBatch) (arrow.RecordBatch, error) {
defer r.Release()
@@ -544,15 +720,14 @@ func processEqualityDeletesColumnar(ctx context.Context, eqDeleteSets []*equalit
var keyBuf bytes.Buffer
- for _, eqDel := range eqDeleteSets {
+ for setIdx, eqDel := range eqDeleteSets {
encoders := make([]colEncoder, len(eqDel.colNames))
for i, name := range eqDel.colNames {
- indices := r.Schema().FieldIndices(name)
- if len(indices) == 0 {
- return nil, fmt.Errorf("equality delete column %q not found in data record", name)
+ var err error
+ encoders[i], err = makeArrowFieldEncoder(r, fieldRefs[setIdx][i], eqDel.fieldIDs[i], name, dataFilePath)
+ if err != nil {
+ return nil, err
}
-
- encoders[i] = makeColEncoder(r.Column(indices[0]))
}
for row := 0; row < numRows; row++ {
diff --git a/table/equality_delete_reader_bench_test.go b/table/equality_delete_reader_bench_test.go
index b86c3cf36..de0c0f697 100644
--- a/table/equality_delete_reader_bench_test.go
+++ b/table/equality_delete_reader_bench_test.go
@@ -28,6 +28,7 @@ import (
"github.com/apache/arrow-go/v18/arrow/array"
"github.com/apache/arrow-go/v18/arrow/compute"
"github.com/apache/arrow-go/v18/arrow/memory"
+ "github.com/apache/iceberg-go"
)
func benchEqDeletes(b *testing.B, buildRec func(memory.Allocator, int) arrow.RecordBatch, buildDel func(int) *equalityDeleteSet) {
@@ -49,7 +50,11 @@ func benchEqDeletes(b *testing.B, buildRec func(memory.Allocator, int) arrow.Rec
defer rec.Release()
delSet := buildDel(nDel)
- filterFn, err := processEqualityDeletes(ctx, []*equalityDeleteSet{delSet})
+ fileSchema := iceberg.NewSchema(0,
+ iceberg.NestedField{ID: 1, Name: "first", Type: iceberg.PrimitiveTypes.Int64},
+ iceberg.NestedField{ID: 2, Name: "second", Type: iceberg.PrimitiveTypes.Int64},
+ )
+ filterFn, err := processEqualityDeletesColumnarForFile(ctx, []*equalityDeleteSet{delSet}, fileSchema, "data.parquet")
if err != nil {
b.Fatal(err)
}
diff --git a/table/equality_delete_reader_internal_test.go b/table/equality_delete_reader_internal_test.go
index e446ad55f..b6764e80d 100644
--- a/table/equality_delete_reader_internal_test.go
+++ b/table/equality_delete_reader_internal_test.go
@@ -19,6 +19,8 @@ package table
import (
"bytes"
+ "context"
+ "errors"
"testing"
"github.com/apache/arrow-go/v18/arrow"
@@ -74,6 +76,42 @@ func TestMakeColEncoderMatchesGenericForNullFastPathTypes(t *testing.T) {
}
}
+func TestMakeArrowFieldEncoderHonorsNullStructParents(t *testing.T) {
+ mem := memory.NewCheckedAllocator(memory.DefaultAllocator)
+ defer mem.AssertSize(t, 0)
+
+ childBuilder := array.NewInt64Builder(mem)
+ childBuilder.Append(123)
+ childBuilder.Append(456)
+ child := childBuilder.NewArray()
+ childBuilder.Release()
+ defer child.Release()
+ structArray, err := array.NewStructArrayWithNulls(
+ []arrow.Array{child}, []string{"id"}, memory.NewBufferBytes([]byte{0x02}), 1, 0)
+ require.NoError(t, err)
+ structType := structArray.DataType().(*arrow.StructType)
+
+ schema := arrow.NewSchema([]arrow.Field{{Name: "person", Type: structType}}, nil)
+ record := array.NewRecordBatch(schema, []arrow.Array{structArray}, 2)
+ structArray.Release()
+ defer record.Release()
+
+ assert.False(t, record.Column(0).(*array.Struct).Field(0).IsNull(0), "child storage must remain valid under the null parent")
+
+ encoder, err := makeArrowFieldEncoder(record, arrowFieldRef{path: []int{0, 0}}, 1, "person.id", "")
+ require.NoError(t, err)
+
+ var got bytes.Buffer
+ encoder(&got, 0)
+ assert.Equal(t, []byte{0}, got.Bytes())
+
+ got.Reset()
+ encoder(&got, 1)
+ var want bytes.Buffer
+ encodeArrowValue(&want, record.Column(0).(*array.Struct).Field(0), 1)
+ assert.Equal(t, want.Bytes(), got.Bytes())
+}
+
type encoderArrayShape int
const (
@@ -186,9 +224,141 @@ func TestReadAllEqualityDeleteFilesRejectsEmptyEqualityFieldIDs(t *testing.T) {
t.Context(),
iceio.NewMemFS(),
schema,
+ nil,
[]FileScanTask{{EqualityDeleteFiles: []iceberg.DataFile{deleteFile}}},
1,
)
require.ErrorIs(t, err, ErrEmptyEqualityFieldIDs)
require.ErrorContains(t, err, "empty-equality-fields.parquet")
}
+
+func TestProcessEqualityDeletesUsesStructuralFieldPaths(t *testing.T) {
+ builder := array.NewInt64Builder(memory.DefaultAllocator)
+ builder.Append(1)
+ first := builder.NewArray()
+ builder.Append(2)
+ second := builder.NewArray()
+ builder.Release()
+ var key bytes.Buffer
+ encodeArrowValue(&key, first, 0)
+
+ schema := arrow.NewSchema([]arrow.Field{
+ {Name: "id", Type: arrow.PrimitiveTypes.Int64},
+ {Name: "id", Type: arrow.PrimitiveTypes.Int64},
+ }, nil)
+ record := array.NewRecordBatch(schema, []arrow.Array{first, second}, 1)
+ first.Release()
+ second.Release()
+
+ fileSchema := iceberg.NewSchema(0,
+ iceberg.NestedField{ID: 1, Name: "left", Type: iceberg.PrimitiveTypes.Int64},
+ iceberg.NestedField{ID: 2, Name: "right", Type: iceberg.PrimitiveTypes.Int64},
+ )
+ process, err := processEqualityDeletesColumnarForFile(context.Background(), []*equalityDeleteSet{{
+ keys: set[string]{key.String(): {}},
+ fieldIDs: []int{1},
+ colNames: []string{"id"},
+ }}, fileSchema, "data.parquet")
+ require.NoError(t, err)
+
+ result, err := process(record)
+ require.NoError(t, err)
+ assert.Zero(t, result.NumRows())
+ result.Release()
+}
+
+func TestProcessEqualityDeletesRejectsMismatchedFieldMetadata(t *testing.T) {
+ process, err := processEqualityDeletesColumnarForFile(context.Background(), []*equalityDeleteSet{{
+ keys: make(set[string]),
+ fieldIDs: []int{1},
+ colNames: []string{"id", "other"},
+ }}, iceberg.NewSchema(0), "data.parquet")
+
+ require.ErrorIs(t, err, iceberg.ErrInvalidArgument)
+ assert.Nil(t, process)
+}
+
+func TestResolveArrowFieldUsesFieldIDBeforeName(t *testing.T) {
+ schema := iceberg.NewSchema(0,
+ iceberg.NestedField{ID: 1, Name: "id", Type: iceberg.PrimitiveTypes.Int64},
+ iceberg.NestedField{ID: 2, Name: "renamed", Type: iceberg.PrimitiveTypes.Int64},
+ iceberg.NestedField{ID: 4, Name: "record", Type: &iceberg.StructType{FieldList: []iceberg.NestedField{
+ {ID: 3, Name: "value", Type: iceberg.PrimitiveTypes.Int64},
+ }}},
+ )
+
+ refs := indexArrowFields(schema)
+ ref, err := resolveArrowField(refs, 2, "id", "data.parquet")
+ require.NoError(t, err)
+ assert.Equal(t, []int{1}, ref.path)
+
+ ref, err = resolveArrowField(refs, 3, "record.value", "data.parquet")
+ require.NoError(t, err)
+ assert.Equal(t, []int{2, 0}, ref.path)
+}
+
+func TestResolveArrowFieldRejectsAmbiguousOrMismatchedIDs(t *testing.T) {
+ tests := []struct {
+ name string
+ schema *iceberg.Schema
+ fieldID int
+ fieldName string
+ wantPath []int
+ wantErr error
+ }{
+ {
+ name: "renamed field uses matching ID",
+ schema: iceberg.NewSchema(0, iceberg.NestedField{
+ ID: 7, Name: "renamed", Type: iceberg.PrimitiveTypes.Int64,
+ }),
+ fieldID: 7, fieldName: "old_name", wantPath: []int{0},
+ },
+ {
+ name: "missing ID is rejected even if a name matches",
+ schema: iceberg.NewSchema(0, iceberg.NestedField{
+ ID: 2, Name: "id", Type: iceberg.PrimitiveTypes.Int64,
+ }),
+ fieldID: 1, fieldName: "id", wantErr: errors.New("not found"),
+ },
+ {
+ name: "literal dotted name is distinct from nested path",
+ schema: iceberg.NewSchema(0,
+ iceberg.NestedField{ID: 1, Name: "user.id", Type: iceberg.PrimitiveTypes.Int64},
+ iceberg.NestedField{ID: 2, Name: "user", Type: &iceberg.StructType{FieldList: []iceberg.NestedField{
+ {ID: 3, Name: "id", Type: iceberg.PrimitiveTypes.Int64},
+ }}},
+ ),
+ fieldID: 1, fieldName: "user.id", wantPath: []int{0},
+ },
+ }
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ ref, err := resolveArrowField(indexArrowFields(test.schema), test.fieldID, test.fieldName, "data.parquet")
+ if test.wantErr != nil {
+ require.Error(t, err)
+ if test.wantErr == ErrAmbiguousEqualityColumn {
+ assert.ErrorIs(t, err, test.wantErr)
+ }
+ assert.Contains(t, err.Error(), test.wantErr.Error())
+ assert.Contains(t, err.Error(), "data.parquet")
+
+ return
+ }
+
+ require.NoError(t, err)
+ assert.Equal(t, test.wantPath, ref.path)
+ })
+ }
+}
+
+func TestResolveArrowFieldRejectsDuplicateMetadataIDs(t *testing.T) {
+ schema := arrow.NewSchema([]arrow.Field{
+ {Name: "left", Type: arrow.PrimitiveTypes.Int64, Metadata: arrow.MetadataFrom(map[string]string{ArrowParquetFieldIDKey: "1"})},
+ {Name: "right", Type: arrow.PrimitiveTypes.Int64, Metadata: arrow.MetadataFrom(map[string]string{ArrowParquetFieldIDKey: "1"})},
+ }, nil)
+
+ _, err := resolveArrowField(indexArrowFieldsByMetadata(schema), 1, "id", "data.parquet")
+ require.ErrorIs(t, err, ErrAmbiguousEqualityColumn)
+ require.ErrorContains(t, err, "data.parquet")
+}
diff --git a/table/equality_delete_reader_test.go b/table/equality_delete_reader_test.go
index 278086547..2b9f937a3 100644
--- a/table/equality_delete_reader_test.go
+++ b/table/equality_delete_reader_test.go
@@ -19,6 +19,7 @@ package table_test
import (
"context"
+ "encoding/json"
"path/filepath"
"strings"
"testing"
@@ -26,6 +27,8 @@ import (
"github.com/apache/arrow-go/v18/arrow"
"github.com/apache/arrow-go/v18/arrow/array"
"github.com/apache/arrow-go/v18/arrow/memory"
+ "github.com/apache/arrow-go/v18/parquet"
+ "github.com/apache/arrow-go/v18/parquet/pqarrow"
"github.com/apache/iceberg-go"
iceio "github.com/apache/iceberg-go/io"
"github.com/apache/iceberg-go/table"
@@ -36,16 +39,26 @@ import (
func newEqDeleteReadTestTable(t *testing.T) *table.Table {
t.Helper()
- location := filepath.ToSlash(t.TempDir())
-
iceSchema := iceberg.NewSchema(0,
iceberg.NestedField{ID: 1, Name: "id", Type: iceberg.PrimitiveTypes.Int64, Required: true},
iceberg.NestedField{ID: 2, Name: "data", Type: iceberg.PrimitiveTypes.String, Required: false},
)
+ return newEqDeleteReadTestTableWithSchema(t, iceSchema, nil)
+}
+
+func newEqDeleteReadTestTableWithSchema(t *testing.T, iceSchema *iceberg.Schema, properties iceberg.Properties) *table.Table {
+ t.Helper()
+
+ location := filepath.ToSlash(t.TempDir())
+ if properties == nil {
+ properties = iceberg.Properties{}
+ }
+ properties[table.PropertyFormatVersion] = "2"
+
meta, err := table.NewMetadata(iceSchema, iceberg.UnpartitionedSpec,
table.UnsortedSortOrder, location,
- iceberg.Properties{table.PropertyFormatVersion: "2"})
+ properties)
require.NoError(t, err)
return table.New(
@@ -58,6 +71,15 @@ func newEqDeleteReadTestTable(t *testing.T) *table.Table {
)
}
+func nameMappingProperties(t *testing.T, mapping iceberg.NameMapping) iceberg.Properties {
+ t.Helper()
+
+ mappingJSON, err := json.Marshal(mapping)
+ require.NoError(t, err)
+
+ return iceberg.Properties{table.DefaultNameMappingKey: string(mappingJSON)}
+}
+
func TestEqualityDeleteReadRoundTrip(t *testing.T) {
tbl := newEqDeleteReadTestTable(t)
@@ -122,6 +144,238 @@ func TestEqualityDeleteReadRoundTrip(t *testing.T) {
assert.Equal(t, []int64{1, 3, 5}, ids, "expected rows with id=2 and id=4 deleted")
}
+func TestEqualityDeleteReadResolvesRenamedDataColumnByFieldID(t *testing.T) {
+ tbl := newEqDeleteReadTestTable(t)
+
+ dataSchema, err := table.SchemaToArrowSchema(tbl.Metadata().CurrentSchema(), nil, true, false)
+ require.NoError(t, err)
+ dataFields := dataSchema.Fields()
+ dataFields[0].Name = "legacy_id"
+ dataMetadata := dataSchema.Metadata()
+ dataSchema = arrow.NewSchema(dataFields, &dataMetadata)
+
+ dataPath := tbl.Location() + "/data/data-renamed.parquet"
+ writeParquetFile(t, dataPath, dataSchema, `[
+ {"legacy_id": 1, "data": "one"},
+ {"legacy_id": 2, "data": "two"},
+ {"legacy_id": 3, "data": "three"}
+ ]`)
+
+ tx := tbl.NewTransaction()
+ require.NoError(t, tx.AddFiles(t.Context(), []string{dataPath}, nil, false))
+ tbl, err = tx.Commit(t.Context())
+ require.NoError(t, err)
+
+ deleteSchema, err := table.SchemaToArrowSchema(
+ iceberg.NewSchema(0, iceberg.NestedField{ID: 1, Name: "id", Type: iceberg.PrimitiveTypes.Int64, Required: true}),
+ nil, true, false)
+ require.NoError(t, err)
+ deletePath := tbl.Location() + "/data/delete-renamed.parquet"
+ writeParquetFile(t, deletePath, deleteSchema, `[{"id": 2}]`)
+
+ deleteBuilder, err := iceberg.NewDataFileBuilder(
+ *iceberg.UnpartitionedSpec, iceberg.EntryContentEqDeletes,
+ deletePath, iceberg.ParquetFile, nil, nil, nil, 1, 128)
+ require.NoError(t, err)
+ deleteBuilder.EqualityFieldIDs([]int{1})
+
+ tx = tbl.NewTransaction()
+ rowDelta := tx.NewRowDelta(nil)
+ rowDelta.AddDeletes(deleteBuilder.Build())
+ require.NoError(t, rowDelta.Commit(t.Context()))
+ tbl, err = tx.Commit(t.Context())
+ require.NoError(t, err)
+
+ _, records, err := tbl.Scan(table.WithSelectedFields("id")).ToArrowRecords(t.Context())
+ require.NoError(t, err)
+
+ var ids []int64
+ for record, err := range records {
+ require.NoError(t, err)
+ column := record.Column(0).(*array.Int64)
+ for i := 0; i < column.Len(); i++ {
+ ids = append(ids, column.Value(i))
+ }
+ record.Release()
+ }
+
+ assert.Equal(t, []int64{1, 3}, ids)
+}
+
+func TestEqualityDeleteReadResolvesRenamedDataColumnByNameMapping(t *testing.T) {
+ iceSchema := iceberg.NewSchema(0,
+ iceberg.NestedField{ID: 1, Name: "id", Type: iceberg.PrimitiveTypes.Int64, Required: true},
+ iceberg.NestedField{ID: 2, Name: "data", Type: iceberg.PrimitiveTypes.String},
+ )
+ mapping := iceSchema.NameMapping()
+ mapping[0].Names = append(mapping[0].Names, "legacy_id")
+ tbl := newEqDeleteReadTestTableWithSchema(t, iceSchema, nameMappingProperties(t, mapping))
+
+ dataSchema := arrow.NewSchema([]arrow.Field{
+ {Name: "legacy_id", Type: arrow.PrimitiveTypes.Int64, Nullable: false},
+ {Name: "data", Type: arrow.BinaryTypes.String, Nullable: true},
+ }, nil)
+ dataPath := tbl.Location() + "/data/data-name-mapped.parquet"
+ writeParquetFile(t, dataPath, dataSchema, `[
+ {"legacy_id": 1, "data": "one"},
+ {"legacy_id": 2, "data": "two"},
+ {"legacy_id": 3, "data": "three"}
+ ]`)
+ dataBuilder, err := iceberg.NewDataFileBuilder(
+ *iceberg.UnpartitionedSpec, iceberg.EntryContentData,
+ dataPath, iceberg.ParquetFile, nil, nil, nil, 3, mustFileSize(t, dataPath))
+ require.NoError(t, err)
+
+ tx := tbl.NewTransaction()
+ require.NoError(t, tx.AddDataFiles(t.Context(), []iceberg.DataFile{dataBuilder.Build()}, nil))
+ tbl, err = tx.Commit(t.Context())
+ require.NoError(t, err)
+
+ deleteSchema, err := table.SchemaToArrowSchema(
+ iceberg.NewSchema(0, iceberg.NestedField{ID: 1, Name: "id", Type: iceberg.PrimitiveTypes.Int64, Required: true}),
+ nil, true, false)
+ require.NoError(t, err)
+ deletePath := tbl.Location() + "/data/delete-name-mapped.parquet"
+ writeParquetFile(t, deletePath, deleteSchema, `[{"id": 2}]`)
+
+ deleteBuilder, err := iceberg.NewDataFileBuilder(
+ *iceberg.UnpartitionedSpec, iceberg.EntryContentEqDeletes,
+ deletePath, iceberg.ParquetFile, nil, nil, nil, 1, 128)
+ require.NoError(t, err)
+ deleteBuilder.EqualityFieldIDs([]int{1})
+
+ tx = tbl.NewTransaction()
+ rowDelta := tx.NewRowDelta(nil)
+ rowDelta.AddDeletes(deleteBuilder.Build())
+ require.NoError(t, rowDelta.Commit(t.Context()))
+ tbl, err = tx.Commit(t.Context())
+ require.NoError(t, err)
+
+ _, records, err := tbl.Scan(table.WithSelectedFields("id")).ToArrowRecords(t.Context())
+ require.NoError(t, err)
+
+ var ids []int64
+ for record, err := range records {
+ require.NoError(t, err)
+ column := record.Column(0).(*array.Int64)
+ for i := 0; i < column.Len(); i++ {
+ ids = append(ids, column.Value(i))
+ }
+ record.Release()
+ }
+
+ assert.Equal(t, []int64{1, 3}, ids)
+}
+
+func TestEqualityDeleteReadResolvesLiteralDottedColumnName(t *testing.T) {
+ iceSchema := iceberg.NewSchema(0,
+ iceberg.NestedField{ID: 1, Name: "user.id", Type: iceberg.PrimitiveTypes.Int64, Required: true},
+ iceberg.NestedField{ID: 2, Name: "data", Type: iceberg.PrimitiveTypes.String},
+ )
+ tbl := newEqDeleteReadTestTableWithSchema(t, iceSchema, nameMappingProperties(t, iceSchema.NameMapping()))
+
+ dataSchema := arrow.NewSchema([]arrow.Field{
+ {Name: "user.id", Type: arrow.PrimitiveTypes.Int64, Nullable: false},
+ {Name: "data", Type: arrow.BinaryTypes.String, Nullable: true},
+ }, nil)
+ dataPath := tbl.Location() + "/data/data-dotted-name.parquet"
+ writeParquetFile(t, dataPath, dataSchema, `[
+ {"user.id": 1, "data": "one"},
+ {"user.id": 2, "data": "two"},
+ {"user.id": 3, "data": "three"}
+ ]`)
+
+ tx := tbl.NewTransaction()
+ require.NoError(t, tx.AddFiles(t.Context(), []string{dataPath}, nil, false))
+ tbl, err := tx.Commit(t.Context())
+ require.NoError(t, err)
+
+ deleteSchema, err := table.SchemaToArrowSchema(
+ iceberg.NewSchema(0, iceberg.NestedField{ID: 1, Name: "user.id", Type: iceberg.PrimitiveTypes.Int64, Required: true}),
+ nil, true, false)
+ require.NoError(t, err)
+ deletePath := tbl.Location() + "/data/delete-dotted-name.parquet"
+ writeParquetFile(t, deletePath, deleteSchema, `[{"user.id": 2}]`)
+
+ deleteBuilder, err := iceberg.NewDataFileBuilder(
+ *iceberg.UnpartitionedSpec, iceberg.EntryContentEqDeletes,
+ deletePath, iceberg.ParquetFile, nil, nil, nil, 1, 128)
+ require.NoError(t, err)
+ deleteBuilder.EqualityFieldIDs([]int{1})
+
+ tx = tbl.NewTransaction()
+ rowDelta := tx.NewRowDelta(nil)
+ rowDelta.AddDeletes(deleteBuilder.Build())
+ require.NoError(t, rowDelta.Commit(t.Context()))
+ tbl, err = tx.Commit(t.Context())
+ require.NoError(t, err)
+
+ _, records, err := tbl.Scan().ToArrowRecords(t.Context())
+ require.NoError(t, err)
+
+ var ids []int64
+ for record, err := range records {
+ require.NoError(t, err)
+ column := record.Column(0).(*array.Int64)
+ for i := 0; i < column.Len(); i++ {
+ ids = append(ids, column.Value(i))
+ }
+ record.Release()
+ }
+
+ assert.Equal(t, []int64{1, 3}, ids)
+}
+
+func TestEqualityDeleteReadRejectsAmbiguousColumns(t *testing.T) {
+ tbl := newEqDeleteReadTestTable(t)
+ arrowSc, err := table.SchemaToArrowSchema(tbl.Metadata().CurrentSchema(), nil, false, false)
+ require.NoError(t, err)
+ dataPath := tbl.Location() + "/data/data.parquet"
+ writeParquetFile(t, dataPath, arrowSc, `[{"id": 1, "data": "one"}]`)
+ tx := tbl.NewTransaction()
+ require.NoError(t, tx.AddFiles(t.Context(), []string{dataPath}, nil, false))
+ tbl, err = tx.Commit(t.Context())
+ require.NoError(t, err)
+
+ deleteSchema, err := table.SchemaToArrowSchema(
+ iceberg.NewSchema(0, iceberg.NestedField{ID: 1, Name: "id", Type: iceberg.PrimitiveTypes.Int64, Required: true}),
+ nil, true, false)
+ require.NoError(t, err)
+ duplicateSchema := arrow.NewSchema([]arrow.Field{deleteSchema.Field(0), deleteSchema.Field(0)}, nil)
+ deletePath := tbl.Location() + "/data/delete.parquet"
+ builder := array.NewInt64Builder(memory.DefaultAllocator)
+ builder.Append(1)
+ first := builder.NewArray()
+ builder.Append(2)
+ second := builder.NewArray()
+ builder.Release()
+ batch := array.NewRecordBatch(duplicateSchema, []arrow.Array{first, second}, 1)
+ first.Release()
+ second.Release()
+ deleteTable := array.NewTableFromRecords(duplicateSchema, []arrow.RecordBatch{batch})
+ batch.Release()
+ file, err := iceio.LocalFS{}.Create(deletePath)
+ require.NoError(t, err)
+ defer file.Close()
+ require.NoError(t, pqarrow.WriteTable(deleteTable, file, 1,
+ parquet.NewWriterProperties(parquet.WithStats(true)), pqarrow.DefaultWriterProps()))
+ deleteTable.Release()
+ deleteBuilder, err := iceberg.NewDataFileBuilder(
+ *iceberg.UnpartitionedSpec, iceberg.EntryContentEqDeletes,
+ deletePath, iceberg.ParquetFile, nil, nil, nil, 1, 128)
+ require.NoError(t, err)
+ deleteBuilder.EqualityFieldIDs([]int{1})
+ tx = tbl.NewTransaction()
+ rd := tx.NewRowDelta(nil)
+ rd.AddDeletes(deleteBuilder.Build())
+ require.NoError(t, rd.Commit(t.Context()))
+ tbl, err = tx.Commit(t.Context())
+ require.NoError(t, err)
+
+ _, _, err = tbl.Scan().ToArrowRecords(t.Context())
+ require.ErrorIs(t, err, table.ErrAmbiguousEqualityColumn)
+}
+
func TestEqualityDeleteDoesNotAffectSameSnapshot(t *testing.T) {
tbl := newEqDeleteReadTestTable(t)