You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by ze...@apache.org on 2026/08/17 17:33:09 UTC
(iceberg-go) branch main updated: fix(table): resolve write planning snapshots from the target branch head (#1638) (#1839)
This is an automated email from the ASF dual-hosted git repository.
zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-go.git
The following commit(s) were added to refs/heads/main by this push:
new 3dcacdab4 fix(table): resolve write planning snapshots from the target branch head (#1638) (#1839)
3dcacdab4 is described below
commit 3dcacdab463cb6865993e7645bc2a425c6670a45
Author: Badal Prasad Singh <ba...@datazip.io>
AuthorDate: Mon Aug 17 23:03:04 2026 +0530
fix(table): resolve write planning snapshots from the target branch head (#1638) (#1839)
fix: branch operations
Signed-off-by: badalprasadsingh <ba...@datazip.io>
---
table/branch_planning_test.go | 634 +++++++++++++++++++++++++++++++++++++
table/metadata.go | 7 +-
table/mor_delete_pruning_test.go | 26 +-
table/rebuild_manifest_test.go | 77 ++++-
table/rewrite_manifests.go | 2 +-
table/row_delta.go | 2 +-
table/table.go | 36 ++-
table/transaction.go | 60 +---
table/transaction_internal_test.go | 44 +--
9 files changed, 772 insertions(+), 116 deletions(-)
diff --git a/table/branch_planning_test.go b/table/branch_planning_test.go
new file mode 100644
index 000000000..d61b117d8
--- /dev/null
+++ b/table/branch_planning_test.go
@@ -0,0 +1,634 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package table_test
+
+// Black-box coverage for branch-aware read-side planning. Every fixture gives
+// the branch and main a file the other lacks, so a plan resolved against main is
+// observably wrong rather than accidentally identical.
+
+import (
+ "context"
+ "fmt"
+ "maps"
+ "path/filepath"
+ "slices"
+ "strings"
+ "testing"
+
+ "github.com/apache/arrow-go/v18/arrow/array"
+ "github.com/apache/arrow-go/v18/arrow/memory"
+ "github.com/apache/iceberg-go"
+ iceio "github.com/apache/iceberg-go/io"
+ "github.com/apache/iceberg-go/table"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+const branchPlanningRef = "feature"
+
+var branchPlanningIdent = table.Identifier{"db", "branch_planning"}
+
+func branchPlanningFS(context.Context) (iceio.IO, error) { return iceio.LocalFS{}, nil }
+
+// newBranchPlanningMetadata builds the fixture's single-column metadata in a
+// fresh location. props overrides the defaults: format version 2, copy-on-write deletes.
+func newBranchPlanningMetadata(t *testing.T, props iceberg.Properties) (location string, meta table.Metadata) {
+ t.Helper()
+
+ location = filepath.ToSlash(t.TempDir())
+ schema := iceberg.NewSchema(0,
+ iceberg.NestedField{ID: 1, Name: "id", Type: iceberg.PrimitiveTypes.Int64, Required: true},
+ )
+
+ tblProps := iceberg.Properties{table.PropertyFormatVersion: "2"}
+ maps.Copy(tblProps, props)
+
+ meta, err := table.NewMetadata(schema, iceberg.UnpartitionedSpec, table.UnsortedSortOrder, location, tblProps)
+ require.NoError(t, err)
+
+ return location, meta
+}
+
+// newBranchPlanningTable wraps that metadata in a table whose catalog stub
+// validates commit requirements.
+func newBranchPlanningTable(t *testing.T, props iceberg.Properties) *table.Table {
+ t.Helper()
+
+ location, meta := newBranchPlanningMetadata(t, props)
+ metaLoc := location + "/metadata/v1.metadata.json"
+ cat := &concurrentTestCatalog{metadata: meta, location: metaLoc, fsF: branchPlanningFS}
+
+ return table.New(branchPlanningIdent, meta, metaLoc, branchPlanningFS, cat)
+}
+
+// idRecordReader returns a reader the caller must release.
+func idRecordReader(t *testing.T, tbl *table.Table, ids ...int64) array.RecordReader {
+ t.Helper()
+
+ arrowSchema, err := table.SchemaToArrowSchema(tbl.Schema(), nil, false, false)
+ require.NoError(t, err)
+
+ rows := make([]string, 0, len(ids))
+ for _, id := range ids {
+ rows = append(rows, fmt.Sprintf(`{"id":%d}`, id))
+ }
+
+ data, err := array.TableFromJSON(memory.DefaultAllocator, arrowSchema,
+ []string{"[" + strings.Join(rows, ",") + "]"})
+ require.NoError(t, err)
+ t.Cleanup(data.Release)
+
+ return array.NewTableReader(data, -1)
+}
+
+func appendRowsOnRef(t *testing.T, tbl *table.Table, ref string, ids ...int64) *table.Table {
+ t.Helper()
+
+ txn, err := tbl.NewTransactionOnBranchWithError(ref)
+ require.NoError(t, err)
+
+ rdr := idRecordReader(t, tbl, ids...)
+ defer rdr.Release()
+
+ require.NoError(t, txn.Append(t.Context(), rdr, nil))
+ committed, err := txn.Commit(t.Context())
+ require.NoError(t, err)
+
+ return committed
+}
+
+// seedDivergentBranch leaves main holding {1,2,5,6} and the branch {1,2,3,4},
+// so each ref owns one data file the other does not reference.
+func seedDivergentBranch(t *testing.T, tbl *table.Table) *table.Table {
+ t.Helper()
+
+ tbl = appendRowsOnRef(t, tbl, table.MainBranch, 1, 2)
+ tbl = appendRowsOnRef(t, tbl, branchPlanningRef, 3, 4)
+ tbl = appendRowsOnRef(t, tbl, table.MainBranch, 5, 6)
+
+ require.ElementsMatch(t, []int64{1, 2, 3, 4}, idsOnRef(t, tbl, branchPlanningRef))
+ require.ElementsMatch(t, []int64{1, 2, 5, 6}, idsOnRef(t, tbl, table.MainBranch))
+
+ return tbl
+}
+
+func idsOnRef(t *testing.T, tbl *table.Table, ref string) []int64 {
+ t.Helper()
+
+ scan, err := tbl.Scan().UseRef(ref)
+ require.NoError(t, err)
+
+ _, itr, err := scan.ToArrowRecords(t.Context())
+ require.NoError(t, err)
+
+ var ids []int64
+ for rec, err := range itr {
+ require.NoError(t, err)
+ idIdx := rec.Schema().FieldIndices("id")
+ require.NotEmpty(t, idIdx)
+ col, ok := rec.Column(idIdx[0]).(*array.Int64)
+ require.True(t, ok, "id column must be int64")
+ for i := range int(rec.NumRows()) {
+ ids = append(ids, col.Value(i))
+ }
+ rec.Release()
+ }
+ slices.Sort(ids)
+
+ return ids
+}
+
+func liveFilesOnRef(t *testing.T, tbl *table.Table, ref string) (dataPaths, deletePaths []string) {
+ t.Helper()
+
+ fs, err := tbl.FS(t.Context())
+ require.NoError(t, err)
+
+ snapshot := tbl.SnapshotByName(ref)
+ require.NotNil(t, snapshot, "ref %q must have a head", ref)
+
+ manifests, err := snapshot.Manifests(fs)
+ require.NoError(t, err)
+
+ for _, m := range manifests {
+ for entry, err := range m.Entries(fs, false) {
+ require.NoError(t, err)
+ if entry.Status() == iceberg.EntryStatusDELETED {
+ continue
+ }
+ df := entry.DataFile()
+ if df.ContentType() == iceberg.EntryContentData {
+ dataPaths = append(dataPaths, df.FilePath())
+ } else {
+ deletePaths = append(deletePaths, df.FilePath())
+ }
+ }
+ }
+
+ return dataPaths, deletePaths
+}
+
+// liveDVCountOnRef counts the live deletion vectors reachable from ref's head.
+func liveDVCountOnRef(t *testing.T, tbl *table.Table, ref string) int {
+ t.Helper()
+
+ fs, err := tbl.FS(t.Context())
+ require.NoError(t, err)
+
+ snapshot := tbl.SnapshotByName(ref)
+ require.NotNil(t, snapshot)
+
+ manifests, err := snapshot.Manifests(fs)
+ require.NoError(t, err)
+
+ count := 0
+ for _, m := range manifests {
+ for entry, err := range m.Entries(fs, false) {
+ require.NoError(t, err)
+ if entry.Status() == iceberg.EntryStatusDELETED {
+ continue
+ }
+ if df := entry.DataFile(); table.IsDeletionVector(df) && df.ReferencedDataFile() != nil {
+ count++
+ }
+ }
+ }
+
+ return count
+}
+
+// branchOnlyDataPath returns the data file only the branch references.
+func branchOnlyDataPath(t *testing.T, tbl *table.Table) string {
+ t.Helper()
+
+ branchPaths, _ := liveFilesOnRef(t, tbl, branchPlanningRef)
+ mainPaths, _ := liveFilesOnRef(t, tbl, table.MainBranch)
+
+ var only []string
+ for _, p := range branchPaths {
+ if !slices.Contains(mainPaths, p) {
+ only = append(only, p)
+ }
+ }
+ require.Len(t, only, 1, "fixture must give the branch exactly one file of its own")
+
+ return only[0]
+}
+
+// mainOnlyDataPath returns the data file only main references.
+func mainOnlyDataPath(t *testing.T, tbl *table.Table) string {
+ t.Helper()
+
+ branchPaths, _ := liveFilesOnRef(t, tbl, branchPlanningRef)
+ mainPaths, _ := liveFilesOnRef(t, tbl, table.MainBranch)
+
+ var only []string
+ for _, p := range mainPaths {
+ if !slices.Contains(branchPaths, p) {
+ only = append(only, p)
+ }
+ }
+ require.Len(t, only, 1, "fixture must give main exactly one file of its own")
+
+ return only[0]
+}
+
+// Planning against main leaves the branch-only file untouched, so rows the
+// filter matched survive the delete.
+func TestBranchDeleteClassifiesBranchFiles(t *testing.T) {
+ tbl := seedDivergentBranch(t, newBranchPlanningTable(t, nil))
+
+ txn, err := tbl.NewTransactionOnBranchWithError(branchPlanningRef)
+ require.NoError(t, err)
+ require.NoError(t, txn.Delete(t.Context(), iceberg.GreaterThanEqual(iceberg.Reference("id"), int64(3)), nil))
+ tbl, err = txn.Commit(t.Context())
+ require.NoError(t, err)
+
+ assert.Equal(t, []int64{1, 2}, idsOnRef(t, tbl, branchPlanningRef),
+ "the branch-only file holding {3,4} matches the filter and must be deleted")
+ assert.Equal(t, []int64{1, 2, 5, 6}, idsOnRef(t, tbl, table.MainBranch),
+ "a delete on the branch must not move or rewrite main")
+}
+
+// A partial match drives the rewrite arm of classification, which reads the
+// source file and writes its survivors back.
+func TestBranchDeleteRewritesPartiallyMatchedBranchFile(t *testing.T) {
+ tbl := seedDivergentBranch(t, newBranchPlanningTable(t, nil))
+
+ txn, err := tbl.NewTransactionOnBranchWithError(branchPlanningRef)
+ require.NoError(t, err)
+ require.NoError(t, txn.Delete(t.Context(), iceberg.EqualTo(iceberg.Reference("id"), int64(3)), nil))
+ tbl, err = txn.Commit(t.Context())
+ require.NoError(t, err)
+
+ assert.Equal(t, []int64{1, 2, 4}, idsOnRef(t, tbl, branchPlanningRef),
+ "the branch-only file must be rewritten without id 3")
+ assert.Equal(t, []int64{1, 2, 5, 6}, idsOnRef(t, tbl, table.MainBranch))
+}
+
+// A full overwrite must replace everything the branch references; a main-based
+// plan never marks the branch-only file deleted.
+func TestBranchOverwriteReplacesBranchOnlyFiles(t *testing.T) {
+ tbl := seedDivergentBranch(t, newBranchPlanningTable(t, nil))
+
+ txn, err := tbl.NewTransactionOnBranchWithError(branchPlanningRef)
+ require.NoError(t, err)
+
+ rdr := idRecordReader(t, tbl, 9)
+ defer rdr.Release()
+ require.NoError(t, txn.Overwrite(t.Context(), rdr, nil))
+ tbl, err = txn.Commit(t.Context())
+ require.NoError(t, err)
+
+ assert.Equal(t, []int64{9}, idsOnRef(t, tbl, branchPlanningRef),
+ "a full overwrite must retain nothing the branch previously referenced")
+ assert.Equal(t, []int64{1, 2, 5, 6}, idsOnRef(t, tbl, table.MainBranch))
+}
+
+// The first write to a new branch forks from main's head, so the plan must come
+// from that same head or the fork's files survive an overwrite of the branch.
+func TestOverwriteCreatingBranchPlansAgainstMainHead(t *testing.T) {
+ tbl := newBranchPlanningTable(t, nil)
+ tbl = appendRowsOnRef(t, tbl, table.MainBranch, 1, 2)
+ require.Nil(t, tbl.SnapshotByName(branchPlanningRef), "the branch must not exist yet")
+
+ txn, err := tbl.NewTransactionOnBranchWithError(branchPlanningRef)
+ require.NoError(t, err)
+
+ rdr := idRecordReader(t, tbl, 9)
+ defer rdr.Release()
+ require.NoError(t, txn.Overwrite(t.Context(), rdr, nil))
+ tbl, err = txn.Commit(t.Context())
+ require.NoError(t, err)
+
+ mainHead := tbl.SnapshotByName(table.MainBranch)
+ branchHead := tbl.SnapshotByName(branchPlanningRef)
+ require.NotNil(t, mainHead)
+ require.NotNil(t, branchHead)
+ require.NotNil(t, branchHead.ParentSnapshotID)
+ assert.Equal(t, mainHead.SnapshotID, *branchHead.ParentSnapshotID, "the new branch forks from main's head")
+ assert.Equal(t, []int64{9}, idsOnRef(t, tbl, branchPlanningRef),
+ "the fork's files were the plan's input, so none may survive the overwrite")
+ assert.Equal(t, []int64{1, 2}, idsOnRef(t, tbl, table.MainBranch))
+}
+
+// A losing attempt replays the plan it already built, and must rebase on the
+// branch's own fresh head: a peer advancing main between attempts may neither
+// reparent the snapshot nor leak its file onto the branch.
+func TestBranchOverwriteReplaysOnBranchHeadAfterConflict(t *testing.T) {
+ location, meta := newBranchPlanningMetadata(t, iceberg.Properties{
+ table.CommitMinRetryWaitMsKey: "0",
+ table.CommitMaxRetryWaitMsKey: "0",
+ table.CommitNumRetriesKey: "2",
+ })
+ metaLoc := location + "/metadata/v1.metadata.json"
+
+ seedCat := &occScenarioCatalog{current: meta, location: location}
+ tbl := table.New(branchPlanningIdent, meta, metaLoc, branchPlanningFS, seedCat)
+ tbl = appendRowsOnRef(t, tbl, table.MainBranch, 1, 2)
+ tbl = appendRowsOnRef(t, tbl, branchPlanningRef, 3, 4)
+
+ branchHeadBefore := tbl.SnapshotByName(branchPlanningRef)
+ require.NotNil(t, branchHeadBefore)
+
+ appendOnMain := func(current table.Metadata) table.Metadata {
+ peerCat := &occScenarioCatalog{current: current, location: location}
+ peerTbl := table.New(branchPlanningIdent, current, metaLoc, branchPlanningFS, peerCat)
+ appendRowsOnRef(t, peerTbl, table.MainBranch, 7)
+
+ return peerCat.current
+ }
+
+ cat := &occScenarioCatalog{
+ current: seedCat.current, conflictsLeft: 1, location: location, onConflict: appendOnMain,
+ }
+ writer := table.New(branchPlanningIdent, seedCat.current, metaLoc, branchPlanningFS, cat)
+
+ txn, err := writer.NewTransactionOnBranchWithError(branchPlanningRef)
+ require.NoError(t, err)
+ rdr := idRecordReader(t, writer, 9)
+ defer rdr.Release()
+ require.NoError(t, txn.Overwrite(t.Context(), rdr, nil))
+ committed, err := txn.Commit(t.Context())
+ require.NoError(t, err)
+ require.Equal(t, int32(2), cat.commitTableCalls.Load(), "one conflict, then success")
+
+ branchHead := committed.Metadata().SnapshotByName(branchPlanningRef)
+ require.NotNil(t, branchHead)
+ require.NotNil(t, branchHead.ParentSnapshotID)
+ assert.Equal(t, branchHeadBefore.SnapshotID, *branchHead.ParentSnapshotID,
+ "the rebuilt snapshot must stay parented on the branch head")
+ assert.Equal(t, []int64{9}, idsOnRef(t, committed, branchPlanningRef),
+ "the overwrite plan must still hold after the replay")
+ assert.Equal(t, []int64{1, 2, 7}, idsOnRef(t, committed, table.MainBranch),
+ "the peer's row belongs to main alone")
+}
+
+// Filtered overwrite classifies through a different manifest walk than the
+// unfiltered one, so it needs its own coverage.
+func TestBranchFilteredOverwriteClassifiesBranchFiles(t *testing.T) {
+ tbl := seedDivergentBranch(t, newBranchPlanningTable(t, nil))
+
+ txn, err := tbl.NewTransactionOnBranchWithError(branchPlanningRef)
+ require.NoError(t, err)
+
+ rdr := idRecordReader(t, tbl, 7)
+ defer rdr.Release()
+ require.NoError(t, txn.Overwrite(t.Context(), rdr, nil,
+ table.WithOverwriteFilter(iceberg.GreaterThanEqual(iceberg.Reference("id"), int64(3)))))
+ tbl, err = txn.Commit(t.Context())
+ require.NoError(t, err)
+
+ assert.Equal(t, []int64{1, 2, 7}, idsOnRef(t, tbl, branchPlanningRef),
+ "only the branch file matching the filter must be replaced")
+ assert.Equal(t, []int64{1, 2, 5, 6}, idsOnRef(t, tbl, table.MainBranch))
+}
+
+// The duplicate check must ask the branch in both directions: main never seeing
+// a file does not make it addable, and main holding one does not block it.
+func TestBranchAddDataFilesDuplicateCheckUsesBranchHead(t *testing.T) {
+ t.Run("rejects a file the branch already references", func(t *testing.T) {
+ tbl := seedDivergentBranch(t, newBranchPlanningTable(t, nil))
+ branchOnly := branchOnlyDataPath(t, tbl)
+
+ txn, err := tbl.NewTransactionOnBranchWithError(branchPlanningRef)
+ require.NoError(t, err)
+
+ err = txn.AddDataFiles(t.Context(), []iceberg.DataFile{buildDataFile(t, branchOnly)}, nil)
+ require.ErrorContains(t, err, "cannot add files that are already referenced by table")
+ assert.Contains(t, err.Error(), branchOnly)
+ })
+
+ t.Run("accepts a file only main references", func(t *testing.T) {
+ tbl := seedDivergentBranch(t, newBranchPlanningTable(t, nil))
+ mainOnly := mainOnlyDataPath(t, tbl)
+
+ txn, err := tbl.NewTransactionOnBranchWithError(branchPlanningRef)
+ require.NoError(t, err)
+
+ require.NoError(t, txn.AddDataFiles(t.Context(), []iceberg.DataFile{buildDataFile(t, mainOnly)}, nil))
+ tbl, err = txn.Commit(t.Context())
+ require.NoError(t, err)
+
+ branchPaths, _ := liveFilesOnRef(t, tbl, branchPlanningRef)
+ assert.Contains(t, branchPaths, mainOnly, "the branch did not reference the file, so the add must land")
+ })
+}
+
+// AddFiles runs the same check from paths, through its own snapshot lookup.
+func TestBranchAddFilesDuplicateCheckUsesBranchHead(t *testing.T) {
+ t.Run("rejects a file the branch already references", func(t *testing.T) {
+ tbl := seedDivergentBranch(t, newBranchPlanningTable(t, nil))
+ branchOnly := branchOnlyDataPath(t, tbl)
+
+ txn, err := tbl.NewTransactionOnBranchWithError(branchPlanningRef)
+ require.NoError(t, err)
+
+ err = txn.AddFiles(t.Context(), []string{branchOnly}, nil, false)
+ require.ErrorContains(t, err, "cannot add files that are already referenced by table")
+ assert.Contains(t, err.Error(), branchOnly)
+ })
+
+ t.Run("accepts a file only main references", func(t *testing.T) {
+ tbl := seedDivergentBranch(t, newBranchPlanningTable(t, nil))
+ mainOnly := mainOnlyDataPath(t, tbl)
+
+ txn, err := tbl.NewTransactionOnBranchWithError(branchPlanningRef)
+ require.NoError(t, err)
+
+ require.NoError(t, txn.AddFiles(t.Context(), []string{mainOnly}, nil, false))
+ tbl, err = txn.Commit(t.Context())
+ require.NoError(t, err)
+
+ assert.Equal(t, []int64{1, 2, 3, 4, 5, 6}, idsOnRef(t, tbl, branchPlanningRef),
+ "the branch must gain the rows of main's file")
+ })
+}
+
+// A table written only through a branch has no main head, so a main-based plan
+// rejects the replace outright instead of performing it.
+func TestReplaceOnBranchOnlyTable(t *testing.T) {
+ t.Run("ReplaceDataFiles", func(t *testing.T) {
+ tbl := newBranchPlanningTable(t, nil)
+ tbl = appendRowsOnRef(t, tbl, branchPlanningRef, 1, 2)
+ require.Nil(t, tbl.CurrentSnapshot(), "fixture must leave main without a head")
+
+ original, _ := liveFilesOnRef(t, tbl, branchPlanningRef)
+ require.Len(t, original, 1)
+
+ replacement := tbl.Location() + "/data/replacement.parquet"
+ arrowSchema, err := table.SchemaToArrowSchema(tbl.Schema(), nil, false, false)
+ require.NoError(t, err)
+ writeParquetFile(t, replacement, arrowSchema, `[{"id":8}]`)
+
+ txn, err := tbl.NewTransactionOnBranchWithError(branchPlanningRef)
+ require.NoError(t, err)
+ require.NoError(t, txn.ReplaceDataFiles(t.Context(), original, []string{replacement}, nil))
+ tbl, err = txn.Commit(t.Context())
+ require.NoError(t, err)
+
+ assert.Equal(t, []int64{8}, idsOnRef(t, tbl, branchPlanningRef))
+ assert.Nil(t, tbl.CurrentSnapshot(), "replacing on the branch must not create a main head")
+ })
+
+ t.Run("ReplaceDataFilesWithDataFiles", func(t *testing.T) {
+ tbl := newBranchPlanningTable(t, nil)
+ staged := buildDataFile(t, tbl.Location()+"/data/branch-1.parquet")
+
+ txn, err := tbl.NewTransactionOnBranchWithError(branchPlanningRef)
+ require.NoError(t, err)
+ require.NoError(t, txn.AddDataFiles(t.Context(), []iceberg.DataFile{staged}, nil))
+ tbl, err = txn.Commit(t.Context())
+ require.NoError(t, err)
+ require.Nil(t, tbl.CurrentSnapshot(), "fixture must leave main without a head")
+
+ replacement := buildDataFile(t, tbl.Location()+"/data/branch-2.parquet")
+ txn, err = tbl.NewTransactionOnBranchWithError(branchPlanningRef)
+ require.NoError(t, err)
+ require.NoError(t, txn.ReplaceDataFilesWithDataFiles(t.Context(),
+ []iceberg.DataFile{staged}, []iceberg.DataFile{replacement}, nil))
+ tbl, err = txn.Commit(t.Context())
+ require.NoError(t, err)
+
+ dataPaths, _ := liveFilesOnRef(t, tbl, branchPlanningRef)
+ assert.Equal(t, []string{replacement.FilePath()}, dataPaths)
+ })
+}
+
+// ReplaceFiles validates the data files and the delete files to remove in one
+// pass; both live only on the branch here.
+func TestBranchReplaceFilesResolvesBranchDeleteFiles(t *testing.T) {
+ tbl := seedDivergentBranch(t, newBranchPlanningTable(t, nil))
+ mainDataPaths, _ := liveFilesOnRef(t, tbl, table.MainBranch)
+
+ staged := buildDataFile(t, tbl.Location()+"/data/branch-staged.parquet")
+ posDelete := buildPosDeleteFile(t, tbl.Location()+"/data/branch-pos-delete.parquet")
+
+ txn, err := tbl.NewTransactionOnBranchWithError(branchPlanningRef)
+ require.NoError(t, err)
+ require.NoError(t, txn.NewRowDelta(nil).AddRows(staged).AddDeletes(posDelete).Commit(t.Context()))
+ tbl, err = txn.Commit(t.Context())
+ require.NoError(t, err)
+
+ replacement := buildDataFile(t, tbl.Location()+"/data/branch-compacted.parquet")
+ txn, err = tbl.NewTransactionOnBranchWithError(branchPlanningRef)
+ require.NoError(t, err)
+ require.NoError(t, txn.ReplaceFiles(t.Context(),
+ []iceberg.DataFile{staged}, []iceberg.DataFile{replacement}, []iceberg.DataFile{posDelete}, nil))
+ tbl, err = txn.Commit(t.Context())
+ require.NoError(t, err)
+
+ branchData, branchDeletes := liveFilesOnRef(t, tbl, branchPlanningRef)
+ assert.Contains(t, branchData, replacement.FilePath())
+ assert.NotContains(t, branchData, staged.FilePath())
+ assert.Empty(t, branchDeletes, "the branch's delete file must be removed by the rewrite")
+
+ mainDataAfter, _ := liveFilesOnRef(t, tbl, table.MainBranch)
+ assert.ElementsMatch(t, mainDataPaths, mainDataAfter, "a rewrite on the branch must not touch main")
+}
+
+// A second delete folds into the existing vector and supersedes it. Missing the
+// branch's vector strands two live vectors on one data file, which violates the
+// spec and resurrects the first delete.
+func TestBranchMergeOnReadDeleteMergesBranchDeletionVector(t *testing.T) {
+ tbl := newBranchPlanningTable(t, iceberg.Properties{
+ table.PropertyFormatVersion: "3",
+ table.WriteDeleteModeKey: table.WriteModeMergeOnRead,
+ })
+ tbl = appendRowsOnRef(t, tbl, table.MainBranch, 1, 2, 3, 4, 5)
+
+ for _, id := range []int64{2, 3} {
+ txn, err := tbl.NewTransactionOnBranchWithError(branchPlanningRef)
+ require.NoError(t, err)
+ require.NoError(t, txn.Delete(t.Context(), iceberg.EqualTo(iceberg.Reference("id"), id), nil))
+ tbl, err = txn.Commit(t.Context())
+ require.NoError(t, err)
+ }
+
+ assert.Equal(t, 1, liveDVCountOnRef(t, tbl, branchPlanningRef),
+ "the second delete must merge into the branch's existing deletion vector")
+ assert.Equal(t, []int64{1, 4, 5}, idsOnRef(t, tbl, branchPlanningRef),
+ "both deletes must hold; a fresh vector would resurrect id 2")
+ assert.Equal(t, []int64{1, 2, 3, 4, 5}, idsOnRef(t, tbl, table.MainBranch),
+ "deletes on the branch must not change main")
+}
+
+// Merge-on-read classifies files before writing any vector: classified against
+// main, a branch-only file matches nothing and the delete hides no row.
+func TestBranchMergeOnReadDeleteTargetsBranchOnlyFile(t *testing.T) {
+ tbl := newBranchPlanningTable(t, iceberg.Properties{
+ table.PropertyFormatVersion: "3",
+ table.WriteDeleteModeKey: table.WriteModeMergeOnRead,
+ })
+ tbl = appendRowsOnRef(t, tbl, table.MainBranch, 1, 2)
+ tbl = appendRowsOnRef(t, tbl, branchPlanningRef, 3, 4)
+
+ txn, err := tbl.NewTransactionOnBranchWithError(branchPlanningRef)
+ require.NoError(t, err)
+ require.NoError(t, txn.Delete(t.Context(), iceberg.EqualTo(iceberg.Reference("id"), int64(3)), nil))
+ tbl, err = txn.Commit(t.Context())
+ require.NoError(t, err)
+
+ assert.Equal(t, []int64{1, 2, 4}, idsOnRef(t, tbl, branchPlanningRef))
+ assert.Equal(t, 1, liveDVCountOnRef(t, tbl, branchPlanningRef),
+ "the delete must write a vector against the branch-only file")
+ assert.Equal(t, []int64{1, 2}, idsOnRef(t, tbl, table.MainBranch))
+}
+
+// RowDelta resolves the vectors it removes against the planning snapshot's
+// delete manifests, where a branch-staged vector is invisible on main.
+func TestBranchRowDeltaResolvesRemovedDeletesOnBranchHead(t *testing.T) {
+ tbl := newBranchPlanningTable(t, iceberg.Properties{table.PropertyFormatVersion: "3"})
+ location := tbl.Location()
+
+ arrowSchema, err := table.SchemaToArrowSchema(tbl.Schema(), nil, false, false)
+ require.NoError(t, err)
+ dataPath := location + "/data/data-001.parquet"
+ writeParquetFile(t, dataPath, arrowSchema, `[{"id":1},{"id":2},{"id":3},{"id":4},{"id":5}]`)
+
+ txn := tbl.NewTransaction()
+ require.NoError(t, txn.AddFiles(t.Context(), []string{dataPath}, nil, false))
+ tbl, err = txn.Commit(t.Context())
+ require.NoError(t, err)
+
+ // The first vector (hiding position 1, id 2) exists only on the branch.
+ dv1 := writeDV(t, location, "dv-001.puffin", dataPath, []int64{1})
+ txn, err = tbl.NewTransactionOnBranchWithError(branchPlanningRef)
+ require.NoError(t, err)
+ require.NoError(t, txn.NewRowDelta(nil).AddDeletes(dv1).Commit(t.Context()))
+ tbl, err = txn.Commit(t.Context())
+ require.NoError(t, err)
+ require.Equal(t, []int64{1, 3, 4, 5}, idsOnRef(t, tbl, branchPlanningRef))
+
+ dv2 := writeDV(t, location, "dv-002.puffin", dataPath, []int64{1, 3})
+ txn, err = tbl.NewTransactionOnBranchWithError(branchPlanningRef)
+ require.NoError(t, err)
+ require.NoError(t, txn.NewRowDelta(nil).AddDeletes(dv2).RemoveDeletes(dv1).Commit(t.Context()))
+ tbl, err = txn.Commit(t.Context())
+ require.NoError(t, err)
+
+ branchHead := tbl.SnapshotByName(branchPlanningRef)
+ require.NotNil(t, branchHead)
+ live, removed := snapshotDVEntries(t, branchHead, iceio.LocalFS{})
+ assert.Equal(t, []string{dv2.FilePath()}, live, "the replacement must be the branch's only live vector")
+ assert.Equal(t, []string{dv1.FilePath()}, removed, "the superseded vector must be recorded as removed")
+
+ assert.Equal(t, []int64{1, 3, 5}, idsOnRef(t, tbl, branchPlanningRef))
+ assert.Equal(t, []int64{1, 2, 3, 4, 5}, idsOnRef(t, tbl, table.MainBranch),
+ "row-delta supersession on the branch must not change main")
+}
diff --git a/table/metadata.go b/table/metadata.go
index 4689c663a..5ffc95d02 100644
--- a/table/metadata.go
+++ b/table/metadata.go
@@ -389,9 +389,10 @@ func (b *MetadataBuilder) currentSnapshot() *Snapshot {
}
// currentSnapshotForRef returns the snapshot a write targeting ref must be
-// parented on (createSnapshotProducer and mergeOverwrite). An unknown branch
-// falls back to currentSnapshot() so a new branch forks from main. This is a
-// parent lookup only: the commit's AssertRefSnapshotID requirement is built
+// parented on (createSnapshotProducer) and plan against
+// (Transaction.planningSnapshot). An unknown branch falls back to
+// currentSnapshot() so a new branch forks from main. This is a head lookup
+// only: the commit's AssertRefSnapshotID requirement is built
// separately from the base table's ref (Transaction.baseRefSnapshotID), which
// returns nil for an absent branch so the requirement can prove the branch does
// not exist yet — never derive one from the other. A ref cannot outlive its
diff --git a/table/mor_delete_pruning_test.go b/table/mor_delete_pruning_test.go
index 0dae2aa77..01dc5585d 100644
--- a/table/mor_delete_pruning_test.go
+++ b/table/mor_delete_pruning_test.go
@@ -262,33 +262,11 @@ func newPartitionedMergeOnReadTestTable(t *testing.T) *table.Table {
}
// liveDVCount returns the number of live deletion-vector entries in the table's
-// current snapshot.
+// current snapshot, which is main's head.
func liveDVCount(t *testing.T, tbl *table.Table) int {
t.Helper()
- fs, err := tbl.FS(context.Background())
- require.NoError(t, err)
-
- snapshot := tbl.CurrentSnapshot()
- require.NotNil(t, snapshot)
- manifests, err := snapshot.Manifests(fs)
- require.NoError(t, err)
-
- count := 0
- for _, m := range manifests {
- for entry, err := range m.Entries(fs, false) {
- require.NoError(t, err)
- if entry.Status() == iceberg.EntryStatusDELETED {
- continue
- }
- df := entry.DataFile()
- if table.IsDeletionVector(df) && df.ReferencedDataFile() != nil {
- count++
- }
- }
- }
-
- return count
+ return liveDVCountOnRef(t, tbl, table.MainBranch)
}
// idsInTable scans the table and returns the surviving id values, sorted.
diff --git a/table/rebuild_manifest_test.go b/table/rebuild_manifest_test.go
index 76b70d5c1..b6df27e95 100644
--- a/table/rebuild_manifest_test.go
+++ b/table/rebuild_manifest_test.go
@@ -155,6 +155,38 @@ func TestRebuildSnapshotUpdates_CallsClosureWithFreshParent(t *testing.T) {
assert.Equal(t, oldManifest, orphaned[0])
}
+func TestStagedSnapshotStillValid(t *testing.T) {
+ v1, err := NewMetadata(
+ iceberg.NewSchema(0, iceberg.NestedField{ID: 1, Name: "id", Type: iceberg.PrimitiveTypes.Int64, Required: true}),
+ iceberg.UnpartitionedSpec, UnsortedSortOrder, "file:///tmp/staged-valid",
+ iceberg.Properties{PropertyFormatVersion: "1"})
+ require.NoError(t, err)
+
+ v2 := newMetadataWithLastSeqNum(t, 3)
+ v3 := newV3MetadataWithNextRowID(t, 50)
+
+ tests := []struct {
+ name string
+ fresh Metadata
+ snap *Snapshot
+ want bool
+ }{
+ {"v1 ignores sequence numbers", v1, &Snapshot{}, true},
+ {"v2 sequence number ahead", v2, &Snapshot{SequenceNumber: v2.LastSequenceNumber() + 1}, true},
+ {"v2 sequence number equal", v2, &Snapshot{SequenceNumber: v2.LastSequenceNumber()}, false},
+ {"v3 first-row-id at the cursor", v3, &Snapshot{SequenceNumber: v3.LastSequenceNumber() + 1, FirstRowID: ptr(v3.NextRowID())}, true},
+ {"v3 first-row-id behind", v3, &Snapshot{SequenceNumber: v3.LastSequenceNumber() + 1, FirstRowID: ptr(v3.NextRowID() - 1)}, false},
+ {"v3 first-row-id missing", v3, &Snapshot{SequenceNumber: v3.LastSequenceNumber() + 1}, false},
+ {"nil fresh metadata", nil, &Snapshot{}, true},
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ assert.Equal(t, tt.want, stagedSnapshotStillValid(tt.snap, tt.fresh))
+ })
+ }
+}
+
// TestRebuildSnapshotUpdates_SkipsWhenParentUnchanged verifies that
// rebuildSnapshotUpdates skips the rebuild when the update's snapshot
// already has the fresh branch head as its parent (no-op retry).
@@ -164,10 +196,13 @@ func TestRebuildSnapshotUpdates_SkipsWhenParentUnchanged(t *testing.T) {
freshHead := int64(42)
freshMeta := newConflictTestMetadata(t, &freshHead)
- // Parent already equals the fresh head — rebuild must be skipped.
+ // Parent already equals the fresh head and the sequence number still
+ // outranks the fresh last-sequence-number, as a snapshot built on this head would -
+ // rebuild must be skipped.
snap := &Snapshot{
SnapshotID: 99,
- ParentSnapshotID: &freshHead, // same as fresh head
+ ParentSnapshotID: &freshHead,
+ SequenceNumber: freshMeta.LastSequenceNumber() + 1,
ManifestList: manifest,
Summary: &Summary{Operation: OpAppend},
}
@@ -197,6 +232,44 @@ func TestRebuildSnapshotUpdates_SkipsWhenParentUnchanged(t *testing.T) {
assert.Same(t, upd, rebuilt[0].(*addSnapshotUpdate), "original update must pass through unchanged")
}
+// The parent is unchanged but table-wide state moved, so the staged snapshot is
+// one AddSnapshot now rejects: skipping its rebuild fails the commit terminally.
+func TestRebuildSnapshotUpdates_RebuildsWhenTableStateMoved(t *testing.T) {
+ const oldManifest = "s3://bucket/manifest-list.avro"
+ const newManifest = "s3://bucket/rebuilt-manifest-list.avro"
+
+ freshHead := int64(42)
+ freshMeta := newConflictTestMetadata(t, &freshHead)
+
+ snap := &Snapshot{
+ SnapshotID: 99,
+ ParentSnapshotID: &freshHead,
+ SequenceNumber: freshMeta.LastSequenceNumber(),
+ ManifestList: oldManifest,
+ Summary: &Summary{Operation: OpAppend},
+ }
+
+ var receivedParent *Snapshot
+ upd := rebuildUpdate(snap, newManifest, &receivedParent)
+
+ rebuilt, orphaned, err := rebuildSnapshotUpdates(
+ t.Context(),
+ []Update{upd},
+ freshMeta,
+ MainBranch,
+ iceio.LocalFS{},
+ 1,
+ )
+ require.NoError(t, err)
+ require.NotNil(t, receivedParent, "the rebuild must run even though the parent is unchanged")
+ assert.Equal(t, freshHead, receivedParent.SnapshotID, "the rebuild must keep the branch head as parent")
+
+ addUpd, ok := rebuilt[0].(*addSnapshotUpdate)
+ require.True(t, ok)
+ assert.Equal(t, newManifest, addUpd.Snapshot.ManifestList)
+ assert.Equal(t, []string{oldManifest}, orphaned)
+}
+
// TestRebuildSnapshotUpdates_PassesThroughNonRebuildUpdates verifies that
// updates without a rebuildManifestList closure are returned unmodified.
func TestRebuildSnapshotUpdates_PassesThroughNonRebuildUpdates(t *testing.T) {
diff --git a/table/rewrite_manifests.go b/table/rewrite_manifests.go
index b71b2ef09..b6692cfc4 100644
--- a/table/rewrite_manifests.go
+++ b/table/rewrite_manifests.go
@@ -367,7 +367,7 @@ func (t *Transaction) RewriteManifests(ctx context.Context, opts ...RewriteManif
// rewritten below come from prod.parentSnapshot(), which resolves the
// branch head. A main-only check reports NoOpNoSnapshot for a branch that
// has manifests to merge whenever main has no head of its own.
- if meta.currentSnapshotForRef(t.branch) == nil {
+ if t.planningSnapshot(meta) == nil {
return &RewriteManifestsResult{NoOpReason: NoOpNoSnapshot}, nil
}
diff --git a/table/row_delta.go b/table/row_delta.go
index 1c0545516..fe6298df2 100644
--- a/table/row_delta.go
+++ b/table/row_delta.go
@@ -406,7 +406,7 @@ func (rd *RowDelta) validateRemovedDeletes(resolved, replacedLive []iceberg.Data
// need; deltas without removals do not pay for it (nor get it — see
// AddDeletes).
func (rd *RowDelta) resolveRemovedDeletes(fs iceio.IO, meta *MetadataBuilder) (resolved, replacedLive []iceberg.DataFile, _ error) {
- snap := meta.currentSnapshot()
+ snap := rd.txn.planningSnapshot(meta)
if snap == nil {
return nil, nil, errors.New("cannot remove delete files from a table without an existing snapshot")
}
diff --git a/table/table.go b/table/table.go
index 9e4f9f150..06b41bdda 100644
--- a/table/table.go
+++ b/table/table.go
@@ -874,9 +874,10 @@ func validateBranchRequirement(reqs []Requirement, branch string, meta Metadata)
// latestSnapshotForBranch returns the head a write targeting branch is parented
// on: a branch that does not exist yet resolves to nil here and falls back to
-// main's head so the first write to a new branch forks from main. The two must
-// stay in sync — the replay path uses this one, and a divergence would reparent
-// a retried snapshot somewhere its first attempt never pointed.
+// main's head so the first write to a new branch forks from main. It is the
+// Metadata-side twin of currentSnapshotForRef, which the retry path cannot use
+// because it has no builder; the two must resolve a ref identically, or a
+// retried snapshot is reparented somewhere its first attempt never pointed.
func latestSnapshotForBranch(meta Metadata, branch string) *Snapshot {
if branch == "" || branch == MainBranch {
return meta.CurrentSnapshot()
@@ -889,6 +890,27 @@ func latestSnapshotForBranch(meta Metadata, branch string) *Snapshot {
return meta.CurrentSnapshot()
}
+// stagedSnapshotStillValid reports whether a snapshot built on an earlier
+// attempt would still be accepted by AddSnapshot against fresh. Both values it
+// checks are table-wide, so a peer committing on ANOTHER branch invalidates them
+// without moving this branch's head — the case the caller's parent check misses,
+// and one AddSnapshot rejects terminally rather than retryably.
+func stagedSnapshotStillValid(snap *Snapshot, fresh Metadata) bool {
+ if snap == nil || fresh == nil {
+ return true
+ }
+
+ if fresh.Version() >= 2 && snap.SequenceNumber <= fresh.LastSequenceNumber() {
+ return false
+ }
+
+ if fresh.Version() >= 3 && (snap.FirstRowID == nil || *snap.FirstRowID < fresh.NextRowID()) {
+ return false
+ }
+
+ return true
+}
+
// rebuildSnapshotUpdates returns a new slice of updates where any
// addSnapshotUpdate that carries a rebuildManifestList closure has its
// snapshot regenerated to inherit all data files committed to the branch
@@ -932,10 +954,12 @@ func rebuildSnapshotUpdates(ctx context.Context, updates []Update, freshMeta Met
}
// Skip only when nothing upstream changed: the running head still
- // matches this snapshot's recorded parent and no earlier snapshot in
- // this chain was rebuilt. Saves an unnecessary S3 write.
+ // matches this snapshot's recorded parent, no earlier snapshot in this
+ // chain was rebuilt, and the table-wide values the rebuild recomputes
+ // are still acceptable. Saves an unnecessary S3 write.
if !chainRebuilt && freshHead != nil && su.Snapshot.ParentSnapshotID != nil &&
- *su.Snapshot.ParentSnapshotID == freshHead.SnapshotID {
+ *su.Snapshot.ParentSnapshotID == freshHead.SnapshotID &&
+ stagedSnapshotStillValid(su.Snapshot, freshMeta) {
// Already parented on the running head; it becomes the parent of
// the next staged snapshot in the chain.
freshHead = su.Snapshot
diff --git a/table/transaction.go b/table/transaction.go
index 9c220d112..7040e99c5 100644
--- a/table/transaction.go
+++ b/table/transaction.go
@@ -63,7 +63,7 @@ func (s snapshotUpdate) fastAppend() *snapshotProducer {
// checks.
func (s snapshotUpdate) mergeOverwrite(commitUUID *uuid.UUID, filter iceberg.BooleanExpression) *snapshotProducer {
op := s.operation
- if s.operation == OpOverwrite && s.txn.meta.currentSnapshotForRef(s.txn.branch) == nil {
+ if s.operation == OpOverwrite && s.txn.planningSnapshot(s.txn.meta) == nil {
op = OpAppend
}
prod := newOverwriteFilesProducer(op, s.txn, s.io, commitUUID, s.snapshotProps)
@@ -317,6 +317,12 @@ func (t *Transaction) baseRefSnapshotID(branch string) *int64 {
return nil
}
+// planningSnapshot must resolve the same head createSnapshotProducer parents on,
+// or an operation plans against files the new snapshot does not inherit.
+func (t *Transaction) planningSnapshot(meta *MetadataBuilder) *Snapshot {
+ return meta.currentSnapshotForRef(t.branch)
+}
+
func transactionRequirements(reqs []Requirement, branch string, base Metadata) []Requirement {
if branch == "" {
branch = MainBranch
@@ -891,29 +897,11 @@ func (t *Transaction) Append(ctx context.Context, rdr array.RecordReader, snapsh
// operation is only valid if the data is exactly the same as the previous snapshot.
//
// For now, we'll keep using an overwrite operation.
-// requireMainBranch fails loudly for operations whose read-side planning (which
-// existing files to delete, replace, or dedupe against) still resolves against
-// main's head and is not yet branch-aware. On a non-main branch it returns
-// ErrInvalidOperation rather than silently planning against the wrong snapshot;
-// branch-aware planning is tracked as a follow-up (#1638). Appends need no such
-// planning and remain supported on a branch.
-func (t *Transaction) requireMainBranch(op string) error {
- if t.branch != "" && t.branch != MainBranch {
- return fmt.Errorf("%w: %s is not yet supported on a branch transaction (branch %q)",
- ErrInvalidOperation, op, t.branch)
- }
-
- return nil
-}
-
func (t *Transaction) ReplaceDataFiles(ctx context.Context, filesToDelete, filesToAdd []string, snapshotProps iceberg.Properties) error {
meta, err := t.txnMeta()
if err != nil {
return err
}
- if err := t.requireMainBranch("ReplaceDataFiles"); err != nil {
- return err
- }
if len(filesToDelete) == 0 {
if len(filesToAdd) > 0 {
return t.AddFiles(ctx, filesToAdd, snapshotProps, false)
@@ -941,7 +929,7 @@ func (t *Transaction) ReplaceDataFiles(ctx context.Context, filesToDelete, files
return errors.New("add file paths must be unique for ReplaceDataFiles")
}
- s := meta.currentSnapshot()
+ s := t.planningSnapshot(meta)
if s == nil {
return fmt.Errorf("%w: cannot replace files in a table without an existing snapshot", ErrInvalidOperation)
}
@@ -1221,9 +1209,6 @@ func (t *Transaction) AddDataFiles(ctx context.Context, dataFiles []iceberg.Data
if _, err := t.txnMeta(); err != nil {
return err
}
- if err := t.requireMainBranch("AddDataFiles"); err != nil {
- return err
- }
if len(dataFiles) == 0 {
return nil
}
@@ -1262,7 +1247,7 @@ func (t *Transaction) AddDataFiles(ctx context.Context, dataFiles []iceberg.Data
}
if !cfg.skipDuplicateCheck {
- if s := meta.currentSnapshot(); s != nil {
+ if s := t.planningSnapshot(meta); s != nil {
referenced := make([]string, 0)
for df, err := range s.dataFiles(fs, nil) {
if err != nil {
@@ -1320,9 +1305,6 @@ func (t *Transaction) ReplaceDataFilesWithDataFiles(ctx context.Context, filesTo
if err != nil {
return err
}
- if err := t.requireMainBranch("ReplaceDataFilesWithDataFiles"); err != nil {
- return err
- }
if len(filesToDelete) == 0 {
if len(filesToAdd) > 0 {
return t.AddDataFiles(ctx, filesToAdd, snapshotProps, opts...)
@@ -1358,7 +1340,7 @@ func (t *Transaction) ReplaceDataFilesWithDataFiles(ctx context.Context, filesTo
setToDelete[path] = struct{}{}
}
- s := meta.currentSnapshot()
+ s := t.planningSnapshot(meta)
if s == nil {
return fmt.Errorf("%w: cannot replace files in a table without an existing snapshot", ErrInvalidOperation)
}
@@ -1434,9 +1416,6 @@ func (t *Transaction) ReplaceFiles(ctx context.Context, dataFilesToDelete, dataF
if err != nil {
return err
}
- if err := t.requireMainBranch("ReplaceFiles"); err != nil {
- return err
- }
// Delegate data file replacement to existing logic.
if len(deleteFilesToRemove) == 0 {
return t.ReplaceDataFilesWithDataFiles(ctx, dataFilesToDelete, dataFilesToAdd, snapshotProps, opts...)
@@ -1495,7 +1474,7 @@ func (t *Transaction) ReplaceFiles(ctx context.Context, dataFilesToDelete, dataF
setDeleteFilesToRemove[path] = struct{}{}
}
- s := meta.currentSnapshot()
+ s := t.planningSnapshot(meta)
if s == nil {
return fmt.Errorf("%w: cannot replace files in a table without an existing snapshot", ErrInvalidOperation)
}
@@ -1609,9 +1588,6 @@ func (t *Transaction) AddFiles(ctx context.Context, filePaths []string, snapshot
if err != nil {
return err
}
- if err := t.requireMainBranch("AddFiles"); err != nil {
- return err
- }
addFilesOp := addFilesOperation{
concurrency: runtime.GOMAXPROCS(0),
}
@@ -1637,7 +1613,7 @@ func (t *Transaction) AddFiles(ctx context.Context, filePaths []string, snapshot
}
if !ignoreDuplicates {
- if s := meta.currentSnapshot(); s != nil {
+ if s := t.planningSnapshot(meta); s != nil {
referenced := make([]string, 0)
for df, err := range s.dataFiles(fs, nil) {
if err != nil {
@@ -1780,9 +1756,6 @@ func (t *Transaction) Overwrite(ctx context.Context, rdr array.RecordReader, sna
if _, err := t.txnMeta(); err != nil {
return err
}
- if err := t.requireMainBranch("Overwrite"); err != nil {
- return err
- }
overwrite := overwriteOperation{
concurrency: runtime.GOMAXPROCS(0),
filter: iceberg.AlwaysTrue{},
@@ -2001,9 +1974,6 @@ func (t *Transaction) Delete(ctx context.Context, filter iceberg.BooleanExpressi
if err != nil {
return err
}
- if err := t.requireMainBranch("Delete"); err != nil {
- return err
- }
deleteOp := deleteOperation{
concurrency: runtime.GOMAXPROCS(0),
caseSensitive: true,
@@ -2052,7 +2022,7 @@ func (t *Transaction) classifyFilesForDeletions(ctx context.Context, fs io.IO, f
return nil, nil, nil, err
}
- s := meta.currentSnapshot()
+ s := t.planningSnapshot(meta)
if s == nil {
return nil, nil, nil, nil
}
@@ -2127,7 +2097,7 @@ func (t *Transaction) classifyFilesForFilteredDeletions(ctx context.Context, fs
classificationTask := newFileClassificationTask(builtMeta, filter, caseSensitive)
manifestEvaluators := newKeyDefaultMapWrapErr(classificationTask.buildManifestEvaluator)
- s := meta.currentSnapshot()
+ s := t.planningSnapshot(meta)
var manifests []iceberg.ManifestFile
if s != nil {
manifests, err = s.Manifests(fs)
@@ -2543,7 +2513,7 @@ func (t *Transaction) collectExistingDVs(fs io.IO, files []iceberg.DataFile) (ma
return nil, err
}
- s := meta.currentSnapshot()
+ s := t.planningSnapshot(meta)
if s == nil {
return nil, nil
}
diff --git a/table/transaction_internal_test.go b/table/transaction_internal_test.go
index cfad01a79..118451b4d 100644
--- a/table/transaction_internal_test.go
+++ b/table/transaction_internal_test.go
@@ -466,41 +466,17 @@ func TestOverwriteOnBranchOnlyTableKeepsOverwriteSemantics(t *testing.T) {
})
}
-// TestBranchWritePlanningRejected enforces the scope boundary in code: until
-// read-side planning is branch-aware (#1638).
-func TestBranchWritePlanningRejected(t *testing.T) {
- ctx := context.Background()
- spec := iceberg.NewPartitionSpec()
- df := newTestDataFile(t, spec, "file://x.parquet", nil)
-
- ops := []struct {
- name string
- run func(txn *Transaction) error
- }{
- {"Delete", func(txn *Transaction) error { return txn.Delete(ctx, iceberg.AlwaysTrue{}, nil) }},
- {"Overwrite", func(txn *Transaction) error { return txn.Overwrite(ctx, nil, nil) }},
- {"ReplaceDataFiles", func(txn *Transaction) error {
- return txn.ReplaceDataFiles(ctx, []string{"file://x.parquet"}, nil, nil)
- }},
- {"ReplaceDataFilesWithDataFiles", func(txn *Transaction) error {
- return txn.ReplaceDataFilesWithDataFiles(ctx, []iceberg.DataFile{df}, nil, nil)
- }},
- {"ReplaceFiles", func(txn *Transaction) error {
- return txn.ReplaceFiles(ctx, []iceberg.DataFile{df}, nil, nil, nil)
- }},
- {"AddDataFiles", func(txn *Transaction) error { return txn.AddDataFiles(ctx, []iceberg.DataFile{df}, nil) }},
- {"AddFiles", func(txn *Transaction) error { return txn.AddFiles(ctx, []string{"file://x.parquet"}, nil, false) }},
- }
-
- for _, tc := range ops {
- t.Run(tc.name, func(t *testing.T) {
- base, _ := createTestTransactionWithMemIO(t, spec)
- txnFeat, err := base.tbl.NewTransactionOnBranchWithError("feature")
- require.NoError(t, err)
+func TestPlanningSnapshotMatchesProducerParent(t *testing.T) {
+ for _, branch := range []string{"", MainBranch, "feature", "does-not-exist"} {
+ t.Run("branch="+branch, func(t *testing.T) {
+ txn := newTransactionWithSnapshotRefs(t)
+ txn.branch = branch
- err = tc.run(txnFeat)
- require.ErrorIs(t, err, ErrInvalidOperation, "%s must be rejected on a branch transaction", tc.name)
- require.ErrorContains(t, err, "branch transaction")
+ planned := txn.planningSnapshot(txn.meta)
+ require.NotNil(t, planned)
+ require.Equal(t, planned.SnapshotID,
+ createSnapshotProducer(OpAppend, txn, nil, nil, nil).parentSnapshotID,
+ "planning must inspect the snapshot the write is parented on")
})
}
}