You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by ze...@apache.org on 2026/04/23 16:45:30 UTC

(iceberg-go) branch main updated: fix(table): seed UpdateSchema id counter from metadata LastColumnID (#936)

This is an automated email from the ASF dual-hosted git repository.

zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-go.git


The following commit(s) were added to refs/heads/main by this push:
     new c576fd17 fix(table): seed UpdateSchema id counter from metadata LastColumnID (#936)
c576fd17 is described below

commit c576fd1726f29845f4810c6d877c34cfc8a870d0
Author: Ashley Jeffs <as...@jeffail.uk>
AuthorDate: Thu Apr 23 17:45:23 2026 +0100

    fix(table): seed UpdateSchema id counter from metadata LastColumnID (#936)
    
    ## Problem
    
    `NewUpdateSchema` seeds its fresh-id counter from
    `txn.meta.CurrentSchema().HighestFieldID()`. The Iceberg spec reserves
    `last-column-id` on table metadata as the monotonic counter for this
    purpose; Java's
    [`SchemaUpdate`](https://github.com/apache/iceberg/blob/main/core/src/main/java/org/apache/iceberg/SchemaUpdate.java)
    seeds from `metadata.lastColumnId()`.
    
    When the current schema's highest id is lower than the lifetime max —
    after a previous evolution added columns that were later dropped, or
    after a schema swap — `AddColumn` allocates ids already referenced by
    fields in historical schemas. Catalogs that index the full schema
    history (e.g. AWS Glue's `schemasToGlueColumns`) then reject the commit
    with a duplicate field id error.
    
    ## Fix
    
    Seed `lastColumnID` from `txn.meta.LastColumnID()` in `NewUpdateSchema`.
    Adds a public `LastColumnID()` getter on `*MetadataBuilder` — the
    `Metadata` interface already exposes this value, but `txn.meta` is the
    builder.
    
    `MetadataBuilder.AddSchema` already keeps `last-column-id` monotonic via
    `max(b.lastColumnId, schema.HighestFieldID())`, so no other allocator
    sites need changes.
    
    ## Testing
    
    Adds `TestAddColumnMonotonicFieldIDs`: constructs metadata whose
    `last-column-id` exceeds the current schema's `HighestFieldID()` (by
    adding a higher-id schema while leaving the original as current) and
    asserts that `AddColumn` allocates above `last-column-id`. Confirmed
    failing on `main` before the fix and passing after.
    
    `go test ./...` passes.
    
    Fixes #935
    
    Signed-off-by: Ashley Jeffs <as...@jeffail.uk>
---
 table/metadata.go           |  4 ++++
 table/update_schema.go      | 11 ++++++++---
 table/update_schema_test.go | 48 +++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 60 insertions(+), 3 deletions(-)

diff --git a/table/metadata.go b/table/metadata.go
index f733343d..48d667b5 100644
--- a/table/metadata.go
+++ b/table/metadata.go
@@ -296,6 +296,10 @@ func (b *MetadataBuilder) CurrentSchema() *iceberg.Schema {
 
 func (b *MetadataBuilder) LastUpdatedMS() int64 { return b.lastUpdatedMS }
 
+// LastColumnID returns the highest field id ever assigned in this table's
+// lifetime, as tracked by the Iceberg spec's last-column-id counter.
+func (b *MetadataBuilder) LastColumnID() int { return b.lastColumnId }
+
 func (b *MetadataBuilder) nextSequenceNumber() int64 {
 	if b.formatVersion > 1 {
 		if b.lastSequenceNumber == nil {
diff --git a/table/update_schema.go b/table/update_schema.go
index dd450328..d5bcc195 100644
--- a/table/update_schema.go
+++ b/table/update_schema.go
@@ -133,9 +133,14 @@ func WithNameMapping(nameMapping iceberg.NameMapping) UpdateSchemaOption {
 // Returns an UpdateSchema instance that can be used to build and apply schema changes.
 func NewUpdateSchema(txn *Transaction, caseSensitive bool, allowIncompatibleChanges bool, opts ...UpdateSchemaOption) *UpdateSchema {
 	u := &UpdateSchema{
-		txn:          txn,
-		schema:       nil,
-		lastColumnID: txn.meta.CurrentSchema().HighestFieldID(),
+		txn:    txn,
+		schema: nil,
+		// Seed from metadata's last-column-id rather than the current schema's
+		// highest field id. Per the Iceberg spec, last-column-id is a monotonic
+		// counter that preserves ids across schema evolution (including
+		// deletions) so that newly-allocated ids never collide with ids still
+		// referenced by historical schemas.
+		lastColumnID: txn.meta.LastColumnID(),
 
 		deletes: make(map[int]struct{}),
 		updates: make(map[int]map[int]iceberg.NestedField),
diff --git a/table/update_schema_test.go b/table/update_schema_test.go
index 1ee56239..4ebd9f14 100644
--- a/table/update_schema_test.go
+++ b/table/update_schema_test.go
@@ -913,3 +913,51 @@ func TestBuildUpdates(t *testing.T) {
 		assert.Equal(t, 0, updates[0].(*setCurrentSchemaUpdate).SchemaID)
 	})
 }
+
+// TestAddColumnMonotonicFieldIDs exercises the case where the table's
+// last-column-id is greater than the current schema's highest field id — for
+// example because a previous schema was added that introduced higher ids, or
+// because the highest-id columns were later dropped. The Iceberg spec requires
+// new field ids to be allocated above last-column-id (never to be reused), so
+// AddColumn must seed its id counter from metadata.LastColumnID() rather than
+// the current schema's HighestFieldID().
+func TestAddColumnMonotonicFieldIDs(t *testing.T) {
+	// Start from originalSchema (field ids 1..11, schema id 1).
+	baseMeta, err := NewMetadata(originalSchema, iceberg.UnpartitionedSpec, UnsortedSortOrder, "", nil)
+	assert.NoError(t, err)
+
+	// Add a second schema that introduces higher field ids. This bumps the
+	// metadata's last-column-id to 13 while the current schema is still the
+	// original (highest field id 11).
+	expanded := iceberg.NewSchema(0,
+		iceberg.NestedField{ID: 1, Name: "id", Type: iceberg.PrimitiveTypes.Int32, Required: true},
+		iceberg.NestedField{ID: 12, Name: "extra_a", Type: iceberg.PrimitiveTypes.String, Required: false},
+		iceberg.NestedField{ID: 13, Name: "extra_b", Type: iceberg.PrimitiveTypes.String, Required: false},
+	)
+
+	builder, err := MetadataBuilderFromBase(baseMeta, "")
+	assert.NoError(t, err)
+	assert.NoError(t, builder.AddSchema(expanded))
+
+	meta, err := builder.Build()
+	assert.NoError(t, err)
+
+	assert.Equal(t, 11, meta.CurrentSchema().HighestFieldID(),
+		"precondition: current schema should still be the original with highest id 11")
+	assert.Equal(t, 13, meta.LastColumnID(),
+		"precondition: last-column-id should have been bumped by the expanded schema")
+
+	tbl := New([]string{"id"}, meta, "", nil, nil)
+	txn := tbl.NewTransaction()
+
+	newSchema, err := NewUpdateSchema(txn, true, true).
+		AddColumn([]string{"fresh"}, iceberg.PrimitiveTypes.String, "", false, nil).
+		Apply()
+	assert.NoError(t, err)
+	assert.NotNil(t, newSchema)
+
+	fresh, ok := newSchema.FindFieldByName("fresh")
+	assert.True(t, ok, "new field should be present in the resulting schema")
+	assert.Equal(t, 14, fresh.ID,
+		"new field id must be allocated above metadata.LastColumnID() (13), not reused from the current schema's highest id (11)")
+}