You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by ze...@apache.org on 2026/04/23 16:45:30 UTC
(iceberg-go) branch main updated: fix(table): seed UpdateSchema id counter from metadata LastColumnID (#936)
This is an automated email from the ASF dual-hosted git repository.
zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-go.git
The following commit(s) were added to refs/heads/main by this push:
new c576fd17 fix(table): seed UpdateSchema id counter from metadata LastColumnID (#936)
c576fd17 is described below
commit c576fd1726f29845f4810c6d877c34cfc8a870d0
Author: Ashley Jeffs <as...@jeffail.uk>
AuthorDate: Thu Apr 23 17:45:23 2026 +0100
fix(table): seed UpdateSchema id counter from metadata LastColumnID (#936)
## Problem
`NewUpdateSchema` seeds its fresh-id counter from
`txn.meta.CurrentSchema().HighestFieldID()`. The Iceberg spec reserves
`last-column-id` on table metadata as the monotonic counter for this
purpose; Java's
[`SchemaUpdate`](https://github.com/apache/iceberg/blob/main/core/src/main/java/org/apache/iceberg/SchemaUpdate.java)
seeds from `metadata.lastColumnId()`.
When the current schema's highest id is lower than the lifetime max —
after a previous evolution added columns that were later dropped, or
after a schema swap — `AddColumn` allocates ids already referenced by
fields in historical schemas. Catalogs that index the full schema
history (e.g. AWS Glue's `schemasToGlueColumns`) then reject the commit
with a duplicate field id error.
## Fix
Seed `lastColumnID` from `txn.meta.LastColumnID()` in `NewUpdateSchema`.
Adds a public `LastColumnID()` getter on `*MetadataBuilder` — the
`Metadata` interface already exposes this value, but `txn.meta` is the
builder.
`MetadataBuilder.AddSchema` already keeps `last-column-id` monotonic via
`max(b.lastColumnId, schema.HighestFieldID())`, so no other allocator
sites need changes.
## Testing
Adds `TestAddColumnMonotonicFieldIDs`: constructs metadata whose
`last-column-id` exceeds the current schema's `HighestFieldID()` (by
adding a higher-id schema while leaving the original as current) and
asserts that `AddColumn` allocates above `last-column-id`. Confirmed
failing on `main` before the fix and passing after.
`go test ./...` passes.
Fixes #935
Signed-off-by: Ashley Jeffs <as...@jeffail.uk>
---
table/metadata.go | 4 ++++
table/update_schema.go | 11 ++++++++---
table/update_schema_test.go | 48 +++++++++++++++++++++++++++++++++++++++++++++
3 files changed, 60 insertions(+), 3 deletions(-)
diff --git a/table/metadata.go b/table/metadata.go
index f733343d..48d667b5 100644
--- a/table/metadata.go
+++ b/table/metadata.go
@@ -296,6 +296,10 @@ func (b *MetadataBuilder) CurrentSchema() *iceberg.Schema {
func (b *MetadataBuilder) LastUpdatedMS() int64 { return b.lastUpdatedMS }
+// LastColumnID returns the highest field id ever assigned in this table's
+// lifetime, as tracked by the Iceberg spec's last-column-id counter.
+func (b *MetadataBuilder) LastColumnID() int { return b.lastColumnId }
+
func (b *MetadataBuilder) nextSequenceNumber() int64 {
if b.formatVersion > 1 {
if b.lastSequenceNumber == nil {
diff --git a/table/update_schema.go b/table/update_schema.go
index dd450328..d5bcc195 100644
--- a/table/update_schema.go
+++ b/table/update_schema.go
@@ -133,9 +133,14 @@ func WithNameMapping(nameMapping iceberg.NameMapping) UpdateSchemaOption {
// Returns an UpdateSchema instance that can be used to build and apply schema changes.
func NewUpdateSchema(txn *Transaction, caseSensitive bool, allowIncompatibleChanges bool, opts ...UpdateSchemaOption) *UpdateSchema {
u := &UpdateSchema{
- txn: txn,
- schema: nil,
- lastColumnID: txn.meta.CurrentSchema().HighestFieldID(),
+ txn: txn,
+ schema: nil,
+ // Seed from metadata's last-column-id rather than the current schema's
+ // highest field id. Per the Iceberg spec, last-column-id is a monotonic
+ // counter that preserves ids across schema evolution (including
+ // deletions) so that newly-allocated ids never collide with ids still
+ // referenced by historical schemas.
+ lastColumnID: txn.meta.LastColumnID(),
deletes: make(map[int]struct{}),
updates: make(map[int]map[int]iceberg.NestedField),
diff --git a/table/update_schema_test.go b/table/update_schema_test.go
index 1ee56239..4ebd9f14 100644
--- a/table/update_schema_test.go
+++ b/table/update_schema_test.go
@@ -913,3 +913,51 @@ func TestBuildUpdates(t *testing.T) {
assert.Equal(t, 0, updates[0].(*setCurrentSchemaUpdate).SchemaID)
})
}
+
+// TestAddColumnMonotonicFieldIDs exercises the case where the table's
+// last-column-id is greater than the current schema's highest field id — for
+// example because a previous schema was added that introduced higher ids, or
+// because the highest-id columns were later dropped. The Iceberg spec requires
+// new field ids to be allocated above last-column-id (never to be reused), so
+// AddColumn must seed its id counter from metadata.LastColumnID() rather than
+// the current schema's HighestFieldID().
+func TestAddColumnMonotonicFieldIDs(t *testing.T) {
+ // Start from originalSchema (field ids 1..11, schema id 1).
+ baseMeta, err := NewMetadata(originalSchema, iceberg.UnpartitionedSpec, UnsortedSortOrder, "", nil)
+ assert.NoError(t, err)
+
+ // Add a second schema that introduces higher field ids. This bumps the
+ // metadata's last-column-id to 13 while the current schema is still the
+ // original (highest field id 11).
+ expanded := iceberg.NewSchema(0,
+ iceberg.NestedField{ID: 1, Name: "id", Type: iceberg.PrimitiveTypes.Int32, Required: true},
+ iceberg.NestedField{ID: 12, Name: "extra_a", Type: iceberg.PrimitiveTypes.String, Required: false},
+ iceberg.NestedField{ID: 13, Name: "extra_b", Type: iceberg.PrimitiveTypes.String, Required: false},
+ )
+
+ builder, err := MetadataBuilderFromBase(baseMeta, "")
+ assert.NoError(t, err)
+ assert.NoError(t, builder.AddSchema(expanded))
+
+ meta, err := builder.Build()
+ assert.NoError(t, err)
+
+ assert.Equal(t, 11, meta.CurrentSchema().HighestFieldID(),
+ "precondition: current schema should still be the original with highest id 11")
+ assert.Equal(t, 13, meta.LastColumnID(),
+ "precondition: last-column-id should have been bumped by the expanded schema")
+
+ tbl := New([]string{"id"}, meta, "", nil, nil)
+ txn := tbl.NewTransaction()
+
+ newSchema, err := NewUpdateSchema(txn, true, true).
+ AddColumn([]string{"fresh"}, iceberg.PrimitiveTypes.String, "", false, nil).
+ Apply()
+ assert.NoError(t, err)
+ assert.NotNil(t, newSchema)
+
+ fresh, ok := newSchema.FindFieldByName("fresh")
+ assert.True(t, ok, "new field should be present in the resulting schema")
+ assert.Equal(t, 14, fresh.ID,
+ "new field id must be allocated above metadata.LastColumnID() (13), not reused from the current schema's highest id (11)")
+}