You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by at...@apache.org on 2026/05/12 20:38:45 UTC

(iceberg-go) branch main updated: feat(table): port remaining v3 metadata-upgrade validation rules (#1070)

This is an automated email from the ASF dual-hosted git repository.

laskoviymishka pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-go.git


The following commit(s) were added to refs/heads/main by this push:
     new 40b81252 feat(table): port remaining v3 metadata-upgrade validation rules (#1070)
40b81252 is described below

commit 40b81252526c658740541d87f78531c7a9f8c70b
Author: Tanmay Rauth <t_...@apple.com>
AuthorDate: Tue May 12 13:38:39 2026 -0700

    feat(table): port remaining v3 metadata-upgrade validation rules (#1070)
    
    Add missing validation rules to SetFormatVersion that mirror Java's
    TableMetadata upgrade checks:
    
    - Initialize lastSequenceNumber to 0 when upgrading from v1 to v2+
    - Initialize nextRowID to 0 when upgrading to v3
    - Assign UUID if not already set when upgrading to v2+
    - Validate lastColumnId is not less than the highest field ID in schemas
    
    Validation runs before state mutation to prevent leaving the builder in
    a half-upgraded state on failure.
    
    Closes #1013
---
 table/metadata.go                       |  39 +++++++++
 table/metadata_builder_internal_test.go | 148 ++++++++++++++++++++++++++++++++
 2 files changed, 187 insertions(+)

diff --git a/table/metadata.go b/table/metadata.go
index e6a06d4a..08f545ca 100644
--- a/table/metadata.go
+++ b/table/metadata.go
@@ -708,9 +708,48 @@ func (b *MetadataBuilder) SetFormatVersion(formatVersion int) error {
 		return nil
 	}
 
+	if err := b.validateLastColumnID(); err != nil {
+		return err
+	}
+
+	previousVersion := b.formatVersion
 	b.updates = append(b.updates, NewUpgradeFormatVersionUpdate(formatVersion))
 	b.formatVersion = formatVersion
 
+	if previousVersion < 2 && formatVersion >= 2 {
+		if b.uuid == (uuid.UUID{}) {
+			b.uuid = uuid.New()
+		}
+
+		if b.lastSequenceNumber == nil {
+			seq := int64(0)
+			b.lastSequenceNumber = &seq
+		}
+	}
+
+	if previousVersion < 3 && formatVersion >= 3 {
+		if b.nextRowID == nil {
+			nextRowID := int64(0)
+			b.nextRowID = &nextRowID
+		}
+	}
+
+	return nil
+}
+
+func (b *MetadataBuilder) validateLastColumnID() error {
+	highestFieldID := 0
+	for _, schema := range b.schemaList {
+		if id := schema.HighestFieldID(); id > highestFieldID {
+			highestFieldID = id
+		}
+	}
+
+	if highestFieldID > 0 && b.lastColumnId < highestFieldID {
+		return fmt.Errorf("%w: last-column-id %d is less than the highest field ID %d in schemas",
+			ErrInvalidMetadata, b.lastColumnId, highestFieldID)
+	}
+
 	return nil
 }
 
diff --git a/table/metadata_builder_internal_test.go b/table/metadata_builder_internal_test.go
index eb8f57ef..991bd50b 100644
--- a/table/metadata_builder_internal_test.go
+++ b/table/metadata_builder_internal_test.go
@@ -1998,3 +1998,151 @@ func TestComplexTypeDefaultValidation(t *testing.T) {
 		require.ErrorContains(t, err, "list type field 'inner' (id: 2) must have null or JSON array initial-default")
 	})
 }
+
+func TestSetFormatVersionV1ToV2InitializesSequenceNumber(t *testing.T) {
+	builder := builderWithoutChanges(1)
+	require.NoError(t, builder.SetFormatVersion(2))
+
+	meta, err := builder.Build()
+	require.NoError(t, err)
+	require.Equal(t, 2, meta.Version())
+	require.Equal(t, int64(0), meta.LastSequenceNumber())
+}
+
+func TestSetFormatVersionV1ToV3InitializesSequenceNumberAndNextRowID(t *testing.T) {
+	builder := builderWithoutChanges(1)
+	require.NoError(t, builder.SetFormatVersion(3))
+
+	meta, err := builder.Build()
+	require.NoError(t, err)
+	require.Equal(t, 3, meta.Version())
+	require.Equal(t, int64(0), meta.LastSequenceNumber())
+	require.Equal(t, int64(0), meta.NextRowID())
+}
+
+func TestSetFormatVersionV2ToV3InitializesNextRowID(t *testing.T) {
+	builder := builderWithoutChanges(2)
+	require.NoError(t, builder.SetFormatVersion(3))
+
+	meta, err := builder.Build()
+	require.NoError(t, err)
+	require.Equal(t, 3, meta.Version())
+	require.Equal(t, int64(0), meta.NextRowID())
+}
+
+func TestSetFormatVersionV1ToV2AssignsUUID(t *testing.T) {
+	builder := builderWithoutChanges(1)
+	require.NoError(t, builder.SetFormatVersion(2))
+
+	meta, err := builder.Build()
+	require.NoError(t, err)
+	require.NotEqual(t, uuid.UUID{}, meta.TableUUID())
+}
+
+func TestSetFormatVersionPreservesExistingUUID(t *testing.T) {
+	builder := builderWithoutChanges(1)
+	existingUUID := uuid.New()
+	require.NoError(t, builder.SetUUID(existingUUID))
+	require.NoError(t, builder.SetFormatVersion(2))
+
+	meta, err := builder.Build()
+	require.NoError(t, err)
+	require.Equal(t, existingUUID, meta.TableUUID())
+}
+
+func TestSetFormatVersionDowngradeNotAllowed(t *testing.T) {
+	builder := builderWithoutChanges(2)
+	err := builder.SetFormatVersion(1)
+	require.Error(t, err)
+	require.ErrorContains(t, err, "downgrading format version from 2 to 1 is not allowed")
+}
+
+func TestSetFormatVersionRejectsUnsupportedVersion(t *testing.T) {
+	builder := builderWithoutChanges(2)
+	err := builder.SetFormatVersion(99)
+	require.Error(t, err)
+	require.ErrorIs(t, err, iceberg.ErrInvalidFormatVersion)
+}
+
+func TestSetFormatVersionNoOpWhenSameVersion(t *testing.T) {
+	builder := builderWithoutChanges(2)
+	require.NoError(t, builder.SetFormatVersion(2))
+	require.False(t, builder.HasChanges())
+}
+
+func TestSetFormatVersionValidatesLastColumnID(t *testing.T) {
+	tableSchema := iceberg.NewSchema(0,
+		iceberg.NestedField{ID: 1, Name: "x", Type: iceberg.PrimitiveTypes.Int64, Required: true},
+		iceberg.NestedField{ID: 2, Name: "y", Type: iceberg.PrimitiveTypes.Int64, Required: true},
+		iceberg.NestedField{ID: 3, Name: "z", Type: iceberg.PrimitiveTypes.Int64, Required: true},
+	)
+	partSpec := iceberg.NewPartitionSpecID(0)
+
+	builder, err := NewMetadataBuilder(1)
+	require.NoError(t, err)
+	require.NoError(t, builder.SetLoc("s3://bucket/test"))
+	require.NoError(t, builder.AddSchema(tableSchema))
+	require.NoError(t, builder.SetCurrentSchemaID(-1))
+	require.NoError(t, builder.AddSortOrder(&UnsortedSortOrder))
+	require.NoError(t, builder.SetDefaultSortOrderID(-1))
+	require.NoError(t, builder.AddPartitionSpec(&partSpec, true))
+	require.NoError(t, builder.SetDefaultSpecID(-1))
+
+	// Corrupt lastColumnId to be less than the highest field ID in the schema
+	builder.lastColumnId = 1
+
+	err = builder.SetFormatVersion(2)
+	require.Error(t, err)
+	require.ErrorContains(t, err, "last-column-id 1 is less than the highest field ID 3")
+
+	// Builder format version must remain unchanged after validation failure
+	require.Equal(t, 1, builder.formatVersion)
+}
+
+func TestSetFormatVersionV2ToV3PreservesSequenceNumber(t *testing.T) {
+	builder := builderWithoutChanges(2)
+
+	meta, err := builder.Build()
+	require.NoError(t, err)
+
+	builder2, err := MetadataBuilderFromBase(meta, "")
+	require.NoError(t, err)
+	require.NoError(t, builder2.SetFormatVersion(3))
+
+	meta3, err := builder2.Build()
+	require.NoError(t, err)
+	require.Equal(t, int64(0), meta3.LastSequenceNumber())
+}
+
+func TestSetFormatVersionV1ToV3FromDeserializedMetadata(t *testing.T) {
+	meta, err := ParseMetadataString(ExampleTableMetadataV1)
+	require.NoError(t, err)
+	require.Equal(t, 1, meta.Version())
+
+	builder, err := MetadataBuilderFromBase(meta, "")
+	require.NoError(t, err)
+	require.NoError(t, builder.SetFormatVersion(3))
+
+	meta3, err := builder.Build()
+	require.NoError(t, err)
+	require.Equal(t, 3, meta3.Version())
+	require.Equal(t, int64(0), meta3.LastSequenceNumber())
+	require.Equal(t, int64(0), meta3.NextRowID())
+	require.NotEqual(t, uuid.UUID{}, meta3.TableUUID())
+}
+
+func TestSetFormatVersionV2ToV3FromDeserializedMetadata(t *testing.T) {
+	meta, err := ParseMetadataString(ExampleTableMetadataV2)
+	require.NoError(t, err)
+	require.Equal(t, 2, meta.Version())
+
+	builder, err := MetadataBuilderFromBase(meta, "")
+	require.NoError(t, err)
+	require.NoError(t, builder.SetFormatVersion(3))
+
+	meta3, err := builder.Build()
+	require.NoError(t, err)
+	require.Equal(t, 3, meta3.Version())
+	require.Equal(t, int64(34), meta3.LastSequenceNumber())
+	require.Equal(t, int64(0), meta3.NextRowID())
+}