You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by at...@apache.org on 2026/07/10 11:55:19 UTC
(iceberg-go) branch main updated: fix(table): validate write defaults under old schema versions (#1417)
This is an automated email from the ASF dual-hosted git repository.
laskoviymishka pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-go.git
The following commit(s) were added to refs/heads/main by this push:
new 84b4a1e0 fix(table): validate write defaults under old schema versions (#1417)
84b4a1e0 is described below
commit 84b4a1e00b93ef699f4c93f9265f043e5d8492ba
Author: Minh Vu <vu...@gmail.com>
AuthorDate: Fri Jul 10 13:55:13 2026 +0200
fix(table): validate write defaults under old schema versions (#1417)
## Summary
`checkSchemaCompatibility` now treats `WriteDefault` like
`InitialDefault` for non-geometry/geography fields when creating or
evolving v1/v2 schemas.
- v1/v2 reject non-null `InitialDefault` and `WriteDefault`, with
separate diagnostics.
- geometry/geography defaults remain required to be null.
- compatibility errors print default values, not pointer addresses.
- tests cover direct compatibility checks and `UpdateColumn(...
WriteDefault ...)` end to end.
## Compatibility scope
This check runs through `MetadataBuilder.AddSchema`, which is used by
new schema construction and schema evolution. `ParseMetadataBytes`
unmarshals existing metadata directly and does not call
`checkSchemaCompatibility`, so this change does not reject
externally-produced v1/v2 metadata during read. The stricter
`WriteDefault` validation is intentional for schemas constructed or
evolved by this package.
## Testing
- `go test ./table -run
'TestNonSpecialDefaultsRequireV3|TestGeometryGeographyNullOnlyDefaults'
-count=1`
- `go test . -count=1`
- `go test ./table -count=1`
- `go run github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.11.4
run --timeout=10m . ./table/...`
---
table/metadata_builder_internal_test.go | 77 +++++++++++++++++++++++++++++++++
table/metadata_schema_compatibility.go | 32 ++++++++++++--
table/update_schema_test.go | 14 ++++++
3 files changed, 119 insertions(+), 4 deletions(-)
diff --git a/table/metadata_builder_internal_test.go b/table/metadata_builder_internal_test.go
index ffce2a68..81400ac3 100644
--- a/table/metadata_builder_internal_test.go
+++ b/table/metadata_builder_internal_test.go
@@ -2232,6 +2232,83 @@ func TestGeometryGeographyNullOnlyDefaults(t *testing.T) {
}
}
+func TestNonSpecialDefaultsRequireV3(t *testing.T) {
+ testTypes := []struct {
+ name string
+ typ iceberg.Type
+ }{
+ {"int", iceberg.Int32Type{}},
+ {"string", iceberg.StringType{}},
+ }
+
+ for _, tt := range testTypes {
+ t.Run(tt.name+" v2 with initial default", func(t *testing.T) {
+ defaultValue := 7
+ sc := iceberg.NewSchema(0,
+ iceberg.NestedField{
+ Type: tt.typ,
+ ID: 1,
+ Name: "col",
+ Required: false,
+ InitialDefault: &defaultValue,
+ },
+ )
+
+ err := checkSchemaCompatibility(sc, 2)
+ require.Error(t, err)
+ require.ErrorIs(t, err, iceberg.ErrInvalidSchema)
+ require.ErrorContains(t, err, "invalid initial default")
+ require.ErrorContains(t, err, "non-null default")
+ require.ErrorContains(t, err, "non-null default (7)")
+ require.ErrorContains(t, err, "is not supported until v3")
+ })
+
+ t.Run(tt.name+" v2 with write default", func(t *testing.T) {
+ defaultValue := 7
+ sc := iceberg.NewSchema(0,
+ iceberg.NestedField{
+ Type: tt.typ,
+ ID: 1,
+ Name: "col",
+ Required: false,
+ WriteDefault: &defaultValue,
+ },
+ )
+
+ err := checkSchemaCompatibility(sc, 2)
+ require.Error(t, err)
+ require.ErrorIs(t, err, iceberg.ErrInvalidSchema)
+ require.ErrorContains(t, err, "invalid write default")
+ require.ErrorContains(t, err, "non-null default")
+ require.ErrorContains(t, err, "non-null default (7)")
+ require.ErrorContains(t, err, "is not supported until v3")
+ })
+
+ t.Run(tt.name+" v2 with initial and write defaults", func(t *testing.T) {
+ initialValue := 7
+ writeValue := 7
+ sc := iceberg.NewSchema(0,
+ iceberg.NestedField{
+ Type: tt.typ,
+ ID: 1,
+ Name: "col",
+ Required: false,
+ InitialDefault: &initialValue,
+ WriteDefault: &writeValue,
+ },
+ )
+
+ err := checkSchemaCompatibility(sc, 2)
+ require.Error(t, err)
+ require.ErrorIs(t, err, iceberg.ErrInvalidSchema)
+
+ errStr := err.Error()
+ assert.Equal(t, 1, strings.Count(errStr, "invalid initial default"), "expected exactly one initial-default line in: %s", errStr)
+ assert.Equal(t, 1, strings.Count(errStr, "invalid write default"), "expected exactly one write-default line in: %s", errStr)
+ })
+ }
+}
+
func TestComplexTypeDefaultValidation(t *testing.T) {
t.Run("InvalidStructInitialDefault", func(t *testing.T) {
schema := iceberg.NewSchema(1,
diff --git a/table/metadata_schema_compatibility.go b/table/metadata_schema_compatibility.go
index 38c764b2..af13ede5 100644
--- a/table/metadata_schema_compatibility.go
+++ b/table/metadata_schema_compatibility.go
@@ -20,6 +20,7 @@ package table
import (
"cmp"
"fmt"
+ "reflect"
"slices"
"strconv"
"strings"
@@ -47,7 +48,12 @@ func (e ErrIncompatibleSchema) Error() string {
fmt.Fprintf(&problems, "\n- invalid write default for %s: %s columns must default to null", f.ColName, f.Field.Type)
}
} else {
- fmt.Fprintf(&problems, "\n- invalid initial default for %s: non-null default (%v) is not supported until v%d", f.ColName, f.Field.InitialDefault, f.InvalidDefault.MinFormatVersion)
+ if f.Field.InitialDefault != nil {
+ fmt.Fprintf(&problems, "\n- invalid initial default for %s: non-null default (%v) is not supported until v%d", f.ColName, formatDefaultValue(f.Field.InitialDefault), f.InvalidDefault.MinFormatVersion)
+ }
+ if f.Field.WriteDefault != nil {
+ fmt.Fprintf(&problems, "\n- invalid write default for %s: non-null default (%v) is not supported until v%d", f.ColName, formatDefaultValue(f.Field.WriteDefault), f.InvalidDefault.MinFormatVersion)
+ }
}
}
}
@@ -72,14 +78,32 @@ type UnsupportedType struct {
type InvalidDefault struct {
MinFormatVersion int
- WriteDefault any
MustBeNullForType bool
}
+func formatDefaultValue(value any) any {
+ v := reflect.ValueOf(value)
+ for v.IsValid() && v.Kind() == reflect.Pointer {
+ if v.IsNil() {
+ return nil
+ }
+ v = v.Elem()
+ }
+ if !v.IsValid() {
+ return nil
+ }
+
+ return v.Interface()
+}
+
// checkSchemaCompatibility checks that the schema is compatible with the table's format version.
// This validates that the schema does not contain types or features that were released
// in later format versions.
// Java: Schema::checkCompatibility
+// This check runs when a schema is added to a MetadataBuilder during table
+// construction or schema evolution. ParseMetadataBytes unmarshals existing
+// metadata directly and does not call this check. We intentionally validate
+// both default fields here, including write-default for pre-v3 schemas.
func checkSchemaCompatibility(sc *iceberg.Schema, formatVersion int) error {
const defaultValuesMinFormatVersion = 3
problems := make([]IncompatibleField, 0)
@@ -133,11 +157,11 @@ func checkSchemaCompatibility(sc *iceberg.Schema, formatVersion int) error {
})
}
default:
- if field.InitialDefault != nil && formatVersion < defaultValuesMinFormatVersion {
+ if (field.InitialDefault != nil || field.WriteDefault != nil) && formatVersion < defaultValuesMinFormatVersion {
problems = append(problems, IncompatibleField{
Field: field,
ColName: colName,
- InvalidDefault: &InvalidDefault{MinFormatVersion: defaultValuesMinFormatVersion, WriteDefault: field.InitialDefault},
+ InvalidDefault: &InvalidDefault{MinFormatVersion: defaultValuesMinFormatVersion},
})
}
}
diff --git a/table/update_schema_test.go b/table/update_schema_test.go
index 6374fad9..6092b703 100644
--- a/table/update_schema_test.go
+++ b/table/update_schema_test.go
@@ -581,6 +581,20 @@ func TestUpdateColumn(t *testing.T) {
assert.Equal(t, "User's age in years", ageField.Doc)
})
+ t.Run("test write default on v2 table returns compatibility error", func(t *testing.T) {
+ table := New([]string{"id"}, testMetadata, "", nil, nil)
+ txn := table.NewTransaction()
+
+ err := NewUpdateSchema(txn, true, true).UpdateColumn([]string{"age"}, ColumnUpdate{
+ WriteDefault: iceberg.Optional[iceberg.Literal]{
+ Valid: true,
+ Val: iceberg.NewLiteral(int32(7)),
+ },
+ }).Commit()
+ require.ErrorContains(t, err, "invalid write default")
+ require.ErrorContains(t, err, "non-null default (7)")
+ })
+
t.Run("test update non-existent column", func(t *testing.T) {
table := New([]string{"id"}, testMetadata, "", nil, nil)
txn := table.NewTransaction()