(iceberg-go) branch main updated: fix: preserve nested field IDs when filling missing columns (#1168)

[ze...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-go.git


The following commit(s) were added to refs/heads/main by this push:
     new 8f4faf40 fix: preserve nested field IDs when filling missing columns (#1168)
8f4faf40 is described below

commit 8f4faf40adee28a63eb282378146e41df193dc6e
Author: Blue Li <10...@users.noreply.github.com>
AuthorDate: Tue Jun 9 03:13:49 2026 +0800

    fix: preserve nested field IDs when filling missing columns (#1168)
    
    Fixes a bug where generating a null array for a completely missing
    nested column (e.g., during schema evolution) accidentally drops the
    inner `PARQUET:field_id` metadata due to a hardcoded `false` flag. This
    correctly propagates `includeFieldIDs` and resolves the `record schema
    does not match writer's` error when writing to `pqarrow`.
---
 table/arrow_utils.go      |  2 +-
 table/arrow_utils_test.go | 60 +++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 61 insertions(+), 1 deletion(-)

diff --git a/table/arrow_utils.go b/table/arrow_utils.go
index d490ad8e..375b6b94 100644
--- a/table/arrow_utils.go
+++ b/table/arrow_utils.go
@@ -1051,7 +1051,7 @@ func (a *arrowProjectionVisitor) Struct(st iceberg.StructType, structArr arrow.A
 			fieldArrs[i] = arr
 			fields[i] = a.constructField(field, arr.DataType())
 		} else {
-			dt := retOrPanic(TypeToArrowType(field.Type, false, a.useLargeTypes))
+			dt := retOrPanic(TypeToArrowType(field.Type, a.includeFieldIDs, a.useLargeTypes))
 			alloc := compute.GetAllocator(a.ctx)
 
 			switch {
diff --git a/table/arrow_utils_test.go b/table/arrow_utils_test.go
index 1ecc5f1f..97a8a3fd 100644
--- a/table/arrow_utils_test.go
+++ b/table/arrow_utils_test.go
@@ -1845,3 +1845,63 @@ func TestToRequestedSchemaWriteDefaultJSONRoundTrip(t *testing.T) {
 		})
 	}
 }
+
+func TestToRequestedSchemaMissingNestedFieldID(t *testing.T) {
+	mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
+	defer mem.AssertSize(t, 0)
+
+	// Create an Arrow schema that lacks the nested list completely
+	schemaWithoutMetadata := arrow.NewSchema([]arrow.Field{
+		{
+			Name: "other_field", Type: arrow.PrimitiveTypes.Int32,
+		},
+	}, nil)
+
+	bldr := array.NewRecordBuilder(mem, schemaWithoutMetadata)
+	defer bldr.Release()
+
+	const data = `{"other_field": 1}
+				  {"other_field": 2}`
+
+	s := bufio.NewScanner(strings.NewReader(data))
+	for s.Scan() {
+		require.NoError(t, bldr.UnmarshalJSON(s.Bytes()))
+	}
+
+	rec := bldr.NewRecordBatch()
+	defer rec.Release()
+
+	// The file schema lacks the nested_list
+	fileIcesc := iceberg.NewSchema(1, iceberg.NestedField{
+		ID: 10, Name: "other_field", Type: iceberg.PrimitiveTypes.Int32, Required: false,
+	})
+
+	// The requested schema has the nested_list
+	reqIcesc := iceberg.NewSchema(1, iceberg.NestedField{
+		ID: 10, Name: "other_field", Type: iceberg.PrimitiveTypes.Int32, Required: false,
+	}, iceberg.NestedField{
+		ID: 11, Name: "nested_list", Type: &iceberg.ListType{
+			ElementID: 12, Element: iceberg.PrimitiveTypes.String, ElementRequired: false,
+		}, Required: false,
+	})
+
+	rec2, err := table.ToRequestedSchema(context.Background(), reqIcesc, fileIcesc, rec, table.SchemaOptions{IncludeFieldIDs: true})
+	require.NoError(t, err)
+	defer rec2.Release()
+
+	targetSchema := arrow.NewSchema([]arrow.Field{
+		{
+			Name: "other_field", Type: arrow.PrimitiveTypes.Int32, Nullable: true,
+			Metadata: arrow.MetadataFrom(map[string]string{table.ArrowParquetFieldIDKey: "10"}),
+		},
+		{
+			Name: "nested_list", Type: arrow.ListOfField(arrow.Field{
+				Name: "element", Type: arrow.BinaryTypes.String, Nullable: true,
+				Metadata: arrow.MetadataFrom(map[string]string{table.ArrowParquetFieldIDKey: "12"}),
+			}),
+			Metadata: arrow.MetadataFrom(map[string]string{table.ArrowParquetFieldIDKey: "11"}),
+			Nullable: true,
+		},
+	}, nil)
+	require.True(t, targetSchema.Equal(rec2.Schema()), "Schema is not perfectly equal")
+}