You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by ze...@apache.org on 2026/08/26 17:52:16 UTC

(iceberg-go) branch main updated: fix(manifest): handle foreign time/timestamp partition logical types (#1861)

This is an automated email from the ASF dual-hosted git repository.

zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-go.git


The following commit(s) were added to refs/heads/main by this push:
     new 8d9b8139b fix(manifest): handle foreign time/timestamp partition logical types (#1861)
8d9b8139b is described below

commit 8d9b8139b445283ad0f2b87e3be1fb1bb4d7120c
Author: Tanmay Rauth <t_...@apple.com>
AuthorDate: Wed Aug 26 10:52:09 2026 -0700

    fix(manifest): handle foreign time/timestamp partition logical types (#1861)
    
    Partition decoding asserted v.(int64) unchecked on every time and
    timestamp arm. A manifest that declares a time/timestamp logicalType on
    an underlying Avro type the spec forbids (e.g. timestamp-millis on an
    int) decodes to a raw int32, panicking Partition() with no way to
    recover. time-millis and timestamp-millis were also returned as
    milliseconds though iceberg.Time and iceberg.Timestamp count
    microseconds, 1000x too small.
    
    Accept the raw int32/int64 value and scale milliseconds to microseconds.
---
 manifest.go                     |  55 ++++++++++--
 manifest_time_partition_test.go | 185 ++++++++++++++++++++++++++++++++++++++++
 2 files changed, 233 insertions(+), 7 deletions(-)

diff --git a/manifest.go b/manifest.go
index 9cb2c46d4..5e3bbff13 100644
--- a/manifest.go
+++ b/manifest.go
@@ -2355,35 +2355,56 @@ func (d *dataFile) convertAvroValueToIcebergType(v any, fieldID int) any {
 			// silently coerces other types, which reintroduces #1200.
 			return v
 		case atype.TimeMillis:
+			// iceberg.Time counts microseconds; a time-millis Duration must be
+			// read as microseconds (val.Milliseconds() undercounts 1000x). The
+			// raw arm converts the millisecond wire value to microseconds too.
 			if val, ok := v.(time.Duration); ok {
-				return Time(val.Milliseconds())
+				return Time(val.Microseconds())
+			}
+			if ms, ok := avroIntAsInt64(v); ok {
+				return Time(ms * 1000)
 			}
 
-			return Time(v.(int64))
+			return v
 		case atype.TimeMicros:
 			if val, ok := v.(time.Duration); ok {
 				return Time(val.Microseconds())
 			}
+			if us, ok := avroIntAsInt64(v); ok {
+				return Time(us)
+			}
 
-			return Time(v.(int64))
+			return v
 		case atype.TimestampMillis:
+			// iceberg.Timestamp counts microseconds; UnixMilli() undercounts
+			// 1000x. Read both the time.Time and the raw millisecond wire value
+			// as microseconds.
 			if val, ok := v.(time.Time); ok {
-				return Timestamp(val.UTC().UnixMilli())
+				return Timestamp(val.UTC().UnixMicro())
+			}
+			if ms, ok := avroIntAsInt64(v); ok {
+				return Timestamp(ms * 1000)
 			}
 
-			return Timestamp(v.(int64))
+			return v
 		case atype.TimestampMicros:
 			if val, ok := v.(time.Time); ok {
 				return Timestamp(val.UTC().UnixMicro())
 			}
+			if us, ok := avroIntAsInt64(v); ok {
+				return Timestamp(us)
+			}
 
-			return Timestamp(v.(int64))
+			return v
 		case atype.TimestampNanos:
 			if val, ok := v.(time.Time); ok {
 				return TimestampNano(val.UTC().UnixNano())
 			}
+			if ns, ok := avroIntAsInt64(v); ok {
+				return TimestampNano(ns)
+			}
 
-			return TimestampNano(v.(int64))
+			return v
 		case atype.Decimal:
 			if r, ok := v.(*big.Rat); ok {
 				scale := d.fieldIDToDecimalScale[fieldID]
@@ -2410,6 +2431,26 @@ func (d *dataFile) convertAvroValueToIcebergType(v any, fieldID int) any {
 	return v
 }
 
+// avroIntAsInt64 extracts an Avro int (int32) or long (int64) partition value
+// as an int64. A manifest from a foreign writer may declare a time/timestamp
+// logicalType on an underlying Avro type the spec does not permit for it (e.g.
+// timestamp-millis on an int). twmb/avro drops such an invalid logicalType when
+// decoding and yields the raw primitive, yet Schema.Root still reports the
+// logicalType, so the conversion arms must accept the raw integer instead of an
+// unchecked assertion that would panic (apache/iceberg-go#1847).
+func avroIntAsInt64(v any) (int64, bool) {
+	switch n := v.(type) {
+	case int64:
+		return n, true
+	case int32:
+		return int64(n), true
+	case int:
+		return int64(n), true
+	}
+
+	return 0, false
+}
+
 func (d *dataFile) setFieldNameToIDMap(m map[string]int) { d.fieldNameToID = m }
 func (d *dataFile) setFieldIDToLogicalTypeMap(m map[int]string) {
 	d.fieldIDToLogicalType = m
diff --git a/manifest_time_partition_test.go b/manifest_time_partition_test.go
new file mode 100644
index 000000000..18ef698c4
--- /dev/null
+++ b/manifest_time_partition_test.go
@@ -0,0 +1,185 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package iceberg
+
+import (
+	"bytes"
+	"encoding/json"
+	"testing"
+
+	"github.com/apache/iceberg-go/internal"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/twmb/avro"
+	"github.com/twmb/avro/atype"
+	"github.com/twmb/avro/ocf"
+)
+
+// timePartitionFieldID is the partition field id used by the time/timestamp
+// logical-type tests.
+const timePartitionFieldID = 1001
+
+// writeTimePartitionManifest encodes a single-entry v2 manifest whose only
+// partition field carries the given Avro schema node and integer value. The
+// node's logical type is written verbatim to the on-disk schema, letting the
+// test reproduce manifests from foreign writers that declare a time/timestamp
+// logical type on an underlying Avro type the spec does not permit for it.
+func writeTimePartitionManifest(t *testing.T, partFieldNode avro.SchemaNode, value any) ([]byte, ManifestFile) {
+	t.Helper()
+
+	field := PartitionField{
+		FieldID:   timePartitionFieldID,
+		SourceIDs: []int{1},
+		Name:      "ts",
+		Transform: IdentityTransform{},
+	}
+	spec := NewPartitionSpecID(0, field)
+
+	builder, err := NewDataFileBuilder(
+		spec,
+		EntryContentData,
+		"s3://bucket/namespace/table/data/00000-0-time.parquet",
+		ParquetFile,
+		map[int]any{timePartitionFieldID: value},
+		map[int]string{},
+		map[int]int{},
+		100,
+		1024,
+	)
+	require.NoError(t, err)
+
+	snapshotID := int64(42)
+	seqNum := int64(1)
+	entry := NewManifestEntry(EntryStatusADDED, &snapshotID, &seqNum, &seqNum, builder.Build())
+
+	partNode := avro.SchemaNode{
+		Type: atype.Record,
+		Name: "r102",
+		Fields: []avro.SchemaField{
+			{Name: "ts", Type: partFieldNode, Props: internal.WithFieldID(timePartitionFieldID)},
+		},
+	}
+	partSchema, err := partNode.Schema()
+	require.NoError(t, err)
+
+	entrySchema, err := internal.NewManifestEntrySchema(partSchema, 2)
+	require.NoError(t, err)
+
+	specFieldsJSON, err := json.Marshal([]PartitionField{field})
+	require.NoError(t, err)
+
+	var buf bytes.Buffer
+	wr, err := ocf.NewWriter(&buf, entrySchema,
+		ocf.WithSchema(entrySchema.String()),
+		ocf.WithMetadata(map[string][]byte{
+			"format-version":    []byte("2"),
+			"content":           []byte("data"),
+			"partition-spec":    specFieldsJSON,
+			"partition-spec-id": []byte("0"),
+		}))
+	require.NoError(t, err)
+	require.NoError(t, wr.Encode(entry))
+	require.NoError(t, wr.Close())
+
+	file := &manifestFile{
+		Path:    "s3://bucket/namespace/table/metadata/00000-time.avro",
+		SpecID:  0,
+		Content: ManifestContentData,
+	}
+	file.setVersion(2)
+
+	return buf.Bytes(), file
+}
+
+// TestTimeLogicalPartitionScaling verifies that time-millis and
+// timestamp-millis partition values decode to microseconds (iceberg.Time and
+// iceberg.Timestamp both count microseconds), not milliseconds. See
+// apache/iceberg-go#1847.
+func TestTimeLogicalPartitionScaling(t *testing.T) {
+	cases := []struct {
+		name     string
+		partNode avro.SchemaNode
+		value    any
+		want     any
+	}{
+		{
+			name:     "time-micros",
+			partNode: internal.NullableNode(internal.TimeNode),
+			value:    int64(3_600_000_000), // 01:00:00 in microseconds
+			want:     Time(3_600_000_000),
+		},
+		{
+			name:     "time-millis",
+			partNode: internal.NullableNode(avro.SchemaNode{Type: atype.Int, LogicalType: atype.TimeMillis}),
+			value:    int32(3_600_000), // 01:00:00 in milliseconds
+			want:     Time(3_600_000_000),
+		},
+		{
+			name:     "timestamp-micros",
+			partNode: internal.NullableNode(internal.TimestampNode),
+			value:    int64(1_700_000_000_000_000),
+			want:     Timestamp(1_700_000_000_000_000),
+		},
+		{
+			name:     "timestamp-millis",
+			partNode: internal.NullableNode(avro.SchemaNode{Type: atype.Long, LogicalType: atype.TimestampMillis, Props: map[string]any{"adjust-to-utc": false}}),
+			value:    int64(1_700_000_000_000), // milliseconds
+			want:     Timestamp(1_700_000_000_000_000),
+		},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			data, file := writeTimePartitionManifest(t, tc.partNode, tc.value)
+
+			entries, err := ReadManifest(file, bytes.NewReader(data), false)
+			require.NoError(t, err)
+			require.Len(t, entries, 1)
+
+			partition := entries[0].DataFile().Partition()
+			got, ok := partition[timePartitionFieldID]
+			require.True(t, ok, "partition value for field %d must be present", timePartitionFieldID)
+			assert.IsType(t, tc.want, got)
+			assert.Equal(t, tc.want, got)
+		})
+	}
+}
+
+// TestTimestampLogicalOnMismatchedTypeDoesNotPanic reproduces a manifest from a
+// foreign writer that declares timestamp-millis on an Avro int, which the spec
+// does not permit. twmb/avro drops the invalid logical type when decoding and
+// yields the raw int32, yet Schema.Root still reports the logical type. Reading
+// such a manifest must not panic (previously an unchecked v.(int64) assertion
+// crashed here). See apache/iceberg-go#1847.
+func TestTimestampLogicalOnMismatchedTypeDoesNotPanic(t *testing.T) {
+	partNode := internal.NullableNode(avro.SchemaNode{Type: atype.Int, LogicalType: atype.TimestampMillis})
+
+	data, file := writeTimePartitionManifest(t, partNode, int32(123456))
+
+	entries, err := ReadManifest(file, bytes.NewReader(data), false)
+	require.NoError(t, err)
+	require.Len(t, entries, 1)
+
+	require.NotPanics(t, func() {
+		partition := entries[0].DataFile().Partition()
+		got, ok := partition[timePartitionFieldID]
+		require.True(t, ok)
+		// The raw millisecond value is scaled to microseconds.
+		assert.Equal(t, Timestamp(int64(123456)*1000), got)
+	})
+}