You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by ze...@apache.org on 2026/08/26 17:52:16 UTC
(iceberg-go) branch main updated: fix(manifest): handle foreign time/timestamp partition logical types (#1861)
This is an automated email from the ASF dual-hosted git repository.
zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-go.git
The following commit(s) were added to refs/heads/main by this push:
new 8d9b8139b fix(manifest): handle foreign time/timestamp partition logical types (#1861)
8d9b8139b is described below
commit 8d9b8139b445283ad0f2b87e3be1fb1bb4d7120c
Author: Tanmay Rauth <t_...@apple.com>
AuthorDate: Wed Aug 26 10:52:09 2026 -0700
fix(manifest): handle foreign time/timestamp partition logical types (#1861)
Partition decoding asserted v.(int64) unchecked on every time and
timestamp arm. A manifest that declares a time/timestamp logicalType on
an underlying Avro type the spec forbids (e.g. timestamp-millis on an
int) decodes to a raw int32, panicking Partition() with no way to
recover. time-millis and timestamp-millis were also returned as
milliseconds though iceberg.Time and iceberg.Timestamp count
microseconds, 1000x too small.
Accept the raw int32/int64 value and scale milliseconds to microseconds.
---
manifest.go | 55 ++++++++++--
manifest_time_partition_test.go | 185 ++++++++++++++++++++++++++++++++++++++++
2 files changed, 233 insertions(+), 7 deletions(-)
diff --git a/manifest.go b/manifest.go
index 9cb2c46d4..5e3bbff13 100644
--- a/manifest.go
+++ b/manifest.go
@@ -2355,35 +2355,56 @@ func (d *dataFile) convertAvroValueToIcebergType(v any, fieldID int) any {
// silently coerces other types, which reintroduces #1200.
return v
case atype.TimeMillis:
+ // iceberg.Time counts microseconds; a time-millis Duration must be
+ // read as microseconds (val.Milliseconds() undercounts 1000x). The
+ // raw arm converts the millisecond wire value to microseconds too.
if val, ok := v.(time.Duration); ok {
- return Time(val.Milliseconds())
+ return Time(val.Microseconds())
+ }
+ if ms, ok := avroIntAsInt64(v); ok {
+ return Time(ms * 1000)
}
- return Time(v.(int64))
+ return v
case atype.TimeMicros:
if val, ok := v.(time.Duration); ok {
return Time(val.Microseconds())
}
+ if us, ok := avroIntAsInt64(v); ok {
+ return Time(us)
+ }
- return Time(v.(int64))
+ return v
case atype.TimestampMillis:
+ // iceberg.Timestamp counts microseconds; UnixMilli() undercounts
+ // 1000x. Read both the time.Time and the raw millisecond wire value
+ // as microseconds.
if val, ok := v.(time.Time); ok {
- return Timestamp(val.UTC().UnixMilli())
+ return Timestamp(val.UTC().UnixMicro())
+ }
+ if ms, ok := avroIntAsInt64(v); ok {
+ return Timestamp(ms * 1000)
}
- return Timestamp(v.(int64))
+ return v
case atype.TimestampMicros:
if val, ok := v.(time.Time); ok {
return Timestamp(val.UTC().UnixMicro())
}
+ if us, ok := avroIntAsInt64(v); ok {
+ return Timestamp(us)
+ }
- return Timestamp(v.(int64))
+ return v
case atype.TimestampNanos:
if val, ok := v.(time.Time); ok {
return TimestampNano(val.UTC().UnixNano())
}
+ if ns, ok := avroIntAsInt64(v); ok {
+ return TimestampNano(ns)
+ }
- return TimestampNano(v.(int64))
+ return v
case atype.Decimal:
if r, ok := v.(*big.Rat); ok {
scale := d.fieldIDToDecimalScale[fieldID]
@@ -2410,6 +2431,26 @@ func (d *dataFile) convertAvroValueToIcebergType(v any, fieldID int) any {
return v
}
+// avroIntAsInt64 extracts an Avro int (int32) or long (int64) partition value
+// as an int64. A manifest from a foreign writer may declare a time/timestamp
+// logicalType on an underlying Avro type the spec does not permit for it (e.g.
+// timestamp-millis on an int). twmb/avro drops such an invalid logicalType when
+// decoding and yields the raw primitive, yet Schema.Root still reports the
+// logicalType, so the conversion arms must accept the raw integer instead of an
+// unchecked assertion that would panic (apache/iceberg-go#1847).
+func avroIntAsInt64(v any) (int64, bool) {
+ switch n := v.(type) {
+ case int64:
+ return n, true
+ case int32:
+ return int64(n), true
+ case int:
+ return int64(n), true
+ }
+
+ return 0, false
+}
+
func (d *dataFile) setFieldNameToIDMap(m map[string]int) { d.fieldNameToID = m }
func (d *dataFile) setFieldIDToLogicalTypeMap(m map[int]string) {
d.fieldIDToLogicalType = m
diff --git a/manifest_time_partition_test.go b/manifest_time_partition_test.go
new file mode 100644
index 000000000..18ef698c4
--- /dev/null
+++ b/manifest_time_partition_test.go
@@ -0,0 +1,185 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package iceberg
+
+import (
+ "bytes"
+ "encoding/json"
+ "testing"
+
+ "github.com/apache/iceberg-go/internal"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+ "github.com/twmb/avro"
+ "github.com/twmb/avro/atype"
+ "github.com/twmb/avro/ocf"
+)
+
+// timePartitionFieldID is the partition field id used by the time/timestamp
+// logical-type tests.
+const timePartitionFieldID = 1001
+
+// writeTimePartitionManifest encodes a single-entry v2 manifest whose only
+// partition field carries the given Avro schema node and integer value. The
+// node's logical type is written verbatim to the on-disk schema, letting the
+// test reproduce manifests from foreign writers that declare a time/timestamp
+// logical type on an underlying Avro type the spec does not permit for it.
+func writeTimePartitionManifest(t *testing.T, partFieldNode avro.SchemaNode, value any) ([]byte, ManifestFile) {
+ t.Helper()
+
+ field := PartitionField{
+ FieldID: timePartitionFieldID,
+ SourceIDs: []int{1},
+ Name: "ts",
+ Transform: IdentityTransform{},
+ }
+ spec := NewPartitionSpecID(0, field)
+
+ builder, err := NewDataFileBuilder(
+ spec,
+ EntryContentData,
+ "s3://bucket/namespace/table/data/00000-0-time.parquet",
+ ParquetFile,
+ map[int]any{timePartitionFieldID: value},
+ map[int]string{},
+ map[int]int{},
+ 100,
+ 1024,
+ )
+ require.NoError(t, err)
+
+ snapshotID := int64(42)
+ seqNum := int64(1)
+ entry := NewManifestEntry(EntryStatusADDED, &snapshotID, &seqNum, &seqNum, builder.Build())
+
+ partNode := avro.SchemaNode{
+ Type: atype.Record,
+ Name: "r102",
+ Fields: []avro.SchemaField{
+ {Name: "ts", Type: partFieldNode, Props: internal.WithFieldID(timePartitionFieldID)},
+ },
+ }
+ partSchema, err := partNode.Schema()
+ require.NoError(t, err)
+
+ entrySchema, err := internal.NewManifestEntrySchema(partSchema, 2)
+ require.NoError(t, err)
+
+ specFieldsJSON, err := json.Marshal([]PartitionField{field})
+ require.NoError(t, err)
+
+ var buf bytes.Buffer
+ wr, err := ocf.NewWriter(&buf, entrySchema,
+ ocf.WithSchema(entrySchema.String()),
+ ocf.WithMetadata(map[string][]byte{
+ "format-version": []byte("2"),
+ "content": []byte("data"),
+ "partition-spec": specFieldsJSON,
+ "partition-spec-id": []byte("0"),
+ }))
+ require.NoError(t, err)
+ require.NoError(t, wr.Encode(entry))
+ require.NoError(t, wr.Close())
+
+ file := &manifestFile{
+ Path: "s3://bucket/namespace/table/metadata/00000-time.avro",
+ SpecID: 0,
+ Content: ManifestContentData,
+ }
+ file.setVersion(2)
+
+ return buf.Bytes(), file
+}
+
+// TestTimeLogicalPartitionScaling verifies that time-millis and
+// timestamp-millis partition values decode to microseconds (iceberg.Time and
+// iceberg.Timestamp both count microseconds), not milliseconds. See
+// apache/iceberg-go#1847.
+func TestTimeLogicalPartitionScaling(t *testing.T) {
+ cases := []struct {
+ name string
+ partNode avro.SchemaNode
+ value any
+ want any
+ }{
+ {
+ name: "time-micros",
+ partNode: internal.NullableNode(internal.TimeNode),
+ value: int64(3_600_000_000), // 01:00:00 in microseconds
+ want: Time(3_600_000_000),
+ },
+ {
+ name: "time-millis",
+ partNode: internal.NullableNode(avro.SchemaNode{Type: atype.Int, LogicalType: atype.TimeMillis}),
+ value: int32(3_600_000), // 01:00:00 in milliseconds
+ want: Time(3_600_000_000),
+ },
+ {
+ name: "timestamp-micros",
+ partNode: internal.NullableNode(internal.TimestampNode),
+ value: int64(1_700_000_000_000_000),
+ want: Timestamp(1_700_000_000_000_000),
+ },
+ {
+ name: "timestamp-millis",
+ partNode: internal.NullableNode(avro.SchemaNode{Type: atype.Long, LogicalType: atype.TimestampMillis, Props: map[string]any{"adjust-to-utc": false}}),
+ value: int64(1_700_000_000_000), // milliseconds
+ want: Timestamp(1_700_000_000_000_000),
+ },
+ }
+
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ data, file := writeTimePartitionManifest(t, tc.partNode, tc.value)
+
+ entries, err := ReadManifest(file, bytes.NewReader(data), false)
+ require.NoError(t, err)
+ require.Len(t, entries, 1)
+
+ partition := entries[0].DataFile().Partition()
+ got, ok := partition[timePartitionFieldID]
+ require.True(t, ok, "partition value for field %d must be present", timePartitionFieldID)
+ assert.IsType(t, tc.want, got)
+ assert.Equal(t, tc.want, got)
+ })
+ }
+}
+
+// TestTimestampLogicalOnMismatchedTypeDoesNotPanic reproduces a manifest from a
+// foreign writer that declares timestamp-millis on an Avro int, which the spec
+// does not permit. twmb/avro drops the invalid logical type when decoding and
+// yields the raw int32, yet Schema.Root still reports the logical type. Reading
+// such a manifest must not panic (previously an unchecked v.(int64) assertion
+// crashed here). See apache/iceberg-go#1847.
+func TestTimestampLogicalOnMismatchedTypeDoesNotPanic(t *testing.T) {
+ partNode := internal.NullableNode(avro.SchemaNode{Type: atype.Int, LogicalType: atype.TimestampMillis})
+
+ data, file := writeTimePartitionManifest(t, partNode, int32(123456))
+
+ entries, err := ReadManifest(file, bytes.NewReader(data), false)
+ require.NoError(t, err)
+ require.Len(t, entries, 1)
+
+ require.NotPanics(t, func() {
+ partition := entries[0].DataFile().Partition()
+ got, ok := partition[timePartitionFieldID]
+ require.True(t, ok)
+ // The raw millisecond value is scaled to microseconds.
+ assert.Equal(t, Timestamp(int64(123456)*1000), got)
+ })
+}