You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by ze...@apache.org on 2026/04/14 16:23:39 UTC
(iceberg-go) branch main updated: feat(avro): replace hamba/avro with twmb/avro v1.5.0 (#871)
This is an automated email from the ASF dual-hosted git repository.
zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-go.git
The following commit(s) were added to refs/heads/main by this push:
new e175604d feat(avro): replace hamba/avro with twmb/avro v1.5.0 (#871)
e175604d is described below
commit e175604d1d0582a7942e9402bd3f49e2acee5c0b
Author: Travis Bischel <tw...@protonmail.com>
AuthorDate: Tue Apr 14 10:23:33 2026 -0600
feat(avro): replace hamba/avro with twmb/avro v1.5.0 (#871)
Drop github.com/hamba/avro/v2 in favor of github.com/twmb/avro v1.5.0
for manifest reading, writing, and partition schema construction,
replacing the archived hamba/avro.
# Behavior differences visible to callers
- DataFile.Partition() returns int32 for plain integer partition fields
(was int). Correct per Avro spec — Avro int is 32-bit signed. Go code
doing v.(int) on partition values needs to switch to v.(int32).
- NewDataFileBuilder takes map[int]string instead of
map[int]avro.LogicalType. The corresponding field on dataFile and the
hasFieldToIDMap interface also now use string. Logical type constants
live in the new github.com/twmb/avro/atype subpackage (atype.Date,
atype.TimestampMicros, etc.), untyped string constants, drop-in wherever
a string is expected.
All other hamba/avro types used by the public surface are replaced:
avro.Schema → *avro.Schema on the handful of exported helpers that
referenced them; avro.SchemaCache (internal.AvroSchemaCache) →
map[string]*avro.Schema (internal.AvroSchemaMap).
# Dependency footprint
Removes five transitive dependencies: json-iterator/go,
modern-go/concurrent, modern-go/reflect2, ettle/strcase, and
go-viper/mapstructure (hamba's chain). twmb/avro's only direct
dependency is klauspost/compress, which iceberg-go already transitively
requires.
# Tests
New TestPartitionTypeToAvroSchemaDuplicateNamedTypes covers partition
spec scenarios with multiple fields of the same named Avro type (two
UUIDs, two same-size Fixed, two same-precision Decimal).
Co-authored-by: Claude Opus 4.6 (1M context) <no...@anthropic.com>
---
go.mod | 5 +-
go.sum | 13 +-
internal/avro_schemas.go | 864 ++++++++++++++++++++-------------------------
manifest.go | 324 +++++++----------
manifest_test.go | 139 ++++----
schema_conversions.go | 53 +--
schema_conversions_test.go | 161 ++++-----
table/internal/utils.go | 16 +-
table/table_test.go | 2 +-
9 files changed, 682 insertions(+), 895 deletions(-)
diff --git a/go.mod b/go.mod
index af07c7d7..ce6d0a16 100644
--- a/go.mod
+++ b/go.mod
@@ -37,11 +37,11 @@ require (
github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815
github.com/google/go-cmp v0.7.0
github.com/google/uuid v1.6.0
- github.com/hamba/avro/v2 v2.31.0
github.com/pterm/pterm v0.12.83
github.com/stretchr/testify v1.11.1
github.com/substrait-io/substrait-go/v7 v7.6.0
github.com/testcontainers/testcontainers-go/modules/compose v0.42.0
+ github.com/twmb/avro v1.5.0
github.com/twmb/murmur3 v1.1.8
github.com/uptrace/bun v1.2.18
github.com/uptrace/bun/dialect/mssqldialect v1.2.18
@@ -169,7 +169,6 @@ require (
github.com/inhies/go-bytesize v0.0.0-20220417184213-4913239db9cf // indirect
github.com/jinzhu/inflection v1.0.0 // indirect
github.com/jonboulle/clockwork v0.5.0 // indirect
- github.com/json-iterator/go v1.1.12 // indirect
github.com/klauspost/asmfmt v1.3.2 // indirect
github.com/klauspost/compress v1.18.5 // indirect
github.com/klauspost/cpuid/v2 v2.3.0 // indirect
@@ -200,8 +199,6 @@ require (
github.com/moby/sys/user v0.4.0 // indirect
github.com/moby/sys/userns v0.1.0 // indirect
github.com/moby/term v0.5.2 // indirect
- github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
- github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect
github.com/morikuni/aec v1.1.0 // indirect
github.com/ncruces/go-strftime v1.0.0 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
diff --git a/go.sum b/go.sum
index b323c2db..895df69a 100644
--- a/go.sum
+++ b/go.sum
@@ -351,7 +351,6 @@ github.com/google/go-replayers/grpcreplay v1.3.0 h1:1Keyy0m1sIpqstQmgz307zhiJ1pV
github.com/google/go-replayers/grpcreplay v1.3.0/go.mod h1:v6NgKtkijC0d3e3RW8il6Sy5sqRVUwoQa4mHOGEy8DI=
github.com/google/go-replayers/httpreplay v1.2.0 h1:VM1wEyyjaoU53BwrOnaf9VhAyQQEEioJvFYxYcLRKzk=
github.com/google/go-replayers/httpreplay v1.2.0/go.mod h1:WahEFFZZ7a1P4VM1qEeHy+tME4bwyqPcwWbNlUI1Mcg=
-github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc=
github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0=
github.com/google/pprof v0.0.0-20250820193118-f64d9cf942d6 h1:EEHtgt9IwisQ2AZ4pIsMjahcegHh6rmhqxzIRQIyepY=
@@ -378,8 +377,6 @@ github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 h1:HWRh5R2+9EifMyIHV7ZV+MIZqgz+PMpZ14Jynv3O2Zs=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0/go.mod h1:JfhWUomR1baixubs02l85lZYYOm7LV6om4ceouMv45c=
-github.com/hamba/avro/v2 v2.31.0 h1:wv3nmua7lCEIwWsb6vqsTS3pXktTxcKg5eoyNu0VhrU=
-github.com/hamba/avro/v2 v2.31.0/go.mod h1:t6lJYAGE5Mswfn17zjtyQsssRQgnqO6TXLBCHHWRqrw=
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
@@ -403,8 +400,6 @@ github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD
github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
github.com/jonboulle/clockwork v0.5.0 h1:Hyh9A8u51kptdkR+cqRpT1EebBwTn1oK9YfGYbdFz6I=
github.com/jonboulle/clockwork v0.5.0/go.mod h1:3mZlmanh0g2NDKO5TWZVJAfofYk64M7XN3SzBPjZF60=
-github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
-github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
github.com/keybase/go-keychain v0.0.1 h1:way+bWYa6lDppZoZcgMbYsvC7GxljxrskdNInRtuthU=
github.com/keybase/go-keychain v0.0.1/go.mod h1:PdEILRW3i9D8JcdM+FmY6RwkHGnhHxXwkPPMeUgOK1k=
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
@@ -486,12 +481,6 @@ github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g
github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28=
github.com/moby/term v0.5.2 h1:6qk3FJAFDs6i/q3W/pQ97SX192qKfZgGjCQqfCJkgzQ=
github.com/moby/term v0.5.2/go.mod h1:d3djjFCrjnB+fl8NJux+EJzu0msscUP+f8it8hPkFLc=
-github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
-github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
-github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
-github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
-github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8=
-github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/morikuni/aec v1.1.0 h1:vBBl0pUnvi/Je71dsRrhMBtreIqNMYErSAbEeb8jrXQ=
github.com/morikuni/aec v1.1.0/go.mod h1:xDRgiq/iw5l+zkao76YTKzKttOp2cwPEne25HDkJnBw=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
@@ -630,6 +619,8 @@ github.com/transparency-dev/formats v0.0.0-20251017110053-404c0d5b696c h1:5a2XDQ
github.com/transparency-dev/formats v0.0.0-20251017110053-404c0d5b696c/go.mod h1:g85IafeFJZLxlzZCDRu4JLpfS7HKzR+Hw9qRh3bVzDI=
github.com/transparency-dev/merkle v0.0.2 h1:Q9nBoQcZcgPamMkGn7ghV8XiTZ/kRxn1yCG81+twTK4=
github.com/transparency-dev/merkle v0.0.2/go.mod h1:pqSy+OXefQ1EDUVmAJ8MUhHB9TXGuzVAT58PqBoHz1A=
+github.com/twmb/avro v1.5.0 h1:9jmbvVQQBcyWHv/6zS+q5+nmASiR8/GwhKF/sU7u71c=
+github.com/twmb/avro v1.5.0/go.mod h1:X0fT1dY2xcbV4YuCE4mYro+qljHl4kUF5uA/2z1rgSk=
github.com/twmb/murmur3 v1.1.8 h1:8Yt9taO/WN3l08xErzjeschgZU2QSrwm1kclYq+0aRg=
github.com/twmb/murmur3 v1.1.8/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ=
github.com/uptrace/bun v1.2.18 h1:3HnRcMfS6OBPMG1eSOzlbFJ/X/AyMEJb7rMxE6VQvDU=
diff --git a/internal/avro_schemas.go b/internal/avro_schemas.go
index 5997ff1d..720f4be4 100644
--- a/internal/avro_schemas.go
+++ b/internal/avro_schemas.go
@@ -19,14 +19,27 @@ package internal
import (
"fmt"
+ "slices"
- "github.com/hamba/avro/v2"
+ "github.com/twmb/avro"
+ "github.com/twmb/avro/atype"
)
-func NullableSchema(schema avro.Schema) avro.Schema {
- return Must(avro.NewUnionSchema([]avro.Schema{
- NullSchema, schema,
- }))
+func NullableNode(node avro.SchemaNode) avro.SchemaNode {
+ return avro.SchemaNode{
+ Type: "union",
+ Branches: []avro.SchemaNode{{Type: "null"}, node},
+ }
+}
+
+// mustSchema compiles a SchemaNode into a Schema, panicking on error.
+// This avoids the pointer-receiver addressability issue with struct literals.
+func mustSchema(n avro.SchemaNode) *avro.Schema {
+ return Must(n.Schema())
+}
+
+func NullableSchema(schema *avro.Schema) *avro.Schema {
+ return mustSchema(NullableNode(schema.Root()))
}
var requiredLength = [...]int{
@@ -46,499 +59,378 @@ func DecimalRequiredBytes(precision int) int {
return requiredLength[precision]
}
-func DecimalSchema(precision, scale int) avro.Schema {
- return Must(avro.NewFixedSchema("fixed", "",
- DecimalRequiredBytes(precision), avro.NewDecimalLogicalSchema(precision, scale)))
+func DecimalNode(precision, scale int) avro.SchemaNode {
+ return avro.SchemaNode{
+ Type: atype.Fixed,
+ Name: "fixed",
+ Size: DecimalRequiredBytes(precision),
+ LogicalType: atype.Decimal,
+ Precision: precision,
+ Scale: scale,
+ }
+}
+
+func DecimalSchema(precision, scale int) *avro.Schema {
+ return mustSchema(DecimalNode(precision, scale))
}
-func FixedSchema(size int) avro.Schema {
- return Must(avro.NewFixedSchema(fmt.Sprintf("fixed_%d", size), "", size, nil))
+func FixedNode(size int) avro.SchemaNode {
+ return avro.SchemaNode{
+ Type: atype.Fixed,
+ Name: fmt.Sprintf("fixed_%d", size),
+ Size: size,
+ }
}
+func FixedSchema(size int) *avro.Schema {
+ return mustSchema(FixedNode(size))
+}
+
+// Schema nodes for reuse in composition.
+var (
+ NullNode = avro.SchemaNode{Type: atype.Null}
+ BoolNode = avro.SchemaNode{Type: atype.Boolean}
+ IntNode = avro.SchemaNode{Type: atype.Int}
+ LongNode = avro.SchemaNode{Type: atype.Long}
+ FloatNode = avro.SchemaNode{Type: atype.Float}
+ DoubleNode = avro.SchemaNode{Type: atype.Double}
+ StringNode = avro.SchemaNode{Type: atype.String}
+ BytesNode = avro.SchemaNode{Type: atype.Bytes}
+
+ DateNode = avro.SchemaNode{Type: atype.Int, LogicalType: atype.Date}
+ TimeNode = avro.SchemaNode{Type: atype.Long, LogicalType: atype.TimeMicros}
+ TimestampNode = avro.SchemaNode{Type: atype.Long, LogicalType: atype.TimestampMicros, Props: map[string]any{"adjust-to-utc": false}}
+ TimestampTzNode = avro.SchemaNode{Type: atype.Long, LogicalType: atype.TimestampMicros, Props: map[string]any{"adjust-to-utc": true}}
+ UUIDNode = avro.SchemaNode{Type: atype.Fixed, Name: "uuid_fixed", Size: 16, LogicalType: atype.UUID}
+)
+
+// Compiled schemas for direct encoding/decoding use.
var (
- NullSchema = avro.NewNullSchema()
- BoolSchema = avro.NewPrimitiveSchema(avro.Boolean, nil)
+ NullSchema = mustSchema(NullNode)
+ BoolSchema = mustSchema(BoolNode)
NullableBoolSchema = NullableSchema(BoolSchema)
- BinarySchema = avro.NewPrimitiveSchema(avro.Bytes, nil)
+ BinarySchema = mustSchema(BytesNode)
NullableBinarySchema = NullableSchema(BinarySchema)
- StringSchema = avro.NewPrimitiveSchema(avro.String, nil)
- IntSchema = avro.NewPrimitiveSchema(avro.Int, nil)
+ StringSchema = mustSchema(StringNode)
+ IntSchema = mustSchema(IntNode)
NullableIntSchema = NullableSchema(IntSchema)
- LongSchema = avro.NewPrimitiveSchema(avro.Long, nil)
+ LongSchema = mustSchema(LongNode)
NullableLongSchema = NullableSchema(LongSchema)
- FloatSchema = avro.NewPrimitiveSchema(avro.Float, nil)
- DoubleSchema = avro.NewPrimitiveSchema(avro.Double, nil)
- DateSchema = avro.NewPrimitiveSchema(avro.Int, avro.NewPrimitiveLogicalSchema(avro.Date))
- TimeSchema = avro.NewPrimitiveSchema(avro.Long, avro.NewPrimitiveLogicalSchema(avro.TimeMicros))
- TimestampSchema = avro.NewPrimitiveSchema(avro.Long, avro.NewPrimitiveLogicalSchema(avro.TimestampMicros),
- avro.WithProps(map[string]any{"adjust-to-utc": false}))
- TimestampTzSchema = avro.NewPrimitiveSchema(avro.Long, avro.NewPrimitiveLogicalSchema(avro.TimestampMicros),
- avro.WithProps(map[string]any{"adjust-to-utc": true}))
- UUIDSchema = Must(avro.NewFixedSchema("uuid", "", 16, avro.NewPrimitiveLogicalSchema(avro.UUID)))
-
- AvroSchemaCache avro.SchemaCache
+ FloatSchema = mustSchema(FloatNode)
+ DoubleSchema = mustSchema(DoubleNode)
+ DateSchema = mustSchema(DateNode)
+ TimeSchema = mustSchema(TimeNode)
+ TimestampSchema = mustSchema(TimestampNode)
+ TimestampTzSchema = mustSchema(TimestampTzNode)
+ UUIDSchema = mustSchema(UUIDNode)
)
-func newMapSchema(name string, keySchema, valueSchema avro.Schema, keyFieldID, valueFieldID int) avro.Schema {
- return avro.NewArraySchema(
- Must(avro.NewRecordSchema(name, "", []*avro.Field{
- Must(avro.NewField("key", keySchema, WithFieldID(keyFieldID))),
- Must(avro.NewField("value", valueSchema, WithFieldID(valueFieldID))),
- })), avro.WithProps(map[string]any{"logicalType": "map"}))
+func fieldNode(name string, typ avro.SchemaNode, fieldID int, opts ...func(*avro.SchemaField)) avro.SchemaField {
+ f := avro.SchemaField{
+ Name: name,
+ Type: typ,
+ Props: map[string]any{"field-id": fieldID},
+ }
+ for _, opt := range opts {
+ opt(&f)
+ }
+
+ return f
}
-func WithFieldID(id int) avro.SchemaOption {
- return avro.WithProps(map[string]any{"field-id": id})
+func withDoc(doc string) func(*avro.SchemaField) {
+ return func(f *avro.SchemaField) {
+ f.Doc = doc
+ }
}
-func WithElementID(id int) avro.SchemaOption {
- return avro.WithProps(map[string]any{"element-id": id})
+func withDefault(val any) func(*avro.SchemaField) {
+ return func(f *avro.SchemaField) {
+ f.Default = val
+ f.HasDefault = true
+ }
+}
+
+func newMapNode(name string, keyNode, valueNode avro.SchemaNode, keyFieldID, valueFieldID int) avro.SchemaNode {
+ return avro.SchemaNode{
+ Type: "array",
+ Items: &avro.SchemaNode{
+ Type: "record",
+ Name: name,
+ Fields: []avro.SchemaField{
+ fieldNode("key", keyNode, keyFieldID),
+ fieldNode("value", valueNode, valueFieldID),
+ },
+ },
+ Props: map[string]any{"logicalType": "map"},
+ }
+}
+
+// AvroSchemaMap stores prebuilt schemas by name for lookup.
+var AvroSchemaMap = make(map[string]*avro.Schema)
+
+// fieldSummaryNode is reused in manifest list schemas.
+var fieldSummaryNode = avro.SchemaNode{
+ Type: "record",
+ Name: "r508",
+ Fields: []avro.SchemaField{
+ fieldNode("contains_null", BoolNode, 509,
+ withDoc("true if the field contains null values")),
+ fieldNode("contains_nan", NullableNode(BoolNode), 518,
+ withDoc("true if the field contains NaN values")),
+ fieldNode("lower_bound", NullableNode(BytesNode), 510,
+ withDoc("serialized lower bound")),
+ fieldNode("upper_bound", NullableNode(BytesNode), 511,
+ withDoc("serialized upper bound")),
+ },
}
func init() {
- AvroSchemaCache.Add("field_summary", Must(avro.NewRecordSchema("r508", "", []*avro.Field{
- Must(avro.NewField("contains_null",
- BoolSchema,
- avro.WithDoc("true if the field contains null values"),
- WithFieldID(509))),
- Must(avro.NewField("contains_nan",
- NullableBoolSchema,
- avro.WithDoc("true if the field contains NaN values"),
- WithFieldID(518))),
- Must(avro.NewField("lower_bound", NullableBinarySchema,
- avro.WithDoc("serialized lower bound"),
- WithFieldID(510))),
- Must(avro.NewField("upper_bound", NullableBinarySchema,
- avro.WithDoc("serialized upper bound"),
- WithFieldID(511))),
- }, WithFieldID(508))))
-
- AvroSchemaCache.Add("manifest_list_file_v1", Must(avro.NewRecordSchema("manifest_file", "", []*avro.Field{
- Must(avro.NewField("manifest_path",
- StringSchema,
- avro.WithDoc("Location URI with FS scheme"),
- WithFieldID(500))),
- Must(avro.NewField("manifest_length",
- LongSchema,
- avro.WithDoc("Total file size in bytes"),
- WithFieldID(501))),
- Must(avro.NewField("partition_spec_id",
- IntSchema,
- avro.WithDoc("Spec ID used to write"),
- WithFieldID(502))),
- Must(avro.NewField("added_snapshot_id",
- LongSchema,
- avro.WithDoc("Snapshot ID that added the manifest"),
- WithFieldID(503))),
- Must(avro.NewField("added_files_count",
- NullableIntSchema,
- avro.WithDoc("Added entry count"),
- WithFieldID(504))),
- Must(avro.NewField("existing_files_count",
- NullableIntSchema,
- avro.WithDoc("Existing entry count"),
- WithFieldID(505))),
- Must(avro.NewField("deleted_files_count",
- NullableIntSchema,
- avro.WithDoc("Deleted entry count"),
- WithFieldID(506))),
- Must(avro.NewField("partitions",
- NullableSchema(
- avro.NewArraySchema(AvroSchemaCache.Get("field_summary"),
- WithElementID(508))),
- avro.WithDoc("Partition field summaries"),
- WithFieldID(507))),
- Must(avro.NewField("added_rows_count",
- NullableLongSchema,
- avro.WithDoc("Added row count"),
- WithFieldID(512))),
- Must(avro.NewField("existing_rows_count",
- NullableLongSchema,
- avro.WithDoc("Existing row count"),
- WithFieldID(513))),
- Must(avro.NewField("deleted_rows_count",
- NullableLongSchema,
- avro.WithDoc("Deleted row count"),
- WithFieldID(514))),
- Must(avro.NewField("key_metadata", NullableBinarySchema,
- avro.WithDoc("Key metadata"),
- WithFieldID(519))),
- })))
-
- AvroSchemaCache.Add("manifest_list_file_v2", Must(avro.NewRecordSchema("manifest_file", "", []*avro.Field{
- Must(avro.NewField("manifest_path",
- StringSchema,
- avro.WithDoc("Location URI with FS scheme"),
- WithFieldID(500))),
- Must(avro.NewField("manifest_length",
- LongSchema,
- avro.WithDoc("Total file size in bytes"),
- WithFieldID(501))),
- Must(avro.NewField("partition_spec_id",
- IntSchema,
- avro.WithDoc("Spec ID used to write"),
- WithFieldID(502))),
- Must(avro.NewField("content", IntSchema,
- avro.WithDoc("Content type"),
- avro.WithDefault(0),
- WithFieldID(517))),
- Must(avro.NewField("sequence_number", LongSchema,
- avro.WithDoc("Sequence number"),
- avro.WithDefault(int64(0)),
- WithFieldID(515))),
- Must(avro.NewField("min_sequence_number", LongSchema,
- avro.WithDoc("Minimum sequence number"),
- avro.WithDefault(int64(0)),
- WithFieldID(516))),
- Must(avro.NewField("added_snapshot_id",
- LongSchema,
- avro.WithDoc("Snapshot ID that added the manifest"),
- WithFieldID(503))),
- Must(avro.NewField("added_files_count",
- IntSchema,
- avro.WithDoc("Added entry count"),
- WithFieldID(504))),
- Must(avro.NewField("existing_files_count",
- IntSchema,
- avro.WithDoc("Existing entry count"),
- WithFieldID(505))),
- Must(avro.NewField("deleted_files_count",
- IntSchema,
- avro.WithDoc("Deleted entry count"),
- WithFieldID(506))),
- Must(avro.NewField("partitions",
- NullableSchema(
- avro.NewArraySchema(AvroSchemaCache.Get("field_summary"),
- WithElementID(508))),
- avro.WithDoc("Partition field summaries"),
- WithFieldID(507))),
- Must(avro.NewField("added_rows_count",
- LongSchema,
- avro.WithDoc("Added row count"),
- WithFieldID(512))),
- Must(avro.NewField("existing_rows_count",
- LongSchema,
- avro.WithDoc("Existing row count"),
- WithFieldID(513))),
- Must(avro.NewField("deleted_rows_count",
- LongSchema,
- avro.WithDoc("Deleted row count"),
- WithFieldID(514))),
- Must(avro.NewField("key_metadata", NullableBinarySchema,
- avro.WithDoc("Key metadata"),
- WithFieldID(519))),
- })))
-
- AvroSchemaCache.Add("data_file_v1", Must(avro.NewRecordSchema("r2", "", []*avro.Field{
- Must(avro.NewField("file_path",
- StringSchema,
- avro.WithDoc("Location URI with FS scheme"),
- WithFieldID(100))),
- Must(avro.NewField("file_format",
- StringSchema,
- avro.WithDoc("File format name: avro, orc, parquet"),
- WithFieldID(101))),
- // skip partition field, we'll add that dynamically as needed
- Must(avro.NewField("record_count",
- LongSchema,
- avro.WithDoc("Number of records in the file"),
- WithFieldID(103))),
- Must(avro.NewField("file_size_in_bytes",
- LongSchema,
- avro.WithDoc("Size of the file in bytes"),
- WithFieldID(104))),
- Must(avro.NewField("block_size_in_bytes",
- LongSchema,
- avro.WithDoc("Deprecated. Always write default in v1. Do not write in v2."),
- avro.WithDefault(int64(64*1024*1024)),
- WithFieldID(105))),
- Must(avro.NewField("column_sizes",
- NullableSchema(newMapSchema("k117_v118", IntSchema, LongSchema, 117, 118)),
- avro.WithDoc("map of column id to total size on disk"),
- WithFieldID(108))),
- Must(avro.NewField("value_counts",
- NullableSchema(newMapSchema("k119_v120", IntSchema, LongSchema, 119, 120)),
- avro.WithDoc("map of value to count"),
- WithFieldID(109))),
- Must(avro.NewField("null_value_counts",
- NullableSchema(newMapSchema("k121_v122", IntSchema, LongSchema, 121, 122)),
- avro.WithDoc("map of value to count"),
- WithFieldID(110))),
- Must(avro.NewField("nan_value_counts",
- NullableSchema(newMapSchema("k138_v139", IntSchema, LongSchema, 138, 139)),
- avro.WithDoc("map of value to count"),
- WithFieldID(137))),
- Must(avro.NewField("lower_bounds",
- NullableSchema(newMapSchema("k126_v127", IntSchema, BinarySchema, 126, 127)),
- avro.WithDoc("map of column id to lower bound"),
- WithFieldID(125))),
- Must(avro.NewField("upper_bounds",
- NullableSchema(newMapSchema("k129_v130", IntSchema, BinarySchema, 129, 130)),
- avro.WithDoc("map of column id to upper bound"),
- WithFieldID(128))),
- Must(avro.NewField("key_metadata", NullableBinarySchema,
- avro.WithDoc("Encryption Key Metadata Blob"),
- WithFieldID(131))),
- Must(avro.NewField("split_offsets",
- NullableSchema(avro.NewArraySchema(LongSchema,
- WithElementID(133))),
- avro.WithDoc("splitable offsets"),
- WithFieldID(132))),
- Must(avro.NewField("sort_order_id",
- NullableIntSchema,
- avro.WithDoc("Sort order ID"),
- WithFieldID(140))),
- })))
-
- AvroSchemaCache.Add("data_file_v2", Must(avro.NewRecordSchema("r2", "", []*avro.Field{
- Must(avro.NewField("content", IntSchema,
- avro.WithDoc("Content type"),
- avro.WithDefault(0),
- WithFieldID(134))),
- Must(avro.NewField("file_path",
- StringSchema,
- avro.WithDoc("Location URI with FS scheme"),
- WithFieldID(100))),
- Must(avro.NewField("file_format",
- StringSchema,
- avro.WithDoc("File format name: avro, orc, parquet"),
- WithFieldID(101))),
- // skip partition field, we'll add that dynamically as needed
- Must(avro.NewField("record_count",
- LongSchema,
- avro.WithDoc("Number of records in the file"),
- WithFieldID(103))),
- Must(avro.NewField("file_size_in_bytes",
- LongSchema,
- avro.WithDoc("Size of the file in bytes"),
- WithFieldID(104))),
- Must(avro.NewField("column_sizes",
- NullableSchema(newMapSchema("k117_v118", IntSchema, LongSchema, 117, 118)),
- avro.WithDoc("map of column id to total size on disk"),
- WithFieldID(108))),
- Must(avro.NewField("value_counts",
- NullableSchema(newMapSchema("k119_v120", IntSchema, LongSchema, 119, 120)),
- avro.WithDoc("map of value to count"),
- WithFieldID(109))),
- Must(avro.NewField("null_value_counts",
- NullableSchema(newMapSchema("k121_v122", IntSchema, LongSchema, 121, 122)),
- avro.WithDoc("map of value to count"),
- WithFieldID(110))),
- Must(avro.NewField("nan_value_counts",
- NullableSchema(newMapSchema("k138_v139", IntSchema, LongSchema, 138, 139)),
- avro.WithDoc("map of value to count"),
- WithFieldID(137))),
- Must(avro.NewField("lower_bounds",
- NullableSchema(newMapSchema("k126_v127", IntSchema, BinarySchema, 126, 127)),
- avro.WithDoc("map of column id to lower bound"),
- WithFieldID(125))),
- Must(avro.NewField("upper_bounds",
- NullableSchema(newMapSchema("k129_v130", IntSchema, BinarySchema, 129, 130)),
- avro.WithDoc("map of column id to upper bound"),
- WithFieldID(128))),
- Must(avro.NewField("key_metadata", NullableBinarySchema,
- avro.WithDoc("Encryption Key Metadata Blob"),
- WithFieldID(131))),
- Must(avro.NewField("split_offsets",
- NullableSchema(avro.NewArraySchema(LongSchema,
- WithElementID(133))),
- avro.WithDoc("splitable offsets"),
- WithFieldID(132))),
- Must(avro.NewField("equality_ids",
- NullableSchema(avro.NewArraySchema(IntSchema,
- WithElementID(136))),
- avro.WithDoc("field ids used to determine row equality in equality delete files"),
- WithFieldID(135))),
- Must(avro.NewField("sort_order_id",
- NullableIntSchema,
- avro.WithDoc("Sort order ID"),
- WithFieldID(140))),
- })))
-
- AvroSchemaCache.Add("manifest_entry_v1", Must(avro.NewRecordSchema("manifest_entry", "", []*avro.Field{
- Must(avro.NewField("status", IntSchema, WithFieldID(0))),
- Must(avro.NewField("snapshot_id", LongSchema, WithFieldID(1))),
- // leave data_file for dyanmic generation
- })))
-
- AvroSchemaCache.Add("manifest_entry_v2", Must(avro.NewRecordSchema("manifest_entry", "", []*avro.Field{
- Must(avro.NewField("status", IntSchema, WithFieldID(0))),
- Must(avro.NewField("snapshot_id", NullableLongSchema, WithFieldID(1))),
- Must(avro.NewField("sequence_number", NullableLongSchema, WithFieldID(3))),
- Must(avro.NewField("file_sequence_number", NullableLongSchema, WithFieldID(4))),
- // leave data_file for dynamic generation
- })))
-
- AvroSchemaCache.Add("manifest_list_file_v3", Must(avro.NewRecordSchema("manifest_file", "", []*avro.Field{
- Must(avro.NewField("manifest_path",
- StringSchema,
- avro.WithDoc("Location URI with FS scheme"),
- WithFieldID(500))),
- Must(avro.NewField("manifest_length",
- LongSchema,
- avro.WithDoc("Total file size in bytes"),
- WithFieldID(501))),
- Must(avro.NewField("partition_spec_id",
- IntSchema,
- avro.WithDoc("Spec ID used to write"),
- WithFieldID(502))),
- Must(avro.NewField("content", IntSchema,
- avro.WithDoc("Content type"),
- avro.WithDefault(0),
- WithFieldID(517))),
- Must(avro.NewField("sequence_number", LongSchema,
- avro.WithDoc("Sequence number"),
- avro.WithDefault(int64(0)),
- WithFieldID(515))),
- Must(avro.NewField("min_sequence_number", LongSchema,
- avro.WithDoc("Minimum sequence number"),
- avro.WithDefault(int64(0)),
- WithFieldID(516))),
- Must(avro.NewField("added_snapshot_id",
- LongSchema,
- avro.WithDoc("Snapshot ID that added the manifest"),
- WithFieldID(503))),
- Must(avro.NewField("added_files_count",
- IntSchema,
- avro.WithDoc("Added entry count"),
- WithFieldID(504))),
- Must(avro.NewField("existing_files_count",
- IntSchema,
- avro.WithDoc("Existing entry count"),
- WithFieldID(505))),
- Must(avro.NewField("deleted_files_count",
- IntSchema,
- avro.WithDoc("Deleted entry count"),
- WithFieldID(506))),
- Must(avro.NewField("partitions",
- NullableSchema(
- avro.NewArraySchema(AvroSchemaCache.Get("field_summary"),
- WithElementID(508))),
- avro.WithDoc("Partition field summaries"),
- WithFieldID(507))),
- Must(avro.NewField("added_rows_count",
- LongSchema,
- avro.WithDoc("Added row count"),
- WithFieldID(512))),
- Must(avro.NewField("existing_rows_count",
- LongSchema,
- avro.WithDoc("Existing row count"),
- WithFieldID(513))),
- Must(avro.NewField("deleted_rows_count",
- LongSchema,
- avro.WithDoc("Deleted row count"),
- WithFieldID(514))),
- Must(avro.NewField("key_metadata", NullableBinarySchema,
- avro.WithDoc("Key metadata"),
- WithFieldID(519))),
- Must(avro.NewField("first_row_id", NullableLongSchema,
- avro.WithDoc("First row ID"),
- WithFieldID(520))),
- })))
- AvroSchemaCache.Add("data_file_v3", Must(avro.NewRecordSchema("r2", "", []*avro.Field{
- Must(avro.NewField("content", IntSchema,
- avro.WithDoc("Content type"),
- avro.WithDefault(0),
- WithFieldID(134))),
- Must(avro.NewField("file_path",
- StringSchema,
- avro.WithDoc("Location URI with FS scheme"),
- WithFieldID(100))),
- Must(avro.NewField("file_format",
- StringSchema,
- avro.WithDoc("File format name: avro, orc, parquet"),
- WithFieldID(101))),
- // skip partition field, we'll add that dynamically as needed
- Must(avro.NewField("record_count",
- LongSchema,
- avro.WithDoc("Number of records in the file"),
- WithFieldID(103))),
- Must(avro.NewField("file_size_in_bytes",
- LongSchema,
- avro.WithDoc("Size of the file in bytes"),
- WithFieldID(104))),
- Must(avro.NewField("column_sizes",
- NullableSchema(newMapSchema("k117_v118", IntSchema, LongSchema, 117, 118)),
- avro.WithDoc("map of column id to total size on disk"),
- WithFieldID(108))),
- Must(avro.NewField("value_counts",
- NullableSchema(newMapSchema("k119_v120", IntSchema, LongSchema, 119, 120)),
- avro.WithDoc("map of value to count"),
- WithFieldID(109))),
- Must(avro.NewField("null_value_counts",
- NullableSchema(newMapSchema("k121_v122", IntSchema, LongSchema, 121, 122)),
- avro.WithDoc("map of value to count"),
- WithFieldID(110))),
- Must(avro.NewField("nan_value_counts",
- NullableSchema(newMapSchema("k138_v139", IntSchema, LongSchema, 138, 139)),
- avro.WithDoc("map of value to count"),
- WithFieldID(137))),
- Must(avro.NewField("lower_bounds",
- NullableSchema(newMapSchema("k126_v127", IntSchema, BinarySchema, 126, 127)),
- avro.WithDoc("map of column id to lower bound"),
- WithFieldID(125))),
- Must(avro.NewField("upper_bounds",
- NullableSchema(newMapSchema("k129_v130", IntSchema, BinarySchema, 129, 130)),
- avro.WithDoc("map of column id to upper bound"),
- WithFieldID(128))),
- Must(avro.NewField("key_metadata", NullableBinarySchema,
- avro.WithDoc("Encryption Key Metadata Blob"),
- WithFieldID(131))),
- Must(avro.NewField("split_offsets",
- NullableSchema(avro.NewArraySchema(LongSchema,
- WithElementID(133))),
- avro.WithDoc("splitable offsets"),
- WithFieldID(132))),
- Must(avro.NewField("equality_ids",
- NullableSchema(avro.NewArraySchema(IntSchema,
- WithElementID(136))),
- avro.WithDoc("field ids used to determine row equality in equality delete files"),
- WithFieldID(135))),
- Must(avro.NewField("sort_order_id",
- NullableIntSchema,
- avro.WithDoc("Sort order ID"),
- WithFieldID(140))),
- Must(avro.NewField("first_row_id",
- NullableLongSchema,
- avro.WithDoc("The _row_id for the first row in the data file"),
- WithFieldID(142))),
- Must(avro.NewField("referenced_data_file",
- NullableSchema(StringSchema),
- avro.WithDoc("Fully qualified location of a data file that all deletes reference"),
- WithFieldID(143))),
- Must(avro.NewField("content_offset",
- NullableLongSchema,
- avro.WithDoc("The offset in the file where the content starts"),
- WithFieldID(144))),
- Must(avro.NewField("content_size_in_bytes",
- NullableLongSchema,
- avro.WithDoc("The length of the referenced content stored in the file"),
- WithFieldID(145))),
- })))
- AvroSchemaCache.Add("manifest_entry_v3", Must(avro.NewRecordSchema("manifest_entry", "", []*avro.Field{
- Must(avro.NewField("status", IntSchema, WithFieldID(0))),
- Must(avro.NewField("snapshot_id", NullableLongSchema, WithFieldID(1))),
- Must(avro.NewField("sequence_number", NullableLongSchema, WithFieldID(3))),
- Must(avro.NewField("file_sequence_number", NullableLongSchema, WithFieldID(4))),
- // leave data_file for dynamic generation
- })))
+ AvroSchemaMap["field_summary"] = mustSchema(fieldSummaryNode)
+
+ partitionsNode := NullableNode(avro.SchemaNode{
+ Type: "array",
+ Items: &fieldSummaryNode,
+ Props: map[string]any{"element-id": 508},
+ })
+
+ AvroSchemaMap["manifest_list_file_v1"] = mustSchema(avro.SchemaNode{
+ Type: "record",
+ Name: "manifest_file",
+ Fields: []avro.SchemaField{
+ fieldNode("manifest_path", StringNode, 500, withDoc("Location URI with FS scheme")),
+ fieldNode("manifest_length", LongNode, 501, withDoc("Total file size in bytes")),
+ fieldNode("partition_spec_id", IntNode, 502, withDoc("Spec ID used to write")),
+ fieldNode("added_snapshot_id", LongNode, 503, withDoc("Snapshot ID that added the manifest")),
+ fieldNode("added_files_count", NullableNode(IntNode), 504, withDoc("Added entry count")),
+ fieldNode("existing_files_count", NullableNode(IntNode), 505, withDoc("Existing entry count")),
+ fieldNode("deleted_files_count", NullableNode(IntNode), 506, withDoc("Deleted entry count")),
+ fieldNode("partitions", partitionsNode, 507, withDoc("Partition field summaries")),
+ fieldNode("added_rows_count", NullableNode(LongNode), 512, withDoc("Added row count")),
+ fieldNode("existing_rows_count", NullableNode(LongNode), 513, withDoc("Existing row count")),
+ fieldNode("deleted_rows_count", NullableNode(LongNode), 514, withDoc("Deleted row count")),
+ fieldNode("key_metadata", NullableNode(BytesNode), 519, withDoc("Key metadata")),
+ },
+ })
+
+ AvroSchemaMap["manifest_list_file_v2"] = mustSchema(avro.SchemaNode{
+ Type: "record",
+ Name: "manifest_file",
+ Fields: []avro.SchemaField{
+ fieldNode("manifest_path", StringNode, 500, withDoc("Location URI with FS scheme")),
+ fieldNode("manifest_length", LongNode, 501, withDoc("Total file size in bytes")),
+ fieldNode("partition_spec_id", IntNode, 502, withDoc("Spec ID used to write")),
+ fieldNode("content", IntNode, 517, withDoc("Content type"), withDefault(0)),
+ fieldNode("sequence_number", LongNode, 515, withDoc("Sequence number"), withDefault(int64(0))),
+ fieldNode("min_sequence_number", LongNode, 516, withDoc("Minimum sequence number"), withDefault(int64(0))),
+ fieldNode("added_snapshot_id", LongNode, 503, withDoc("Snapshot ID that added the manifest")),
+ fieldNode("added_files_count", IntNode, 504, withDoc("Added entry count")),
+ fieldNode("existing_files_count", IntNode, 505, withDoc("Existing entry count")),
+ fieldNode("deleted_files_count", IntNode, 506, withDoc("Deleted entry count")),
+ fieldNode("partitions", partitionsNode, 507, withDoc("Partition field summaries")),
+ fieldNode("added_rows_count", LongNode, 512, withDoc("Added row count")),
+ fieldNode("existing_rows_count", LongNode, 513, withDoc("Existing row count")),
+ fieldNode("deleted_rows_count", LongNode, 514, withDoc("Deleted row count")),
+ fieldNode("key_metadata", NullableNode(BytesNode), 519, withDoc("Key metadata")),
+ },
+ })
+
+ AvroSchemaMap["data_file_v1"] = mustSchema(avro.SchemaNode{
+ Type: "record",
+ Name: "r2",
+ Fields: []avro.SchemaField{
+ fieldNode("file_path", StringNode, 100, withDoc("Location URI with FS scheme")),
+ fieldNode("file_format", StringNode, 101, withDoc("File format name: avro, orc, parquet")),
+ // skip partition field, we'll add that dynamically as needed
+ fieldNode("record_count", LongNode, 103, withDoc("Number of records in the file")),
+ fieldNode("file_size_in_bytes", LongNode, 104, withDoc("Size of the file in bytes")),
+ fieldNode("block_size_in_bytes", LongNode, 105,
+ withDoc("Deprecated. Always write default in v1. Do not write in v2."),
+ withDefault(int64(64*1024*1024))),
+ fieldNode("column_sizes",
+ NullableNode(newMapNode("k117_v118", IntNode, LongNode, 117, 118)),
+ 108, withDoc("map of column id to total size on disk")),
+ fieldNode("value_counts",
+ NullableNode(newMapNode("k119_v120", IntNode, LongNode, 119, 120)),
+ 109, withDoc("map of value to count")),
+ fieldNode("null_value_counts",
+ NullableNode(newMapNode("k121_v122", IntNode, LongNode, 121, 122)),
+ 110, withDoc("map of value to count")),
+ fieldNode("nan_value_counts",
+ NullableNode(newMapNode("k138_v139", IntNode, LongNode, 138, 139)),
+ 137, withDoc("map of value to count")),
+ fieldNode("lower_bounds",
+ NullableNode(newMapNode("k126_v127", IntNode, BytesNode, 126, 127)),
+ 125, withDoc("map of column id to lower bound")),
+ fieldNode("upper_bounds",
+ NullableNode(newMapNode("k129_v130", IntNode, BytesNode, 129, 130)),
+ 128, withDoc("map of column id to upper bound")),
+ fieldNode("key_metadata", NullableNode(BytesNode), 131, withDoc("Encryption Key Metadata Blob")),
+ fieldNode("split_offsets",
+ NullableNode(avro.SchemaNode{Type: "array", Items: &LongNode, Props: map[string]any{"element-id": 133}}),
+ 132, withDoc("splitable offsets")),
+ fieldNode("sort_order_id", NullableNode(IntNode), 140, withDoc("Sort order ID")),
+ },
+ })
+
+ AvroSchemaMap["data_file_v2"] = mustSchema(avro.SchemaNode{
+ Type: "record",
+ Name: "r2",
+ Fields: []avro.SchemaField{
+ fieldNode("content", IntNode, 134, withDoc("Content type"), withDefault(0)),
+ fieldNode("file_path", StringNode, 100, withDoc("Location URI with FS scheme")),
+ fieldNode("file_format", StringNode, 101, withDoc("File format name: avro, orc, parquet")),
+ // skip partition field, we'll add that dynamically as needed
+ fieldNode("record_count", LongNode, 103, withDoc("Number of records in the file")),
+ fieldNode("file_size_in_bytes", LongNode, 104, withDoc("Size of the file in bytes")),
+ fieldNode("column_sizes",
+ NullableNode(newMapNode("k117_v118", IntNode, LongNode, 117, 118)),
+ 108, withDoc("map of column id to total size on disk")),
+ fieldNode("value_counts",
+ NullableNode(newMapNode("k119_v120", IntNode, LongNode, 119, 120)),
+ 109, withDoc("map of value to count")),
+ fieldNode("null_value_counts",
+ NullableNode(newMapNode("k121_v122", IntNode, LongNode, 121, 122)),
+ 110, withDoc("map of value to count")),
+ fieldNode("nan_value_counts",
+ NullableNode(newMapNode("k138_v139", IntNode, LongNode, 138, 139)),
+ 137, withDoc("map of value to count")),
+ fieldNode("lower_bounds",
+ NullableNode(newMapNode("k126_v127", IntNode, BytesNode, 126, 127)),
+ 125, withDoc("map of column id to lower bound")),
+ fieldNode("upper_bounds",
+ NullableNode(newMapNode("k129_v130", IntNode, BytesNode, 129, 130)),
+ 128, withDoc("map of column id to upper bound")),
+ fieldNode("key_metadata", NullableNode(BytesNode), 131, withDoc("Encryption Key Metadata Blob")),
+ fieldNode("split_offsets",
+ NullableNode(avro.SchemaNode{Type: "array", Items: &LongNode, Props: map[string]any{"element-id": 133}}),
+ 132, withDoc("splitable offsets")),
+ fieldNode("equality_ids",
+ NullableNode(avro.SchemaNode{Type: "array", Items: &IntNode, Props: map[string]any{"element-id": 136}}),
+ 135, withDoc("field ids used to determine row equality in equality delete files")),
+ fieldNode("sort_order_id", NullableNode(IntNode), 140, withDoc("Sort order ID")),
+ },
+ })
+
+ AvroSchemaMap["manifest_entry_v1"] = mustSchema(avro.SchemaNode{
+ Type: "record",
+ Name: "manifest_entry",
+ Fields: []avro.SchemaField{
+ fieldNode("status", IntNode, 0),
+ fieldNode("snapshot_id", LongNode, 1),
+ // leave data_file for dynamic generation
+ },
+ })
+
+ AvroSchemaMap["manifest_entry_v2"] = mustSchema(avro.SchemaNode{
+ Type: "record",
+ Name: "manifest_entry",
+ Fields: []avro.SchemaField{
+ fieldNode("status", IntNode, 0),
+ fieldNode("snapshot_id", NullableNode(LongNode), 1),
+ fieldNode("sequence_number", NullableNode(LongNode), 3),
+ fieldNode("file_sequence_number", NullableNode(LongNode), 4),
+ // leave data_file for dynamic generation
+ },
+ })
+
+ AvroSchemaMap["manifest_list_file_v3"] = mustSchema(avro.SchemaNode{
+ Type: "record",
+ Name: "manifest_file",
+ Fields: []avro.SchemaField{
+ fieldNode("manifest_path", StringNode, 500, withDoc("Location URI with FS scheme")),
+ fieldNode("manifest_length", LongNode, 501, withDoc("Total file size in bytes")),
+ fieldNode("partition_spec_id", IntNode, 502, withDoc("Spec ID used to write")),
+ fieldNode("content", IntNode, 517, withDoc("Content type"), withDefault(0)),
+ fieldNode("sequence_number", LongNode, 515, withDoc("Sequence number"), withDefault(int64(0))),
+ fieldNode("min_sequence_number", LongNode, 516, withDoc("Minimum sequence number"), withDefault(int64(0))),
+ fieldNode("added_snapshot_id", LongNode, 503, withDoc("Snapshot ID that added the manifest")),
+ fieldNode("added_files_count", IntNode, 504, withDoc("Added entry count")),
+ fieldNode("existing_files_count", IntNode, 505, withDoc("Existing entry count")),
+ fieldNode("deleted_files_count", IntNode, 506, withDoc("Deleted entry count")),
+ fieldNode("partitions", partitionsNode, 507, withDoc("Partition field summaries")),
+ fieldNode("added_rows_count", LongNode, 512, withDoc("Added row count")),
+ fieldNode("existing_rows_count", LongNode, 513, withDoc("Existing row count")),
+ fieldNode("deleted_rows_count", LongNode, 514, withDoc("Deleted row count")),
+ fieldNode("key_metadata", NullableNode(BytesNode), 519, withDoc("Key metadata")),
+ fieldNode("first_row_id", NullableNode(LongNode), 520, withDoc("First row ID")),
+ },
+ })
+
+ AvroSchemaMap["data_file_v3"] = mustSchema(avro.SchemaNode{
+ Type: "record",
+ Name: "r2",
+ Fields: []avro.SchemaField{
+ fieldNode("content", IntNode, 134, withDoc("Content type"), withDefault(0)),
+ fieldNode("file_path", StringNode, 100, withDoc("Location URI with FS scheme")),
+ fieldNode("file_format", StringNode, 101, withDoc("File format name: avro, orc, parquet")),
+ // skip partition field, we'll add that dynamically as needed
+ fieldNode("record_count", LongNode, 103, withDoc("Number of records in the file")),
+ fieldNode("file_size_in_bytes", LongNode, 104, withDoc("Size of the file in bytes")),
+ fieldNode("column_sizes",
+ NullableNode(newMapNode("k117_v118", IntNode, LongNode, 117, 118)),
+ 108, withDoc("map of column id to total size on disk")),
+ fieldNode("value_counts",
+ NullableNode(newMapNode("k119_v120", IntNode, LongNode, 119, 120)),
+ 109, withDoc("map of value to count")),
+ fieldNode("null_value_counts",
+ NullableNode(newMapNode("k121_v122", IntNode, LongNode, 121, 122)),
+ 110, withDoc("map of value to count")),
+ fieldNode("nan_value_counts",
+ NullableNode(newMapNode("k138_v139", IntNode, LongNode, 138, 139)),
+ 137, withDoc("map of value to count")),
+ fieldNode("lower_bounds",
+ NullableNode(newMapNode("k126_v127", IntNode, BytesNode, 126, 127)),
+ 125, withDoc("map of column id to lower bound")),
+ fieldNode("upper_bounds",
+ NullableNode(newMapNode("k129_v130", IntNode, BytesNode, 129, 130)),
+ 128, withDoc("map of column id to upper bound")),
+ fieldNode("key_metadata", NullableNode(BytesNode), 131, withDoc("Encryption Key Metadata Blob")),
+ fieldNode("split_offsets",
+ NullableNode(avro.SchemaNode{Type: "array", Items: &LongNode, Props: map[string]any{"element-id": 133}}),
+ 132, withDoc("splitable offsets")),
+ fieldNode("equality_ids",
+ NullableNode(avro.SchemaNode{Type: "array", Items: &IntNode, Props: map[string]any{"element-id": 136}}),
+ 135, withDoc("field ids used to determine row equality in equality delete files")),
+ fieldNode("sort_order_id", NullableNode(IntNode), 140, withDoc("Sort order ID")),
+ fieldNode("first_row_id", NullableNode(LongNode), 142, withDoc("The _row_id for the first row in the data file")),
+ fieldNode("referenced_data_file", NullableNode(StringNode), 143,
+ withDoc("Fully qualified location of a data file that all deletes reference")),
+ fieldNode("content_offset", NullableNode(LongNode), 144,
+ withDoc("The offset in the file where the content starts")),
+ fieldNode("content_size_in_bytes", NullableNode(LongNode), 145,
+ withDoc("The length of the referenced content stored in the file")),
+ },
+ })
+
+ AvroSchemaMap["manifest_entry_v3"] = mustSchema(avro.SchemaNode{
+ Type: "record",
+ Name: "manifest_entry",
+ Fields: []avro.SchemaField{
+ fieldNode("status", IntNode, 0),
+ fieldNode("snapshot_id", NullableNode(LongNode), 1),
+ fieldNode("sequence_number", NullableNode(LongNode), 3),
+ fieldNode("file_sequence_number", NullableNode(LongNode), 4),
+ // leave data_file for dynamic generation
+ },
+ })
}
-func newDataFileSchema(partitionType avro.Schema, version int) (avro.Schema, error) {
+func newDataFileSchema(partitionType *avro.Schema, version int) (*avro.Schema, error) {
key := fmt.Sprintf("data_file_v%d", version)
- schema := AvroSchemaCache.Get(key)
+ schema := AvroSchemaMap[key]
- partField, err := avro.NewField("partition",
- partitionType, WithFieldID(102))
- if err != nil {
- return nil, err
- }
+ node := schema.Root()
+ node.Fields = append(slices.Clone(node.Fields), avro.SchemaField{
+ Name: "partition",
+ Type: partitionType.Root(),
+ Props: map[string]any{"field-id": 102},
+ })
- return avro.NewRecordSchema("r2", "",
- append(schema.(*avro.RecordSchema).Fields(), partField))
+ return node.Schema()
}
-func NewManifestFileSchema(version int) (avro.Schema, error) {
+func NewManifestFileSchema(version int) (*avro.Schema, error) {
switch version {
case 1, 2, 3:
default:
@@ -547,10 +439,10 @@ func NewManifestFileSchema(version int) (avro.Schema, error) {
key := fmt.Sprintf("manifest_list_file_v%d", version)
- return AvroSchemaCache.Get(key), nil
+ return AvroSchemaMap[key], nil
}
-func NewManifestEntrySchema(partitionType avro.Schema, version int) (avro.Schema, error) {
+func NewManifestEntrySchema(partitionType *avro.Schema, version int) (*avro.Schema, error) {
switch version {
case 1, 2, 3:
default:
@@ -562,14 +454,20 @@ func NewManifestEntrySchema(partitionType avro.Schema, version int) (avro.Schema
return nil, err
}
- dfField, err := avro.NewField("data_file", dfschema, WithFieldID(2))
- if err != nil {
- return nil, err
- }
-
key := fmt.Sprintf("manifest_entry_v%d", version)
- schema := AvroSchemaCache.Get(key)
+ schema := AvroSchemaMap[key]
+
+ node := schema.Root()
+ node.Fields = append(slices.Clone(node.Fields), avro.SchemaField{
+ Name: "data_file",
+ Type: dfschema.Root(),
+ Props: map[string]any{"field-id": 2},
+ })
+
+ return node.Schema()
+}
- return avro.NewRecordSchema("manifest_entry", "",
- append(schema.(*avro.RecordSchema).Fields(), dfField))
+// WithFieldID returns a props map for use with SchemaField for Iceberg field IDs.
+func WithFieldID(id int) map[string]any {
+ return map[string]any{"field-id": id}
}
diff --git a/manifest.go b/manifest.go
index e2109a37..6c0775ba 100644
--- a/manifest.go
+++ b/manifest.go
@@ -35,9 +35,9 @@ import (
"github.com/apache/iceberg-go/internal"
iceio "github.com/apache/iceberg-go/io"
"github.com/google/uuid"
-
- "github.com/hamba/avro/v2"
- "github.com/hamba/avro/v2/ocf"
+ "github.com/twmb/avro"
+ "github.com/twmb/avro/atype"
+ "github.com/twmb/avro/ocf"
)
// ManifestContent indicates the type of data inside of the files
@@ -407,11 +407,12 @@ func (m *manifestFile) FetchEntries(fs iceio.IO, discardDeleted bool) ([]Manifes
return fetchManifestEntries(m, fs, discardDeleted)
}
-func getFieldIDMap(sc avro.Schema) (map[string]int, map[int]avro.LogicalType, map[int]int) {
- getField := func(rs *avro.RecordSchema, name string) *avro.Field {
- for _, f := range rs.Fields() {
- if f.Name() == name {
- return f
+func getFieldIDMap(sc *avro.Schema) (map[string]int, map[int]string, map[int]int) {
+ root := sc.Root()
+ getField := func(node avro.SchemaNode, name string) *avro.SchemaField {
+ for i := range node.Fields {
+ if node.Fields[i].Name == name {
+ return &node.Fields[i]
}
}
@@ -419,15 +420,15 @@ func getFieldIDMap(sc avro.Schema) (map[string]int, map[int]avro.LogicalType, ma
}
result := make(map[string]int)
- logicalTypes := make(map[int]avro.LogicalType)
+ logicalTypes := make(map[int]string)
fixedSizes := make(map[int]int)
- entryField := getField(sc.(*avro.RecordSchema), "data_file")
- partitionField := getField(entryField.Type().(*avro.RecordSchema), "partition")
+ entryField := getField(root, "data_file")
+ partitionField := getField(entryField.Type, "partition")
- for _, field := range partitionField.Type().(*avro.RecordSchema).Fields() {
+ for _, field := range partitionField.Type.Fields {
var fid int
- switch v := field.Prop("field-id").(type) {
+ switch v := field.Props["field-id"].(type) {
case int:
fid = v
case float64:
@@ -436,22 +437,15 @@ func getFieldIDMap(sc avro.Schema) (map[string]int, map[int]avro.LogicalType, ma
continue
}
- result[field.Name()] = fid
- avroTyp := field.Type()
- if us, ok := avroTyp.(*avro.UnionSchema); ok {
- typeList := us.Types()
- avroTyp = typeList[len(typeList)-1]
+ result[field.Name] = fid
+ typ := field.Type
+ if typ.Type == atype.Union {
+ typ = typ.Branches[len(typ.Branches)-1]
}
- if ps, ok := avroTyp.(*avro.PrimitiveSchema); ok && ps.Logical() != nil {
- logicalTypes[fid] = ps.Logical().Type()
- } else if fs, ok := avroTyp.(*avro.FixedSchema); ok {
- if fs.Logical() != nil {
- logicalTypes[int(fid)] = fs.Logical().Type()
- if decimalLogical, ok := fs.Logical().(*avro.DecimalLogicalSchema); ok {
- fixedSizes[int(fid)] = decimalLogical.Scale()
- }
- } else {
- fixedSizes[int(fid)] = fs.Size()
+ if typ.LogicalType != "" {
+ logicalTypes[fid] = typ.LogicalType
+ if typ.LogicalType == atype.Decimal {
+ fixedSizes[fid] = typ.Scale
}
}
}
@@ -461,7 +455,7 @@ func getFieldIDMap(sc avro.Schema) (map[string]int, map[int]avro.LogicalType, ma
type hasFieldToIDMap interface {
setFieldNameToIDMap(map[string]int)
- setFieldIDToLogicalTypeMap(map[int]avro.LogicalType)
+ setFieldIDToLogicalTypeMap(map[int]string)
setFieldIDToFixedSizeMap(map[int]int)
}
@@ -549,49 +543,53 @@ type fallbackManifest[T any] interface {
*T
}
-func decodeManifestsWithFallback[P fallbackManifest[T], T any](dec *ocf.Decoder) ([]ManifestFile, error) {
+func decodeManifestsWithFallback[P fallbackManifest[T], T any](rd *ocf.Reader) ([]ManifestFile, error) {
results := make([]ManifestFile, 0)
- for dec.HasNext() {
+ for {
tmp := P(new(T))
- if err := dec.Decode(tmp); err != nil {
+ if err := rd.Decode(tmp); err != nil {
+ if errors.Is(err, io.EOF) {
+ return results, nil
+ }
+
return nil, err
}
results = append(results, tmp.toFile())
}
-
- return results, dec.Error()
}
func decodeManifests[I interface {
ManifestFile
*T
-}, T any](dec *ocf.Decoder, version int) ([]ManifestFile, error) {
+}, T any](rd *ocf.Reader, version int) ([]ManifestFile, error) {
results := make([]ManifestFile, 0)
- for dec.HasNext() {
+ for {
tmp := I(new(T))
- if err := dec.Decode(tmp); err != nil {
+ if err := rd.Decode(tmp); err != nil {
+ if errors.Is(err, io.EOF) {
+ return results, nil
+ }
+
return nil, err
}
tmp.setVersion(version)
results = append(results, tmp)
}
-
- return results, dec.Error()
}
// ManifestReader reads the metadata and data from an avro manifest file.
// This type is not thread-safe; its methods should not be called from
// multiple goroutines.
type ManifestReader struct {
- dec *ocf.Decoder
+ rd *ocf.Reader
file ManifestFile
formatVersion int
isFallback bool
content ManifestContent
fieldNameToID map[string]int
- fieldIDToType map[int]avro.LogicalType
+ fieldIDToType map[int]string
fieldIDToSize map[int]int
// The rest are lazily populated, on demand. Most readers
@@ -606,16 +604,16 @@ type ManifestReader struct {
// file. If the caller is interested in the manifest entries in the file, it must call
// [ManifestReader.Entries] before closing the provided reader.
func NewManifestReader(file ManifestFile, in io.Reader) (*ManifestReader, error) {
- dec, err := ocf.NewDecoder(in, ocf.WithDecoderSchemaCache(&avro.SchemaCache{}))
+ rd, err := ocf.NewReader(in)
if err != nil {
return nil, err
}
- metadata := dec.Metadata()
- sc := dec.Schema()
+ metadata := rd.Metadata()
+ sc := rd.Schema()
formatVersion := 1
- // format-version is optional for v1 manifest files, so default to v1
+ // format-version is optional for v1 manifest files, so default to v1.
if raw := metadata["format-version"]; len(raw) > 0 {
formatVersion, err = strconv.Atoi(string(raw))
if err != nil {
@@ -651,9 +649,10 @@ func NewManifestReader(file ManifestFile, in io.Reader) (*ManifestReader, error)
isFallback := false
if formatVersion == 1 {
- for _, f := range sc.(*avro.RecordSchema).Fields() {
- if f.Name() == "snapshot_id" {
- if f.Type().Type() != avro.Union {
+ root := sc.Root()
+ for _, f := range root.Fields {
+ if f.Name == "snapshot_id" {
+ if f.Type.Type != atype.Union {
isFallback = true
}
@@ -664,7 +663,7 @@ func NewManifestReader(file ManifestFile, in io.Reader) (*ManifestReader, error)
fieldNameToID, fieldIDToType, fieldIDToSize := getFieldIDMap(sc)
return &ManifestReader{
- dec: dec,
+ rd: rd,
file: file,
formatVersion: formatVersion,
isFallback: isFallback,
@@ -677,7 +676,7 @@ func NewManifestReader(file ManifestFile, in io.Reader) (*ManifestReader, error)
// Close releases decoder resources associated with this manifest reader.
func (c *ManifestReader) Close() error {
- return c.dec.Close()
+ return c.rd.Close()
}
// Version returns the file's format version.
@@ -692,7 +691,7 @@ func (c *ManifestReader) ManifestContent() ManifestContent {
// SchemaID returns the schema ID encoded in the avro file's metadata.
func (c *ManifestReader) SchemaID() (int, error) {
- id, err := strconv.Atoi(string(c.dec.Metadata()["schema-id"]))
+ id, err := strconv.Atoi(string(c.rd.Metadata()["schema-id"]))
if err != nil {
return 0, fmt.Errorf("manifest file's 'schema-id' metadata is invalid: %w", err)
}
@@ -707,7 +706,7 @@ func (c *ManifestReader) Schema() (*Schema, error) {
if err != nil {
return nil, err
}
- if err := json.Unmarshal(c.dec.Metadata()["schema"], &c.schema); err != nil {
+ if err := json.Unmarshal(c.rd.Metadata()["schema"], &c.schema); err != nil {
return nil, fmt.Errorf("manifest file's 'schema' metadata is invalid: %w", err)
}
c.schema.ID = schemaID
@@ -719,7 +718,7 @@ func (c *ManifestReader) Schema() (*Schema, error) {
// PartitionSpecID returns the partition spec ID encoded in the avro file's metadata.
func (c *ManifestReader) PartitionSpecID() (int, error) {
- id, err := strconv.Atoi(string(c.dec.Metadata()["partition-spec-id"]))
+ id, err := strconv.Atoi(string(c.rd.Metadata()["partition-spec-id"]))
if err != nil {
return 0, fmt.Errorf("manifest file's 'partition-spec-id' metadata is invalid: %w", err)
}
@@ -738,7 +737,7 @@ func (c *ManifestReader) PartitionSpec() (*PartitionSpec, error) {
if err != nil {
return nil, err
}
- if err := json.Unmarshal(c.dec.Metadata()["partition-spec"], &c.partitionSpec.fields); err != nil {
+ if err := json.Unmarshal(c.rd.Metadata()["partition-spec"], &c.partitionSpec.fields); err != nil {
return nil, fmt.Errorf("manifest file's 'partition-spec' metadata is invalid: %w", err)
}
c.partitionSpec.id = partitionSpecID
@@ -751,12 +750,6 @@ func (c *ManifestReader) PartitionSpec() (*PartitionSpec, error) {
// ReadEntry reads the next manifest entry in the avro file's data.
func (c *ManifestReader) ReadEntry() (ManifestEntry, error) {
- if err := c.dec.Error(); err != nil {
- return nil, err
- }
- if !c.dec.HasNext() {
- return nil, io.EOF
- }
var tmp ManifestEntry
if c.isFallback {
tmp = &fallbackManifestEntry{
@@ -766,7 +759,7 @@ func (c *ManifestReader) ReadEntry() (ManifestEntry, error) {
tmp = &manifestEntry{Data: &dataFile{}}
}
- if err := c.dec.Decode(tmp); err != nil {
+ if err := c.rd.Decode(tmp); err != nil {
return nil, err
}
if c.isFallback {
@@ -818,18 +811,18 @@ func ReadManifest(m ManifestFile, f io.Reader, discardDeleted bool) ([]ManifestE
// "format-version" metadata key (only manifest files are). When the key is
// absent, version 1 is assumed.
func ReadManifestList(in io.Reader) ([]ManifestFile, error) {
- dec, err := ocf.NewDecoder(in, ocf.WithDecoderSchemaCache(&avro.SchemaCache{}))
+ rd, err := ocf.NewReader(in)
if err != nil {
return nil, err
}
- sc, err := avro.ParseBytes(dec.Metadata()["avro.schema"])
+ sc, err := avro.Parse(string(rd.Metadata()["avro.schema"]))
if err != nil {
return nil, err
}
version := 1
- if raw := dec.Metadata()["format-version"]; len(raw) > 0 {
+ if raw := rd.Metadata()["format-version"]; len(raw) > 0 {
version, err = strconv.Atoi(string(raw))
if err != nil {
return nil, fmt.Errorf("invalid format-version: %w", err)
@@ -837,10 +830,11 @@ func ReadManifestList(in io.Reader) ([]ManifestFile, error) {
}
if version == 1 {
- for _, f := range sc.(*avro.RecordSchema).Fields() {
- if f.Name() == "added_snapshot_id" {
- if f.Type().Type() == avro.Union {
- return decodeManifestsWithFallback[*fallbackManifestFileV1](dec)
+ root := sc.Root()
+ for _, f := range root.Fields {
+ if f.Name == "added_snapshot_id" {
+ if f.Type.Type == "union" {
+ return decodeManifestsWithFallback[*fallbackManifestFileV1](rd)
}
break
@@ -850,9 +844,9 @@ func ReadManifestList(in io.Reader) ([]ManifestFile, error) {
switch version {
case 1:
- return decodeManifestsWithFallback[*manifestFileV1](dec)
+ return decodeManifestsWithFallback[*manifestFileV1](rd)
default:
- return decodeManifests[*manifestFile](dec, version)
+ return decodeManifests[*manifestFile](rd, version)
}
}
@@ -1067,15 +1061,14 @@ type ManifestWriter struct {
impl writerImpl
output io.Writer
- writer *ocf.Encoder
+ writer *ocf.Writer
spec PartitionSpec
schema *Schema
content ManifestContent
- partFieldNameToID map[string]int
- partFieldIDToType map[int]avro.LogicalType
- partFieldIDToFixSize map[int]int
+ partFieldNameToID map[string]int
+ partFieldIDToType map[int]string
snapshotID int64
addedFiles int32
@@ -1122,21 +1115,20 @@ func NewManifestWriter(version int, out io.Writer, spec PartitionSpec, schema *S
return nil, err
}
- nameToID, idToType, idToFixSize := getFieldIDMap(fileSchema)
+ nameToID, idToType, _ := getFieldIDMap(fileSchema)
w := &ManifestWriter{
- impl: impl,
- version: version,
- output: out,
- spec: spec,
- content: ManifestContentData,
- schema: schema,
- partFieldNameToID: nameToID,
- partFieldIDToType: idToType,
- partFieldIDToFixSize: idToFixSize,
- snapshotID: snapshotID,
- minSeqNum: -1,
- partitions: make([]map[int]any, 0),
+ impl: impl,
+ version: version,
+ output: out,
+ spec: spec,
+ content: ManifestContentData,
+ schema: schema,
+ partFieldNameToID: nameToID,
+ partFieldIDToType: idToType,
+ snapshotID: snapshotID,
+ minSeqNum: -1,
+ partitions: make([]map[int]any, 0),
}
for _, apply := range opts {
@@ -1151,13 +1143,12 @@ func NewManifestWriter(version int, out io.Writer, spec PartitionSpec, schema *S
return nil, err
}
- enc, err := ocf.NewEncoderWithSchema(fileSchema, out,
- ocf.WithSchemaMarshaler(ocf.FullSchemaMarshaler),
- ocf.WithEncoderSchemaCache(&avro.SchemaCache{}),
+ wr, err := ocf.NewWriter(out, fileSchema,
+ ocf.WithSchema(fileSchema.String()),
ocf.WithMetadata(md),
- ocf.WithCodec(ocf.Deflate))
+ ocf.WithCodec(ocf.DeflateCodec(-1)))
- w.writer = enc
+ w.writer = wr
return w, err
}
@@ -1274,11 +1265,10 @@ func (w *ManifestWriter) addEntry(entry *manifestEntry) error {
if setter, ok := entry.DataFile().(hasFieldToIDMap); ok {
setter.setFieldNameToIDMap(w.partFieldNameToID)
setter.setFieldIDToLogicalTypeMap(w.partFieldIDToType)
- setter.setFieldIDToFixedSizeMap(w.partFieldIDToFixSize)
}
w.partitions = append(w.partitions, entry.Data.Partition())
- partitionData := avroPartitionData(entry.Data.Partition(), w.partFieldIDToType, w.partFieldIDToFixSize)
+ partitionData := avroPartitionData(entry.Data.Partition(), w.partFieldIDToType)
if dataFile, ok := entry.DataFile().(*dataFile); ok {
convertedPartitionData := make(map[string]any)
@@ -1331,7 +1321,7 @@ type ManifestListWriter struct {
out io.Writer
commitSnapshotID int64
sequenceNumber int64
- writer *ocf.Encoder
+ writer *ocf.Writer
nextRowID *int64
}
@@ -1404,16 +1394,15 @@ func (m *ManifestListWriter) init(meta map[string][]byte) error {
return err
}
- enc, err := ocf.NewEncoderWithSchema(fileSchema, m.out,
- ocf.WithSchemaMarshaler(ocf.FullSchemaMarshaler),
- ocf.WithEncoderSchemaCache(&avro.SchemaCache{}),
+ wr, err := ocf.NewWriter(m.out, fileSchema,
+ ocf.WithSchema(fileSchema.String()),
ocf.WithMetadata(meta),
- ocf.WithCodec(ocf.Deflate))
+ ocf.WithCodec(ocf.DeflateCodec(-1)))
if err != nil {
return err
}
- m.writer = enc
+ m.writer = wr
return nil
}
@@ -1648,13 +1637,11 @@ func mapToAvroColMap[K comparable, V any](m map[K]V) *[]colMap[K, V] {
return &out
}
-func avroPartitionData(input map[int]any, logicalTypes map[int]avro.LogicalType, fixedSizes map[int]int) map[int]any {
+func avroPartitionData(input map[int]any, logicalTypes map[int]string) map[int]any {
out := make(map[int]any)
for k, v := range input {
if logical, ok := logicalTypes[k]; ok {
out[k] = convertLogicalTypeValue(v, logical)
- } else if size, ok := fixedSizes[k]; ok {
- out[k] = convertFixedValue(v, size)
} else {
out[k] = v
}
@@ -1663,31 +1650,17 @@ func avroPartitionData(input map[int]any, logicalTypes map[int]avro.LogicalType,
return out
}
-func convertFixedValue(v any, size int) any {
- if v == nil {
- return map[string]any{"null": nil}
- }
-
- if bytes, ok := v.([]byte); ok {
- fixedArray := convertToFixedArray(padOrTruncateBytes(bytes, size), size)
-
- return map[string]any{fmt.Sprintf("fixed_%d", size): fixedArray}
- }
-
- return v
-}
-
-func convertLogicalTypeValue(v any, logicalType avro.LogicalType) any {
+func convertLogicalTypeValue(v any, logicalType string) any {
switch logicalType {
- case avro.Date:
+ case atype.Date:
return convertDateValue(v)
- case avro.TimeMicros:
+ case atype.TimeMicros:
return convertTimeMicrosValue(v)
- case avro.TimestampMicros:
+ case atype.TimestampMicros:
return convertTimestampMicrosValue(v)
- case avro.Decimal:
+ case atype.Decimal:
return convertDecimalValue(v)
- case avro.UUID:
+ case atype.UUID:
return convertUUIDValue(v)
default:
return v
@@ -1695,67 +1668,46 @@ func convertLogicalTypeValue(v any, logicalType avro.LogicalType) any {
}
func convertDateValue(v any) any {
- if v == nil {
- return map[string]any{"null": nil}
- }
-
if d, ok := v.(Date); ok {
- return map[string]any{"int.date": int32(d)}
+ return int32(d)
}
return v
}
func convertTimeMicrosValue(v any) any {
- if v == nil {
- return map[string]any{"null": nil}
- }
-
if t, ok := v.(Time); ok {
- return map[string]any{"long.time-micros": int64(t)}
+ return int64(t)
}
return v
}
func convertTimestampMicrosValue(v any) any {
- if v == nil {
- return map[string]any{"null": nil}
- }
-
if ts, ok := v.(Timestamp); ok {
- return map[string]any{"long.timestamp-micros": int64(ts)}
+ return int64(ts)
}
return v
}
func convertDecimalValue(v any) any {
- if v == nil {
- return map[string]any{"null": nil}
- }
-
if dec, ok := v.(Decimal); ok {
fixedSize := internal.DecimalRequiredBytes(len(dec.String()))
bytes, err := DecimalLiteral(dec).MarshalBinary()
if err != nil {
return v
}
- fixedArray := convertToFixedArray(padOrTruncateBytes(bytes, fixedSize), fixedSize)
- return map[string]any{"fixed": fixedArray}
+ return padOrTruncateBytes(bytes, fixedSize)
}
return v
}
func convertUUIDValue(v any) any {
- if v == nil {
- return map[string]any{"null": nil}
- }
-
if uuidVal, ok := v.(uuid.UUID); ok {
- return map[string]any{"uuid": [16]byte(uuidVal)}
+ return [16]byte(uuidVal)
}
return v
@@ -1770,13 +1722,6 @@ func padOrTruncateBytes(bytes []byte, size int) []byte {
return append(make([]byte, size-len(bytes)), padded...)
}
-func convertToFixedArray(bytes []byte, size int) any {
- arr := reflect.New(reflect.ArrayOf(size, reflect.TypeOf(byte(0)))).Elem()
- reflect.Copy(arr, reflect.ValueOf(bytes))
-
- return arr.Interface()
-}
-
type dataFile struct {
Content ManifestEntryContent `avro:"content"`
Path string `avro:"file_path"`
@@ -1811,7 +1756,7 @@ type dataFile struct {
// used for partition retrieval
fieldNameToID map[string]int
- fieldIDToLogicalType map[int]avro.LogicalType
+ fieldIDToLogicalType map[int]string
fieldIDToPartitionData map[int]any
fieldIDToFixedSize map[int]int
@@ -1843,86 +1788,69 @@ func (d *dataFile) initializeMapData() {
func (d *dataFile) convertAvroValueToIcebergType(v any, fieldID int) any {
if logicalType, ok := d.fieldIDToLogicalType[fieldID]; ok {
+ // twmb/avro returns rich Go types (time.Time, time.Duration,
+ // *big.Rat, [16]byte) when the file schema includes a logicalType,
+ // but raw primitives (int32, int64, []byte, string) when it does
+ // not. Each case handles both possibilities.
switch logicalType {
- case avro.Date:
+ case atype.Date:
if val, ok := v.(time.Time); ok {
return Date(val.Truncate(24*time.Hour).Unix() / int64((time.Hour * 24).Seconds()))
}
return Date(v.(int32))
- case avro.TimeMillis:
+ case atype.TimeMillis:
if val, ok := v.(time.Duration); ok {
return Time(val.Milliseconds())
}
return Time(v.(int64))
- case avro.TimeMicros:
+ case atype.TimeMicros:
if val, ok := v.(time.Duration); ok {
return Time(val.Microseconds())
}
return Time(v.(int64))
- case avro.TimestampMillis:
+ case atype.TimestampMillis:
if val, ok := v.(time.Time); ok {
return Timestamp(val.UTC().UnixMilli())
}
return Timestamp(v.(int64))
- case avro.TimestampMicros:
+ case atype.TimestampMicros:
if val, ok := v.(time.Time); ok {
return Timestamp(val.UTC().UnixMicro())
}
return Timestamp(v.(int64))
- case avro.Decimal:
- if unionMap, ok := v.(map[string]any); ok {
- if val, ok := unionMap["fixed"]; ok {
- if bigRatValue, ok := val.(*big.Rat); ok {
- scale := d.fieldIDToFixedSize[fieldID]
- scaleFactor := new(big.Int).Exp(big.NewInt(10), big.NewInt(int64(scale)), nil)
- unscaled := new(big.Int).Mul(bigRatValue.Num(), scaleFactor)
- unscaled = unscaled.Div(unscaled, bigRatValue.Denom())
- decimal128Val := decimal128.FromBigInt(unscaled)
-
- return DecimalLiteral{
- Scale: scale,
- Val: decimal128Val,
- }
- }
+ case atype.Decimal:
+ if r, ok := v.(*big.Rat); ok {
+ scale := d.fieldIDToFixedSize[fieldID]
+ scaleFactor := new(big.Int).Exp(big.NewInt(10), big.NewInt(int64(scale)), nil)
+ unscaled := new(big.Int).Mul(r.Num(), scaleFactor)
+ unscaled = unscaled.Div(unscaled, r.Denom())
+
+ return DecimalLiteral{
+ Scale: scale,
+ Val: decimal128.FromBigInt(unscaled),
}
}
return v
- case avro.UUID:
- if unionMap, ok := v.(map[string]any); ok {
- if val, ok := unionMap["uuid"]; ok {
- if uuidArr, ok := val.([16]byte); ok {
- return uuid.UUID(uuidArr)
- }
- }
+ case atype.UUID:
+ if uuidArr, ok := v.([16]byte); ok {
+ return uuid.UUID(uuidArr)
}
return v
}
}
- if size, ok := d.fieldIDToFixedSize[fieldID]; ok {
- if unionMap, ok := v.(map[string]any); ok {
- key := fmt.Sprintf("fixed_%d", size)
- if val, ok := unionMap[key]; ok {
- rv := reflect.ValueOf(val)
- if rv.Kind() == reflect.Array {
- return rv.Slice(0, rv.Len()).Bytes()
- }
- }
- }
- }
-
return v
}
func (d *dataFile) setFieldNameToIDMap(m map[string]int) { d.fieldNameToID = m }
-func (d *dataFile) setFieldIDToLogicalTypeMap(m map[int]avro.LogicalType) {
+func (d *dataFile) setFieldIDToLogicalTypeMap(m map[int]string) {
d.fieldIDToLogicalType = m
}
func (d *dataFile) setFieldIDToFixedSizeMap(m map[int]int) { d.fieldIDToFixedSize = m }
@@ -2146,7 +2074,7 @@ func NewDataFileBuilder(
path string,
format FileFormat,
fieldIDToPartitionData map[int]any,
- fieldIDToLogicalType map[int]avro.LogicalType,
+ fieldIDToLogicalType map[int]string,
fieldIDToFixedSize map[int]int,
recordCount int64,
fileSize int64,
diff --git a/manifest_test.go b/manifest_test.go
index b3fdbf97..e9db2969 100644
--- a/manifest_test.go
+++ b/manifest_test.go
@@ -19,6 +19,7 @@ package iceberg
import (
"bytes"
+ "compress/flate"
"encoding/json"
"errors"
"io"
@@ -27,9 +28,9 @@ import (
"time"
"github.com/apache/iceberg-go/internal"
- "github.com/hamba/avro/v2"
- "github.com/hamba/avro/v2/ocf"
"github.com/stretchr/testify/suite"
+ "github.com/twmb/avro"
+ "github.com/twmb/avro/ocf"
)
var (
@@ -784,18 +785,17 @@ func writeManifestListNoFormatVersion(t *testing.T, version int) bytes.Buffer {
}
var buf bytes.Buffer
- enc, err := ocf.NewEncoderWithSchema(fileSchema, &buf,
- ocf.WithSchemaMarshaler(ocf.FullSchemaMarshaler),
- ocf.WithEncoderSchemaCache(&avro.SchemaCache{}),
+ wr, err := ocf.NewWriter(&buf, fileSchema,
+ ocf.WithSchema(fileSchema.String()),
ocf.WithMetadata(map[string][]byte{
"snapshot-id": []byte("1234"),
"sequence-number": []byte("0"),
}),
- ocf.WithCodec(ocf.Deflate))
+ ocf.WithCodec(ocf.DeflateCodec(-1)))
if err != nil {
t.Fatal(err)
}
- if err := enc.Close(); err != nil {
+ if err := wr.Close(); err != nil {
t.Fatal(err)
}
@@ -832,9 +832,7 @@ func writeManifestNoFormatVersion(t *testing.T) bytes.Buffer {
}
var buf bytes.Buffer
- enc, err := ocf.NewEncoderWithSchema(entrySchema, &buf,
- ocf.WithSchemaMarshaler(ocf.FullSchemaMarshaler),
- ocf.WithEncoderSchemaCache(&avro.SchemaCache{}),
+ w, err := ocf.NewWriter(&buf, entrySchema,
ocf.WithMetadata(map[string][]byte{
// intentionally omit "format-version" to simulate Java Iceberg v1 files
"schema": schemaJSON,
@@ -843,11 +841,11 @@ func writeManifestNoFormatVersion(t *testing.T) bytes.Buffer {
"partition-spec-id": []byte("0"),
"content": []byte("data"),
}),
- ocf.WithCodec(ocf.Deflate))
+ ocf.WithCodec(ocf.DeflateCodec(flate.DefaultCompression)))
if err != nil {
t.Fatal(err)
}
- if err := enc.Close(); err != nil {
+ if err := w.Close(); err != nil {
t.Fatal(err)
}
@@ -866,15 +864,10 @@ func (m *ManifestTestSuite) TestNewManifestReaderMissingFormatVersion() {
}
func (m *ManifestTestSuite) TestReadManifestListIncompleteSchema() {
- // This prevents a regression that could be caused by using a schema cache
- // across multiple read/write operations of an avro file. While it may sound
- // like a reasonable idea (caches speed things up, right?), it isn't that
- // sort of cache: it's really a resolver to allow files with incomplete
- // schemas, which we don't want.
-
- // If a schema cache *were* in use, this would populate it with a definition for
- // the missing record type in the incomplete schema. So we'll first "warm up"
- // any cache. (Note: if working correctly, this will have no such side effect.)
+ // Verify that reading a manifest list whose embedded schema references
+ // an undefined named type ("field_summary" without its definition)
+ // fails. A stray cache or cross-file resolver could mask this by
+ // reusing a definition from a previously-read file.
var buf bytes.Buffer
seqNum := int64(9876)
err := WriteManifestList(2, &buf, 1234, nil, &seqNum, 0, []ManifestFile{
@@ -988,14 +981,10 @@ func (m *ManifestTestSuite) TestReadManifestListIncompleteSchema() {
}`
// We'll generate a file that is missing part of its schema
- cache := &avro.SchemaCache{}
sch, err := internal.NewManifestFileSchema(2)
m.NoError(err)
- enc, err := ocf.NewEncoderWithSchema(sch, &buf,
- ocf.WithEncoderSchemaCache(cache),
- ocf.WithSchemaMarshaler(func(schema avro.Schema) ([]byte, error) {
- return []byte(incompleteSchema), nil
- }),
+ wr, err := ocf.NewWriter(&buf, sch,
+ ocf.WithSchema(incompleteSchema),
ocf.WithMetadata(map[string][]byte{
"format-version": {'2'},
"snapshot-id": []byte("1234"),
@@ -1005,24 +994,19 @@ func (m *ManifestTestSuite) TestReadManifestListIncompleteSchema() {
)
m.NoError(err)
for _, file := range files {
- m.NoError(enc.Encode(file))
+ m.NoError(wr.Encode(file))
}
// This should fail because the file's schema is incomplete.
_, err = ReadManifestList(&buf)
- m.ErrorContains(err, "unknown type: field_summary")
+ m.Error(err)
}
func (m *ManifestTestSuite) TestReadManifestIncompleteSchema() {
- // This prevents a regression that could be caused by using a schema cache
- // across multiple read/write operations of an avro file. While it may sound
- // like a reasonable idea (caches speed things up, right?), it isn't that
- // sort of cache: it's really a resolver to allow files with incomplete
- // schemas, which we don't want.
-
- // If a schema cache *were* in use, this would populate it with a definition for
- // the missing record type in the incomplete schema. So we'll first "warm up"
- // any cache. (Note: if working correctly, this will have no such side effect.)
+ // Verify that reading a manifest entry whose embedded schema references
+ // an undefined named type ("r2" without its definition) fails. A stray
+ // cache or cross-file resolver could mask this by reusing a definition
+ // from a previously-read file.
var buf bytes.Buffer
partitionSpec := NewPartitionSpecID(1)
snapshotID := int64(12345678)
@@ -1033,7 +1017,7 @@ func (m *ManifestTestSuite) TestReadManifestIncompleteSchema() {
"s3://bucket/namespace/table/data/abcd-0123.parquet",
ParquetFile,
map[int]any{},
- map[int]avro.LogicalType{},
+ map[int]string{},
map[int]int{},
100,
100*1000*1000,
@@ -1109,16 +1093,15 @@ func (m *ManifestTestSuite) TestReadManifestIncompleteSchema() {
}`
// We'll generate a file that is missing part of its schema
- cache := &avro.SchemaCache{}
- partitionSchema, err := avro.NewRecordSchema("r102", "", nil) // empty struct
+ partNode := avro.SchemaNode{
+ Type: "record", Name: "r102", Fields: nil,
+ }
+ partitionSchema, err := partNode.Schema()
m.NoError(err)
sch, err := internal.NewManifestEntrySchema(partitionSchema, 2)
m.NoError(err)
- enc, err := ocf.NewEncoderWithSchema(sch, &buf,
- ocf.WithEncoderSchemaCache(cache),
- ocf.WithSchemaMarshaler(func(schema avro.Schema) ([]byte, error) {
- return []byte(incompleteSchema), nil
- }),
+ wr, err := ocf.NewWriter(&buf, sch,
+ ocf.WithSchema(incompleteSchema),
ocf.WithMetadata(map[string][]byte{
"format-version": {'2'},
// TODO: spec says other things are required, like schema and partition-spec info,
@@ -1127,12 +1110,12 @@ func (m *ManifestTestSuite) TestReadManifestIncompleteSchema() {
)
m.NoError(err)
for _, entry := range entries {
- m.NoError(enc.Encode(entry))
+ m.NoError(wr.Encode(entry))
}
// This should fail because the file's schema is incomplete.
_, err = ReadManifest(file, &buf, false)
- m.ErrorContains(err, "unknown type: r2")
+ m.Error(err)
}
func (m *ManifestTestSuite) TestManifestEntriesV2() {
@@ -1291,7 +1274,7 @@ func (m *ManifestTestSuite) TestManifestEntriesV2() {
m.Equal(testEqualityIDs, datafile.EqualityFieldIDs())
m.Zero(*datafile.SortOrderID())
- m.equalityIDsSchemaIsInt(manifestReader.dec.Schema())
+ m.equalityIDsSchemaIsInt(manifestReader.rd.Schema())
}
func (m *ManifestTestSuite) TestManifestEntriesV3() {
@@ -1373,7 +1356,7 @@ func (m *ManifestTestSuite) TestManifestEntriesV3() {
m.Equal(testEqualityIDs, datafile.EqualityFieldIDs())
m.Zero(*datafile.SortOrderID())
- m.equalityIDsSchemaIsInt(manifestReader.dec.Schema())
+ m.equalityIDsSchemaIsInt(manifestReader.rd.Schema())
}
func (m *ManifestTestSuite) TestNewManifestReaderZstdManifestEntriesV2() {
@@ -1403,16 +1386,15 @@ func (m *ManifestTestSuite) TestNewManifestReaderZstdManifestEntriesV2() {
m.Require().NoError(err)
var buf bytes.Buffer
- enc, err := ocf.NewEncoderWithSchema(entrySchema, &buf,
- ocf.WithSchemaMarshaler(ocf.FullSchemaMarshaler),
- ocf.WithEncoderSchemaCache(&avro.SchemaCache{}),
+ wr, err := ocf.NewWriter(&buf, entrySchema,
+ ocf.WithSchema(entrySchema.String()),
ocf.WithMetadata(md),
- ocf.WithCodec(ocf.ZStandard))
+ ocf.WithCodec(ocf.MustZstdCodec(nil, nil)))
m.Require().NoError(err)
- m.Require().NoError(enc.Encode(manifestEntryV2Records[0]))
- m.Require().NoError(enc.Encode(manifestEntryV2Records[1]))
- m.Require().NoError(enc.Close())
+ m.Require().NoError(wr.Encode(manifestEntryV2Records[0]))
+ m.Require().NoError(wr.Encode(manifestEntryV2Records[1]))
+ m.Require().NoError(wr.Close())
manifestReader, err := NewManifestReader(&manifest, bytes.NewReader(buf.Bytes()))
m.Require().NoError(err)
@@ -1439,7 +1421,7 @@ func (m *ManifestTestSuite) TestManifestEntryBuilder() {
"sample.parquet",
ParquetFile,
map[int]any{1001: int(1), 1002: time.Unix(1925, 0).UnixMicro()},
- map[int]avro.LogicalType{},
+ map[int]string{},
map[int]int{},
1,
2,
@@ -1520,42 +1502,43 @@ func (m *ManifestTestSuite) TestManifestEntryBuilder() {
}
// equalityIDsSchemaIsInt asserts equality_ids uses Avro "int", not "long".
-func (m *ManifestTestSuite) equalityIDsSchemaIsInt(sc avro.Schema) {
+func (m *ManifestTestSuite) equalityIDsSchemaIsInt(sc *avro.Schema) {
m.T().Helper()
- entrySchema := sc.(*avro.RecordSchema)
- var dataFileSchema *avro.RecordSchema
- for _, f := range entrySchema.Fields() {
- if f.Name() == "data_file" {
- dataFileSchema = f.Type().(*avro.RecordSchema)
+ root := sc.Root()
+ var dataFileField *avro.SchemaField
+ for i := range root.Fields {
+ if root.Fields[i].Name == "data_file" {
+ dataFileField = &root.Fields[i]
break
}
}
- m.Require().NotNil(dataFileSchema, "data_file field not found in manifest_entry schema")
+ m.Require().NotNil(dataFileField, "data_file field not found in manifest_entry schema")
- var eqIDsField *avro.Field
- for _, f := range dataFileSchema.Fields() {
- if f.Name() == "equality_ids" {
- eqIDsField = f
+ dfType := dataFileField.Type
+ var eqIDsField *avro.SchemaField
+ for i := range dfType.Fields {
+ if dfType.Fields[i].Name == "equality_ids" {
+ eqIDsField = &dfType.Fields[i]
break
}
}
m.Require().NotNil(eqIDsField, "equality_ids field not found in data_file schema")
- unionSchema := eqIDsField.Type().(*avro.UnionSchema)
- var arraySchema *avro.ArraySchema
- for _, ts := range unionSchema.Types() {
- if ts.Type() == avro.Array {
- arraySchema = ts.(*avro.ArraySchema)
+ // equality_ids is ["null", {"type":"array","items":"int"}]
+ m.Require().Equal("union", eqIDsField.Type.Type)
+ var arrayBranch *avro.SchemaNode
+ for i := range eqIDsField.Type.Branches {
+ if eqIDsField.Type.Branches[i].Type == "array" {
+ arrayBranch = &eqIDsField.Type.Branches[i]
break
}
}
- m.Require().NotNil(arraySchema, "equality_ids union should contain an array schema")
-
- m.Equal(avro.Int, arraySchema.Items().Type(),
+ m.Require().NotNil(arrayBranch, "equality_ids union should contain an array schema")
+ m.Equal("int", arrayBranch.Items.Type,
"equality_ids array elements must be Avro int (not long) per the Iceberg spec")
}
@@ -1774,7 +1757,7 @@ func (m *ManifestTestSuite) TestManifestRoundTripSortOrderID() {
"s3://bucket/ns/table/data/round-trip.parquet",
ParquetFile,
map[int]any{},
- map[int]avro.LogicalType{},
+ map[int]string{},
map[int]int{},
10,
10*1000,
diff --git a/schema_conversions.go b/schema_conversions.go
index 6e91ce52..70563426 100644
--- a/schema_conversions.go
+++ b/schema_conversions.go
@@ -21,50 +21,59 @@ import (
"fmt"
"github.com/apache/iceberg-go/internal"
- "github.com/hamba/avro/v2"
+ "github.com/twmb/avro"
+ "github.com/twmb/avro/atype"
)
-func partitionTypeToAvroSchema(t *StructType) (avro.Schema, error) {
- fields := make([]*avro.Field, len(t.FieldList))
+func partitionTypeToAvroSchema(t *StructType) (*avro.Schema, error) {
+ fields := make([]avro.SchemaField, len(t.FieldList))
for i, f := range t.FieldList {
- var sc avro.Schema
+ var node avro.SchemaNode
switch typ := f.Type.(type) {
case Int32Type:
- sc = internal.NullableSchema(internal.IntSchema)
+ node = internal.NullableNode(internal.IntNode)
case Int64Type:
- sc = internal.NullableSchema(internal.LongSchema)
+ node = internal.NullableNode(internal.LongNode)
case Float32Type:
- sc = internal.NullableSchema(internal.FloatSchema)
+ node = internal.NullableNode(internal.FloatNode)
case Float64Type:
- sc = internal.NullableSchema(internal.DoubleSchema)
+ node = internal.NullableNode(internal.DoubleNode)
case StringType:
- sc = internal.NullableSchema(internal.StringSchema)
+ node = internal.NullableNode(internal.StringNode)
case DateType:
- sc = internal.NullableSchema(internal.DateSchema)
+ node = internal.NullableNode(internal.DateNode)
case TimeType:
- sc = internal.NullableSchema(internal.TimeSchema)
+ node = internal.NullableNode(internal.TimeNode)
case TimestampType:
- sc = internal.NullableSchema(internal.TimestampSchema)
+ node = internal.NullableNode(internal.TimestampNode)
case TimestampTzType:
- sc = internal.NullableSchema(internal.TimestampTzSchema)
+ node = internal.NullableNode(internal.TimestampTzNode)
case UUIDType:
- sc = internal.NullableSchema(internal.UUIDSchema)
+ node = internal.NullableNode(internal.UUIDNode)
case BooleanType:
- sc = internal.NullableSchema(internal.BoolSchema)
+ node = internal.NullableNode(internal.BoolNode)
case BinaryType:
- sc = internal.NullableSchema(internal.BinarySchema)
+ node = internal.NullableNode(internal.BytesNode)
case FixedType:
- fixedSchema := internal.FixedSchema(typ.Len())
- sc = internal.NullableSchema(fixedSchema)
+ node = internal.NullableNode(internal.FixedNode(typ.Len()))
case DecimalType:
- decimalSchema := internal.DecimalSchema(typ.precision, typ.scale)
- sc = internal.NullableSchema(decimalSchema)
+ node = internal.NullableNode(internal.DecimalNode(typ.precision, typ.scale))
default:
return nil, fmt.Errorf("unsupported partition type: %s", f.Type.String())
}
- fields[i], _ = avro.NewField(f.Name, sc, internal.WithFieldID(f.ID))
+ fields[i] = avro.SchemaField{
+ Name: f.Name,
+ Type: node,
+ Props: internal.WithFieldID(f.ID),
+ }
+ }
+
+ node := avro.SchemaNode{
+ Type: atype.Record,
+ Name: "r102",
+ Fields: fields,
}
- return avro.NewRecordSchema("r102", "", fields)
+ return node.Schema()
}
diff --git a/schema_conversions_test.go b/schema_conversions_test.go
index 2323b418..74d32a12 100644
--- a/schema_conversions_test.go
+++ b/schema_conversions_test.go
@@ -22,121 +22,62 @@ import (
"testing"
"github.com/apache/iceberg-go/internal"
- "github.com/hamba/avro/v2"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
+ "github.com/twmb/avro"
)
-func partitionTypeToAvroSchemaNonNullable(t *StructType) (avro.Schema, error) {
- fields := make([]*avro.Field, len(t.FieldList))
+func partitionTypeToAvroSchemaNonNullable(t *StructType) (*avro.Schema, error) {
+ fields := make([]avro.SchemaField, len(t.FieldList))
for i, f := range t.FieldList {
- var sc avro.Schema
+ var node avro.SchemaNode
switch typ := f.Type.(type) {
case Int32Type:
- sc = internal.IntSchema
+ node = internal.IntNode
case Int64Type:
- sc = internal.LongSchema
+ node = internal.LongNode
case Float32Type:
- sc = internal.FloatSchema
+ node = internal.FloatNode
case Float64Type:
- sc = internal.DoubleSchema
+ node = internal.DoubleNode
case StringType:
- sc = internal.StringSchema
+ node = internal.StringNode
case DateType:
- sc = internal.DateSchema
+ node = internal.DateNode
case TimeType:
- sc = internal.TimeSchema
+ node = internal.TimeNode
case TimestampType:
- sc = internal.TimestampSchema
+ node = internal.TimestampNode
case TimestampTzType:
- sc = internal.TimestampTzSchema
+ node = internal.TimestampTzNode
case UUIDType:
- sc = internal.UUIDSchema
+ node = internal.UUIDNode
case BooleanType:
- sc = internal.BoolSchema
+ node = internal.BoolNode
case BinaryType:
- sc = internal.BinarySchema
+ node = internal.BytesNode
case FixedType:
- fixedSchema := internal.FixedSchema(typ.Len())
- sc = fixedSchema
+ node = internal.FixedNode(typ.Len())
case DecimalType:
- decimalSchema := internal.DecimalSchema(typ.precision, typ.scale)
- sc = decimalSchema
+ node = internal.DecimalNode(typ.precision, typ.scale)
default:
return nil, fmt.Errorf("unsupported partition type: %s", f.Type.String())
}
- fields[i], _ = avro.NewField(f.Name, sc, internal.WithFieldID(f.ID))
- }
-
- return avro.NewRecordSchema("r102", "", fields)
-}
-
-func TestPartitionTypeToAvroSchemaFixedType(t *testing.T) {
- partitionType := &StructType{
- FieldList: []NestedField{
- {ID: 1, Name: "fixed4_col", Type: FixedType{len: 4}, Required: false},
- {ID: 2, Name: "fixed16_col", Type: FixedType{len: 16}, Required: false},
- },
- }
-
- schema, err := partitionTypeToAvroSchema(partitionType)
- require.NoError(t, err)
- require.NotNil(t, schema)
-
- rec := schema.(*avro.RecordSchema)
- require.Len(t, rec.Fields(), 2)
-
- for _, f := range rec.Fields() {
- union, ok := f.Type().(*avro.UnionSchema)
- require.True(t, ok, "field %s should be a union schema", f.Name())
- require.Len(t, union.Types(), 2)
-
- fixed, ok := union.Types()[1].(*avro.FixedSchema)
- require.True(t, ok, "non-null branch of field %s should be avro.FixedSchema, got %T", f.Name(), union.Types()[1])
-
- switch f.Name() {
- case "fixed4_col":
- assert.Equal(t, 4, fixed.Size())
- case "fixed16_col":
- assert.Equal(t, 16, fixed.Size())
+ fields[i] = avro.SchemaField{
+ Name: f.Name,
+ Type: node,
+ Props: internal.WithFieldID(f.ID),
}
}
- t.Run("round-trip non-nil fixed data", func(t *testing.T) {
- data := map[string]any{
- "fixed4_col": map[string]any{"fixed_4": [4]byte{0x01, 0x02, 0x03, 0x04}},
- "fixed16_col": map[string]any{"fixed_16": [16]byte{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f}},
- }
-
- encoded, err := avro.Marshal(schema, data)
- require.NoError(t, err)
- require.NotEmpty(t, encoded)
-
- var decoded map[string]any
- err = avro.Unmarshal(schema, encoded, &decoded)
- require.NoError(t, err)
-
- assert.Equal(t, data["fixed4_col"], decoded["fixed4_col"])
- assert.Equal(t, data["fixed16_col"], decoded["fixed16_col"])
- })
-
- t.Run("round-trip nil fixed data", func(t *testing.T) {
- data := map[string]any{
- "fixed4_col": nil,
- "fixed16_col": nil,
- }
-
- encoded, err := avro.Marshal(schema, data)
- require.NoError(t, err)
-
- var decoded map[string]any
- err = avro.Unmarshal(schema, encoded, &decoded)
- require.NoError(t, err)
+ node := avro.SchemaNode{
+ Type: "record",
+ Name: "r102",
+ Fields: fields,
+ }
- assert.Nil(t, decoded["fixed4_col"])
- assert.Nil(t, decoded["fixed16_col"])
- })
+ return node.Schema()
}
func TestPartitionTypeToAvroSchemaNullableAndNonNullable(t *testing.T) {
@@ -181,12 +122,12 @@ func TestPartitionTypeToAvroSchemaNullableAndNonNullable(t *testing.T) {
require.NoError(t, err)
require.NotNil(t, schemaNullable)
- encoded, err := avro.Marshal(schemaNullable, partitionData)
+ encoded, err := schemaNullable.Encode(partitionData)
require.NoError(t, err)
require.NotEmpty(t, encoded)
var decoded map[string]any
- err = avro.Unmarshal(schemaNullable, encoded, &decoded)
+ _, err = schemaNullable.Decode(encoded, &decoded)
require.NoError(t, err)
assert.Nil(t, decoded["int32_col"])
@@ -210,8 +151,48 @@ func TestPartitionTypeToAvroSchemaNullableAndNonNullable(t *testing.T) {
require.NoError(t, err)
require.NotNil(t, schemaNonNullable)
- encoded, err := avro.Marshal(schemaNonNullable, partitionData)
+ encoded, err := schemaNonNullable.Encode(partitionData)
require.Error(t, err, "expected marshal to fail when values are nil for non-nullable schema")
assert.Empty(t, encoded)
})
}
+
+// TestPartitionTypeToAvroSchemaDuplicateNamedTypes verifies that a
+// partition spec containing multiple fields of the same named Avro type
+// (UUID, Fixed, Decimal) compiles without "duplicate named type" errors.
+// Relies on twmb/avro v1.4.0's automatic named type deduplication.
+func TestPartitionTypeToAvroSchemaDuplicateNamedTypes(t *testing.T) {
+ cases := []struct {
+ name string
+ ptyp *StructType
+ }{
+ {
+ "two uuid fields",
+ &StructType{FieldList: []NestedField{
+ {ID: 1, Name: "a", Type: UUIDType{}, Required: false},
+ {ID: 2, Name: "b", Type: UUIDType{}, Required: false},
+ }},
+ },
+ {
+ "two fixed fields same size",
+ &StructType{FieldList: []NestedField{
+ {ID: 1, Name: "a", Type: FixedType{len: 8}, Required: false},
+ {ID: 2, Name: "b", Type: FixedType{len: 8}, Required: false},
+ }},
+ },
+ {
+ "two decimal fields same precision/scale",
+ &StructType{FieldList: []NestedField{
+ {ID: 1, Name: "a", Type: DecimalType{precision: 10, scale: 2}, Required: false},
+ {ID: 2, Name: "b", Type: DecimalType{precision: 10, scale: 2}, Required: false},
+ }},
+ },
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ s, err := partitionTypeToAvroSchema(tc.ptyp)
+ require.NoError(t, err)
+ require.NotNil(t, s)
+ })
+ }
+}
diff --git a/table/internal/utils.go b/table/internal/utils.go
index 02e3d949..4b13c077 100644
--- a/table/internal/utils.go
+++ b/table/internal/utils.go
@@ -35,7 +35,7 @@ import (
"github.com/apache/arrow-go/v18/arrow/decimal"
"github.com/apache/arrow-go/v18/arrow/decimal128"
"github.com/apache/iceberg-go"
- "github.com/hamba/avro/v2"
+ "github.com/twmb/avro/atype"
"golang.org/x/sync/errgroup"
)
@@ -253,7 +253,7 @@ type DataFileOpts struct {
func (d *DataFileStatistics) ToDataFile(opts DataFileOpts) iceberg.DataFile {
var fieldIDToPartitionData map[int]any
- fieldIDToLogicalType := make(map[int]avro.LogicalType)
+ fieldIDToLogicalType := make(map[int]string)
fieldIDToFixedSize := make(map[int]int)
if !opts.Spec.Equals(*iceberg.UnpartitionedSpec) {
@@ -276,18 +276,18 @@ func (d *DataFileStatistics) ToDataFile(opts DataFileOpts) iceberg.DataFile {
switch rt := resultType.(type) {
case iceberg.DateType:
- fieldIDToLogicalType[field.FieldID] = avro.Date
+ fieldIDToLogicalType[field.FieldID] = atype.Date
case iceberg.TimeType:
- fieldIDToLogicalType[field.FieldID] = avro.TimeMicros
+ fieldIDToLogicalType[field.FieldID] = atype.TimeMicros
case iceberg.TimestampType:
- fieldIDToLogicalType[field.FieldID] = avro.TimestampMicros
+ fieldIDToLogicalType[field.FieldID] = atype.TimestampMicros
case iceberg.TimestampTzType:
- fieldIDToLogicalType[field.FieldID] = avro.TimestampMicros
+ fieldIDToLogicalType[field.FieldID] = atype.TimestampMicros
case iceberg.DecimalType:
- fieldIDToLogicalType[field.FieldID] = avro.Decimal
+ fieldIDToLogicalType[field.FieldID] = atype.Decimal
fieldIDToFixedSize[field.FieldID] = rt.Scale()
case iceberg.UUIDType:
- fieldIDToLogicalType[field.FieldID] = avro.UUID
+ fieldIDToLogicalType[field.FieldID] = atype.UUID
}
}
}
diff --git a/table/table_test.go b/table/table_test.go
index e6eb8b70..a58c630f 100644
--- a/table/table_test.go
+++ b/table/table_test.go
@@ -666,7 +666,7 @@ func (t *TableWritingTestSuite) TestAddFilesPartitionedTable() {
for _, e := range entries {
t.Equal(map[int]any{
- 1000: 123, 1001: 650,
+ 1000: int32(123), 1001: int32(650),
}, e.DataFile().Partition())
}
}