You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by ze...@apache.org on 2026/03/05 21:12:28 UTC
(iceberg-go) branch main updated: refactor: make partition spec and sort order fields range friendly (#772)
This is an automated email from the ASF dual-hosted git repository.
zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-go.git
The following commit(s) were added to refs/heads/main by this push:
new e3b130bb refactor: make partition spec and sort order fields range friendly (#772)
e3b130bb is described below
commit e3b130bbc38a9655ad87a968c82fd9d311a4d7aa
Author: ferhat elmas <el...@gmail.com>
AuthorDate: Thu Mar 5 22:12:22 2026 +0100
refactor: make partition spec and sort order fields range friendly (#772)
related to #767
Signed-off-by: ferhat elmas <el...@gmail.com>
---
catalog/sql/sql_test.go | 2 +-
cmd/iceberg/utils_test.go | 2 +-
partitions.go | 12 ++++--------
table/arrow_utils.go | 2 +-
table/internal/utils.go | 2 +-
table/metadata.go | 6 +++---
table/metadata_internal_test.go | 7 ++++++-
table/sorting.go | 4 ++--
table/transaction.go | 2 +-
table/update_spec.go | 9 +++++----
10 files changed, 25 insertions(+), 23 deletions(-)
diff --git a/catalog/sql/sql_test.go b/catalog/sql/sql_test.go
index 039189bc..cbafc28d 100644
--- a/catalog/sql/sql_test.go
+++ b/catalog/sql/sql_test.go
@@ -423,7 +423,7 @@ func (s *SqliteCatalogTestSuite) TestCreateTableCustomSortOrder() {
s.Equal(1, tbl.SortOrder().OrderID())
s.Equal(tbl.SortOrder().Len(), 1)
- for f := range tbl.SortOrder().Fields() {
+ for _, f := range tbl.SortOrder().Fields() {
s.Equal(table.SortASC, f.Direction)
s.Equal(table.NullsFirst, f.NullOrder)
s.Equal("identity", f.Transform.String())
diff --git a/cmd/iceberg/utils_test.go b/cmd/iceberg/utils_test.go
index 087deb52..ce1868b6 100644
--- a/cmd/iceberg/utils_test.go
+++ b/cmd/iceberg/utils_test.go
@@ -229,7 +229,7 @@ func TestParseSortOrder(t *testing.T) {
// Validate sort directions and null orders
i := 0
- for field := range got.Fields() {
+ for _, field := range got.Fields() {
if i < len(tt.expectedDirections) && field.Direction != tt.expectedDirections[i] {
t.Errorf("parseSortOrder() field %d direction = %v, expected %v", i, field.Direction, tt.expectedDirections[i])
}
diff --git a/partitions.go b/partitions.go
index ecba79ae..bd556cdd 100644
--- a/partitions.go
+++ b/partitions.go
@@ -142,7 +142,7 @@ func (p *PartitionSpec) BindToSchema(schema *Schema, lastPartitionID *int, newSp
opts = append(opts, WithSpecID(p.id))
}
- for field := range p.Fields() {
+ for _, field := range p.Fields() {
opts = append(opts, AddPartitionFieldBySourceID(field.SourceID, field.Name, field.Transform, schema, &field.FieldID))
}
@@ -344,13 +344,9 @@ func (ps PartitionSpec) Equals(other PartitionSpec) bool {
return ps.id == other.id && slices.Equal(ps.fields, other.fields)
}
-// Fields returns a clone of the partition fields in this spec.
-func (ps *PartitionSpec) Fields() iter.Seq[PartitionField] {
- if ps.fields == nil {
- return slices.Values([]PartitionField{})
- }
-
- return slices.Values(ps.fields)
+// Fields returns an iterator over the partition fields in this spec.
+func (ps *PartitionSpec) Fields() iter.Seq2[int, PartitionField] {
+ return slices.All(ps.fields)
}
func (ps PartitionSpec) MarshalJSON() ([]byte, error) {
diff --git a/table/arrow_utils.go b/table/arrow_utils.go
index 66cdf29d..82d49e2e 100644
--- a/table/arrow_utils.go
+++ b/table/arrow_utils.go
@@ -1244,7 +1244,7 @@ func filesToDataFiles(ctx context.Context, fileIO iceio.IO, meta *MetadataBuilde
partitionValues := make(map[int]any)
if !currentSpec.Equals(*iceberg.UnpartitionedSpec) {
- for field := range currentSpec.Fields() {
+ for _, field := range currentSpec.Fields() {
if !field.Transform.PreservesOrder() {
yield(nil, fmt.Errorf("cannot infer partition value from parquet metadata for a non-linear partition field: %s with transform %s", field.Name, field.Transform))
diff --git a/table/internal/utils.go b/table/internal/utils.go
index 4e9a5a05..95a1336d 100644
--- a/table/internal/utils.go
+++ b/table/internal/utils.go
@@ -241,7 +241,7 @@ func (d *DataFileStatistics) ToDataFile(schema *iceberg.Schema, spec iceberg.Par
if !spec.Equals(*iceberg.UnpartitionedSpec) {
fieldIDToPartitionData = make(map[int]any)
- for field := range spec.Fields() {
+ for _, field := range spec.Fields() {
partitionVal := partitionValues[field.FieldID]
if partitionVal != nil {
val := d.PartitionValue(field, schema)
diff --git a/table/metadata.go b/table/metadata.go
index 2dcdf6c1..5d303c45 100644
--- a/table/metadata.go
+++ b/table/metadata.go
@@ -368,7 +368,7 @@ func (b *MetadataBuilder) AddPartitionSpec(spec *iceberg.PartitionSpec, initial
maxFieldID := 0
fieldCount := 0
- for f := range freshSpec.Fields() {
+ for _, f := range freshSpec.Fields() {
maxFieldID = max(maxFieldID, f.FieldID)
if b.formatVersion <= 1 {
expectedID := partitionFieldStartID + fieldCount
@@ -1006,7 +1006,7 @@ func (b *MetadataBuilder) Build() (Metadata, error) {
}
partitionFields := make([]iceberg.PartitionField, 0)
- for f := range partition.Fields() {
+ for _, f := range partition.Fields() {
partitionFields = append(partitionFields, f)
}
@@ -1986,7 +1986,7 @@ func reassignIDs(sc *iceberg.Schema, partitions *iceberg.PartitionSpec, sortOrde
partitions = iceberg.UnpartitionedSpec
}
opts := make([]iceberg.PartitionOption, 0)
- for f := range partitions.Fields() {
+ for _, f := range partitions.Fields() {
var s string
var ok bool
if s, ok = previousMapFn(f.SourceID); !ok {
diff --git a/table/metadata_internal_test.go b/table/metadata_internal_test.go
index 5dd27098..22292f1a 100644
--- a/table/metadata_internal_test.go
+++ b/table/metadata_internal_test.go
@@ -750,6 +750,11 @@ func TestNewMetadataWithExplicitV1Format(t *testing.T) {
require.NoError(t, err)
lastPartitionID := 1000
+ partitionFields := make([]iceberg.PartitionField, 0)
+ for _, field := range expectedSpec.Fields() {
+ partitionFields = append(partitionFields, field)
+ }
+
expected := &metadataV1{
commonMetadata: commonMetadata{
Loc: "s3://some_v1_location/",
@@ -766,7 +771,7 @@ func TestNewMetadataWithExplicitV1Format(t *testing.T) {
FormatVersion: 1,
},
Schema: expectedSchema,
- Partition: slices.Collect(expectedSpec.Fields()),
+ Partition: partitionFields,
}
assert.Truef(t, expected.Equals(actual), "expected: %s\ngot: %s", expected, actual)
diff --git a/table/sorting.go b/table/sorting.go
index b43bfcf9..5e2446e7 100644
--- a/table/sorting.go
+++ b/table/sorting.go
@@ -144,8 +144,8 @@ func (s SortOrder) OrderID() int {
return s.orderID
}
-func (s SortOrder) Fields() iter.Seq[SortField] {
- return slices.Values(s.fields)
+func (s SortOrder) Fields() iter.Seq2[int, SortField] {
+ return slices.All(s.fields)
}
func (s SortOrder) Len() int {
diff --git a/table/transaction.go b/table/transaction.go
index 49081792..c946c17b 100644
--- a/table/transaction.go
+++ b/table/transaction.go
@@ -445,7 +445,7 @@ func validateDataFilePartitionData(df iceberg.DataFile, spec *iceberg.PartitionS
partitionData := df.Partition()
expectedFieldIDs := make(map[int]string)
- for field := range spec.Fields() {
+ for _, field := range spec.Fields() {
expectedFieldIDs[field.FieldID] = field.Name
if _, ok := partitionData[field.FieldID]; !ok {
return fmt.Errorf("missing partition value for field id %d (%s)", field.FieldID, field.Name)
diff --git a/table/update_spec.go b/table/update_spec.go
index a5493247..71fe0d1d 100644
--- a/table/update_spec.go
+++ b/table/update_spec.go
@@ -20,7 +20,6 @@ package table
import (
"errors"
"fmt"
- "slices"
"github.com/apache/iceberg-go"
)
@@ -58,7 +57,7 @@ func NewUpdateSpec(t *Transaction, caseSensitive bool) *UpdateSpec {
transformToField := make(map[transformKey]iceberg.PartitionField)
nameToField := make(map[string]iceberg.PartitionField)
partitionSpec := t.tbl.Metadata().PartitionSpec()
- for partitionField := range partitionSpec.Fields() {
+ for _, partitionField := range partitionSpec.Fields() {
transformToField[transformKey{
SourceId: partitionField.SourceID,
Transform: partitionField.Transform.String(),
@@ -140,7 +139,7 @@ func (us *UpdateSpec) Apply() (iceberg.PartitionSpec, error) {
partitionFields := make([]iceberg.PartitionField, 0)
partitionNames := make(map[string]bool)
spec := us.txn.tbl.Metadata().PartitionSpec()
- for field := range spec.Fields() {
+ for _, field := range spec.Fields() {
var newField iceberg.PartitionField
var err error
if _, deleted := us.deletes[field.FieldID]; !deleted {
@@ -326,7 +325,9 @@ func (us *UpdateSpec) partitionField(key transformKey, name string) (iceberg.Par
sourceId, transform := key.SourceId, key.Transform
historicalFields := make([]iceberg.PartitionField, 0)
for _, spec := range us.txn.tbl.Metadata().PartitionSpecs() {
- historicalFields = slices.AppendSeq(historicalFields, spec.Fields())
+ for _, field := range spec.Fields() {
+ historicalFields = append(historicalFields, field)
+ }
}
for _, field := range historicalFields {
if field.SourceID == sourceId && field.Transform.String() == transform {