You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by ze...@apache.org on 2026/06/08 22:39:37 UTC
(iceberg-go) branch main updated: feat(table): enforce sort order on write #833 (#1157)
This is an automated email from the ASF dual-hosted git repository.
zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-go.git
The following commit(s) were added to refs/heads/main by this push:
new e4bee1e0 feat(table): enforce sort order on write #833 (#1157)
e4bee1e0 is described below
commit e4bee1e0d6c0040846168b117aa3073b1bea2b5d
Author: Tanmay Rauth <t_...@apple.com>
AuthorDate: Tue Jun 9 04:09:32 2026 +0530
feat(table): enforce sort order on write #833 (#1157)
Resolve the table's default sort order to Arrow compute SortKeys and
apply compute.SortRecordBatch to each batch before flushing to a data
file. Skip for unsorted tables (sort_order_id 0) and for delete files,
where the table-level order does not apply.
All data write paths route through writerFactory + RollingDataWriter
(unpartitioned, clustered, and hash-fanout), so Append, Overwrite,
Delete data writes, WriteRecords, and compaction (which calls
WriteRecords) all pick up the sort.
Limitations:
- Per-batch, not per-file: each Add() is sorted internally; rows are not
merged across batches. Matches PyIceberg.
- Bucket transform falls back to source-column order (the manifest still
records sort_order_id verbatim).
- Multi-arg transform sort fields use only the first source column.
- Nested-field sort sources are rejected with a clear error.
Closes #833 (https://github.com/apache/iceberg-go/issues/833).
---
table/rolling_data_writer.go | 32 +++++
table/rolling_data_writer_test.go | 199 +++++++++++++++++++++++++++++
table/sort_keys.go | 80 ++++++++++++
table/sort_keys_test.go | 75 +++++++++++
table/write_records_sort_test.go | 260 ++++++++++++++++++++++++++++++++++++++
5 files changed, 646 insertions(+)
diff --git a/table/rolling_data_writer.go b/table/rolling_data_writer.go
index a8fe795e..66d93542 100644
--- a/table/rolling_data_writer.go
+++ b/table/rolling_data_writer.go
@@ -28,6 +28,7 @@ import (
"sync/atomic"
"github.com/apache/arrow-go/v18/arrow"
+ "github.com/apache/arrow-go/v18/arrow/compute"
"github.com/apache/iceberg-go"
iceio "github.com/apache/iceberg-go/io"
tblutils "github.com/apache/iceberg-go/table/internal"
@@ -57,6 +58,7 @@ type writerFactory struct {
content iceberg.ManifestEntryContent
equalityFieldIDs []int
sortOrderID int
+ sortKeys []compute.SortKey
writers sync.Map
partitionLocProviders sync.Map
@@ -176,6 +178,21 @@ func newWriterFactory(rootLocation string, args recordWritingArgs, meta *Metadat
return nil, err
}
+ if f.content == iceberg.EntryContentData && f.sortOrderID != UnsortedSortOrderID {
+ sortOrder, err := meta.GetSortOrderByID(f.sortOrderID)
+ if err != nil {
+ stopCount()
+
+ return nil, err
+ }
+ f.sortKeys, err = resolveSortKeys(*sortOrder, f.fileSchema)
+ if err != nil {
+ stopCount()
+
+ return nil, err
+ }
+ }
+
return f, nil
}
@@ -358,6 +375,21 @@ func (r *RollingDataWriter) stream(outputDataFilesCh chan<- iceberg.DataFile) {
return
}
+ // Sort each batch independently before writing. This is per-batch, not
+ // per-file: rows are not merged into one globally sorted run across
+ // batches. See resolveSortKeys for the full list of limitations.
+ if len(r.factory.sortKeys) > 0 {
+ sorted, err := compute.SortRecordBatch(r.ctx, converted, r.factory.sortKeys)
+ converted.Release()
+ if err != nil {
+ record.Release()
+ r.sendError(err)
+
+ return
+ }
+ converted = sorted
+ }
+
err = currentWriter.Write(converted)
converted.Release()
record.Release()
diff --git a/table/rolling_data_writer_test.go b/table/rolling_data_writer_test.go
index a3f06b68..9ff57080 100644
--- a/table/rolling_data_writer_test.go
+++ b/table/rolling_data_writer_test.go
@@ -26,6 +26,8 @@ import (
"github.com/apache/arrow-go/v18/arrow"
"github.com/apache/arrow-go/v18/arrow/array"
"github.com/apache/arrow-go/v18/arrow/memory"
+ "github.com/apache/arrow-go/v18/parquet/file"
+ "github.com/apache/arrow-go/v18/parquet/pqarrow"
"github.com/apache/iceberg-go"
iceio "github.com/apache/iceberg-go/io"
tblutils "github.com/apache/iceberg-go/table/internal"
@@ -452,3 +454,200 @@ func (s *RollingDataWriterTestSuite) TestPartitionLocProviderPreservesObjectStor
s.NotEqual("table_location/data/dt=2026-06-04/a", dataLoc,
"object storage layout should inject an entropy hash prefix between partition dir and file")
}
+
+// TestSortsRowsBeforeWriting exercises the sort-on-write path: the rolling
+// data writer must reorder rows according to the table's default sort order
+// before flushing them to the data file. We use a multi-key order with mixed
+// directions and null placement to cover the full SortField surface.
+func (s *RollingDataWriterTestSuite) TestSortsRowsBeforeWriting() {
+ arrSchema := arrow.NewSchema([]arrow.Field{
+ {Name: "id", Type: arrow.PrimitiveTypes.Int32, Nullable: true},
+ {Name: "name", Type: arrow.BinaryTypes.String, Nullable: true},
+ }, nil)
+
+ icebergSchema, err := ArrowSchemaToIcebergWithFreshIDs(arrSchema, false)
+ s.Require().NoError(err)
+ idFieldID := icebergSchema.Fields()[0].ID
+ nameFieldID := icebergSchema.Fields()[1].ID
+
+ // id ASC NULLS LAST, name DESC NULLS FIRST.
+ sortOrder, err := NewSortOrder(1, []SortField{
+ {SourceIDs: []int{idFieldID}, Transform: iceberg.IdentityTransform{}, Direction: SortASC, NullOrder: NullsLast},
+ {SourceIDs: []int{nameFieldID}, Transform: iceberg.IdentityTransform{}, Direction: SortDESC, NullOrder: NullsFirst},
+ })
+ s.Require().NoError(err)
+
+ loc := filepath.ToSlash(s.T().TempDir())
+ spec := iceberg.NewPartitionSpec()
+ meta, err := NewMetadata(icebergSchema, &spec, UnsortedSortOrder, loc, iceberg.Properties{})
+ s.Require().NoError(err)
+ metaBuilder, err := MetadataBuilderFromBase(meta, "")
+ s.Require().NoError(err)
+ s.Require().NoError(metaBuilder.AddSortOrder(&sortOrder))
+ s.Require().NoError(metaBuilder.SetDefaultSortOrderID(-1))
+
+ writeUUID := uuid.New()
+ args := recordWritingArgs{
+ sc: arrSchema,
+ fs: iceio.LocalFS{},
+ writeUUID: &writeUUID,
+ counter: func(yield func(int) bool) {
+ for i := 0; ; i++ {
+ if !yield(i) {
+ break
+ }
+ }
+ },
+ }
+
+ factory, err := newWriterFactory(loc, args, metaBuilder, icebergSchema, 1024*1024)
+ s.Require().NoError(err)
+ defer factory.closeAll()
+
+ bldr := array.NewRecordBuilder(s.mem, arrSchema)
+ defer bldr.Release()
+ idBldr := bldr.Field(0).(*array.Int32Builder)
+ nameBldr := bldr.Field(1).(*array.StringBuilder)
+ type row struct {
+ id int32
+ idOK bool
+ name string
+ nmOK bool
+ }
+ rows := []row{
+ {id: 3, idOK: true, name: "c", nmOK: true},
+ {id: 1, idOK: true, name: "b", nmOK: true},
+ {id: 1, idOK: true, name: "a", nmOK: true},
+ {idOK: false, name: "x", nmOK: true},
+ {id: 2, idOK: true, nmOK: false},
+ {id: 1, idOK: true, name: "z", nmOK: true},
+ }
+ for _, r := range rows {
+ if r.idOK {
+ idBldr.Append(r.id)
+ } else {
+ idBldr.AppendNull()
+ }
+ if r.nmOK {
+ nameBldr.Append(r.name)
+ } else {
+ nameBldr.AppendNull()
+ }
+ }
+ record := bldr.NewRecordBatch()
+ defer record.Release()
+
+ outputCh := make(chan iceberg.DataFile, 10)
+ writer := factory.newRollingDataWriter(s.ctx, "", nil, outputCh)
+ s.Require().NoError(writer.Add(record))
+ s.Require().NoError(writer.closeAndWait())
+ close(outputCh)
+
+ var dataFiles []iceberg.DataFile
+ for df := range outputCh {
+ dataFiles = append(dataFiles, df)
+ }
+ s.Require().Len(dataFiles, 1)
+
+ rdr, err := file.OpenParquetFile(dataFiles[0].FilePath(), false)
+ s.Require().NoError(err)
+ defer rdr.Close()
+ arrowRdr, err := pqarrow.NewFileReader(rdr, pqarrow.ArrowReadProperties{}, memory.DefaultAllocator)
+ s.Require().NoError(err)
+ tbl, err := arrowRdr.ReadTable(s.ctx)
+ s.Require().NoError(err)
+ defer tbl.Release()
+
+ s.Equal(int64(len(rows)), tbl.NumRows())
+
+ type observed struct {
+ id int32
+ idOK bool
+ name string
+ nmOK bool
+ }
+ var got []observed
+ idChunks := tbl.Column(0).Data().Chunks()
+ nameChunks := tbl.Column(1).Data().Chunks()
+ for c := range idChunks {
+ idArr := idChunks[c].(*array.Int32)
+ nameArr := nameChunks[c].(*array.String)
+ for i := 0; i < idArr.Len(); i++ {
+ o := observed{}
+ if idArr.IsValid(i) {
+ o.id = idArr.Value(i)
+ o.idOK = true
+ }
+ if nameArr.IsValid(i) {
+ o.name = nameArr.Value(i)
+ o.nmOK = true
+ }
+ got = append(got, o)
+ }
+ }
+
+ // Expected order: id ASC NULLS LAST, then name DESC NULLS FIRST.
+ want := []observed{
+ {id: 1, idOK: true, name: "z", nmOK: true},
+ {id: 1, idOK: true, name: "b", nmOK: true},
+ {id: 1, idOK: true, name: "a", nmOK: true},
+ {id: 2, idOK: true, nmOK: false},
+ {id: 3, idOK: true, name: "c", nmOK: true},
+ {idOK: false, name: "x", nmOK: true},
+ }
+ s.Equal(want, got)
+}
+
+// TestUnsortedOrderIsNoOp confirms that when the table has no sort order
+// (id 0), batches are written in arrival order — i.e. the sort path costs
+// nothing for unsorted tables.
+func (s *RollingDataWriterTestSuite) TestUnsortedOrderIsNoOp() {
+ arrSchema := arrow.NewSchema([]arrow.Field{
+ {Name: "id", Type: arrow.PrimitiveTypes.Int32, Nullable: true},
+ {Name: "name", Type: arrow.BinaryTypes.String, Nullable: true},
+ }, nil)
+
+ loc := filepath.ToSlash(s.T().TempDir())
+ factory, _ := s.createWriterFactory(loc, arrSchema, 1024*1024)
+ defer factory.closeAll()
+ s.Require().Nil(factory.sortKeys, "unsorted tables must skip sort key resolution")
+
+ bldr := array.NewRecordBuilder(s.mem, arrSchema)
+ defer bldr.Release()
+ for _, v := range []int32{3, 1, 2} {
+ bldr.Field(0).(*array.Int32Builder).Append(v)
+ bldr.Field(1).(*array.StringBuilder).Append("x")
+ }
+ record := bldr.NewRecordBatch()
+ defer record.Release()
+
+ outputCh := make(chan iceberg.DataFile, 10)
+ writer := factory.newRollingDataWriter(s.ctx, "", nil, outputCh)
+ s.Require().NoError(writer.Add(record))
+ s.Require().NoError(writer.closeAndWait())
+ close(outputCh)
+
+ var dataFiles []iceberg.DataFile
+ for df := range outputCh {
+ dataFiles = append(dataFiles, df)
+ }
+ s.Require().Len(dataFiles, 1)
+
+ rdr, err := file.OpenParquetFile(dataFiles[0].FilePath(), false)
+ s.Require().NoError(err)
+ defer rdr.Close()
+ arrowRdr, err := pqarrow.NewFileReader(rdr, pqarrow.ArrowReadProperties{}, memory.DefaultAllocator)
+ s.Require().NoError(err)
+ tbl, err := arrowRdr.ReadTable(s.ctx)
+ s.Require().NoError(err)
+ defer tbl.Release()
+
+ var ids []int32
+ for _, chunk := range tbl.Column(0).Data().Chunks() {
+ arr := chunk.(*array.Int32)
+ for i := 0; i < arr.Len(); i++ {
+ ids = append(ids, arr.Value(i))
+ }
+ }
+ s.Equal([]int32{3, 1, 2}, ids, "unsorted order must preserve arrival order")
+}
diff --git a/table/sort_keys.go b/table/sort_keys.go
new file mode 100644
index 00000000..11270050
--- /dev/null
+++ b/table/sort_keys.go
@@ -0,0 +1,80 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package table
+
+import (
+ "fmt"
+
+ "github.com/apache/arrow-go/v18/arrow/compute"
+ "github.com/apache/iceberg-go"
+)
+
+// resolveSortKeys converts an Iceberg SortOrder into Arrow compute sort keys
+// against the given file schema. Each sort field's source column is used as
+// the sort key directly: order-preserving transforms (identity, truncate,
+// year/month/day/hour) sort the same as their source values.
+//
+// Returns nil keys for an unsorted SortOrder. Returns an error when a sort
+// field's source id is not a top-level column of fileSchema.
+//
+// Limitations (these match PyIceberg's current behavior):
+//
+// - Per-batch, not per-file: the resolved keys are applied to each record
+// batch independently as it is flushed; rows are not merged into a single
+// globally sorted run across batches within a file.
+// - Bucket transform falls back to source-column order, since the hash
+// bucket is not order-preserving. The manifest entry still records the
+// table's sort_order_id verbatim.
+// - Multi-argument transform sort fields use only the first source column.
+// - Nested-field sort sources are not supported: a source id that is not a
+// top-level column of fileSchema is rejected with an error.
+func resolveSortKeys(order SortOrder, fileSchema *iceberg.Schema) ([]compute.SortKey, error) {
+ if order.IsUnsorted() {
+ return nil, nil
+ }
+
+ keys := make([]compute.SortKey, 0, order.Len())
+ for _, field := range order.fields {
+ idx, ok := topLevelFieldIndex(fileSchema, field.SourceID())
+ if !ok {
+ return nil, fmt.Errorf("sort order %d: source id %d is not a top-level column in schema",
+ order.OrderID(), field.SourceID())
+ }
+
+ key := compute.SortKey{ColumnIndex: idx, Order: compute.SortOrderAscending, NullPlacement: compute.SortNullsAtEnd}
+ if field.Direction == SortDESC {
+ key.Order = compute.SortOrderDescending
+ }
+ if field.NullOrder == NullsFirst {
+ key.NullPlacement = compute.SortNullsAtStart
+ }
+ keys = append(keys, key)
+ }
+
+ return keys, nil
+}
+
+func topLevelFieldIndex(schema *iceberg.Schema, sourceID int) (int, bool) {
+ for i, f := range schema.Fields() {
+ if f.ID == sourceID {
+ return i, true
+ }
+ }
+
+ return 0, false
+}
diff --git a/table/sort_keys_test.go b/table/sort_keys_test.go
new file mode 100644
index 00000000..709e54bd
--- /dev/null
+++ b/table/sort_keys_test.go
@@ -0,0 +1,75 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package table
+
+import (
+ "testing"
+
+ "github.com/apache/arrow-go/v18/arrow/compute"
+ "github.com/apache/iceberg-go"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+func TestResolveSortKeys_Unsorted(t *testing.T) {
+ schema := iceberg.NewSchema(0,
+ iceberg.NestedField{ID: 1, Name: "id", Type: iceberg.PrimitiveTypes.Int64, Required: true},
+ )
+ keys, err := resolveSortKeys(UnsortedSortOrder, schema)
+ require.NoError(t, err)
+ assert.Nil(t, keys, "unsorted order must produce no sort keys")
+}
+
+func TestResolveSortKeys_DirectionAndNullOrder(t *testing.T) {
+ schema := iceberg.NewSchema(0,
+ iceberg.NestedField{ID: 1, Name: "a", Type: iceberg.PrimitiveTypes.Int64, Required: false},
+ iceberg.NestedField{ID: 2, Name: "b", Type: iceberg.PrimitiveTypes.String, Required: false},
+ )
+
+ order, err := NewSortOrder(1, []SortField{
+ {SourceIDs: []int{2}, Transform: iceberg.IdentityTransform{}, Direction: SortDESC, NullOrder: NullsFirst},
+ {SourceIDs: []int{1}, Transform: iceberg.IdentityTransform{}, Direction: SortASC, NullOrder: NullsLast},
+ })
+ require.NoError(t, err)
+
+ keys, err := resolveSortKeys(order, schema)
+ require.NoError(t, err)
+ require.Len(t, keys, 2)
+
+ assert.Equal(t, 1, keys[0].ColumnIndex, "first key should target column index 1 (source id 2)")
+ assert.Equal(t, compute.SortOrderDescending, keys[0].Order)
+ assert.Equal(t, compute.SortNullsAtStart, keys[0].NullPlacement)
+
+ assert.Equal(t, 0, keys[1].ColumnIndex, "second key should target column index 0 (source id 1)")
+ assert.Equal(t, compute.SortOrderAscending, keys[1].Order)
+ assert.Equal(t, compute.SortNullsAtEnd, keys[1].NullPlacement)
+}
+
+func TestResolveSortKeys_MissingSourceID(t *testing.T) {
+ schema := iceberg.NewSchema(0,
+ iceberg.NestedField{ID: 1, Name: "a", Type: iceberg.PrimitiveTypes.Int64, Required: true},
+ )
+ order, err := NewSortOrder(1, []SortField{
+ {SourceIDs: []int{99}, Transform: iceberg.IdentityTransform{}, Direction: SortASC, NullOrder: NullsFirst},
+ })
+ require.NoError(t, err)
+
+ _, err = resolveSortKeys(order, schema)
+ require.Error(t, err)
+ assert.Contains(t, err.Error(), "source id 99")
+}
diff --git a/table/write_records_sort_test.go b/table/write_records_sort_test.go
new file mode 100644
index 00000000..c4721d26
--- /dev/null
+++ b/table/write_records_sort_test.go
@@ -0,0 +1,260 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package table_test
+
+import (
+ "context"
+ "path/filepath"
+ "sort"
+ "testing"
+
+ "github.com/apache/arrow-go/v18/arrow"
+ "github.com/apache/arrow-go/v18/arrow/array"
+ "github.com/apache/arrow-go/v18/arrow/compute"
+ "github.com/apache/arrow-go/v18/arrow/memory"
+ "github.com/apache/arrow-go/v18/parquet/file"
+ "github.com/apache/arrow-go/v18/parquet/pqarrow"
+ "github.com/apache/iceberg-go"
+ iceio "github.com/apache/iceberg-go/io"
+ "github.com/apache/iceberg-go/table"
+ "github.com/stretchr/testify/require"
+ "github.com/stretchr/testify/suite"
+)
+
+type WriteRecordsSortTestSuite struct {
+ suite.Suite
+
+ mem *memory.CheckedAllocator
+ ctx context.Context
+}
+
+func TestWriteRecordsSort(t *testing.T) {
+ suite.Run(t, new(WriteRecordsSortTestSuite))
+}
+
+func (s *WriteRecordsSortTestSuite) SetupTest() {
+ s.mem = memory.NewCheckedAllocator(memory.DefaultAllocator)
+ s.ctx = compute.WithAllocator(context.Background(), s.mem)
+}
+
+func (s *WriteRecordsSortTestSuite) TearDownTest() {
+ s.mem.AssertSize(s.T(), 0)
+}
+
+// TestPartitionedTableSortedPerFile exercises the hash-fanout write path:
+// a partition-by-category table with a sort order on id ASC must produce
+// one data file per partition, each with rows sorted by id.
+func (s *WriteRecordsSortTestSuite) TestPartitionedTableSortedPerFile() {
+ loc := filepath.ToSlash(s.T().TempDir())
+
+ iceSch := iceberg.NewSchema(0,
+ iceberg.NestedField{ID: 1, Name: "id", Type: iceberg.PrimitiveTypes.Int32, Required: false},
+ iceberg.NestedField{ID: 2, Name: "category", Type: iceberg.PrimitiveTypes.String, Required: true},
+ )
+ spec := iceberg.NewPartitionSpec(
+ iceberg.PartitionField{SourceIDs: []int{2}, FieldID: 1000, Name: "category", Transform: iceberg.IdentityTransform{}},
+ )
+ sortOrder, err := table.NewSortOrder(1, []table.SortField{
+ {SourceIDs: []int{1}, Transform: iceberg.IdentityTransform{}, Direction: table.SortASC, NullOrder: table.NullsLast},
+ })
+ s.Require().NoError(err)
+
+ meta, err := table.NewMetadata(iceSch, &spec, sortOrder, loc, iceberg.Properties{
+ table.PropertyFormatVersion: "2",
+ })
+ s.Require().NoError(err)
+
+ tbl := table.New(
+ table.Identifier{"db", "partitioned_sorted"},
+ meta,
+ filepath.Join(loc, "metadata", "v1.metadata.json"),
+ func(ctx context.Context) (iceio.IO, error) { return iceio.LocalFS{}, nil },
+ nil,
+ )
+
+ arrSchema := arrow.NewSchema([]arrow.Field{
+ {Name: "id", Type: arrow.PrimitiveTypes.Int32, Nullable: true},
+ {Name: "category", Type: arrow.BinaryTypes.String, Nullable: false},
+ }, nil)
+
+ // Mixed partitions, rows intentionally unsorted within and across.
+ type row struct {
+ id int32
+ cat string
+ }
+ rows := []row{
+ {5, "a"},
+ {2, "b"},
+ {1, "a"},
+ {7, "b"},
+ {3, "a"},
+ {4, "b"},
+ {6, "a"},
+ {0, "b"},
+ }
+ bldr := array.NewRecordBuilder(s.mem, arrSchema)
+ for _, r := range rows {
+ bldr.Field(0).(*array.Int32Builder).Append(r.id)
+ bldr.Field(1).(*array.StringBuilder).Append(r.cat)
+ }
+ rec := bldr.NewRecordBatch()
+ bldr.Release()
+
+ records := func(yield func(arrow.RecordBatch, error) bool) {
+ yield(rec, nil)
+ }
+
+ var dataFiles []iceberg.DataFile
+ for df, err := range table.WriteRecords(s.ctx, tbl, arrSchema, records) {
+ s.Require().NoError(err)
+ dataFiles = append(dataFiles, df)
+ }
+ s.Require().Len(dataFiles, 2, "expected one file per partition")
+
+ // Group expected ids by category for cross-checking.
+ expectedByCat := map[string][]int32{}
+ for _, r := range rows {
+ expectedByCat[r.cat] = append(expectedByCat[r.cat], r.id)
+ }
+ for cat := range expectedByCat {
+ sort.Slice(expectedByCat[cat], func(i, j int) bool {
+ return expectedByCat[cat][i] < expectedByCat[cat][j]
+ })
+ }
+
+ for _, df := range dataFiles {
+ s.NotNil(df.SortOrderID(), "data file must carry the sort order id")
+ s.Equal(meta.DefaultSortOrder(), *df.SortOrderID())
+
+ ids, cats := readIDsAndCategories(s.T(), df.FilePath())
+ s.Require().NotEmpty(cats, "file should have at least one row")
+
+ // All rows in a partitioned file share the partition value.
+ cat := cats[0]
+ for _, c := range cats {
+ s.Equal(cat, c, "partitioned file should contain a single category")
+ }
+
+ s.Equal(expectedByCat[cat], ids, "rows must be sorted by id ASC within partition %q", cat)
+ }
+}
+
+// TestUnpartitionedTableMultipleBatchesEachSorted documents the per-batch
+// sort guarantee: each Add() call is locally sorted, but rows aren't merged
+// across batches. Two unsorted batches in arrival order produce a file with
+// two locally-sorted runs concatenated. This matches PyIceberg's behavior.
+func (s *WriteRecordsSortTestSuite) TestUnpartitionedTableMultipleBatchesEachSorted() {
+ loc := filepath.ToSlash(s.T().TempDir())
+
+ iceSch := iceberg.NewSchema(0,
+ iceberg.NestedField{ID: 1, Name: "id", Type: iceberg.PrimitiveTypes.Int32, Required: false},
+ )
+ spec := iceberg.NewPartitionSpec()
+ sortOrder, err := table.NewSortOrder(1, []table.SortField{
+ {SourceIDs: []int{1}, Transform: iceberg.IdentityTransform{}, Direction: table.SortASC, NullOrder: table.NullsLast},
+ })
+ s.Require().NoError(err)
+ meta, err := table.NewMetadata(iceSch, &spec, sortOrder, loc, iceberg.Properties{})
+ s.Require().NoError(err)
+
+ tbl := table.New(
+ table.Identifier{"db", "unpartitioned_sorted"},
+ meta,
+ filepath.Join(loc, "metadata", "v1.metadata.json"),
+ func(ctx context.Context) (iceio.IO, error) { return iceio.LocalFS{}, nil },
+ nil,
+ )
+
+ arrSchema := arrow.NewSchema([]arrow.Field{
+ {Name: "id", Type: arrow.PrimitiveTypes.Int32, Nullable: true},
+ }, nil)
+
+ makeBatch := func(vals []int32) arrow.RecordBatch {
+ bldr := array.NewRecordBuilder(s.mem, arrSchema)
+ for _, v := range vals {
+ bldr.Field(0).(*array.Int32Builder).Append(v)
+ }
+ rec := bldr.NewRecordBatch()
+ bldr.Release()
+
+ return rec
+ }
+
+ batch1 := makeBatch([]int32{3, 1, 2})
+ batch2 := makeBatch([]int32{6, 4, 5})
+ records := func(yield func(arrow.RecordBatch, error) bool) {
+ if !yield(batch1, nil) {
+ return
+ }
+ yield(batch2, nil)
+ }
+
+ var dataFiles []iceberg.DataFile
+ for df, err := range table.WriteRecords(s.ctx, tbl, arrSchema, records) {
+ s.Require().NoError(err)
+ dataFiles = append(dataFiles, df)
+ }
+ s.Require().Len(dataFiles, 1)
+
+ ids, _ := readIDsAndCategories(s.T(), dataFiles[0].FilePath())
+ // Two locally-sorted runs concatenated: [1,2,3] then [4,5,6]. The
+ // boundary between them isn't a global sort, but each run is sorted.
+ s.Equal([]int32{1, 2, 3, 4, 5, 6}, ids,
+ "each batch is sorted independently and emitted in arrival order")
+}
+
+func readIDsAndCategories(t *testing.T, path string) ([]int32, []string) {
+ t.Helper()
+ rdr, err := file.OpenParquetFile(path, false)
+ require.NoError(t, err)
+ defer rdr.Close()
+
+ arrowRdr, err := pqarrow.NewFileReader(rdr, pqarrow.ArrowReadProperties{}, memory.DefaultAllocator)
+ require.NoError(t, err)
+ tbl, err := arrowRdr.ReadTable(context.Background())
+ require.NoError(t, err)
+ defer tbl.Release()
+
+ var ids []int32
+ var cats []string
+
+ idCol := tbl.Schema().FieldIndices("id")
+ if len(idCol) > 0 {
+ for _, chunk := range tbl.Column(idCol[0]).Data().Chunks() {
+ arr := chunk.(*array.Int32)
+ for i := 0; i < arr.Len(); i++ {
+ if arr.IsValid(i) {
+ ids = append(ids, arr.Value(i))
+ }
+ }
+ }
+ }
+ catCol := tbl.Schema().FieldIndices("category")
+ if len(catCol) > 0 {
+ for _, chunk := range tbl.Column(catCol[0]).Data().Chunks() {
+ arr := chunk.(*array.String)
+ for i := 0; i < arr.Len(); i++ {
+ if arr.IsValid(i) {
+ cats = append(cats, arr.Value(i))
+ }
+ }
+ }
+ }
+
+ return ids, cats
+}