You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by ze...@apache.org on 2026/08/19 20:45:45 UTC
(iceberg-go) branch main updated: feat(table): add all manifests metadata table (#1745)
This is an automated email from the ASF dual-hosted git repository.
zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-go.git
The following commit(s) were added to refs/heads/main by this push:
new a3829e8d6 feat(table): add all manifests metadata table (#1745)
a3829e8d6 is described below
commit a3829e8d61df0457b2251d979dc733ec6dad8edf
Author: Minh Vu <vu...@gmail.com>
AuthorDate: Wed Aug 19 22:45:40 2026 +0200
feat(table): add all manifests metadata table (#1745)
* feat(table): add manifests metadata table
* chore: fix lint findings
* fix: match manifest partition summary schema
* test(table): cover manifest metadata inspection
* fix(table): validate manifest metadata inspection
* fix(table): decode promoted manifest partition bounds
* fix(table): satisfy manifest inspection formatter
* fix(table): validate manifest list counts
* feat(table): add all manifests metadata table
Signed-off-by: Minh Vu <vu...@gmail.com>
* fix(table): stream all manifests inspection batches
Signed-off-by: Minh Vu <vu...@gmail.com>
* test(table): cover all manifests reader abandonment
Signed-off-by: Minh Vu <vu...@gmail.com>
---------
Signed-off-by: Minh Vu <vu...@gmail.com>
---
table/inspect.go | 65 ++++++++++++++------
table/inspect_all_manifests.go | 134 +++++++++++++++++++++++++++++++++++++++++
table/inspect_internal_test.go | 132 ++++++++++++++++++++++++++++++++++++++++
3 files changed, 312 insertions(+), 19 deletions(-)
diff --git a/table/inspect.go b/table/inspect.go
index 7274f0e4c..1b2475645 100644
--- a/table/inspect.go
+++ b/table/inspect.go
@@ -270,6 +270,24 @@ func (i InspectTable) Manifests(ctx context.Context) (array.RecordReader, error)
bldr := array.NewRecordBuilder(i.alloc, arrowSchema)
defer bldr.Release()
+ if err := i.appendManifestRows(ctx, bldr, manifests, nil); err != nil {
+ return nil, fmt.Errorf("inspect manifests: %w", err)
+ }
+
+ rr, err := singleBatchReader(arrowSchema, bldr)
+ if err != nil {
+ return nil, fmt.Errorf("inspect manifests: %w", err)
+ }
+
+ return rr, nil
+}
+
+func (i InspectTable) appendManifestRows(
+ ctx context.Context,
+ bldr *array.RecordBuilder,
+ manifests []iceberg.ManifestFile,
+ referenceSnapshotID *int64,
+) error {
content := bldr.Field(0).(*array.Int32Builder)
path := bldr.Field(1).(*array.StringBuilder)
length := bldr.Field(2).(*array.Int64Builder)
@@ -287,21 +305,27 @@ func (i InspectTable) Manifests(ctx context.Context) (array.RecordReader, error)
summaryContainsNaN := summaryStruct.FieldBuilder(1).(*array.BooleanBuilder)
summaryLower := summaryStruct.FieldBuilder(2).(*array.StringBuilder)
summaryUpper := summaryStruct.FieldBuilder(3).(*array.StringBuilder)
+ var referenceSnapshot *array.Int64Builder
+ var keyMetadata *array.BinaryBuilder
+ if referenceSnapshotID != nil {
+ referenceSnapshot = bldr.Field(12).(*array.Int64Builder)
+ keyMetadata = bldr.Field(13).(*array.BinaryBuilder)
+ }
for _, manifest := range manifests {
if err := ctx.Err(); err != nil {
- return nil, err
+ return err
}
manifestContent := manifest.ManifestContent()
switch manifestContent {
case iceberg.ManifestContentData, iceberg.ManifestContentDeletes:
default:
- return nil, fmt.Errorf("manifest %s has unknown content %d", manifest.FilePath(), manifestContent)
+ return fmt.Errorf("manifest %s has unknown content %d", manifest.FilePath(), manifestContent)
}
snapshotID := manifest.SnapshotID()
if snapshotID < 0 {
- return nil, fmt.Errorf("manifest %s has negative added_snapshot_id %d", manifest.FilePath(), snapshotID)
+ return fmt.Errorf("manifest %s has negative added_snapshot_id %d", manifest.FilePath(), snapshotID)
}
content.Append(int32(manifestContent))
@@ -320,13 +344,13 @@ func (i InspectTable) Manifests(ctx context.Context) (array.RecordReader, error)
switch manifestContent {
case iceberg.ManifestContentData:
if err := appendCount(addedDataFiles, "added_data_files", manifest.AddedDataFiles()); err != nil {
- return nil, err
+ return err
}
if err := appendCount(existingDataFiles, "existing_data_files", manifest.ExistingDataFiles()); err != nil {
- return nil, err
+ return err
}
if err := appendCount(deletedDataFiles, "deleted_data_files", manifest.DeletedDataFiles()); err != nil {
- return nil, err
+ return err
}
addedDeleteFiles.Append(0)
existingDeleteFiles.Append(0)
@@ -338,13 +362,21 @@ func (i InspectTable) Manifests(ctx context.Context) (array.RecordReader, error)
// ManifestFile's data-file accessors expose the generic manifest-list
// file counts, which represent delete files for delete manifests.
if err := appendCount(addedDeleteFiles, "added_delete_files", manifest.AddedDataFiles()); err != nil {
- return nil, err
+ return err
}
if err := appendCount(existingDeleteFiles, "existing_delete_files", manifest.ExistingDataFiles()); err != nil {
- return nil, err
+ return err
}
if err := appendCount(deletedDeleteFiles, "deleted_delete_files", manifest.DeletedDataFiles()); err != nil {
- return nil, err
+ return err
+ }
+ }
+ if referenceSnapshotID != nil {
+ referenceSnapshot.Append(*referenceSnapshotID)
+ if metadata := manifest.KeyMetadata(); metadata != nil {
+ keyMetadata.Append(metadata)
+ } else {
+ keyMetadata.AppendNull()
}
}
@@ -357,12 +389,12 @@ func (i InspectTable) Manifests(ctx context.Context) (array.RecordReader, error)
spec := i.tbl.metadata.PartitionSpecByID(int(manifest.PartitionSpecID()))
if spec == nil {
- return nil, fmt.Errorf("manifest %s references missing partition spec %d",
+ return fmt.Errorf("manifest %s references missing partition spec %d",
manifest.FilePath(), manifest.PartitionSpecID())
}
partType := spec.PartitionType(i.tbl.metadata.CurrentSchema())
if len(partitions) > spec.NumFields() {
- return nil, fmt.Errorf("manifest %s has %d partition summaries for partition spec %d with %d fields",
+ return fmt.Errorf("manifest %s has %d partition summaries for partition spec %d with %d fields",
manifest.FilePath(), len(partitions), manifest.PartitionSpecID(), spec.NumFields())
}
@@ -379,22 +411,17 @@ func (i InspectTable) Manifests(ctx context.Context) (array.RecordReader, error)
fieldType := partType.FieldList[idx].Type
transform := spec.Field(idx).Transform
if err := appendManifestBound(summaryLower, fieldType, transform, summary.LowerBound); err != nil {
- return nil, fmt.Errorf("manifest %s partition field %q lower bound: %w",
+ return fmt.Errorf("manifest %s partition field %q lower bound: %w",
manifest.FilePath(), partType.FieldList[idx].Name, err)
}
if err := appendManifestBound(summaryUpper, fieldType, transform, summary.UpperBound); err != nil {
- return nil, fmt.Errorf("manifest %s partition field %q upper bound: %w",
+ return fmt.Errorf("manifest %s partition field %q upper bound: %w",
manifest.FilePath(), partType.FieldList[idx].Name, err)
}
}
}
- rr, err := singleBatchReader(arrowSchema, bldr)
- if err != nil {
- return nil, fmt.Errorf("inspect manifests: %w", err)
- }
-
- return rr, nil
+ return nil
}
func appendManifestCount(builder *array.Int32Builder, version int, name string, count int32) error {
diff --git a/table/inspect_all_manifests.go b/table/inspect_all_manifests.go
new file mode 100644
index 000000000..1c59ecd02
--- /dev/null
+++ b/table/inspect_all_manifests.go
@@ -0,0 +1,134 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package table
+
+import (
+ "context"
+ "errors"
+ "fmt"
+
+ "github.com/apache/arrow-go/v18/arrow"
+ "github.com/apache/arrow-go/v18/arrow/array"
+ "github.com/apache/iceberg-go"
+)
+
+// AllManifests returns every manifest referenced by every snapshot currently
+// tracked by the table. A manifest referenced by multiple snapshots produces
+// one row per reference, identified by reference_snapshot_id.
+// Results are streamed in bounded record batches while each snapshot's
+// manifest list is traversed.
+func (i InspectTable) AllManifests(ctx context.Context) (array.RecordReader, error) {
+ arrowSchema, err := SchemaToArrowSchema(AllManifestsSchema(), nil, true, false)
+ if err != nil {
+ return nil, fmt.Errorf("inspect all manifests: build arrow schema: %w", err)
+ }
+
+ snapshots := i.tbl.metadata.Snapshots()
+ var readSnapshotManifests func(Snapshot) ([]iceberg.ManifestFile, error)
+ if len(snapshots) > 0 {
+ if i.tbl.fsF == nil {
+ return nil, errors.New("inspect all manifests: table file IO is not configured")
+ }
+ fs, err := i.tbl.fsF(ctx)
+ if err != nil {
+ return nil, fmt.Errorf("inspect all manifests: %w", err)
+ }
+ readSnapshotManifests = func(snapshot Snapshot) ([]iceberg.ManifestFile, error) {
+ return snapshot.Manifests(fs)
+ }
+ }
+
+ return array.ReaderFromIter(arrowSchema, func(yield func(arrow.RecordBatch, error) bool) {
+ bldr := array.NewRecordBuilder(i.alloc, arrowSchema)
+ defer bldr.Release()
+
+ rows := 0
+ emitted := false
+ emit := func() bool {
+ if rows == 0 {
+ return true
+ }
+
+ batch := bldr.NewRecordBatch()
+ rows = 0
+ emitted = true
+
+ return yield(batch, nil)
+ }
+ emitEmpty := func() {
+ batch := bldr.NewRecordBatch()
+ emitted = true
+ _ = yield(batch, nil)
+ }
+ yieldError := func(err error) {
+ _ = yield(nil, err)
+ }
+
+ for _, snapshot := range snapshots {
+ if err := ctx.Err(); err != nil {
+ yieldError(err)
+
+ return
+ }
+
+ manifests, err := readSnapshotManifests(snapshot)
+ if err != nil {
+ yieldError(fmt.Errorf("inspect all manifests: read snapshot %d manifests: %w",
+ snapshot.SnapshotID, err))
+
+ return
+ }
+
+ referenceSnapshotID := snapshot.SnapshotID
+ for start := 0; start < len(manifests); {
+ end := min(start+inspectRecordBatchSize-rows, len(manifests))
+ if err := i.appendManifestRows(ctx, bldr, manifests[start:end], &referenceSnapshotID); err != nil {
+ yieldError(fmt.Errorf("inspect all manifests: snapshot %d: %w", snapshot.SnapshotID, err))
+
+ return
+ }
+ rows += end - start
+ start = end
+
+ if rows == inspectRecordBatchSize && !emit() {
+ return
+ }
+ }
+ }
+
+ if rows > 0 {
+ _ = emit()
+ } else if !emitted {
+ emitEmpty()
+ }
+ }), nil
+}
+
+// AllManifestsSchema returns the schema of the all_manifests metadata table.
+func AllManifestsSchema() *iceberg.Schema {
+ fields := ManifestsSchema().Fields()
+ fields = append(fields,
+ iceberg.NestedField{
+ ID: 18, Name: "reference_snapshot_id", Type: iceberg.PrimitiveTypes.Int64, Required: true,
+ },
+ iceberg.NestedField{
+ ID: 19, Name: "key_metadata", Type: iceberg.PrimitiveTypes.Binary, Required: false,
+ },
+ )
+
+ return iceberg.NewSchema(0, fields...)
+}
diff --git a/table/inspect_internal_test.go b/table/inspect_internal_test.go
index 2444cf700..9aceb3dfe 100644
--- a/table/inspect_internal_test.go
+++ b/table/inspect_internal_test.go
@@ -1165,6 +1165,138 @@ func TestInspectManifestsSchema(t *testing.T) {
require.Equal(t, 13, partitionSummary.FieldList[3].ID)
}
+func TestInspectAllManifestsSchema(t *testing.T) {
+ sc := AllManifestsSchema()
+ require.Equal(t,
+ append(testFieldNames(ManifestsSchema()), "reference_snapshot_id", "key_metadata"),
+ testFieldNames(sc))
+
+ fields := sc.Fields()
+ require.Equal(t, 18, fields[12].ID)
+ require.True(t, fields[12].Required)
+ require.Equal(t, iceberg.PrimitiveTypes.Int64, fields[12].Type)
+ require.Equal(t, 19, fields[13].ID)
+ require.False(t, fields[13].Required)
+ require.Equal(t, iceberg.PrimitiveTypes.Binary, fields[13].Type)
+}
+
+func TestInspectAllManifestsPreservesSnapshotReferences(t *testing.T) {
+ memFS := iceio.NewMemFS()
+ schema := simpleSchema()
+ const tableLocation = "mem://default/inspect-all-manifests"
+ meta, err := NewMetadata(schema, iceberg.UnpartitionedSpec, UnsortedSortOrder, tableLocation,
+ iceberg.Properties{PropertyFormatVersion: "2"})
+ require.NoError(t, err)
+ builder, err := MetadataBuilderFromBase(meta, "")
+ require.NoError(t, err)
+
+ manifest := iceberg.NewManifestFile(2, tableLocation+"/metadata/shared-manifest.avro", 100, 0, 1).
+ SequenceNum(1, 1).
+ AddedFiles(1).
+ KeyMetadata([]byte{1, 2, 3}).
+ Build()
+ manifestListPath := tableLocation + "/metadata/shared-list.avro"
+ var list bytes.Buffer
+ sequenceNumber := int64(1)
+ require.NoError(t, iceberg.WriteManifestList(2, &list, 1, nil, &sequenceNumber, 0,
+ []iceberg.ManifestFile{manifest}))
+ require.NoError(t, memFS.WriteFile(manifestListPath, list.Bytes()))
+
+ schemaID := schema.ID
+ snapshotOne := int64(1)
+ timestamp := meta.LastUpdatedMillis()
+ require.NoError(t, builder.AddSnapshot(&Snapshot{
+ SnapshotID: snapshotOne, SequenceNumber: 1, TimestampMs: timestamp + 1,
+ ManifestList: manifestListPath, SchemaID: &schemaID,
+ }))
+ require.NoError(t, builder.AddSnapshot(&Snapshot{
+ SnapshotID: 2, ParentSnapshotID: &snapshotOne, SequenceNumber: 2, TimestampMs: timestamp + 2,
+ ManifestList: manifestListPath, SchemaID: &schemaID,
+ }))
+ require.NoError(t, builder.SetSnapshotRef(MainBranch, 2, BranchRef))
+ built, err := builder.Build()
+ require.NoError(t, err)
+ tbl := New(Identifier{"db", "all_manifests"}, built, tableLocation+"/metadata/metadata.json",
+ func(context.Context) (iceio.IO, error) { return memFS, nil }, nil)
+
+ rr, err := tbl.Inspect().AllManifests(context.Background())
+ require.NoError(t, err)
+ defer rr.Release()
+ record := collectRecord(t, rr)
+ defer record.Release()
+
+ require.EqualValues(t, 2, record.NumRows())
+ paths := record.Column(1).(*array.String)
+ require.Equal(t, manifest.FilePath(), paths.Value(0))
+ require.Equal(t, manifest.FilePath(), paths.Value(1))
+ references := record.Column(12).(*array.Int64)
+ require.Equal(t, []int64{1, 2}, references.Int64Values())
+ keyMetadata := record.Column(13).(*array.Binary)
+ require.Equal(t, []byte{1, 2, 3}, keyMetadata.Value(0))
+ require.Equal(t, []byte{1, 2, 3}, keyMetadata.Value(1))
+}
+
+func TestInspectAllManifestsStreamsRecordBatches(t *testing.T) {
+ const manifestCount = inspectRecordBatchSize + 1
+
+ manifests := make([]iceberg.ManifestFile, manifestCount)
+ for idx := range manifests {
+ manifests[idx] = iceberg.NewManifestFile(2,
+ fmt.Sprintf("mem://default/table-location/metadata/manifest-%04d.avro", idx),
+ 100, 0, 1).
+ SequenceNum(1, 1).
+ AddedFiles(1).
+ Build()
+ }
+ tbl := inspectTableWithManifestList(t, *iceberg.UnpartitionedSpec, 2, manifests)
+
+ rr, err := tbl.Inspect().AllManifests(context.Background())
+ require.NoError(t, err)
+ defer rr.Release()
+
+ var batchSizes []int64
+ for rr.Next() {
+ batchSizes = append(batchSizes, rr.RecordBatch().NumRows())
+ }
+ require.NoError(t, rr.Err())
+ require.Equal(t, []int64{inspectRecordBatchSize, 1}, batchSizes)
+}
+
+func TestInspectAllManifestsAllocator(t *testing.T) {
+ for _, tt := range []struct {
+ name string
+ manifestCount int
+ next bool
+ }{
+ {name: "before first batch", manifestCount: 1},
+ {name: "after first batch", manifestCount: inspectRecordBatchSize + 1, next: true},
+ } {
+ t.Run(tt.name, func(t *testing.T) {
+ checked := memory.NewCheckedAllocator(memory.DefaultAllocator)
+ t.Cleanup(func() { checked.AssertSize(t, 0) })
+
+ manifests := make([]iceberg.ManifestFile, tt.manifestCount)
+ for idx := range manifests {
+ manifests[idx] = iceberg.NewManifestFile(2,
+ fmt.Sprintf("mem://default/table-location/metadata/manifest-%04d.avro", idx),
+ 100, 0, 1).
+ SequenceNum(1, 1).
+ AddedFiles(1).
+ Build()
+ }
+ tbl := inspectTableWithManifestList(t, *iceberg.UnpartitionedSpec, 2,
+ manifests)
+
+ rr, err := tbl.Inspect(WithInspectAllocator(checked)).AllManifests(context.Background())
+ require.NoError(t, err)
+ if tt.next {
+ require.True(t, rr.Next())
+ }
+ rr.Release()
+ })
+ }
+}
+
func TestInspectManifests(t *testing.T) {
const snapshotID = int64(1)
spec := partitionedSpec()