You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by ze...@apache.org on 2026/08/19 20:45:45 UTC

(iceberg-go) branch main updated: feat(table): add all manifests metadata table (#1745)

This is an automated email from the ASF dual-hosted git repository.

zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-go.git


The following commit(s) were added to refs/heads/main by this push:
     new a3829e8d6 feat(table): add all manifests metadata table (#1745)
a3829e8d6 is described below

commit a3829e8d61df0457b2251d979dc733ec6dad8edf
Author: Minh Vu <vu...@gmail.com>
AuthorDate: Wed Aug 19 22:45:40 2026 +0200

    feat(table): add all manifests metadata table (#1745)
    
    * feat(table): add manifests metadata table
    
    * chore: fix lint findings
    
    * fix: match manifest partition summary schema
    
    * test(table): cover manifest metadata inspection
    
    * fix(table): validate manifest metadata inspection
    
    * fix(table): decode promoted manifest partition bounds
    
    * fix(table): satisfy manifest inspection formatter
    
    * fix(table): validate manifest list counts
    
    * feat(table): add all manifests metadata table
    
    Signed-off-by: Minh Vu <vu...@gmail.com>
    
    * fix(table): stream all manifests inspection batches
    
    Signed-off-by: Minh Vu <vu...@gmail.com>
    
    * test(table): cover all manifests reader abandonment
    
    Signed-off-by: Minh Vu <vu...@gmail.com>
    
    ---------
    
    Signed-off-by: Minh Vu <vu...@gmail.com>
---
 table/inspect.go               |  65 ++++++++++++++------
 table/inspect_all_manifests.go | 134 +++++++++++++++++++++++++++++++++++++++++
 table/inspect_internal_test.go | 132 ++++++++++++++++++++++++++++++++++++++++
 3 files changed, 312 insertions(+), 19 deletions(-)

diff --git a/table/inspect.go b/table/inspect.go
index 7274f0e4c..1b2475645 100644
--- a/table/inspect.go
+++ b/table/inspect.go
@@ -270,6 +270,24 @@ func (i InspectTable) Manifests(ctx context.Context) (array.RecordReader, error)
 	bldr := array.NewRecordBuilder(i.alloc, arrowSchema)
 	defer bldr.Release()
 
+	if err := i.appendManifestRows(ctx, bldr, manifests, nil); err != nil {
+		return nil, fmt.Errorf("inspect manifests: %w", err)
+	}
+
+	rr, err := singleBatchReader(arrowSchema, bldr)
+	if err != nil {
+		return nil, fmt.Errorf("inspect manifests: %w", err)
+	}
+
+	return rr, nil
+}
+
+func (i InspectTable) appendManifestRows(
+	ctx context.Context,
+	bldr *array.RecordBuilder,
+	manifests []iceberg.ManifestFile,
+	referenceSnapshotID *int64,
+) error {
 	content := bldr.Field(0).(*array.Int32Builder)
 	path := bldr.Field(1).(*array.StringBuilder)
 	length := bldr.Field(2).(*array.Int64Builder)
@@ -287,21 +305,27 @@ func (i InspectTable) Manifests(ctx context.Context) (array.RecordReader, error)
 	summaryContainsNaN := summaryStruct.FieldBuilder(1).(*array.BooleanBuilder)
 	summaryLower := summaryStruct.FieldBuilder(2).(*array.StringBuilder)
 	summaryUpper := summaryStruct.FieldBuilder(3).(*array.StringBuilder)
+	var referenceSnapshot *array.Int64Builder
+	var keyMetadata *array.BinaryBuilder
+	if referenceSnapshotID != nil {
+		referenceSnapshot = bldr.Field(12).(*array.Int64Builder)
+		keyMetadata = bldr.Field(13).(*array.BinaryBuilder)
+	}
 
 	for _, manifest := range manifests {
 		if err := ctx.Err(); err != nil {
-			return nil, err
+			return err
 		}
 
 		manifestContent := manifest.ManifestContent()
 		switch manifestContent {
 		case iceberg.ManifestContentData, iceberg.ManifestContentDeletes:
 		default:
-			return nil, fmt.Errorf("manifest %s has unknown content %d", manifest.FilePath(), manifestContent)
+			return fmt.Errorf("manifest %s has unknown content %d", manifest.FilePath(), manifestContent)
 		}
 		snapshotID := manifest.SnapshotID()
 		if snapshotID < 0 {
-			return nil, fmt.Errorf("manifest %s has negative added_snapshot_id %d", manifest.FilePath(), snapshotID)
+			return fmt.Errorf("manifest %s has negative added_snapshot_id %d", manifest.FilePath(), snapshotID)
 		}
 
 		content.Append(int32(manifestContent))
@@ -320,13 +344,13 @@ func (i InspectTable) Manifests(ctx context.Context) (array.RecordReader, error)
 		switch manifestContent {
 		case iceberg.ManifestContentData:
 			if err := appendCount(addedDataFiles, "added_data_files", manifest.AddedDataFiles()); err != nil {
-				return nil, err
+				return err
 			}
 			if err := appendCount(existingDataFiles, "existing_data_files", manifest.ExistingDataFiles()); err != nil {
-				return nil, err
+				return err
 			}
 			if err := appendCount(deletedDataFiles, "deleted_data_files", manifest.DeletedDataFiles()); err != nil {
-				return nil, err
+				return err
 			}
 			addedDeleteFiles.Append(0)
 			existingDeleteFiles.Append(0)
@@ -338,13 +362,21 @@ func (i InspectTable) Manifests(ctx context.Context) (array.RecordReader, error)
 			// ManifestFile's data-file accessors expose the generic manifest-list
 			// file counts, which represent delete files for delete manifests.
 			if err := appendCount(addedDeleteFiles, "added_delete_files", manifest.AddedDataFiles()); err != nil {
-				return nil, err
+				return err
 			}
 			if err := appendCount(existingDeleteFiles, "existing_delete_files", manifest.ExistingDataFiles()); err != nil {
-				return nil, err
+				return err
 			}
 			if err := appendCount(deletedDeleteFiles, "deleted_delete_files", manifest.DeletedDataFiles()); err != nil {
-				return nil, err
+				return err
+			}
+		}
+		if referenceSnapshotID != nil {
+			referenceSnapshot.Append(*referenceSnapshotID)
+			if metadata := manifest.KeyMetadata(); metadata != nil {
+				keyMetadata.Append(metadata)
+			} else {
+				keyMetadata.AppendNull()
 			}
 		}
 
@@ -357,12 +389,12 @@ func (i InspectTable) Manifests(ctx context.Context) (array.RecordReader, error)
 
 		spec := i.tbl.metadata.PartitionSpecByID(int(manifest.PartitionSpecID()))
 		if spec == nil {
-			return nil, fmt.Errorf("manifest %s references missing partition spec %d",
+			return fmt.Errorf("manifest %s references missing partition spec %d",
 				manifest.FilePath(), manifest.PartitionSpecID())
 		}
 		partType := spec.PartitionType(i.tbl.metadata.CurrentSchema())
 		if len(partitions) > spec.NumFields() {
-			return nil, fmt.Errorf("manifest %s has %d partition summaries for partition spec %d with %d fields",
+			return fmt.Errorf("manifest %s has %d partition summaries for partition spec %d with %d fields",
 				manifest.FilePath(), len(partitions), manifest.PartitionSpecID(), spec.NumFields())
 		}
 
@@ -379,22 +411,17 @@ func (i InspectTable) Manifests(ctx context.Context) (array.RecordReader, error)
 			fieldType := partType.FieldList[idx].Type
 			transform := spec.Field(idx).Transform
 			if err := appendManifestBound(summaryLower, fieldType, transform, summary.LowerBound); err != nil {
-				return nil, fmt.Errorf("manifest %s partition field %q lower bound: %w",
+				return fmt.Errorf("manifest %s partition field %q lower bound: %w",
 					manifest.FilePath(), partType.FieldList[idx].Name, err)
 			}
 			if err := appendManifestBound(summaryUpper, fieldType, transform, summary.UpperBound); err != nil {
-				return nil, fmt.Errorf("manifest %s partition field %q upper bound: %w",
+				return fmt.Errorf("manifest %s partition field %q upper bound: %w",
 					manifest.FilePath(), partType.FieldList[idx].Name, err)
 			}
 		}
 	}
 
-	rr, err := singleBatchReader(arrowSchema, bldr)
-	if err != nil {
-		return nil, fmt.Errorf("inspect manifests: %w", err)
-	}
-
-	return rr, nil
+	return nil
 }
 
 func appendManifestCount(builder *array.Int32Builder, version int, name string, count int32) error {
diff --git a/table/inspect_all_manifests.go b/table/inspect_all_manifests.go
new file mode 100644
index 000000000..1c59ecd02
--- /dev/null
+++ b/table/inspect_all_manifests.go
@@ -0,0 +1,134 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package table
+
+import (
+	"context"
+	"errors"
+	"fmt"
+
+	"github.com/apache/arrow-go/v18/arrow"
+	"github.com/apache/arrow-go/v18/arrow/array"
+	"github.com/apache/iceberg-go"
+)
+
+// AllManifests returns every manifest referenced by every snapshot currently
+// tracked by the table. A manifest referenced by multiple snapshots produces
+// one row per reference, identified by reference_snapshot_id.
+// Results are streamed in bounded record batches while each snapshot's
+// manifest list is traversed.
+func (i InspectTable) AllManifests(ctx context.Context) (array.RecordReader, error) {
+	arrowSchema, err := SchemaToArrowSchema(AllManifestsSchema(), nil, true, false)
+	if err != nil {
+		return nil, fmt.Errorf("inspect all manifests: build arrow schema: %w", err)
+	}
+
+	snapshots := i.tbl.metadata.Snapshots()
+	var readSnapshotManifests func(Snapshot) ([]iceberg.ManifestFile, error)
+	if len(snapshots) > 0 {
+		if i.tbl.fsF == nil {
+			return nil, errors.New("inspect all manifests: table file IO is not configured")
+		}
+		fs, err := i.tbl.fsF(ctx)
+		if err != nil {
+			return nil, fmt.Errorf("inspect all manifests: %w", err)
+		}
+		readSnapshotManifests = func(snapshot Snapshot) ([]iceberg.ManifestFile, error) {
+			return snapshot.Manifests(fs)
+		}
+	}
+
+	return array.ReaderFromIter(arrowSchema, func(yield func(arrow.RecordBatch, error) bool) {
+		bldr := array.NewRecordBuilder(i.alloc, arrowSchema)
+		defer bldr.Release()
+
+		rows := 0
+		emitted := false
+		emit := func() bool {
+			if rows == 0 {
+				return true
+			}
+
+			batch := bldr.NewRecordBatch()
+			rows = 0
+			emitted = true
+
+			return yield(batch, nil)
+		}
+		emitEmpty := func() {
+			batch := bldr.NewRecordBatch()
+			emitted = true
+			_ = yield(batch, nil)
+		}
+		yieldError := func(err error) {
+			_ = yield(nil, err)
+		}
+
+		for _, snapshot := range snapshots {
+			if err := ctx.Err(); err != nil {
+				yieldError(err)
+
+				return
+			}
+
+			manifests, err := readSnapshotManifests(snapshot)
+			if err != nil {
+				yieldError(fmt.Errorf("inspect all manifests: read snapshot %d manifests: %w",
+					snapshot.SnapshotID, err))
+
+				return
+			}
+
+			referenceSnapshotID := snapshot.SnapshotID
+			for start := 0; start < len(manifests); {
+				end := min(start+inspectRecordBatchSize-rows, len(manifests))
+				if err := i.appendManifestRows(ctx, bldr, manifests[start:end], &referenceSnapshotID); err != nil {
+					yieldError(fmt.Errorf("inspect all manifests: snapshot %d: %w", snapshot.SnapshotID, err))
+
+					return
+				}
+				rows += end - start
+				start = end
+
+				if rows == inspectRecordBatchSize && !emit() {
+					return
+				}
+			}
+		}
+
+		if rows > 0 {
+			_ = emit()
+		} else if !emitted {
+			emitEmpty()
+		}
+	}), nil
+}
+
+// AllManifestsSchema returns the schema of the all_manifests metadata table.
+func AllManifestsSchema() *iceberg.Schema {
+	fields := ManifestsSchema().Fields()
+	fields = append(fields,
+		iceberg.NestedField{
+			ID: 18, Name: "reference_snapshot_id", Type: iceberg.PrimitiveTypes.Int64, Required: true,
+		},
+		iceberg.NestedField{
+			ID: 19, Name: "key_metadata", Type: iceberg.PrimitiveTypes.Binary, Required: false,
+		},
+	)
+
+	return iceberg.NewSchema(0, fields...)
+}
diff --git a/table/inspect_internal_test.go b/table/inspect_internal_test.go
index 2444cf700..9aceb3dfe 100644
--- a/table/inspect_internal_test.go
+++ b/table/inspect_internal_test.go
@@ -1165,6 +1165,138 @@ func TestInspectManifestsSchema(t *testing.T) {
 	require.Equal(t, 13, partitionSummary.FieldList[3].ID)
 }
 
+func TestInspectAllManifestsSchema(t *testing.T) {
+	sc := AllManifestsSchema()
+	require.Equal(t,
+		append(testFieldNames(ManifestsSchema()), "reference_snapshot_id", "key_metadata"),
+		testFieldNames(sc))
+
+	fields := sc.Fields()
+	require.Equal(t, 18, fields[12].ID)
+	require.True(t, fields[12].Required)
+	require.Equal(t, iceberg.PrimitiveTypes.Int64, fields[12].Type)
+	require.Equal(t, 19, fields[13].ID)
+	require.False(t, fields[13].Required)
+	require.Equal(t, iceberg.PrimitiveTypes.Binary, fields[13].Type)
+}
+
+func TestInspectAllManifestsPreservesSnapshotReferences(t *testing.T) {
+	memFS := iceio.NewMemFS()
+	schema := simpleSchema()
+	const tableLocation = "mem://default/inspect-all-manifests"
+	meta, err := NewMetadata(schema, iceberg.UnpartitionedSpec, UnsortedSortOrder, tableLocation,
+		iceberg.Properties{PropertyFormatVersion: "2"})
+	require.NoError(t, err)
+	builder, err := MetadataBuilderFromBase(meta, "")
+	require.NoError(t, err)
+
+	manifest := iceberg.NewManifestFile(2, tableLocation+"/metadata/shared-manifest.avro", 100, 0, 1).
+		SequenceNum(1, 1).
+		AddedFiles(1).
+		KeyMetadata([]byte{1, 2, 3}).
+		Build()
+	manifestListPath := tableLocation + "/metadata/shared-list.avro"
+	var list bytes.Buffer
+	sequenceNumber := int64(1)
+	require.NoError(t, iceberg.WriteManifestList(2, &list, 1, nil, &sequenceNumber, 0,
+		[]iceberg.ManifestFile{manifest}))
+	require.NoError(t, memFS.WriteFile(manifestListPath, list.Bytes()))
+
+	schemaID := schema.ID
+	snapshotOne := int64(1)
+	timestamp := meta.LastUpdatedMillis()
+	require.NoError(t, builder.AddSnapshot(&Snapshot{
+		SnapshotID: snapshotOne, SequenceNumber: 1, TimestampMs: timestamp + 1,
+		ManifestList: manifestListPath, SchemaID: &schemaID,
+	}))
+	require.NoError(t, builder.AddSnapshot(&Snapshot{
+		SnapshotID: 2, ParentSnapshotID: &snapshotOne, SequenceNumber: 2, TimestampMs: timestamp + 2,
+		ManifestList: manifestListPath, SchemaID: &schemaID,
+	}))
+	require.NoError(t, builder.SetSnapshotRef(MainBranch, 2, BranchRef))
+	built, err := builder.Build()
+	require.NoError(t, err)
+	tbl := New(Identifier{"db", "all_manifests"}, built, tableLocation+"/metadata/metadata.json",
+		func(context.Context) (iceio.IO, error) { return memFS, nil }, nil)
+
+	rr, err := tbl.Inspect().AllManifests(context.Background())
+	require.NoError(t, err)
+	defer rr.Release()
+	record := collectRecord(t, rr)
+	defer record.Release()
+
+	require.EqualValues(t, 2, record.NumRows())
+	paths := record.Column(1).(*array.String)
+	require.Equal(t, manifest.FilePath(), paths.Value(0))
+	require.Equal(t, manifest.FilePath(), paths.Value(1))
+	references := record.Column(12).(*array.Int64)
+	require.Equal(t, []int64{1, 2}, references.Int64Values())
+	keyMetadata := record.Column(13).(*array.Binary)
+	require.Equal(t, []byte{1, 2, 3}, keyMetadata.Value(0))
+	require.Equal(t, []byte{1, 2, 3}, keyMetadata.Value(1))
+}
+
+func TestInspectAllManifestsStreamsRecordBatches(t *testing.T) {
+	const manifestCount = inspectRecordBatchSize + 1
+
+	manifests := make([]iceberg.ManifestFile, manifestCount)
+	for idx := range manifests {
+		manifests[idx] = iceberg.NewManifestFile(2,
+			fmt.Sprintf("mem://default/table-location/metadata/manifest-%04d.avro", idx),
+			100, 0, 1).
+			SequenceNum(1, 1).
+			AddedFiles(1).
+			Build()
+	}
+	tbl := inspectTableWithManifestList(t, *iceberg.UnpartitionedSpec, 2, manifests)
+
+	rr, err := tbl.Inspect().AllManifests(context.Background())
+	require.NoError(t, err)
+	defer rr.Release()
+
+	var batchSizes []int64
+	for rr.Next() {
+		batchSizes = append(batchSizes, rr.RecordBatch().NumRows())
+	}
+	require.NoError(t, rr.Err())
+	require.Equal(t, []int64{inspectRecordBatchSize, 1}, batchSizes)
+}
+
+func TestInspectAllManifestsAllocator(t *testing.T) {
+	for _, tt := range []struct {
+		name          string
+		manifestCount int
+		next          bool
+	}{
+		{name: "before first batch", manifestCount: 1},
+		{name: "after first batch", manifestCount: inspectRecordBatchSize + 1, next: true},
+	} {
+		t.Run(tt.name, func(t *testing.T) {
+			checked := memory.NewCheckedAllocator(memory.DefaultAllocator)
+			t.Cleanup(func() { checked.AssertSize(t, 0) })
+
+			manifests := make([]iceberg.ManifestFile, tt.manifestCount)
+			for idx := range manifests {
+				manifests[idx] = iceberg.NewManifestFile(2,
+					fmt.Sprintf("mem://default/table-location/metadata/manifest-%04d.avro", idx),
+					100, 0, 1).
+					SequenceNum(1, 1).
+					AddedFiles(1).
+					Build()
+			}
+			tbl := inspectTableWithManifestList(t, *iceberg.UnpartitionedSpec, 2,
+				manifests)
+
+			rr, err := tbl.Inspect(WithInspectAllocator(checked)).AllManifests(context.Background())
+			require.NoError(t, err)
+			if tt.next {
+				require.True(t, rr.Next())
+			}
+			rr.Release()
+		})
+	}
+}
+
 func TestInspectManifests(t *testing.T) {
 	const snapshotID = int64(1)
 	spec := partitionedSpec()