You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by ze...@apache.org on 2026/08/11 14:46:41 UTC

(iceberg-go) branch main updated: feat(table): add metadata log entries table (#1696)

This is an automated email from the ASF dual-hosted git repository.

zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-go.git


The following commit(s) were added to refs/heads/main by this push:
     new 61f153a21 feat(table): add metadata log entries table (#1696)
61f153a21 is described below

commit 61f153a21297fa4e85de10740bafbe14949cf439
Author: Minh Vu <vu...@gmail.com>
AuthorDate: Tue Aug 11 16:46:34 2026 +0200

    feat(table): add metadata log entries table (#1696)
    
    * feat(table): add metadata log entries table
    
    * chore: fix lint findings
    
    * fix(table): select latest eligible metadata snapshot
    
    * fix(table): keep first metadata snapshot on timestamp ties
    
    * fix(table): preserve expired metadata snapshot IDs
    
    * fix(table): reuse snapshot log lookup helper
    
    * fix(table): address metadata log inspection review
---
 table/inspect.go               | 109 ++++++++++++++++
 table/inspect_internal_test.go | 275 +++++++++++++++++++++++++++++++++++++++++
 2 files changed, 384 insertions(+)

diff --git a/table/inspect.go b/table/inspect.go
index a8bbf6fd3..dff1a5ea9 100644
--- a/table/inspect.go
+++ b/table/inspect.go
@@ -20,6 +20,7 @@ package table
 import (
 	"context"
 	"fmt"
+	"slices"
 
 	"github.com/apache/arrow-go/v18/arrow"
 	"github.com/apache/arrow-go/v18/arrow/array"
@@ -247,6 +248,101 @@ func (i InspectTable) Snapshots(ctx context.Context) (array.RecordReader, error)
 	return rr, nil
 }
 
+// MetadataLogEntries returns one row for every metadata file in the table's
+// metadata log, plus the current metadata file when its location is set.
+// Snapshot information is resolved from the snapshot log at each metadata
+// file's timestamp.
+//
+// Columns:
+//   - timestamp (timestamptz, required): when the metadata file was written
+//   - file (string, required): metadata file location
+//   - latest_snapshot_id (long, optional): latest snapshot visible then
+//   - latest_schema_id (int, optional): schema used by that snapshot
+//   - latest_sequence_number (long, optional): sequence number of that snapshot
+//
+// The returned reader holds a single record batch. The caller must Release it.
+func (i InspectTable) MetadataLogEntries(ctx context.Context) (array.RecordReader, error) {
+	arrowSchema, err := SchemaToArrowSchema(MetadataLogEntriesSchema(), nil, true, false)
+	if err != nil {
+		return nil, fmt.Errorf("inspect metadata log entries: build arrow schema: %w", err)
+	}
+
+	entries := slices.Collect(i.tbl.metadata.PreviousFiles())
+	if i.tbl.metadataLocation != "" {
+		entries = append(entries, MetadataLogEntry{
+			MetadataFile: i.tbl.metadataLocation,
+			TimestampMs:  i.tbl.metadata.LastUpdatedMillis(),
+		})
+	}
+
+	bldr := array.NewRecordBuilder(i.alloc, arrowSchema)
+	defer bldr.Release()
+
+	timestamp := bldr.Field(0).(*array.TimestampBuilder)
+	file := bldr.Field(1).(*array.StringBuilder)
+	latestSnapshotID := bldr.Field(2).(*array.Int64Builder)
+	latestSchemaID := bldr.Field(3).(*array.Int32Builder)
+	latestSequenceNumber := bldr.Field(4).(*array.Int64Builder)
+
+	for _, entry := range entries {
+		if err := ctx.Err(); err != nil {
+			return nil, err
+		}
+
+		timestamp.Append(arrow.Timestamp(entry.TimestampMs * 1000))
+		file.Append(entry.MetadataFile)
+
+		snapshotID, snapshot, found := latestSnapshotAt(i.tbl.metadata, entry.TimestampMs)
+		if !found {
+			latestSnapshotID.AppendNull()
+			latestSchemaID.AppendNull()
+			latestSequenceNumber.AppendNull()
+
+			continue
+		}
+
+		latestSnapshotID.Append(snapshotID)
+		if snapshot == nil {
+			latestSchemaID.AppendNull()
+			latestSequenceNumber.AppendNull()
+
+			continue
+		}
+		if snapshot.SchemaID != nil {
+			// Iceberg schema IDs are bounded to int32 by the specification.
+			//nolint:gosec // schema IDs are spec-bounded to int32
+			latestSchemaID.Append(int32(*snapshot.SchemaID))
+		} else {
+			latestSchemaID.AppendNull()
+		}
+		latestSequenceNumber.Append(snapshot.SequenceNumber)
+	}
+
+	rr, err := singleBatchReader(arrowSchema, bldr)
+	if err != nil {
+		return nil, fmt.Errorf("inspect metadata log entries: %w", err)
+	}
+
+	return rr, nil
+}
+
+// latestSnapshotAt follows the metadata-table behavior used by Java's
+// MetadataLogEntriesTable: it resolves against the current snapshot log, not
+// a point-in-time copy of that log. Trimming old entries can therefore shift
+// the result to a later eligible entry or make it unavailable. Equal
+// timestamps keep the first entry encountered, matching Java's
+// SnapshotUtil.snapshotIdAsOfTime; PyIceberg keeps the last entry instead.
+// The snapshot ID remains available when the snapshot itself has expired from
+// metadata.
+func latestSnapshotAt(metadata Metadata, timestampMs int64) (int64, *Snapshot, bool) {
+	entry, found := snapshotLogEntryAsOf(metadata.SnapshotLogs(), timestampMs, true)
+	if !found {
+		return 0, nil, false
+	}
+
+	return entry.SnapshotID, metadata.SnapshotByID(entry.SnapshotID), true
+}
+
 // HistorySchema returns the Iceberg schema of the history metadata table. The
 // field IDs are fixed by the Iceberg metadata-tables spec and match the Java,
 // PyIceberg, and Rust clients for cross-client parity. A fresh schema value is
@@ -285,3 +381,16 @@ func SnapshotsSchema() *iceberg.Schema {
 		}},
 	)
 }
+
+// MetadataLogEntriesSchema returns a fresh Iceberg schema for the
+// metadata-log-entries metadata table. The field IDs and names match Java's
+// implementation; callers should not rely on pointer identity.
+func MetadataLogEntriesSchema() *iceberg.Schema {
+	return iceberg.NewSchema(0,
+		iceberg.NestedField{ID: 1, Name: "timestamp", Type: iceberg.PrimitiveTypes.TimestampTz, Required: true},
+		iceberg.NestedField{ID: 2, Name: "file", Type: iceberg.PrimitiveTypes.String, Required: true},
+		iceberg.NestedField{ID: 3, Name: "latest_snapshot_id", Type: iceberg.PrimitiveTypes.Int64, Required: false},
+		iceberg.NestedField{ID: 4, Name: "latest_schema_id", Type: iceberg.PrimitiveTypes.Int32, Required: false},
+		iceberg.NestedField{ID: 5, Name: "latest_sequence_number", Type: iceberg.PrimitiveTypes.Int64, Required: false},
+	)
+}
diff --git a/table/inspect_internal_test.go b/table/inspect_internal_test.go
index 1450499c5..f619926dc 100644
--- a/table/inspect_internal_test.go
+++ b/table/inspect_internal_test.go
@@ -404,6 +404,258 @@ func TestInspectSnapshotsEmpty(t *testing.T) {
 	require.EqualValues(t, 6, rec.NumCols())
 }
 
+func TestInspectMetadataLogEntriesSchema(t *testing.T) {
+	sc := MetadataLogEntriesSchema()
+
+	require.Equal(t,
+		[]string{"timestamp", "file", "latest_snapshot_id", "latest_schema_id", "latest_sequence_number"},
+		testFieldNames(sc))
+
+	fields := sc.Fields()
+	for i := range fields {
+		require.Equal(t, i+1, fields[i].ID)
+	}
+
+	require.True(t, fields[0].Required, "timestamp is required")
+	require.True(t, fields[1].Required, "file is required")
+	require.False(t, fields[2].Required, "latest_snapshot_id is optional")
+	require.False(t, fields[3].Required, "latest_schema_id is optional")
+	require.False(t, fields[4].Required, "latest_sequence_number is optional")
+
+	require.Equal(t, iceberg.PrimitiveTypes.TimestampTz, fields[0].Type)
+	require.Equal(t, iceberg.PrimitiveTypes.String, fields[1].Type)
+	require.Equal(t, iceberg.PrimitiveTypes.Int64, fields[2].Type)
+	require.Equal(t, iceberg.PrimitiveTypes.Int32, fields[3].Type)
+	require.Equal(t, iceberg.PrimitiveTypes.Int64, fields[4].Type)
+}
+
+func TestInspectMetadataLogEntries(t *testing.T) {
+	const (
+		s1      = int64(101)
+		s2      = int64(102)
+		expired = int64(999)
+	)
+	schema1 := 1
+	schema2 := 2
+	current := s2
+	lastPartitionID := 999
+	meta := &metadataV2{commonMetadata: commonMetadata{
+		FormatVersion: 2,
+		UUID:          uuid.New(),
+		Loc:           "s3://test/metadata-log-entries",
+		LastUpdatedMS: 4000,
+		LastColumnId:  1,
+		SchemaList: []*iceberg.Schema{
+			iceberg.NewSchema(0),
+			iceberg.NewSchema(schema1),
+			iceberg.NewSchema(schema2),
+		},
+		CurrentSchemaID: schema2,
+		Specs:           []iceberg.PartitionSpec{*iceberg.UnpartitionedSpec},
+		DefaultSpecID:   0,
+		LastPartitionID: &lastPartitionID,
+		Props:           iceberg.Properties{},
+		MetadataLog: []MetadataLogEntry{
+			{MetadataFile: "/metadata/v1.json", TimestampMs: 1000},
+			{MetadataFile: "/metadata/v2.json", TimestampMs: 2000},
+		},
+		SnapshotList: []Snapshot{
+			{SnapshotID: s1, SequenceNumber: 7, TimestampMs: 1500, SchemaID: &schema1},
+			{SnapshotID: s2, SequenceNumber: 8, TimestampMs: 3000, SchemaID: &schema2},
+		},
+		SnapshotLog: []SnapshotLogEntry{
+			{SnapshotID: s1, TimestampMs: 1500},
+			{SnapshotID: s2, TimestampMs: 3000},
+			{SnapshotID: expired, TimestampMs: 3500},
+		},
+		CurrentSnapshotID:  &current,
+		SortOrderList:      []SortOrder{UnsortedSortOrder},
+		DefaultSortOrderID: 0,
+		SnapshotRefs:       map[string]SnapshotRef{MainBranch: {SnapshotID: s2, SnapshotRefType: BranchRef}},
+	}}
+	tbl := New(Identifier{"metadata-log-entries"}, meta, "/metadata/v3.json", nil, nil)
+
+	rr, err := tbl.Inspect().MetadataLogEntries(context.Background())
+	require.NoError(t, err)
+	defer rr.Release()
+
+	rec := collectRecord(t, rr)
+	defer rec.Release()
+
+	require.EqualValues(t, 3, rec.NumRows())
+	require.EqualValues(t, 5, rec.NumCols())
+
+	tsType, ok := rec.Schema().Field(0).Type.(*arrow.TimestampType)
+	require.True(t, ok, "timestamp must be an Arrow timestamp")
+	require.Equal(t, arrow.Microsecond, tsType.Unit)
+	require.Equal(t, "UTC", tsType.TimeZone)
+
+	timestamp := rec.Column(0).(*array.Timestamp)
+	file := rec.Column(1).(*array.String)
+	latestSnapshotID := rec.Column(2).(*array.Int64)
+	latestSchemaID := rec.Column(3).(*array.Int32)
+	latestSequenceNumber := rec.Column(4).(*array.Int64)
+
+	require.EqualValues(t, 1000*1000, timestamp.Value(0))
+	require.EqualValues(t, 2000*1000, timestamp.Value(1))
+	require.EqualValues(t, 4000*1000, timestamp.Value(2))
+	require.Equal(t, "/metadata/v1.json", file.Value(0))
+	require.Equal(t, "/metadata/v2.json", file.Value(1))
+	require.Equal(t, "/metadata/v3.json", file.Value(2))
+
+	// No snapshot had been committed at the first metadata-file timestamp.
+	require.True(t, latestSnapshotID.IsNull(0))
+	require.True(t, latestSchemaID.IsNull(0))
+	require.True(t, latestSequenceNumber.IsNull(0))
+
+	require.EqualValues(t, s1, latestSnapshotID.Value(1))
+	require.EqualValues(t, schema1, latestSchemaID.Value(1))
+	require.EqualValues(t, 7, latestSequenceNumber.Value(1))
+	// The snapshot log can retain an entry after the snapshot itself expires.
+	// Keep its ID while leaving details that require the missing snapshot null.
+	require.EqualValues(t, expired, latestSnapshotID.Value(2))
+	require.True(t, latestSchemaID.IsNull(2))
+	require.True(t, latestSequenceNumber.IsNull(2))
+}
+
+func TestLatestSnapshotAtKeepsExpiredSnapshotID(t *testing.T) {
+	const expiredSnapshot = int64(101)
+	meta := &metadataV2{commonMetadata: commonMetadata{
+		SnapshotLog: []SnapshotLogEntry{{SnapshotID: expiredSnapshot, TimestampMs: 2000}},
+	}}
+
+	snapshotID, snapshot, found := latestSnapshotAt(meta, 2500)
+	require.True(t, found)
+	require.Equal(t, expiredSnapshot, snapshotID)
+	require.Nil(t, snapshot)
+}
+
+func TestLatestSnapshotAtScansAllSnapshotLogEntries(t *testing.T) {
+	const (
+		firstSnapshot  = int64(101)
+		secondSnapshot = int64(102)
+		lateSnapshot   = int64(103)
+	)
+	meta := &metadataV2{commonMetadata: commonMetadata{
+		SnapshotList: []Snapshot{
+			{SnapshotID: firstSnapshot},
+			{SnapshotID: secondSnapshot},
+			{SnapshotID: lateSnapshot},
+		},
+		// The 500ms inversion is intentional and models clock skew within the
+		// metadata validator's one-minute tolerance.
+		SnapshotLog: []SnapshotLogEntry{
+			{SnapshotID: firstSnapshot, TimestampMs: 2000},
+			{SnapshotID: secondSnapshot, TimestampMs: 3000},
+			{SnapshotID: lateSnapshot, TimestampMs: 2500},
+		},
+	}}
+
+	snapshotID, snapshot, found := latestSnapshotAt(meta, 2600)
+	require.True(t, found)
+	require.NotNil(t, snapshot)
+	require.Equal(t, lateSnapshot, snapshotID)
+	require.Equal(t, lateSnapshot, snapshot.SnapshotID)
+}
+
+func TestLatestSnapshotAtUsesFirstEntryForEqualTimestamps(t *testing.T) {
+	const (
+		firstSnapshot  = int64(101)
+		secondSnapshot = int64(102)
+		timestamp      = int64(2000)
+	)
+	meta := &metadataV2{commonMetadata: commonMetadata{
+		SnapshotList: []Snapshot{
+			{SnapshotID: firstSnapshot},
+			{SnapshotID: secondSnapshot},
+		},
+		SnapshotLog: []SnapshotLogEntry{
+			{SnapshotID: firstSnapshot, TimestampMs: timestamp},
+			{SnapshotID: secondSnapshot, TimestampMs: timestamp},
+		},
+	}}
+
+	snapshotID, snapshot, found := latestSnapshotAt(meta, timestamp)
+	require.True(t, found)
+	require.NotNil(t, snapshot)
+	require.Equal(t, firstSnapshot, snapshotID)
+	require.Equal(t, firstSnapshot, snapshot.SnapshotID)
+}
+
+func TestInspectMetadataLogEntriesAllowsLiveSnapshotWithoutSchemaID(t *testing.T) {
+	const snapshotID = int64(101)
+	meta := &metadataV2{commonMetadata: commonMetadata{
+		LastUpdatedMS: 2000,
+		SnapshotList:  []Snapshot{{SnapshotID: snapshotID, SequenceNumber: 7}},
+		SnapshotLog:   []SnapshotLogEntry{{SnapshotID: snapshotID, TimestampMs: 1500}},
+		SnapshotRefs:  map[string]SnapshotRef{},
+	}}
+	tbl := New(Identifier{"metadata-log-entries-nil-schema"}, meta, "/metadata/v2.json", nil, nil)
+
+	rr, err := tbl.Inspect().MetadataLogEntries(context.Background())
+	require.NoError(t, err)
+	defer rr.Release()
+
+	rec := collectRecord(t, rr)
+	defer rec.Release()
+
+	latestSnapshotID := rec.Column(2).(*array.Int64)
+	latestSchemaID := rec.Column(3).(*array.Int32)
+	latestSequenceNumber := rec.Column(4).(*array.Int64)
+	require.EqualValues(t, snapshotID, latestSnapshotID.Value(0))
+	require.True(t, latestSchemaID.IsNull(0))
+	require.EqualValues(t, 7, latestSequenceNumber.Value(0))
+}
+
+func TestInspectMetadataLogEntriesEmpty(t *testing.T) {
+	lastPartitionID := 999
+	meta := &metadataV2{commonMetadata: commonMetadata{
+		FormatVersion:      2,
+		UUID:               uuid.New(),
+		Loc:                "s3://test/empty-metadata-log",
+		LastUpdatedMS:      1000,
+		LastColumnId:       1,
+		SchemaList:         []*iceberg.Schema{iceberg.NewSchema(0)},
+		CurrentSchemaID:    0,
+		Specs:              []iceberg.PartitionSpec{*iceberg.UnpartitionedSpec},
+		DefaultSpecID:      0,
+		LastPartitionID:    &lastPartitionID,
+		Props:              iceberg.Properties{},
+		SnapshotRefs:       map[string]SnapshotRef{},
+		SortOrderList:      []SortOrder{UnsortedSortOrder},
+		DefaultSortOrderID: 0,
+	}}
+	tbl := New(Identifier{"empty-metadata-log"}, meta, "/metadata/v1.json", nil, nil)
+
+	rr, err := tbl.Inspect().MetadataLogEntries(context.Background())
+	require.NoError(t, err)
+	defer rr.Release()
+
+	rec := collectRecord(t, rr)
+	defer rec.Release()
+
+	require.EqualValues(t, 1, rec.NumRows())
+	require.Equal(t, "/metadata/v1.json", rec.Column(1).(*array.String).Value(0))
+	for column := 2; column < int(rec.NumCols()); column++ {
+		require.True(t, rec.Column(column).IsNull(0), "column %d should be null", column)
+	}
+}
+
+func TestInspectMetadataLogEntriesSkipsEmptyCurrentMetadataLocation(t *testing.T) {
+	meta := &metadataV2{commonMetadata: commonMetadata{LastUpdatedMS: 1000}}
+	tbl := New(Identifier{"empty-metadata-location"}, meta, "", nil, nil)
+
+	rr, err := tbl.Inspect().MetadataLogEntries(context.Background())
+	require.NoError(t, err)
+	defer rr.Release()
+
+	rec := collectRecord(t, rr)
+	defer rec.Release()
+
+	require.EqualValues(t, 0, rec.NumRows())
+	require.EqualValues(t, 5, rec.NumCols())
+}
+
 // TestInspectAllocatorOption verifies WithInspectAllocator routes allocations
 // through the supplied allocator, and that all buffers are released.
 func TestInspectAllocatorOption(t *testing.T) {
@@ -423,6 +675,29 @@ func TestInspectAllocatorOption(t *testing.T) {
 	require.EqualValues(t, 2, rec.NumRows())
 }
 
+func TestInspectMetadataLogEntriesAllocator(t *testing.T) {
+	checked := memory.NewCheckedAllocator(memory.DefaultAllocator)
+	t.Cleanup(func() { checked.AssertSize(t, 0) })
+	meta := &metadataV2{commonMetadata: commonMetadata{
+		LastUpdatedMS: 2000,
+		MetadataLog: []MetadataLogEntry{
+			{MetadataFile: "/metadata/v1.json", TimestampMs: 1000},
+		},
+		SnapshotList: []Snapshot{{SnapshotID: 101, SequenceNumber: 7}},
+		SnapshotLog:  []SnapshotLogEntry{{SnapshotID: 101, TimestampMs: 1500}},
+	}}
+	tbl := New(Identifier{"metadata-log-entries-allocator"}, meta, "/metadata/v2.json", nil, nil)
+
+	rr, err := tbl.Inspect(WithInspectAllocator(checked)).MetadataLogEntries(context.Background())
+	require.NoError(t, err)
+	defer rr.Release()
+
+	rec := collectRecord(t, rr)
+	defer rec.Release()
+
+	require.EqualValues(t, 2, rec.NumRows())
+}
+
 // mapRow reads one row of a string->string Arrow map column into a Go map.
 func mapRow(t *testing.T, m *array.Map, row int) map[string]string {
 	t.Helper()