You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by ze...@apache.org on 2026/08/11 14:46:41 UTC
(iceberg-go) branch main updated: feat(table): add metadata log entries table (#1696)
This is an automated email from the ASF dual-hosted git repository.
zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-go.git
The following commit(s) were added to refs/heads/main by this push:
new 61f153a21 feat(table): add metadata log entries table (#1696)
61f153a21 is described below
commit 61f153a21297fa4e85de10740bafbe14949cf439
Author: Minh Vu <vu...@gmail.com>
AuthorDate: Tue Aug 11 16:46:34 2026 +0200
feat(table): add metadata log entries table (#1696)
* feat(table): add metadata log entries table
* chore: fix lint findings
* fix(table): select latest eligible metadata snapshot
* fix(table): keep first metadata snapshot on timestamp ties
* fix(table): preserve expired metadata snapshot IDs
* fix(table): reuse snapshot log lookup helper
* fix(table): address metadata log inspection review
---
table/inspect.go | 109 ++++++++++++++++
table/inspect_internal_test.go | 275 +++++++++++++++++++++++++++++++++++++++++
2 files changed, 384 insertions(+)
diff --git a/table/inspect.go b/table/inspect.go
index a8bbf6fd3..dff1a5ea9 100644
--- a/table/inspect.go
+++ b/table/inspect.go
@@ -20,6 +20,7 @@ package table
import (
"context"
"fmt"
+ "slices"
"github.com/apache/arrow-go/v18/arrow"
"github.com/apache/arrow-go/v18/arrow/array"
@@ -247,6 +248,101 @@ func (i InspectTable) Snapshots(ctx context.Context) (array.RecordReader, error)
return rr, nil
}
+// MetadataLogEntries returns one row for every metadata file in the table's
+// metadata log, plus the current metadata file when its location is set.
+// Snapshot information is resolved from the snapshot log at each metadata
+// file's timestamp.
+//
+// Columns:
+// - timestamp (timestamptz, required): when the metadata file was written
+// - file (string, required): metadata file location
+// - latest_snapshot_id (long, optional): latest snapshot visible then
+// - latest_schema_id (int, optional): schema used by that snapshot
+// - latest_sequence_number (long, optional): sequence number of that snapshot
+//
+// The returned reader holds a single record batch. The caller must Release it.
+func (i InspectTable) MetadataLogEntries(ctx context.Context) (array.RecordReader, error) {
+ arrowSchema, err := SchemaToArrowSchema(MetadataLogEntriesSchema(), nil, true, false)
+ if err != nil {
+ return nil, fmt.Errorf("inspect metadata log entries: build arrow schema: %w", err)
+ }
+
+ entries := slices.Collect(i.tbl.metadata.PreviousFiles())
+ if i.tbl.metadataLocation != "" {
+ entries = append(entries, MetadataLogEntry{
+ MetadataFile: i.tbl.metadataLocation,
+ TimestampMs: i.tbl.metadata.LastUpdatedMillis(),
+ })
+ }
+
+ bldr := array.NewRecordBuilder(i.alloc, arrowSchema)
+ defer bldr.Release()
+
+ timestamp := bldr.Field(0).(*array.TimestampBuilder)
+ file := bldr.Field(1).(*array.StringBuilder)
+ latestSnapshotID := bldr.Field(2).(*array.Int64Builder)
+ latestSchemaID := bldr.Field(3).(*array.Int32Builder)
+ latestSequenceNumber := bldr.Field(4).(*array.Int64Builder)
+
+ for _, entry := range entries {
+ if err := ctx.Err(); err != nil {
+ return nil, err
+ }
+
+ timestamp.Append(arrow.Timestamp(entry.TimestampMs * 1000))
+ file.Append(entry.MetadataFile)
+
+ snapshotID, snapshot, found := latestSnapshotAt(i.tbl.metadata, entry.TimestampMs)
+ if !found {
+ latestSnapshotID.AppendNull()
+ latestSchemaID.AppendNull()
+ latestSequenceNumber.AppendNull()
+
+ continue
+ }
+
+ latestSnapshotID.Append(snapshotID)
+ if snapshot == nil {
+ latestSchemaID.AppendNull()
+ latestSequenceNumber.AppendNull()
+
+ continue
+ }
+ if snapshot.SchemaID != nil {
+ // Iceberg schema IDs are bounded to int32 by the specification.
+ //nolint:gosec // schema IDs are spec-bounded to int32
+ latestSchemaID.Append(int32(*snapshot.SchemaID))
+ } else {
+ latestSchemaID.AppendNull()
+ }
+ latestSequenceNumber.Append(snapshot.SequenceNumber)
+ }
+
+ rr, err := singleBatchReader(arrowSchema, bldr)
+ if err != nil {
+ return nil, fmt.Errorf("inspect metadata log entries: %w", err)
+ }
+
+ return rr, nil
+}
+
+// latestSnapshotAt follows the metadata-table behavior used by Java's
+// MetadataLogEntriesTable: it resolves against the current snapshot log, not
+// a point-in-time copy of that log. Trimming old entries can therefore shift
+// the result to a later eligible entry or make it unavailable. Equal
+// timestamps keep the first entry encountered, matching Java's
+// SnapshotUtil.snapshotIdAsOfTime; PyIceberg keeps the last entry instead.
+// The snapshot ID remains available when the snapshot itself has expired from
+// metadata.
+func latestSnapshotAt(metadata Metadata, timestampMs int64) (int64, *Snapshot, bool) {
+ entry, found := snapshotLogEntryAsOf(metadata.SnapshotLogs(), timestampMs, true)
+ if !found {
+ return 0, nil, false
+ }
+
+ return entry.SnapshotID, metadata.SnapshotByID(entry.SnapshotID), true
+}
+
// HistorySchema returns the Iceberg schema of the history metadata table. The
// field IDs are fixed by the Iceberg metadata-tables spec and match the Java,
// PyIceberg, and Rust clients for cross-client parity. A fresh schema value is
@@ -285,3 +381,16 @@ func SnapshotsSchema() *iceberg.Schema {
}},
)
}
+
+// MetadataLogEntriesSchema returns a fresh Iceberg schema for the
+// metadata-log-entries metadata table. The field IDs and names match Java's
+// implementation; callers should not rely on pointer identity.
+func MetadataLogEntriesSchema() *iceberg.Schema {
+ return iceberg.NewSchema(0,
+ iceberg.NestedField{ID: 1, Name: "timestamp", Type: iceberg.PrimitiveTypes.TimestampTz, Required: true},
+ iceberg.NestedField{ID: 2, Name: "file", Type: iceberg.PrimitiveTypes.String, Required: true},
+ iceberg.NestedField{ID: 3, Name: "latest_snapshot_id", Type: iceberg.PrimitiveTypes.Int64, Required: false},
+ iceberg.NestedField{ID: 4, Name: "latest_schema_id", Type: iceberg.PrimitiveTypes.Int32, Required: false},
+ iceberg.NestedField{ID: 5, Name: "latest_sequence_number", Type: iceberg.PrimitiveTypes.Int64, Required: false},
+ )
+}
diff --git a/table/inspect_internal_test.go b/table/inspect_internal_test.go
index 1450499c5..f619926dc 100644
--- a/table/inspect_internal_test.go
+++ b/table/inspect_internal_test.go
@@ -404,6 +404,258 @@ func TestInspectSnapshotsEmpty(t *testing.T) {
require.EqualValues(t, 6, rec.NumCols())
}
+func TestInspectMetadataLogEntriesSchema(t *testing.T) {
+ sc := MetadataLogEntriesSchema()
+
+ require.Equal(t,
+ []string{"timestamp", "file", "latest_snapshot_id", "latest_schema_id", "latest_sequence_number"},
+ testFieldNames(sc))
+
+ fields := sc.Fields()
+ for i := range fields {
+ require.Equal(t, i+1, fields[i].ID)
+ }
+
+ require.True(t, fields[0].Required, "timestamp is required")
+ require.True(t, fields[1].Required, "file is required")
+ require.False(t, fields[2].Required, "latest_snapshot_id is optional")
+ require.False(t, fields[3].Required, "latest_schema_id is optional")
+ require.False(t, fields[4].Required, "latest_sequence_number is optional")
+
+ require.Equal(t, iceberg.PrimitiveTypes.TimestampTz, fields[0].Type)
+ require.Equal(t, iceberg.PrimitiveTypes.String, fields[1].Type)
+ require.Equal(t, iceberg.PrimitiveTypes.Int64, fields[2].Type)
+ require.Equal(t, iceberg.PrimitiveTypes.Int32, fields[3].Type)
+ require.Equal(t, iceberg.PrimitiveTypes.Int64, fields[4].Type)
+}
+
+func TestInspectMetadataLogEntries(t *testing.T) {
+ const (
+ s1 = int64(101)
+ s2 = int64(102)
+ expired = int64(999)
+ )
+ schema1 := 1
+ schema2 := 2
+ current := s2
+ lastPartitionID := 999
+ meta := &metadataV2{commonMetadata: commonMetadata{
+ FormatVersion: 2,
+ UUID: uuid.New(),
+ Loc: "s3://test/metadata-log-entries",
+ LastUpdatedMS: 4000,
+ LastColumnId: 1,
+ SchemaList: []*iceberg.Schema{
+ iceberg.NewSchema(0),
+ iceberg.NewSchema(schema1),
+ iceberg.NewSchema(schema2),
+ },
+ CurrentSchemaID: schema2,
+ Specs: []iceberg.PartitionSpec{*iceberg.UnpartitionedSpec},
+ DefaultSpecID: 0,
+ LastPartitionID: &lastPartitionID,
+ Props: iceberg.Properties{},
+ MetadataLog: []MetadataLogEntry{
+ {MetadataFile: "/metadata/v1.json", TimestampMs: 1000},
+ {MetadataFile: "/metadata/v2.json", TimestampMs: 2000},
+ },
+ SnapshotList: []Snapshot{
+ {SnapshotID: s1, SequenceNumber: 7, TimestampMs: 1500, SchemaID: &schema1},
+ {SnapshotID: s2, SequenceNumber: 8, TimestampMs: 3000, SchemaID: &schema2},
+ },
+ SnapshotLog: []SnapshotLogEntry{
+ {SnapshotID: s1, TimestampMs: 1500},
+ {SnapshotID: s2, TimestampMs: 3000},
+ {SnapshotID: expired, TimestampMs: 3500},
+ },
+ CurrentSnapshotID: ¤t,
+ SortOrderList: []SortOrder{UnsortedSortOrder},
+ DefaultSortOrderID: 0,
+ SnapshotRefs: map[string]SnapshotRef{MainBranch: {SnapshotID: s2, SnapshotRefType: BranchRef}},
+ }}
+ tbl := New(Identifier{"metadata-log-entries"}, meta, "/metadata/v3.json", nil, nil)
+
+ rr, err := tbl.Inspect().MetadataLogEntries(context.Background())
+ require.NoError(t, err)
+ defer rr.Release()
+
+ rec := collectRecord(t, rr)
+ defer rec.Release()
+
+ require.EqualValues(t, 3, rec.NumRows())
+ require.EqualValues(t, 5, rec.NumCols())
+
+ tsType, ok := rec.Schema().Field(0).Type.(*arrow.TimestampType)
+ require.True(t, ok, "timestamp must be an Arrow timestamp")
+ require.Equal(t, arrow.Microsecond, tsType.Unit)
+ require.Equal(t, "UTC", tsType.TimeZone)
+
+ timestamp := rec.Column(0).(*array.Timestamp)
+ file := rec.Column(1).(*array.String)
+ latestSnapshotID := rec.Column(2).(*array.Int64)
+ latestSchemaID := rec.Column(3).(*array.Int32)
+ latestSequenceNumber := rec.Column(4).(*array.Int64)
+
+ require.EqualValues(t, 1000*1000, timestamp.Value(0))
+ require.EqualValues(t, 2000*1000, timestamp.Value(1))
+ require.EqualValues(t, 4000*1000, timestamp.Value(2))
+ require.Equal(t, "/metadata/v1.json", file.Value(0))
+ require.Equal(t, "/metadata/v2.json", file.Value(1))
+ require.Equal(t, "/metadata/v3.json", file.Value(2))
+
+ // No snapshot had been committed at the first metadata-file timestamp.
+ require.True(t, latestSnapshotID.IsNull(0))
+ require.True(t, latestSchemaID.IsNull(0))
+ require.True(t, latestSequenceNumber.IsNull(0))
+
+ require.EqualValues(t, s1, latestSnapshotID.Value(1))
+ require.EqualValues(t, schema1, latestSchemaID.Value(1))
+ require.EqualValues(t, 7, latestSequenceNumber.Value(1))
+ // The snapshot log can retain an entry after the snapshot itself expires.
+ // Keep its ID while leaving details that require the missing snapshot null.
+ require.EqualValues(t, expired, latestSnapshotID.Value(2))
+ require.True(t, latestSchemaID.IsNull(2))
+ require.True(t, latestSequenceNumber.IsNull(2))
+}
+
+func TestLatestSnapshotAtKeepsExpiredSnapshotID(t *testing.T) {
+ const expiredSnapshot = int64(101)
+ meta := &metadataV2{commonMetadata: commonMetadata{
+ SnapshotLog: []SnapshotLogEntry{{SnapshotID: expiredSnapshot, TimestampMs: 2000}},
+ }}
+
+ snapshotID, snapshot, found := latestSnapshotAt(meta, 2500)
+ require.True(t, found)
+ require.Equal(t, expiredSnapshot, snapshotID)
+ require.Nil(t, snapshot)
+}
+
+func TestLatestSnapshotAtScansAllSnapshotLogEntries(t *testing.T) {
+ const (
+ firstSnapshot = int64(101)
+ secondSnapshot = int64(102)
+ lateSnapshot = int64(103)
+ )
+ meta := &metadataV2{commonMetadata: commonMetadata{
+ SnapshotList: []Snapshot{
+ {SnapshotID: firstSnapshot},
+ {SnapshotID: secondSnapshot},
+ {SnapshotID: lateSnapshot},
+ },
+ // The 500ms inversion is intentional and models clock skew within the
+ // metadata validator's one-minute tolerance.
+ SnapshotLog: []SnapshotLogEntry{
+ {SnapshotID: firstSnapshot, TimestampMs: 2000},
+ {SnapshotID: secondSnapshot, TimestampMs: 3000},
+ {SnapshotID: lateSnapshot, TimestampMs: 2500},
+ },
+ }}
+
+ snapshotID, snapshot, found := latestSnapshotAt(meta, 2600)
+ require.True(t, found)
+ require.NotNil(t, snapshot)
+ require.Equal(t, lateSnapshot, snapshotID)
+ require.Equal(t, lateSnapshot, snapshot.SnapshotID)
+}
+
+func TestLatestSnapshotAtUsesFirstEntryForEqualTimestamps(t *testing.T) {
+ const (
+ firstSnapshot = int64(101)
+ secondSnapshot = int64(102)
+ timestamp = int64(2000)
+ )
+ meta := &metadataV2{commonMetadata: commonMetadata{
+ SnapshotList: []Snapshot{
+ {SnapshotID: firstSnapshot},
+ {SnapshotID: secondSnapshot},
+ },
+ SnapshotLog: []SnapshotLogEntry{
+ {SnapshotID: firstSnapshot, TimestampMs: timestamp},
+ {SnapshotID: secondSnapshot, TimestampMs: timestamp},
+ },
+ }}
+
+ snapshotID, snapshot, found := latestSnapshotAt(meta, timestamp)
+ require.True(t, found)
+ require.NotNil(t, snapshot)
+ require.Equal(t, firstSnapshot, snapshotID)
+ require.Equal(t, firstSnapshot, snapshot.SnapshotID)
+}
+
+func TestInspectMetadataLogEntriesAllowsLiveSnapshotWithoutSchemaID(t *testing.T) {
+ const snapshotID = int64(101)
+ meta := &metadataV2{commonMetadata: commonMetadata{
+ LastUpdatedMS: 2000,
+ SnapshotList: []Snapshot{{SnapshotID: snapshotID, SequenceNumber: 7}},
+ SnapshotLog: []SnapshotLogEntry{{SnapshotID: snapshotID, TimestampMs: 1500}},
+ SnapshotRefs: map[string]SnapshotRef{},
+ }}
+ tbl := New(Identifier{"metadata-log-entries-nil-schema"}, meta, "/metadata/v2.json", nil, nil)
+
+ rr, err := tbl.Inspect().MetadataLogEntries(context.Background())
+ require.NoError(t, err)
+ defer rr.Release()
+
+ rec := collectRecord(t, rr)
+ defer rec.Release()
+
+ latestSnapshotID := rec.Column(2).(*array.Int64)
+ latestSchemaID := rec.Column(3).(*array.Int32)
+ latestSequenceNumber := rec.Column(4).(*array.Int64)
+ require.EqualValues(t, snapshotID, latestSnapshotID.Value(0))
+ require.True(t, latestSchemaID.IsNull(0))
+ require.EqualValues(t, 7, latestSequenceNumber.Value(0))
+}
+
+func TestInspectMetadataLogEntriesEmpty(t *testing.T) {
+ lastPartitionID := 999
+ meta := &metadataV2{commonMetadata: commonMetadata{
+ FormatVersion: 2,
+ UUID: uuid.New(),
+ Loc: "s3://test/empty-metadata-log",
+ LastUpdatedMS: 1000,
+ LastColumnId: 1,
+ SchemaList: []*iceberg.Schema{iceberg.NewSchema(0)},
+ CurrentSchemaID: 0,
+ Specs: []iceberg.PartitionSpec{*iceberg.UnpartitionedSpec},
+ DefaultSpecID: 0,
+ LastPartitionID: &lastPartitionID,
+ Props: iceberg.Properties{},
+ SnapshotRefs: map[string]SnapshotRef{},
+ SortOrderList: []SortOrder{UnsortedSortOrder},
+ DefaultSortOrderID: 0,
+ }}
+ tbl := New(Identifier{"empty-metadata-log"}, meta, "/metadata/v1.json", nil, nil)
+
+ rr, err := tbl.Inspect().MetadataLogEntries(context.Background())
+ require.NoError(t, err)
+ defer rr.Release()
+
+ rec := collectRecord(t, rr)
+ defer rec.Release()
+
+ require.EqualValues(t, 1, rec.NumRows())
+ require.Equal(t, "/metadata/v1.json", rec.Column(1).(*array.String).Value(0))
+ for column := 2; column < int(rec.NumCols()); column++ {
+ require.True(t, rec.Column(column).IsNull(0), "column %d should be null", column)
+ }
+}
+
+func TestInspectMetadataLogEntriesSkipsEmptyCurrentMetadataLocation(t *testing.T) {
+ meta := &metadataV2{commonMetadata: commonMetadata{LastUpdatedMS: 1000}}
+ tbl := New(Identifier{"empty-metadata-location"}, meta, "", nil, nil)
+
+ rr, err := tbl.Inspect().MetadataLogEntries(context.Background())
+ require.NoError(t, err)
+ defer rr.Release()
+
+ rec := collectRecord(t, rr)
+ defer rec.Release()
+
+ require.EqualValues(t, 0, rec.NumRows())
+ require.EqualValues(t, 5, rec.NumCols())
+}
+
// TestInspectAllocatorOption verifies WithInspectAllocator routes allocations
// through the supplied allocator, and that all buffers are released.
func TestInspectAllocatorOption(t *testing.T) {
@@ -423,6 +675,29 @@ func TestInspectAllocatorOption(t *testing.T) {
require.EqualValues(t, 2, rec.NumRows())
}
+func TestInspectMetadataLogEntriesAllocator(t *testing.T) {
+ checked := memory.NewCheckedAllocator(memory.DefaultAllocator)
+ t.Cleanup(func() { checked.AssertSize(t, 0) })
+ meta := &metadataV2{commonMetadata: commonMetadata{
+ LastUpdatedMS: 2000,
+ MetadataLog: []MetadataLogEntry{
+ {MetadataFile: "/metadata/v1.json", TimestampMs: 1000},
+ },
+ SnapshotList: []Snapshot{{SnapshotID: 101, SequenceNumber: 7}},
+ SnapshotLog: []SnapshotLogEntry{{SnapshotID: 101, TimestampMs: 1500}},
+ }}
+ tbl := New(Identifier{"metadata-log-entries-allocator"}, meta, "/metadata/v2.json", nil, nil)
+
+ rr, err := tbl.Inspect(WithInspectAllocator(checked)).MetadataLogEntries(context.Background())
+ require.NoError(t, err)
+ defer rr.Release()
+
+ rec := collectRecord(t, rr)
+ defer rec.Release()
+
+ require.EqualValues(t, 2, rec.NumRows())
+}
+
// mapRow reads one row of a string->string Arrow map column into a Go map.
func mapRow(t *testing.T, m *array.Map, row int) map[string]string {
t.Helper()