You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by ze...@apache.org on 2026/04/13 18:54:17 UTC

(iceberg-go) branch main updated: fix(table): prune StatisticsList and PartitionStatsList in RemoveSnapshots (#873)

This is an automated email from the ASF dual-hosted git repository.

zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-go.git


The following commit(s) were added to refs/heads/main by this push:
     new c82d2546 fix(table): prune StatisticsList and PartitionStatsList in RemoveSnapshots (#873)
c82d2546 is described below

commit c82d2546be93b0a87f6f5805f62ffe15f91e871f
Author: Ali Asghar <98...@users.noreply.github.com>
AuthorDate: Mon Apr 13 23:54:11 2026 +0500

    fix(table): prune StatisticsList and PartitionStatsList in RemoveSnapshots (#873)
    
    ## Summary
    
    This PR fixes two related bugs:
    
    **1. `buildCommonMetadata` silently dropped all statistics (primary
    bug)**
    
    `buildCommonMetadata` in `metadata.go` was not copying `StatisticsList`
    or `PartitionStatsList` into the rebuilt metadata. This meant *every*
    `Build()` call — not just `RemoveSnapshots` — silently dropped all
    statistics files. Any builder operation (add snapshot, update schema,
    set properties, etc.) would lose statistics on the resulting metadata.
    
    **2. `RemoveSnapshots` left orphaned statistics entries**
    
    Even after fixing the builder, `RemoveSnapshots` would leave stale
    `StatisticsList`/`PartitionStatsList` entries pointing at snapshots that
    had just been removed. Added pruning logic consistent with the existing
    `snapshotList`/`snapshotLog` pattern.
    
    Fixes #836
    
    ## Test plan
    
    - [x] `go test ./...`
    - [x] `gofmt` clean
    - [x] `TestRemoveSnapshotsPrunesStatistics` — seeds statistics for two
    snapshots, removes one, asserts the right entry is pruned and the result
    survives `Build()`.
    - [x] `TestBuildPreservesStatistics` — no-op `MetadataBuilderFromBase →
    Build()` round-trip, asserts statistics are preserved. Pins the
    `buildCommonMetadata` fix independently of `RemoveSnapshots`.
    
    Signed-off-by: Ali <al...@gmail.com>
    
    ---------
    
    Signed-off-by: Ali <al...@gmail.com>
---
 table/metadata.go                       |  15 ++++
 table/metadata_builder_internal_test.go | 124 ++++++++++++++++++++++++++++++++
 2 files changed, 139 insertions(+)

diff --git a/table/metadata.go b/table/metadata.go
index d3b18f21..2b611934 100644
--- a/table/metadata.go
+++ b/table/metadata.go
@@ -186,6 +186,8 @@ type MetadataBuilder struct {
 	sortOrderList      []SortOrder
 	defaultSortOrderID int
 	refs               map[string]SnapshotRef
+	statisticsList     []StatisticsFile
+	partitionStatsList []PartitionStatisticsFile
 
 	previousFileEntry *MetadataLogEntry
 	// >v1 specific
@@ -261,6 +263,8 @@ func MetadataBuilderFromBase(metadata Metadata, currentFileLocation string) (*Me
 	b.refs = maps.Collect(metadata.Refs())
 	b.snapshotLog = slices.Collect(metadata.SnapshotLogs())
 	b.metadataLog = slices.Collect(metadata.PreviousFiles())
+	b.statisticsList = slices.Collect(metadata.Statistics())
+	b.partitionStatsList = slices.Collect(metadata.PartitionStatistics())
 
 	if currentFileLocation != "" {
 		b.previousFileEntry = &MetadataLogEntry{
@@ -505,6 +509,15 @@ func (b *MetadataBuilder) RemoveSnapshots(snapshotIds []int64, postCommit bool)
 	}
 	b.refs = newRefs
 
+	// Prune statistics entries whose snapshot was removed so that table
+	// metadata does not retain stale statistics references.
+	b.statisticsList = slices.DeleteFunc(b.statisticsList, func(e StatisticsFile) bool {
+		return slices.Contains(snapshotIds, e.SnapshotID)
+	})
+	b.partitionStatsList = slices.DeleteFunc(b.partitionStatsList, func(e PartitionStatisticsFile) bool {
+		return slices.Contains(snapshotIds, e.SnapshotID)
+	})
+
 	b.updates = append(b.updates, NewRemoveSnapshotsUpdate(snapshotIds, postCommit))
 
 	return nil
@@ -871,6 +884,8 @@ func (b *MetadataBuilder) buildCommonMetadata() (*commonMetadata, error) {
 		SortOrderList:      b.sortOrderList,
 		DefaultSortOrderID: b.defaultSortOrderID,
 		SnapshotRefs:       b.refs,
+		StatisticsList:     b.statisticsList,
+		PartitionStatsList: b.partitionStatsList,
 	}, nil
 }
 
diff --git a/table/metadata_builder_internal_test.go b/table/metadata_builder_internal_test.go
index 44990209..30a7ef8b 100644
--- a/table/metadata_builder_internal_test.go
+++ b/table/metadata_builder_internal_test.go
@@ -19,11 +19,13 @@ package table
 
 import (
 	"fmt"
+	"slices"
 	"testing"
 	"time"
 
 	"github.com/apache/iceberg-go"
 	"github.com/davecgh/go-spew/spew"
+	"github.com/google/uuid"
 	"github.com/stretchr/testify/require"
 )
 
@@ -622,6 +624,128 @@ func TestRemoveSnapshotRemovesBranch(t *testing.T) {
 	}
 }
 
+// TestRemoveSnapshotsPrunesStatistics verifies that RemoveSnapshots also
+// prunes StatisticsList and PartitionStatsList entries whose SnapshotID
+// matches a removed snapshot. See iceberg-go#836.
+func TestRemoveSnapshotsPrunesStatistics(t *testing.T) {
+	const (
+		removedID = int64(100)
+		keptID    = int64(200)
+	)
+
+	// Build minimal metadata directly so we can seed statistics and
+	// snapshots without going through AddSnapshot's timestamp checks.
+	currentID := int64(300)
+	lastPartitionID := 999
+	commonMeta := commonMetadata{
+		FormatVersion:   2,
+		UUID:            uuid.New(),
+		Loc:             "s3://test/table",
+		LastUpdatedMS:   1000,
+		LastColumnId:    1,
+		SchemaList:      []*iceberg.Schema{iceberg.NewSchema(0)},
+		CurrentSchemaID: 0,
+		Specs:           []iceberg.PartitionSpec{*iceberg.UnpartitionedSpec},
+		DefaultSpecID:   0,
+		LastPartitionID: &lastPartitionID,
+		Props:           iceberg.Properties{},
+		SnapshotList: []Snapshot{
+			{SnapshotID: removedID, TimestampMs: 1001, ManifestList: "/snap-100.avro"},
+			{SnapshotID: keptID, TimestampMs: 1002, ManifestList: "/snap-200.avro"},
+			{SnapshotID: currentID, TimestampMs: 1003, ManifestList: "/snap-300.avro"},
+		},
+		CurrentSnapshotID:  &currentID,
+		SnapshotLog:        []SnapshotLogEntry{{SnapshotID: currentID, TimestampMs: 1003}},
+		SortOrderList:      []SortOrder{UnsortedSortOrder},
+		DefaultSortOrderID: 0,
+		SnapshotRefs:       map[string]SnapshotRef{MainBranch: {SnapshotID: currentID, SnapshotRefType: BranchRef}},
+		StatisticsList: []StatisticsFile{
+			{SnapshotID: removedID, StatisticsPath: "s3://stats/removed.puffin"},
+			{SnapshotID: keptID, StatisticsPath: "s3://stats/kept.puffin"},
+		},
+		PartitionStatsList: []PartitionStatisticsFile{
+			{SnapshotID: removedID, StatisticsPath: "s3://partstats/removed.parquet"},
+			{SnapshotID: keptID, StatisticsPath: "s3://partstats/kept.parquet"},
+		},
+	}
+	meta := &metadataV2{LastSeqNum: 0, commonMetadata: commonMeta}
+
+	builder, err := MetadataBuilderFromBase(meta, "")
+	require.NoError(t, err)
+
+	// Sanity: MetadataBuilderFromBase must load statistics from the base.
+	require.Len(t, builder.statisticsList, 2,
+		"MetadataBuilderFromBase should load StatisticsList from the base metadata")
+	require.Len(t, builder.partitionStatsList, 2,
+		"MetadataBuilderFromBase should load PartitionStatsList from the base metadata")
+
+	require.NoError(t, builder.RemoveSnapshots([]int64{removedID}, false))
+
+	require.Len(t, builder.statisticsList, 1, "statistics for removed snapshot should be pruned")
+	require.Equal(t, keptID, builder.statisticsList[0].SnapshotID)
+
+	require.Len(t, builder.partitionStatsList, 1, "partition statistics for removed snapshot should be pruned")
+	require.Equal(t, keptID, builder.partitionStatsList[0].SnapshotID)
+
+	// And the kept entries survive a Build round trip.
+	rebuilt, err := builder.Build()
+	require.NoError(t, err)
+	require.Len(t, slices.Collect(rebuilt.Statistics()), 1)
+	require.Len(t, slices.Collect(rebuilt.PartitionStatistics()), 1)
+}
+
+// TestBuildPreservesStatistics pins the buildCommonMetadata fix: a plain
+// MetadataBuilderFromBase → Build() round-trip must preserve StatisticsList
+// and PartitionStatsList without any RemoveSnapshots call.  Before the fix,
+// Build() silently dropped all statistics on every builder operation.
+func TestBuildPreservesStatistics(t *testing.T) {
+	const snapID = int64(42)
+	currentID := snapID
+	lastPartitionID := 0
+	commonMeta := commonMetadata{
+		FormatVersion:   2,
+		UUID:            uuid.New(),
+		Loc:             "s3://test/table",
+		LastUpdatedMS:   1000,
+		LastColumnId:    1,
+		SchemaList:      []*iceberg.Schema{iceberg.NewSchema(0)},
+		CurrentSchemaID: 0,
+		Specs:           []iceberg.PartitionSpec{*iceberg.UnpartitionedSpec},
+		DefaultSpecID:   0,
+		LastPartitionID: &lastPartitionID,
+		Props:           iceberg.Properties{},
+		SnapshotList: []Snapshot{
+			{SnapshotID: snapID, TimestampMs: 1001, ManifestList: "/snap.avro"},
+		},
+		CurrentSnapshotID:  &currentID,
+		SnapshotLog:        []SnapshotLogEntry{{SnapshotID: snapID, TimestampMs: 1001}},
+		SortOrderList:      []SortOrder{UnsortedSortOrder},
+		DefaultSortOrderID: 0,
+		SnapshotRefs:       map[string]SnapshotRef{MainBranch: {SnapshotID: snapID, SnapshotRefType: BranchRef}},
+		StatisticsList: []StatisticsFile{
+			{SnapshotID: snapID, StatisticsPath: "s3://stats/snap.puffin"},
+		},
+		PartitionStatsList: []PartitionStatisticsFile{
+			{SnapshotID: snapID, StatisticsPath: "s3://partstats/snap.parquet"},
+		},
+	}
+	meta := &metadataV2{commonMetadata: commonMeta}
+
+	builder, err := MetadataBuilderFromBase(meta, "")
+	require.NoError(t, err)
+
+	// No mutations — just rebuild.
+	rebuilt, err := builder.Build()
+	require.NoError(t, err)
+
+	require.Len(t, slices.Collect(rebuilt.Statistics()), 1,
+		"Build() must not silently drop StatisticsList")
+	require.Len(t, slices.Collect(rebuilt.PartitionStatistics()), 1,
+		"Build() must not silently drop PartitionStatsList")
+	require.Equal(t, snapID, slices.Collect(rebuilt.Statistics())[0].SnapshotID)
+	require.Equal(t, snapID, slices.Collect(rebuilt.PartitionStatistics())[0].SnapshotID)
+}
+
 func TestExpireMetadataLog(t *testing.T) {
 	builder1 := builderWithoutChanges(2)
 	meta, err := builder1.Build()