You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by ze...@apache.org on 2026/08/17 23:10:22 UTC

(iceberg-go) branch main updated: perf(compaction): compute survivor minimum once (#1820)

This is an automated email from the ASF dual-hosted git repository.

zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-go.git


The following commit(s) were added to refs/heads/main by this push:
     new 4390d2af2 perf(compaction): compute survivor minimum once (#1820)
4390d2af2 is described below

commit 4390d2af2bb011c0df9797e06e7db6c4a7ef551b
Author: Minh Vu <vu...@gmail.com>
AuthorDate: Tue Aug 18 01:10:15 2026 +0200

    perf(compaction): compute survivor minimum once (#1820)
    
    * perf(compaction): compute survivor minimum once
    
    * test(compaction): cover conservative delete decisions
    
    * fix(compaction): satisfy integer range lint
---
 table/compaction/eq_delete_decision.go            |  3 +-
 table/compaction/eq_delete_decision_bench_test.go | 67 +++++++++++++++++++
 table/compaction/eq_delete_decision_test.go       | 81 ++++++++++++++++++++++-
 3 files changed, 149 insertions(+), 2 deletions(-)

diff --git a/table/compaction/eq_delete_decision.go b/table/compaction/eq_delete_decision.go
index 5fedd7e8f..92759ad2e 100644
--- a/table/compaction/eq_delete_decision.go
+++ b/table/compaction/eq_delete_decision.go
@@ -138,6 +138,7 @@ func DecideDeadEqualityDeletes(survey *SurvivorSurvey, candidates []iceberg.Mani
 
 	dead := make([]iceberg.DataFile, 0, len(candidates))
 	seen := make(map[string]struct{}, len(candidates))
+	minSeq := survey.conservativeMinSeq()
 	for _, e := range candidates {
 		df := e.DataFile()
 		path := df.FilePath()
@@ -147,7 +148,7 @@ func DecideDeadEqualityDeletes(survey *SurvivorSurvey, candidates []iceberg.Mani
 		if e.SequenceNum() < 0 {
 			continue
 		}
-		if survey.conservativeMinSeq() >= e.SequenceNum() {
+		if minSeq >= e.SequenceNum() {
 			seen[path] = struct{}{}
 			dead = append(dead, df)
 		}
diff --git a/table/compaction/eq_delete_decision_bench_test.go b/table/compaction/eq_delete_decision_bench_test.go
new file mode 100644
index 000000000..d2dcfa17b
--- /dev/null
+++ b/table/compaction/eq_delete_decision_bench_test.go
@@ -0,0 +1,67 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package compaction_test
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/apache/iceberg-go"
+	"github.com/apache/iceberg-go/table/compaction"
+)
+
+var benchmarkDeadEqualityDeletes []iceberg.DataFile
+
+func BenchmarkDecideDeadEqualityDeletes(b *testing.B) {
+	for _, tc := range []struct {
+		name       string
+		partitions int
+		deletes    int
+	}{
+		{name: "P=1/D=1", partitions: 1, deletes: 1},
+		{name: "P=100/D=100", partitions: 100, deletes: 100},
+		{name: "P=1000/D=1000", partitions: 1000, deletes: 1000},
+		{name: "P=10000/D=1000", partitions: 10000, deletes: 1000},
+		{name: "P=1000/D=10000", partitions: 1000, deletes: 10000},
+	} {
+		b.Run(tc.name, func(b *testing.B) {
+			survey := compaction.NewSurvivorSurvey()
+			for i := range tc.partitions {
+				survey.AddSurvivor(map[int]any{1000: i}, 10)
+			}
+
+			specs := compactionTestSpecs()
+			candidates := make([]iceberg.ManifestEntry, tc.deletes)
+			for i := range candidates {
+				candidates[i] = makeEqDeleteEntry(
+					b, specs[0], nil, 1, fmt.Sprintf("/eq-%d.parquet", i))
+			}
+
+			b.ReportAllocs()
+			b.ResetTimer()
+			for range b.N {
+				benchmarkDeadEqualityDeletes = compaction.DecideDeadEqualityDeletes(
+					survey, candidates)
+			}
+			b.StopTimer()
+			if len(benchmarkDeadEqualityDeletes) != tc.deletes {
+				b.Fatalf("expected %d dead equality deletes, got %d", tc.deletes, len(benchmarkDeadEqualityDeletes))
+			}
+		})
+	}
+}
diff --git a/table/compaction/eq_delete_decision_test.go b/table/compaction/eq_delete_decision_test.go
index 9b117075a..9b82dc18a 100644
--- a/table/compaction/eq_delete_decision_test.go
+++ b/table/compaction/eq_delete_decision_test.go
@@ -235,6 +235,85 @@ func TestDecideDeadEqualityDeletesConservativelyMatchesAcrossSpecs(t *testing.T)
 		survey, []iceberg.ManifestEntry{voidCandidate}))
 }
 
+func TestDecideDeadEqualityDeletesUsesConservativeMinimum(t *testing.T) {
+	type candidate struct {
+		path string
+		seq  int64
+	}
+
+	tests := []struct {
+		name       string
+		survivors  []survivor
+		candidates []candidate
+		wantPaths  []string
+	}{
+		{
+			name: "minimum across partition buckets",
+			survivors: []survivor{
+				{partition: map[int]any{1000: "us"}, seq: 3},
+				{partition: map[int]any{1000: "eu"}, seq: 9},
+			},
+			candidates: []candidate{
+				{path: "/dead-before-min.parquet", seq: 2},
+				{path: "/dead-at-min.parquet", seq: 3},
+				{path: "/alive-after-min.parquet", seq: 4},
+				{path: "/unknown-seq.parquet", seq: -1},
+			},
+			wantPaths: []string{
+				"/dead-before-min.parquet",
+				"/dead-at-min.parquet",
+			},
+		},
+		{
+			name: "includes empty-partition bucket",
+			survivors: []survivor{
+				{partition: nil, seq: 7},
+				{partition: map[int]any{1000: "us"}, seq: 10},
+			},
+			candidates: []candidate{
+				{path: "/dead-at-empty-min.parquet", seq: 7},
+				{path: "/alive-after-empty-min.parquet", seq: 8},
+			},
+			wantPaths: []string{"/dead-at-empty-min.parquet"},
+		},
+		{
+			name: "deduplicates paths",
+			survivors: []survivor{
+				{partition: map[int]any{1000: "us"}, seq: 5},
+			},
+			candidates: []candidate{
+				{path: "/duplicate.parquet", seq: 5},
+				{path: "/duplicate.parquet", seq: 5},
+				{path: "/unknown-seq.parquet", seq: -1},
+			},
+			wantPaths: []string{"/duplicate.parquet"},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			specs := compactionTestSpecs()
+			survey := compaction.NewSurvivorSurvey()
+			for _, s := range tt.survivors {
+				survey.AddSurvivor(s.partition, s.seq)
+			}
+
+			candidates := make([]iceberg.ManifestEntry, 0, len(tt.candidates))
+			for _, c := range tt.candidates {
+				candidates = append(candidates,
+					makeEqDeleteEntry(t, specs[0], nil, c.seq, c.path))
+			}
+
+			got := compaction.DecideDeadEqualityDeletes(survey, candidates)
+			gotPaths := make([]string, len(got))
+			for i, file := range got {
+				gotPaths[i] = file.FilePath()
+			}
+			assert.Equal(t, tt.wantPaths, gotPaths)
+		})
+	}
+}
+
 // TestSurvivorSurvey_AddSurvivor_DefensiveSeq asserts that a survivor
 // with sequence number < 0 is recorded as if seq=0 — guaranteeing it
 // stays "alive" against every eq-delete.
@@ -293,7 +372,7 @@ func compactionTestSpecs() compactionTestSpecLookup {
 // reads only path, partition, content type, and seq, so a minimal
 // builder configuration is enough.
 func makeEqDeleteEntry(
-	t *testing.T,
+	t testing.TB,
 	spec iceberg.PartitionSpec,
 	part map[int]any,
 	seq int64,