You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by ze...@apache.org on 2026/07/13 17:49:06 UTC
(iceberg-go) branch main updated: fix(table): include orphan file sizes in cleanup results (#1412)
This is an automated email from the ASF dual-hosted git repository.
zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-go.git
The following commit(s) were added to refs/heads/main by this push:
new 772aa2b4 fix(table): include orphan file sizes in cleanup results (#1412)
772aa2b4 is described below
commit 772aa2b4b61d6954e3c37f950c8fbb3022258275
Author: Minh Vu <vu...@gmail.com>
AuthorDate: Mon Jul 13 19:49:01 2026 +0200
fix(table): include orphan file sizes in cleanup results (#1412)
## Summary
- Add per-orphan file sizes to table orphan cleanup result as .
- Populate file sizes during scan so retains size metadata, not just
paths.
- Map sizes into CLI output entries and render per-file size in text
output.
- Extend JSON output to always include and add regression tests for
table/CLI.
## Testing
- go test ./...
---
cmd/iceberg/clean_orphan_files.go | 35 ++++++++++++----
cmd/iceberg/clean_orphan_files_test.go | 74 +++++++++++++++++++++++++++++++---
cmd/iceberg/maintenance.go | 2 +-
table/orphan_cleanup.go | 17 +++++++-
table/orphan_cleanup_test.go | 62 ++++++++++++++++++++++++++++
5 files changed, 174 insertions(+), 16 deletions(-)
diff --git a/cmd/iceberg/clean_orphan_files.go b/cmd/iceberg/clean_orphan_files.go
index e55e7c4c..f2751558 100644
--- a/cmd/iceberg/clean_orphan_files.go
+++ b/cmd/iceberg/clean_orphan_files.go
@@ -94,14 +94,32 @@ func runCleanOrphanFiles(ctx context.Context, output Output, cat catalog.Catalog
}
func buildCleanOrphanFilesResult(tbl *table.Table, result table.OrphanCleanupResult, dryRun bool) CleanOrphanFilesResult {
- files := result.OrphanFileLocations
- if !dryRun {
- files = result.DeletedFiles
- }
+ var entries []OrphanFileEntry
+ if dryRun {
+ entries = make([]OrphanFileEntry, 0, len(result.OrphanFiles))
+ for _, f := range result.OrphanFiles {
+ entries = append(entries, OrphanFileEntry{
+ Path: f.Path,
+ SizeBytes: f.SizeBytes,
+ })
+ }
+ } else {
+ deletedSet := make(map[string]struct{}, len(result.DeletedFiles))
+ for _, f := range result.DeletedFiles {
+ deletedSet[f] = struct{}{}
+ }
+
+ entries = make([]OrphanFileEntry, 0, len(result.DeletedFiles))
+ for _, f := range result.OrphanFiles {
+ if _, ok := deletedSet[f.Path]; !ok {
+ continue
+ }
- entries := make([]OrphanFileEntry, 0, len(files))
- for _, f := range files {
- entries = append(entries, OrphanFileEntry{Path: f})
+ entries = append(entries, OrphanFileEntry{
+ Path: f.Path,
+ SizeBytes: f.SizeBytes,
+ })
+ }
}
return CleanOrphanFilesResult{
@@ -128,12 +146,13 @@ func (t textOutput) CleanOrphanFilesResult(result CleanOrphanFilesResult) {
pterm.Printfln("Deleted %d orphan files (%s) from %s.", result.OrphanFileCount, sizeStr, result.Table)
}
- data := pterm.TableData{{"#", "PATH"}}
+ data := pterm.TableData{{"#", "PATH", "SIZE_BYTES"}}
for i, f := range result.OrphanFiles {
data = append(data, []string{
strconv.Itoa(i + 1),
f.Path,
+ formatBytes(f.SizeBytes),
})
}
diff --git a/cmd/iceberg/clean_orphan_files_test.go b/cmd/iceberg/clean_orphan_files_test.go
index 490dc033..2cc414af 100644
--- a/cmd/iceberg/clean_orphan_files_test.go
+++ b/cmd/iceberg/clean_orphan_files_test.go
@@ -58,7 +58,11 @@ func TestBuildCleanOrphanFilesResultDryRun(t *testing.T) {
orphanResult := table.OrphanCleanupResult{
OrphanFileLocations: []string{"s3://bucket/data/file1.parquet", "s3://bucket/data/file2.parquet"},
- TotalSizeBytes: 4096,
+ OrphanFiles: []table.OrphanFile{
+ {Path: "s3://bucket/data/file1.parquet", SizeBytes: 1024},
+ {Path: "s3://bucket/data/file2.parquet", SizeBytes: 2048},
+ },
+ TotalSizeBytes: 4096,
}
result := buildCleanOrphanFilesResult(tbl, orphanResult, true)
@@ -69,6 +73,7 @@ func TestBuildCleanOrphanFilesResultDryRun(t *testing.T) {
assert.Equal(t, int64(4096), result.TotalSizeBytes)
require.Len(t, result.OrphanFiles, 2)
assert.Equal(t, "s3://bucket/data/file1.parquet", result.OrphanFiles[0].Path)
+ assert.Equal(t, int64(1024), result.OrphanFiles[0].SizeBytes)
}
func TestBuildCleanOrphanFilesResultDeleted(t *testing.T) {
@@ -101,8 +106,11 @@ func TestBuildCleanOrphanFilesResultDeleted(t *testing.T) {
orphanResult := table.OrphanCleanupResult{
OrphanFileLocations: []string{"s3://bucket/data/file1.parquet"},
- DeletedFiles: []string{"s3://bucket/data/file1.parquet"},
- TotalSizeBytes: 2048,
+ OrphanFiles: []table.OrphanFile{
+ {Path: "s3://bucket/data/file1.parquet", SizeBytes: 2048},
+ },
+ DeletedFiles: []string{"s3://bucket/data/file1.parquet"},
+ TotalSizeBytes: 2048,
}
result := buildCleanOrphanFilesResult(tbl, orphanResult, false)
@@ -110,6 +118,57 @@ func TestBuildCleanOrphanFilesResultDeleted(t *testing.T) {
assert.False(t, result.DryRun)
assert.Equal(t, 1, result.OrphanFileCount)
assert.Equal(t, "s3://bucket/data/file1.parquet", result.OrphanFiles[0].Path)
+ assert.Equal(t, int64(2048), result.OrphanFiles[0].SizeBytes)
+}
+
+func TestBuildCleanOrphanFilesResultDeletedFiltersToDeletedFiles(t *testing.T) {
+ const metadata = `{
+ "format-version": 2,
+ "table-uuid": "9c12d441-03fe-4693-9a96-a0705ddf69c1",
+ "location": "s3://bucket/test/location",
+ "last-sequence-number": 0,
+ "last-updated-ms": 1602638573590,
+ "last-column-id": 1,
+ "current-schema-id": 0,
+ "schemas": [{"type": "struct", "schema-id": 0, "fields": [{"id": 1, "name": "x", "required": true, "type": "long"}]}],
+ "default-spec-id": 0,
+ "partition-specs": [{"spec-id": 0, "fields": []}],
+ "last-partition-id": 0,
+ "default-sort-order-id": 0,
+ "sort-orders": [{"order-id": 0, "fields": []}],
+ "properties": {},
+ "current-snapshot-id": -1,
+ "snapshots": [],
+ "snapshot-log": [],
+ "metadata-log": [],
+ "refs": {}
+ }`
+
+ meta, err := table.ParseMetadataBytes([]byte(metadata))
+ require.NoError(t, err)
+
+ tbl := table.New([]string{"db", "orphans"}, meta, "", nil, nil)
+
+ orphanResult := table.OrphanCleanupResult{
+ OrphanFileLocations: []string{
+ "s3://bucket/data/file1.parquet",
+ "s3://bucket/data/file2.parquet",
+ },
+ OrphanFiles: []table.OrphanFile{
+ {Path: "s3://bucket/data/file1.parquet", SizeBytes: 1024},
+ {Path: "s3://bucket/data/file2.parquet", SizeBytes: 2048},
+ },
+ DeletedFiles: []string{"s3://bucket/data/file2.parquet"},
+ TotalSizeBytes: 3072,
+ }
+
+ result := buildCleanOrphanFilesResult(tbl, orphanResult, false)
+
+ assert.False(t, result.DryRun)
+ assert.Equal(t, 1, result.OrphanFileCount)
+ require.Len(t, result.OrphanFiles, 1)
+ assert.Equal(t, "s3://bucket/data/file2.parquet", result.OrphanFiles[0].Path)
+ assert.Equal(t, int64(2048), result.OrphanFiles[0].SizeBytes)
}
func TestTextOutputCleanOrphanFilesResultEmpty(t *testing.T) {
@@ -142,8 +201,8 @@ func TestTextOutputCleanOrphanFilesResultDryRun(t *testing.T) {
OrphanFileCount: 2,
TotalSizeBytes: 1048576,
OrphanFiles: []OrphanFileEntry{
- {Path: "s3://bucket/data/a.parquet"},
- {Path: "s3://bucket/data/b.parquet"},
+ {Path: "s3://bucket/data/a.parquet", SizeBytes: 1024},
+ {Path: "s3://bucket/data/b.parquet", SizeBytes: 4096},
},
}
@@ -155,6 +214,8 @@ func TestTextOutputCleanOrphanFilesResultDryRun(t *testing.T) {
assert.Contains(t, output, "2 orphan files found")
assert.Contains(t, output, "1.0 MB")
assert.Contains(t, output, "s3://bucket/data/a.parquet")
+ assert.Contains(t, output, "1.0 KB")
+ assert.Contains(t, output, "4.0 KB")
}
func TestJSONOutputCleanOrphanFilesResult(t *testing.T) {
@@ -169,7 +230,7 @@ func TestJSONOutputCleanOrphanFilesResult(t *testing.T) {
OrphanFileCount: 1,
TotalSizeBytes: 512,
OrphanFiles: []OrphanFileEntry{
- {Path: "s3://bucket/data/orphan.parquet"},
+ {Path: "s3://bucket/data/orphan.parquet", SizeBytes: 512},
},
}
@@ -185,4 +246,5 @@ func TestJSONOutputCleanOrphanFilesResult(t *testing.T) {
assert.Contains(t, output, `"orphan_file_count":1`)
assert.Contains(t, output, `"total_size_bytes":512`)
assert.Contains(t, output, `"path":"s3://bucket/data/orphan.parquet"`)
+ assert.Contains(t, output, `"size_bytes":512`)
}
diff --git a/cmd/iceberg/maintenance.go b/cmd/iceberg/maintenance.go
index 62a803a6..4842befb 100644
--- a/cmd/iceberg/maintenance.go
+++ b/cmd/iceberg/maintenance.go
@@ -172,7 +172,7 @@ type CleanOrphanFilesResult struct {
type OrphanFileEntry struct {
Path string `json:"path"`
- SizeBytes int64 `json:"size_bytes,omitempty"`
+ SizeBytes int64 `json:"size_bytes"`
}
type UpgradeResult struct {
diff --git a/table/orphan_cleanup.go b/table/orphan_cleanup.go
index e6f5cdc4..3c68a53a 100644
--- a/table/orphan_cleanup.go
+++ b/table/orphan_cleanup.go
@@ -160,12 +160,21 @@ func WithEqualAuthorities(authorities map[string]string) OrphanCleanupOption {
}
type OrphanCleanupResult struct {
+ // OrphanFileLocations is retained for backward compatibility with callers
+ // that consume only orphan paths. Prefer OrphanFiles for canonical path+size data.
OrphanFileLocations []string
- DeletedFiles []string
+ // OrphanFiles is the canonical richer orphan result, carrying both path and size.
+ OrphanFiles []OrphanFile
+ DeletedFiles []string
// TotalSizeBytes is the combined size of orphan files only, not all scanned files.
TotalSizeBytes int64
}
+type OrphanFile struct {
+ Path string
+ SizeBytes int64
+}
+
// DeleteOrphanFiles identifies files under a table location that are no longer
// referenced by table metadata and deletes them unless dry-run is enabled.
//
@@ -249,6 +258,7 @@ func (t Table) executeOrphanCleanup(ctx context.Context, cfg *orphanCleanupConfi
}
var orphanFiles []string
+ orphanFileEntries := make([]OrphanFile, 0)
var totalOrphanSize int64
for _, f := range scannedFiles {
isOrphan, err := isFileOrphan(f.path, referencedFiles, normalizedRef, cfg)
@@ -257,12 +267,17 @@ func (t Table) executeOrphanCleanup(ctx context.Context, cfg *orphanCleanupConfi
}
if isOrphan {
orphanFiles = append(orphanFiles, f.path)
+ orphanFileEntries = append(orphanFileEntries, OrphanFile{
+ Path: f.path,
+ SizeBytes: f.size,
+ })
totalOrphanSize += f.size
}
}
result := OrphanCleanupResult{
OrphanFileLocations: orphanFiles,
+ OrphanFiles: orphanFileEntries,
TotalSizeBytes: totalOrphanSize,
}
diff --git a/table/orphan_cleanup_test.go b/table/orphan_cleanup_test.go
index 1bea4680..ebbf5742 100644
--- a/table/orphan_cleanup_test.go
+++ b/table/orphan_cleanup_test.go
@@ -726,6 +726,68 @@ func TestWalkDirectoryUsesListableIO(t *testing.T) {
}, sizes)
}
+func TestDeleteOrphanFilesPopulatesOrphanFileSizes(t *testing.T) {
+ const metaJSON = `{
+ "format-version": 2,
+ "table-uuid": "9c12d441-03fe-4693-9a96-a0705ddf69c1",
+ "location": "s3://bucket/table",
+ "last-sequence-number": 0,
+ "last-updated-ms": 1602638573590,
+ "last-column-id": 1,
+ "current-schema-id": 0,
+ "schemas": [{"type": "struct", "schema-id": 0, "fields": [{"id": 1, "name": "x", "required": true, "type": "long"}]}],
+ "default-spec-id": 0,
+ "partition-specs": [{"spec-id": 0, "fields": []}],
+ "last-partition-id": 0,
+ "default-sort-order-id": 0,
+ "sort-orders": [{"order-id": 0, "fields": []}],
+ "properties": {},
+ "current-snapshot-id": -1,
+ "snapshots": [],
+ "snapshot-log": [],
+ "metadata-log": [],
+ "refs": {}
+ }`
+
+ meta, err := ParseMetadataString(metaJSON)
+ require.NoError(t, err)
+
+ mockFS := &mockListableIO{
+ entries: []mockWalkEntry{
+ {path: "s3://bucket/table", info: mockFileInfo{name: "table", mode: stdfs.ModeDir}},
+ {path: "s3://bucket/table/a.parquet", info: mockFileInfo{name: "a.parquet", size: 1024}},
+ {path: "s3://bucket/table/nested", info: mockFileInfo{name: "nested", mode: stdfs.ModeDir}},
+ {path: "s3://bucket/table/nested/b.parquet", info: mockFileInfo{name: "b.parquet", size: 4096}},
+ },
+ }
+
+ tbl := New(
+ Identifier{"db", "tbl"},
+ meta,
+ "s3://bucket/table/metadata/v1.metadata.json",
+ func(context.Context) (io.IO, error) { return mockFS, nil },
+ nil,
+ )
+
+ result, err := tbl.DeleteOrphanFiles(context.Background(),
+ WithDryRun(true),
+ WithLocation("s3://bucket/table"),
+ WithMaxConcurrency(1),
+ WithFilesOlderThan(-1*time.Second),
+ )
+
+ require.NoError(t, err)
+ require.Len(t, result.OrphanFiles, 2)
+
+ orphanSizes := make(map[string]int64, len(result.OrphanFiles))
+ for _, f := range result.OrphanFiles {
+ orphanSizes[f.Path] = f.SizeBytes
+ }
+
+ assert.Equal(t, int64(1024), orphanSizes["s3://bucket/table/a.parquet"])
+ assert.Equal(t, int64(4096), orphanSizes["s3://bucket/table/nested/b.parquet"])
+}
+
func TestWalkDirectoryRequiresListableIO(t *testing.T) {
err := walkDirectory(&mockPlainIO{}, "s3://bucket/data", func(string, stdfs.FileInfo) error {
require.Fail(t, "walk callback should not run for non-listable IO")