You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by ze...@apache.org on 2026/08/14 16:01:07 UTC
(iceberg-go) branch main updated: perf(io): delete GoCloud files concurrently (#1811)
This is an automated email from the ASF dual-hosted git repository.
zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-go.git
The following commit(s) were added to refs/heads/main by this push:
new 42ccd4635 perf(io): delete GoCloud files concurrently (#1811)
42ccd4635 is described below
commit 42ccd46350de4c192dc6370d80afe494a2abde39
Author: Minh Vu <vu...@gmail.com>
AuthorDate: Fri Aug 14 18:01:02 2026 +0200
perf(io): delete GoCloud files concurrently (#1811)
perf(io): delete gocloud files concurrently
---
io/gocloud/blob.go | 72 +++++++++++++++++++-------
io/gocloud/blob_test.go | 131 ++++++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 186 insertions(+), 17 deletions(-)
diff --git a/io/gocloud/blob.go b/io/gocloud/blob.go
index 46a3083b6..ce4172ea5 100644
--- a/io/gocloud/blob.go
+++ b/io/gocloud/blob.go
@@ -26,6 +26,7 @@ import (
pathpkg "path"
"slices"
"strings"
+ "sync"
"time"
icebergio "github.com/apache/iceberg-go/io"
@@ -188,6 +189,10 @@ type BlobFileIO struct {
var _ icebergio.ListableIO = (*BlobFileIO)(nil)
+// deleteFilesMaxConcurrency bounds the number of in-flight object-store
+// deletes without creating one goroutine per path for large cleanup jobs.
+const deleteFilesMaxConcurrency = 16
+
type blobFileInfo struct {
name string
size int64
@@ -420,37 +425,70 @@ func (bfs *BlobFileIO) WalkDir(root string, fn fs.WalkDirFunc) error {
})
}
+func (bfs *BlobFileIO) deleteFile(ctx context.Context, p string) (bool, error) {
+ key, err := bfs.preprocess(p)
+ if err != nil {
+ return false, fmt.Errorf("failed to delete %s: %w", p, err)
+ }
+
+ if err := bfs.Delete(ctx, key); err != nil {
+ // Missing files are not errors per the interface contract.
+ if gcerrors.Code(err) == gcerrors.NotFound {
+ return true, nil
+ }
+
+ return false, fmt.Errorf("failed to delete %s: %w", p, err)
+ }
+
+ return true, nil
+}
+
func (bfs *BlobFileIO) DeleteFiles(ctx context.Context, paths []string) ([]string, error) {
if len(paths) == 0 {
return nil, nil
}
- deleted := make([]string, 0, len(paths))
-
- var errs error
+ type result struct {
+ deleted bool
+ err error
+ }
- for _, p := range paths {
- key, err := bfs.preprocess(p)
- if err != nil {
- errs = errors.Join(errs, fmt.Errorf("failed to delete %s: %w", p, err))
+ results := make([]result, len(paths))
+ workers := len(paths)
+ if workers > deleteFilesMaxConcurrency {
+ workers = deleteFilesMaxConcurrency
+ }
- continue
- }
+ jobs := make(chan int)
+ var wg sync.WaitGroup
+ wg.Add(workers)
- if err := bfs.Delete(ctx, key); err != nil {
- // Missing files are not errors per the interface contract.
- if gcerrors.Code(err) == gcerrors.NotFound {
- deleted = append(deleted, p)
+ for i := 0; i < workers; i++ {
+ go func() {
+ defer wg.Done()
- continue
+ for idx := range jobs {
+ results[idx].deleted, results[idx].err = bfs.deleteFile(ctx, paths[idx])
}
+ }()
+ }
- errs = errors.Join(errs, fmt.Errorf("failed to delete %s: %w", p, err))
+ for idx := range paths {
+ jobs <- idx
+ }
+ close(jobs)
+ wg.Wait()
- continue
+ deleted := make([]string, 0, len(paths))
+ var errs error
+ for idx, result := range results {
+ if result.deleted {
+ deleted = append(deleted, paths[idx])
}
- deleted = append(deleted, p)
+ if result.err != nil {
+ errs = errors.Join(errs, result.err)
+ }
}
return deleted, errs
diff --git a/io/gocloud/blob_test.go b/io/gocloud/blob_test.go
index ba6bf399d..4f4e60ca5 100644
--- a/io/gocloud/blob_test.go
+++ b/io/gocloud/blob_test.go
@@ -19,17 +19,23 @@ package gocloud
import (
"context"
+ "errors"
+ "fmt"
"io"
"io/fs"
"net/url"
+ "sync"
"testing"
+ "time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
icebergio "github.com/apache/iceberg-go/io"
"gocloud.dev/blob"
+ "gocloud.dev/blob/driver"
"gocloud.dev/blob/memblob"
+ "gocloud.dev/gcerrors"
)
func TestDefaultKeyExtractor(t *testing.T) {
@@ -725,6 +731,131 @@ func TestBlobFileIODeleteFilesEmpty(t *testing.T) {
assert.Nil(t, deleted)
}
+type trackingDeleteBucket struct {
+ started chan<- struct{}
+ release chan struct{}
+
+ mu sync.Mutex
+ inFlight int
+ maxInFlight int
+}
+
+var errTrackingDeleteUnsupported = errors.New("unsupported test operation")
+
+func (b *trackingDeleteBucket) ErrorCode(error) gcerrors.ErrorCode { return gcerrors.Unknown }
+func (b *trackingDeleteBucket) As(any) bool { return false }
+func (b *trackingDeleteBucket) ErrorAs(error, any) bool { return false }
+func (b *trackingDeleteBucket) Attributes(context.Context, string) (*driver.Attributes, error) {
+ return nil, errTrackingDeleteUnsupported
+}
+
+func (b *trackingDeleteBucket) ListPaged(context.Context, *driver.ListOptions) (*driver.ListPage, error) {
+ return nil, errTrackingDeleteUnsupported
+}
+
+func (b *trackingDeleteBucket) NewRangeReader(context.Context, string, int64, int64, *driver.ReaderOptions) (driver.Reader, error) {
+ return nil, errTrackingDeleteUnsupported
+}
+
+func (b *trackingDeleteBucket) NewTypedWriter(context.Context, string, string, *driver.WriterOptions) (driver.Writer, error) {
+ return nil, errTrackingDeleteUnsupported
+}
+
+func (b *trackingDeleteBucket) Copy(context.Context, string, string, *driver.CopyOptions) error {
+ return errTrackingDeleteUnsupported
+}
+
+func (b *trackingDeleteBucket) Delete(ctx context.Context, _ string) error {
+ b.mu.Lock()
+ b.inFlight++
+ if b.inFlight > b.maxInFlight {
+ b.maxInFlight = b.inFlight
+ }
+ b.mu.Unlock()
+ defer func() {
+ b.mu.Lock()
+ b.inFlight--
+ b.mu.Unlock()
+ }()
+
+ select {
+ case b.started <- struct{}{}:
+ case <-ctx.Done():
+ return ctx.Err()
+ }
+
+ select {
+ case <-b.release:
+ return nil
+ case <-ctx.Done():
+ return ctx.Err()
+ }
+}
+
+func (b *trackingDeleteBucket) SignedURL(context.Context, string, *driver.SignedURLOptions) (string, error) {
+ return "", errTrackingDeleteUnsupported
+}
+func (b *trackingDeleteBucket) Close() error { return nil }
+
+func TestBlobFileIODeleteFilesIsConcurrentAndBounded(t *testing.T) {
+ const pathCount = deleteFilesMaxConcurrency * 2
+
+ started := make(chan struct{}, pathCount)
+ tracker := &trackingDeleteBucket{
+ started: started,
+ release: make(chan struct{}),
+ }
+ bucket := blob.NewBucket(tracker)
+ defer bucket.Close()
+
+ bfs := &BlobFileIO{
+ Bucket: bucket,
+ extractObject: defaultObjectLocationExtractor("test-bucket"),
+ ctx: context.Background(),
+ }
+ paths := make([]string, pathCount)
+ for i := range paths {
+ paths[i] = fmt.Sprintf("s3://test-bucket/data/file%d.parquet", i)
+ }
+
+ deletedCh := make(chan struct{})
+ var deleted []string
+ var deleteErr error
+ go func() {
+ deleted, deleteErr = bfs.DeleteFiles(context.Background(), paths)
+ close(deletedCh)
+ }()
+
+ for i := 0; i < deleteFilesMaxConcurrency; i++ {
+ select {
+ case <-started:
+ case <-time.After(5 * time.Second):
+ t.Fatal("DeleteFiles did not start the expected concurrent deletes")
+ }
+ }
+
+ tracker.mu.Lock()
+ maxInFlight := tracker.maxInFlight
+ tracker.mu.Unlock()
+ assert.Equal(t, deleteFilesMaxConcurrency, maxInFlight)
+
+ select {
+ case <-started:
+ t.Fatal("DeleteFiles exceeded its concurrency limit")
+ default:
+ }
+
+ close(tracker.release)
+ select {
+ case <-deletedCh:
+ case <-time.After(5 * time.Second):
+ t.Fatal("DeleteFiles did not finish after deletes were released")
+ }
+
+ require.NoError(t, deleteErr)
+ assert.Equal(t, paths, deleted)
+}
+
func TestBlobFileIOStat(t *testing.T) {
ctx := context.Background()
bucket := memblob.OpenBucket(nil)