You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by ze...@apache.org on 2026/08/14 16:01:07 UTC

(iceberg-go) branch main updated: perf(io): delete GoCloud files concurrently (#1811)

This is an automated email from the ASF dual-hosted git repository.

zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-go.git


The following commit(s) were added to refs/heads/main by this push:
     new 42ccd4635 perf(io): delete GoCloud files concurrently (#1811)
42ccd4635 is described below

commit 42ccd46350de4c192dc6370d80afe494a2abde39
Author: Minh Vu <vu...@gmail.com>
AuthorDate: Fri Aug 14 18:01:02 2026 +0200

    perf(io): delete GoCloud files concurrently (#1811)
    
    perf(io): delete gocloud files concurrently
---
 io/gocloud/blob.go      |  72 +++++++++++++++++++-------
 io/gocloud/blob_test.go | 131 ++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 186 insertions(+), 17 deletions(-)

diff --git a/io/gocloud/blob.go b/io/gocloud/blob.go
index 46a3083b6..ce4172ea5 100644
--- a/io/gocloud/blob.go
+++ b/io/gocloud/blob.go
@@ -26,6 +26,7 @@ import (
 	pathpkg "path"
 	"slices"
 	"strings"
+	"sync"
 	"time"
 
 	icebergio "github.com/apache/iceberg-go/io"
@@ -188,6 +189,10 @@ type BlobFileIO struct {
 
 var _ icebergio.ListableIO = (*BlobFileIO)(nil)
 
+// deleteFilesMaxConcurrency bounds the number of in-flight object-store
+// deletes without creating one goroutine per path for large cleanup jobs.
+const deleteFilesMaxConcurrency = 16
+
 type blobFileInfo struct {
 	name    string
 	size    int64
@@ -420,37 +425,70 @@ func (bfs *BlobFileIO) WalkDir(root string, fn fs.WalkDirFunc) error {
 	})
 }
 
+func (bfs *BlobFileIO) deleteFile(ctx context.Context, p string) (bool, error) {
+	key, err := bfs.preprocess(p)
+	if err != nil {
+		return false, fmt.Errorf("failed to delete %s: %w", p, err)
+	}
+
+	if err := bfs.Delete(ctx, key); err != nil {
+		// Missing files are not errors per the interface contract.
+		if gcerrors.Code(err) == gcerrors.NotFound {
+			return true, nil
+		}
+
+		return false, fmt.Errorf("failed to delete %s: %w", p, err)
+	}
+
+	return true, nil
+}
+
 func (bfs *BlobFileIO) DeleteFiles(ctx context.Context, paths []string) ([]string, error) {
 	if len(paths) == 0 {
 		return nil, nil
 	}
 
-	deleted := make([]string, 0, len(paths))
-
-	var errs error
+	type result struct {
+		deleted bool
+		err     error
+	}
 
-	for _, p := range paths {
-		key, err := bfs.preprocess(p)
-		if err != nil {
-			errs = errors.Join(errs, fmt.Errorf("failed to delete %s: %w", p, err))
+	results := make([]result, len(paths))
+	workers := len(paths)
+	if workers > deleteFilesMaxConcurrency {
+		workers = deleteFilesMaxConcurrency
+	}
 
-			continue
-		}
+	jobs := make(chan int)
+	var wg sync.WaitGroup
+	wg.Add(workers)
 
-		if err := bfs.Delete(ctx, key); err != nil {
-			// Missing files are not errors per the interface contract.
-			if gcerrors.Code(err) == gcerrors.NotFound {
-				deleted = append(deleted, p)
+	for i := 0; i < workers; i++ {
+		go func() {
+			defer wg.Done()
 
-				continue
+			for idx := range jobs {
+				results[idx].deleted, results[idx].err = bfs.deleteFile(ctx, paths[idx])
 			}
+		}()
+	}
 
-			errs = errors.Join(errs, fmt.Errorf("failed to delete %s: %w", p, err))
+	for idx := range paths {
+		jobs <- idx
+	}
+	close(jobs)
+	wg.Wait()
 
-			continue
+	deleted := make([]string, 0, len(paths))
+	var errs error
+	for idx, result := range results {
+		if result.deleted {
+			deleted = append(deleted, paths[idx])
 		}
 
-		deleted = append(deleted, p)
+		if result.err != nil {
+			errs = errors.Join(errs, result.err)
+		}
 	}
 
 	return deleted, errs
diff --git a/io/gocloud/blob_test.go b/io/gocloud/blob_test.go
index ba6bf399d..4f4e60ca5 100644
--- a/io/gocloud/blob_test.go
+++ b/io/gocloud/blob_test.go
@@ -19,17 +19,23 @@ package gocloud
 
 import (
 	"context"
+	"errors"
+	"fmt"
 	"io"
 	"io/fs"
 	"net/url"
+	"sync"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
 	icebergio "github.com/apache/iceberg-go/io"
 	"gocloud.dev/blob"
+	"gocloud.dev/blob/driver"
 	"gocloud.dev/blob/memblob"
+	"gocloud.dev/gcerrors"
 )
 
 func TestDefaultKeyExtractor(t *testing.T) {
@@ -725,6 +731,131 @@ func TestBlobFileIODeleteFilesEmpty(t *testing.T) {
 	assert.Nil(t, deleted)
 }
 
+type trackingDeleteBucket struct {
+	started chan<- struct{}
+	release chan struct{}
+
+	mu          sync.Mutex
+	inFlight    int
+	maxInFlight int
+}
+
+var errTrackingDeleteUnsupported = errors.New("unsupported test operation")
+
+func (b *trackingDeleteBucket) ErrorCode(error) gcerrors.ErrorCode { return gcerrors.Unknown }
+func (b *trackingDeleteBucket) As(any) bool                        { return false }
+func (b *trackingDeleteBucket) ErrorAs(error, any) bool            { return false }
+func (b *trackingDeleteBucket) Attributes(context.Context, string) (*driver.Attributes, error) {
+	return nil, errTrackingDeleteUnsupported
+}
+
+func (b *trackingDeleteBucket) ListPaged(context.Context, *driver.ListOptions) (*driver.ListPage, error) {
+	return nil, errTrackingDeleteUnsupported
+}
+
+func (b *trackingDeleteBucket) NewRangeReader(context.Context, string, int64, int64, *driver.ReaderOptions) (driver.Reader, error) {
+	return nil, errTrackingDeleteUnsupported
+}
+
+func (b *trackingDeleteBucket) NewTypedWriter(context.Context, string, string, *driver.WriterOptions) (driver.Writer, error) {
+	return nil, errTrackingDeleteUnsupported
+}
+
+func (b *trackingDeleteBucket) Copy(context.Context, string, string, *driver.CopyOptions) error {
+	return errTrackingDeleteUnsupported
+}
+
+func (b *trackingDeleteBucket) Delete(ctx context.Context, _ string) error {
+	b.mu.Lock()
+	b.inFlight++
+	if b.inFlight > b.maxInFlight {
+		b.maxInFlight = b.inFlight
+	}
+	b.mu.Unlock()
+	defer func() {
+		b.mu.Lock()
+		b.inFlight--
+		b.mu.Unlock()
+	}()
+
+	select {
+	case b.started <- struct{}{}:
+	case <-ctx.Done():
+		return ctx.Err()
+	}
+
+	select {
+	case <-b.release:
+		return nil
+	case <-ctx.Done():
+		return ctx.Err()
+	}
+}
+
+func (b *trackingDeleteBucket) SignedURL(context.Context, string, *driver.SignedURLOptions) (string, error) {
+	return "", errTrackingDeleteUnsupported
+}
+func (b *trackingDeleteBucket) Close() error { return nil }
+
+func TestBlobFileIODeleteFilesIsConcurrentAndBounded(t *testing.T) {
+	const pathCount = deleteFilesMaxConcurrency * 2
+
+	started := make(chan struct{}, pathCount)
+	tracker := &trackingDeleteBucket{
+		started: started,
+		release: make(chan struct{}),
+	}
+	bucket := blob.NewBucket(tracker)
+	defer bucket.Close()
+
+	bfs := &BlobFileIO{
+		Bucket:        bucket,
+		extractObject: defaultObjectLocationExtractor("test-bucket"),
+		ctx:           context.Background(),
+	}
+	paths := make([]string, pathCount)
+	for i := range paths {
+		paths[i] = fmt.Sprintf("s3://test-bucket/data/file%d.parquet", i)
+	}
+
+	deletedCh := make(chan struct{})
+	var deleted []string
+	var deleteErr error
+	go func() {
+		deleted, deleteErr = bfs.DeleteFiles(context.Background(), paths)
+		close(deletedCh)
+	}()
+
+	for i := 0; i < deleteFilesMaxConcurrency; i++ {
+		select {
+		case <-started:
+		case <-time.After(5 * time.Second):
+			t.Fatal("DeleteFiles did not start the expected concurrent deletes")
+		}
+	}
+
+	tracker.mu.Lock()
+	maxInFlight := tracker.maxInFlight
+	tracker.mu.Unlock()
+	assert.Equal(t, deleteFilesMaxConcurrency, maxInFlight)
+
+	select {
+	case <-started:
+		t.Fatal("DeleteFiles exceeded its concurrency limit")
+	default:
+	}
+
+	close(tracker.release)
+	select {
+	case <-deletedCh:
+	case <-time.After(5 * time.Second):
+		t.Fatal("DeleteFiles did not finish after deletes were released")
+	}
+
+	require.NoError(t, deleteErr)
+	assert.Equal(t, paths, deleted)
+}
+
 func TestBlobFileIOStat(t *testing.T) {
 	ctx := context.Background()
 	bucket := memblob.OpenBucket(nil)