(iceberg-go) branch main updated: feat: added override of http client as well as readme on utils.WithAw… (#981)

[at...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

laskoviymishka pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-go.git


The following commit(s) were added to refs/heads/main by this push:
     new 89f36c16 feat: added override of http client as well as readme on utils.WithAw… (#981)
89f36c16 is described below

commit 89f36c164c3e1cd65c7b3857b16464d779b0d522
Author: Ankit Sharma <67...@users.noreply.github.com>
AuthorDate: Sun May 10 23:15:03 2026 +0530

    feat: added override of http client as well as readme on utils.WithAw… (#981)
    
    This pr intends to solve issue:
    (https://github.com/apache/iceberg-go/issues/958)
    
    -> overriding of the HTTP client when no configuration is provided
    -> as well as adding a comment on the glue NewCatalog function
---
 catalog/glue/glue.go | 11 +++++++++++
 io/gocloud/s3.go     | 15 +++++++++++++++
 2 files changed, 26 insertions(+)

diff --git a/catalog/glue/glue.go b/catalog/glue/glue.go
index 2167651a..0611aef3 100644
--- a/catalog/glue/glue.go
+++ b/catalog/glue/glue.go
@@ -145,6 +145,17 @@ type Catalog struct {
 }
 
 // NewCatalog creates a new instance of glue.Catalog with the given options.
+// To override the AWS config below is an example:
+//
+// import awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http"
+//
+// client := awshttp.NewBuildableClient().WithTransportOptions(func(t *http.Transport) {
+//     t.IdleConnTimeout     = 90 * time.Second
+//     ...other transport options...
+// })
+// awsCfg.HTTPClient = client
+// catalog := glue.NewCatalog(glue.WithAwsConfig(awsCfg))
+
 func NewCatalog(opts ...Option) *Catalog {
 	glueOps := &options{}
 
diff --git a/io/gocloud/s3.go b/io/gocloud/s3.go
index ea979d5d..309513fc 100644
--- a/io/gocloud/s3.go
+++ b/io/gocloud/s3.go
@@ -26,6 +26,7 @@ import (
 	"os"
 	"slices"
 	"strconv"
+	"time"
 
 	"github.com/apache/iceberg-go/io"
 	"github.com/apache/iceberg-go/utils"
@@ -131,6 +132,20 @@ func createS3Bucket(ctx context.Context, parsed *url.URL, props map[string]strin
 		}
 	}
 
+	// Default HTTP client when not configured: use the SDK buildable client so
+	// proxy, TLS, dial, and HTTP/2 behavior match the usual AWS defaults, but
+	// raise per-host idle limits (Go's DefaultTransport uses 2 per host).
+	if awscfg.HTTPClient == nil {
+		awscfg.HTTPClient = awshttp.NewBuildableClient().WithTransportOptions(
+			func(t *http.Transport) {
+				t.MaxIdleConns = 256
+				t.MaxIdleConnsPerHost = 256
+				t.MaxConnsPerHost = 256
+				t.IdleConnTimeout = 90 * time.Second
+			},
+		)
+	}
+
 	endpoint, ok := props[io.S3EndpointURL]
 	if !ok {
 		endpoint = os.Getenv("AWS_S3_ENDPOINT")