(iceberg-go) branch main updated: chore(s3): update to remove deprecated s3 sdk (#122)

[et...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

etudenhoefner pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-go.git


The following commit(s) were added to refs/heads/main by this push:
     new a11721c  chore(s3): update to remove deprecated s3 sdk (#122)
a11721c is described below

commit a11721ce82ccaa3c8bbbe0957654deb363fce1c3
Author: Matt Topol <zo...@gmail.com>
AuthorDate: Sun Aug 18 06:22:16 2024 -0400

    chore(s3): update to remove deprecated s3 sdk (#122)
---
 io/s3.go | 46 ++++++++++++++++++++--------------------------
 1 file changed, 20 insertions(+), 26 deletions(-)

diff --git a/io/s3.go b/io/s3.go
index a2bab6c..6702080 100644
--- a/io/s3.go
+++ b/io/s3.go
@@ -25,7 +25,6 @@ import (
 	"os"
 	"strings"
 
-	"github.com/aws/aws-sdk-go-v2/aws"
 	awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http"
 	"github.com/aws/aws-sdk-go-v2/config"
 	"github.com/aws/aws-sdk-go-v2/credentials"
@@ -45,43 +44,38 @@ const (
 )
 
 func createS3FileIO(parsed *url.URL, props map[string]string) (IO, error) {
-	opts := []func(*config.LoadOptions) error{}
+	cfgOpts := []func(*config.LoadOptions) error{}
+	opts := []func(*s3.Options){}
+
 	endpoint, ok := props[S3EndpointURL]
 	if !ok {
 		endpoint = os.Getenv("AWS_S3_ENDPOINT")
 	}
 
 	if endpoint != "" {
-		opts = append(opts, config.WithEndpointResolverWithOptions(aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) {
-			if service != s3.ServiceID {
-				// fallback to default resolution for the service
-				return aws.Endpoint{}, &aws.EndpointNotFoundError{}
-			}
-
-			return aws.Endpoint{
-				URL:               endpoint,
-				SigningRegion:     region,
-				HostnameImmutable: true,
-			}, nil
-		})))
+		opts = append(opts, func(o *s3.Options) {
+			o.BaseEndpoint = &endpoint
+		})
 	}
 
 	if tok, ok := props["token"]; ok {
-		opts = append(opts, config.WithBearerAuthTokenProvider(
+		cfgOpts = append(cfgOpts, config.WithBearerAuthTokenProvider(
 			&bearer.StaticTokenProvider{Token: bearer.Token{Value: tok}}))
 	}
 
 	if region, ok := props[S3Region]; ok {
-		opts = append(opts, config.WithRegion(region))
+		opts = append(opts, func(o *s3.Options) { o.Region = region })
 	} else if region, ok := props["client.region"]; ok {
-		opts = append(opts, config.WithRegion(region))
+		opts = append(opts, func(o *s3.Options) { o.Region = region })
 	}
 
 	accessKey, secretAccessKey := props[S3AccessKeyID], props[S3SecretAccessKey]
 	token := props[S3SessionToken]
 	if accessKey != "" || secretAccessKey != "" || token != "" {
-		opts = append(opts, config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(
-			props[S3AccessKeyID], props[S3SecretAccessKey], props[S3SessionToken])))
+		opts = append(opts, func(o *s3.Options) {
+			o.Credentials = credentials.NewStaticCredentialsProvider(
+				props[S3AccessKeyID], props[S3SecretAccessKey], props[S3SessionToken])
+		})
 	}
 
 	if proxy, ok := props[S3ProxyURI]; ok {
@@ -90,18 +84,18 @@ func createS3FileIO(parsed *url.URL, props map[string]string) (IO, error) {
 			return nil, fmt.Errorf("invalid s3 proxy url '%s'", proxy)
 		}
 
-		opts = append(opts, config.WithHTTPClient(awshttp.NewBuildableClient().WithTransportOptions(
-			func(t *http.Transport) {
-				t.Proxy = http.ProxyURL(proxyURL)
-			},
-		)))
+		opts = append(opts, func(o *s3.Options) {
+			o.HTTPClient = awshttp.NewBuildableClient().WithTransportOptions(
+				func(t *http.Transport) { t.Proxy = http.ProxyURL(proxyURL) })
+		})
 	}
 
-	awscfg, err := config.LoadDefaultConfig(context.Background(), opts...)
+	awscfg, err := config.LoadDefaultConfig(context.Background(), cfgOpts...)
 	if err != nil {
 		return nil, err
 	}
 
+	s3Client := s3.NewFromConfig(awscfg, opts...)
 	preprocess := func(n string) string {
 		_, after, found := strings.Cut(n, "://")
 		if found {
@@ -111,6 +105,6 @@ func createS3FileIO(parsed *url.URL, props map[string]string) (IO, error) {
 		return strings.TrimPrefix(n, parsed.Host)
 	}
 
-	s3fs := s3iofs.New(parsed.Host, awscfg)
+	s3fs := s3iofs.NewWithClient(parsed.Host, s3Client)
 	return FSPreProcName(s3fs, preprocess), nil
 }