You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by ja...@apache.org on 2022/03/23 23:48:23 UTC
[iceberg] branch master updated: AWS: support choosing Apache HTTP client as default (#4371)
This is an automated email from the ASF dual-hosted git repository.
jackye pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/master by this push:
new 578b443 AWS: support choosing Apache HTTP client as default (#4371)
578b443 is described below
commit 578b4436fdb890176e6daae30ca02420165e64cd
Author: Xiaoxuan <xi...@amazon.com>
AuthorDate: Wed Mar 23 16:48:08 2022 -0700
AWS: support choosing Apache HTTP client as default (#4371)
---
.../aws/TestAssumeRoleAwsClientFactory.java | 1 +
.../iceberg/aws/AssumeRoleAwsClientFactory.java | 14 ++++++++---
.../org/apache/iceberg/aws/AwsClientFactories.java | 29 +++++++++++++++++++---
.../java/org/apache/iceberg/aws/AwsProperties.java | 22 ++++++++++++++++
build.gradle | 1 +
5 files changed, 60 insertions(+), 7 deletions(-)
diff --git a/aws/src/integration/java/org/apache/iceberg/aws/TestAssumeRoleAwsClientFactory.java b/aws/src/integration/java/org/apache/iceberg/aws/TestAssumeRoleAwsClientFactory.java
index 30b0056..c853fb3 100644
--- a/aws/src/integration/java/org/apache/iceberg/aws/TestAssumeRoleAwsClientFactory.java
+++ b/aws/src/integration/java/org/apache/iceberg/aws/TestAssumeRoleAwsClientFactory.java
@@ -74,6 +74,7 @@ public class TestAssumeRoleAwsClientFactory {
.build());
assumeRoleProperties = Maps.newHashMap();
assumeRoleProperties.put(AwsProperties.CLIENT_FACTORY, AssumeRoleAwsClientFactory.class.getName());
+ assumeRoleProperties.put(AwsProperties.HTTP_CLIENT_TYPE, AwsProperties.HTTP_CLIENT_TYPE_APACHE);
assumeRoleProperties.put(AwsProperties.CLIENT_ASSUME_ROLE_REGION, "us-east-1");
assumeRoleProperties.put(AwsProperties.CLIENT_ASSUME_ROLE_ARN, response.role().arn());
assumeRoleProperties.put(AwsProperties.CLIENT_ASSUME_ROLE_TAGS_PREFIX + "key1", "value1");
diff --git a/aws/src/main/java/org/apache/iceberg/aws/AssumeRoleAwsClientFactory.java b/aws/src/main/java/org/apache/iceberg/aws/AssumeRoleAwsClientFactory.java
index 1433761..680eb61 100644
--- a/aws/src/main/java/org/apache/iceberg/aws/AssumeRoleAwsClientFactory.java
+++ b/aws/src/main/java/org/apache/iceberg/aws/AssumeRoleAwsClientFactory.java
@@ -27,7 +27,6 @@ import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
import org.apache.iceberg.util.PropertyUtil;
import software.amazon.awssdk.awscore.client.builder.AwsClientBuilder;
import software.amazon.awssdk.awscore.client.builder.AwsSyncClientBuilder;
-import software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient;
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.dynamodb.DynamoDbClient;
import software.amazon.awssdk.services.glue.GlueClient;
@@ -46,6 +45,7 @@ public class AssumeRoleAwsClientFactory implements AwsClientFactory {
private int timeout;
private String region;
private String s3Endpoint;
+ private String httpClientType;
@Override
public S3Client s3() {
@@ -84,6 +84,8 @@ public class AssumeRoleAwsClientFactory implements AwsClientFactory {
this.s3Endpoint = properties.get(AwsProperties.S3FILEIO_ENDPOINT);
this.tags = toTags(properties);
+ this.httpClientType = PropertyUtil.propertyAsString(properties,
+ AwsProperties.HTTP_CLIENT_TYPE, AwsProperties.HTTP_CLIENT_TYPE_DEFAULT);
}
private <T extends AwsClientBuilder & AwsSyncClientBuilder> T configure(T clientBuilder) {
@@ -97,16 +99,22 @@ public class AssumeRoleAwsClientFactory implements AwsClientFactory {
clientBuilder.credentialsProvider(
StsAssumeRoleCredentialsProvider.builder()
- .stsClient(StsClient.builder().httpClientBuilder(UrlConnectionHttpClient.builder()).build())
+ .stsClient(sts())
.refreshRequest(request)
.build());
clientBuilder.region(Region.of(region));
- clientBuilder.httpClientBuilder(UrlConnectionHttpClient.builder());
+ clientBuilder.httpClientBuilder(AwsClientFactories.configureHttpClientBuilder(httpClientType));
return clientBuilder;
}
+ private StsClient sts() {
+ return StsClient.builder()
+ .httpClientBuilder(AwsClientFactories.configureHttpClientBuilder(httpClientType))
+ .build();
+ }
+
private String genSessionName() {
return String.format("iceberg-aws-%s", UUID.randomUUID());
}
diff --git a/aws/src/main/java/org/apache/iceberg/aws/AwsClientFactories.java b/aws/src/main/java/org/apache/iceberg/aws/AwsClientFactories.java
index fdbc258..3969b4b 100644
--- a/aws/src/main/java/org/apache/iceberg/aws/AwsClientFactories.java
+++ b/aws/src/main/java/org/apache/iceberg/aws/AwsClientFactories.java
@@ -23,6 +23,7 @@ import java.net.URI;
import java.util.Map;
import org.apache.iceberg.common.DynConstructors;
import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
import org.apache.iceberg.util.PropertyUtil;
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
@@ -30,6 +31,8 @@ import software.amazon.awssdk.auth.credentials.AwsSessionCredentials;
import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider;
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
import software.amazon.awssdk.core.client.builder.SdkClientBuilder;
+import software.amazon.awssdk.http.SdkHttpClient;
+import software.amazon.awssdk.http.apache.ApacheHttpClient;
import software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient;
import software.amazon.awssdk.services.dynamodb.DynamoDbClient;
import software.amazon.awssdk.services.glue.GlueClient;
@@ -80,6 +83,7 @@ public class AwsClientFactories {
private String s3AccessKeyId;
private String s3SecretAccessKey;
private String s3SessionToken;
+ private String httpClientType;
DefaultAwsClientFactory() {
}
@@ -87,7 +91,7 @@ public class AwsClientFactories {
@Override
public S3Client s3() {
return S3Client.builder()
- .httpClientBuilder(UrlConnectionHttpClient.builder())
+ .httpClientBuilder(configureHttpClientBuilder(httpClientType))
.applyMutation(builder -> configureEndpoint(builder, s3Endpoint))
.credentialsProvider(credentialsProvider(s3AccessKeyId, s3SecretAccessKey, s3SessionToken))
.build();
@@ -95,17 +99,17 @@ public class AwsClientFactories {
@Override
public GlueClient glue() {
- return GlueClient.builder().httpClientBuilder(UrlConnectionHttpClient.builder()).build();
+ return GlueClient.builder().httpClientBuilder(configureHttpClientBuilder(httpClientType)).build();
}
@Override
public KmsClient kms() {
- return KmsClient.builder().httpClientBuilder(UrlConnectionHttpClient.builder()).build();
+ return KmsClient.builder().httpClientBuilder(configureHttpClientBuilder(httpClientType)).build();
}
@Override
public DynamoDbClient dynamo() {
- return DynamoDbClient.builder().httpClientBuilder(UrlConnectionHttpClient.builder()).build();
+ return DynamoDbClient.builder().httpClientBuilder(configureHttpClientBuilder(httpClientType)).build();
}
@Override
@@ -118,6 +122,23 @@ public class AwsClientFactories {
ValidationException.check((s3AccessKeyId == null && s3SecretAccessKey == null) ||
(s3AccessKeyId != null && s3SecretAccessKey != null),
"S3 client access key ID and secret access key must be set at the same time");
+ this.httpClientType = PropertyUtil.propertyAsString(properties,
+ AwsProperties.HTTP_CLIENT_TYPE, AwsProperties.HTTP_CLIENT_TYPE_DEFAULT);
+ }
+ }
+
+ static SdkHttpClient.Builder configureHttpClientBuilder(String httpClientType) {
+ String clientType = httpClientType;
+ if (Strings.isNullOrEmpty(clientType)) {
+ clientType = AwsProperties.HTTP_CLIENT_TYPE_DEFAULT;
+ }
+ switch (clientType) {
+ case AwsProperties.HTTP_CLIENT_TYPE_URLCONNECTION:
+ return UrlConnectionHttpClient.builder();
+ case AwsProperties.HTTP_CLIENT_TYPE_APACHE:
+ return ApacheHttpClient.builder();
+ default:
+ throw new IllegalArgumentException("Unrecognized HTTP client type " + httpClientType);
}
}
diff --git a/aws/src/main/java/org/apache/iceberg/aws/AwsProperties.java b/aws/src/main/java/org/apache/iceberg/aws/AwsProperties.java
index 5112f51..8997c1b 100644
--- a/aws/src/main/java/org/apache/iceberg/aws/AwsProperties.java
+++ b/aws/src/main/java/org/apache/iceberg/aws/AwsProperties.java
@@ -246,6 +246,28 @@ public class AwsProperties implements Serializable {
public static final String CLIENT_ASSUME_ROLE_REGION = "client.assume-role.region";
/**
+ * The type of {@link software.amazon.awssdk.http.SdkHttpClient} implementation used by {@link AwsClientFactory}
+ * If set, all AWS clients will use this specified HTTP client.
+ * If not set, {@link #HTTP_CLIENT_TYPE_DEFAULT} will be used.
+ * For specific types supported, see HTTP_CLIENT_TYPE_* defined below.
+ */
+ public static final String HTTP_CLIENT_TYPE = "http-client.type";
+
+ /**
+ * If this is set under {@link #HTTP_CLIENT_TYPE},
+ * {@link software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient}
+ * will be used as the HTTP Client in {@link AwsClientFactory}
+ */
+ public static final String HTTP_CLIENT_TYPE_URLCONNECTION = "urlconnection";
+
+ /**
+ * If this is set under {@link #HTTP_CLIENT_TYPE}, {@link software.amazon.awssdk.http.apache.ApacheHttpClient}
+ * will be used as the HTTP Client in {@link AwsClientFactory}
+ */
+ public static final String HTTP_CLIENT_TYPE_APACHE = "apache";
+ public static final String HTTP_CLIENT_TYPE_DEFAULT = HTTP_CLIENT_TYPE_URLCONNECTION;
+
+ /**
* Used by {@link S3FileIO} to tag objects when writing. To set, we can pass a catalog property.
* <p>
* For more details, see https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-tagging.html
diff --git a/build.gradle b/build.gradle
index 46eeb2b..cc769f9 100644
--- a/build.gradle
+++ b/build.gradle
@@ -312,6 +312,7 @@ project(':iceberg-aws') {
implementation project(':iceberg-core')
compileOnly 'software.amazon.awssdk:url-connection-client'
+ compileOnly 'software.amazon.awssdk:apache-client'
compileOnly 'software.amazon.awssdk:s3'
compileOnly 'software.amazon.awssdk:kms'
compileOnly 'software.amazon.awssdk:glue'