You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by et...@apache.org on 2025/03/13 05:24:02 UTC
(iceberg) branch main updated: AWS: Don't fetch credential from endpoint if properties contain a valid credential (#12504)
This is an automated email from the ASF dual-hosted git repository.
etudenhoefner pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/main by this push:
new 9a98de0410 AWS: Don't fetch credential from endpoint if properties contain a valid credential (#12504)
9a98de0410 is described below
commit 9a98de04103755ad4833ac6c3918c39022d54a99
Author: Eduard Tudenhoefner <et...@gmail.com>
AuthorDate: Thu Mar 13 06:23:56 2025 +0100
AWS: Don't fetch credential from endpoint if properties contain a valid credential (#12504)
When the `VendedCredentialProvider` is created, the `properties` typically already contain a valid credential from the first time a table is loaded.
This PR uses the credential from the properties and otherwise falls back to loading a valid credential from the refresh endpoint in case the credential in `properties` is incomplete or expired.
---
.../iceberg/aws/s3/VendedCredentialsProvider.java | 37 ++++-
.../aws/s3/TestVendedCredentialsProvider.java | 170 +++++++++++++++++++++
2 files changed, 206 insertions(+), 1 deletion(-)
diff --git a/aws/src/main/java/org/apache/iceberg/aws/s3/VendedCredentialsProvider.java b/aws/src/main/java/org/apache/iceberg/aws/s3/VendedCredentialsProvider.java
index 792a6ff27b..769d187875 100644
--- a/aws/src/main/java/org/apache/iceberg/aws/s3/VendedCredentialsProvider.java
+++ b/aws/src/main/java/org/apache/iceberg/aws/s3/VendedCredentialsProvider.java
@@ -22,8 +22,10 @@ import java.time.Instant;
import java.time.temporal.ChronoUnit;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
import java.util.stream.Collectors;
import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
import org.apache.iceberg.rest.ErrorHandlers;
import org.apache.iceberg.rest.HTTPClient;
import org.apache.iceberg.rest.HTTPHeaders;
@@ -52,7 +54,7 @@ public class VendedCredentialsProvider implements AwsCredentialsProvider, SdkAut
Preconditions.checkArgument(null != properties.get(URI), "Invalid URI: null");
this.properties = properties;
this.credentialCache =
- CachedSupplier.builder(this::refreshCredential)
+ CachedSupplier.builder(() -> credentialFromProperties().orElseGet(this::refreshCredential))
.cachedValueName(VendedCredentialsProvider.class.getName())
.build();
}
@@ -101,6 +103,39 @@ public class VendedCredentialsProvider implements AwsCredentialsProvider, SdkAut
ErrorHandlers.defaultErrorHandler());
}
+ private Optional<RefreshResult<AwsCredentials>> credentialFromProperties() {
+ String accessKeyId = properties.get(S3FileIOProperties.ACCESS_KEY_ID);
+ String secretAccessKey = properties.get(S3FileIOProperties.SECRET_ACCESS_KEY);
+ String sessionToken = properties.get(S3FileIOProperties.SESSION_TOKEN);
+ String tokenExpiresAtMillis = properties.get(S3FileIOProperties.SESSION_TOKEN_EXPIRES_AT_MS);
+ if (Strings.isNullOrEmpty(accessKeyId)
+ || Strings.isNullOrEmpty(secretAccessKey)
+ || Strings.isNullOrEmpty(sessionToken)
+ || Strings.isNullOrEmpty(tokenExpiresAtMillis)) {
+ return Optional.empty();
+ }
+
+ Instant expiresAt = Instant.ofEpochMilli(Long.parseLong(tokenExpiresAtMillis));
+ Instant prefetchAt = expiresAt.minus(5, ChronoUnit.MINUTES);
+
+ if (Instant.now().isAfter(prefetchAt)) {
+ return Optional.empty();
+ }
+
+ return Optional.of(
+ RefreshResult.builder(
+ (AwsCredentials)
+ AwsSessionCredentials.builder()
+ .accessKeyId(accessKeyId)
+ .secretAccessKey(secretAccessKey)
+ .sessionToken(sessionToken)
+ .expirationTime(expiresAt)
+ .build())
+ .staleTime(expiresAt)
+ .prefetchTime(prefetchAt)
+ .build());
+ }
+
private RefreshResult<AwsCredentials> refreshCredential() {
LoadCredentialsResponse response = fetchCredentials();
diff --git a/aws/src/test/java/org/apache/iceberg/aws/s3/TestVendedCredentialsProvider.java b/aws/src/test/java/org/apache/iceberg/aws/s3/TestVendedCredentialsProvider.java
index 67cd1cb552..51aca88943 100644
--- a/aws/src/test/java/org/apache/iceberg/aws/s3/TestVendedCredentialsProvider.java
+++ b/aws/src/test/java/org/apache/iceberg/aws/s3/TestVendedCredentialsProvider.java
@@ -302,6 +302,176 @@ public class TestVendedCredentialsProvider {
}
}
+ @Test
+ public void nonExpiredTokenInProperties() {
+ HttpRequest mockRequest = request("/v1/credentials").withMethod(HttpMethod.GET.name());
+ String expiresAt = Long.toString(Instant.now().plus(10, ChronoUnit.HOURS).toEpochMilli());
+ Credential credentialFromProperties =
+ ImmutableCredential.builder()
+ .prefix("s3")
+ .config(
+ ImmutableMap.of(
+ S3FileIOProperties.ACCESS_KEY_ID,
+ "randomAccessKeyFromProperties",
+ S3FileIOProperties.SECRET_ACCESS_KEY,
+ "randomSecretAccessKeyFromProperties",
+ S3FileIOProperties.SESSION_TOKEN,
+ "sessionTokenFromProperties",
+ S3FileIOProperties.SESSION_TOKEN_EXPIRES_AT_MS,
+ expiresAt))
+ .build();
+
+ Credential credential =
+ ImmutableCredential.builder()
+ .prefix("s3")
+ .config(
+ ImmutableMap.of(
+ S3FileIOProperties.ACCESS_KEY_ID,
+ "randomAccessKey",
+ S3FileIOProperties.SECRET_ACCESS_KEY,
+ "randomSecretAccessKey",
+ S3FileIOProperties.SESSION_TOKEN,
+ "sessionToken",
+ S3FileIOProperties.SESSION_TOKEN_EXPIRES_AT_MS,
+ Long.toString(Instant.now().plus(1, ChronoUnit.HOURS).toEpochMilli())))
+ .build();
+ LoadCredentialsResponse response =
+ ImmutableLoadCredentialsResponse.builder().addCredentials(credential).build();
+
+ HttpResponse mockResponse =
+ response(LoadCredentialsResponseParser.toJson(response)).withStatusCode(200);
+ mockServer.when(mockRequest).respond(mockResponse);
+
+ try (VendedCredentialsProvider provider =
+ VendedCredentialsProvider.create(
+ ImmutableMap.of(
+ VendedCredentialsProvider.URI,
+ URI,
+ S3FileIOProperties.ACCESS_KEY_ID,
+ "randomAccessKeyFromProperties",
+ S3FileIOProperties.SECRET_ACCESS_KEY,
+ "randomSecretAccessKeyFromProperties",
+ S3FileIOProperties.SESSION_TOKEN,
+ "sessionTokenFromProperties",
+ S3FileIOProperties.SESSION_TOKEN_EXPIRES_AT_MS,
+ expiresAt))) {
+ AwsCredentials awsCredentials = provider.resolveCredentials();
+
+ verifyCredentials(awsCredentials, credentialFromProperties);
+
+ for (int i = 0; i < 5; i++) {
+ // resolving credentials multiple times should not hit the credentials endpoint again
+ assertThat(provider.resolveCredentials()).isSameAs(awsCredentials);
+ }
+ }
+
+ // token endpoint isn't hit, because the credentials are extracted from the properties
+ mockServer.verify(mockRequest, VerificationTimes.never());
+ }
+
+ @Test
+ public void expiredTokenInProperties() {
+ HttpRequest mockRequest = request("/v1/credentials").withMethod(HttpMethod.GET.name());
+
+ Credential credential =
+ ImmutableCredential.builder()
+ .prefix("s3")
+ .config(
+ ImmutableMap.of(
+ S3FileIOProperties.ACCESS_KEY_ID,
+ "randomAccessKey",
+ S3FileIOProperties.SECRET_ACCESS_KEY,
+ "randomSecretAccessKey",
+ S3FileIOProperties.SESSION_TOKEN,
+ "sessionToken",
+ S3FileIOProperties.SESSION_TOKEN_EXPIRES_AT_MS,
+ Long.toString(Instant.now().plus(1, ChronoUnit.HOURS).toEpochMilli())))
+ .build();
+ LoadCredentialsResponse response =
+ ImmutableLoadCredentialsResponse.builder().addCredentials(credential).build();
+
+ HttpResponse mockResponse =
+ response(LoadCredentialsResponseParser.toJson(response)).withStatusCode(200);
+ mockServer.when(mockRequest).respond(mockResponse);
+
+ try (VendedCredentialsProvider provider =
+ VendedCredentialsProvider.create(
+ ImmutableMap.of(
+ VendedCredentialsProvider.URI,
+ URI,
+ S3FileIOProperties.ACCESS_KEY_ID,
+ "randomAccessKeyFromProperties",
+ S3FileIOProperties.SECRET_ACCESS_KEY,
+ "randomSecretAccessKeyFromProperties",
+ S3FileIOProperties.SESSION_TOKEN,
+ "sessionTokenFromProperties",
+ S3FileIOProperties.SESSION_TOKEN_EXPIRES_AT_MS,
+ Long.toString(Instant.now().minus(1, ChronoUnit.HOURS).toEpochMilli())))) {
+ AwsCredentials awsCredentials = provider.resolveCredentials();
+
+ verifyCredentials(awsCredentials, credential);
+
+ for (int i = 0; i < 5; i++) {
+ // resolving credentials multiple times should not hit the credentials endpoint again
+ assertThat(provider.resolveCredentials()).isSameAs(awsCredentials);
+ }
+ }
+
+ // token endpoint is hit once due to the properties containing an expired token
+ mockServer.verify(mockRequest, VerificationTimes.once());
+ }
+
+ @Test
+ public void invalidTokenInProperties() {
+ HttpRequest mockRequest = request("/v1/credentials").withMethod(HttpMethod.GET.name());
+
+ Credential credential =
+ ImmutableCredential.builder()
+ .prefix("s3")
+ .config(
+ ImmutableMap.of(
+ S3FileIOProperties.ACCESS_KEY_ID,
+ "randomAccessKey",
+ S3FileIOProperties.SECRET_ACCESS_KEY,
+ "randomSecretAccessKey",
+ S3FileIOProperties.SESSION_TOKEN,
+ "sessionToken",
+ S3FileIOProperties.SESSION_TOKEN_EXPIRES_AT_MS,
+ Long.toString(Instant.now().plus(1, ChronoUnit.HOURS).toEpochMilli())))
+ .build();
+ LoadCredentialsResponse response =
+ ImmutableLoadCredentialsResponse.builder().addCredentials(credential).build();
+
+ HttpResponse mockResponse =
+ response(LoadCredentialsResponseParser.toJson(response)).withStatusCode(200);
+ mockServer.when(mockRequest).respond(mockResponse);
+
+ // token expiration is missing from the properties
+ try (VendedCredentialsProvider provider =
+ VendedCredentialsProvider.create(
+ ImmutableMap.of(
+ VendedCredentialsProvider.URI,
+ URI,
+ S3FileIOProperties.ACCESS_KEY_ID,
+ "randomAccessKeyFromProperties",
+ S3FileIOProperties.SECRET_ACCESS_KEY,
+ "randomSecretAccessKeyFromProperties",
+ S3FileIOProperties.SESSION_TOKEN,
+ "sessionTokenFromProperties"))) {
+ AwsCredentials awsCredentials = provider.resolveCredentials();
+
+ verifyCredentials(awsCredentials, credential);
+
+ for (int i = 0; i < 5; i++) {
+ // resolving credentials multiple times should not hit the credentials endpoint again
+ assertThat(provider.resolveCredentials()).isSameAs(awsCredentials);
+ }
+ }
+
+ // token endpoint is hit once due to the properties not containing the token's expiration
+ mockServer.verify(mockRequest, VerificationTimes.once());
+ }
+
private void verifyCredentials(AwsCredentials awsCredentials, Credential credential) {
assertThat(awsCredentials).isInstanceOf(AwsSessionCredentials.class);
AwsSessionCredentials creds = (AwsSessionCredentials) awsCredentials;