You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by et...@apache.org on 2025/01/15 10:42:47 UTC
(iceberg) branch 1.7.x updated: AWS, Core, GCP: Support relative credential endpoint / pass OAuth2 token to credential provider (#11971)
This is an automated email from the ASF dual-hosted git repository.
etudenhoefner pushed a commit to branch 1.7.x
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/1.7.x by this push:
new 06bbdb1ec3 AWS, Core, GCP: Support relative credential endpoint / pass OAuth2 token to credential provider (#11971)
06bbdb1ec3 is described below
commit 06bbdb1ec3841b7208e7e6161a6e0453eca648de
Author: Eduard Tudenhoefner <et...@gmail.com>
AuthorDate: Wed Jan 15 11:42:41 2025 +0100
AWS, Core, GCP: Support relative credential endpoint / pass OAuth2 token to credential provider (#11971)
---
.../apache/iceberg/aws/AwsClientProperties.java | 16 ++++-
.../iceberg/aws/AwsClientPropertiesTest.java | 78 ++++++++++++++++++++++
.../java/org/apache/iceberg/rest/RESTUtil.java | 27 ++++++++
.../java/org/apache/iceberg/rest/TestRESTUtil.java | 27 ++++++++
.../java/org/apache/iceberg/gcp/GCPProperties.java | 7 +-
5 files changed, 153 insertions(+), 2 deletions(-)
diff --git a/aws/src/main/java/org/apache/iceberg/aws/AwsClientProperties.java b/aws/src/main/java/org/apache/iceberg/aws/AwsClientProperties.java
index 4f2d4d6a5a..b14fe4747d 100644
--- a/aws/src/main/java/org/apache/iceberg/aws/AwsClientProperties.java
+++ b/aws/src/main/java/org/apache/iceberg/aws/AwsClientProperties.java
@@ -20,12 +20,17 @@ package org.apache.iceberg.aws;
import java.io.Serializable;
import java.util.Map;
+import java.util.Optional;
+import org.apache.iceberg.CatalogProperties;
import org.apache.iceberg.aws.s3.VendedCredentialsProvider;
import org.apache.iceberg.common.DynClasses;
import org.apache.iceberg.common.DynMethods;
import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.rest.RESTUtil;
+import org.apache.iceberg.rest.auth.OAuth2Properties;
import org.apache.iceberg.util.PropertyUtil;
+import org.apache.iceberg.util.SerializableMap;
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
import software.amazon.awssdk.auth.credentials.AwsSessionCredentials;
@@ -81,6 +86,7 @@ public class AwsClientProperties implements Serializable {
private final Map<String, String> clientCredentialsProviderProperties;
private final String refreshCredentialsEndpoint;
private final boolean refreshCredentialsEnabled;
+ private final Map<String, String> allProperties;
public AwsClientProperties() {
this.clientRegion = null;
@@ -88,14 +94,18 @@ public class AwsClientProperties implements Serializable {
this.clientCredentialsProviderProperties = null;
this.refreshCredentialsEndpoint = null;
this.refreshCredentialsEnabled = true;
+ this.allProperties = null;
}
public AwsClientProperties(Map<String, String> properties) {
+ this.allProperties = SerializableMap.copyOf(properties);
this.clientRegion = properties.get(CLIENT_REGION);
this.clientCredentialsProvider = properties.get(CLIENT_CREDENTIALS_PROVIDER);
this.clientCredentialsProviderProperties =
PropertyUtil.propertiesWithPrefix(properties, CLIENT_CREDENTIAL_PROVIDER_PREFIX);
- this.refreshCredentialsEndpoint = properties.get(REFRESH_CREDENTIALS_ENDPOINT);
+ this.refreshCredentialsEndpoint =
+ RESTUtil.resolveEndpoint(
+ properties.get(CatalogProperties.URI), properties.get(REFRESH_CREDENTIALS_ENDPOINT));
this.refreshCredentialsEnabled =
PropertyUtil.propertyAsBoolean(properties, REFRESH_CREDENTIALS_ENABLED, true);
}
@@ -157,6 +167,10 @@ public class AwsClientProperties implements Serializable {
if (refreshCredentialsEnabled && !Strings.isNullOrEmpty(refreshCredentialsEndpoint)) {
clientCredentialsProviderProperties.put(
VendedCredentialsProvider.URI, refreshCredentialsEndpoint);
+ Optional.ofNullable(allProperties.get(OAuth2Properties.TOKEN))
+ .ifPresent(
+ token ->
+ clientCredentialsProviderProperties.putIfAbsent(OAuth2Properties.TOKEN, token));
return credentialsProvider(VendedCredentialsProvider.class.getName());
}
diff --git a/aws/src/test/java/org/apache/iceberg/aws/AwsClientPropertiesTest.java b/aws/src/test/java/org/apache/iceberg/aws/AwsClientPropertiesTest.java
index 5cf9dd810c..64b91c0526 100644
--- a/aws/src/test/java/org/apache/iceberg/aws/AwsClientPropertiesTest.java
+++ b/aws/src/test/java/org/apache/iceberg/aws/AwsClientPropertiesTest.java
@@ -21,9 +21,11 @@ package org.apache.iceberg.aws;
import static org.assertj.core.api.Assertions.assertThat;
import java.util.Map;
+import org.apache.iceberg.CatalogProperties;
import org.apache.iceberg.aws.s3.VendedCredentialsProvider;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.rest.auth.OAuth2Properties;
import org.junit.jupiter.api.Test;
import org.mockito.ArgumentCaptor;
import org.mockito.Mockito;
@@ -140,4 +142,80 @@ public class AwsClientPropertiesTest {
assertThat(awsClientProperties.credentialsProvider("key", "secret", "token"))
.isInstanceOf(StaticCredentialsProvider.class);
}
+
+ @Test
+ public void refreshCredentialsEndpointWithOAuthToken() {
+ AwsClientProperties awsClientProperties =
+ new AwsClientProperties(
+ ImmutableMap.of(
+ AwsClientProperties.REFRESH_CREDENTIALS_ENDPOINT,
+ "http://localhost:1234/v1/credentials",
+ OAuth2Properties.TOKEN,
+ "oauth-token"));
+
+ AwsCredentialsProvider provider =
+ awsClientProperties.credentialsProvider("key", "secret", "token");
+ assertThat(provider).isInstanceOf(VendedCredentialsProvider.class);
+ VendedCredentialsProvider vendedCredentialsProvider = (VendedCredentialsProvider) provider;
+ assertThat(vendedCredentialsProvider)
+ .extracting("properties")
+ .isEqualTo(
+ ImmutableMap.of(
+ "credentials.uri",
+ "http://localhost:1234/v1/credentials",
+ OAuth2Properties.TOKEN,
+ "oauth-token"));
+ }
+
+ @Test
+ public void refreshCredentialsEndpointWithOverridingOAuthToken() {
+ AwsClientProperties awsClientProperties =
+ new AwsClientProperties(
+ ImmutableMap.of(
+ AwsClientProperties.REFRESH_CREDENTIALS_ENDPOINT,
+ "http://localhost:1234/v1/credentials",
+ OAuth2Properties.TOKEN,
+ "oauth-token",
+ "client.credentials-provider.token",
+ "specific-token"));
+
+ AwsCredentialsProvider provider =
+ awsClientProperties.credentialsProvider("key", "secret", "token");
+ assertThat(provider).isInstanceOf(VendedCredentialsProvider.class);
+ VendedCredentialsProvider vendedCredentialsProvider = (VendedCredentialsProvider) provider;
+ assertThat(vendedCredentialsProvider)
+ .extracting("properties")
+ .isEqualTo(
+ ImmutableMap.of(
+ "credentials.uri",
+ "http://localhost:1234/v1/credentials",
+ OAuth2Properties.TOKEN,
+ "specific-token"));
+ }
+
+ @Test
+ public void refreshCredentialsEndpointWithRelativePath() {
+ AwsClientProperties awsClientProperties =
+ new AwsClientProperties(
+ ImmutableMap.of(
+ CatalogProperties.URI,
+ "http://localhost:1234/v1",
+ AwsClientProperties.REFRESH_CREDENTIALS_ENDPOINT,
+ "/relative/credentials/endpoint",
+ OAuth2Properties.TOKEN,
+ "oauth-token"));
+
+ AwsCredentialsProvider provider =
+ awsClientProperties.credentialsProvider("key", "secret", "token");
+ assertThat(provider).isInstanceOf(VendedCredentialsProvider.class);
+ VendedCredentialsProvider vendedCredentialsProvider = (VendedCredentialsProvider) provider;
+ assertThat(vendedCredentialsProvider)
+ .extracting("properties")
+ .isEqualTo(
+ ImmutableMap.of(
+ "credentials.uri",
+ "http://localhost:1234/v1/relative/credentials/endpoint",
+ OAuth2Properties.TOKEN,
+ "oauth-token"));
+ }
}
diff --git a/core/src/main/java/org/apache/iceberg/rest/RESTUtil.java b/core/src/main/java/org/apache/iceberg/rest/RESTUtil.java
index fab01162ca..bbf17b6b20 100644
--- a/core/src/main/java/org/apache/iceberg/rest/RESTUtil.java
+++ b/core/src/main/java/org/apache/iceberg/rest/RESTUtil.java
@@ -215,4 +215,31 @@ public class RESTUtil {
return Namespace.of(levels);
}
+
+ /**
+ * Returns the catalog URI suffixed by the relative endpoint path. If the endpoint path is an
+ * absolute path, then the absolute endpoint path is returned without using the catalog URI.
+ *
+ * @param catalogUri The catalog URI that is typically passed through {@link
+ * org.apache.iceberg.CatalogProperties#URI}
+ * @param endpointPath Either an absolute or relative endpoint path
+ * @return The actual endpoint path if it's an absolute path or the catalog uri suffixed by the
+ * endpoint path if the path is relative.
+ */
+ public static String resolveEndpoint(String catalogUri, String endpointPath) {
+ if (null == endpointPath) {
+ return null;
+ }
+
+ if (null == catalogUri
+ || endpointPath.startsWith("http://")
+ || endpointPath.startsWith("https://")) {
+ return endpointPath;
+ }
+
+ return String.format(
+ "%s%s",
+ RESTUtil.stripTrailingSlash(catalogUri),
+ endpointPath.startsWith("/") ? endpointPath : "/" + endpointPath);
+ }
}
diff --git a/core/src/test/java/org/apache/iceberg/rest/TestRESTUtil.java b/core/src/test/java/org/apache/iceberg/rest/TestRESTUtil.java
index c7667d90ac..6eb8f59bf6 100644
--- a/core/src/test/java/org/apache/iceberg/rest/TestRESTUtil.java
+++ b/core/src/test/java/org/apache/iceberg/rest/TestRESTUtil.java
@@ -139,4 +139,31 @@ public class TestRESTUtil {
assertThat(RESTUtil.decodeFormData(formString)).isEqualTo(expected);
}
+
+ @Test
+ public void testNullEndpointPath() {
+ assertThat(RESTUtil.resolveEndpoint("http://catalog-uri", null)).isNull();
+ }
+
+ @Test
+ public void testAbsoluteEndpointPath() {
+ assertThat(
+ RESTUtil.resolveEndpoint("http://catalog-uri", "http://catalog-uri/refresh-endpoint"))
+ .isEqualTo("http://catalog-uri/refresh-endpoint");
+ assertThat(
+ RESTUtil.resolveEndpoint("http://catalog-uri/", "http://catalog-uri/refresh-endpoint"))
+ .isEqualTo("http://catalog-uri/refresh-endpoint");
+ }
+
+ @Test
+ public void testRelativeEndpointPath() {
+ assertThat(RESTUtil.resolveEndpoint(null, "/refresh-endpoint")).isEqualTo("/refresh-endpoint");
+ assertThat(RESTUtil.resolveEndpoint("http://catalog-uri", "/refresh-endpoint"))
+ .isEqualTo("http://catalog-uri/refresh-endpoint");
+ assertThat(RESTUtil.resolveEndpoint("http://catalog-uri/", "/refresh-endpoint"))
+ .isEqualTo("http://catalog-uri/refresh-endpoint");
+ assertThat(
+ RESTUtil.resolveEndpoint("http://catalog-uri/", "relative-endpoint/refresh-endpoint"))
+ .isEqualTo("http://catalog-uri/relative-endpoint/refresh-endpoint");
+ }
}
diff --git a/gcp/src/main/java/org/apache/iceberg/gcp/GCPProperties.java b/gcp/src/main/java/org/apache/iceberg/gcp/GCPProperties.java
index c03906ae5d..cbc446525c 100644
--- a/gcp/src/main/java/org/apache/iceberg/gcp/GCPProperties.java
+++ b/gcp/src/main/java/org/apache/iceberg/gcp/GCPProperties.java
@@ -22,7 +22,9 @@ import java.io.Serializable;
import java.util.Date;
import java.util.Map;
import java.util.Optional;
+import org.apache.iceberg.CatalogProperties;
import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.rest.RESTUtil;
import org.apache.iceberg.util.PropertyUtil;
public class GCPProperties implements Serializable {
@@ -104,7 +106,10 @@ public class GCPProperties implements Serializable {
new Date(Long.parseLong(properties.get(GCS_OAUTH2_TOKEN_EXPIRES_AT)));
}
- gcsOauth2RefreshCredentialsEndpoint = properties.get(GCS_OAUTH2_REFRESH_CREDENTIALS_ENDPOINT);
+ gcsOauth2RefreshCredentialsEndpoint =
+ RESTUtil.resolveEndpoint(
+ properties.get(CatalogProperties.URI),
+ properties.get(GCS_OAUTH2_REFRESH_CREDENTIALS_ENDPOINT));
gcsOauth2RefreshCredentialsEnabled =
PropertyUtil.propertyAsBoolean(properties, GCS_OAUTH2_REFRESH_CREDENTIALS_ENABLED, true);
gcsNoAuth = Boolean.parseBoolean(properties.getOrDefault(GCS_NO_AUTH, "false"));