You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@asterixdb.apache.org by AsterixDB Code Review <do...@asterix-gerrit.ics.uci.edu> on 2026/08/26 11:55:01 UTC
[L] Change in asterixdb[master]: [ASTERIXDB-3816][EXT]: Iceberg credential vending
From Hussain Towaileb <hu...@gmail.com>:
Hussain Towaileb has uploaded this change for review. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/21617?usp=email )
Change subject: [ASTERIXDB-3816][EXT]: Iceberg credential vending
......................................................................
[ASTERIXDB-3816][EXT]: Iceberg credential vending
Requests credential vending from an Iceberg REST catalog with the
X-Iceberg-Access-Delegation header, and leaves what the catalog
returns in place: no FileIO is configured and no AWS client factory
is installed for such a collection, since either would override the
vended credentials. Skipping the FileIO covers S3, GCS and ADLS
alike -- Iceberg then resolves one from the data file's location
scheme and refreshes the vended credentials through that cloud's
own provider.
Validates that vending is asked for only on an Iceberg table, and
only on a catalog source that is backed by a REST catalog.
Fixes the AWS client factory set for the Glue REST and S3 Tables
catalogs being applied regardless, which silently fell back to
ambient credentials instead of the vended ones.
Adds unit tests over both validations, the property passthrough,
the delegation header and the FileIO skip for every storage type,
plus a runtime test rejecting a vending request made against a
non-REST catalog source.
Ext-ref: MB-72435
Co-Authored-By: Claude Opus 5 <no...@anthropic.com>
Change-Id: I3bac5a7f83b033eb892c6fe61116415597635d0a
---
M asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/helpers/IcebergStatementValidationHelper.java
A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/iceberg/negative/vended-credentials/test.000.ddl.sqlpp
A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/iceberg/negative/vended-credentials/test.010.ddl.sqlpp
A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/iceberg/negative/vended-credentials/test.990.ddl.sqlpp
M asterixdb/asterix-app/src/test/resources/runtimets/testsuite_iceberg.xml
M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
M asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/iceberg/glue/GlueUtils.java
M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/iceberg/IcebergConstants.java
M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/iceberg/IcebergUtils.java
A asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/util/iceberg/IcebergVendedCredentialsTest.java
11 files changed, 522 insertions(+), 2 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/17/21617/1
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/helpers/IcebergStatementValidationHelper.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/helpers/IcebergStatementValidationHelper.java
index 7ac35b6..a8fea73 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/helpers/IcebergStatementValidationHelper.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/helpers/IcebergStatementValidationHelper.java
@@ -34,6 +34,7 @@
import org.apache.asterix.metadata.entities.Catalog;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.api.exceptions.SourceLocation;
+import org.apache.hyracks.util.annotations.AiProvenance;
public class IcebergStatementValidationHelper {
@@ -47,6 +48,7 @@
adapter);
}
+ @AiProvenance(agent = AiProvenance.Agent.CLAUDE_OPUS_5, tool = AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind = AiProvenance.ContributionKind.ASSISTED, notes = "Reject vended credentials on a catalog source that cannot vend them")
public static void validateIfIcebergTable(ICcApplicationContext appCtx, MetadataProvider metadataProvider,
MetadataTransactionContext mdTxnCtx, Map<String, String> properties, Map<String, String> extraProperties,
SourceLocation srcLoc, String adapter) throws AlgebricksException {
@@ -68,6 +70,8 @@
// validate snapshot exists if provided
properties.putAll(extraProperties);
metadataProvider.addIcebergCatalogPropertiesIfNeeded(appCtx, properties);
+ // only possible once the catalog properties are merged in, since that is where the source comes from
+ IcebergUtils.validateVendedCredentialsSource(properties, srcLoc);
IcebergSnapshotUtils.validateSnapshotExists(properties);
}
}
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/iceberg/negative/vended-credentials/test.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/iceberg/negative/vended-credentials/test.000.ddl.sqlpp
new file mode 100644
index 0000000..9e36f92
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/iceberg/negative/vended-credentials/test.000.ddl.sqlpp
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// A NESSIE (native) catalog. Vending is part of the Iceberg REST protocol, so this one cannot do it.
+CREATE CATALOG myNessieCatalog
+TYPE Iceberg
+SOURCE NESSIE
+WITH {
+ "uri": "%NESSIE_SERVER_URI%",
+ "warehouse": "s3://iceberg-container/nessie/warehouse"
+};
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/iceberg/negative/vended-credentials/test.010.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/iceberg/negative/vended-credentials/test.010.ddl.sqlpp
new file mode 100644
index 0000000..9537da9
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/iceberg/negative/vended-credentials/test.010.ddl.sqlpp
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// Asking a non-REST catalog to vend must be rejected at DDL time rather than failing on first read.
+CREATE TYPE vendedType AS open {};
+CREATE EXTERNAL COLLECTION vendedOnNonRestCatalog(vendedType)
+USING S3
+(
+ ("table-format"="iceberg"),
+ ("namespace"="my_namespace"),
+ ("tableName"="users"),
+ ("catalogName"="myNessieCatalog"),
+ ("vendedCredentials"="true"),
+ ("region"="custom-region"),
+ ("serviceEndpoint"="%DOCKER_S3_CONTAINER%"),
+ ("pathStyleAddressing"="true")
+);
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/iceberg/negative/vended-credentials/test.990.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/iceberg/negative/vended-credentials/test.990.ddl.sqlpp
new file mode 100644
index 0000000..9cd8289
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/iceberg/negative/vended-credentials/test.990.ddl.sqlpp
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+DROP COLLECTION vendedOnNonRestCatalog IF EXISTS;
+DROP TYPE vendedType IF EXISTS;
+DROP CATALOG myNessieCatalog;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_iceberg.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_iceberg.xml
index 9705e0e..3932e7b 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_iceberg.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_iceberg.xml
@@ -145,6 +145,13 @@
</compilation-unit>
</test-case>
<test-case FilePath="iceberg/negative">
+ <compilation-unit name="vended-credentials">
+ <output-dir compare="Text">vended-credentials</output-dir>
+ <!-- vending is part of the Iceberg REST protocol; a native NESSIE catalog cannot do it -->
+ <expected-error>ASX1263: Iceberg catalog source 'NESSIE' cannot vend credentials</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="iceberg/negative">
<compilation-unit name="bad-source-type">
<output-dir compare="Text">bad-source-type</output-dir>
<expected-error>ASX1230: Unsupported Iceberg catalog source: 'FAKE_AWS_GLUE'</expected-error>
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
index fdfa61f..9027b7a 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
@@ -371,6 +371,8 @@
UNSUPPORTED_MULTIPLE_STATEMENTS_FOR_DELIVERY_MODE(1259),
UNSUPPORTED_VECTOR_ELEMENT_TYPE(1260),
VECTOR_DIMENSION_MISMATCH(1261),
+ VENDED_CREDENTIALS_REQUIRE_ICEBERG_TABLE(1262),
+ VENDED_CREDENTIALS_UNSUPPORTED_CATALOG_SOURCE(1263),
// Feed errors
DATAFLOW_ILLEGAL_STATE(3001),
diff --git a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
index 0ff4d5a..c88c015 100644
--- a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
+++ b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
@@ -373,6 +373,8 @@
1259 = %1$s execution is not supported for a request with multiple statements
1260 = Unsupported type %1$s in the vector value; the supported element types are numbers
1261 = Vector dimension mismatch: expected %1$s dimensions, but got %2$s
+1262 = Vended credentials can only be used with an Iceberg table
+1263 = Iceberg catalog source '%1$s' cannot vend credentials
# Feed Errors
3001 = Illegal state.
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/iceberg/glue/GlueUtils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/iceberg/glue/GlueUtils.java
index 2e047d0..af1f34d 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/iceberg/glue/GlueUtils.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/iceberg/glue/GlueUtils.java
@@ -48,6 +48,7 @@
import org.apache.asterix.external.util.aws.iceberg.auth.EnsureCloseClientsRESTSigV4AuthManager;
import org.apache.asterix.external.util.iceberg.IcebergConstants;
import org.apache.asterix.external.util.iceberg.IcebergUtils;
+import org.apache.hyracks.util.annotations.AiProvenance;
import org.apache.iceberg.CatalogProperties;
import org.apache.iceberg.aws.glue.GlueCatalog;
import org.apache.iceberg.rest.RESTCatalog;
@@ -93,13 +94,20 @@
catalogProperties.put(CLIENT_FACTORY, EnsureCloseAWSClientFactory.class.getName());
}
+ @AiProvenance(agent = AiProvenance.Agent.CLAUDE_OPUS_5, tool = AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind = AiProvenance.ContributionKind.ASSISTED, notes = "Leave the storage client factory unset when the catalog vends the credentials, so the "
+ + "vended ones are what the table's FileIO is built from")
public static void setGlueRestCatalogProperties(Map<String, String> catalogProperties) throws CompilationException {
catalogProperties.put(CATALOG_IMPL, RESTCatalog.class.getName());
catalogProperties.put(URI, catalogProperties.get(ICEBERG_URI_PROPERTY_KEY));
+ // sigv4 reaches the catalog endpoint itself, so it stays ours whether or not credentials are vended
catalogProperties.put(AUTH_TYPE, EnsureCloseClientsRESTSigV4AuthManager.class.getName());
catalogProperties.put(REST_SIG4_SIGNING_NAME, getSigningName(catalogProperties));
catalogProperties.put(REST_SIG4_SIGNING_REGION, catalogProperties.get(REST_SIG4_SIGNING_REGION_PROPERTY_NAME));
- catalogProperties.put(CLIENT_FACTORY, EnsureCloseAWSClientFactory.class.getName());
+ if (!IcebergUtils.isVendedCredentials(catalogProperties)) {
+ // this factory builds the storage clients backing the table's FileIO from the collection's own
+ // credentials, which a vended-credentials collection does not have
+ catalogProperties.put(CLIENT_FACTORY, EnsureCloseAWSClientFactory.class.getName());
+ }
}
private static String getSigningName(Map<String, String> catalogProperties) throws CompilationException {
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/iceberg/IcebergConstants.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/iceberg/IcebergConstants.java
index a468e5b..ac1ea61 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/iceberg/IcebergConstants.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/iceberg/IcebergConstants.java
@@ -23,6 +23,7 @@
import org.apache.asterix.external.util.aws.AwsConstants;
import org.apache.asterix.external.util.azure.AzureConstants;
import org.apache.asterix.external.util.google.GCSConstants;
+import org.apache.hyracks.util.annotations.AiProvenance;
import org.apache.iceberg.aws.s3.S3FileIO;
import org.apache.iceberg.azure.adlsv2.ADLSFileIO;
@@ -47,6 +48,9 @@
public static final String ICEBERG_SNAPSHOT_ID_PROPERTY_KEY = "snapshotId";
public static final String ICEBERG_SCHEMA_ID_PROPERTY_KEY = "schemaId";
public static final String ICEBERG_SNAPSHOT_TIMESTAMP_PROPERTY_KEY = "snapshotTimestamp";
+ // Whether the collection is to use vended credentials, i.e. take its storage credentials from the catalog.
+ @AiProvenance(agent = AiProvenance.Agent.CLAUDE_OPUS_5, tool = AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind = AiProvenance.ContributionKind.GENERATED, notes = "Marker property for a collection whose storage credentials come from the catalog")
+ public static final String ICEBERG_VENDED_CREDENTIALS_PROPERTY_KEY = "vendedCredentials";
public static final String ICEBERG_CATALOG_PROPERTY_PREFIX_INTERNAL = "catalog-property#";
public static final String ICEBERG_COLLECTION_PROPERTY_PREFIX_INTERNAL = "collection-property#";
@@ -76,7 +80,11 @@
}
public static class Rest {
-
+ // Access delegation is requested as an ordinary HTTP header on catalog requests. Iceberg defines no
+ // constant for it, so it is passed to the REST client through its "header." catalog-property prefix.
+ @AiProvenance(agent = AiProvenance.Agent.CLAUDE_OPUS_5, tool = AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind = AiProvenance.ContributionKind.GENERATED, notes = "Catalog property that asks the REST catalog to vend storage credentials")
+ public static final String ACCESS_DELEGATION_HEADER_PROPERTY = "header.X-Iceberg-Access-Delegation";
+ public static final String ACCESS_DELEGATION_VENDED_CREDENTIALS = "vended-credentials";
}
// we need to split the creds for the catalog from the iceberg table, we will prefix the
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/iceberg/IcebergUtils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/iceberg/IcebergUtils.java
index 328717a..c999078 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/iceberg/IcebergUtils.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/iceberg/IcebergUtils.java
@@ -55,6 +55,7 @@
import org.apache.asterix.external.util.iceberg.nessie.NessieUtils;
import org.apache.asterix.external.util.iceberg.rest.RestUtils;
import org.apache.asterix.om.types.ARecordType;
+import org.apache.hyracks.api.exceptions.SourceLocation;
import org.apache.hyracks.util.annotations.AiProvenance;
import org.apache.iceberg.CatalogProperties;
import org.apache.iceberg.Table;
@@ -99,6 +100,71 @@
return false;
}
+ /**
+ * Checks whether the collection is to use vended credentials, i.e. take its storage credentials from the
+ * Iceberg catalog. This is independent of the credentials used to reach the catalog itself.
+ *
+ * @param configuration external data configuration
+ */
+ @AiProvenance(agent = AiProvenance.Agent.CLAUDE_OPUS_5, tool = AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind = AiProvenance.ContributionKind.GENERATED, notes = "Predicate for the vended-credentials marker, alongside the other Iceberg property semantics")
+ public static boolean isVendedCredentials(Map<String, String> configuration) {
+ return Boolean.parseBoolean(configuration.get(IcebergConstants.ICEBERG_VENDED_CREDENTIALS_PROPERTY_KEY));
+ }
+
+ /**
+ * Validates that a collection asking for vended credentials is an Iceberg table. The catalog is what vends
+ * the credentials, so the request is meaningless for any other format.
+ *
+ * @param configuration external data configuration, including any WITH-clause properties, since that is
+ * where the table format is declared
+ * @param sourceLoc location of the declaration, for error reporting
+ */
+ @AiProvenance(agent = AiProvenance.Agent.CLAUDE_OPUS_5, tool = AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind = AiProvenance.ContributionKind.GENERATED, notes = "Rejects vended credentials on a non-Iceberg collection")
+ public static void validateVendedCredentials(Map<String, String> configuration, SourceLocation sourceLoc)
+ throws CompilationException {
+ if (!isIcebergTable(configuration)) {
+ throw new CompilationException(ErrorCode.VENDED_CREDENTIALS_REQUIRE_ICEBERG_TABLE, sourceLoc);
+ }
+ }
+
+ /**
+ * Whether a catalog of the given source is able to vend credentials. Credential vending is part of the
+ * Iceberg REST protocol, so only the sources backed by a REST catalog can offer it.
+ *
+ * @param catalogSource catalog source
+ */
+ @AiProvenance(agent = AiProvenance.Agent.CLAUDE_OPUS_5, tool = AiProvenance.Tool.CLAUDE_CODE_UI,
+ contributionKind = AiProvenance.ContributionKind.GENERATED,
+ notes = "Identifies which catalog sources are REST-backed and can therefore vend credentials")
+ public static boolean supportsVendedCredentials(IcebergCatalogSource catalogSource) {
+ // kept exhaustive on purpose, so a newly added source has to make this choice explicitly
+ return switch (catalogSource) {
+ case REST, NESSIE_REST, AWS_GLUE_REST, S3_TABLES, BIGLAKE_METASTORE -> true;
+ case AWS_GLUE, NESSIE -> false;
+ };
+ }
+
+ /**
+ * Validates that a collection asking for vended credentials sits on a catalog that can actually vend them.
+ * Must be called once the catalog's own properties have been merged in, since that is where the catalog
+ * source is carried.
+ *
+ * @param configuration collection configuration, including the merged catalog properties
+ * @param sourceLoc location of the declaration, for error reporting
+ */
+ @AiProvenance(agent = AiProvenance.Agent.CLAUDE_OPUS_5, tool = AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind = AiProvenance.ContributionKind.GENERATED, notes = "Rejects vended credentials on a catalog source that cannot vend them")
+ public static void validateVendedCredentialsSource(Map<String, String> configuration, SourceLocation sourceLoc)
+ throws CompilationException {
+ if (!isVendedCredentials(configuration)) {
+ return;
+ }
+ String source = configuration
+ .get(ICEBERG_CATALOG_PROPERTY_PREFIX_INTERNAL + IcebergConstants.ICEBERG_SOURCE_PROPERTY_KEY);
+ if (!supportsVendedCredentials(validateAndGetCatalogSource(source))) {
+ throw new CompilationException(ErrorCode.VENDED_CREDENTIALS_UNSUPPORTED_CATALOG_SOURCE, sourceLoc, source);
+ }
+ }
+
public static void validateCatalogProperties(Map<String, String> properties) throws CompilationException {
validatePropertyExists(properties, IcebergConstants.ICEBERG_SOURCE_PROPERTY_KEY, ErrorCode.PARAMETERS_REQUIRED);
@@ -243,6 +309,7 @@
* @param configuration configuration
* @return catalog properties
*/
+ @AiProvenance(agent = AiProvenance.Agent.CLAUDE_OPUS_5, tool = AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind = AiProvenance.ContributionKind.ASSISTED, notes = "Carry the vended-credentials marker through to the catalog properties so setFileIoProperties can see it")
public static Map<String, String> filterCatalogProperties(Map<String, String> configuration) {
Map<String, String> properties = new HashMap<>();
String ioReader = configuration.get(ExternalDataConstants.KEY_EXTERNAL_SOURCE_TYPE);
@@ -258,6 +325,10 @@
// we only need reader type from collection properties (other than auth params) for setting FileIO later
properties.put(IcebergConstants.ICEBERG_IO_READER_TYPE, ioReader);
+ // likewise, setFileIoProperties needs to know whether the catalog vends the credentials, in which case
+ // it must not configure a FileIO at all
+ properties.put(IcebergConstants.ICEBERG_VENDED_CREDENTIALS_PROPERTY_KEY,
+ configuration.get(IcebergConstants.ICEBERG_VENDED_CREDENTIALS_PROPERTY_KEY));
return properties;
}
@@ -332,6 +403,7 @@
try {
catalog = createAndSetCatalogProperties(catalogProperties, catalogSource);
setWarehouseIfPresent(catalogProperties);
+ setAccessDelegationIfVended(catalogProperties);
if (initCatalogIo) {
setFileIoProperties(catalogProperties, catalogSource);
}
@@ -466,6 +538,22 @@
}
}
+ @AiProvenance(agent = AiProvenance.Agent.CLAUDE_OPUS_5, tool = AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind = AiProvenance.ContributionKind.ASSISTED, notes = "Skip FileIO configuration when the catalog vends the credentials")
+ /**
+ * Asks the catalog to vend storage credentials for the tables it hands out. Only the request side is ours:
+ * the credentials come back on the load-table response and the Iceberg client applies them to the table's
+ * FileIO itself, which is also why {@link #setFileIoProperties} must not configure one.
+ *
+ * @param catalogProperties catalog properties
+ */
+ @AiProvenance(agent = AiProvenance.Agent.CLAUDE_OPUS_5, tool = AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind = AiProvenance.ContributionKind.GENERATED, notes = "Requests credential vending via the REST access-delegation header")
+ private static void setAccessDelegationIfVended(Map<String, String> catalogProperties) {
+ if (isVendedCredentials(catalogProperties)) {
+ catalogProperties.put(IcebergConstants.Rest.ACCESS_DELEGATION_HEADER_PROPERTY,
+ IcebergConstants.Rest.ACCESS_DELEGATION_VENDED_CREDENTIALS);
+ }
+ }
+
public static void setFileIoProperties(Map<String, String> catalogProperties, IcebergCatalogSource catalogSource)
throws CompilationException {
if (catalogSource == IcebergCatalogSource.NESSIE_REST) {
@@ -473,6 +561,12 @@
return;
}
+ if (isVendedCredentials(catalogProperties)) {
+ // the catalog vends the storage credentials and supplies the FileIO configuration along with them,
+ // so configuring one here would override what is vended
+ return;
+ }
+
String ioType = catalogProperties.get(IcebergConstants.ICEBERG_IO_READER_TYPE);
if (ioType == null) {
throw new IllegalStateException("Iceberg IO reader type is not set");
diff --git a/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/util/iceberg/IcebergVendedCredentialsTest.java b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/util/iceberg/IcebergVendedCredentialsTest.java
new file mode 100644
index 0000000..1dd40e4
--- /dev/null
+++ b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/util/iceberg/IcebergVendedCredentialsTest.java
@@ -0,0 +1,313 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.util.iceberg;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.asterix.common.config.CatalogConfig.IcebergCatalogSource;
+import org.apache.asterix.common.exceptions.CompilationException;
+import org.apache.asterix.common.exceptions.ErrorCode;
+import org.apache.asterix.external.util.ExternalDataConstants;
+import org.apache.asterix.external.util.aws.iceberg.glue.GlueUtils;
+import org.apache.asterix.external.util.google.iceberg.fileio.GCSFileIO;
+import org.apache.hyracks.util.annotations.AiProvenance;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.aws.AwsProperties;
+import org.junit.Test;
+
+/**
+ * Compile-time and validation-time behaviour of Iceberg credential vending. Nothing here reads data or
+ * contacts a catalog; the point is the decisions made before any I/O happens.
+ *
+ * <p>Every "X is not configured" assertion is paired with a control showing that X <em>is</em> configured
+ * when credentials are not vended — otherwise the assertion would pass just as happily if the feature were
+ * deleted.
+ */
+@AiProvenance(agent = AiProvenance.Agent.CLAUDE_OPUS_5, tool = AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind = AiProvenance.ContributionKind.TEST_GENERATED, notes = "Validation and property-plumbing coverage for credential vending, with no data access")
+public class IcebergVendedCredentialsTest {
+
+ private static final String MARKER = IcebergConstants.ICEBERG_VENDED_CREDENTIALS_PROPERTY_KEY;
+
+ /**
+ * Every storage type the FileIO dispatch knows, with the FileIO each configures when credentials are not
+ * vended. When they are vended none of these is configured, and Iceberg's ResolvingFileIO picks the
+ * implementation from the data file's location scheme instead -- s3, gs or abfss.
+ */
+ private static final Map<String, String> STORAGE_FILE_IO = Map.of(ExternalDataConstants.KEY_ADAPTER_NAME_AWS_S3,
+ IcebergConstants.Aws.S3_FILE_IO, ExternalDataConstants.KEY_ADAPTER_NAME_GCS, GCSFileIO.class.getName(),
+ ExternalDataConstants.KEY_ADAPTER_NAME_AZURE_DATALAKE, IcebergConstants.Azure.ADLS_FILE_IO,
+ ExternalDataConstants.KEY_ADAPTER_NAME_AZURE_BLOB, IcebergConstants.Azure.ADLS_FILE_IO);
+
+ /** Sources backed by a REST catalog, and therefore able to vend. */
+ private static final Set<IcebergCatalogSource> REST_BACKED =
+ EnumSet.of(IcebergCatalogSource.REST, IcebergCatalogSource.NESSIE_REST, IcebergCatalogSource.AWS_GLUE_REST,
+ IcebergCatalogSource.S3_TABLES, IcebergCatalogSource.BIGLAKE_METASTORE);
+
+ // ------------------------------------------------------------------------------------------------
+ // the marker
+ // ------------------------------------------------------------------------------------------------
+
+ @Test
+ public void markerIsReadAsABoolean() {
+ assertFalse("absent must not read as vended", IcebergUtils.isVendedCredentials(new HashMap<>()));
+ assertFalse(IcebergUtils.isVendedCredentials(props(MARKER, "false")));
+ assertFalse("a non-boolean must not read as vended", IcebergUtils.isVendedCredentials(props(MARKER, "yes")));
+ assertTrue(IcebergUtils.isVendedCredentials(props(MARKER, "true")));
+ assertTrue("the value is parsed case-insensitively", IcebergUtils.isVendedCredentials(props(MARKER, "TRUE")));
+ }
+
+ // ------------------------------------------------------------------------------------------------
+ // which sources can vend
+ // ------------------------------------------------------------------------------------------------
+
+ /**
+ * Exhaustive on purpose: a newly added catalog source must fail here until someone decides whether it
+ * can vend, rather than silently inheriting a default.
+ */
+ @Test
+ public void everySourceIsClassified() {
+ for (IcebergCatalogSource source : IcebergCatalogSource.values()) {
+ assertEquals("unexpected vending support for " + source, REST_BACKED.contains(source),
+ IcebergUtils.supportsVendedCredentials(source));
+ }
+ assertEquals("a source was added or removed; classify it above", 7, IcebergCatalogSource.values().length);
+ }
+
+ // ------------------------------------------------------------------------------------------------
+ // vending requires an Iceberg table
+ // ------------------------------------------------------------------------------------------------
+
+ @Test
+ public void icebergTableIsAccepted() throws CompilationException {
+ IcebergUtils.validateVendedCredentials(icebergProps(), null);
+ }
+
+ @Test
+ public void nonIcebergTableIsRejected() {
+ Map<String, String> config = props(ExternalDataConstants.KEY_FORMAT, "json");
+ try {
+ IcebergUtils.validateVendedCredentials(config, null);
+ fail("expected vending to be rejected for a non-Iceberg collection");
+ } catch (CompilationException e) {
+ assertEquals(ErrorCode.VENDED_CREDENTIALS_REQUIRE_ICEBERG_TABLE.intValue(), e.getErrorCode());
+ }
+ }
+
+ // ------------------------------------------------------------------------------------------------
+ // vending requires a REST-backed catalog
+ // ------------------------------------------------------------------------------------------------
+
+ @Test
+ public void sourceIsNotCheckedWhenNotVending() throws CompilationException {
+ // control: the source below cannot vend, but without the marker this must not be consulted at all
+ Map<String, String> config = sourceProps(IcebergCatalogSource.AWS_GLUE);
+ config.remove(MARKER);
+ IcebergUtils.validateVendedCredentialsSource(config, null);
+ }
+
+ @Test
+ public void restBackedSourcesAreAccepted() throws CompilationException {
+ for (IcebergCatalogSource source : REST_BACKED) {
+ IcebergUtils.validateVendedCredentialsSource(sourceProps(source), null);
+ }
+ }
+
+ @Test
+ public void nonRestSourcesAreRejected() {
+ for (IcebergCatalogSource source : EnumSet.complementOf(EnumSet.copyOf(REST_BACKED))) {
+ try {
+ IcebergUtils.validateVendedCredentialsSource(sourceProps(source), null);
+ fail("expected " + source + " to be rejected for vending");
+ } catch (CompilationException e) {
+ assertEquals("unexpected error for " + source,
+ ErrorCode.VENDED_CREDENTIALS_UNSUPPORTED_CATALOG_SOURCE.intValue(), e.getErrorCode());
+ }
+ }
+ }
+
+ // ------------------------------------------------------------------------------------------------
+ // the marker has to survive the catalog-property filter
+ // ------------------------------------------------------------------------------------------------
+
+ /**
+ * {@code filterCatalogProperties} keeps only catalog-prefixed keys and a short allowlist, so a marker
+ * that is not explicitly carried over is dropped silently and every check downstream of it stops
+ * firing. That happened once already.
+ */
+ @Test
+ public void markerSurvivesCatalogPropertyFiltering() {
+ Map<String, String> collection = icebergProps();
+ collection.put(ExternalDataConstants.KEY_EXTERNAL_SOURCE_TYPE, ExternalDataConstants.KEY_ADAPTER_NAME_AWS_S3);
+
+ Map<String, String> filtered = IcebergUtils.filterCatalogProperties(collection);
+ assertTrue("the marker was dropped by filtering", IcebergUtils.isVendedCredentials(filtered));
+
+ // control: it is carried, not fabricated
+ collection.remove(MARKER);
+ assertFalse(IcebergUtils.isVendedCredentials(IcebergUtils.filterCatalogProperties(collection)));
+ }
+
+ // ------------------------------------------------------------------------------------------------
+ // no FileIO is configured when the catalog supplies it
+ // ------------------------------------------------------------------------------------------------
+
+ /**
+ * The skip covers every storage type, not only S3: it happens before the reader-type dispatch, so no
+ * storage type gets a FileIO of its own. Each cloud then refreshes the vended credentials through its own
+ * provider, keyed off a refresh endpoint the catalog returns -- VendedCredentialsProvider on S3,
+ * OAuth2RefreshCredentialsHandler on GCS, VendedAdlsCredentialProvider on ADLS.
+ */
+ @Test
+ public void fileIoIsNotConfiguredWhenVendingForAnyStorageType() throws CompilationException {
+ for (String storageType : STORAGE_FILE_IO.keySet()) {
+ Map<String, String> catalogProps = ioReaderProps(storageType);
+ catalogProps.put(MARKER, "true");
+
+ IcebergUtils.setFileIoProperties(catalogProps, IcebergCatalogSource.REST);
+ assertNull("configuring a FileIO would override what the catalog vended for " + storageType,
+ catalogProps.get(CatalogProperties.FILE_IO_IMPL));
+ }
+ }
+
+ @Test
+ public void fileIoIsConfiguredWhenNotVending() throws CompilationException {
+ // control for the test above: without the marker a FileIO is configured as usual. Azure is excluded
+ // because its branch does more than name a FileIO -- see azureCredentialsAreNotDemandedWhenVending.
+ for (Map.Entry<String, String> storage : STORAGE_FILE_IO.entrySet()) {
+ String storageType = storage.getKey();
+ if (IcebergConstants.Azure.ADLS_FILE_IO.equals(storage.getValue())) {
+ continue;
+ }
+ Map<String, String> catalogProps = ioReaderProps(storageType);
+
+ IcebergUtils.setFileIoProperties(catalogProps, IcebergCatalogSource.REST);
+ assertEquals(storageType, storage.getValue(), catalogProps.get(CatalogProperties.FILE_IO_IMPL));
+ }
+ }
+
+ /**
+ * Azure is the storage type where skipping the FileIO is load-bearing rather than merely tidy. Its branch
+ * resolves the collection's own Azure credentials into ADLS auth properties, which a vended-credentials
+ * collection does not have -- so reaching it at all fails the DDL. The control below shows it does fail
+ * without the marker, which is what makes the assertion above meaningful.
+ */
+ @Test
+ public void azureCredentialsAreNotDemandedWhenVending() throws CompilationException {
+ for (String storageType : azureStorageTypes()) {
+ Map<String, String> catalogProps = ioReaderProps(storageType);
+ catalogProps.put(MARKER, "true");
+
+ IcebergUtils.setFileIoProperties(catalogProps, IcebergCatalogSource.REST);
+ assertNull(storageType, catalogProps.get(CatalogProperties.FILE_IO_IMPL));
+ }
+ }
+
+ @Test
+ public void azureCredentialsAreDemandedWhenNotVending() {
+ // control for the test above: the same configuration without the marker cannot resolve Azure auth
+ for (String storageType : azureStorageTypes()) {
+ try {
+ IcebergUtils.setFileIoProperties(ioReaderProps(storageType), IcebergCatalogSource.REST);
+ fail("expected " + storageType + " to demand the collection's own Azure credentials");
+ } catch (CompilationException e) {
+ assertEquals(storageType, ErrorCode.PARAMETERS_REQUIRED.intValue(), e.getErrorCode());
+ }
+ }
+ }
+
+ // ------------------------------------------------------------------------------------------------
+ // no storage client factory when the catalog supplies the credentials
+ // ------------------------------------------------------------------------------------------------
+
+ /**
+ * The Glue REST and S3 Tables catalogs install the client factory in {@code createAndSetCatalogProperties},
+ * which runs before FileIO setup — so skipping FileIO does not cover them. Left installed, it would build
+ * S3 clients from the collection's own credentials and quietly ignore the vended ones.
+ */
+ @Test
+ public void glueRestClientFactoryIsNotInstalledWhenVending() throws CompilationException {
+ Map<String, String> catalogProps = glueRestProps();
+ catalogProps.put(MARKER, "true");
+
+ GlueUtils.setGlueRestCatalogProperties(catalogProps);
+ assertNull("the client factory would ignore the vended credentials",
+ catalogProps.get(AwsProperties.CLIENT_FACTORY));
+ assertEquals("sigv4 reaches the catalog itself and must stay", IcebergConstants.Aws.REST_SIG4_GLUE_SIGNING_NAME,
+ catalogProps.get(IcebergConstants.Aws.REST_SIG4_SIGNING_NAME));
+ }
+
+ @Test
+ public void glueRestClientFactoryIsInstalledWhenNotVending() throws CompilationException {
+ // control for the test above
+ Map<String, String> catalogProps = glueRestProps();
+
+ GlueUtils.setGlueRestCatalogProperties(catalogProps);
+ assertEquals("org.apache.asterix.external.awsclient.EnsureCloseAWSClientFactory",
+ catalogProps.get(AwsProperties.CLIENT_FACTORY));
+ }
+
+ // ------------------------------------------------------------------------------------------------
+ // helpers
+ // ------------------------------------------------------------------------------------------------
+
+ private static Map<String, String> props(String key, String value) {
+ Map<String, String> config = new HashMap<>();
+ config.put(key, value);
+ return config;
+ }
+
+ private static Map<String, String> icebergProps() {
+ Map<String, String> config = props(ExternalDataConstants.TABLE_FORMAT, IcebergConstants.ICEBERG_TABLE_FORMAT);
+ config.put(MARKER, "true");
+ return config;
+ }
+
+ private static Map<String, String> sourceProps(IcebergCatalogSource source) {
+ Map<String, String> config = icebergProps();
+ config.put(IcebergConstants.ICEBERG_CATALOG_PROPERTY_PREFIX_INTERNAL
+ + IcebergConstants.ICEBERG_SOURCE_PROPERTY_KEY, source.name());
+ return config;
+ }
+
+ private static Map<String, String> ioReaderProps(String storageType) {
+ return props(IcebergConstants.ICEBERG_IO_READER_TYPE, storageType);
+ }
+
+ private static Set<String> azureStorageTypes() {
+ return STORAGE_FILE_IO.entrySet().stream().filter(e -> IcebergConstants.Azure.ADLS_FILE_IO.equals(e.getValue()))
+ .map(Map.Entry::getKey).collect(java.util.stream.Collectors.toSet());
+ }
+
+ private static Map<String, String> glueRestProps() {
+ Map<String, String> config =
+ props(IcebergConstants.ICEBERG_URI_PROPERTY_KEY, "https://glue.eu-central-1.amazonaws.com/iceberg");
+ config.put(IcebergConstants.ICEBERG_SOURCE_PROPERTY_KEY, IcebergCatalogSource.AWS_GLUE_REST.name());
+ config.put(IcebergConstants.Aws.REST_SIG4_SIGNING_REGION_PROPERTY_NAME, "eu-central-1");
+ return config;
+ }
+}
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/21617?usp=email
To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings?usp=email
Gerrit-MessageType: newchange
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Change-Id: I3bac5a7f83b033eb892c6fe61116415597635d0a
Gerrit-Change-Number: 21617
Gerrit-PatchSet: 1
Gerrit-Owner: Hussain Towaileb <hu...@gmail.com>