You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by gi...@apache.org on 2026/05/28 11:46:27 UTC
(datafusion-comet) branch asf-site updated: Publish built docs triggered by a08cb4eca7dc4fbf107cf10bb6dc7f32e56f2aa1
This is an automated email from the ASF dual-hosted git repository.
github-actions[bot] pushed a commit to branch asf-site
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
The following commit(s) were added to refs/heads/asf-site by this push:
new e5389b5351 Publish built docs triggered by a08cb4eca7dc4fbf107cf10bb6dc7f32e56f2aa1
e5389b5351 is described below
commit e5389b5351eccef20a8bc504ef5b3593440ce60b
Author: github-actions[bot] <gi...@users.noreply.github.com>
AuthorDate: Thu May 28 11:46:21 2026 +0000
Publish built docs triggered by a08cb4eca7dc4fbf107cf10bb6dc7f32e56f2aa1
---
_sources/contributor-guide/index.md.txt | 1 +
.../s3-credential-provider-design.md.txt | 106 +++
_sources/user-guide/latest/index.rst.txt | 1 +
.../latest/s3-credential-providers.md.txt | 292 +++++++
contributor-guide/adding_a_new_expression.html | 1 +
contributor-guide/adding_a_new_operator.html | 7 +-
contributor-guide/adding_a_new_spark_version.html | 1 +
contributor-guide/benchmark-results/tpc-ds.html | 1 +
contributor-guide/benchmark-results/tpc-h.html | 1 +
contributor-guide/benchmarking.html | 1 +
contributor-guide/benchmarking_aws_ec2.html | 1 +
contributor-guide/benchmarking_macos.html | 1 +
contributor-guide/benchmarking_spark_sql_perf.html | 1 +
contributor-guide/bug_triage.html | 1 +
contributor-guide/contributing.html | 1 +
contributor-guide/debugging.html | 1 +
contributor-guide/development.html | 1 +
contributor-guide/ffi.html | 1 +
contributor-guide/iceberg-spark-tests.html | 1 +
contributor-guide/index.html | 1 +
contributor-guide/jvm_shuffle.html | 1 +
contributor-guide/native_shuffle.html | 1 +
contributor-guide/plugin_overview.html | 1 +
contributor-guide/profiling.html | 1 +
contributor-guide/release_process.html | 1 +
contributor-guide/roadmap.html | 1 +
...fle.html => s3-credential-provider-design.html} | 342 +++------
contributor-guide/spark-sql-tests.html | 1 +
contributor-guide/spark_configs_support.html | 1 +
contributor-guide/spark_expressions_support.html | 1 +
contributor-guide/sql-file-tests.html | 1 +
contributor-guide/sql_error_propagation.html | 7 +-
contributor-guide/tracing.html | 1 +
genindex.html | 2 +
index.html | 2 +
objects.inv | Bin 2399 -> 2463 bytes
search.html | 2 +
searchindex.js | 2 +-
user-guide/0.13/compatibility.html | 1 +
user-guide/0.13/configs.html | 1 +
user-guide/0.13/datasources.html | 1 +
user-guide/0.13/datatypes.html | 1 +
user-guide/0.13/expressions.html | 1 +
user-guide/0.13/iceberg.html | 1 +
user-guide/0.13/index.html | 1 +
user-guide/0.13/installation.html | 1 +
user-guide/0.13/kubernetes.html | 1 +
user-guide/0.13/metrics.html | 1 +
user-guide/0.13/operators.html | 1 +
user-guide/0.13/source.html | 1 +
user-guide/0.13/tuning.html | 1 +
user-guide/0.14/compatibility.html | 1 +
user-guide/0.14/configs.html | 1 +
user-guide/0.14/datasources.html | 1 +
user-guide/0.14/datatypes.html | 1 +
user-guide/0.14/expressions.html | 1 +
user-guide/0.14/iceberg.html | 1 +
user-guide/0.14/index.html | 1 +
user-guide/0.14/installation.html | 1 +
user-guide/0.14/kubernetes.html | 1 +
user-guide/0.14/metrics.html | 1 +
user-guide/0.14/operators.html | 1 +
user-guide/0.14/source.html | 1 +
user-guide/0.14/tuning.html | 1 +
user-guide/0.15/compatibility.html | 1 +
user-guide/0.15/configs.html | 1 +
user-guide/0.15/datasources.html | 1 +
user-guide/0.15/datatypes.html | 1 +
user-guide/0.15/expressions.html | 1 +
user-guide/0.15/iceberg.html | 1 +
user-guide/0.15/index.html | 1 +
user-guide/0.15/installation.html | 1 +
user-guide/0.15/kubernetes.html | 1 +
user-guide/0.15/metrics.html | 1 +
user-guide/0.15/operators.html | 1 +
user-guide/0.15/source.html | 1 +
user-guide/0.15/tuning.html | 1 +
.../0.16/compatibility/expressions/aggregate.html | 1 +
.../0.16/compatibility/expressions/array.html | 1 +
.../0.16/compatibility/expressions/cast.html | 1 +
.../0.16/compatibility/expressions/datetime.html | 1 +
.../0.16/compatibility/expressions/index.html | 1 +
user-guide/0.16/compatibility/expressions/map.html | 1 +
.../0.16/compatibility/expressions/math.html | 1 +
.../0.16/compatibility/expressions/misc.html | 1 +
.../0.16/compatibility/expressions/string.html | 1 +
.../0.16/compatibility/expressions/struct.html | 1 +
user-guide/0.16/compatibility/floating-point.html | 1 +
user-guide/0.16/compatibility/index.html | 1 +
user-guide/0.16/compatibility/operators.html | 1 +
user-guide/0.16/compatibility/regex.html | 1 +
user-guide/0.16/compatibility/scans.html | 1 +
user-guide/0.16/compatibility/spark-versions.html | 1 +
user-guide/0.16/configs.html | 1 +
user-guide/0.16/datasources.html | 1 +
user-guide/0.16/datatypes.html | 1 +
user-guide/0.16/expressions.html | 1 +
user-guide/0.16/iceberg.html | 1 +
user-guide/0.16/index.html | 1 +
user-guide/0.16/installation.html | 1 +
user-guide/0.16/kubernetes.html | 1 +
user-guide/0.16/metrics.html | 1 +
user-guide/0.16/operators.html | 1 +
user-guide/0.16/source.html | 1 +
user-guide/0.16/tuning.html | 1 +
user-guide/0.16/understanding-comet-plans.html | 1 +
user-guide/index.html | 1 +
.../compatibility/expressions/aggregate.html | 1 +
.../latest/compatibility/expressions/array.html | 1 +
.../latest/compatibility/expressions/cast.html | 1 +
.../latest/compatibility/expressions/datetime.html | 1 +
.../latest/compatibility/expressions/index.html | 1 +
.../latest/compatibility/expressions/map.html | 1 +
.../latest/compatibility/expressions/math.html | 1 +
.../latest/compatibility/expressions/misc.html | 1 +
.../latest/compatibility/expressions/string.html | 1 +
.../latest/compatibility/expressions/struct.html | 1 +
.../latest/compatibility/expressions/url.html | 1 +
.../latest/compatibility/floating-point.html | 1 +
user-guide/latest/compatibility/index.html | 1 +
user-guide/latest/compatibility/operators.html | 1 +
user-guide/latest/compatibility/regex.html | 1 +
user-guide/latest/compatibility/scans.html | 1 +
.../latest/compatibility/spark-versions.html | 1 +
user-guide/latest/configs.html | 1 +
user-guide/latest/datasources.html | 1 +
user-guide/latest/datatypes.html | 1 +
user-guide/latest/expressions.html | 1 +
user-guide/latest/iceberg.html | 7 +-
user-guide/latest/index.html | 1 +
user-guide/latest/installation.html | 1 +
user-guide/latest/kubernetes.html | 7 +-
user-guide/latest/metrics.html | 1 +
user-guide/latest/operators.html | 1 +
user-guide/latest/s3-credential-providers.html | 855 +++++++++++++++++++++
user-guide/latest/scala_java_udfs.html | 1 +
user-guide/latest/source.html | 1 +
user-guide/latest/tuning.html | 1 +
user-guide/latest/understanding-comet-plans.html | 1 +
user-guide/older-versions.html | 1 +
140 files changed, 1495 insertions(+), 263 deletions(-)
diff --git a/_sources/contributor-guide/index.md.txt b/_sources/contributor-guide/index.md.txt
index c601dab87d..55b6552371 100644
--- a/_sources/contributor-guide/index.md.txt
+++ b/_sources/contributor-guide/index.md.txt
@@ -49,6 +49,7 @@ Arrow FFI <ffi>
JVM Shuffle <jvm_shuffle>
Native Shuffle <native_shuffle>
ANSI Error Propagation <sql_error_propagation>
+S3 Credential Provider Design <s3-credential-provider-design>
```
```{toctree}
diff --git a/_sources/contributor-guide/s3-credential-provider-design.md.txt b/_sources/contributor-guide/s3-credential-provider-design.md.txt
new file mode 100644
index 0000000000..00504c3ea3
--- /dev/null
+++ b/_sources/contributor-guide/s3-credential-provider-design.md.txt
@@ -0,0 +1,106 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# S3 Credential Provider SPI: Design Notes
+
+This page captures why the `org.apache.comet.cloud.s3.CometS3CredentialProvider` SPI is shaped the way it is. The user-facing contract and operator setup live in the user guide page on S3 credential providers; this page is for maintainers and reviewers who want the design rationale.
+
+## The gap the SPI fills
+
+Comet's native scan paths (`object_store` for raw Parquet, `opendal` via `iceberg-rust` for Iceberg) bypass Spark's Hadoop S3A code path. That means credentials cannot flow through any of the contracts that vendors typically wire into for S3A:
+
+- `org.apache.spark.deploy.security.cloud.CloudCredentialsProvider` yields a single JWT per service name. No path argument, no AWS credential.
+- Hadoop S3A custom signers hide path-aware logic inside `Signer.sign(request, credentials)`. The credential never leaves the signing pipeline, and the underlying secret is an HMAC key that is not present in the signed output, so running the signer against a synthesized request cannot recover it.
+- `AWSCredentialsProvider.getCredentials()` (AWS SDK v1) and `AwsCredentialsProvider.resolveCredentials()` (v2) are parameterless. They cannot vend per-path credentials.
+- Reflecting into vendor singletons would encode per-vendor class names and lifecycles in Comet and would silently break on vendor upgrades.
+
+A Comet-specific SPI is the narrowest fit: a single Java method that takes a `CometS3CredentialContext` (today wrapping `bucket`, `path`, and access `mode`; new fields can be added without breaking vendors compiled against earlier versions) and returns `CometS3Credentials`.
+
+## Why config-driven activation, not `META-INF/services`
+
+An earlier iteration used `ServiceLoader` discovery. That was rejected because:
+
+- Peer SPIs in the same space (Hadoop `AWSCredentialsProvider`, AWS SDK v2 `AwsCredentialsProvider`, Iceberg `AwsClientFactory`, S3A custom signers) are all class-name-in-config. Vendors are already familiar with that model.
+- ServiceLoader makes activation implicit on classpath presence. A vendor JAR drifting onto a cluster could silently change S3 auth behavior. The config key makes activation explicit.
+- The activation key (`fs.s3a.comet.credential.provider.class`, with per-bucket override) follows the same shape as `fs.s3a.bucket.<name>.aws.credentials.provider`, so operators do not learn a new pattern.
+
+Activation is modeled on `parquet.crypto.factory.class` (Parquet Modular Encryption KMS, see Comet #2447): the user names a single vendor class and the vendor dispatches across multiple credential backends inside that class if they need to. This mirrors how Iceberg's `DecryptionPropertiesFactory` already behaves for Parquet keys.
+
+## Why `(FQCN, dispatchKey, catalogProperties)` keying
+
+Comet caches one provider instance per `(FQCN, dispatchKey, catalogProperties)` triple. The dispatch key is the Spark V2 catalog name on the Iceberg path and the bucket on the Parquet path.
+
+- Two catalogs that share one provider class (typical in multi-tenant deployments) need isolated `initialize` maps because their `catalogProperties` differ. Without `dispatchKey`, the second `initialize` would either overwrite the first or be silently skipped.
+- The bucket as `dispatchKey` for Parquet gives vendors per-bucket isolation when the same provider is named under several `fs.s3a.bucket.<name>.comet.credential.provider.class` keys.
+- `catalogProperties` enters the key to handle multi-tenant JVMs (Spark Connect, Thrift Server, `SparkSession.newSession()`) where two sessions can configure the same provider class against the same `dispatchKey` but with different REST endpoints, OAuth tokens, or vendor keys. Without it the second session would silently use the first session's credentials.
+- Keying solely by FQCN would force vendors to encode multi-tenant routing in static state. The triple-key keeps each call site independent.
+
+`ensureInitialized` returns a `long` handle that the native bridge stashes and replays on every per-request call. Routing per-request lookups by handle avoids re-sending the property bag across JNI on the hot path and unambiguously selects the right provider when the same `(FQCN, dispatchKey)` pair maps to multiple instances.
+
+## Why fresh construction in `initialize`, not probing a JVM-wide static
+
+A provider implementation might be tempted to probe an existing static populated elsewhere (e.g. by a Hadoop S3A signer's `registerStore` callback) and reuse the credential cache that the Hadoop path uses. That fails on Comet-only executors:
+
+- The driver JVM hits `S3AFileSystem.initialize` during analysis (raw `s3a://` paths) or during Hadoop catalog manifest reads (Iceberg with Hadoop catalog), so the static is populated there.
+- The driver may not hit `S3AFileSystem` at all under Iceberg with REST catalog plus `S3FileIO`, because `S3FileIO` calls AWS SDK directly without going through the Hadoop layer. The static stays null.
+- Executors with Comet-only reads never instantiate `S3AFileSystem`. The data path is `object_store` (raw Parquet) or `opendal` via `iceberg-rust` (Iceberg native scan). Neither touches Hadoop S3A. The static stays null on every executor.
+
+Constructing a fresh provider from `catalogProperties` plus `SparkEnv` is the only strategy that works across all four cases. The trade-off is that on the driver (and any JVM where Hadoop S3A is also active), two credential caches now exist for the same identity: one inside the Hadoop signer's provider, one inside the SPI implementation's. The vendor pays for this with a small number of extra AS round-trips on cold starts and TTL boundaries. A future optional optimization could probe the [...]
+
+## Why no Comet-side cache
+
+Comet's bridge does not maintain a TTL cache, schedule refresh, or broadcast catalog state. All of that is the vendor's responsibility:
+
+- Iceberg vendors get `software.amazon.awssdk.utils.cache.CachedSupplier` for free inside `org.apache.iceberg.aws.s3.VendedCredentialsProvider`.
+- Custom-STS vendors write whatever cache fits their refresh model.
+- Driver-only state is distributed via `initialize`'s `catalogProperties` (Iceberg path) or read from Hadoop conf via `SparkEnv` (Parquet path). Both are plan-time snapshots: Comet does not re-execute the catalog or push fresh values to running scans. Vendors that need a refreshing bearer compose with Spark's `HadoopDelegationTokenProvider`, which mints and renews on the driver and propagates to executors via `UserGroupInformation`. The two SPIs are orthogonal: Spark covers bearer lifecy [...]
+
+A Comet-side cache would have to either expose a tuning knob (TTL, max size, eviction policy) and grow over time, or be hardcoded and surprise vendors whose policies disagree. The bridge intentionally has neither and forwards every call.
+
+## Path-specific behavior
+
+`object_store::CredentialProvider` and `reqsign_core::ProvideCredential` differ in what they consume:
+
+| Concern | Parquet (`object_store`) | Iceberg (`opendal` via `reqsign-core`) |
+| ----------------------- | -------------------------------------------------- | ------------------------------------------------------------ |
+| Trait method | `get_credential() -> AwsCredential` | `provide_credential(...) -> Option<IcebergAwsCredential>` |
+| Returns expiry? | No (only key/secret/token) | Yes (`expires_in: Option<Timestamp>`) |
+| Comet-side TTL wrapper? | None. Bridge passed straight to `with_credentials` | None. `opendal` schedules the next refresh from `expires_in` |
+| When SPI is called | Every `get_credential()` call | When `expires_in` is exceeded |
+| Vendor returns 0 expiry | Field has no use | Bridge substitutes 5 minutes to bound staleness |
+
+The 5-minute fallback is a safety net so a vendor that omits expiry cannot leave `opendal` caching a stale token indefinitely. It is intentionally not a configuration knob.
+
+## Property-bag handling on the Iceberg path
+
+The full unfiltered FileIO property bag crosses JNI as `catalog_properties`. The storage-prefix filter (`s3.`/`gcs.`/`adls.`/`client.`) is applied native-side in `iceberg_scan.rs::load_file_io` immediately before `FileIOBuilder.with_prop`. This means the bridge sees `credentials.uri`, OAuth tokens, and any vendor-custom keys with no parallel field on the operator and no driver-side broadcast. Vendors set their own keys on the catalog config and read them back inside `initialize(Map)`.
+
+`IcebergScanExec` derives a redacting `Debug` so plan dumps and tracing do not leak the property bag.
+
+## Returns or throws, not a fall-through value
+
+The SPI returns a `CometS3Credentials` or throws. There is no sentinel "I do not know" return. Vendors that are only authoritative for some paths resolve the default AWS chain themselves for the rest and return the result. This matches the contract on every other AWS credential SPI in the JVM ecosystem (AWS SDK v1/v2, Hadoop S3A, Iceberg `VendedCredentialsProvider`).
+
+## Lifecycle: `AutoCloseable` plus a JVM shutdown hook
+
+`CometS3CredentialProvider` extends `AutoCloseable` with a default no-op `close()`. The dispatcher installs a JVM shutdown hook that iterates every cached instance and calls `close()`, swallowing per-provider exceptions so a slow or buggy vendor cannot block other providers from cleaning up. Stateless providers ignore this entirely; vendors that hold long-lived resources (HTTP clients, scheduled-refresh executors, STS connection pools) override `close()` to release them. Shutdown hooks a [...]
+
+## Iceberg path: error message fidelity caveat
+
+When the bridge is wired into `iceberg-rust`, the outer `reqsign-core::ProvideCredentialChain` currently swallows thrown exceptions into "no credential" before the request reaches `opendal`. The credential is still not issued and the request still fails, but the message is degraded to an opaque anonymous-request failure. No Comet change fixes this; it is resolved upstream when `iceberg-rust` stops wrapping custom loaders in its outer chain or moves its S3 backend to `object_store`.
diff --git a/_sources/user-guide/latest/index.rst.txt b/_sources/user-guide/latest/index.rst.txt
index 6cc5778717..f97537c6e4 100644
--- a/_sources/user-guide/latest/index.rst.txt
+++ b/_sources/user-guide/latest/index.rst.txt
@@ -79,6 +79,7 @@ to read more.
:hidden:
Iceberg Guide <iceberg>
+ S3 Credential Providers <s3-credential-providers>
Kubernetes Guide <kubernetes>
.. toctree::
diff --git a/_sources/user-guide/latest/s3-credential-providers.md.txt b/_sources/user-guide/latest/s3-credential-providers.md.txt
new file mode 100644
index 0000000000..33777c497a
--- /dev/null
+++ b/_sources/user-guide/latest/s3-credential-providers.md.txt
@@ -0,0 +1,292 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# S3 Credential Providers
+
+Comet's native S3 readers normally fetch credentials from the standard AWS credential chain (static keys, instance profiles, environment variables, etc.). Some clusters use a vendor-managed mechanism instead, where credentials are issued per request based on a JWT or per S3 path. For those clusters, Comet supports loading a vendor-supplied bridge class that routes every native credential request through the vendor's Java code.
+
+## Do I need this?
+
+You don't, if any of the following describe your cluster:
+
+- You use static AWS credentials (`fs.s3a.access.key` / `fs.s3a.secret.key`).
+- You use EC2 instance profiles, EKS pod identities, ECS task roles, or environment variables.
+- Your S3 access works in Spark today via the default AWS credential chain.
+
+You probably do, if any of these are true:
+
+- You have a Hadoop S3A custom signer configured (`fs.s3a.custom.signers=...`).
+- You have a Spark `CloudCredentialsProvider` that issues a JWT for a vendor STS service.
+- You have a custom Iceberg `client.factory` that injects a configured S3 client.
+- Spark queries against your S3 paths work, but the same queries with Comet enabled fail with 403.
+
+## Enabling a bridge
+
+A bridge is activated by naming the vendor's class in a Spark config. Putting a JAR on the classpath alone has no effect; the config key must be set.
+
+For raw Parquet (the `object_store` path), set the Hadoop S3A config:
+
+```
+spark.hadoop.fs.s3a.comet.credential.provider.class=com.vendor.MyCometCredentialProvider
+```
+
+A per-bucket override is supported and follows the same shape as `fs.s3a.bucket.<name>.aws.credentials.provider`:
+
+```
+spark.hadoop.fs.s3a.bucket.<bucket>.comet.credential.provider.class=com.vendor.MyCometCredentialProvider
+```
+
+For Iceberg (the `opendal` path), set the per-catalog property under the `s3.` namespace Iceberg already uses for its S3 settings:
+
+```
+spark.sql.catalog.<catalog>.s3.comet.credential.provider.class=com.vendor.MyCometCredentialProvider
+```
+
+Add the vendor JAR to your Spark executor classpath:
+
+```sh
+spark-submit --jars vendor-comet-bridge.jar ...
+```
+
+OSS Comet ships no vendor-specific bridges. Get one from the same vendor that supplies your Hadoop S3A signer or Iceberg client factory. If they do not yet provide one, send them to the "Writing a bridge" section below.
+
+## Wire encryption
+
+Catalog configuration you set under `spark.sql.catalog.<catalog>.*` (which may include endpoint URLs, OAuth tokens, and other vendor keys your provider reads) is sent from the driver to executors over Spark's Netty RPC channel so the provider can run there. Live AWS credentials fetched by the provider stay on the executor that fetched them.
+
+Spark's RPC channel is plaintext by default and offers two opt-in encryption mechanisms, both off by default:
+
+- [`spark.network.crypto.enabled`](https://spark.apache.org/docs/latest/security.html#authentication-and-encryption) for AES-based RPC encryption keyed off the auth shared secret.
+- [`spark.ssl.rpc.enabled`](https://spark.apache.org/docs/latest/security.html#ssl-configuration) for TLS on the same channel (Spark 3.4+).
+
+The same defaults apply to Hadoop delegation tokens, `CloudCredentialsProvider` JWTs, and any other secrets Spark already ships driver-to-executor, so this is a deployment-wide call rather than something specific to this SPI. See Spark's [security guide](https://spark.apache.org/docs/latest/security.html) for the full set of knobs.
+
+## Verification
+
+With the config set and the JAR on the classpath, executor logs show on first S3 access:
+
+- Info level: `Instantiated CometS3CredentialProvider <fully.qualified.VendorClassName>`
+- Debug level: `Fetching credentials via <class> (dispatchKey=<key>) for bucket=... path=... mode=...`
+
+Without the config set, no credential-related log lines appear at startup; native readers use the default AWS credential chain.
+
+## Troubleshooting
+
+**`CometS3CredentialProvider class not found: <name>`**. The class named in the config is not on the executor classpath. Re-check `--jars` / `spark.jars`. On YARN or Kubernetes, confirm the JAR actually reached the executor and not only the driver.
+
+**`<class> does not implement org.apache.comet.cloud.s3.CometS3CredentialProvider`**. The configured class exists but does not implement the SPI. Double-check the FQCN against the vendor's documentation.
+
+**`<class> must declare a public no-arg constructor`**. Vendor classes are instantiated reflectively with `Class.forName(name).getDeclaredConstructor().newInstance()`. A non-default constructor is not supported; ask the vendor to expose a no-arg form that reads any state it needs from `initialize(Map)` or environment.
+
+**`CometS3CredentialProvider <class> (dispatchKey=...) was not initialized`**. `initialize(Map)` was not called before a credential request. Comet should always invoke `ensureInitialized` synchronously when it builds the bridge at plan time, so this indicates the bridge skipped the init call (a Comet bug) or the vendor's `initialize` threw and the bridge fell through to the default chain.
+
+**`403 AccessDenied` with the bridge configured.** The provider returned credentials but they were rejected by S3. Most often a region mismatch (see Iceberg section below) or expired session token; enable debug logging on the vendor's class to confirm what it returned.
+
+**Credentials silently going stale during long-running jobs.** When a vendor returns `expirationEpochMillis=0`, the bridge substitutes a 5-minute expiry before handing the credential to `opendal`, so `opendal`'s cache cannot hold a stale credential indefinitely. Returning a real expiry is preferred; the 5-minute fallback is a safety net, not a knob.
+
+## Iceberg: explicit S3 region required
+
+With the bridge configured, Comet wires a custom credential loader into `iceberg-storage-opendal`. `opendal`'s built-in S3 region auto-detection only runs when no custom loader is configured, so on the bridge path the region (and endpoint for non-AWS) must be set explicitly on the Spark catalog:
+
+```
+spark.sql.catalog.<catalog>.s3.region = us-east-1
+spark.sql.catalog.<catalog>.s3.endpoint = https://... (non-AWS only)
+spark.sql.catalog.<catalog>.s3.path-style-access = true (path-style endpoints only)
+```
+
+If you hit `region is missing. Please find it by S3::detect_region() or set them in env`, this is the missing config.
+
+## Writing a bridge
+
+Comet's native scan paths (`object_store` for raw Parquet, `opendal` via `iceberg-rust` for Iceberg) bypass Hadoop S3A entirely. The standard `AWSCredentialsProvider.getCredentials()` has no path argument, so vendors that issue per-path STS credentials cannot expose them through it. The `CometS3CredentialProvider` SPI fills that gap.
+
+Implement `org.apache.comet.cloud.s3.CometS3CredentialProvider`:
+
+```java
+package org.apache.comet.cloud.s3;
+
+public interface CometS3CredentialProvider extends AutoCloseable {
+ /** Called once per (FQCN, dispatchKey, catalogProperties) before any per-request call. Optional. */
+ default void initialize(java.util.Map<String, String> catalogProperties) {}
+
+ CometS3Credentials getCredentialsForPath(CometS3CredentialContext context) throws Exception;
+
+ /** Invoked from the dispatcher's JVM shutdown hook. Default is a no-op. */
+ @Override default void close() throws Exception {}
+}
+```
+
+The class must have a public no-arg constructor. `getCredentialsForPath` may be invoked concurrently from many native tokio worker threads; the implementation must be thread-safe. The supplied `CometS3CredentialContext` exposes `getBucket()`, `getPath()`, and `getMode()`; future Comet releases may add accessors here without changing the method signature, so vendors compiled against today's API stay binary-compatible.
+
+### Lifecycle
+
+Comet keys provider instances by `(FQCN, dispatchKey, catalogProperties)`. The dispatch key is the Spark V2 catalog name on the Iceberg path and the S3 bucket name on the Parquet path. The first time a given key is seen on an executor, Comet reflects the class, calls `initialize(Map)` exactly once, and caches the instance for the JVM lifetime. Two catalogs sharing one provider FQCN therefore get isolated instances with their own `initialize` maps. Including `catalogProperties` in the key [...]
+
+`initialize` should be cheap and non-blocking. Defer real credential fetches (REST round-trips, STS calls) to the first `getCredentialsForPath` invocation. On the Iceberg path the supplied `catalogProperties` carries the unfiltered FileIO bag, including REST-vended fields like `credentials.uri`, OAuth tokens, and any vendor-custom keys you set on the catalog config. The map may contain secrets, so do not log it.
+
+`close()` is invoked from a JVM shutdown hook installed by the dispatcher. The default no-op is fine for stateless providers. Override it to release HTTP clients, scheduled-refresh executors, or STS connection pools. Shutdown hooks are best-effort: a `SIGKILL` or abrupt JVM termination skips them, so do not depend on `close()` for correctness.
+
+### Caching, refresh, and distribution
+
+Comet does not maintain a TTL cache, broadcast catalog state, or schedule refresh. Vendors decide:
+
+- Whether to cache credentials and for how long. Iceberg vendors get `software.amazon.awssdk.utils.cache.CachedSupplier` for free inside `VendedCredentialsProvider`; vendors with custom STS write whatever cache fits.
+- When to refresh: proactive timer, on-demand at expiry, on `403` retry, etc.
+- How to distribute and refresh state. `catalogProperties` is a plan-time snapshot. Comet does not re-execute the catalog or push fresh values to running scans, so any state that must refresh during a scan is the vendor's responsibility. Options: call back to a vendor service from the executor on each `getCredentialsForPath`, run your own Spark broadcast inside the class, or compose with Spark's [`HadoopDelegationTokenProvider`](https://spark.apache.org/docs/latest/security.html#kerberos [...]
+
+When using `HadoopDelegationTokenProvider`, Spark mints and renews the token on the driver and propagates it to executors via `UserGroupInformation`. The provider reads the current value inside `getCredentialsForPath` and exchanges it for AWS credentials:
+
+```java
+@Override
+public CometS3Credentials getCredentialsForPath(CometS3CredentialContext ctx) throws Exception {
+ Token<?> token = UserGroupInformation.getCurrentUser()
+ .getCredentials()
+ .getToken(new Text("vendor-service"));
+ return vendorSts.exchangeForS3Credentials(token, ctx.getBucket(), ctx.getPath(), ctx.getMode());
+}
+```
+
+Spark delegation token propagation is supported on YARN and Kubernetes only. Standalone deployments need a different refresh path, typically a vendor-side service callback authenticated by long-lived state in `catalogProperties` or Hadoop conf.
+
+`expirationEpochMillis` only matters on the Iceberg/`opendal` path. There the bridge implements `reqsign_core::ProvideCredential`, which carries an `expires_in` field that `opendal` uses to schedule the next refresh. Publish a real expiry when you have one. `0` means "unknown"; the bridge then substitutes a 5-minute expiry to bound staleness.
+
+The Parquet/`object_store` path has no expiry concept: `object_store::CredentialProvider` returns just `AwsCredential` (key/secret/token). The bridge is passed to `with_credentials` without a TTL wrapper, so `object_store` calls into the SPI on every request and relies on the vendor's own cache for hit rates. Expiry handling is fully the vendor's responsibility: the vendor decides when its internal cache refreshes. If `object_store` receives a 403 from an expired session token, its retry [...]
+
+### Returned fields
+
+| Field | Notes |
+| ----------------------- | ------------------------------------------------------------------------------------------------------------------------- |
+| `accessKeyId` | Required. |
+| `secretAccessKey` | Required. |
+| `sessionToken` | `null` for non-STS credentials. |
+| `expirationEpochMillis` | Iceberg path only. `0` means "unknown"; the bridge substitutes a 5-minute expiry. The Parquet path has no expiry concept. |
+
+Provide a real `expirationEpochMillis` whenever you have one on the Iceberg path. The Parquet path's `object_store::CredentialProvider` does not consume an expiry, and the bridge invokes the SPI on every `get_credential()` call.
+
+### Returns or throws
+
+The SPI follows the same shape as the other JVM AWS-credential SPIs (AWS SDK v1/v2, Hadoop S3A, Iceberg `VendedCredentialsProvider`): return credentials or throw. There is no "fall-through" return value.
+
+If your provider is authoritative only for some paths, resolve the default AWS chain yourself for the rest:
+
+```java
+private final DefaultCredentialsProvider defaultChain = DefaultCredentialsProvider.create();
+
+@Override
+public CometS3Credentials getCredentialsForPath(CometS3CredentialContext ctx) throws Exception {
+ if (handlesPath(ctx.getBucket(), ctx.getPath())) {
+ return mintFromMyVendorService(ctx.getBucket(), ctx.getPath(), ctx.getMode());
+ }
+ AwsCredentials c = defaultChain.resolveCredentials();
+ String token = (c instanceof AwsSessionCredentials)
+ ? ((AwsSessionCredentials) c).sessionToken()
+ : null;
+ return new CometS3Credentials(c.accessKeyId(), c.secretAccessKey(), token, 0L);
+}
+```
+
+### Composing multiple credential backends
+
+A single configured provider class is the dispatcher. If a vendor needs to route across several credential backends (per bucket, per path prefix, per tenant), the dispatch lives inside the vendor's class:
+
+```java
+public final class MyCometCredentialProvider implements CometS3CredentialProvider {
+ private final ProdVendor prod = ...;
+ private final StsVendor sts = ...;
+ private final DefaultVendor fallback = ...;
+
+ @Override
+ public CometS3Credentials getCredentialsForPath(CometS3CredentialContext ctx) throws Exception {
+ if (ctx.getBucket().startsWith("data-prod-")) return prod.fetch(ctx);
+ if (ctx.getBucket().equals("partner-shared")) return sts.assumeRole(ctx);
+ return fallback.fetch(ctx.getBucket(), ctx.getPath());
+ }
+}
+```
+
+Per-bucket Hadoop overrides (`fs.s3a.bucket.<name>.comet.credential.provider.class`) are also available if you prefer to ship multiple vendor classes and pick by bucket in config rather than in code.
+
+For Iceberg deployments where two catalogs share one provider class but need isolated state, configure the same FQCN on both catalogs and read your discriminator from `initialize`'s `catalogProperties`. Each catalog gets its own provider instance because Comet keys by `(FQCN, catalogName, catalogProperties)`:
+
+```java
+public final class MyMultiTenantProvider implements CometS3CredentialProvider {
+ private volatile String tenantId;
+
+ @Override
+ public void initialize(Map<String, String> catalogProperties) {
+ this.tenantId = catalogProperties.get("vendor.tenant-id");
+ }
+
+ @Override
+ public CometS3Credentials getCredentialsForPath(CometS3CredentialContext ctx) {
+ return mintForTenant(tenantId, ctx.getBucket(), ctx.getPath(), ctx.getMode());
+ }
+}
+```
+
+### Reference implementation: Iceberg REST vended credentials
+
+For Iceberg REST catalogs that vend AWS credentials (`LoadTableResponse.credentials`), the canonical implementation wraps Iceberg's existing `VendedCredentialsProvider`:
+
+```java
+public final class IcebergRESTVendedS3Provider implements CometS3CredentialProvider {
+ private volatile VendedCredentialsProvider provider;
+
+ @Override
+ public void initialize(Map<String, String> catalogProperties) {
+ this.provider = VendedCredentialsProvider.create(catalogProperties);
+ }
+
+ @Override
+ public CometS3Credentials getCredentialsForPath(CometS3CredentialContext ctx) {
+ AwsCredentials c = provider.resolveCredentials();
+ String token = (c instanceof AwsSessionCredentials)
+ ? ((AwsSessionCredentials) c).sessionToken() : null;
+ return new CometS3Credentials(c.accessKeyId(), c.secretAccessKey(), token, 0L);
+ }
+}
+```
+
+`VendedCredentialsProvider` reads `credentials.uri`, the catalog endpoint, and OAuth tokens from the supplied map (Comet forwards the unfiltered FileIO bag to `initialize`), and refreshes through its own `CachedSupplier`. Caching, refresh-near-expiry, and the REST round-trip all live in Iceberg, not in Comet. Comet ships a copy of this class under `spark/src/test` as a reference; copy it into your runtime jar alongside `iceberg-aws` and AWS SDK v2.
+
+### Access mode
+
+| Value | Used for |
+| ------- | -------------------------------------------------------------------------- |
+| `READ` | All native scan paths (raw Parquet, Iceberg). Comet today only sends READ. |
+| `WRITE` | Reserved for future native write paths. |
+
+A `WRITE` credential is not implicitly read-capable. Vendors that need read-during-write workflows include the required read permissions in the IAM policy attached to their `WRITE` credentials.
+
+### Build setup
+
+Vendor implementations need the Comet SPI classes at compile time only. Use `provided`-scope:
+
+```xml
+<dependency>
+ <groupId>org.apache.datafusion</groupId>
+ <artifactId>comet-spark-spark${spark.version.short}_${scala.binary.version}</artifactId>
+ <version>${comet.version}</version>
+ <scope>provided</scope>
+</dependency>
+```
+
+### Iceberg path: error message fidelity
+
+When the bridge is wired into `iceberg-rust`, the outer `reqsign-core::ProvideCredentialChain` currently swallows thrown exceptions into "no credential" before the request reaches `opendal`. The credential is still not issued and the request still fails, only the message is degraded to an opaque anonymous-request failure. No Comet change fixes this; it is resolved upstream if `iceberg-rust` stops wrapping custom loaders in its outer chain or moves its S3 backend to `object_store`.
diff --git a/contributor-guide/adding_a_new_expression.html b/contributor-guide/adding_a_new_expression.html
index 15a85cfdb3..9f52d7a7aa 100644
--- a/contributor-guide/adding_a_new_expression.html
+++ b/contributor-guide/adding_a_new_expression.html
@@ -403,6 +403,7 @@ under the License.
<li class="toctree-l1"><a class="reference internal" href="jvm_shuffle.html">JVM Shuffle</a></li>
<li class="toctree-l1"><a class="reference internal" href="native_shuffle.html">Native Shuffle</a></li>
<li class="toctree-l1"><a class="reference internal" href="sql_error_propagation.html">ANSI Error Propagation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="s3-credential-provider-design.html">S3 Credential Provider Design</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Adding Functionality</span></p>
<ul class="current nav bd-sidenav">
diff --git a/contributor-guide/adding_a_new_operator.html b/contributor-guide/adding_a_new_operator.html
index 93c60ef707..3dc7d15ed3 100644
--- a/contributor-guide/adding_a_new_operator.html
+++ b/contributor-guide/adding_a_new_operator.html
@@ -66,7 +66,7 @@ under the License.
<link rel="index" title="Index" href="../genindex.html" />
<link rel="search" title="Search" href="../search.html" />
<link rel="next" title="Adding a New Expression" href="adding_a_new_expression.html" />
- <link rel="prev" title="ANSI SQL Error Propagation in Comet" href="sql_error_propagation.html" />
+ <link rel="prev" title="S3 Credential Provider SPI: Design Notes" href="s3-credential-provider-design.html" />
<meta name="viewport" content="width=device-width, initial-scale=1"/>
<meta name="docsearch:language" content="en"/>
<meta name="docsearch:version" content="" />
@@ -403,6 +403,7 @@ under the License.
<li class="toctree-l1"><a class="reference internal" href="jvm_shuffle.html">JVM Shuffle</a></li>
<li class="toctree-l1"><a class="reference internal" href="native_shuffle.html">Native Shuffle</a></li>
<li class="toctree-l1"><a class="reference internal" href="sql_error_propagation.html">ANSI Error Propagation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="s3-credential-provider-design.html">S3 Credential Provider Design</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Adding Functionality</span></p>
<ul class="current nav bd-sidenav">
@@ -1098,12 +1099,12 @@ manages the boundary between JVM and native execution.</p>
<div class="prev-next-area">
<a class="left-prev"
- href="sql_error_propagation.html"
+ href="s3-credential-provider-design.html"
title="previous page">
<i class="fa-solid fa-angle-left"></i>
<div class="prev-next-info">
<p class="prev-next-subtitle">previous</p>
- <p class="prev-next-title">ANSI SQL Error Propagation in Comet</p>
+ <p class="prev-next-title">S3 Credential Provider SPI: Design Notes</p>
</div>
</a>
<a class="right-next"
diff --git a/contributor-guide/adding_a_new_spark_version.html b/contributor-guide/adding_a_new_spark_version.html
index 104f704bbf..afab82c958 100644
--- a/contributor-guide/adding_a_new_spark_version.html
+++ b/contributor-guide/adding_a_new_spark_version.html
@@ -403,6 +403,7 @@ under the License.
<li class="toctree-l1"><a class="reference internal" href="jvm_shuffle.html">JVM Shuffle</a></li>
<li class="toctree-l1"><a class="reference internal" href="native_shuffle.html">Native Shuffle</a></li>
<li class="toctree-l1"><a class="reference internal" href="sql_error_propagation.html">ANSI Error Propagation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="s3-credential-provider-design.html">S3 Credential Provider Design</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Adding Functionality</span></p>
<ul class="current nav bd-sidenav">
diff --git a/contributor-guide/benchmark-results/tpc-ds.html b/contributor-guide/benchmark-results/tpc-ds.html
index 8850be72b8..4d5bf9b1c4 100644
--- a/contributor-guide/benchmark-results/tpc-ds.html
+++ b/contributor-guide/benchmark-results/tpc-ds.html
@@ -403,6 +403,7 @@ under the License.
<li class="toctree-l1"><a class="reference internal" href="../jvm_shuffle.html">JVM Shuffle</a></li>
<li class="toctree-l1"><a class="reference internal" href="../native_shuffle.html">Native Shuffle</a></li>
<li class="toctree-l1"><a class="reference internal" href="../sql_error_propagation.html">ANSI Error Propagation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../s3-credential-provider-design.html">S3 Credential Provider Design</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Adding Functionality</span></p>
<ul class="nav bd-sidenav">
diff --git a/contributor-guide/benchmark-results/tpc-h.html b/contributor-guide/benchmark-results/tpc-h.html
index de840975f0..b75aa4ff2a 100644
--- a/contributor-guide/benchmark-results/tpc-h.html
+++ b/contributor-guide/benchmark-results/tpc-h.html
@@ -403,6 +403,7 @@ under the License.
<li class="toctree-l1"><a class="reference internal" href="../jvm_shuffle.html">JVM Shuffle</a></li>
<li class="toctree-l1"><a class="reference internal" href="../native_shuffle.html">Native Shuffle</a></li>
<li class="toctree-l1"><a class="reference internal" href="../sql_error_propagation.html">ANSI Error Propagation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../s3-credential-provider-design.html">S3 Credential Provider Design</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Adding Functionality</span></p>
<ul class="nav bd-sidenav">
diff --git a/contributor-guide/benchmarking.html b/contributor-guide/benchmarking.html
index 038c560773..12b6e298cf 100644
--- a/contributor-guide/benchmarking.html
+++ b/contributor-guide/benchmarking.html
@@ -403,6 +403,7 @@ under the License.
<li class="toctree-l1"><a class="reference internal" href="jvm_shuffle.html">JVM Shuffle</a></li>
<li class="toctree-l1"><a class="reference internal" href="native_shuffle.html">Native Shuffle</a></li>
<li class="toctree-l1"><a class="reference internal" href="sql_error_propagation.html">ANSI Error Propagation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="s3-credential-provider-design.html">S3 Credential Provider Design</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Adding Functionality</span></p>
<ul class="nav bd-sidenav">
diff --git a/contributor-guide/benchmarking_aws_ec2.html b/contributor-guide/benchmarking_aws_ec2.html
index bd4943c30d..5009d150cb 100644
--- a/contributor-guide/benchmarking_aws_ec2.html
+++ b/contributor-guide/benchmarking_aws_ec2.html
@@ -403,6 +403,7 @@ under the License.
<li class="toctree-l1"><a class="reference internal" href="jvm_shuffle.html">JVM Shuffle</a></li>
<li class="toctree-l1"><a class="reference internal" href="native_shuffle.html">Native Shuffle</a></li>
<li class="toctree-l1"><a class="reference internal" href="sql_error_propagation.html">ANSI Error Propagation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="s3-credential-provider-design.html">S3 Credential Provider Design</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Adding Functionality</span></p>
<ul class="nav bd-sidenav">
diff --git a/contributor-guide/benchmarking_macos.html b/contributor-guide/benchmarking_macos.html
index 4f3c0b30fe..611dc6d8a3 100644
--- a/contributor-guide/benchmarking_macos.html
+++ b/contributor-guide/benchmarking_macos.html
@@ -403,6 +403,7 @@ under the License.
<li class="toctree-l1"><a class="reference internal" href="jvm_shuffle.html">JVM Shuffle</a></li>
<li class="toctree-l1"><a class="reference internal" href="native_shuffle.html">Native Shuffle</a></li>
<li class="toctree-l1"><a class="reference internal" href="sql_error_propagation.html">ANSI Error Propagation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="s3-credential-provider-design.html">S3 Credential Provider Design</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Adding Functionality</span></p>
<ul class="nav bd-sidenav">
diff --git a/contributor-guide/benchmarking_spark_sql_perf.html b/contributor-guide/benchmarking_spark_sql_perf.html
index b885076d92..989f18aef8 100644
--- a/contributor-guide/benchmarking_spark_sql_perf.html
+++ b/contributor-guide/benchmarking_spark_sql_perf.html
@@ -403,6 +403,7 @@ under the License.
<li class="toctree-l1"><a class="reference internal" href="jvm_shuffle.html">JVM Shuffle</a></li>
<li class="toctree-l1"><a class="reference internal" href="native_shuffle.html">Native Shuffle</a></li>
<li class="toctree-l1"><a class="reference internal" href="sql_error_propagation.html">ANSI Error Propagation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="s3-credential-provider-design.html">S3 Credential Provider Design</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Adding Functionality</span></p>
<ul class="nav bd-sidenav">
diff --git a/contributor-guide/bug_triage.html b/contributor-guide/bug_triage.html
index 7ac6b23de2..4288959183 100644
--- a/contributor-guide/bug_triage.html
+++ b/contributor-guide/bug_triage.html
@@ -403,6 +403,7 @@ under the License.
<li class="toctree-l1"><a class="reference internal" href="jvm_shuffle.html">JVM Shuffle</a></li>
<li class="toctree-l1"><a class="reference internal" href="native_shuffle.html">Native Shuffle</a></li>
<li class="toctree-l1"><a class="reference internal" href="sql_error_propagation.html">ANSI Error Propagation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="s3-credential-provider-design.html">S3 Credential Provider Design</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Adding Functionality</span></p>
<ul class="nav bd-sidenav">
diff --git a/contributor-guide/contributing.html b/contributor-guide/contributing.html
index c74b49961b..eb95b9e38c 100644
--- a/contributor-guide/contributing.html
+++ b/contributor-guide/contributing.html
@@ -403,6 +403,7 @@ under the License.
<li class="toctree-l1"><a class="reference internal" href="jvm_shuffle.html">JVM Shuffle</a></li>
<li class="toctree-l1"><a class="reference internal" href="native_shuffle.html">Native Shuffle</a></li>
<li class="toctree-l1"><a class="reference internal" href="sql_error_propagation.html">ANSI Error Propagation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="s3-credential-provider-design.html">S3 Credential Provider Design</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Adding Functionality</span></p>
<ul class="nav bd-sidenav">
diff --git a/contributor-guide/debugging.html b/contributor-guide/debugging.html
index a543de8bb4..c14f17daec 100644
--- a/contributor-guide/debugging.html
+++ b/contributor-guide/debugging.html
@@ -403,6 +403,7 @@ under the License.
<li class="toctree-l1"><a class="reference internal" href="jvm_shuffle.html">JVM Shuffle</a></li>
<li class="toctree-l1"><a class="reference internal" href="native_shuffle.html">Native Shuffle</a></li>
<li class="toctree-l1"><a class="reference internal" href="sql_error_propagation.html">ANSI Error Propagation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="s3-credential-provider-design.html">S3 Credential Provider Design</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Adding Functionality</span></p>
<ul class="nav bd-sidenav">
diff --git a/contributor-guide/development.html b/contributor-guide/development.html
index 0751e2a2a4..7eb947cd87 100644
--- a/contributor-guide/development.html
+++ b/contributor-guide/development.html
@@ -403,6 +403,7 @@ under the License.
<li class="toctree-l1"><a class="reference internal" href="jvm_shuffle.html">JVM Shuffle</a></li>
<li class="toctree-l1"><a class="reference internal" href="native_shuffle.html">Native Shuffle</a></li>
<li class="toctree-l1"><a class="reference internal" href="sql_error_propagation.html">ANSI Error Propagation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="s3-credential-provider-design.html">S3 Credential Provider Design</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Adding Functionality</span></p>
<ul class="nav bd-sidenav">
diff --git a/contributor-guide/ffi.html b/contributor-guide/ffi.html
index b389c9a77e..17551db3ad 100644
--- a/contributor-guide/ffi.html
+++ b/contributor-guide/ffi.html
@@ -403,6 +403,7 @@ under the License.
<li class="toctree-l1"><a class="reference internal" href="jvm_shuffle.html">JVM Shuffle</a></li>
<li class="toctree-l1"><a class="reference internal" href="native_shuffle.html">Native Shuffle</a></li>
<li class="toctree-l1"><a class="reference internal" href="sql_error_propagation.html">ANSI Error Propagation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="s3-credential-provider-design.html">S3 Credential Provider Design</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Adding Functionality</span></p>
<ul class="nav bd-sidenav">
diff --git a/contributor-guide/iceberg-spark-tests.html b/contributor-guide/iceberg-spark-tests.html
index af4dd2306b..de971c846b 100644
--- a/contributor-guide/iceberg-spark-tests.html
+++ b/contributor-guide/iceberg-spark-tests.html
@@ -403,6 +403,7 @@ under the License.
<li class="toctree-l1"><a class="reference internal" href="jvm_shuffle.html">JVM Shuffle</a></li>
<li class="toctree-l1"><a class="reference internal" href="native_shuffle.html">Native Shuffle</a></li>
<li class="toctree-l1"><a class="reference internal" href="sql_error_propagation.html">ANSI Error Propagation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="s3-credential-provider-design.html">S3 Credential Provider Design</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Adding Functionality</span></p>
<ul class="nav bd-sidenav">
diff --git a/contributor-guide/index.html b/contributor-guide/index.html
index 59db098a09..192f854c45 100644
--- a/contributor-guide/index.html
+++ b/contributor-guide/index.html
@@ -403,6 +403,7 @@ under the License.
<li class="toctree-l1"><a class="reference internal" href="jvm_shuffle.html">JVM Shuffle</a></li>
<li class="toctree-l1"><a class="reference internal" href="native_shuffle.html">Native Shuffle</a></li>
<li class="toctree-l1"><a class="reference internal" href="sql_error_propagation.html">ANSI Error Propagation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="s3-credential-provider-design.html">S3 Credential Provider Design</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Adding Functionality</span></p>
<ul class="nav bd-sidenav">
diff --git a/contributor-guide/jvm_shuffle.html b/contributor-guide/jvm_shuffle.html
index b07dc2b777..4ed8ead705 100644
--- a/contributor-guide/jvm_shuffle.html
+++ b/contributor-guide/jvm_shuffle.html
@@ -403,6 +403,7 @@ under the License.
<li class="toctree-l1 current active"><a class="current reference internal" href="#">JVM Shuffle</a></li>
<li class="toctree-l1"><a class="reference internal" href="native_shuffle.html">Native Shuffle</a></li>
<li class="toctree-l1"><a class="reference internal" href="sql_error_propagation.html">ANSI Error Propagation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="s3-credential-provider-design.html">S3 Credential Provider Design</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Adding Functionality</span></p>
<ul class="nav bd-sidenav">
diff --git a/contributor-guide/native_shuffle.html b/contributor-guide/native_shuffle.html
index 0b0c5ddf0b..6d70699620 100644
--- a/contributor-guide/native_shuffle.html
+++ b/contributor-guide/native_shuffle.html
@@ -403,6 +403,7 @@ under the License.
<li class="toctree-l1"><a class="reference internal" href="jvm_shuffle.html">JVM Shuffle</a></li>
<li class="toctree-l1 current active"><a class="current reference internal" href="#">Native Shuffle</a></li>
<li class="toctree-l1"><a class="reference internal" href="sql_error_propagation.html">ANSI Error Propagation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="s3-credential-provider-design.html">S3 Credential Provider Design</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Adding Functionality</span></p>
<ul class="nav bd-sidenav">
diff --git a/contributor-guide/plugin_overview.html b/contributor-guide/plugin_overview.html
index de4c6c4992..06733c1651 100644
--- a/contributor-guide/plugin_overview.html
+++ b/contributor-guide/plugin_overview.html
@@ -403,6 +403,7 @@ under the License.
<li class="toctree-l1"><a class="reference internal" href="jvm_shuffle.html">JVM Shuffle</a></li>
<li class="toctree-l1"><a class="reference internal" href="native_shuffle.html">Native Shuffle</a></li>
<li class="toctree-l1"><a class="reference internal" href="sql_error_propagation.html">ANSI Error Propagation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="s3-credential-provider-design.html">S3 Credential Provider Design</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Adding Functionality</span></p>
<ul class="nav bd-sidenav">
diff --git a/contributor-guide/profiling.html b/contributor-guide/profiling.html
index 645fa7f527..051dfa188d 100644
--- a/contributor-guide/profiling.html
+++ b/contributor-guide/profiling.html
@@ -403,6 +403,7 @@ under the License.
<li class="toctree-l1"><a class="reference internal" href="jvm_shuffle.html">JVM Shuffle</a></li>
<li class="toctree-l1"><a class="reference internal" href="native_shuffle.html">Native Shuffle</a></li>
<li class="toctree-l1"><a class="reference internal" href="sql_error_propagation.html">ANSI Error Propagation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="s3-credential-provider-design.html">S3 Credential Provider Design</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Adding Functionality</span></p>
<ul class="nav bd-sidenav">
diff --git a/contributor-guide/release_process.html b/contributor-guide/release_process.html
index 942f4af7b9..a7401323cd 100644
--- a/contributor-guide/release_process.html
+++ b/contributor-guide/release_process.html
@@ -403,6 +403,7 @@ under the License.
<li class="toctree-l1"><a class="reference internal" href="jvm_shuffle.html">JVM Shuffle</a></li>
<li class="toctree-l1"><a class="reference internal" href="native_shuffle.html">Native Shuffle</a></li>
<li class="toctree-l1"><a class="reference internal" href="sql_error_propagation.html">ANSI Error Propagation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="s3-credential-provider-design.html">S3 Credential Provider Design</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Adding Functionality</span></p>
<ul class="nav bd-sidenav">
diff --git a/contributor-guide/roadmap.html b/contributor-guide/roadmap.html
index 7a190bbb82..c773897ef0 100644
--- a/contributor-guide/roadmap.html
+++ b/contributor-guide/roadmap.html
@@ -403,6 +403,7 @@ under the License.
<li class="toctree-l1"><a class="reference internal" href="jvm_shuffle.html">JVM Shuffle</a></li>
<li class="toctree-l1"><a class="reference internal" href="native_shuffle.html">Native Shuffle</a></li>
<li class="toctree-l1"><a class="reference internal" href="sql_error_propagation.html">ANSI Error Propagation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="s3-credential-provider-design.html">S3 Credential Provider Design</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Adding Functionality</span></p>
<ul class="nav bd-sidenav">
diff --git a/contributor-guide/jvm_shuffle.html b/contributor-guide/s3-credential-provider-design.html
similarity index 50%
copy from contributor-guide/jvm_shuffle.html
copy to contributor-guide/s3-credential-provider-design.html
index b07dc2b777..f5ff2cdd60 100644
--- a/contributor-guide/jvm_shuffle.html
+++ b/contributor-guide/s3-credential-provider-design.html
@@ -27,7 +27,7 @@ under the License.
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="viewport" content="width=device-width, initial-scale=1" />
- <title>JVM Shuffle — Apache DataFusion Comet documentation</title>
+ <title>S3 Credential Provider SPI: Design Notes — Apache DataFusion Comet documentation</title>
@@ -61,12 +61,12 @@ under the License.
<script src="../_static/documentation_options.js?v=5929fcd5"></script>
<script src="../_static/doctools.js?v=9a2dae69"></script>
<script src="../_static/sphinx_highlight.js?v=dc90522c"></script>
- <script>DOCUMENTATION_OPTIONS.pagename = 'contributor-guide/jvm_shuffle';</script>
+ <script>DOCUMENTATION_OPTIONS.pagename = 'contributor-guide/s3-credential-provider-design';</script>
<script async="true" defer="true" src="https://buttons.github.io/buttons.js"></script>
<link rel="index" title="Index" href="../genindex.html" />
<link rel="search" title="Search" href="../search.html" />
- <link rel="next" title="Native Shuffle" href="native_shuffle.html" />
- <link rel="prev" title="Arrow FFI Usage in Comet" href="ffi.html" />
+ <link rel="next" title="Adding a New Operator" href="adding_a_new_operator.html" />
+ <link rel="prev" title="ANSI SQL Error Propagation in Comet" href="sql_error_propagation.html" />
<meta name="viewport" content="width=device-width, initial-scale=1"/>
<meta name="docsearch:language" content="en"/>
<meta name="docsearch:version" content="" />
@@ -400,9 +400,10 @@ under the License.
<li class="toctree-l1"><a class="reference internal" href="plugin_overview.html">Comet Plugin Architecture</a></li>
<li class="toctree-l1"><a class="reference internal" href="ffi.html">Arrow FFI</a></li>
-<li class="toctree-l1 current active"><a class="current reference internal" href="#">JVM Shuffle</a></li>
+<li class="toctree-l1"><a class="reference internal" href="jvm_shuffle.html">JVM Shuffle</a></li>
<li class="toctree-l1"><a class="reference internal" href="native_shuffle.html">Native Shuffle</a></li>
<li class="toctree-l1"><a class="reference internal" href="sql_error_propagation.html">ANSI Error Propagation</a></li>
+<li class="toctree-l1 current active"><a class="current reference internal" href="#">S3 Credential Provider Design</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Adding Functionality</span></p>
<ul class="nav bd-sidenav">
@@ -486,7 +487,7 @@ under the License.
<li class="breadcrumb-item"><a href="index.html" class="nav-link">Comet Contributor Guide</a></li>
- <li class="breadcrumb-item active" aria-current="page"><span class="ellipsis">JVM Shuffle</span></li>
+ <li class="breadcrumb-item active" aria-current="page"><span class="ellipsis">S3 Credential Provider SPI: Design Notes</span></li>
</ul>
</nav>
</div>
@@ -521,272 +522,113 @@ KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
-<section id="jvm-shuffle">
-<h1>JVM Shuffle<a class="headerlink" href="#jvm-shuffle" title="Link to this heading">#</a></h1>
-<p>This document describes Comet’s JVM-based columnar shuffle implementation (<code class="docutils literal notranslate"><span class="pre">CometColumnarShuffle</span></code>), which
-writes shuffle data in Arrow IPC format using JVM code with native encoding. For the fully native
-alternative, see <a class="reference internal" href="native_shuffle.html"><span class="std std-doc">Native Shuffle</span></a>.</p>
-<section id="overview">
-<h2>Overview<a class="headerlink" href="#overview" title="Link to this heading">#</a></h2>
-<p>Comet provides two shuffle implementations:</p>
+<section id="s3-credential-provider-spi-design-notes">
+<h1>S3 Credential Provider SPI: Design Notes<a class="headerlink" href="#s3-credential-provider-spi-design-notes" title="Link to this heading">#</a></h1>
+<p>This page captures why the <code class="docutils literal notranslate"><span class="pre">org.apache.comet.cloud.s3.CometS3CredentialProvider</span></code> SPI is shaped the way it is. The user-facing contract and operator setup live in the user guide page on S3 credential providers; this page is for maintainers and reviewers who want the design rationale.</p>
+<section id="the-gap-the-spi-fills">
+<h2>The gap the SPI fills<a class="headerlink" href="#the-gap-the-spi-fills" title="Link to this heading">#</a></h2>
+<p>Comet’s native scan paths (<code class="docutils literal notranslate"><span class="pre">object_store</span></code> for raw Parquet, <code class="docutils literal notranslate"><span class="pre">opendal</span></code> via <code class="docutils literal notranslate"><span class="pre">iceberg-rust</span></code> for Iceberg) bypass Spark’s Hadoop S3A code path. That means credentials cannot flow through any of the contracts that vendors typically wire into for S3A:</p>
<ul class="simple">
-<li><p><strong>CometNativeShuffle</strong> (<code class="docutils literal notranslate"><span class="pre">CometExchange</span></code>): Fully native shuffle using Rust. Takes columnar input directly
-from Comet native operators and performs partitioning in native code.</p></li>
-<li><p><strong>CometColumnarShuffle</strong> (<code class="docutils literal notranslate"><span class="pre">CometColumnarExchange</span></code>): JVM-based shuffle that operates on rows internally,
-buffers <code class="docutils literal notranslate"><span class="pre">UnsafeRow</span></code>s in memory pages, and uses native code (via JNI) to encode them to Arrow IPC format.
-Uses Spark’s partitioner for partition assignment. Can accept either row-based or columnar input
-(columnar input is converted to rows via <code class="docutils literal notranslate"><span class="pre">ColumnarToRowExec</span></code>).</p></li>
+<li><p><code class="docutils literal notranslate"><span class="pre">org.apache.spark.deploy.security.cloud.CloudCredentialsProvider</span></code> yields a single JWT per service name. No path argument, no AWS credential.</p></li>
+<li><p>Hadoop S3A custom signers hide path-aware logic inside <code class="docutils literal notranslate"><span class="pre">Signer.sign(request,</span> <span class="pre">credentials)</span></code>. The credential never leaves the signing pipeline, and the underlying secret is an HMAC key that is not present in the signed output, so running the signer against a synthesized request cannot recover it.</p></li>
+<li><p><code class="docutils literal notranslate"><span class="pre">AWSCredentialsProvider.getCredentials()</span></code> (AWS SDK v1) and <code class="docutils literal notranslate"><span class="pre">AwsCredentialsProvider.resolveCredentials()</span></code> (v2) are parameterless. They cannot vend per-path credentials.</p></li>
+<li><p>Reflecting into vendor singletons would encode per-vendor class names and lifecycles in Comet and would silently break on vendor upgrades.</p></li>
</ul>
-<p>The JVM shuffle is selected via <code class="docutils literal notranslate"><span class="pre">CometShuffleDependency.shuffleType</span></code>.</p>
-</section>
-<section id="when-jvm-shuffle-is-used">
-<h2>When JVM Shuffle is Used<a class="headerlink" href="#when-jvm-shuffle-is-used" title="Link to this heading">#</a></h2>
-<p>JVM shuffle (<code class="docutils literal notranslate"><span class="pre">CometColumnarExchange</span></code>) is used instead of native shuffle (<code class="docutils literal notranslate"><span class="pre">CometExchange</span></code>) in the following cases:</p>
-<ol class="arabic simple">
-<li><p><strong>Shuffle mode is explicitly set to “jvm”</strong>: When <code class="docutils literal notranslate"><span class="pre">spark.comet.exec.shuffle.mode</span></code> is set to <code class="docutils literal notranslate"><span class="pre">jvm</span></code>.</p></li>
-<li><p><strong>Child plan is not a Comet native operator</strong>: When the child plan is a Spark row-based operator
-(not a <code class="docutils literal notranslate"><span class="pre">CometPlan</span></code>), JVM shuffle is the only option since native shuffle requires columnar input
-from Comet operators.</p></li>
-<li><p><strong>Unsupported partition key types</strong>: For <code class="docutils literal notranslate"><span class="pre">HashPartitioning</span></code> and <code class="docutils literal notranslate"><span class="pre">RangePartitioning</span></code>, native shuffle
-only supports primitive types as partition keys. Complex types (struct, array, map) cannot be used
-as partition keys in native shuffle and will fall back to JVM columnar shuffle. Note that complex types are
-fully supported as data columns in both implementations.</p></li>
-</ol>
-</section>
-<section id="input-handling">
-<h2>Input Handling<a class="headerlink" href="#input-handling" title="Link to this heading">#</a></h2>
-<section id="spark-row-based-input">
-<h3>Spark Row-Based Input<a class="headerlink" href="#spark-row-based-input" title="Link to this heading">#</a></h3>
-<p>When the child plan is a Spark row-based operator, <code class="docutils literal notranslate"><span class="pre">CometColumnarExchange</span></code> calls <code class="docutils literal notranslate"><span class="pre">child.execute()</span></code> which
-returns an <code class="docutils literal notranslate"><span class="pre">RDD[InternalRow]</span></code>. The rows flow directly to the JVM shuffle writers.</p>
-</section>
-<section id="comet-columnar-input">
-<h3>Comet Columnar Input<a class="headerlink" href="#comet-columnar-input" title="Link to this heading">#</a></h3>
-<p>When the child plan is a Comet native operator (e.g., <code class="docutils literal notranslate"><span class="pre">CometHashAggregate</span></code>) but JVM shuffle is selected
-(due to shuffle mode setting or unsupported partitioning), <code class="docutils literal notranslate"><span class="pre">CometColumnarExchange</span></code> still calls
-<code class="docutils literal notranslate"><span class="pre">child.execute()</span></code>. Comet operators implement <code class="docutils literal notranslate"><span class="pre">doExecute()</span></code> by wrapping themselves with <code class="docutils literal notranslate"><span class="pre">ColumnarToRowExec</span></code>:</p>
-<div class="highlight-scala notranslate"><div class="highlight"><pre><span></span><span class="c1">// In CometExec base class</span>
-<span class="k">override</span><span class="w"> </span><span class="k">def</span><span class="w"> </span><span class="nf">doExecute</span><span class="p">():</span><span class="w"> </span><span class="nc">RDD</span><span class="p">[</span><span class="nc">InternalRow</span><span class="p">]</span><span class="w"> </span><span class="o">=</span>
-<span class="w"> </span><span class="nc">ColumnarToRowExec</span><span class="p">(</span><span class="bp">this</span><span class="p">).</span><span class="n">doExecute</span><span class="p">()</span>
-</pre></div>
-</div>
-<p>This means the data path becomes:</p>
-<div class="highlight-default notranslate"><div class="highlight"><pre><span></span>Comet Native (columnar) → ColumnarToRowExec → rows → JVM Shuffle → Arrow IPC → columnar
-</pre></div>
-</div>
-<p>This is less efficient than native shuffle which avoids the columnar-to-row conversion:</p>
-<div class="highlight-default notranslate"><div class="highlight"><pre><span></span>Comet Native (columnar) → Native Shuffle → Arrow IPC → columnar
-</pre></div>
-</div>
-</section>
-<section id="why-use-spark-s-partitioner">
-<h3>Why Use Spark’s Partitioner?<a class="headerlink" href="#why-use-spark-s-partitioner" title="Link to this heading">#</a></h3>
-<p>JVM shuffle uses row-based input so it can leverage Spark’s existing partitioner infrastructure
-(<code class="docutils literal notranslate"><span class="pre">partitioner.getPartition(key)</span></code>). This allows Comet to support all of Spark’s partitioning schemes
-without reimplementing them in Rust. Native shuffle, by contrast, serializes the partitioning scheme
-to protobuf and implements the partitioning logic in native code.</p>
+<p>A Comet-specific SPI is the narrowest fit: a single Java method that takes a <code class="docutils literal notranslate"><span class="pre">CometS3CredentialContext</span></code> (today wrapping <code class="docutils literal notranslate"><span class="pre">bucket</span></code>, <code class="docutils literal notranslate"><span class="pre">path</span></code>, and access <code class="docutils literal notranslate"><span class="pre">mode</span></code>; new fields can be added without breaking [...]
</section>
+<section id="why-config-driven-activation-not-meta-inf-services">
+<h2>Why config-driven activation, not <code class="docutils literal notranslate"><span class="pre">META-INF/services</span></code><a class="headerlink" href="#why-config-driven-activation-not-meta-inf-services" title="Link to this heading">#</a></h2>
+<p>An earlier iteration used <code class="docutils literal notranslate"><span class="pre">ServiceLoader</span></code> discovery. That was rejected because:</p>
+<ul class="simple">
+<li><p>Peer SPIs in the same space (Hadoop <code class="docutils literal notranslate"><span class="pre">AWSCredentialsProvider</span></code>, AWS SDK v2 <code class="docutils literal notranslate"><span class="pre">AwsCredentialsProvider</span></code>, Iceberg <code class="docutils literal notranslate"><span class="pre">AwsClientFactory</span></code>, S3A custom signers) are all class-name-in-config. Vendors are already familiar with that model.</p></li>
+<li><p>ServiceLoader makes activation implicit on classpath presence. A vendor JAR drifting onto a cluster could silently change S3 auth behavior. The config key makes activation explicit.</p></li>
+<li><p>The activation key (<code class="docutils literal notranslate"><span class="pre">fs.s3a.comet.credential.provider.class</span></code>, with per-bucket override) follows the same shape as <code class="docutils literal notranslate"><span class="pre">fs.s3a.bucket.<name>.aws.credentials.provider</span></code>, so operators do not learn a new pattern.</p></li>
+</ul>
+<p>Activation is modeled on <code class="docutils literal notranslate"><span class="pre">parquet.crypto.factory.class</span></code> (Parquet Modular Encryption KMS, see Comet #2447): the user names a single vendor class and the vendor dispatches across multiple credential backends inside that class if they need to. This mirrors how Iceberg’s <code class="docutils literal notranslate"><span class="pre">DecryptionPropertiesFactory</span></code> already behaves for Parquet keys.</p>
</section>
-<section id="architecture">
-<h2>Architecture<a class="headerlink" href="#architecture" title="Link to this heading">#</a></h2>
-<div class="highlight-default notranslate"><div class="highlight"><pre><span></span>┌─────────────────────────────────────────────────────────────────────────┐
-│ CometShuffleManager │
-│ - Extends Spark's ShuffleManager │
-│ - Routes to appropriate writer/reader based on ShuffleHandle type │
-└─────────────────────────────────────────────────────────────────────────┘
- │
- ┌────────────────────────┼────────────────────────┐
- ▼ ▼ ▼
-┌─────────────────────┐ ┌─────────────────────┐ ┌─────────────────────┐
-│ CometBypassMerge- │ │ CometUnsafe- │ │ CometNative- │
-│ SortShuffleWriter │ │ ShuffleWriter │ │ ShuffleWriter │
-│ (hash-based) │ │ (sort-based) │ │ (fully native) │
-└─────────────────────┘ └─────────────────────┘ └─────────────────────┘
- │ │
- ▼ ▼
-┌─────────────────────┐ ┌─────────────────────┐
-│ CometDiskBlock- │ │ CometShuffleExternal│
-│ Writer │ │ Sorter │
-└─────────────────────┘ └─────────────────────┘
- │ │
- └────────────┬───────────┘
- ▼
- ┌─────────────────────┐
- │ SpillWriter │
- │ (native encoding │
- │ via JNI) │
- └─────────────────────┘
-</pre></div>
-</div>
+<section id="why-fqcn-dispatchkey-catalogproperties-keying">
+<h2>Why <code class="docutils literal notranslate"><span class="pre">(FQCN,</span> <span class="pre">dispatchKey,</span> <span class="pre">catalogProperties)</span></code> keying<a class="headerlink" href="#why-fqcn-dispatchkey-catalogproperties-keying" title="Link to this heading">#</a></h2>
+<p>Comet caches one provider instance per <code class="docutils literal notranslate"><span class="pre">(FQCN,</span> <span class="pre">dispatchKey,</span> <span class="pre">catalogProperties)</span></code> triple. The dispatch key is the Spark V2 catalog name on the Iceberg path and the bucket on the Parquet path.</p>
+<ul class="simple">
+<li><p>Two catalogs that share one provider class (typical in multi-tenant deployments) need isolated <code class="docutils literal notranslate"><span class="pre">initialize</span></code> maps because their <code class="docutils literal notranslate"><span class="pre">catalogProperties</span></code> differ. Without <code class="docutils literal notranslate"><span class="pre">dispatchKey</span></code>, the second <code class="docutils literal notranslate"><span class="pre">initialize</span [...]
+<li><p>The bucket as <code class="docutils literal notranslate"><span class="pre">dispatchKey</span></code> for Parquet gives vendors per-bucket isolation when the same provider is named under several <code class="docutils literal notranslate"><span class="pre">fs.s3a.bucket.<name>.comet.credential.provider.class</span></code> keys.</p></li>
+<li><p><code class="docutils literal notranslate"><span class="pre">catalogProperties</span></code> enters the key to handle multi-tenant JVMs (Spark Connect, Thrift Server, <code class="docutils literal notranslate"><span class="pre">SparkSession.newSession()</span></code>) where two sessions can configure the same provider class against the same <code class="docutils literal notranslate"><span class="pre">dispatchKey</span></code> but with different REST endpoints, OAuth tokens, or ven [...]
+<li><p>Keying solely by FQCN would force vendors to encode multi-tenant routing in static state. The triple-key keeps each call site independent.</p></li>
+</ul>
+<p><code class="docutils literal notranslate"><span class="pre">ensureInitialized</span></code> returns a <code class="docutils literal notranslate"><span class="pre">long</span></code> handle that the native bridge stashes and replays on every per-request call. Routing per-request lookups by handle avoids re-sending the property bag across JNI on the hot path and unambiguously selects the right provider when the same <code class="docutils literal notranslate"><span class="pre">(FQCN,</s [...]
</section>
-<section id="key-classes">
-<h2>Key Classes<a class="headerlink" href="#key-classes" title="Link to this heading">#</a></h2>
-<section id="shuffle-manager">
-<h3>Shuffle Manager<a class="headerlink" href="#shuffle-manager" title="Link to this heading">#</a></h3>
-<div class="pst-scrollable-table-container"><table class="table">
-<thead>
-<tr class="row-odd"><th class="head"><p>Class</p></th>
-<th class="head"><p>Location</p></th>
-<th class="head"><p>Description</p></th>
-</tr>
-</thead>
-<tbody>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">CometShuffleManager</span></code></p></td>
-<td><p><code class="docutils literal notranslate"><span class="pre">.../shuffle/CometShuffleManager.scala</span></code></p></td>
-<td><p>Entry point. Extends Spark’s <code class="docutils literal notranslate"><span class="pre">ShuffleManager</span></code>. Selects writer/reader based on handle type. Delegates non-Comet shuffles to <code class="docutils literal notranslate"><span class="pre">SortShuffleManager</span></code>.</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">CometShuffleDependency</span></code></p></td>
-<td><p><code class="docutils literal notranslate"><span class="pre">.../shuffle/CometShuffleDependency.scala</span></code></p></td>
-<td><p>Extends <code class="docutils literal notranslate"><span class="pre">ShuffleDependency</span></code>. Contains <code class="docutils literal notranslate"><span class="pre">shuffleType</span></code> (<code class="docutils literal notranslate"><span class="pre">CometColumnarShuffle</span></code> or <code class="docutils literal notranslate"><span class="pre">CometNativeShuffle</span></code>) and schema info.</p></td>
-</tr>
-</tbody>
-</table>
-</div>
+<section id="why-fresh-construction-in-initialize-not-probing-a-jvm-wide-static">
+<h2>Why fresh construction in <code class="docutils literal notranslate"><span class="pre">initialize</span></code>, not probing a JVM-wide static<a class="headerlink" href="#why-fresh-construction-in-initialize-not-probing-a-jvm-wide-static" title="Link to this heading">#</a></h2>
+<p>A provider implementation might be tempted to probe an existing static populated elsewhere (e.g. by a Hadoop S3A signer’s <code class="docutils literal notranslate"><span class="pre">registerStore</span></code> callback) and reuse the credential cache that the Hadoop path uses. That fails on Comet-only executors:</p>
+<ul class="simple">
+<li><p>The driver JVM hits <code class="docutils literal notranslate"><span class="pre">S3AFileSystem.initialize</span></code> during analysis (raw <code class="docutils literal notranslate"><span class="pre">s3a://</span></code> paths) or during Hadoop catalog manifest reads (Iceberg with Hadoop catalog), so the static is populated there.</p></li>
+<li><p>The driver may not hit <code class="docutils literal notranslate"><span class="pre">S3AFileSystem</span></code> at all under Iceberg with REST catalog plus <code class="docutils literal notranslate"><span class="pre">S3FileIO</span></code>, because <code class="docutils literal notranslate"><span class="pre">S3FileIO</span></code> calls AWS SDK directly without going through the Hadoop layer. The static stays null.</p></li>
+<li><p>Executors with Comet-only reads never instantiate <code class="docutils literal notranslate"><span class="pre">S3AFileSystem</span></code>. The data path is <code class="docutils literal notranslate"><span class="pre">object_store</span></code> (raw Parquet) or <code class="docutils literal notranslate"><span class="pre">opendal</span></code> via <code class="docutils literal notranslate"><span class="pre">iceberg-rust</span></code> (Iceberg native scan). Neither touches Hadoop S3 [...]
+</ul>
+<p>Constructing a fresh provider from <code class="docutils literal notranslate"><span class="pre">catalogProperties</span></code> plus <code class="docutils literal notranslate"><span class="pre">SparkEnv</span></code> is the only strategy that works across all four cases. The trade-off is that on the driver (and any JVM where Hadoop S3A is also active), two credential caches now exist for the same identity: one inside the Hadoop signer’s provider, one inside the SPI implementation’s. T [...]
</section>
-<section id="shuffle-handles">
-<h3>Shuffle Handles<a class="headerlink" href="#shuffle-handles" title="Link to this heading">#</a></h3>
-<div class="pst-scrollable-table-container"><table class="table">
-<thead>
-<tr class="row-odd"><th class="head"><p>Handle</p></th>
-<th class="head"><p>Writer Strategy</p></th>
-</tr>
-</thead>
-<tbody>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">CometBypassMergeSortShuffleHandle</span></code></p></td>
-<td><p>Hash-based: one file per partition, merged at end</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">CometSerializedShuffleHandle</span></code></p></td>
-<td><p>Sort-based: records sorted by partition ID, single output</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">CometNativeShuffleHandle</span></code></p></td>
-<td><p>Fully native shuffle</p></td>
-</tr>
-</tbody>
-</table>
-</div>
-<p>Selection logic in <code class="docutils literal notranslate"><span class="pre">CometShuffleManager.shouldBypassMergeSort()</span></code>:</p>
+<section id="why-no-comet-side-cache">
+<h2>Why no Comet-side cache<a class="headerlink" href="#why-no-comet-side-cache" title="Link to this heading">#</a></h2>
+<p>Comet’s bridge does not maintain a TTL cache, schedule refresh, or broadcast catalog state. All of that is the vendor’s responsibility:</p>
<ul class="simple">
-<li><p>Uses bypass if partitions < threshold AND partitions × cores ≤ max threads</p></li>
-<li><p>Otherwise uses sort-based to avoid OOM from many concurrent writers</p></li>
+<li><p>Iceberg vendors get <code class="docutils literal notranslate"><span class="pre">software.amazon.awssdk.utils.cache.CachedSupplier</span></code> for free inside <code class="docutils literal notranslate"><span class="pre">org.apache.iceberg.aws.s3.VendedCredentialsProvider</span></code>.</p></li>
+<li><p>Custom-STS vendors write whatever cache fits their refresh model.</p></li>
+<li><p>Driver-only state is distributed via <code class="docutils literal notranslate"><span class="pre">initialize</span></code>’s <code class="docutils literal notranslate"><span class="pre">catalogProperties</span></code> (Iceberg path) or read from Hadoop conf via <code class="docutils literal notranslate"><span class="pre">SparkEnv</span></code> (Parquet path). Both are plan-time snapshots: Comet does not re-execute the catalog or push fresh values to running scans. Vendors that nee [...]
</ul>
+<p>A Comet-side cache would have to either expose a tuning knob (TTL, max size, eviction policy) and grow over time, or be hardcoded and surprise vendors whose policies disagree. The bridge intentionally has neither and forwards every call.</p>
</section>
-<section id="writers">
-<h3>Writers<a class="headerlink" href="#writers" title="Link to this heading">#</a></h3>
+<section id="path-specific-behavior">
+<h2>Path-specific behavior<a class="headerlink" href="#path-specific-behavior" title="Link to this heading">#</a></h2>
+<p><code class="docutils literal notranslate"><span class="pre">object_store::CredentialProvider</span></code> and <code class="docutils literal notranslate"><span class="pre">reqsign_core::ProvideCredential</span></code> differ in what they consume:</p>
<div class="pst-scrollable-table-container"><table class="table">
<thead>
-<tr class="row-odd"><th class="head"><p>Class</p></th>
-<th class="head"><p>Location</p></th>
-<th class="head"><p>Description</p></th>
+<tr class="row-odd"><th class="head"><p>Concern</p></th>
+<th class="head"><p>Parquet (<code class="docutils literal notranslate"><span class="pre">object_store</span></code>)</p></th>
+<th class="head"><p>Iceberg (<code class="docutils literal notranslate"><span class="pre">opendal</span></code> via <code class="docutils literal notranslate"><span class="pre">reqsign-core</span></code>)</p></th>
</tr>
</thead>
<tbody>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">CometBypassMergeSortShuffleWriter</span></code></p></td>
-<td><p><code class="docutils literal notranslate"><span class="pre">.../shuffle/CometBypassMergeSortShuffleWriter.java</span></code></p></td>
-<td><p>Hash-based writer. Creates one <code class="docutils literal notranslate"><span class="pre">CometDiskBlockWriter</span></code> per partition. Supports async writes.</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">CometUnsafeShuffleWriter</span></code></p></td>
-<td><p><code class="docutils literal notranslate"><span class="pre">.../shuffle/CometUnsafeShuffleWriter.java</span></code></p></td>
-<td><p>Sort-based writer. Uses <code class="docutils literal notranslate"><span class="pre">CometShuffleExternalSorter</span></code> to buffer and sort records, then merges spill files.</p></td>
+<tr class="row-even"><td><p>Trait method</p></td>
+<td><p><code class="docutils literal notranslate"><span class="pre">get_credential()</span> <span class="pre">-></span> <span class="pre">AwsCredential</span></code></p></td>
+<td><p><code class="docutils literal notranslate"><span class="pre">provide_credential(...)</span> <span class="pre">-></span> <span class="pre">Option<IcebergAwsCredential></span></code></p></td>
</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">CometDiskBlockWriter</span></code></p></td>
-<td><p><code class="docutils literal notranslate"><span class="pre">.../shuffle/CometDiskBlockWriter.java</span></code></p></td>
-<td><p>Buffers rows in memory pages for a single partition. Spills to disk via native encoding. Used by bypass writer.</p></td>
+<tr class="row-odd"><td><p>Returns expiry?</p></td>
+<td><p>No (only key/secret/token)</p></td>
+<td><p>Yes (<code class="docutils literal notranslate"><span class="pre">expires_in:</span> <span class="pre">Option<Timestamp></span></code>)</p></td>
</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">CometShuffleExternalSorter</span></code></p></td>
-<td><p><code class="docutils literal notranslate"><span class="pre">.../shuffle/sort/CometShuffleExternalSorter.java</span></code></p></td>
-<td><p>Buffers records across all partitions, sorts by partition ID, spills sorted data. Used by unsafe writer.</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">SpillWriter</span></code></p></td>
-<td><p><code class="docutils literal notranslate"><span class="pre">.../shuffle/SpillWriter.java</span></code></p></td>
-<td><p>Base class for spill logic. Manages memory pages and calls <code class="docutils literal notranslate"><span class="pre">Native.writeSortedFileNative()</span></code> for Arrow IPC encoding.</p></td>
-</tr>
-</tbody>
-</table>
-</div>
-</section>
-<section id="reader">
-<h3>Reader<a class="headerlink" href="#reader" title="Link to this heading">#</a></h3>
-<div class="pst-scrollable-table-container"><table class="table">
-<thead>
-<tr class="row-odd"><th class="head"><p>Class</p></th>
-<th class="head"><p>Location</p></th>
-<th class="head"><p>Description</p></th>
+<tr class="row-even"><td><p>Comet-side TTL wrapper?</p></td>
+<td><p>None. Bridge passed straight to <code class="docutils literal notranslate"><span class="pre">with_credentials</span></code></p></td>
+<td><p>None. <code class="docutils literal notranslate"><span class="pre">opendal</span></code> schedules the next refresh from <code class="docutils literal notranslate"><span class="pre">expires_in</span></code></p></td>
</tr>
-</thead>
-<tbody>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">CometBlockStoreShuffleReader</span></code></p></td>
-<td><p><code class="docutils literal notranslate"><span class="pre">.../shuffle/CometBlockStoreShuffleReader.scala</span></code></p></td>
-<td><p>Fetches shuffle blocks via <code class="docutils literal notranslate"><span class="pre">ShuffleBlockFetcherIterator</span></code>. Decodes Arrow IPC to <code class="docutils literal notranslate"><span class="pre">ColumnarBatch</span></code>.</p></td>
+<tr class="row-odd"><td><p>When SPI is called</p></td>
+<td><p>Every <code class="docutils literal notranslate"><span class="pre">get_credential()</span></code> call</p></td>
+<td><p>When <code class="docutils literal notranslate"><span class="pre">expires_in</span></code> is exceeded</p></td>
</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">NativeBatchDecoderIterator</span></code></p></td>
-<td><p><code class="docutils literal notranslate"><span class="pre">.../shuffle/NativeBatchDecoderIterator.scala</span></code></p></td>
-<td><p>Reads compressed Arrow IPC batches from input stream. Calls <code class="docutils literal notranslate"><span class="pre">Native.decodeShuffleBlock()</span></code> via JNI.</p></td>
+<tr class="row-even"><td><p>Vendor returns 0 expiry</p></td>
+<td><p>Field has no use</p></td>
+<td><p>Bridge substitutes 5 minutes to bound staleness</p></td>
</tr>
</tbody>
</table>
</div>
+<p>The 5-minute fallback is a safety net so a vendor that omits expiry cannot leave <code class="docutils literal notranslate"><span class="pre">opendal</span></code> caching a stale token indefinitely. It is intentionally not a configuration knob.</p>
</section>
+<section id="property-bag-handling-on-the-iceberg-path">
+<h2>Property-bag handling on the Iceberg path<a class="headerlink" href="#property-bag-handling-on-the-iceberg-path" title="Link to this heading">#</a></h2>
+<p>The full unfiltered FileIO property bag crosses JNI as <code class="docutils literal notranslate"><span class="pre">catalog_properties</span></code>. The storage-prefix filter (<code class="docutils literal notranslate"><span class="pre">s3.</span></code>/<code class="docutils literal notranslate"><span class="pre">gcs.</span></code>/<code class="docutils literal notranslate"><span class="pre">adls.</span></code>/<code class="docutils literal notranslate"><span class="pre">client.</sp [...]
+<p><code class="docutils literal notranslate"><span class="pre">IcebergScanExec</span></code> derives a redacting <code class="docutils literal notranslate"><span class="pre">Debug</span></code> so plan dumps and tracing do not leak the property bag.</p>
</section>
-<section id="data-flow">
-<h2>Data Flow<a class="headerlink" href="#data-flow" title="Link to this heading">#</a></h2>
-<section id="write-path">
-<h3>Write Path<a class="headerlink" href="#write-path" title="Link to this heading">#</a></h3>
-<ol class="arabic simple">
-<li><p><code class="docutils literal notranslate"><span class="pre">ShuffleWriteProcessor</span></code> calls <code class="docutils literal notranslate"><span class="pre">CometShuffleManager.getWriter()</span></code></p></li>
-<li><p>Writer receives <code class="docutils literal notranslate"><span class="pre">Iterator[Product2[K,</span> <span class="pre">V]]</span></code> where V is <code class="docutils literal notranslate"><span class="pre">UnsafeRow</span></code></p></li>
-<li><p>Rows are serialized and buffered in off-heap memory pages</p></li>
-<li><p>When memory threshold or batch size is reached, <code class="docutils literal notranslate"><span class="pre">SpillWriter.doSpilling()</span></code> is called</p></li>
-<li><p>Native code (<code class="docutils literal notranslate"><span class="pre">Native.writeSortedFileNative()</span></code>) converts rows to Arrow arrays and writes IPC format</p></li>
-<li><p>For bypass writer: partition files are concatenated into final output</p></li>
-<li><p>For sort writer: spill files are merged</p></li>
-</ol>
+<section id="returns-or-throws-not-a-fall-through-value">
+<h2>Returns or throws, not a fall-through value<a class="headerlink" href="#returns-or-throws-not-a-fall-through-value" title="Link to this heading">#</a></h2>
+<p>The SPI returns a <code class="docutils literal notranslate"><span class="pre">CometS3Credentials</span></code> or throws. There is no sentinel “I do not know” return. Vendors that are only authoritative for some paths resolve the default AWS chain themselves for the rest and return the result. This matches the contract on every other AWS credential SPI in the JVM ecosystem (AWS SDK v1/v2, Hadoop S3A, Iceberg <code class="docutils literal notranslate"><span class="pre">VendedCredentia [...]
</section>
-<section id="read-path">
-<h3>Read Path<a class="headerlink" href="#read-path" title="Link to this heading">#</a></h3>
-<ol class="arabic simple">
-<li><p><code class="docutils literal notranslate"><span class="pre">CometBlockStoreShuffleReader.read()</span></code> creates <code class="docutils literal notranslate"><span class="pre">ShuffleBlockFetcherIterator</span></code></p></li>
-<li><p>For each block, <code class="docutils literal notranslate"><span class="pre">NativeBatchDecoderIterator</span></code> reads the IPC stream</p></li>
-<li><p>Native code (<code class="docutils literal notranslate"><span class="pre">Native.decodeShuffleBlock()</span></code>) decompresses and decodes to Arrow arrays</p></li>
-<li><p>Arrow FFI imports arrays as <code class="docutils literal notranslate"><span class="pre">ColumnarBatch</span></code></p></li>
-</ol>
+<section id="lifecycle-autocloseable-plus-a-jvm-shutdown-hook">
+<h2>Lifecycle: <code class="docutils literal notranslate"><span class="pre">AutoCloseable</span></code> plus a JVM shutdown hook<a class="headerlink" href="#lifecycle-autocloseable-plus-a-jvm-shutdown-hook" title="Link to this heading">#</a></h2>
+<p><code class="docutils literal notranslate"><span class="pre">CometS3CredentialProvider</span></code> extends <code class="docutils literal notranslate"><span class="pre">AutoCloseable</span></code> with a default no-op <code class="docutils literal notranslate"><span class="pre">close()</span></code>. The dispatcher installs a JVM shutdown hook that iterates every cached instance and calls <code class="docutils literal notranslate"><span class="pre">close()</span></code>, swallowing p [...]
</section>
-</section>
-<section id="memory-management">
-<h2>Memory Management<a class="headerlink" href="#memory-management" title="Link to this heading">#</a></h2>
-<ul class="simple">
-<li><p><code class="docutils literal notranslate"><span class="pre">CometShuffleMemoryAllocator</span></code>: Custom allocator for off-heap memory pages</p></li>
-<li><p>Memory is allocated in pages; when allocation fails, writers spill to disk</p></li>
-<li><p><code class="docutils literal notranslate"><span class="pre">CometDiskBlockWriter</span></code> coordinates spilling across all partition writers (largest first)</p></li>
-<li><p>Async spilling is supported via <code class="docutils literal notranslate"><span class="pre">ShuffleThreadPool</span></code></p></li>
-</ul>
-</section>
-<section id="configuration">
-<h2>Configuration<a class="headerlink" href="#configuration" title="Link to this heading">#</a></h2>
-<div class="pst-scrollable-table-container"><table class="table">
-<thead>
-<tr class="row-odd"><th class="head"><p>Config</p></th>
-<th class="head"><p>Description</p></th>
-</tr>
-</thead>
-<tbody>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.columnar.shuffle.async.enabled</span></code></p></td>
-<td><p>Enable async spill writes</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.columnar.shuffle.async.thread.num</span></code></p></td>
-<td><p>Threads per writer for async</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.columnar.shuffle.batch.size</span></code></p></td>
-<td><p>Rows per Arrow batch</p></td>
-</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.columnar.shuffle.spill.threshold</span></code></p></td>
-<td><p>Row count threshold for spill</p></td>
-</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">spark.comet.exec.shuffle.compression.codec</span></code></p></td>
-<td><p>Compression codec (zstd, lz4, etc.)</p></td>
-</tr>
-</tbody>
-</table>
-</div>
+<section id="iceberg-path-error-message-fidelity-caveat">
+<h2>Iceberg path: error message fidelity caveat<a class="headerlink" href="#iceberg-path-error-message-fidelity-caveat" title="Link to this heading">#</a></h2>
+<p>When the bridge is wired into <code class="docutils literal notranslate"><span class="pre">iceberg-rust</span></code>, the outer <code class="docutils literal notranslate"><span class="pre">reqsign-core::ProvideCredentialChain</span></code> currently swallows thrown exceptions into “no credential” before the request reaches <code class="docutils literal notranslate"><span class="pre">opendal</span></code>. The credential is still not issued and the request still fails, but the message [...]
</section>
</section>
@@ -801,20 +643,20 @@ to protobuf and implements the partitioning logic in native code.</p>
<div class="prev-next-area">
<a class="left-prev"
- href="ffi.html"
+ href="sql_error_propagation.html"
title="previous page">
<i class="fa-solid fa-angle-left"></i>
<div class="prev-next-info">
<p class="prev-next-subtitle">previous</p>
- <p class="prev-next-title">Arrow FFI Usage in Comet</p>
+ <p class="prev-next-title">ANSI SQL Error Propagation in Comet</p>
</div>
</a>
<a class="right-next"
- href="native_shuffle.html"
+ href="adding_a_new_operator.html"
title="next page">
<div class="prev-next-info">
<p class="prev-next-subtitle">next</p>
- <p class="prev-next-title">Native Shuffle</p>
+ <p class="prev-next-title">Adding a New Operator</p>
</div>
<i class="fa-solid fa-angle-right"></i>
</a>
diff --git a/contributor-guide/spark-sql-tests.html b/contributor-guide/spark-sql-tests.html
index 057e2a1410..4a6652327b 100644
--- a/contributor-guide/spark-sql-tests.html
+++ b/contributor-guide/spark-sql-tests.html
@@ -403,6 +403,7 @@ under the License.
<li class="toctree-l1"><a class="reference internal" href="jvm_shuffle.html">JVM Shuffle</a></li>
<li class="toctree-l1"><a class="reference internal" href="native_shuffle.html">Native Shuffle</a></li>
<li class="toctree-l1"><a class="reference internal" href="sql_error_propagation.html">ANSI Error Propagation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="s3-credential-provider-design.html">S3 Credential Provider Design</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Adding Functionality</span></p>
<ul class="nav bd-sidenav">
diff --git a/contributor-guide/spark_configs_support.html b/contributor-guide/spark_configs_support.html
index 3ba02d5b8f..095d8b44a8 100644
--- a/contributor-guide/spark_configs_support.html
+++ b/contributor-guide/spark_configs_support.html
@@ -403,6 +403,7 @@ under the License.
<li class="toctree-l1"><a class="reference internal" href="jvm_shuffle.html">JVM Shuffle</a></li>
<li class="toctree-l1"><a class="reference internal" href="native_shuffle.html">Native Shuffle</a></li>
<li class="toctree-l1"><a class="reference internal" href="sql_error_propagation.html">ANSI Error Propagation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="s3-credential-provider-design.html">S3 Credential Provider Design</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Adding Functionality</span></p>
<ul class="nav bd-sidenav">
diff --git a/contributor-guide/spark_expressions_support.html b/contributor-guide/spark_expressions_support.html
index a67344afa0..ab61fb7e1a 100644
--- a/contributor-guide/spark_expressions_support.html
+++ b/contributor-guide/spark_expressions_support.html
@@ -403,6 +403,7 @@ under the License.
<li class="toctree-l1"><a class="reference internal" href="jvm_shuffle.html">JVM Shuffle</a></li>
<li class="toctree-l1"><a class="reference internal" href="native_shuffle.html">Native Shuffle</a></li>
<li class="toctree-l1"><a class="reference internal" href="sql_error_propagation.html">ANSI Error Propagation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="s3-credential-provider-design.html">S3 Credential Provider Design</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Adding Functionality</span></p>
<ul class="nav bd-sidenav">
diff --git a/contributor-guide/sql-file-tests.html b/contributor-guide/sql-file-tests.html
index 21c075c4ec..25f52eca29 100644
--- a/contributor-guide/sql-file-tests.html
+++ b/contributor-guide/sql-file-tests.html
@@ -403,6 +403,7 @@ under the License.
<li class="toctree-l1"><a class="reference internal" href="jvm_shuffle.html">JVM Shuffle</a></li>
<li class="toctree-l1"><a class="reference internal" href="native_shuffle.html">Native Shuffle</a></li>
<li class="toctree-l1"><a class="reference internal" href="sql_error_propagation.html">ANSI Error Propagation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="s3-credential-provider-design.html">S3 Credential Provider Design</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Adding Functionality</span></p>
<ul class="nav bd-sidenav">
diff --git a/contributor-guide/sql_error_propagation.html b/contributor-guide/sql_error_propagation.html
index 5b3313e930..24bdc90547 100644
--- a/contributor-guide/sql_error_propagation.html
+++ b/contributor-guide/sql_error_propagation.html
@@ -65,7 +65,7 @@ under the License.
<script async="true" defer="true" src="https://buttons.github.io/buttons.js"></script>
<link rel="index" title="Index" href="../genindex.html" />
<link rel="search" title="Search" href="../search.html" />
- <link rel="next" title="Adding a New Operator" href="adding_a_new_operator.html" />
+ <link rel="next" title="S3 Credential Provider SPI: Design Notes" href="s3-credential-provider-design.html" />
<link rel="prev" title="Native Shuffle" href="native_shuffle.html" />
<meta name="viewport" content="width=device-width, initial-scale=1"/>
<meta name="docsearch:language" content="en"/>
@@ -403,6 +403,7 @@ under the License.
<li class="toctree-l1"><a class="reference internal" href="jvm_shuffle.html">JVM Shuffle</a></li>
<li class="toctree-l1"><a class="reference internal" href="native_shuffle.html">Native Shuffle</a></li>
<li class="toctree-l1 current active"><a class="current reference internal" href="#">ANSI Error Propagation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="s3-credential-provider-design.html">S3 Credential Provider Design</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Adding Functionality</span></p>
<ul class="nav bd-sidenav">
@@ -1063,11 +1064,11 @@ maintain.</p>
</div>
</a>
<a class="right-next"
- href="adding_a_new_operator.html"
+ href="s3-credential-provider-design.html"
title="next page">
<div class="prev-next-info">
<p class="prev-next-subtitle">next</p>
- <p class="prev-next-title">Adding a New Operator</p>
+ <p class="prev-next-title">S3 Credential Provider SPI: Design Notes</p>
</div>
<i class="fa-solid fa-angle-right"></i>
</a>
diff --git a/contributor-guide/tracing.html b/contributor-guide/tracing.html
index 72f0f5a333..1eeafa8c7f 100644
--- a/contributor-guide/tracing.html
+++ b/contributor-guide/tracing.html
@@ -403,6 +403,7 @@ under the License.
<li class="toctree-l1"><a class="reference internal" href="jvm_shuffle.html">JVM Shuffle</a></li>
<li class="toctree-l1"><a class="reference internal" href="native_shuffle.html">Native Shuffle</a></li>
<li class="toctree-l1"><a class="reference internal" href="sql_error_propagation.html">ANSI Error Propagation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="s3-credential-provider-design.html">S3 Credential Provider Design</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Adding Functionality</span></p>
<ul class="nav bd-sidenav">
diff --git a/genindex.html b/genindex.html
index 6a16ec265d..82c4864ef9 100644
--- a/genindex.html
+++ b/genindex.html
@@ -447,6 +447,7 @@ under the License.
<li class="toctree-l3"><a class="reference internal" href="user-guide/latest/tuning.html">Tuning Guide</a></li>
<li class="toctree-l3"><a class="reference internal" href="user-guide/latest/metrics.html">Metrics Guide</a></li>
<li class="toctree-l3"><a class="reference internal" href="user-guide/latest/iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l3"><a class="reference internal" href="user-guide/latest/s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l3"><a class="reference internal" href="user-guide/latest/kubernetes.html">Kubernetes Guide</a></li>
<li class="toctree-l3"><a class="reference internal" href="user-guide/latest/source.html">Building From Source</a></li>
</ul>
@@ -510,6 +511,7 @@ under the License.
<li class="toctree-l2"><a class="reference internal" href="contributor-guide/jvm_shuffle.html">JVM Shuffle</a></li>
<li class="toctree-l2"><a class="reference internal" href="contributor-guide/native_shuffle.html">Native Shuffle</a></li>
<li class="toctree-l2"><a class="reference internal" href="contributor-guide/sql_error_propagation.html">ANSI Error Propagation</a></li>
+<li class="toctree-l2"><a class="reference internal" href="contributor-guide/s3-credential-provider-design.html">S3 Credential Provider Design</a></li>
<li class="toctree-l2"><a class="reference internal" href="contributor-guide/adding_a_new_operator.html">Adding a New Operator</a></li>
<li class="toctree-l2"><a class="reference internal" href="contributor-guide/adding_a_new_expression.html">Adding a New Expression</a></li>
<li class="toctree-l2"><a class="reference internal" href="contributor-guide/adding_a_new_spark_version.html">Adding a New Spark Version</a></li>
diff --git a/index.html b/index.html
index 6552b6ecfc..ba2f35137e 100644
--- a/index.html
+++ b/index.html
@@ -449,6 +449,7 @@ under the License.
<li class="toctree-l3"><a class="reference internal" href="user-guide/latest/tuning.html">Tuning Guide</a></li>
<li class="toctree-l3"><a class="reference internal" href="user-guide/latest/metrics.html">Metrics Guide</a></li>
<li class="toctree-l3"><a class="reference internal" href="user-guide/latest/iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l3"><a class="reference internal" href="user-guide/latest/s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l3"><a class="reference internal" href="user-guide/latest/kubernetes.html">Kubernetes Guide</a></li>
<li class="toctree-l3"><a class="reference internal" href="user-guide/latest/source.html">Building From Source</a></li>
</ul>
@@ -512,6 +513,7 @@ under the License.
<li class="toctree-l2"><a class="reference internal" href="contributor-guide/jvm_shuffle.html">JVM Shuffle</a></li>
<li class="toctree-l2"><a class="reference internal" href="contributor-guide/native_shuffle.html">Native Shuffle</a></li>
<li class="toctree-l2"><a class="reference internal" href="contributor-guide/sql_error_propagation.html">ANSI Error Propagation</a></li>
+<li class="toctree-l2"><a class="reference internal" href="contributor-guide/s3-credential-provider-design.html">S3 Credential Provider Design</a></li>
<li class="toctree-l2"><a class="reference internal" href="contributor-guide/adding_a_new_operator.html">Adding a New Operator</a></li>
<li class="toctree-l2"><a class="reference internal" href="contributor-guide/adding_a_new_expression.html">Adding a New Expression</a></li>
<li class="toctree-l2"><a class="reference internal" href="contributor-guide/adding_a_new_spark_version.html">Adding a New Spark Version</a></li>
diff --git a/objects.inv b/objects.inv
index e54634c90b..789484e785 100644
Binary files a/objects.inv and b/objects.inv differ
diff --git a/search.html b/search.html
index f1f83de36a..c07aa5b83a 100644
--- a/search.html
+++ b/search.html
@@ -449,6 +449,7 @@ under the License.
<li class="toctree-l3"><a class="reference internal" href="user-guide/latest/tuning.html">Tuning Guide</a></li>
<li class="toctree-l3"><a class="reference internal" href="user-guide/latest/metrics.html">Metrics Guide</a></li>
<li class="toctree-l3"><a class="reference internal" href="user-guide/latest/iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l3"><a class="reference internal" href="user-guide/latest/s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l3"><a class="reference internal" href="user-guide/latest/kubernetes.html">Kubernetes Guide</a></li>
<li class="toctree-l3"><a class="reference internal" href="user-guide/latest/source.html">Building From Source</a></li>
</ul>
@@ -512,6 +513,7 @@ under the License.
<li class="toctree-l2"><a class="reference internal" href="contributor-guide/jvm_shuffle.html">JVM Shuffle</a></li>
<li class="toctree-l2"><a class="reference internal" href="contributor-guide/native_shuffle.html">Native Shuffle</a></li>
<li class="toctree-l2"><a class="reference internal" href="contributor-guide/sql_error_propagation.html">ANSI Error Propagation</a></li>
+<li class="toctree-l2"><a class="reference internal" href="contributor-guide/s3-credential-provider-design.html">S3 Credential Provider Design</a></li>
<li class="toctree-l2"><a class="reference internal" href="contributor-guide/adding_a_new_operator.html">Adding a New Operator</a></li>
<li class="toctree-l2"><a class="reference internal" href="contributor-guide/adding_a_new_expression.html">Adding a New Expression</a></li>
<li class="toctree-l2"><a class="reference internal" href="contributor-guide/adding_a_new_spark_version.html">Adding a New Spark Version</a></li>
diff --git a/searchindex.js b/searchindex.js
index cd0377ea7e..a22a9f560c 100644
--- a/searchindex.js
+++ b/searchindex.js
@@ -1 +1 @@
-Search.setIndex({"alltitles": {"1. Format Your Code": [[34, "format-your-code"]], "1. Install Comet": [[36, "install-comet"], [44, "install-comet"]], "1. Native Operators (nativeExecs map)": [[23, "native-operators-nativeexecs-map"]], "2. Build and Verify": [[34, "build-and-verify"]], "2. Clone Iceberg and Apply Diff": [[36, "clone-iceberg-and-apply-diff"]], "2. Clone Spark and Apply Diff": [[44, "clone-spark-and-apply-diff"]], "2. Sink Operators (sinks map)": [[23, "sink-operators-sinks [...]
\ No newline at end of file
+Search.setIndex({"alltitles": {"1. Format Your Code": [[34, "format-your-code"]], "1. Install Comet": [[36, "install-comet"], [45, "install-comet"]], "1. Native Operators (nativeExecs map)": [[23, "native-operators-nativeexecs-map"]], "2. Build and Verify": [[34, "build-and-verify"]], "2. Clone Iceberg and Apply Diff": [[36, "clone-iceberg-and-apply-diff"]], "2. Clone Spark and Apply Diff": [[45, "clone-spark-and-apply-diff"]], "2. Sink Operators (sinks map)": [[23, "sink-operators-sinks [...]
\ No newline at end of file
diff --git a/user-guide/0.13/compatibility.html b/user-guide/0.13/compatibility.html
index a04182971c..40ccd74d2f 100644
--- a/user-guide/0.13/compatibility.html
+++ b/user-guide/0.13/compatibility.html
@@ -458,6 +458,7 @@ under the License.
<li class="toctree-l2"><a class="reference internal" href="../latest/tuning.html">Tuning Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/metrics.html">Metrics Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../latest/s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/kubernetes.html">Kubernetes Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/source.html">Building From Source</a></li>
</ul>
diff --git a/user-guide/0.13/configs.html b/user-guide/0.13/configs.html
index fe8ad66c1f..4b92b7a7f8 100644
--- a/user-guide/0.13/configs.html
+++ b/user-guide/0.13/configs.html
@@ -458,6 +458,7 @@ under the License.
<li class="toctree-l2"><a class="reference internal" href="../latest/tuning.html">Tuning Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/metrics.html">Metrics Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../latest/s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/kubernetes.html">Kubernetes Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/source.html">Building From Source</a></li>
</ul>
diff --git a/user-guide/0.13/datasources.html b/user-guide/0.13/datasources.html
index 558d21b4cb..bd8fdf1f1e 100644
--- a/user-guide/0.13/datasources.html
+++ b/user-guide/0.13/datasources.html
@@ -458,6 +458,7 @@ under the License.
<li class="toctree-l2"><a class="reference internal" href="../latest/tuning.html">Tuning Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/metrics.html">Metrics Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../latest/s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/kubernetes.html">Kubernetes Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/source.html">Building From Source</a></li>
</ul>
diff --git a/user-guide/0.13/datatypes.html b/user-guide/0.13/datatypes.html
index 3962c5384c..63cb8fbe15 100644
--- a/user-guide/0.13/datatypes.html
+++ b/user-guide/0.13/datatypes.html
@@ -458,6 +458,7 @@ under the License.
<li class="toctree-l2"><a class="reference internal" href="../latest/tuning.html">Tuning Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/metrics.html">Metrics Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../latest/s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/kubernetes.html">Kubernetes Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/source.html">Building From Source</a></li>
</ul>
diff --git a/user-guide/0.13/expressions.html b/user-guide/0.13/expressions.html
index 72cd3cbaf9..8c83a92c34 100644
--- a/user-guide/0.13/expressions.html
+++ b/user-guide/0.13/expressions.html
@@ -458,6 +458,7 @@ under the License.
<li class="toctree-l2"><a class="reference internal" href="../latest/tuning.html">Tuning Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/metrics.html">Metrics Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../latest/s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/kubernetes.html">Kubernetes Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/source.html">Building From Source</a></li>
</ul>
diff --git a/user-guide/0.13/iceberg.html b/user-guide/0.13/iceberg.html
index 34c3ca7834..4887ccfa20 100644
--- a/user-guide/0.13/iceberg.html
+++ b/user-guide/0.13/iceberg.html
@@ -458,6 +458,7 @@ under the License.
<li class="toctree-l2"><a class="reference internal" href="../latest/tuning.html">Tuning Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/metrics.html">Metrics Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../latest/s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/kubernetes.html">Kubernetes Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/source.html">Building From Source</a></li>
</ul>
diff --git a/user-guide/0.13/index.html b/user-guide/0.13/index.html
index d2e28c4ee9..269239f179 100644
--- a/user-guide/0.13/index.html
+++ b/user-guide/0.13/index.html
@@ -458,6 +458,7 @@ under the License.
<li class="toctree-l2"><a class="reference internal" href="../latest/tuning.html">Tuning Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/metrics.html">Metrics Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../latest/s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/kubernetes.html">Kubernetes Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/source.html">Building From Source</a></li>
</ul>
diff --git a/user-guide/0.13/installation.html b/user-guide/0.13/installation.html
index 59410a8f71..b152c7fb9d 100644
--- a/user-guide/0.13/installation.html
+++ b/user-guide/0.13/installation.html
@@ -458,6 +458,7 @@ under the License.
<li class="toctree-l2"><a class="reference internal" href="../latest/tuning.html">Tuning Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/metrics.html">Metrics Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../latest/s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/kubernetes.html">Kubernetes Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/source.html">Building From Source</a></li>
</ul>
diff --git a/user-guide/0.13/kubernetes.html b/user-guide/0.13/kubernetes.html
index 76d51551ed..51b3a07cd7 100644
--- a/user-guide/0.13/kubernetes.html
+++ b/user-guide/0.13/kubernetes.html
@@ -458,6 +458,7 @@ under the License.
<li class="toctree-l2"><a class="reference internal" href="../latest/tuning.html">Tuning Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/metrics.html">Metrics Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../latest/s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/kubernetes.html">Kubernetes Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/source.html">Building From Source</a></li>
</ul>
diff --git a/user-guide/0.13/metrics.html b/user-guide/0.13/metrics.html
index 20ae25c008..91a8af51c4 100644
--- a/user-guide/0.13/metrics.html
+++ b/user-guide/0.13/metrics.html
@@ -458,6 +458,7 @@ under the License.
<li class="toctree-l2"><a class="reference internal" href="../latest/tuning.html">Tuning Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/metrics.html">Metrics Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../latest/s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/kubernetes.html">Kubernetes Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/source.html">Building From Source</a></li>
</ul>
diff --git a/user-guide/0.13/operators.html b/user-guide/0.13/operators.html
index e46124cefb..e5af108d2c 100644
--- a/user-guide/0.13/operators.html
+++ b/user-guide/0.13/operators.html
@@ -458,6 +458,7 @@ under the License.
<li class="toctree-l2"><a class="reference internal" href="../latest/tuning.html">Tuning Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/metrics.html">Metrics Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../latest/s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/kubernetes.html">Kubernetes Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/source.html">Building From Source</a></li>
</ul>
diff --git a/user-guide/0.13/source.html b/user-guide/0.13/source.html
index 62221085a2..999210ac17 100644
--- a/user-guide/0.13/source.html
+++ b/user-guide/0.13/source.html
@@ -458,6 +458,7 @@ under the License.
<li class="toctree-l2"><a class="reference internal" href="../latest/tuning.html">Tuning Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/metrics.html">Metrics Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../latest/s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/kubernetes.html">Kubernetes Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/source.html">Building From Source</a></li>
</ul>
diff --git a/user-guide/0.13/tuning.html b/user-guide/0.13/tuning.html
index b53dd56f1b..5a33fc1dc8 100644
--- a/user-guide/0.13/tuning.html
+++ b/user-guide/0.13/tuning.html
@@ -458,6 +458,7 @@ under the License.
<li class="toctree-l2"><a class="reference internal" href="../latest/tuning.html">Tuning Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/metrics.html">Metrics Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../latest/s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/kubernetes.html">Kubernetes Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/source.html">Building From Source</a></li>
</ul>
diff --git a/user-guide/0.14/compatibility.html b/user-guide/0.14/compatibility.html
index cf6c416c4b..1456ad9662 100644
--- a/user-guide/0.14/compatibility.html
+++ b/user-guide/0.14/compatibility.html
@@ -458,6 +458,7 @@ under the License.
<li class="toctree-l2"><a class="reference internal" href="../latest/tuning.html">Tuning Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/metrics.html">Metrics Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../latest/s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/kubernetes.html">Kubernetes Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/source.html">Building From Source</a></li>
</ul>
diff --git a/user-guide/0.14/configs.html b/user-guide/0.14/configs.html
index 3df86a1c5e..feca5f0579 100644
--- a/user-guide/0.14/configs.html
+++ b/user-guide/0.14/configs.html
@@ -458,6 +458,7 @@ under the License.
<li class="toctree-l2"><a class="reference internal" href="../latest/tuning.html">Tuning Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/metrics.html">Metrics Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../latest/s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/kubernetes.html">Kubernetes Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/source.html">Building From Source</a></li>
</ul>
diff --git a/user-guide/0.14/datasources.html b/user-guide/0.14/datasources.html
index e89fc988bb..62e9e0cfe5 100644
--- a/user-guide/0.14/datasources.html
+++ b/user-guide/0.14/datasources.html
@@ -458,6 +458,7 @@ under the License.
<li class="toctree-l2"><a class="reference internal" href="../latest/tuning.html">Tuning Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/metrics.html">Metrics Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../latest/s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/kubernetes.html">Kubernetes Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/source.html">Building From Source</a></li>
</ul>
diff --git a/user-guide/0.14/datatypes.html b/user-guide/0.14/datatypes.html
index 171c98d30e..855db751f2 100644
--- a/user-guide/0.14/datatypes.html
+++ b/user-guide/0.14/datatypes.html
@@ -458,6 +458,7 @@ under the License.
<li class="toctree-l2"><a class="reference internal" href="../latest/tuning.html">Tuning Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/metrics.html">Metrics Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../latest/s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/kubernetes.html">Kubernetes Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/source.html">Building From Source</a></li>
</ul>
diff --git a/user-guide/0.14/expressions.html b/user-guide/0.14/expressions.html
index 537e977c57..40797c7e0b 100644
--- a/user-guide/0.14/expressions.html
+++ b/user-guide/0.14/expressions.html
@@ -458,6 +458,7 @@ under the License.
<li class="toctree-l2"><a class="reference internal" href="../latest/tuning.html">Tuning Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/metrics.html">Metrics Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../latest/s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/kubernetes.html">Kubernetes Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/source.html">Building From Source</a></li>
</ul>
diff --git a/user-guide/0.14/iceberg.html b/user-guide/0.14/iceberg.html
index 9c6000ecb6..7d9d74e4a8 100644
--- a/user-guide/0.14/iceberg.html
+++ b/user-guide/0.14/iceberg.html
@@ -458,6 +458,7 @@ under the License.
<li class="toctree-l2"><a class="reference internal" href="../latest/tuning.html">Tuning Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/metrics.html">Metrics Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../latest/s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/kubernetes.html">Kubernetes Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/source.html">Building From Source</a></li>
</ul>
diff --git a/user-guide/0.14/index.html b/user-guide/0.14/index.html
index b1d70860f2..ff721c3765 100644
--- a/user-guide/0.14/index.html
+++ b/user-guide/0.14/index.html
@@ -458,6 +458,7 @@ under the License.
<li class="toctree-l2"><a class="reference internal" href="../latest/tuning.html">Tuning Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/metrics.html">Metrics Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../latest/s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/kubernetes.html">Kubernetes Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/source.html">Building From Source</a></li>
</ul>
diff --git a/user-guide/0.14/installation.html b/user-guide/0.14/installation.html
index 30a68534f0..8996612e1c 100644
--- a/user-guide/0.14/installation.html
+++ b/user-guide/0.14/installation.html
@@ -458,6 +458,7 @@ under the License.
<li class="toctree-l2"><a class="reference internal" href="../latest/tuning.html">Tuning Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/metrics.html">Metrics Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../latest/s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/kubernetes.html">Kubernetes Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/source.html">Building From Source</a></li>
</ul>
diff --git a/user-guide/0.14/kubernetes.html b/user-guide/0.14/kubernetes.html
index 440c1d66ef..ce1e815fd5 100644
--- a/user-guide/0.14/kubernetes.html
+++ b/user-guide/0.14/kubernetes.html
@@ -458,6 +458,7 @@ under the License.
<li class="toctree-l2"><a class="reference internal" href="../latest/tuning.html">Tuning Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/metrics.html">Metrics Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../latest/s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/kubernetes.html">Kubernetes Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/source.html">Building From Source</a></li>
</ul>
diff --git a/user-guide/0.14/metrics.html b/user-guide/0.14/metrics.html
index 8d7a2d29ec..34e2877177 100644
--- a/user-guide/0.14/metrics.html
+++ b/user-guide/0.14/metrics.html
@@ -458,6 +458,7 @@ under the License.
<li class="toctree-l2"><a class="reference internal" href="../latest/tuning.html">Tuning Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/metrics.html">Metrics Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../latest/s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/kubernetes.html">Kubernetes Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/source.html">Building From Source</a></li>
</ul>
diff --git a/user-guide/0.14/operators.html b/user-guide/0.14/operators.html
index 966999954e..ed9f1af885 100644
--- a/user-guide/0.14/operators.html
+++ b/user-guide/0.14/operators.html
@@ -458,6 +458,7 @@ under the License.
<li class="toctree-l2"><a class="reference internal" href="../latest/tuning.html">Tuning Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/metrics.html">Metrics Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../latest/s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/kubernetes.html">Kubernetes Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/source.html">Building From Source</a></li>
</ul>
diff --git a/user-guide/0.14/source.html b/user-guide/0.14/source.html
index 02ac86b0dd..c17e5de7d5 100644
--- a/user-guide/0.14/source.html
+++ b/user-guide/0.14/source.html
@@ -458,6 +458,7 @@ under the License.
<li class="toctree-l2"><a class="reference internal" href="../latest/tuning.html">Tuning Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/metrics.html">Metrics Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../latest/s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/kubernetes.html">Kubernetes Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/source.html">Building From Source</a></li>
</ul>
diff --git a/user-guide/0.14/tuning.html b/user-guide/0.14/tuning.html
index b11ae5a960..0dfadb5329 100644
--- a/user-guide/0.14/tuning.html
+++ b/user-guide/0.14/tuning.html
@@ -458,6 +458,7 @@ under the License.
<li class="toctree-l2"><a class="reference internal" href="../latest/tuning.html">Tuning Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/metrics.html">Metrics Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../latest/s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/kubernetes.html">Kubernetes Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/source.html">Building From Source</a></li>
</ul>
diff --git a/user-guide/0.15/compatibility.html b/user-guide/0.15/compatibility.html
index d10af7b5b8..80f0848233 100644
--- a/user-guide/0.15/compatibility.html
+++ b/user-guide/0.15/compatibility.html
@@ -458,6 +458,7 @@ under the License.
<li class="toctree-l2"><a class="reference internal" href="../latest/tuning.html">Tuning Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/metrics.html">Metrics Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../latest/s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/kubernetes.html">Kubernetes Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/source.html">Building From Source</a></li>
</ul>
diff --git a/user-guide/0.15/configs.html b/user-guide/0.15/configs.html
index 78cd70cb1b..d5afb61113 100644
--- a/user-guide/0.15/configs.html
+++ b/user-guide/0.15/configs.html
@@ -458,6 +458,7 @@ under the License.
<li class="toctree-l2"><a class="reference internal" href="../latest/tuning.html">Tuning Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/metrics.html">Metrics Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../latest/s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/kubernetes.html">Kubernetes Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/source.html">Building From Source</a></li>
</ul>
diff --git a/user-guide/0.15/datasources.html b/user-guide/0.15/datasources.html
index 48c25dcbb8..ffa2bc9758 100644
--- a/user-guide/0.15/datasources.html
+++ b/user-guide/0.15/datasources.html
@@ -458,6 +458,7 @@ under the License.
<li class="toctree-l2"><a class="reference internal" href="../latest/tuning.html">Tuning Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/metrics.html">Metrics Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../latest/s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/kubernetes.html">Kubernetes Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/source.html">Building From Source</a></li>
</ul>
diff --git a/user-guide/0.15/datatypes.html b/user-guide/0.15/datatypes.html
index 95ee16a66a..da2fc5f1fc 100644
--- a/user-guide/0.15/datatypes.html
+++ b/user-guide/0.15/datatypes.html
@@ -458,6 +458,7 @@ under the License.
<li class="toctree-l2"><a class="reference internal" href="../latest/tuning.html">Tuning Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/metrics.html">Metrics Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../latest/s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/kubernetes.html">Kubernetes Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/source.html">Building From Source</a></li>
</ul>
diff --git a/user-guide/0.15/expressions.html b/user-guide/0.15/expressions.html
index 7e2fef0cad..162da7abe3 100644
--- a/user-guide/0.15/expressions.html
+++ b/user-guide/0.15/expressions.html
@@ -458,6 +458,7 @@ under the License.
<li class="toctree-l2"><a class="reference internal" href="../latest/tuning.html">Tuning Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/metrics.html">Metrics Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../latest/s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/kubernetes.html">Kubernetes Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/source.html">Building From Source</a></li>
</ul>
diff --git a/user-guide/0.15/iceberg.html b/user-guide/0.15/iceberg.html
index c71e807681..0219f5b8e4 100644
--- a/user-guide/0.15/iceberg.html
+++ b/user-guide/0.15/iceberg.html
@@ -458,6 +458,7 @@ under the License.
<li class="toctree-l2"><a class="reference internal" href="../latest/tuning.html">Tuning Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/metrics.html">Metrics Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../latest/s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/kubernetes.html">Kubernetes Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/source.html">Building From Source</a></li>
</ul>
diff --git a/user-guide/0.15/index.html b/user-guide/0.15/index.html
index 3ebaf9d005..f8beb5d478 100644
--- a/user-guide/0.15/index.html
+++ b/user-guide/0.15/index.html
@@ -458,6 +458,7 @@ under the License.
<li class="toctree-l2"><a class="reference internal" href="../latest/tuning.html">Tuning Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/metrics.html">Metrics Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../latest/s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/kubernetes.html">Kubernetes Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/source.html">Building From Source</a></li>
</ul>
diff --git a/user-guide/0.15/installation.html b/user-guide/0.15/installation.html
index 22c73a9c4c..2e80a7ad05 100644
--- a/user-guide/0.15/installation.html
+++ b/user-guide/0.15/installation.html
@@ -458,6 +458,7 @@ under the License.
<li class="toctree-l2"><a class="reference internal" href="../latest/tuning.html">Tuning Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/metrics.html">Metrics Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../latest/s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/kubernetes.html">Kubernetes Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/source.html">Building From Source</a></li>
</ul>
diff --git a/user-guide/0.15/kubernetes.html b/user-guide/0.15/kubernetes.html
index fc0f8fd480..e0ed2501b8 100644
--- a/user-guide/0.15/kubernetes.html
+++ b/user-guide/0.15/kubernetes.html
@@ -458,6 +458,7 @@ under the License.
<li class="toctree-l2"><a class="reference internal" href="../latest/tuning.html">Tuning Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/metrics.html">Metrics Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../latest/s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/kubernetes.html">Kubernetes Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/source.html">Building From Source</a></li>
</ul>
diff --git a/user-guide/0.15/metrics.html b/user-guide/0.15/metrics.html
index 5a0e9d5e09..b4e5652495 100644
--- a/user-guide/0.15/metrics.html
+++ b/user-guide/0.15/metrics.html
@@ -458,6 +458,7 @@ under the License.
<li class="toctree-l2"><a class="reference internal" href="../latest/tuning.html">Tuning Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/metrics.html">Metrics Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../latest/s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/kubernetes.html">Kubernetes Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/source.html">Building From Source</a></li>
</ul>
diff --git a/user-guide/0.15/operators.html b/user-guide/0.15/operators.html
index a5e1d6745b..5b30f1a751 100644
--- a/user-guide/0.15/operators.html
+++ b/user-guide/0.15/operators.html
@@ -458,6 +458,7 @@ under the License.
<li class="toctree-l2"><a class="reference internal" href="../latest/tuning.html">Tuning Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/metrics.html">Metrics Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../latest/s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/kubernetes.html">Kubernetes Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/source.html">Building From Source</a></li>
</ul>
diff --git a/user-guide/0.15/source.html b/user-guide/0.15/source.html
index 045915b690..8b4ef8a8c5 100644
--- a/user-guide/0.15/source.html
+++ b/user-guide/0.15/source.html
@@ -458,6 +458,7 @@ under the License.
<li class="toctree-l2"><a class="reference internal" href="../latest/tuning.html">Tuning Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/metrics.html">Metrics Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../latest/s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/kubernetes.html">Kubernetes Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/source.html">Building From Source</a></li>
</ul>
diff --git a/user-guide/0.15/tuning.html b/user-guide/0.15/tuning.html
index f5ec2201a6..063b91da67 100644
--- a/user-guide/0.15/tuning.html
+++ b/user-guide/0.15/tuning.html
@@ -458,6 +458,7 @@ under the License.
<li class="toctree-l2"><a class="reference internal" href="../latest/tuning.html">Tuning Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/metrics.html">Metrics Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../latest/s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/kubernetes.html">Kubernetes Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/source.html">Building From Source</a></li>
</ul>
diff --git a/user-guide/0.16/compatibility/expressions/aggregate.html b/user-guide/0.16/compatibility/expressions/aggregate.html
index 951627f5b1..0058a7aeac 100644
--- a/user-guide/0.16/compatibility/expressions/aggregate.html
+++ b/user-guide/0.16/compatibility/expressions/aggregate.html
@@ -458,6 +458,7 @@ under the License.
<li class="toctree-l2"><a class="reference internal" href="../../../latest/tuning.html">Tuning Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../../latest/metrics.html">Metrics Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../../latest/iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../latest/s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../../latest/kubernetes.html">Kubernetes Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../../latest/source.html">Building From Source</a></li>
</ul>
diff --git a/user-guide/0.16/compatibility/expressions/array.html b/user-guide/0.16/compatibility/expressions/array.html
index 3054cc6067..5754d2f554 100644
--- a/user-guide/0.16/compatibility/expressions/array.html
+++ b/user-guide/0.16/compatibility/expressions/array.html
@@ -458,6 +458,7 @@ under the License.
<li class="toctree-l2"><a class="reference internal" href="../../../latest/tuning.html">Tuning Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../../latest/metrics.html">Metrics Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../../latest/iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../latest/s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../../latest/kubernetes.html">Kubernetes Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../../latest/source.html">Building From Source</a></li>
</ul>
diff --git a/user-guide/0.16/compatibility/expressions/cast.html b/user-guide/0.16/compatibility/expressions/cast.html
index 0922a5da3e..d48576087e 100644
--- a/user-guide/0.16/compatibility/expressions/cast.html
+++ b/user-guide/0.16/compatibility/expressions/cast.html
@@ -458,6 +458,7 @@ under the License.
<li class="toctree-l2"><a class="reference internal" href="../../../latest/tuning.html">Tuning Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../../latest/metrics.html">Metrics Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../../latest/iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../latest/s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../../latest/kubernetes.html">Kubernetes Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../../latest/source.html">Building From Source</a></li>
</ul>
diff --git a/user-guide/0.16/compatibility/expressions/datetime.html b/user-guide/0.16/compatibility/expressions/datetime.html
index 90490b382a..c6e773e4da 100644
--- a/user-guide/0.16/compatibility/expressions/datetime.html
+++ b/user-guide/0.16/compatibility/expressions/datetime.html
@@ -458,6 +458,7 @@ under the License.
<li class="toctree-l2"><a class="reference internal" href="../../../latest/tuning.html">Tuning Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../../latest/metrics.html">Metrics Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../../latest/iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../latest/s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../../latest/kubernetes.html">Kubernetes Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../../latest/source.html">Building From Source</a></li>
</ul>
diff --git a/user-guide/0.16/compatibility/expressions/index.html b/user-guide/0.16/compatibility/expressions/index.html
index 89618eec04..264984d641 100644
--- a/user-guide/0.16/compatibility/expressions/index.html
+++ b/user-guide/0.16/compatibility/expressions/index.html
@@ -458,6 +458,7 @@ under the License.
<li class="toctree-l2"><a class="reference internal" href="../../../latest/tuning.html">Tuning Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../../latest/metrics.html">Metrics Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../../latest/iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../latest/s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../../latest/kubernetes.html">Kubernetes Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../../latest/source.html">Building From Source</a></li>
</ul>
diff --git a/user-guide/0.16/compatibility/expressions/map.html b/user-guide/0.16/compatibility/expressions/map.html
index c5201ca0b5..622bac47ce 100644
--- a/user-guide/0.16/compatibility/expressions/map.html
+++ b/user-guide/0.16/compatibility/expressions/map.html
@@ -458,6 +458,7 @@ under the License.
<li class="toctree-l2"><a class="reference internal" href="../../../latest/tuning.html">Tuning Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../../latest/metrics.html">Metrics Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../../latest/iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../latest/s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../../latest/kubernetes.html">Kubernetes Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../../latest/source.html">Building From Source</a></li>
</ul>
diff --git a/user-guide/0.16/compatibility/expressions/math.html b/user-guide/0.16/compatibility/expressions/math.html
index 00c09e58d4..ca3eb1bb53 100644
--- a/user-guide/0.16/compatibility/expressions/math.html
+++ b/user-guide/0.16/compatibility/expressions/math.html
@@ -458,6 +458,7 @@ under the License.
<li class="toctree-l2"><a class="reference internal" href="../../../latest/tuning.html">Tuning Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../../latest/metrics.html">Metrics Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../../latest/iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../latest/s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../../latest/kubernetes.html">Kubernetes Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../../latest/source.html">Building From Source</a></li>
</ul>
diff --git a/user-guide/0.16/compatibility/expressions/misc.html b/user-guide/0.16/compatibility/expressions/misc.html
index c833c3d1c4..31001a513b 100644
--- a/user-guide/0.16/compatibility/expressions/misc.html
+++ b/user-guide/0.16/compatibility/expressions/misc.html
@@ -458,6 +458,7 @@ under the License.
<li class="toctree-l2"><a class="reference internal" href="../../../latest/tuning.html">Tuning Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../../latest/metrics.html">Metrics Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../../latest/iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../latest/s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../../latest/kubernetes.html">Kubernetes Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../../latest/source.html">Building From Source</a></li>
</ul>
diff --git a/user-guide/0.16/compatibility/expressions/string.html b/user-guide/0.16/compatibility/expressions/string.html
index 6fa64889f2..7335c8b9b5 100644
--- a/user-guide/0.16/compatibility/expressions/string.html
+++ b/user-guide/0.16/compatibility/expressions/string.html
@@ -458,6 +458,7 @@ under the License.
<li class="toctree-l2"><a class="reference internal" href="../../../latest/tuning.html">Tuning Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../../latest/metrics.html">Metrics Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../../latest/iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../latest/s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../../latest/kubernetes.html">Kubernetes Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../../latest/source.html">Building From Source</a></li>
</ul>
diff --git a/user-guide/0.16/compatibility/expressions/struct.html b/user-guide/0.16/compatibility/expressions/struct.html
index 5e1dd2f52c..7a26848080 100644
--- a/user-guide/0.16/compatibility/expressions/struct.html
+++ b/user-guide/0.16/compatibility/expressions/struct.html
@@ -458,6 +458,7 @@ under the License.
<li class="toctree-l2"><a class="reference internal" href="../../../latest/tuning.html">Tuning Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../../latest/metrics.html">Metrics Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../../latest/iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../latest/s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../../latest/kubernetes.html">Kubernetes Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../../latest/source.html">Building From Source</a></li>
</ul>
diff --git a/user-guide/0.16/compatibility/floating-point.html b/user-guide/0.16/compatibility/floating-point.html
index 602c396400..fd6037007d 100644
--- a/user-guide/0.16/compatibility/floating-point.html
+++ b/user-guide/0.16/compatibility/floating-point.html
@@ -458,6 +458,7 @@ under the License.
<li class="toctree-l2"><a class="reference internal" href="../../latest/tuning.html">Tuning Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../latest/metrics.html">Metrics Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../latest/iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../latest/s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../latest/kubernetes.html">Kubernetes Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../latest/source.html">Building From Source</a></li>
</ul>
diff --git a/user-guide/0.16/compatibility/index.html b/user-guide/0.16/compatibility/index.html
index 8d1dd6ad48..24974ea262 100644
--- a/user-guide/0.16/compatibility/index.html
+++ b/user-guide/0.16/compatibility/index.html
@@ -458,6 +458,7 @@ under the License.
<li class="toctree-l2"><a class="reference internal" href="../../latest/tuning.html">Tuning Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../latest/metrics.html">Metrics Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../latest/iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../latest/s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../latest/kubernetes.html">Kubernetes Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../latest/source.html">Building From Source</a></li>
</ul>
diff --git a/user-guide/0.16/compatibility/operators.html b/user-guide/0.16/compatibility/operators.html
index 43cbf9dd0e..295d15eff7 100644
--- a/user-guide/0.16/compatibility/operators.html
+++ b/user-guide/0.16/compatibility/operators.html
@@ -458,6 +458,7 @@ under the License.
<li class="toctree-l2"><a class="reference internal" href="../../latest/tuning.html">Tuning Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../latest/metrics.html">Metrics Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../latest/iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../latest/s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../latest/kubernetes.html">Kubernetes Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../latest/source.html">Building From Source</a></li>
</ul>
diff --git a/user-guide/0.16/compatibility/regex.html b/user-guide/0.16/compatibility/regex.html
index 459b1a2cd5..d34bf407a8 100644
--- a/user-guide/0.16/compatibility/regex.html
+++ b/user-guide/0.16/compatibility/regex.html
@@ -458,6 +458,7 @@ under the License.
<li class="toctree-l2"><a class="reference internal" href="../../latest/tuning.html">Tuning Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../latest/metrics.html">Metrics Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../latest/iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../latest/s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../latest/kubernetes.html">Kubernetes Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../latest/source.html">Building From Source</a></li>
</ul>
diff --git a/user-guide/0.16/compatibility/scans.html b/user-guide/0.16/compatibility/scans.html
index e2f66e6016..18d57dd98d 100644
--- a/user-guide/0.16/compatibility/scans.html
+++ b/user-guide/0.16/compatibility/scans.html
@@ -458,6 +458,7 @@ under the License.
<li class="toctree-l2"><a class="reference internal" href="../../latest/tuning.html">Tuning Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../latest/metrics.html">Metrics Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../latest/iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../latest/s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../latest/kubernetes.html">Kubernetes Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../latest/source.html">Building From Source</a></li>
</ul>
diff --git a/user-guide/0.16/compatibility/spark-versions.html b/user-guide/0.16/compatibility/spark-versions.html
index 82b7e66a2c..a16bb94e06 100644
--- a/user-guide/0.16/compatibility/spark-versions.html
+++ b/user-guide/0.16/compatibility/spark-versions.html
@@ -458,6 +458,7 @@ under the License.
<li class="toctree-l2"><a class="reference internal" href="../../latest/tuning.html">Tuning Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../latest/metrics.html">Metrics Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../latest/iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../latest/s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../latest/kubernetes.html">Kubernetes Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../latest/source.html">Building From Source</a></li>
</ul>
diff --git a/user-guide/0.16/configs.html b/user-guide/0.16/configs.html
index bde833ad35..ade8694e3c 100644
--- a/user-guide/0.16/configs.html
+++ b/user-guide/0.16/configs.html
@@ -458,6 +458,7 @@ under the License.
<li class="toctree-l2"><a class="reference internal" href="../latest/tuning.html">Tuning Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/metrics.html">Metrics Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../latest/s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/kubernetes.html">Kubernetes Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/source.html">Building From Source</a></li>
</ul>
diff --git a/user-guide/0.16/datasources.html b/user-guide/0.16/datasources.html
index 4f384caa5a..02c727cb5d 100644
--- a/user-guide/0.16/datasources.html
+++ b/user-guide/0.16/datasources.html
@@ -458,6 +458,7 @@ under the License.
<li class="toctree-l2"><a class="reference internal" href="../latest/tuning.html">Tuning Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/metrics.html">Metrics Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../latest/s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/kubernetes.html">Kubernetes Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/source.html">Building From Source</a></li>
</ul>
diff --git a/user-guide/0.16/datatypes.html b/user-guide/0.16/datatypes.html
index 691815bd79..81ef1c69d1 100644
--- a/user-guide/0.16/datatypes.html
+++ b/user-guide/0.16/datatypes.html
@@ -458,6 +458,7 @@ under the License.
<li class="toctree-l2"><a class="reference internal" href="../latest/tuning.html">Tuning Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/metrics.html">Metrics Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../latest/s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/kubernetes.html">Kubernetes Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/source.html">Building From Source</a></li>
</ul>
diff --git a/user-guide/0.16/expressions.html b/user-guide/0.16/expressions.html
index a33ff555c1..0746ed2d98 100644
--- a/user-guide/0.16/expressions.html
+++ b/user-guide/0.16/expressions.html
@@ -458,6 +458,7 @@ under the License.
<li class="toctree-l2"><a class="reference internal" href="../latest/tuning.html">Tuning Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/metrics.html">Metrics Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../latest/s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/kubernetes.html">Kubernetes Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/source.html">Building From Source</a></li>
</ul>
diff --git a/user-guide/0.16/iceberg.html b/user-guide/0.16/iceberg.html
index 05ce3d2506..b7a372b6b5 100644
--- a/user-guide/0.16/iceberg.html
+++ b/user-guide/0.16/iceberg.html
@@ -458,6 +458,7 @@ under the License.
<li class="toctree-l2"><a class="reference internal" href="../latest/tuning.html">Tuning Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/metrics.html">Metrics Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../latest/s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/kubernetes.html">Kubernetes Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/source.html">Building From Source</a></li>
</ul>
diff --git a/user-guide/0.16/index.html b/user-guide/0.16/index.html
index c7d4bf8232..e7398a185a 100644
--- a/user-guide/0.16/index.html
+++ b/user-guide/0.16/index.html
@@ -458,6 +458,7 @@ under the License.
<li class="toctree-l2"><a class="reference internal" href="../latest/tuning.html">Tuning Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/metrics.html">Metrics Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../latest/s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/kubernetes.html">Kubernetes Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/source.html">Building From Source</a></li>
</ul>
diff --git a/user-guide/0.16/installation.html b/user-guide/0.16/installation.html
index 283c66cae6..a67fb83441 100644
--- a/user-guide/0.16/installation.html
+++ b/user-guide/0.16/installation.html
@@ -458,6 +458,7 @@ under the License.
<li class="toctree-l2"><a class="reference internal" href="../latest/tuning.html">Tuning Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/metrics.html">Metrics Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../latest/s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/kubernetes.html">Kubernetes Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/source.html">Building From Source</a></li>
</ul>
diff --git a/user-guide/0.16/kubernetes.html b/user-guide/0.16/kubernetes.html
index 34f267b9c3..21f73455b2 100644
--- a/user-guide/0.16/kubernetes.html
+++ b/user-guide/0.16/kubernetes.html
@@ -458,6 +458,7 @@ under the License.
<li class="toctree-l2"><a class="reference internal" href="../latest/tuning.html">Tuning Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/metrics.html">Metrics Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../latest/s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/kubernetes.html">Kubernetes Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/source.html">Building From Source</a></li>
</ul>
diff --git a/user-guide/0.16/metrics.html b/user-guide/0.16/metrics.html
index 17013e0a90..98a658ed95 100644
--- a/user-guide/0.16/metrics.html
+++ b/user-guide/0.16/metrics.html
@@ -458,6 +458,7 @@ under the License.
<li class="toctree-l2"><a class="reference internal" href="../latest/tuning.html">Tuning Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/metrics.html">Metrics Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../latest/s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/kubernetes.html">Kubernetes Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/source.html">Building From Source</a></li>
</ul>
diff --git a/user-guide/0.16/operators.html b/user-guide/0.16/operators.html
index 5e2b1e3fbb..13c2ee6be2 100644
--- a/user-guide/0.16/operators.html
+++ b/user-guide/0.16/operators.html
@@ -458,6 +458,7 @@ under the License.
<li class="toctree-l2"><a class="reference internal" href="../latest/tuning.html">Tuning Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/metrics.html">Metrics Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../latest/s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/kubernetes.html">Kubernetes Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/source.html">Building From Source</a></li>
</ul>
diff --git a/user-guide/0.16/source.html b/user-guide/0.16/source.html
index 6b5884a667..540daea35b 100644
--- a/user-guide/0.16/source.html
+++ b/user-guide/0.16/source.html
@@ -458,6 +458,7 @@ under the License.
<li class="toctree-l2"><a class="reference internal" href="../latest/tuning.html">Tuning Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/metrics.html">Metrics Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../latest/s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/kubernetes.html">Kubernetes Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/source.html">Building From Source</a></li>
</ul>
diff --git a/user-guide/0.16/tuning.html b/user-guide/0.16/tuning.html
index 8a7855bc6a..129b635218 100644
--- a/user-guide/0.16/tuning.html
+++ b/user-guide/0.16/tuning.html
@@ -458,6 +458,7 @@ under the License.
<li class="toctree-l2"><a class="reference internal" href="../latest/tuning.html">Tuning Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/metrics.html">Metrics Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../latest/s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/kubernetes.html">Kubernetes Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/source.html">Building From Source</a></li>
</ul>
diff --git a/user-guide/0.16/understanding-comet-plans.html b/user-guide/0.16/understanding-comet-plans.html
index bfc87839d2..77ad29f83f 100644
--- a/user-guide/0.16/understanding-comet-plans.html
+++ b/user-guide/0.16/understanding-comet-plans.html
@@ -458,6 +458,7 @@ under the License.
<li class="toctree-l2"><a class="reference internal" href="../latest/tuning.html">Tuning Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/metrics.html">Metrics Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../latest/s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/kubernetes.html">Kubernetes Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="../latest/source.html">Building From Source</a></li>
</ul>
diff --git a/user-guide/index.html b/user-guide/index.html
index 6d85dafa53..0729050fbb 100644
--- a/user-guide/index.html
+++ b/user-guide/index.html
@@ -458,6 +458,7 @@ under the License.
<li class="toctree-l2"><a class="reference internal" href="latest/tuning.html">Tuning Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="latest/metrics.html">Metrics Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="latest/iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l2"><a class="reference internal" href="latest/s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l2"><a class="reference internal" href="latest/kubernetes.html">Kubernetes Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="latest/source.html">Building From Source</a></li>
</ul>
diff --git a/user-guide/latest/compatibility/expressions/aggregate.html b/user-guide/latest/compatibility/expressions/aggregate.html
index 356cbf4ae2..4c7db9522e 100644
--- a/user-guide/latest/compatibility/expressions/aggregate.html
+++ b/user-guide/latest/compatibility/expressions/aggregate.html
@@ -434,6 +434,7 @@ under the License.
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Integrations</span></p>
<ul class="nav bd-sidenav">
<li class="toctree-l1"><a class="reference internal" href="../../iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../kubernetes.html">Kubernetes Guide</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Advanced</span></p>
diff --git a/user-guide/latest/compatibility/expressions/array.html b/user-guide/latest/compatibility/expressions/array.html
index 1fd9f85bdc..26cd6d0fd7 100644
--- a/user-guide/latest/compatibility/expressions/array.html
+++ b/user-guide/latest/compatibility/expressions/array.html
@@ -434,6 +434,7 @@ under the License.
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Integrations</span></p>
<ul class="nav bd-sidenav">
<li class="toctree-l1"><a class="reference internal" href="../../iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../kubernetes.html">Kubernetes Guide</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Advanced</span></p>
diff --git a/user-guide/latest/compatibility/expressions/cast.html b/user-guide/latest/compatibility/expressions/cast.html
index 8247534e12..a96afd7c19 100644
--- a/user-guide/latest/compatibility/expressions/cast.html
+++ b/user-guide/latest/compatibility/expressions/cast.html
@@ -434,6 +434,7 @@ under the License.
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Integrations</span></p>
<ul class="nav bd-sidenav">
<li class="toctree-l1"><a class="reference internal" href="../../iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../kubernetes.html">Kubernetes Guide</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Advanced</span></p>
diff --git a/user-guide/latest/compatibility/expressions/datetime.html b/user-guide/latest/compatibility/expressions/datetime.html
index f7cb9d5821..14d9faf28c 100644
--- a/user-guide/latest/compatibility/expressions/datetime.html
+++ b/user-guide/latest/compatibility/expressions/datetime.html
@@ -434,6 +434,7 @@ under the License.
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Integrations</span></p>
<ul class="nav bd-sidenav">
<li class="toctree-l1"><a class="reference internal" href="../../iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../kubernetes.html">Kubernetes Guide</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Advanced</span></p>
diff --git a/user-guide/latest/compatibility/expressions/index.html b/user-guide/latest/compatibility/expressions/index.html
index 37030ae09d..1428078d3f 100644
--- a/user-guide/latest/compatibility/expressions/index.html
+++ b/user-guide/latest/compatibility/expressions/index.html
@@ -434,6 +434,7 @@ under the License.
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Integrations</span></p>
<ul class="nav bd-sidenav">
<li class="toctree-l1"><a class="reference internal" href="../../iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../kubernetes.html">Kubernetes Guide</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Advanced</span></p>
diff --git a/user-guide/latest/compatibility/expressions/map.html b/user-guide/latest/compatibility/expressions/map.html
index ff828b0acf..83e9b8d65d 100644
--- a/user-guide/latest/compatibility/expressions/map.html
+++ b/user-guide/latest/compatibility/expressions/map.html
@@ -434,6 +434,7 @@ under the License.
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Integrations</span></p>
<ul class="nav bd-sidenav">
<li class="toctree-l1"><a class="reference internal" href="../../iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../kubernetes.html">Kubernetes Guide</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Advanced</span></p>
diff --git a/user-guide/latest/compatibility/expressions/math.html b/user-guide/latest/compatibility/expressions/math.html
index 413e8336b2..dea7e4f5b7 100644
--- a/user-guide/latest/compatibility/expressions/math.html
+++ b/user-guide/latest/compatibility/expressions/math.html
@@ -434,6 +434,7 @@ under the License.
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Integrations</span></p>
<ul class="nav bd-sidenav">
<li class="toctree-l1"><a class="reference internal" href="../../iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../kubernetes.html">Kubernetes Guide</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Advanced</span></p>
diff --git a/user-guide/latest/compatibility/expressions/misc.html b/user-guide/latest/compatibility/expressions/misc.html
index 6f0389b7df..168b472534 100644
--- a/user-guide/latest/compatibility/expressions/misc.html
+++ b/user-guide/latest/compatibility/expressions/misc.html
@@ -434,6 +434,7 @@ under the License.
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Integrations</span></p>
<ul class="nav bd-sidenav">
<li class="toctree-l1"><a class="reference internal" href="../../iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../kubernetes.html">Kubernetes Guide</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Advanced</span></p>
diff --git a/user-guide/latest/compatibility/expressions/string.html b/user-guide/latest/compatibility/expressions/string.html
index e2899cbb9a..9a725475eb 100644
--- a/user-guide/latest/compatibility/expressions/string.html
+++ b/user-guide/latest/compatibility/expressions/string.html
@@ -434,6 +434,7 @@ under the License.
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Integrations</span></p>
<ul class="nav bd-sidenav">
<li class="toctree-l1"><a class="reference internal" href="../../iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../kubernetes.html">Kubernetes Guide</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Advanced</span></p>
diff --git a/user-guide/latest/compatibility/expressions/struct.html b/user-guide/latest/compatibility/expressions/struct.html
index 95546d53fc..89c8928d67 100644
--- a/user-guide/latest/compatibility/expressions/struct.html
+++ b/user-guide/latest/compatibility/expressions/struct.html
@@ -434,6 +434,7 @@ under the License.
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Integrations</span></p>
<ul class="nav bd-sidenav">
<li class="toctree-l1"><a class="reference internal" href="../../iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../kubernetes.html">Kubernetes Guide</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Advanced</span></p>
diff --git a/user-guide/latest/compatibility/expressions/url.html b/user-guide/latest/compatibility/expressions/url.html
index 00bd1e0212..9b68c0e3b8 100644
--- a/user-guide/latest/compatibility/expressions/url.html
+++ b/user-guide/latest/compatibility/expressions/url.html
@@ -434,6 +434,7 @@ under the License.
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Integrations</span></p>
<ul class="nav bd-sidenav">
<li class="toctree-l1"><a class="reference internal" href="../../iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../kubernetes.html">Kubernetes Guide</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Advanced</span></p>
diff --git a/user-guide/latest/compatibility/floating-point.html b/user-guide/latest/compatibility/floating-point.html
index a1d0b50c7d..b3a4c9a136 100644
--- a/user-guide/latest/compatibility/floating-point.html
+++ b/user-guide/latest/compatibility/floating-point.html
@@ -434,6 +434,7 @@ under the License.
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Integrations</span></p>
<ul class="nav bd-sidenav">
<li class="toctree-l1"><a class="reference internal" href="../iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l1"><a class="reference internal" href="../kubernetes.html">Kubernetes Guide</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Advanced</span></p>
diff --git a/user-guide/latest/compatibility/index.html b/user-guide/latest/compatibility/index.html
index 2da1d38236..44309c9d67 100644
--- a/user-guide/latest/compatibility/index.html
+++ b/user-guide/latest/compatibility/index.html
@@ -434,6 +434,7 @@ under the License.
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Integrations</span></p>
<ul class="nav bd-sidenav">
<li class="toctree-l1"><a class="reference internal" href="../iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l1"><a class="reference internal" href="../kubernetes.html">Kubernetes Guide</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Advanced</span></p>
diff --git a/user-guide/latest/compatibility/operators.html b/user-guide/latest/compatibility/operators.html
index 838767ef64..4a51957a73 100644
--- a/user-guide/latest/compatibility/operators.html
+++ b/user-guide/latest/compatibility/operators.html
@@ -434,6 +434,7 @@ under the License.
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Integrations</span></p>
<ul class="nav bd-sidenav">
<li class="toctree-l1"><a class="reference internal" href="../iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l1"><a class="reference internal" href="../kubernetes.html">Kubernetes Guide</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Advanced</span></p>
diff --git a/user-guide/latest/compatibility/regex.html b/user-guide/latest/compatibility/regex.html
index e25438896a..eb4a2e8f12 100644
--- a/user-guide/latest/compatibility/regex.html
+++ b/user-guide/latest/compatibility/regex.html
@@ -434,6 +434,7 @@ under the License.
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Integrations</span></p>
<ul class="nav bd-sidenav">
<li class="toctree-l1"><a class="reference internal" href="../iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l1"><a class="reference internal" href="../kubernetes.html">Kubernetes Guide</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Advanced</span></p>
diff --git a/user-guide/latest/compatibility/scans.html b/user-guide/latest/compatibility/scans.html
index 55b4a7e22d..0fabc1db68 100644
--- a/user-guide/latest/compatibility/scans.html
+++ b/user-guide/latest/compatibility/scans.html
@@ -434,6 +434,7 @@ under the License.
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Integrations</span></p>
<ul class="nav bd-sidenav">
<li class="toctree-l1"><a class="reference internal" href="../iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l1"><a class="reference internal" href="../kubernetes.html">Kubernetes Guide</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Advanced</span></p>
diff --git a/user-guide/latest/compatibility/spark-versions.html b/user-guide/latest/compatibility/spark-versions.html
index 211f3388b4..832dafa3ff 100644
--- a/user-guide/latest/compatibility/spark-versions.html
+++ b/user-guide/latest/compatibility/spark-versions.html
@@ -434,6 +434,7 @@ under the License.
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Integrations</span></p>
<ul class="nav bd-sidenav">
<li class="toctree-l1"><a class="reference internal" href="../iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l1"><a class="reference internal" href="../kubernetes.html">Kubernetes Guide</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Advanced</span></p>
diff --git a/user-guide/latest/configs.html b/user-guide/latest/configs.html
index 2bd2861e85..90726eb347 100644
--- a/user-guide/latest/configs.html
+++ b/user-guide/latest/configs.html
@@ -434,6 +434,7 @@ under the License.
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Integrations</span></p>
<ul class="nav bd-sidenav">
<li class="toctree-l1"><a class="reference internal" href="iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l1"><a class="reference internal" href="s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l1"><a class="reference internal" href="kubernetes.html">Kubernetes Guide</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Advanced</span></p>
diff --git a/user-guide/latest/datasources.html b/user-guide/latest/datasources.html
index fa003a4ef3..0051f53816 100644
--- a/user-guide/latest/datasources.html
+++ b/user-guide/latest/datasources.html
@@ -434,6 +434,7 @@ under the License.
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Integrations</span></p>
<ul class="nav bd-sidenav">
<li class="toctree-l1"><a class="reference internal" href="iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l1"><a class="reference internal" href="s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l1"><a class="reference internal" href="kubernetes.html">Kubernetes Guide</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Advanced</span></p>
diff --git a/user-guide/latest/datatypes.html b/user-guide/latest/datatypes.html
index 6354472e7f..aed4382f26 100644
--- a/user-guide/latest/datatypes.html
+++ b/user-guide/latest/datatypes.html
@@ -434,6 +434,7 @@ under the License.
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Integrations</span></p>
<ul class="nav bd-sidenav">
<li class="toctree-l1"><a class="reference internal" href="iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l1"><a class="reference internal" href="s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l1"><a class="reference internal" href="kubernetes.html">Kubernetes Guide</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Advanced</span></p>
diff --git a/user-guide/latest/expressions.html b/user-guide/latest/expressions.html
index ee05177cfc..dd0d9af523 100644
--- a/user-guide/latest/expressions.html
+++ b/user-guide/latest/expressions.html
@@ -434,6 +434,7 @@ under the License.
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Integrations</span></p>
<ul class="nav bd-sidenav">
<li class="toctree-l1"><a class="reference internal" href="iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l1"><a class="reference internal" href="s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l1"><a class="reference internal" href="kubernetes.html">Kubernetes Guide</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Advanced</span></p>
diff --git a/user-guide/latest/iceberg.html b/user-guide/latest/iceberg.html
index 9429ef1f19..cce65eb6c2 100644
--- a/user-guide/latest/iceberg.html
+++ b/user-guide/latest/iceberg.html
@@ -65,7 +65,7 @@ under the License.
<script async="true" defer="true" src="https://buttons.github.io/buttons.js"></script>
<link rel="index" title="Index" href="../../genindex.html" />
<link rel="search" title="Search" href="../../search.html" />
- <link rel="next" title="Comet Kubernetes Support" href="kubernetes.html" />
+ <link rel="next" title="S3 Credential Providers" href="s3-credential-providers.html" />
<link rel="prev" title="Comet Metrics" href="metrics.html" />
<meta name="viewport" content="width=device-width, initial-scale=1"/>
<meta name="docsearch:language" content="en"/>
@@ -434,6 +434,7 @@ under the License.
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Integrations</span></p>
<ul class="current nav bd-sidenav">
<li class="toctree-l1 current active"><a class="current reference internal" href="#">Iceberg Guide</a></li>
+<li class="toctree-l1"><a class="reference internal" href="s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l1"><a class="reference internal" href="kubernetes.html">Kubernetes Guide</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Advanced</span></p>
@@ -691,11 +692,11 @@ the project, exchange, and sort operators around them stay on the Comet path end
</div>
</a>
<a class="right-next"
- href="kubernetes.html"
+ href="s3-credential-providers.html"
title="next page">
<div class="prev-next-info">
<p class="prev-next-subtitle">next</p>
- <p class="prev-next-title">Comet Kubernetes Support</p>
+ <p class="prev-next-title">S3 Credential Providers</p>
</div>
<i class="fa-solid fa-angle-right"></i>
</a>
diff --git a/user-guide/latest/index.html b/user-guide/latest/index.html
index 310aefc897..add5441626 100644
--- a/user-guide/latest/index.html
+++ b/user-guide/latest/index.html
@@ -434,6 +434,7 @@ under the License.
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Integrations</span></p>
<ul class="nav bd-sidenav">
<li class="toctree-l1"><a class="reference internal" href="iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l1"><a class="reference internal" href="s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l1"><a class="reference internal" href="kubernetes.html">Kubernetes Guide</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Advanced</span></p>
diff --git a/user-guide/latest/installation.html b/user-guide/latest/installation.html
index 9c17e89251..ee293ef2fd 100644
--- a/user-guide/latest/installation.html
+++ b/user-guide/latest/installation.html
@@ -434,6 +434,7 @@ under the License.
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Integrations</span></p>
<ul class="nav bd-sidenav">
<li class="toctree-l1"><a class="reference internal" href="iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l1"><a class="reference internal" href="s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l1"><a class="reference internal" href="kubernetes.html">Kubernetes Guide</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Advanced</span></p>
diff --git a/user-guide/latest/kubernetes.html b/user-guide/latest/kubernetes.html
index 93c18a11f6..be0ef46c83 100644
--- a/user-guide/latest/kubernetes.html
+++ b/user-guide/latest/kubernetes.html
@@ -66,7 +66,7 @@ under the License.
<link rel="index" title="Index" href="../../genindex.html" />
<link rel="search" title="Search" href="../../search.html" />
<link rel="next" title="Building Comet From Source" href="source.html" />
- <link rel="prev" title="Accelerating Apache Iceberg Parquet Scans using Comet" href="iceberg.html" />
+ <link rel="prev" title="S3 Credential Providers" href="s3-credential-providers.html" />
<meta name="viewport" content="width=device-width, initial-scale=1"/>
<meta name="docsearch:language" content="en"/>
<meta name="docsearch:version" content="" />
@@ -434,6 +434,7 @@ under the License.
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Integrations</span></p>
<ul class="current nav bd-sidenav">
<li class="toctree-l1"><a class="reference internal" href="iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l1"><a class="reference internal" href="s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l1 current active"><a class="current reference internal" href="#">Kubernetes Guide</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Advanced</span></p>
@@ -644,12 +645,12 @@ Events:
<div class="prev-next-area">
<a class="left-prev"
- href="iceberg.html"
+ href="s3-credential-providers.html"
title="previous page">
<i class="fa-solid fa-angle-left"></i>
<div class="prev-next-info">
<p class="prev-next-subtitle">previous</p>
- <p class="prev-next-title">Accelerating Apache Iceberg Parquet Scans using Comet</p>
+ <p class="prev-next-title">S3 Credential Providers</p>
</div>
</a>
<a class="right-next"
diff --git a/user-guide/latest/metrics.html b/user-guide/latest/metrics.html
index 05886bb854..774fd2af93 100644
--- a/user-guide/latest/metrics.html
+++ b/user-guide/latest/metrics.html
@@ -434,6 +434,7 @@ under the License.
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Integrations</span></p>
<ul class="nav bd-sidenav">
<li class="toctree-l1"><a class="reference internal" href="iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l1"><a class="reference internal" href="s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l1"><a class="reference internal" href="kubernetes.html">Kubernetes Guide</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Advanced</span></p>
diff --git a/user-guide/latest/operators.html b/user-guide/latest/operators.html
index f9dbd1108b..06a91252ac 100644
--- a/user-guide/latest/operators.html
+++ b/user-guide/latest/operators.html
@@ -434,6 +434,7 @@ under the License.
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Integrations</span></p>
<ul class="nav bd-sidenav">
<li class="toctree-l1"><a class="reference internal" href="iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l1"><a class="reference internal" href="s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l1"><a class="reference internal" href="kubernetes.html">Kubernetes Guide</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Advanced</span></p>
diff --git a/user-guide/latest/s3-credential-providers.html b/user-guide/latest/s3-credential-providers.html
new file mode 100644
index 0000000000..9ffbb69e87
--- /dev/null
+++ b/user-guide/latest/s3-credential-providers.html
@@ -0,0 +1,855 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+
+<!DOCTYPE html>
+
+
+<html lang="en" data-content_root="../../" data-theme="light">
+
+ <head>
+ <meta charset="utf-8" />
+ <meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="viewport" content="width=device-width, initial-scale=1" />
+
+ <title>S3 Credential Providers — Apache DataFusion Comet documentation</title>
+
+
+
+ <script data-cfasync="false">
+ document.documentElement.dataset.mode = localStorage.getItem("mode") || "light";
+ document.documentElement.dataset.theme = localStorage.getItem("theme") || "light";
+ </script>
+ <!--
+ this give us a css class that will be invisible only if js is disabled
+ -->
+ <noscript>
+ <style>
+ .pst-js-only { display: none !important; }
+
+ </style>
+ </noscript>
+
+ <!-- Loaded before other Sphinx assets -->
+ <link href="../../_static/styles/theme.css?digest=8878045cc6db502f8baf" rel="stylesheet" />
+<link href="../../_static/styles/pydata-sphinx-theme.css?digest=8878045cc6db502f8baf" rel="stylesheet" />
+
+ <link rel="stylesheet" type="text/css" href="../../_static/pygments.css?v=8f2a1f02" />
+ <link rel="stylesheet" type="text/css" href="../../_static/theme_overrides.css?v=cd442bcd" />
+
+ <!-- So that users can add custom icons -->
+ <script src="../../_static/scripts/fontawesome.js?digest=8878045cc6db502f8baf"></script>
+ <!-- Pre-loaded scripts that we'll load fully later -->
+ <link rel="preload" as="script" href="../../_static/scripts/bootstrap.js?digest=8878045cc6db502f8baf" />
+<link rel="preload" as="script" href="../../_static/scripts/pydata-sphinx-theme.js?digest=8878045cc6db502f8baf" />
+
+ <script src="../../_static/documentation_options.js?v=5929fcd5"></script>
+ <script src="../../_static/doctools.js?v=9a2dae69"></script>
+ <script src="../../_static/sphinx_highlight.js?v=dc90522c"></script>
+ <script>DOCUMENTATION_OPTIONS.pagename = 'user-guide/latest/s3-credential-providers';</script>
+ <script async="true" defer="true" src="https://buttons.github.io/buttons.js"></script>
+ <link rel="index" title="Index" href="../../genindex.html" />
+ <link rel="search" title="Search" href="../../search.html" />
+ <link rel="next" title="Comet Kubernetes Support" href="kubernetes.html" />
+ <link rel="prev" title="Accelerating Apache Iceberg Parquet Scans using Comet" href="iceberg.html" />
+ <meta name="viewport" content="width=device-width, initial-scale=1"/>
+ <meta name="docsearch:language" content="en"/>
+ <meta name="docsearch:version" content="" />
+ </head>
+
+
+ <body data-bs-spy="scroll" data-bs-target=".bd-toc-nav" data-offset="180" data-bs-root-margin="0px 0px -60%" data-default-mode="light">
+
+
+
+ <div id="pst-skip-link" class="skip-link d-print-none"><a href="#main-content">Skip to main content</a></div>
+
+ <div id="pst-scroll-pixel-helper"></div>
+
+ <button type="button" class="btn rounded-pill" id="pst-back-to-top">
+ <i class="fa-solid fa-arrow-up"></i>Back to top</button>
+
+
+ <dialog id="pst-search-dialog">
+
+<form class="bd-search d-flex align-items-center"
+ action="../../search.html"
+ method="get">
+ <i class="fa-solid fa-magnifying-glass"></i>
+ <input type="search"
+ class="form-control"
+ name="q"
+ placeholder="Search the docs ..."
+ aria-label="Search the docs ..."
+ autocomplete="off"
+ autocorrect="off"
+ autocapitalize="off"
+ spellcheck="false"/>
+ <span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd>K</kbd></span>
+</form>
+ </dialog>
+
+ <div class="pst-async-banner-revealer d-none">
+ <aside id="bd-header-version-warning" class="d-none d-print-none" aria-label="Version warning"></aside>
+</div>
+
+
+ <header class="bd-header navbar navbar-expand-lg bd-navbar d-print-none">
+<div class="bd-header__inner bd-page-width">
+ <button class="pst-navbar-icon sidebar-toggle primary-toggle" aria-label="Site navigation">
+ <span class="fa-solid fa-bars"></span>
+ </button>
+
+
+ <div class="col-lg-3 navbar-header-items__start">
+
+ <div class="navbar-item">
+
+
+
+
+
+<a class="navbar-brand logo" href="../../index.html">
+
+
+
+
+
+
+
+
+ <img src="../../_static/DataFusionComet-Logo-Light.png" class="logo__image only-light" alt="Apache DataFusion Comet documentation - Home"/>
+ <img src="../../_static/DataFusionComet-Logo-Dark.png" class="logo__image only-dark pst-js-only" alt="Apache DataFusion Comet documentation - Home"/>
+
+
+</a></div>
+
+ </div>
+
+ <div class="col-lg-9 navbar-header-items">
+
+ <div class="me-auto navbar-header-items__center">
+
+ <div class="navbar-item">
+<nav>
+ <ul class="bd-navbar-elements navbar-nav">
+
+<li class="nav-item current active">
+ <a class="nav-link nav-internal" href="../index.html">
+ User Guide
+ </a>
+</li>
+
+
+<li class="nav-item ">
+ <a class="nav-link nav-internal" href="../../contributor-guide/index.html">
+ Contributor Guide
+ </a>
+</li>
+
+
+<li class="nav-item ">
+ <a class="nav-link nav-internal" href="../../changelog/index.html">
+ Changelog
+ </a>
+</li>
+
+
+<li class="nav-item ">
+ <a class="nav-link nav-internal" href="../../about/gluten_comparison.html">
+ Comparison with Gluten
+ </a>
+</li>
+
+
+<li class="nav-item ">
+ <a class="nav-link nav-internal" href="../../about/versioning_policy.html">
+ Versioning Policy
+ </a>
+</li>
+
+ <li class="nav-item dropdown">
+ <button class="btn dropdown-toggle nav-item" type="button"
+ data-bs-toggle="dropdown" aria-expanded="false"
+ aria-controls="pst-nav-more-links">
+ More
+ </button>
+ <ul id="pst-nav-more-links" class="dropdown-menu">
+
+<li class=" ">
+ <a class="nav-link dropdown-item nav-internal" href="../../asf/index.html">
+ ASF Links
+ </a>
+</li>
+
+ </ul>
+ </li>
+
+ </ul>
+</nav></div>
+
+ </div>
+
+
+ <div class="navbar-header-items__end">
+
+ <div class="navbar-item navbar-persistent--container">
+
+
+<button class="btn search-button-field search-button__button pst-js-only" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip">
+ <i class="fa-solid fa-magnifying-glass"></i>
+ <span class="search-button__default-text">Search</span>
+ <span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span>
+</button>
+ </div>
+
+
+ <div class="navbar-item"><ul class="navbar-icon-links"
+ aria-label="Icon Links">
+ <li class="nav-item">
+
+
+
+
+
+
+
+
+ <a href="https://github.com/apache/datafusion-comet" title="GitHub" class="nav-link pst-navbar-icon" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><i class="fa-brands fa-github fa-lg" aria-hidden="true"></i>
+ <span class="sr-only">GitHub</span></a>
+ </li>
+</ul></div>
+
+ <div class="navbar-item">
+
+<button class="btn btn-sm nav-link pst-navbar-icon theme-switch-button pst-js-only" aria-label="Color mode" data-bs-title="Color mode" data-bs-placement="bottom" data-bs-toggle="tooltip">
+ <i class="theme-switch fa-solid fa-sun fa-lg" data-mode="light" title="Light"></i>
+ <i class="theme-switch fa-solid fa-moon fa-lg" data-mode="dark" title="Dark"></i>
+ <i class="theme-switch fa-solid fa-circle-half-stroke fa-lg" data-mode="auto" title="System Settings"></i>
+</button></div>
+
+ </div>
+
+ </div>
+
+
+ <div class="navbar-persistent--mobile">
+
+<button class="btn search-button-field search-button__button pst-js-only" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip">
+ <i class="fa-solid fa-magnifying-glass"></i>
+ <span class="search-button__default-text">Search</span>
+ <span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span>
+</button>
+ </div>
+
+
+
+</div>
+
+ </header>
+
+
+ <div class="bd-container">
+ <div class="bd-container__inner bd-page-width">
+
+
+
+ <dialog id="pst-primary-sidebar-modal"></dialog>
+ <div id="pst-primary-sidebar" class="bd-sidebar-primary bd-sidebar">
+
+
+
+ <div class="sidebar-header-items sidebar-primary__section">
+
+
+ <div class="sidebar-header-items__center">
+
+
+
+ <div class="navbar-item">
+<nav>
+ <ul class="bd-navbar-elements navbar-nav">
+
+<li class="nav-item current active">
+ <a class="nav-link nav-internal" href="../index.html">
+ User Guide
+ </a>
+</li>
+
+
+<li class="nav-item ">
+ <a class="nav-link nav-internal" href="../../contributor-guide/index.html">
+ Contributor Guide
+ </a>
+</li>
+
+
+<li class="nav-item ">
+ <a class="nav-link nav-internal" href="../../changelog/index.html">
+ Changelog
+ </a>
+</li>
+
+
+<li class="nav-item ">
+ <a class="nav-link nav-internal" href="../../about/gluten_comparison.html">
+ Comparison with Gluten
+ </a>
+</li>
+
+
+<li class="nav-item ">
+ <a class="nav-link nav-internal" href="../../about/versioning_policy.html">
+ Versioning Policy
+ </a>
+</li>
+
+
+<li class="nav-item ">
+ <a class="nav-link nav-internal" href="../../asf/index.html">
+ ASF Links
+ </a>
+</li>
+
+ </ul>
+</nav></div>
+
+
+ </div>
+
+
+
+ <div class="sidebar-header-items__end">
+
+ <div class="navbar-item"><ul class="navbar-icon-links"
+ aria-label="Icon Links">
+ <li class="nav-item">
+
+
+
+
+
+
+
+
+ <a href="https://github.com/apache/datafusion-comet" title="GitHub" class="nav-link pst-navbar-icon" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><i class="fa-brands fa-github fa-lg" aria-hidden="true"></i>
+ <span class="sr-only">GitHub</span></a>
+ </li>
+</ul></div>
+
+ <div class="navbar-item">
+
+<button class="btn btn-sm nav-link pst-navbar-icon theme-switch-button pst-js-only" aria-label="Color mode" data-bs-title="Color mode" data-bs-placement="bottom" data-bs-toggle="tooltip">
+ <i class="theme-switch fa-solid fa-sun fa-lg" data-mode="light" title="Light"></i>
+ <i class="theme-switch fa-solid fa-moon fa-lg" data-mode="dark" title="Dark"></i>
+ <i class="theme-switch fa-solid fa-circle-half-stroke fa-lg" data-mode="auto" title="System Settings"></i>
+</button></div>
+
+ </div>
+
+ </div>
+
+ <div class="sidebar-primary-items__start sidebar-primary__section">
+ <div class="sidebar-primary-item"><!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+
+
+
+
+<nav class="bd-links" id="bd-docs-nav" aria-label="Main navigation">
+ <div class="bd-toc-item active">
+ <p aria-level="2" class="caption" role="heading"><span class="caption-text">Getting Started</span></p>
+<ul class="nav bd-sidenav">
+<li class="toctree-l1"><a class="reference internal" href="installation.html">Installing Comet</a></li>
+<li class="toctree-l1"><a class="reference internal" href="configs.html">Configuration Settings</a></li>
+</ul>
+<p aria-level="2" class="caption" role="heading"><span class="caption-text">What Comet Supports</span></p>
+<ul class="nav bd-sidenav">
+<li class="toctree-l1"><a class="reference internal" href="datasources.html">Supported Data Sources</a></li>
+<li class="toctree-l1"><a class="reference internal" href="datatypes.html">Supported Data Types</a></li>
+<li class="toctree-l1"><a class="reference internal" href="operators.html">Supported Operators</a></li>
+<li class="toctree-l1"><a class="reference internal" href="expressions.html">Supported Expressions</a></li>
+<li class="toctree-l1"><a class="reference internal" href="scala_java_udfs.html">ScalaUDF and Java UDF Support</a></li>
+</ul>
+<p aria-level="2" class="caption" role="heading"><span class="caption-text">Compatibility</span></p>
+<ul class="nav bd-sidenav">
+<li class="toctree-l1"><a class="reference internal" href="compatibility/index.html">Overview</a></li>
+<li class="toctree-l1"><a class="reference internal" href="compatibility/scans.html">Parquet Compatibility</a></li>
+<li class="toctree-l1"><a class="reference internal" href="compatibility/floating-point.html">Floating-point Number Comparison</a></li>
+<li class="toctree-l1"><a class="reference internal" href="compatibility/regex.html">Regular Expressions</a></li>
+<li class="toctree-l1"><a class="reference internal" href="compatibility/operators.html">Operator Compatibility</a></li>
+<li class="toctree-l1 has-children"><a class="reference internal" href="compatibility/expressions/index.html">Expression Compatibility</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
+<li class="toctree-l2"><a class="reference internal" href="compatibility/expressions/aggregate.html">Aggregate Expressions</a></li>
+<li class="toctree-l2"><a class="reference internal" href="compatibility/expressions/array.html">Array Expressions</a></li>
+<li class="toctree-l2"><a class="reference internal" href="compatibility/expressions/datetime.html">Date/Time Expressions</a></li>
+<li class="toctree-l2"><a class="reference internal" href="compatibility/expressions/map.html">Map Expressions</a></li>
+<li class="toctree-l2"><a class="reference internal" href="compatibility/expressions/math.html">Math Expressions</a></li>
+<li class="toctree-l2"><a class="reference internal" href="compatibility/expressions/misc.html">Miscellaneous Expressions</a></li>
+<li class="toctree-l2"><a class="reference internal" href="compatibility/expressions/string.html">String Expressions</a></li>
+<li class="toctree-l2"><a class="reference internal" href="compatibility/expressions/struct.html">Struct Expressions</a></li>
+<li class="toctree-l2"><a class="reference internal" href="compatibility/expressions/url.html">URL Expressions</a></li>
+<li class="toctree-l2"><a class="reference internal" href="compatibility/expressions/cast.html">Cast</a></li>
+</ul>
+</details></li>
+<li class="toctree-l1"><a class="reference internal" href="compatibility/spark-versions.html">Spark Version Compatibility</a></li>
+</ul>
+<p aria-level="2" class="caption" role="heading"><span class="caption-text">Operating Comet</span></p>
+<ul class="nav bd-sidenav">
+<li class="toctree-l1"><a class="reference internal" href="understanding-comet-plans.html">Understanding Comet Plans</a></li>
+<li class="toctree-l1"><a class="reference internal" href="tuning.html">Tuning Guide</a></li>
+<li class="toctree-l1"><a class="reference internal" href="metrics.html">Metrics Guide</a></li>
+</ul>
+<p aria-level="2" class="caption" role="heading"><span class="caption-text">Integrations</span></p>
+<ul class="current nav bd-sidenav">
+<li class="toctree-l1"><a class="reference internal" href="iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l1 current active"><a class="current reference internal" href="#">S3 Credential Providers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="kubernetes.html">Kubernetes Guide</a></li>
+</ul>
+<p aria-level="2" class="caption" role="heading"><span class="caption-text">Advanced</span></p>
+<ul class="nav bd-sidenav">
+<li class="toctree-l1"><a class="reference internal" href="source.html">Building From Source</a></li>
+</ul>
+
+ </div>
+</nav>
+</div>
+ </div>
+
+
+ <div class="sidebar-primary-items__end sidebar-primary__section">
+ <div class="sidebar-primary-item">
+<div id="ethical-ad-placement"
+ class="flat"
+ data-ea-publisher="readthedocs"
+ data-ea-type="readthedocs-sidebar"
+ data-ea-manual="true">
+</div></div>
+ </div>
+
+
+ </div>
+
+ <main id="main-content" class="bd-main" role="main">
+
+
+ <div class="bd-content">
+ <div class="bd-article-container">
+
+ <div class="bd-header-article d-print-none">
+<div class="header-article-items header-article__inner">
+
+ <div class="header-article-items__start">
+
+ <div class="header-article-item">
+
+<nav aria-label="Breadcrumb" class="d-print-none">
+ <ul class="bd-breadcrumbs">
+
+ <li class="breadcrumb-item breadcrumb-home">
+ <a href="../../index.html" class="nav-link" aria-label="Home">
+ <i class="fa-solid fa-home"></i>
+ </a>
+ </li>
+
+ <li class="breadcrumb-item"><a href="../index.html" class="nav-link">Comet User Guide</a></li>
+
+
+ <li class="breadcrumb-item"><a href="index.html" class="nav-link">Comet 0.17.0-SNAPSHOT User Guide</a></li>
+
+ <li class="breadcrumb-item active" aria-current="page"><span class="ellipsis">S3 Credential Providers</span></li>
+ </ul>
+</nav>
+</div>
+
+ </div>
+
+
+</div>
+</div>
+
+
+
+
+<div id="searchbox"></div>
+ <article class="bd-article">
+
+ <!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<section id="s3-credential-providers">
+<h1>S3 Credential Providers<a class="headerlink" href="#s3-credential-providers" title="Link to this heading">#</a></h1>
+<p>Comet’s native S3 readers normally fetch credentials from the standard AWS credential chain (static keys, instance profiles, environment variables, etc.). Some clusters use a vendor-managed mechanism instead, where credentials are issued per request based on a JWT or per S3 path. For those clusters, Comet supports loading a vendor-supplied bridge class that routes every native credential request through the vendor’s Java code.</p>
+<section id="do-i-need-this">
+<h2>Do I need this?<a class="headerlink" href="#do-i-need-this" title="Link to this heading">#</a></h2>
+<p>You don’t, if any of the following describe your cluster:</p>
+<ul class="simple">
+<li><p>You use static AWS credentials (<code class="docutils literal notranslate"><span class="pre">fs.s3a.access.key</span></code> / <code class="docutils literal notranslate"><span class="pre">fs.s3a.secret.key</span></code>).</p></li>
+<li><p>You use EC2 instance profiles, EKS pod identities, ECS task roles, or environment variables.</p></li>
+<li><p>Your S3 access works in Spark today via the default AWS credential chain.</p></li>
+</ul>
+<p>You probably do, if any of these are true:</p>
+<ul class="simple">
+<li><p>You have a Hadoop S3A custom signer configured (<code class="docutils literal notranslate"><span class="pre">fs.s3a.custom.signers=...</span></code>).</p></li>
+<li><p>You have a Spark <code class="docutils literal notranslate"><span class="pre">CloudCredentialsProvider</span></code> that issues a JWT for a vendor STS service.</p></li>
+<li><p>You have a custom Iceberg <code class="docutils literal notranslate"><span class="pre">client.factory</span></code> that injects a configured S3 client.</p></li>
+<li><p>Spark queries against your S3 paths work, but the same queries with Comet enabled fail with 403.</p></li>
+</ul>
+</section>
+<section id="enabling-a-bridge">
+<h2>Enabling a bridge<a class="headerlink" href="#enabling-a-bridge" title="Link to this heading">#</a></h2>
+<p>A bridge is activated by naming the vendor’s class in a Spark config. Putting a JAR on the classpath alone has no effect; the config key must be set.</p>
+<p>For raw Parquet (the <code class="docutils literal notranslate"><span class="pre">object_store</span></code> path), set the Hadoop S3A config:</p>
+<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">spark</span><span class="o">.</span><span class="n">hadoop</span><span class="o">.</span><span class="n">fs</span><span class="o">.</span><span class="n">s3a</span><span class="o">.</span><span class="n">comet</span><span class="o">.</span><span class="n">credential</span><span class="o">.</span><span class="n">provider</span><span class="o">.</span><span class="n">class</span><span class= [...]
+</pre></div>
+</div>
+<p>A per-bucket override is supported and follows the same shape as <code class="docutils literal notranslate"><span class="pre">fs.s3a.bucket.<name>.aws.credentials.provider</span></code>:</p>
+<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">spark</span><span class="o">.</span><span class="n">hadoop</span><span class="o">.</span><span class="n">fs</span><span class="o">.</span><span class="n">s3a</span><span class="o">.</span><span class="n">bucket</span><span class="o">.<</span><span class="n">bucket</span><span class="o">>.</span><span class="n">comet</span><span class="o">.</span><span class="n">credential</span><span [...]
+</pre></div>
+</div>
+<p>For Iceberg (the <code class="docutils literal notranslate"><span class="pre">opendal</span></code> path), set the per-catalog property under the <code class="docutils literal notranslate"><span class="pre">s3.</span></code> namespace Iceberg already uses for its S3 settings:</p>
+<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">spark</span><span class="o">.</span><span class="n">sql</span><span class="o">.</span><span class="n">catalog</span><span class="o">.<</span><span class="n">catalog</span><span class="o">>.</span><span class="n">s3</span><span class="o">.</span><span class="n">comet</span><span class="o">.</span><span class="n">credential</span><span class="o">.</span><span class="n">provider</span>< [...]
+</pre></div>
+</div>
+<p>Add the vendor JAR to your Spark executor classpath:</p>
+<div class="highlight-sh notranslate"><div class="highlight"><pre><span></span>spark-submit<span class="w"> </span>--jars<span class="w"> </span>vendor-comet-bridge.jar<span class="w"> </span>...
+</pre></div>
+</div>
+<p>OSS Comet ships no vendor-specific bridges. Get one from the same vendor that supplies your Hadoop S3A signer or Iceberg client factory. If they do not yet provide one, send them to the “Writing a bridge” section below.</p>
+</section>
+<section id="wire-encryption">
+<h2>Wire encryption<a class="headerlink" href="#wire-encryption" title="Link to this heading">#</a></h2>
+<p>Catalog configuration you set under <code class="docutils literal notranslate"><span class="pre">spark.sql.catalog.<catalog>.*</span></code> (which may include endpoint URLs, OAuth tokens, and other vendor keys your provider reads) is sent from the driver to executors over Spark’s Netty RPC channel so the provider can run there. Live AWS credentials fetched by the provider stay on the executor that fetched them.</p>
+<p>Spark’s RPC channel is plaintext by default and offers two opt-in encryption mechanisms, both off by default:</p>
+<ul class="simple">
+<li><p><a class="reference external" href="https://spark.apache.org/docs/latest/security.html#authentication-and-encryption"><code class="docutils literal notranslate"><span class="pre">spark.network.crypto.enabled</span></code></a> for AES-based RPC encryption keyed off the auth shared secret.</p></li>
+<li><p><a class="reference external" href="https://spark.apache.org/docs/latest/security.html#ssl-configuration"><code class="docutils literal notranslate"><span class="pre">spark.ssl.rpc.enabled</span></code></a> for TLS on the same channel (Spark 3.4+).</p></li>
+</ul>
+<p>The same defaults apply to Hadoop delegation tokens, <code class="docutils literal notranslate"><span class="pre">CloudCredentialsProvider</span></code> JWTs, and any other secrets Spark already ships driver-to-executor, so this is a deployment-wide call rather than something specific to this SPI. See Spark’s <a class="reference external" href="https://spark.apache.org/docs/latest/security.html">security guide</a> for the full set of knobs.</p>
+</section>
+<section id="verification">
+<h2>Verification<a class="headerlink" href="#verification" title="Link to this heading">#</a></h2>
+<p>With the config set and the JAR on the classpath, executor logs show on first S3 access:</p>
+<ul class="simple">
+<li><p>Info level: <code class="docutils literal notranslate"><span class="pre">Instantiated</span> <span class="pre">CometS3CredentialProvider</span> <span class="pre"><fully.qualified.VendorClassName></span></code></p></li>
+<li><p>Debug level: <code class="docutils literal notranslate"><span class="pre">Fetching</span> <span class="pre">credentials</span> <span class="pre">via</span> <span class="pre"><class></span> <span class="pre">(dispatchKey=<key>)</span> <span class="pre">for</span> <span class="pre">bucket=...</span> <span class="pre">path=...</span> <span class="pre">mode=...</span></code></p></li>
+</ul>
+<p>Without the config set, no credential-related log lines appear at startup; native readers use the default AWS credential chain.</p>
+</section>
+<section id="troubleshooting">
+<h2>Troubleshooting<a class="headerlink" href="#troubleshooting" title="Link to this heading">#</a></h2>
+<p><strong><code class="docutils literal notranslate"><span class="pre">CometS3CredentialProvider</span> <span class="pre">class</span> <span class="pre">not</span> <span class="pre">found:</span> <span class="pre"><name></span></code></strong>. The class named in the config is not on the executor classpath. Re-check <code class="docutils literal notranslate"><span class="pre">--jars</span></code> / <code class="docutils literal notranslate"><span class="pre">spark.jars</span></cod [...]
+<p><strong><code class="docutils literal notranslate"><span class="pre"><class></span> <span class="pre">does</span> <span class="pre">not</span> <span class="pre">implement</span> <span class="pre">org.apache.comet.cloud.s3.CometS3CredentialProvider</span></code></strong>. The configured class exists but does not implement the SPI. Double-check the FQCN against the vendor’s documentation.</p>
+<p><strong><code class="docutils literal notranslate"><span class="pre"><class></span> <span class="pre">must</span> <span class="pre">declare</span> <span class="pre">a</span> <span class="pre">public</span> <span class="pre">no-arg</span> <span class="pre">constructor</span></code></strong>. Vendor classes are instantiated reflectively with <code class="docutils literal notranslate"><span class="pre">Class.forName(name).getDeclaredConstructor().newInstance()</span></code>. A non- [...]
+<p><strong><code class="docutils literal notranslate"><span class="pre">CometS3CredentialProvider</span> <span class="pre"><class></span> <span class="pre">(dispatchKey=...)</span> <span class="pre">was</span> <span class="pre">not</span> <span class="pre">initialized</span></code></strong>. <code class="docutils literal notranslate"><span class="pre">initialize(Map)</span></code> was not called before a credential request. Comet should always invoke <code class="docutils literal n [...]
+<p><strong><code class="docutils literal notranslate"><span class="pre">403</span> <span class="pre">AccessDenied</span></code> with the bridge configured.</strong> The provider returned credentials but they were rejected by S3. Most often a region mismatch (see Iceberg section below) or expired session token; enable debug logging on the vendor’s class to confirm what it returned.</p>
+<p><strong>Credentials silently going stale during long-running jobs.</strong> When a vendor returns <code class="docutils literal notranslate"><span class="pre">expirationEpochMillis=0</span></code>, the bridge substitutes a 5-minute expiry before handing the credential to <code class="docutils literal notranslate"><span class="pre">opendal</span></code>, so <code class="docutils literal notranslate"><span class="pre">opendal</span></code>’s cache cannot hold a stale credential indefini [...]
+</section>
+<section id="iceberg-explicit-s3-region-required">
+<h2>Iceberg: explicit S3 region required<a class="headerlink" href="#iceberg-explicit-s3-region-required" title="Link to this heading">#</a></h2>
+<p>With the bridge configured, Comet wires a custom credential loader into <code class="docutils literal notranslate"><span class="pre">iceberg-storage-opendal</span></code>. <code class="docutils literal notranslate"><span class="pre">opendal</span></code>’s built-in S3 region auto-detection only runs when no custom loader is configured, so on the bridge path the region (and endpoint for non-AWS) must be set explicitly on the Spark catalog:</p>
+<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">spark</span><span class="o">.</span><span class="n">sql</span><span class="o">.</span><span class="n">catalog</span><span class="o">.<</span><span class="n">catalog</span><span class="o">>.</span><span class="n">s3</span><span class="o">.</span><span class="n">region</span> <span class="o">=</span> <span class="n">us</span><span class="o">-</span><span class="n">east</span><sp [...]
+<span class="n">spark</span><span class="o">.</span><span class="n">sql</span><span class="o">.</span><span class="n">catalog</span><span class="o">.<</span><span class="n">catalog</span><span class="o">>.</span><span class="n">s3</span><span class="o">.</span><span class="n">endpoint</span> <span class="o">=</span> <span class="n">https</span><span class="p">:</span><span class="o">//...</span> <span class="p">(</span><span class="n">non</span><span class="o">-</span><span [...]
+<span class="n">spark</span><span class="o">.</span><span class="n">sql</span><span class="o">.</span><span class="n">catalog</span><span class="o">.<</span><span class="n">catalog</span><span class="o">>.</span><span class="n">s3</span><span class="o">.</span><span class="n">path</span><span class="o">-</span><span class="n">style</span><span class="o">-</span><span class="n">access</span> <span class="o">=</span> <span class="n">true</span> <span class="p">(</span><span clas [...]
+</pre></div>
+</div>
+<p>If you hit <code class="docutils literal notranslate"><span class="pre">region</span> <span class="pre">is</span> <span class="pre">missing.</span> <span class="pre">Please</span> <span class="pre">find</span> <span class="pre">it</span> <span class="pre">by</span> <span class="pre">S3::detect_region()</span> <span class="pre">or</span> <span class="pre">set</span> <span class="pre">them</span> <span class="pre">in</span> <span class="pre">env</span></code>, this is the missing config.</p>
+</section>
+<section id="writing-a-bridge">
+<h2>Writing a bridge<a class="headerlink" href="#writing-a-bridge" title="Link to this heading">#</a></h2>
+<p>Comet’s native scan paths (<code class="docutils literal notranslate"><span class="pre">object_store</span></code> for raw Parquet, <code class="docutils literal notranslate"><span class="pre">opendal</span></code> via <code class="docutils literal notranslate"><span class="pre">iceberg-rust</span></code> for Iceberg) bypass Hadoop S3A entirely. The standard <code class="docutils literal notranslate"><span class="pre">AWSCredentialsProvider.getCredentials()</span></code> has no path a [...]
+<p>Implement <code class="docutils literal notranslate"><span class="pre">org.apache.comet.cloud.s3.CometS3CredentialProvider</span></code>:</p>
+<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="kn">package</span><span class="w"> </span><span class="nn">org.apache.comet.cloud.s3</span><span class="p">;</span>
+
+<span class="kd">public</span><span class="w"> </span><span class="kd">interface</span> <span class="nc">CometS3CredentialProvider</span><span class="w"> </span><span class="kd">extends</span><span class="w"> </span><span class="n">AutoCloseable</span><span class="w"> </span><span class="p">{</span>
+<span class="w"> </span><span class="cm">/** Called once per (FQCN, dispatchKey, catalogProperties) before any per-request call. Optional. */</span>
+<span class="w"> </span><span class="k">default</span><span class="w"> </span><span class="kt">void</span><span class="w"> </span><span class="nf">initialize</span><span class="p">(</span><span class="n">java</span><span class="p">.</span><span class="na">util</span><span class="p">.</span><span class="na">Map</span><span class="o"><</span><span class="n">String</span><span class="p">,</span><span class="w"> </span><span class="n">String</span><span class="o">></span><span class [...]
+
+<span class="w"> </span><span class="n">CometS3Credentials</span><span class="w"> </span><span class="nf">getCredentialsForPath</span><span class="p">(</span><span class="n">CometS3CredentialContext</span><span class="w"> </span><span class="n">context</span><span class="p">)</span><span class="w"> </span><span class="kd">throws</span><span class="w"> </span><span class="n">Exception</span><span class="p">;</span>
+
+<span class="w"> </span><span class="cm">/** Invoked from the dispatcher's JVM shutdown hook. Default is a no-op. */</span>
+<span class="w"> </span><span class="nd">@Override</span><span class="w"> </span><span class="k">default</span><span class="w"> </span><span class="kt">void</span><span class="w"> </span><span class="nf">close</span><span class="p">()</span><span class="w"> </span><span class="kd">throws</span><span class="w"> </span><span class="n">Exception</span><span class="w"> </span><span class="p">{}</span>
+<span class="p">}</span>
+</pre></div>
+</div>
+<p>The class must have a public no-arg constructor. <code class="docutils literal notranslate"><span class="pre">getCredentialsForPath</span></code> may be invoked concurrently from many native tokio worker threads; the implementation must be thread-safe. The supplied <code class="docutils literal notranslate"><span class="pre">CometS3CredentialContext</span></code> exposes <code class="docutils literal notranslate"><span class="pre">getBucket()</span></code>, <code class="docutils liter [...]
+<section id="lifecycle">
+<h3>Lifecycle<a class="headerlink" href="#lifecycle" title="Link to this heading">#</a></h3>
+<p>Comet keys provider instances by <code class="docutils literal notranslate"><span class="pre">(FQCN,</span> <span class="pre">dispatchKey,</span> <span class="pre">catalogProperties)</span></code>. The dispatch key is the Spark V2 catalog name on the Iceberg path and the S3 bucket name on the Parquet path. The first time a given key is seen on an executor, Comet reflects the class, calls <code class="docutils literal notranslate"><span class="pre">initialize(Map)</span></code> exactly [...]
+<p><code class="docutils literal notranslate"><span class="pre">initialize</span></code> should be cheap and non-blocking. Defer real credential fetches (REST round-trips, STS calls) to the first <code class="docutils literal notranslate"><span class="pre">getCredentialsForPath</span></code> invocation. On the Iceberg path the supplied <code class="docutils literal notranslate"><span class="pre">catalogProperties</span></code> carries the unfiltered FileIO bag, including REST-vended fiel [...]
+<p><code class="docutils literal notranslate"><span class="pre">close()</span></code> is invoked from a JVM shutdown hook installed by the dispatcher. The default no-op is fine for stateless providers. Override it to release HTTP clients, scheduled-refresh executors, or STS connection pools. Shutdown hooks are best-effort: a <code class="docutils literal notranslate"><span class="pre">SIGKILL</span></code> or abrupt JVM termination skips them, so do not depend on <code class="docutils li [...]
+</section>
+<section id="caching-refresh-and-distribution">
+<h3>Caching, refresh, and distribution<a class="headerlink" href="#caching-refresh-and-distribution" title="Link to this heading">#</a></h3>
+<p>Comet does not maintain a TTL cache, broadcast catalog state, or schedule refresh. Vendors decide:</p>
+<ul class="simple">
+<li><p>Whether to cache credentials and for how long. Iceberg vendors get <code class="docutils literal notranslate"><span class="pre">software.amazon.awssdk.utils.cache.CachedSupplier</span></code> for free inside <code class="docutils literal notranslate"><span class="pre">VendedCredentialsProvider</span></code>; vendors with custom STS write whatever cache fits.</p></li>
+<li><p>When to refresh: proactive timer, on-demand at expiry, on <code class="docutils literal notranslate"><span class="pre">403</span></code> retry, etc.</p></li>
+<li><p>How to distribute and refresh state. <code class="docutils literal notranslate"><span class="pre">catalogProperties</span></code> is a plan-time snapshot. Comet does not re-execute the catalog or push fresh values to running scans, so any state that must refresh during a scan is the vendor’s responsibility. Options: call back to a vendor service from the executor on each <code class="docutils literal notranslate"><span class="pre">getCredentialsForPath</span></code>, run your own [...]
+</ul>
+<p>When using <code class="docutils literal notranslate"><span class="pre">HadoopDelegationTokenProvider</span></code>, Spark mints and renews the token on the driver and propagates it to executors via <code class="docutils literal notranslate"><span class="pre">UserGroupInformation</span></code>. The provider reads the current value inside <code class="docutils literal notranslate"><span class="pre">getCredentialsForPath</span></code> and exchanges it for AWS credentials:</p>
+<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="nd">@Override</span>
+<span class="kd">public</span><span class="w"> </span><span class="n">CometS3Credentials</span><span class="w"> </span><span class="nf">getCredentialsForPath</span><span class="p">(</span><span class="n">CometS3CredentialContext</span><span class="w"> </span><span class="n">ctx</span><span class="p">)</span><span class="w"> </span><span class="kd">throws</span><span class="w"> </span><span class="n">Exception</span><span class="w"> </span><span class="p">{</span>
+<span class="w"> </span><span class="n">Token</span><span class="o"><?></span><span class="w"> </span><span class="n">token</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">UserGroupInformation</span><span class="p">.</span><span class="na">getCurrentUser</span><span class="p">()</span>
+<span class="w"> </span><span class="p">.</span><span class="na">getCredentials</span><span class="p">()</span>
+<span class="w"> </span><span class="p">.</span><span class="na">getToken</span><span class="p">(</span><span class="k">new</span><span class="w"> </span><span class="n">Text</span><span class="p">(</span><span class="s">"vendor-service"</span><span class="p">));</span>
+<span class="w"> </span><span class="k">return</span><span class="w"> </span><span class="n">vendorSts</span><span class="p">.</span><span class="na">exchangeForS3Credentials</span><span class="p">(</span><span class="n">token</span><span class="p">,</span><span class="w"> </span><span class="n">ctx</span><span class="p">.</span><span class="na">getBucket</span><span class="p">(),</span><span class="w"> </span><span class="n">ctx</span><span class="p">.</span><span class="na">getPath< [...]
+<span class="p">}</span>
+</pre></div>
+</div>
+<p>Spark delegation token propagation is supported on YARN and Kubernetes only. Standalone deployments need a different refresh path, typically a vendor-side service callback authenticated by long-lived state in <code class="docutils literal notranslate"><span class="pre">catalogProperties</span></code> or Hadoop conf.</p>
+<p><code class="docutils literal notranslate"><span class="pre">expirationEpochMillis</span></code> only matters on the Iceberg/<code class="docutils literal notranslate"><span class="pre">opendal</span></code> path. There the bridge implements <code class="docutils literal notranslate"><span class="pre">reqsign_core::ProvideCredential</span></code>, which carries an <code class="docutils literal notranslate"><span class="pre">expires_in</span></code> field that <code class="docutils lit [...]
+<p>The Parquet/<code class="docutils literal notranslate"><span class="pre">object_store</span></code> path has no expiry concept: <code class="docutils literal notranslate"><span class="pre">object_store::CredentialProvider</span></code> returns just <code class="docutils literal notranslate"><span class="pre">AwsCredential</span></code> (key/secret/token). The bridge is passed to <code class="docutils literal notranslate"><span class="pre">with_credentials</span></code> without a TTL w [...]
+</section>
+<section id="returned-fields">
+<h3>Returned fields<a class="headerlink" href="#returned-fields" title="Link to this heading">#</a></h3>
+<div class="pst-scrollable-table-container"><table class="table">
+<thead>
+<tr class="row-odd"><th class="head"><p>Field</p></th>
+<th class="head"><p>Notes</p></th>
+</tr>
+</thead>
+<tbody>
+<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">accessKeyId</span></code></p></td>
+<td><p>Required.</p></td>
+</tr>
+<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">secretAccessKey</span></code></p></td>
+<td><p>Required.</p></td>
+</tr>
+<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">sessionToken</span></code></p></td>
+<td><p><code class="docutils literal notranslate"><span class="pre">null</span></code> for non-STS credentials.</p></td>
+</tr>
+<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">expirationEpochMillis</span></code></p></td>
+<td><p>Iceberg path only. <code class="docutils literal notranslate"><span class="pre">0</span></code> means “unknown”; the bridge substitutes a 5-minute expiry. The Parquet path has no expiry concept.</p></td>
+</tr>
+</tbody>
+</table>
+</div>
+<p>Provide a real <code class="docutils literal notranslate"><span class="pre">expirationEpochMillis</span></code> whenever you have one on the Iceberg path. The Parquet path’s <code class="docutils literal notranslate"><span class="pre">object_store::CredentialProvider</span></code> does not consume an expiry, and the bridge invokes the SPI on every <code class="docutils literal notranslate"><span class="pre">get_credential()</span></code> call.</p>
+</section>
+<section id="returns-or-throws">
+<h3>Returns or throws<a class="headerlink" href="#returns-or-throws" title="Link to this heading">#</a></h3>
+<p>The SPI follows the same shape as the other JVM AWS-credential SPIs (AWS SDK v1/v2, Hadoop S3A, Iceberg <code class="docutils literal notranslate"><span class="pre">VendedCredentialsProvider</span></code>): return credentials or throw. There is no “fall-through” return value.</p>
+<p>If your provider is authoritative only for some paths, resolve the default AWS chain yourself for the rest:</p>
+<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="kd">private</span><span class="w"> </span><span class="kd">final</span><span class="w"> </span><span class="n">DefaultCredentialsProvider</span><span class="w"> </span><span class="n">defaultChain</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">DefaultCredentialsProvider</span><span class="p">.</span><span class="na">create</span><span class="p">();</span>
+
+<span class="nd">@Override</span>
+<span class="kd">public</span><span class="w"> </span><span class="n">CometS3Credentials</span><span class="w"> </span><span class="nf">getCredentialsForPath</span><span class="p">(</span><span class="n">CometS3CredentialContext</span><span class="w"> </span><span class="n">ctx</span><span class="p">)</span><span class="w"> </span><span class="kd">throws</span><span class="w"> </span><span class="n">Exception</span><span class="w"> </span><span class="p">{</span>
+<span class="w"> </span><span class="k">if</span><span class="w"> </span><span class="p">(</span><span class="n">handlesPath</span><span class="p">(</span><span class="n">ctx</span><span class="p">.</span><span class="na">getBucket</span><span class="p">(),</span><span class="w"> </span><span class="n">ctx</span><span class="p">.</span><span class="na">getPath</span><span class="p">()))</span><span class="w"> </span><span class="p">{</span>
+<span class="w"> </span><span class="k">return</span><span class="w"> </span><span class="n">mintFromMyVendorService</span><span class="p">(</span><span class="n">ctx</span><span class="p">.</span><span class="na">getBucket</span><span class="p">(),</span><span class="w"> </span><span class="n">ctx</span><span class="p">.</span><span class="na">getPath</span><span class="p">(),</span><span class="w"> </span><span class="n">ctx</span><span class="p">.</span><span class="na">getMode [...]
+<span class="w"> </span><span class="p">}</span>
+<span class="w"> </span><span class="n">AwsCredentials</span><span class="w"> </span><span class="n">c</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">defaultChain</span><span class="p">.</span><span class="na">resolveCredentials</span><span class="p">();</span>
+<span class="w"> </span><span class="n">String</span><span class="w"> </span><span class="n">token</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="p">(</span><span class="n">c</span><span class="w"> </span><span class="k">instanceof</span><span class="w"> </span><span class="n">AwsSessionCredentials</span><span class="p">)</span>
+<span class="w"> </span><span class="o">?</span><span class="w"> </span><span class="p">((</span><span class="n">AwsSessionCredentials</span><span class="p">)</span><span class="w"> </span><span class="n">c</span><span class="p">).</span><span class="na">sessionToken</span><span class="p">()</span>
+<span class="w"> </span><span class="p">:</span><span class="w"> </span><span class="kc">null</span><span class="p">;</span>
+<span class="w"> </span><span class="k">return</span><span class="w"> </span><span class="k">new</span><span class="w"> </span><span class="n">CometS3Credentials</span><span class="p">(</span><span class="n">c</span><span class="p">.</span><span class="na">accessKeyId</span><span class="p">(),</span><span class="w"> </span><span class="n">c</span><span class="p">.</span><span class="na">secretAccessKey</span><span class="p">(),</span><span class="w"> </span><span class="n">token</span [...]
+<span class="p">}</span>
+</pre></div>
+</div>
+</section>
+<section id="composing-multiple-credential-backends">
+<h3>Composing multiple credential backends<a class="headerlink" href="#composing-multiple-credential-backends" title="Link to this heading">#</a></h3>
+<p>A single configured provider class is the dispatcher. If a vendor needs to route across several credential backends (per bucket, per path prefix, per tenant), the dispatch lives inside the vendor’s class:</p>
+<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="kd">public</span><span class="w"> </span><span class="kd">final</span><span class="w"> </span><span class="kd">class</span> <span class="nc">MyCometCredentialProvider</span><span class="w"> </span><span class="kd">implements</span><span class="w"> </span><span class="n">CometS3CredentialProvider</span><span class="w"> </span><span class="p">{</span>
+<span class="w"> </span><span class="kd">private</span><span class="w"> </span><span class="kd">final</span><span class="w"> </span><span class="n">ProdVendor</span><span class="w"> </span><span class="n">prod</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="p">...;</span>
+<span class="w"> </span><span class="kd">private</span><span class="w"> </span><span class="kd">final</span><span class="w"> </span><span class="n">StsVendor</span><span class="w"> </span><span class="n">sts</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="p">...;</span>
+<span class="w"> </span><span class="kd">private</span><span class="w"> </span><span class="kd">final</span><span class="w"> </span><span class="n">DefaultVendor</span><span class="w"> </span><span class="n">fallback</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="p">...;</span>
+
+<span class="w"> </span><span class="nd">@Override</span>
+<span class="w"> </span><span class="kd">public</span><span class="w"> </span><span class="n">CometS3Credentials</span><span class="w"> </span><span class="nf">getCredentialsForPath</span><span class="p">(</span><span class="n">CometS3CredentialContext</span><span class="w"> </span><span class="n">ctx</span><span class="p">)</span><span class="w"> </span><span class="kd">throws</span><span class="w"> </span><span class="n">Exception</span><span class="w"> </span><span class="p">{</span>
+<span class="w"> </span><span class="k">if</span><span class="w"> </span><span class="p">(</span><span class="n">ctx</span><span class="p">.</span><span class="na">getBucket</span><span class="p">().</span><span class="na">startsWith</span><span class="p">(</span><span class="s">"data-prod-"</span><span class="p">))</span><span class="w"> </span><span class="k">return</span><span class="w"> </span><span class="n">prod</span><span class="p">.</span><span class="na">fetc [...]
+<span class="w"> </span><span class="k">if</span><span class="w"> </span><span class="p">(</span><span class="n">ctx</span><span class="p">.</span><span class="na">getBucket</span><span class="p">().</span><span class="na">equals</span><span class="p">(</span><span class="s">"partner-shared"</span><span class="p">))</span><span class="w"> </span><span class="k">return</span><span class="w"> </span><span class="n">sts</span><span class="p">.</span><span class="na">assum [...]
+<span class="w"> </span><span class="k">return</span><span class="w"> </span><span class="n">fallback</span><span class="p">.</span><span class="na">fetch</span><span class="p">(</span><span class="n">ctx</span><span class="p">.</span><span class="na">getBucket</span><span class="p">(),</span><span class="w"> </span><span class="n">ctx</span><span class="p">.</span><span class="na">getPath</span><span class="p">());</span>
+<span class="w"> </span><span class="p">}</span>
+<span class="p">}</span>
+</pre></div>
+</div>
+<p>Per-bucket Hadoop overrides (<code class="docutils literal notranslate"><span class="pre">fs.s3a.bucket.<name>.comet.credential.provider.class</span></code>) are also available if you prefer to ship multiple vendor classes and pick by bucket in config rather than in code.</p>
+<p>For Iceberg deployments where two catalogs share one provider class but need isolated state, configure the same FQCN on both catalogs and read your discriminator from <code class="docutils literal notranslate"><span class="pre">initialize</span></code>’s <code class="docutils literal notranslate"><span class="pre">catalogProperties</span></code>. Each catalog gets its own provider instance because Comet keys by <code class="docutils literal notranslate"><span class="pre">(FQCN,</span> [...]
+<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="kd">public</span><span class="w"> </span><span class="kd">final</span><span class="w"> </span><span class="kd">class</span> <span class="nc">MyMultiTenantProvider</span><span class="w"> </span><span class="kd">implements</span><span class="w"> </span><span class="n">CometS3CredentialProvider</span><span class="w"> </span><span class="p">{</span>
+<span class="w"> </span><span class="kd">private</span><span class="w"> </span><span class="kd">volatile</span><span class="w"> </span><span class="n">String</span><span class="w"> </span><span class="n">tenantId</span><span class="p">;</span>
+
+<span class="w"> </span><span class="nd">@Override</span>
+<span class="w"> </span><span class="kd">public</span><span class="w"> </span><span class="kt">void</span><span class="w"> </span><span class="nf">initialize</span><span class="p">(</span><span class="n">Map</span><span class="o"><</span><span class="n">String</span><span class="p">,</span><span class="w"> </span><span class="n">String</span><span class="o">></span><span class="w"> </span><span class="n">catalogProperties</span><span class="p">)</span><span class="w"> </span><sp [...]
+<span class="w"> </span><span class="k">this</span><span class="p">.</span><span class="na">tenantId</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">catalogProperties</span><span class="p">.</span><span class="na">get</span><span class="p">(</span><span class="s">"vendor.tenant-id"</span><span class="p">);</span>
+<span class="w"> </span><span class="p">}</span>
+
+<span class="w"> </span><span class="nd">@Override</span>
+<span class="w"> </span><span class="kd">public</span><span class="w"> </span><span class="n">CometS3Credentials</span><span class="w"> </span><span class="nf">getCredentialsForPath</span><span class="p">(</span><span class="n">CometS3CredentialContext</span><span class="w"> </span><span class="n">ctx</span><span class="p">)</span><span class="w"> </span><span class="p">{</span>
+<span class="w"> </span><span class="k">return</span><span class="w"> </span><span class="n">mintForTenant</span><span class="p">(</span><span class="n">tenantId</span><span class="p">,</span><span class="w"> </span><span class="n">ctx</span><span class="p">.</span><span class="na">getBucket</span><span class="p">(),</span><span class="w"> </span><span class="n">ctx</span><span class="p">.</span><span class="na">getPath</span><span class="p">(),</span><span class="w"> </span><span [...]
+<span class="w"> </span><span class="p">}</span>
+<span class="p">}</span>
+</pre></div>
+</div>
+</section>
+<section id="reference-implementation-iceberg-rest-vended-credentials">
+<h3>Reference implementation: Iceberg REST vended credentials<a class="headerlink" href="#reference-implementation-iceberg-rest-vended-credentials" title="Link to this heading">#</a></h3>
+<p>For Iceberg REST catalogs that vend AWS credentials (<code class="docutils literal notranslate"><span class="pre">LoadTableResponse.credentials</span></code>), the canonical implementation wraps Iceberg’s existing <code class="docutils literal notranslate"><span class="pre">VendedCredentialsProvider</span></code>:</p>
+<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="kd">public</span><span class="w"> </span><span class="kd">final</span><span class="w"> </span><span class="kd">class</span> <span class="nc">IcebergRESTVendedS3Provider</span><span class="w"> </span><span class="kd">implements</span><span class="w"> </span><span class="n">CometS3CredentialProvider</span><span class="w"> </span><span class="p">{</span>
+<span class="w"> </span><span class="kd">private</span><span class="w"> </span><span class="kd">volatile</span><span class="w"> </span><span class="n">VendedCredentialsProvider</span><span class="w"> </span><span class="n">provider</span><span class="p">;</span>
+
+<span class="w"> </span><span class="nd">@Override</span>
+<span class="w"> </span><span class="kd">public</span><span class="w"> </span><span class="kt">void</span><span class="w"> </span><span class="nf">initialize</span><span class="p">(</span><span class="n">Map</span><span class="o"><</span><span class="n">String</span><span class="p">,</span><span class="w"> </span><span class="n">String</span><span class="o">></span><span class="w"> </span><span class="n">catalogProperties</span><span class="p">)</span><span class="w"> </span><sp [...]
+<span class="w"> </span><span class="k">this</span><span class="p">.</span><span class="na">provider</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">VendedCredentialsProvider</span><span class="p">.</span><span class="na">create</span><span class="p">(</span><span class="n">catalogProperties</span><span class="p">);</span>
+<span class="w"> </span><span class="p">}</span>
+
+<span class="w"> </span><span class="nd">@Override</span>
+<span class="w"> </span><span class="kd">public</span><span class="w"> </span><span class="n">CometS3Credentials</span><span class="w"> </span><span class="nf">getCredentialsForPath</span><span class="p">(</span><span class="n">CometS3CredentialContext</span><span class="w"> </span><span class="n">ctx</span><span class="p">)</span><span class="w"> </span><span class="p">{</span>
+<span class="w"> </span><span class="n">AwsCredentials</span><span class="w"> </span><span class="n">c</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">provider</span><span class="p">.</span><span class="na">resolveCredentials</span><span class="p">();</span>
+<span class="w"> </span><span class="n">String</span><span class="w"> </span><span class="n">token</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="p">(</span><span class="n">c</span><span class="w"> </span><span class="k">instanceof</span><span class="w"> </span><span class="n">AwsSessionCredentials</span><span class="p">)</span>
+<span class="w"> </span><span class="o">?</span><span class="w"> </span><span class="p">((</span><span class="n">AwsSessionCredentials</span><span class="p">)</span><span class="w"> </span><span class="n">c</span><span class="p">).</span><span class="na">sessionToken</span><span class="p">()</span><span class="w"> </span><span class="p">:</span><span class="w"> </span><span class="kc">null</span><span class="p">;</span>
+<span class="w"> </span><span class="k">return</span><span class="w"> </span><span class="k">new</span><span class="w"> </span><span class="n">CometS3Credentials</span><span class="p">(</span><span class="n">c</span><span class="p">.</span><span class="na">accessKeyId</span><span class="p">(),</span><span class="w"> </span><span class="n">c</span><span class="p">.</span><span class="na">secretAccessKey</span><span class="p">(),</span><span class="w"> </span><span class="n">token</ [...]
+<span class="w"> </span><span class="p">}</span>
+<span class="p">}</span>
+</pre></div>
+</div>
+<p><code class="docutils literal notranslate"><span class="pre">VendedCredentialsProvider</span></code> reads <code class="docutils literal notranslate"><span class="pre">credentials.uri</span></code>, the catalog endpoint, and OAuth tokens from the supplied map (Comet forwards the unfiltered FileIO bag to <code class="docutils literal notranslate"><span class="pre">initialize</span></code>), and refreshes through its own <code class="docutils literal notranslate"><span class="pre">Cache [...]
+</section>
+<section id="access-mode">
+<h3>Access mode<a class="headerlink" href="#access-mode" title="Link to this heading">#</a></h3>
+<div class="pst-scrollable-table-container"><table class="table">
+<thead>
+<tr class="row-odd"><th class="head"><p>Value</p></th>
+<th class="head"><p>Used for</p></th>
+</tr>
+</thead>
+<tbody>
+<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">READ</span></code></p></td>
+<td><p>All native scan paths (raw Parquet, Iceberg). Comet today only sends READ.</p></td>
+</tr>
+<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">WRITE</span></code></p></td>
+<td><p>Reserved for future native write paths.</p></td>
+</tr>
+</tbody>
+</table>
+</div>
+<p>A <code class="docutils literal notranslate"><span class="pre">WRITE</span></code> credential is not implicitly read-capable. Vendors that need read-during-write workflows include the required read permissions in the IAM policy attached to their <code class="docutils literal notranslate"><span class="pre">WRITE</span></code> credentials.</p>
+</section>
+<section id="build-setup">
+<h3>Build setup<a class="headerlink" href="#build-setup" title="Link to this heading">#</a></h3>
+<p>Vendor implementations need the Comet SPI classes at compile time only. Use <code class="docutils literal notranslate"><span class="pre">provided</span></code>-scope:</p>
+<div class="highlight-xml notranslate"><div class="highlight"><pre><span></span><span class="nt"><dependency></span>
+<span class="w"> </span><span class="nt"><groupId></span>org.apache.datafusion<span class="nt"></groupId></span>
+<span class="w"> </span><span class="nt"><artifactId></span>comet-spark-spark${spark.version.short}_${scala.binary.version}<span class="nt"></artifactId></span>
+<span class="w"> </span><span class="nt"><version></span>${comet.version}<span class="nt"></version></span>
+<span class="w"> </span><span class="nt"><scope></span>provided<span class="nt"></scope></span>
+<span class="nt"></dependency></span>
+</pre></div>
+</div>
+</section>
+<section id="iceberg-path-error-message-fidelity">
+<h3>Iceberg path: error message fidelity<a class="headerlink" href="#iceberg-path-error-message-fidelity" title="Link to this heading">#</a></h3>
+<p>When the bridge is wired into <code class="docutils literal notranslate"><span class="pre">iceberg-rust</span></code>, the outer <code class="docutils literal notranslate"><span class="pre">reqsign-core::ProvideCredentialChain</span></code> currently swallows thrown exceptions into “no credential” before the request reaches <code class="docutils literal notranslate"><span class="pre">opendal</span></code>. The credential is still not issued and the request still fails, only the messag [...]
+</section>
+</section>
+</section>
+
+
+ </article>
+
+
+
+
+
+ <footer class="prev-next-footer d-print-none">
+
+<div class="prev-next-area">
+ <a class="left-prev"
+ href="iceberg.html"
+ title="previous page">
+ <i class="fa-solid fa-angle-left"></i>
+ <div class="prev-next-info">
+ <p class="prev-next-subtitle">previous</p>
+ <p class="prev-next-title">Accelerating Apache Iceberg Parquet Scans using Comet</p>
+ </div>
+ </a>
+ <a class="right-next"
+ href="kubernetes.html"
+ title="next page">
+ <div class="prev-next-info">
+ <p class="prev-next-subtitle">next</p>
+ <p class="prev-next-title">Comet Kubernetes Support</p>
+ </div>
+ <i class="fa-solid fa-angle-right"></i>
+ </a>
+</div>
+ </footer>
+
+ </div>
+
+
+
+
+ </div>
+ <footer class="bd-footer-content">
+
+ </footer>
+
+ </main>
+ </div>
+ </div>
+
+ <!-- Scripts loaded after <body> so the DOM is not blocked -->
+ <script defer src="../../_static/scripts/bootstrap.js?digest=8878045cc6db502f8baf"></script>
+<script defer src="../../_static/scripts/pydata-sphinx-theme.js?digest=8878045cc6db502f8baf"></script>
+
+<!-- Based on pydata_sphinx_theme/footer.html -->
+<footer class="footer mt-5 mt-md-0">
+ <div class="container">
+
+ <div class="footer-item">
+ <p>Apache DataFusion, Apache DataFusion Comet, Apache, the Apache feather logo, and the Apache DataFusion project logo</p>
+ <p>are either registered trademarks or trademarks of The Apache Software Foundation in the United States and other countries.</p>
+ </div>
+ </div>
+</footer>
+
+
+ </body>
+</html>
\ No newline at end of file
diff --git a/user-guide/latest/scala_java_udfs.html b/user-guide/latest/scala_java_udfs.html
index 3bf7928efa..237f1e7a45 100644
--- a/user-guide/latest/scala_java_udfs.html
+++ b/user-guide/latest/scala_java_udfs.html
@@ -434,6 +434,7 @@ under the License.
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Integrations</span></p>
<ul class="nav bd-sidenav">
<li class="toctree-l1"><a class="reference internal" href="iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l1"><a class="reference internal" href="s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l1"><a class="reference internal" href="kubernetes.html">Kubernetes Guide</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Advanced</span></p>
diff --git a/user-guide/latest/source.html b/user-guide/latest/source.html
index c908d0eee9..1f97a71225 100644
--- a/user-guide/latest/source.html
+++ b/user-guide/latest/source.html
@@ -434,6 +434,7 @@ under the License.
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Integrations</span></p>
<ul class="nav bd-sidenav">
<li class="toctree-l1"><a class="reference internal" href="iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l1"><a class="reference internal" href="s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l1"><a class="reference internal" href="kubernetes.html">Kubernetes Guide</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Advanced</span></p>
diff --git a/user-guide/latest/tuning.html b/user-guide/latest/tuning.html
index cf07a0e29a..7963f31cd8 100644
--- a/user-guide/latest/tuning.html
+++ b/user-guide/latest/tuning.html
@@ -434,6 +434,7 @@ under the License.
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Integrations</span></p>
<ul class="nav bd-sidenav">
<li class="toctree-l1"><a class="reference internal" href="iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l1"><a class="reference internal" href="s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l1"><a class="reference internal" href="kubernetes.html">Kubernetes Guide</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Advanced</span></p>
diff --git a/user-guide/latest/understanding-comet-plans.html b/user-guide/latest/understanding-comet-plans.html
index ba716a81e7..9e0ddf6ca7 100644
--- a/user-guide/latest/understanding-comet-plans.html
+++ b/user-guide/latest/understanding-comet-plans.html
@@ -434,6 +434,7 @@ under the License.
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Integrations</span></p>
<ul class="nav bd-sidenav">
<li class="toctree-l1"><a class="reference internal" href="iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l1"><a class="reference internal" href="s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l1"><a class="reference internal" href="kubernetes.html">Kubernetes Guide</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Advanced</span></p>
diff --git a/user-guide/older-versions.html b/user-guide/older-versions.html
index e8d0c3a8cb..7a783fe136 100644
--- a/user-guide/older-versions.html
+++ b/user-guide/older-versions.html
@@ -458,6 +458,7 @@ under the License.
<li class="toctree-l2"><a class="reference internal" href="latest/tuning.html">Tuning Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="latest/metrics.html">Metrics Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="latest/iceberg.html">Iceberg Guide</a></li>
+<li class="toctree-l2"><a class="reference internal" href="latest/s3-credential-providers.html">S3 Credential Providers</a></li>
<li class="toctree-l2"><a class="reference internal" href="latest/kubernetes.html">Kubernetes Guide</a></li>
<li class="toctree-l2"><a class="reference internal" href="latest/source.html">Building From Source</a></li>
</ul>
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org