You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@polaris.apache.org by nk...@apache.org on 2026/07/15 09:57:38 UTC
(polaris) branch main updated: Propagate storage HTTP client settings to S3FileIO for table operations (#5004)
This is an automated email from the ASF dual-hosted git repository.
nandorKollar pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/polaris.git
The following commit(s) were added to refs/heads/main by this push:
new 26a33dc9cd Propagate storage HTTP client settings to S3FileIO for table operations (#5004)
26a33dc9cd is described below
commit 26a33dc9cd13e572d2b4caeba1eb269043c62155
Author: Prithvi S <pr...@gmail.com>
AuthorDate: Wed Jul 15 15:27:33 2026 +0530
Propagate storage HTTP client settings to S3FileIO for table operations (#5004)
* Propagate storage HTTP client settings to S3FileIO for table operations
* Apply storage HTTP client settings on wasb FileIOFactory path
Wire S3AccessConfig into WasbTranslatingFileIOFactory so production
non-default factories also propagate polaris.storage.* HTTP settings
to S3FileIO. Clarify that the no-arg DefaultFileIOFactory path is for
tests and fixtures only.
* review changes
Address review feedback: use S3AccessConfig.empty() via @PolarisImmutable
instead of null, rename loadFileIO tableProperties param, and reuse Iceberg
HttpClientProperties constants. Keep regenerated polaris.storage config docs
in the same commit for CI.
---
CHANGELOG.md | 1 +
.../service/catalog/io/DefaultFileIOFactory.java | 63 ++++++++++++++++++++--
.../catalog/io/WasbTranslatingFileIOFactory.java | 5 +-
.../service/storage/aws/S3AccessConfig.java | 11 +++-
.../service/catalog/io/FileIOFactoryTest.java | 3 +-
.../service/catalog/io/MeasuredFileIOFactory.java | 3 +-
.../config-sections/smallrye-polaris_storage.md | 2 +-
7 files changed, 78 insertions(+), 10 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 396f34013a..1debe47062 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -102,6 +102,7 @@ request adding CHANGELOG notes for breaking (!) changes and possibly other secti
- The token broker now builds the JWT `Algorithm` and `JWTVerifier` once per realm in the `TokenBrokerFactory` and reuses them across requests, instead of rebuilding them on every `verify()`/`sign()` call on the request-scoped broker. For deployments using file-based symmetric secrets, the secret is now read once per realm (at first use) rather than on every JWT operation; rotating the on-disk secret requires a restart.
### Fixes
+- `polaris.storage.max-http-connections` (and related read/connect/acquisition/idle timeouts) now take effect for Iceberg S3FileIO clients used in table operations (previously only affected the STS client pool).
- Fixed a boundary condition in GCS downscoped credential generation (`GcpCredentialsStorageIntegration`). Locations without a trailing slash could previously grant access to sibling object prefixes via the generated CEL conditions for `resource.name` and list prefixes. Granted paths are now normalized to a directory prefix (with a trailing slash) before the CEL conditions are built, so sibling prefixes can no longer satisfy the `startsWith` checks.
- Fixed `NullPointerException` during `dropEntity` when an entity referenced by a grant had been concurrently removed (or purged). `lookupEntities` can return null entries for dropped entities; these are now skipped safely.
- `RateLimiterFilter` now returns an Iceberg-compatible `ErrorResponse` JSON body on HTTP 429, with `Content-Type: application/json`. Previously the body was empty, causing Iceberg REST clients to surface an opaque error.
diff --git a/runtime/service/src/main/java/org/apache/polaris/service/catalog/io/DefaultFileIOFactory.java b/runtime/service/src/main/java/org/apache/polaris/service/catalog/io/DefaultFileIOFactory.java
index cf334b2afd..99459ee854 100644
--- a/runtime/service/src/main/java/org/apache/polaris/service/catalog/io/DefaultFileIOFactory.java
+++ b/runtime/service/src/main/java/org/apache/polaris/service/catalog/io/DefaultFileIOFactory.java
@@ -25,8 +25,10 @@ import jakarta.inject.Inject;
import java.util.HashMap;
import java.util.Map;
import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.aws.HttpClientProperties;
import org.apache.iceberg.io.FileIO;
import org.apache.polaris.core.storage.StorageAccessConfig;
+import org.apache.polaris.service.storage.aws.S3AccessConfig;
import org.jspecify.annotations.NonNull;
/**
@@ -35,22 +37,30 @@ import org.jspecify.annotations.NonNull;
*
* <p>This class acts as a translation layer between Polaris properties and the properties required
* by Iceberg's {@link FileIO}.
+ *
+ * <p>{@code polaris.storage.*} HTTP client settings from {@link S3AccessConfig} are applied to
+ * Iceberg AWS FileIOs. Production CDI paths inject the live config; tests/fixtures can pass {@link
+ * S3AccessConfig#empty()}.
*/
@RequestScoped
@Identifier("default")
public class DefaultFileIOFactory implements FileIOFactory {
+ private final S3AccessConfig s3AccessConfig;
+
@Inject
- public DefaultFileIOFactory() {}
+ public DefaultFileIOFactory(S3AccessConfig s3AccessConfig) {
+ this.s3AccessConfig = s3AccessConfig;
+ }
@Override
public FileIO loadFileIO(
@NonNull StorageAccessConfig storageAccessConfig,
@NonNull String ioImplClassName,
- @NonNull Map<String, String> properties) {
+ @NonNull Map<String, String> tableProperties) {
// Get subcoped creds
- properties = new HashMap<>(properties);
+ Map<String, String> properties = new HashMap<>(tableProperties);
// Update the FileIO with the subscoped credentials
// Update with properties in case there are table-level overrides the credentials should
@@ -60,6 +70,53 @@ public class DefaultFileIOFactory implements FileIOFactory {
properties.putAll(storageAccessConfig.extraProperties());
properties.putAll(storageAccessConfig.internalProperties());
+ // Apply polaris.storage.* HTTP client settings to Iceberg S3FileIO (and other AWS FileIOs).
+ // Previously only applied to the STS client pool. Empty config is a no-op (tests/fixtures).
+ s3AccessConfig
+ .maxHttpConnections()
+ .ifPresent(
+ v -> properties.put(HttpClientProperties.APACHE_MAX_CONNECTIONS, String.valueOf(v)));
+ s3AccessConfig
+ .readTimeout()
+ .ifPresent(
+ d ->
+ properties.put(
+ HttpClientProperties.APACHE_SOCKET_TIMEOUT_MS, String.valueOf(d.toMillis())));
+ s3AccessConfig
+ .connectTimeout()
+ .ifPresent(
+ d ->
+ properties.put(
+ HttpClientProperties.APACHE_CONNECTION_TIMEOUT_MS,
+ String.valueOf(d.toMillis())));
+ s3AccessConfig
+ .connectionAcquisitionTimeout()
+ .ifPresent(
+ d ->
+ properties.put(
+ HttpClientProperties.APACHE_CONNECTION_ACQUISITION_TIMEOUT_MS,
+ String.valueOf(d.toMillis())));
+ s3AccessConfig
+ .connectionMaxIdleTime()
+ .ifPresent(
+ d ->
+ properties.put(
+ HttpClientProperties.APACHE_CONNECTION_MAX_IDLE_TIME_MS,
+ String.valueOf(d.toMillis())));
+ s3AccessConfig
+ .connectionTimeToLive()
+ .ifPresent(
+ d ->
+ properties.put(
+ HttpClientProperties.APACHE_CONNECTION_TIME_TO_LIVE_MS,
+ String.valueOf(d.toMillis())));
+ s3AccessConfig
+ .expectContinueEnabled()
+ .ifPresent(
+ v ->
+ properties.put(
+ HttpClientProperties.APACHE_EXPECT_CONTINUE_ENABLED, String.valueOf(v)));
+
return loadFileIOInternal(ioImplClassName, properties);
}
diff --git a/runtime/service/src/main/java/org/apache/polaris/service/catalog/io/WasbTranslatingFileIOFactory.java b/runtime/service/src/main/java/org/apache/polaris/service/catalog/io/WasbTranslatingFileIOFactory.java
index cb11456160..d5f98ee223 100644
--- a/runtime/service/src/main/java/org/apache/polaris/service/catalog/io/WasbTranslatingFileIOFactory.java
+++ b/runtime/service/src/main/java/org/apache/polaris/service/catalog/io/WasbTranslatingFileIOFactory.java
@@ -24,6 +24,7 @@ import jakarta.inject.Inject;
import java.util.Map;
import org.apache.iceberg.io.FileIO;
import org.apache.polaris.core.storage.StorageAccessConfig;
+import org.apache.polaris.service.storage.aws.S3AccessConfig;
import org.jspecify.annotations.NonNull;
/** A {@link FileIOFactory} that translates WASB paths to ABFS ones */
@@ -34,8 +35,8 @@ public class WasbTranslatingFileIOFactory implements FileIOFactory {
private final FileIOFactory defaultFileIOFactory;
@Inject
- public WasbTranslatingFileIOFactory() {
- defaultFileIOFactory = new DefaultFileIOFactory();
+ public WasbTranslatingFileIOFactory(S3AccessConfig s3AccessConfig) {
+ defaultFileIOFactory = new DefaultFileIOFactory(s3AccessConfig);
}
@Override
diff --git a/runtime/service/src/main/java/org/apache/polaris/service/storage/aws/S3AccessConfig.java b/runtime/service/src/main/java/org/apache/polaris/service/storage/aws/S3AccessConfig.java
index 940d03229a..935770be80 100644
--- a/runtime/service/src/main/java/org/apache/polaris/service/storage/aws/S3AccessConfig.java
+++ b/runtime/service/src/main/java/org/apache/polaris/service/storage/aws/S3AccessConfig.java
@@ -22,18 +22,25 @@ package org.apache.polaris.service.storage.aws;
import java.time.Duration;
import java.util.Optional;
import java.util.OptionalInt;
+import org.apache.polaris.immutables.PolarisImmutable;
/**
* Configuration interface containing parameters for clients accessing S3 services from Polaris
* servers.
*
- * <p>Currently, this configuration does not apply to all of Polaris code, but only to select
- * services.
+ * <p>Applies to the STS client pool and to Iceberg S3 clients created for table operations via
+ * {@code S3FileIO}.
*/
+@PolarisImmutable
public interface S3AccessConfig {
/** Default value for {@link #clientsCacheMaxSize()}. */
int DEFAULT_MAX_STS_CLIENT_CACHE_ENTRIES = 50;
+ /** An empty config with no HTTP client overrides (e.g. tests and fixtures). */
+ static S3AccessConfig empty() {
+ return ImmutableS3AccessConfig.builder().build();
+ }
+
/** Maximum number of entries to keep in the STS clients cache. */
OptionalInt clientsCacheMaxSize();
diff --git a/runtime/service/src/test/java/org/apache/polaris/service/catalog/io/FileIOFactoryTest.java b/runtime/service/src/test/java/org/apache/polaris/service/catalog/io/FileIOFactoryTest.java
index f52503d43c..0a48ec941f 100644
--- a/runtime/service/src/test/java/org/apache/polaris/service/catalog/io/FileIOFactoryTest.java
+++ b/runtime/service/src/test/java/org/apache/polaris/service/catalog/io/FileIOFactoryTest.java
@@ -48,6 +48,7 @@ import org.apache.polaris.core.persistence.pagination.PageToken;
import org.apache.polaris.service.TestServices;
import org.apache.polaris.service.catalog.PolarisPassthroughResolutionView;
import org.apache.polaris.service.catalog.iceberg.LocalIcebergCatalog;
+import org.apache.polaris.service.storage.aws.S3AccessConfig;
import org.apache.polaris.service.task.TaskFileIOSupplier;
import org.assertj.core.api.Assertions;
import org.jspecify.annotations.NonNull;
@@ -105,7 +106,7 @@ public class FileIOFactoryTest {
Supplier<FileIOFactory> fileIOFactorySupplier =
() ->
Mockito.spy(
- new DefaultFileIOFactory() {
+ new DefaultFileIOFactory(S3AccessConfig.empty()) {
@Override
FileIO loadFileIOInternal(
@NonNull String ioImplClassName, @NonNull Map<String, String> properties) {
diff --git a/runtime/service/src/testFixtures/java/org/apache/polaris/service/catalog/io/MeasuredFileIOFactory.java b/runtime/service/src/testFixtures/java/org/apache/polaris/service/catalog/io/MeasuredFileIOFactory.java
index 29c29625ed..cbdce1d299 100644
--- a/runtime/service/src/testFixtures/java/org/apache/polaris/service/catalog/io/MeasuredFileIOFactory.java
+++ b/runtime/service/src/testFixtures/java/org/apache/polaris/service/catalog/io/MeasuredFileIOFactory.java
@@ -27,6 +27,7 @@ import java.util.Optional;
import java.util.function.Supplier;
import org.apache.iceberg.io.FileIO;
import org.apache.polaris.core.storage.StorageAccessConfig;
+import org.apache.polaris.service.storage.aws.S3AccessConfig;
import org.jspecify.annotations.NonNull;
/**
@@ -47,7 +48,7 @@ public class MeasuredFileIOFactory implements FileIOFactory {
@Inject
public MeasuredFileIOFactory() {
- defaultFileIOFactory = new DefaultFileIOFactory();
+ defaultFileIOFactory = new DefaultFileIOFactory(S3AccessConfig.empty());
}
@Override
diff --git a/site/content/in-dev/unreleased/configuration/config-sections/smallrye-polaris_storage.md b/site/content/in-dev/unreleased/configuration/config-sections/smallrye-polaris_storage.md
index 9e03916023..f7af9089a8 100644
--- a/site/content/in-dev/unreleased/configuration/config-sections/smallrye-polaris_storage.md
+++ b/site/content/in-dev/unreleased/configuration/config-sections/smallrye-polaris_storage.md
@@ -25,7 +25,7 @@ build:
Configuration interface containing parameters for clients accessing S3 services from Polaris servers.
-Currently, this configuration does not apply to all of Polaris code, but only to select services.
+Applies to the STS client pool and to Iceberg S3 clients created for table operations via `S3FileIO`.
| Property | Default Value | Type | Description |
|----------|---------------|------|-------------|