You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@polaris.apache.org by yu...@apache.org on 2026/10/26 17:30:12 UTC
(polaris) branch main updated: Apply allow-listed connection properties to HADOOP and HIVE federated catalogs (#5680)
This is an automated email from the ASF dual-hosted git repository.
flyrain pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/polaris.git
The following commit(s) were added to refs/heads/main by this push:
new 1af3c00b12 Apply allow-listed connection properties to HADOOP and HIVE federated catalogs (#5680)
1af3c00b12 is described below
commit 1af3c00b12b50cb1cf053bfae1b8d3aba5004e84
Author: youngrae kim <16...@users.noreply.github.com>
AuthorDate: Tue Oct 27 02:30:07 2026 +0900
Apply allow-listed connection properties to HADOOP and HIVE federated catalogs (#5680)
---
CHANGELOG.md | 5 +
.../core/connection/ConnectionConfigInfoDpo.java | 14 +++
.../hadoop/HadoopConnectionConfigInfoDpo.java | 17 ++++
.../hive/HiveConnectionConfigInfoDpo.java | 22 +++++
.../IcebergRestConnectionConfigInfoDpo.java | 6 +-
.../hadoop/HadoopConnectionConfigInfoDpoTest.java | 106 +++++++++++++++++++++
.../hive/HiveConnectionConfigInfoDpoTest.java | 103 ++++++++++++++++++++
.../federation/hive-metastore-federation.md | 22 +++++
8 files changed, 290 insertions(+), 5 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 96d0023b67..1d2b7f46e5 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -175,6 +175,11 @@ request adding CHANGELOG notes for breaking (!) changes and possibly other secti
- Catalog federation: `connectionConfigInfo.properties` is now persisted and returned for `HADOOP`
and `HIVE` connection configurations. It was previously accepted by the management API but
silently dropped, so `GET /catalogs/{name}` never showed it.
+- Catalog federation: `connectionConfigInfo.properties` is now applied when Polaris initializes
+ the Iceberg client for `HADOOP` and `HIVE` catalogs. As for `ICEBERG_REST`, only an explicit
+ allow-list is forwarded: `suppress-permission-error` (Hadoop), `clients` and `list-all-tables`
+ (Hive), plus the non-secret FileIO settings `s3.endpoint`,
+ `s3.path-style-access` and `client.region` for both. Other keys are stored but ignored.
- Semantic models: a `datasets` field that is not a JSON array now returns `400 Bad Request`
instead of being silently skipped, which bypassed every `dataset.source` check.
- Conditional `loadTable` (`If-None-Match` → HTTP 304) no longer attaches a null
diff --git a/polaris-core/src/main/java/org/apache/polaris/core/connection/ConnectionConfigInfoDpo.java b/polaris-core/src/main/java/org/apache/polaris/core/connection/ConnectionConfigInfoDpo.java
index 747f7103b2..af8d9e5c1c 100644
--- a/polaris-core/src/main/java/org/apache/polaris/core/connection/ConnectionConfigInfoDpo.java
+++ b/polaris-core/src/main/java/org/apache/polaris/core/connection/ConnectionConfigInfoDpo.java
@@ -26,6 +26,7 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo;
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URL;
+import java.util.Collection;
import java.util.Map;
import org.apache.polaris.core.admin.model.BigQueryMetastoreConnectionConfigInfo;
import org.apache.polaris.core.admin.model.ConnectionConfigInfo;
@@ -139,6 +140,19 @@ public abstract class ConnectionConfigInfoDpo implements IcebergCatalogPropertie
return properties;
}
+ /**
+ * Copies the connection properties whose keys are in {@code allowedKeys} into {@code target}.
+ * Connection properties are set by catalog administrators, so each connection type forwards only
+ * an explicit allow-list to the Iceberg catalog client instead of every key.
+ */
+ protected void copyAllowedProperties(Map<String, String> target, Collection<String> allowedKeys) {
+ for (String key : allowedKeys) {
+ if (properties.containsKey(key)) {
+ target.put(key, properties.get(key));
+ }
+ }
+ }
+
private static final ObjectMapper DEFAULT_MAPPER;
static {
diff --git a/polaris-core/src/main/java/org/apache/polaris/core/connection/hadoop/HadoopConnectionConfigInfoDpo.java b/polaris-core/src/main/java/org/apache/polaris/core/connection/hadoop/HadoopConnectionConfigInfoDpo.java
index 0de556841d..ca0fcf28a0 100644
--- a/polaris-core/src/main/java/org/apache/polaris/core/connection/hadoop/HadoopConnectionConfigInfoDpo.java
+++ b/polaris-core/src/main/java/org/apache/polaris/core/connection/hadoop/HadoopConnectionConfigInfoDpo.java
@@ -21,9 +21,12 @@ package org.apache.polaris.core.connection.hadoop;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.MoreObjects;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import java.util.Optional;
import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.aws.AwsClientProperties;
+import org.apache.iceberg.aws.s3.S3FileIOProperties;
import org.apache.polaris.core.admin.model.ConnectionConfigInfo;
import org.apache.polaris.core.admin.model.HadoopConnectionConfigInfo;
import org.apache.polaris.core.connection.AuthenticationParametersDpo;
@@ -42,6 +45,19 @@ import org.jspecify.annotations.Nullable;
*/
public class HadoopConnectionConfigInfoDpo extends ConnectionConfigInfoDpo {
+ /**
+ * Connection properties forwarded to the Iceberg {@code HadoopCatalog}: catalog client settings
+ * and non-secret FileIO client settings. Keys that select implementations or carry credentials
+ * are deliberately not forwarded. The FileIO keys only take effect when the catalog is configured
+ * to use {@code S3FileIO} via the catalog-level {@code io-impl} property.
+ */
+ static final List<String> ALLOWED_PROPERTIES =
+ List.of(
+ "suppress-permission-error",
+ S3FileIOProperties.ENDPOINT,
+ S3FileIOProperties.PATH_STYLE_ACCESS,
+ AwsClientProperties.CLIENT_REGION);
+
private final String warehouse;
public HadoopConnectionConfigInfoDpo(
@@ -88,6 +104,7 @@ public class HadoopConnectionConfigInfoDpo extends ConnectionConfigInfoDpo {
}
// Add authentication-specific metadata (non-credential properties)
properties.putAll(getAuthenticationParameters().asIcebergCatalogProperties(credentialManager));
+ copyAllowedProperties(properties, ALLOWED_PROPERTIES);
// Add connection credentials from Polaris credential manager
ConnectionCredentials connectionCredentials = credentialManager.getConnectionCredentials(this);
properties.putAll(connectionCredentials.credentials());
diff --git a/polaris-core/src/main/java/org/apache/polaris/core/connection/hive/HiveConnectionConfigInfoDpo.java b/polaris-core/src/main/java/org/apache/polaris/core/connection/hive/HiveConnectionConfigInfoDpo.java
index 9ab1f73f53..d295a49f44 100644
--- a/polaris-core/src/main/java/org/apache/polaris/core/connection/hive/HiveConnectionConfigInfoDpo.java
+++ b/polaris-core/src/main/java/org/apache/polaris/core/connection/hive/HiveConnectionConfigInfoDpo.java
@@ -21,9 +21,12 @@ package org.apache.polaris.core.connection.hive;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.MoreObjects;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import java.util.Optional;
import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.aws.AwsClientProperties;
+import org.apache.iceberg.aws.s3.S3FileIOProperties;
import org.apache.polaris.core.admin.model.ConnectionConfigInfo;
import org.apache.polaris.core.admin.model.HiveConnectionConfigInfo;
import org.apache.polaris.core.connection.AuthenticationParametersDpo;
@@ -42,6 +45,20 @@ import org.jspecify.annotations.Nullable;
*/
public class HiveConnectionConfigInfoDpo extends ConnectionConfigInfoDpo {
+ /**
+ * Connection properties forwarded to the Iceberg {@code HiveCatalog}: catalog client settings and
+ * non-secret FileIO client settings. Keys that select implementations or carry credentials are
+ * deliberately not forwarded. The FileIO keys only take effect when the catalog is configured to
+ * use {@code S3FileIO} via the catalog-level {@code io-impl} property.
+ */
+ static final List<String> ALLOWED_PROPERTIES =
+ List.of(
+ CatalogProperties.CLIENT_POOL_SIZE,
+ "list-all-tables",
+ S3FileIOProperties.ENDPOINT,
+ S3FileIOProperties.PATH_STYLE_ACCESS,
+ AwsClientProperties.CLIENT_REGION);
+
private final String warehouse;
public HiveConnectionConfigInfoDpo(
@@ -85,6 +102,11 @@ public class HiveConnectionConfigInfoDpo extends ConnectionConfigInfoDpo {
// Add authentication-specific metadata (non-credential properties)
properties.putAll(
getAuthenticationParameters().asIcebergCatalogProperties(polarisCredentialManager));
+ }
+ // Same order as the other connection types: allow-listed properties after the authentication
+ // metadata, and credentials last.
+ copyAllowedProperties(properties, ALLOWED_PROPERTIES);
+ if (getAuthenticationParameters() != null) {
// Add connection credentials from Polaris credential manager
ConnectionCredentials connectionCredentials =
polarisCredentialManager.getConnectionCredentials(this);
diff --git a/polaris-core/src/main/java/org/apache/polaris/core/connection/iceberg/IcebergRestConnectionConfigInfoDpo.java b/polaris-core/src/main/java/org/apache/polaris/core/connection/iceberg/IcebergRestConnectionConfigInfoDpo.java
index e6da3cf65e..32cd495c7c 100644
--- a/polaris-core/src/main/java/org/apache/polaris/core/connection/iceberg/IcebergRestConnectionConfigInfoDpo.java
+++ b/polaris-core/src/main/java/org/apache/polaris/core/connection/iceberg/IcebergRestConnectionConfigInfoDpo.java
@@ -86,11 +86,7 @@ public class IcebergRestConnectionConfigInfoDpo extends ConnectionConfigInfoDpo
// Add authentication-specific metadata (non-credential properties)
properties.putAll(getAuthenticationParameters().asIcebergCatalogProperties(credentialManager));
- for (String headerKey : ALLOWED_PROPERTIES) {
- if (getProperties().containsKey(headerKey)) {
- properties.put(headerKey, getProperties().get(headerKey));
- }
- }
+ copyAllowedProperties(properties, ALLOWED_PROPERTIES);
// Add connection credentials from Polaris credential manager
ConnectionCredentials connectionCredentials = credentialManager.getConnectionCredentials(this);
diff --git a/polaris-core/src/test/java/org/apache/polaris/core/connection/hadoop/HadoopConnectionConfigInfoDpoTest.java b/polaris-core/src/test/java/org/apache/polaris/core/connection/hadoop/HadoopConnectionConfigInfoDpoTest.java
new file mode 100644
index 0000000000..1b1674a135
--- /dev/null
+++ b/polaris-core/src/test/java/org/apache/polaris/core/connection/hadoop/HadoopConnectionConfigInfoDpoTest.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.core.connection.hadoop;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.util.Map;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.aws.AwsClientProperties;
+import org.apache.iceberg.aws.s3.S3FileIOProperties;
+import org.apache.polaris.core.connection.ImplicitAuthenticationParametersDpo;
+import org.apache.polaris.core.credentials.PolarisCredentialManager;
+import org.apache.polaris.core.credentials.connection.ConnectionCredentials;
+import org.junit.jupiter.api.Test;
+
+class HadoopConnectionConfigInfoDpoTest {
+
+ @Test
+ void testAllowedConnectionPropertiesAreForwarded() {
+ HadoopConnectionConfigInfoDpo dpo =
+ createDpo(
+ Map.of(
+ "suppress-permission-error",
+ "true",
+ S3FileIOProperties.ENDPOINT,
+ "http://minio:9000",
+ S3FileIOProperties.PATH_STYLE_ACCESS,
+ "true",
+ AwsClientProperties.CLIENT_REGION,
+ "us-east-1"));
+ Map<String, String> properties = dpo.asIcebergCatalogProperties(mockCredentialManager(dpo));
+ assertThat(properties)
+ .containsEntry("suppress-permission-error", "true")
+ .containsEntry(S3FileIOProperties.ENDPOINT, "http://minio:9000")
+ .containsEntry(S3FileIOProperties.PATH_STYLE_ACCESS, "true")
+ .containsEntry(AwsClientProperties.CLIENT_REGION, "us-east-1");
+ }
+
+ @Test
+ void testOtherConnectionPropertiesAreNotForwarded() {
+ HadoopConnectionConfigInfoDpo dpo =
+ createDpo(
+ Map.of(
+ CatalogProperties.FILE_IO_IMPL,
+ "com.example.Custom",
+ CatalogProperties.LOCK_IMPL,
+ "com.example.Custom",
+ "s3.access-key-id",
+ "AKIA...",
+ "custom-key",
+ "custom-value"));
+
+ Map<String, String> properties = dpo.asIcebergCatalogProperties(mockCredentialManager(dpo));
+
+ assertThat(properties)
+ .doesNotContainKeys(
+ CatalogProperties.FILE_IO_IMPL,
+ CatalogProperties.LOCK_IMPL,
+ "s3.access-key-id",
+ "custom-key");
+ }
+
+ @Test
+ void testNullConnectionPropertiesHandledGracefully() {
+ HadoopConnectionConfigInfoDpo dpo = createDpo(null);
+
+ Map<String, String> properties = dpo.asIcebergCatalogProperties(mockCredentialManager(dpo));
+
+ assertThat(properties).containsEntry(CatalogProperties.URI, dpo.getUri());
+ }
+
+ private static PolarisCredentialManager mockCredentialManager(HadoopConnectionConfigInfoDpo dpo) {
+ PolarisCredentialManager credentialManager = mock(PolarisCredentialManager.class);
+ ConnectionCredentials credentials = mock(ConnectionCredentials.class);
+ when(credentials.credentials()).thenReturn(Map.of());
+ when(credentialManager.getConnectionCredentials(dpo)).thenReturn(credentials);
+ return credentialManager;
+ }
+
+ private static HadoopConnectionConfigInfoDpo createDpo(Map<String, String> properties) {
+ return new HadoopConnectionConfigInfoDpo(
+ "file:///warehouse",
+ new ImplicitAuthenticationParametersDpo(),
+ null,
+ "warehouse",
+ properties);
+ }
+}
diff --git a/polaris-core/src/test/java/org/apache/polaris/core/connection/hive/HiveConnectionConfigInfoDpoTest.java b/polaris-core/src/test/java/org/apache/polaris/core/connection/hive/HiveConnectionConfigInfoDpoTest.java
new file mode 100644
index 0000000000..ba822c4341
--- /dev/null
+++ b/polaris-core/src/test/java/org/apache/polaris/core/connection/hive/HiveConnectionConfigInfoDpoTest.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.core.connection.hive;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.util.Map;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.aws.AwsClientProperties;
+import org.apache.iceberg.aws.s3.S3FileIOProperties;
+import org.apache.polaris.core.connection.ImplicitAuthenticationParametersDpo;
+import org.apache.polaris.core.credentials.PolarisCredentialManager;
+import org.apache.polaris.core.credentials.connection.ConnectionCredentials;
+import org.junit.jupiter.api.Test;
+
+class HiveConnectionConfigInfoDpoTest {
+
+ @Test
+ void testAllowedConnectionPropertiesAreForwarded() {
+ HiveConnectionConfigInfoDpo dpo =
+ createDpo(
+ Map.of(
+ CatalogProperties.CLIENT_POOL_SIZE,
+ "5",
+ "list-all-tables",
+ "true",
+ S3FileIOProperties.ENDPOINT,
+ "http://minio:9000",
+ S3FileIOProperties.PATH_STYLE_ACCESS,
+ "true",
+ AwsClientProperties.CLIENT_REGION,
+ "us-east-1"));
+ Map<String, String> properties = dpo.asIcebergCatalogProperties(mockCredentialManager(dpo));
+ assertThat(properties)
+ .containsEntry(CatalogProperties.CLIENT_POOL_SIZE, "5")
+ .containsEntry("list-all-tables", "true")
+ .containsEntry(S3FileIOProperties.ENDPOINT, "http://minio:9000")
+ .containsEntry(S3FileIOProperties.PATH_STYLE_ACCESS, "true")
+ .containsEntry(AwsClientProperties.CLIENT_REGION, "us-east-1");
+ }
+
+ @Test
+ void testOtherConnectionPropertiesAreNotForwarded() {
+ HiveConnectionConfigInfoDpo dpo =
+ createDpo(
+ Map.of(
+ CatalogProperties.FILE_IO_IMPL,
+ "com.example.Custom",
+ "s3.access-key-id",
+ "AKIA...",
+ "custom-key",
+ "custom-value"));
+
+ Map<String, String> properties = dpo.asIcebergCatalogProperties(mockCredentialManager(dpo));
+
+ assertThat(properties)
+ .doesNotContainKeys(CatalogProperties.FILE_IO_IMPL, "s3.access-key-id", "custom-key");
+ }
+
+ @Test
+ void testNullConnectionPropertiesHandledGracefully() {
+ HiveConnectionConfigInfoDpo dpo = createDpo(null);
+
+ Map<String, String> properties = dpo.asIcebergCatalogProperties(mockCredentialManager(dpo));
+
+ assertThat(properties).containsEntry(CatalogProperties.URI, dpo.getUri());
+ }
+
+ private static PolarisCredentialManager mockCredentialManager(HiveConnectionConfigInfoDpo dpo) {
+ PolarisCredentialManager credentialManager = mock(PolarisCredentialManager.class);
+ ConnectionCredentials credentials = mock(ConnectionCredentials.class);
+ when(credentials.credentials()).thenReturn(Map.of());
+ when(credentialManager.getConnectionCredentials(dpo)).thenReturn(credentials);
+ return credentialManager;
+ }
+
+ private static HiveConnectionConfigInfoDpo createDpo(Map<String, String> properties) {
+ return new HiveConnectionConfigInfoDpo(
+ "thrift://hms:9083",
+ new ImplicitAuthenticationParametersDpo(),
+ "s3://bucket/warehouse",
+ null,
+ properties);
+ }
+}
diff --git a/site/content/in-dev/unreleased/federation/hive-metastore-federation.md b/site/content/in-dev/unreleased/federation/hive-metastore-federation.md
index 06d19b2252..7c84e024f1 100644
--- a/site/content/in-dev/unreleased/federation/hive-metastore-federation.md
+++ b/site/content/in-dev/unreleased/federation/hive-metastore-federation.md
@@ -151,6 +151,28 @@ prefixes. If your IAM trust policy requires an `externalId` or explicit `userArn
optional fields in `storageConfigInfo`. Polaris persists them and supplies them when assuming the
role cited by `roleArn` during metadata commits.
+## Connection properties
+
+`connectionConfigInfo.properties` holds settings for the Hive client that Polaris creates for the
+federated catalog. Unlike catalog-level `properties`, these values are visible only through the
+Management API and are never returned to catalog clients via `/v1/config`, so they are the right
+place for non-secret client settings. Only the following keys are forwarded to the Iceberg
+`HiveCatalog`; any other key is stored but ignored:
+
+| Key | Purpose |
+|-----|---------|
+| `clients` | Hive metastore client pool size |
+| `list-all-tables` | List non-Iceberg tables as well |
+| `s3.endpoint`, `s3.path-style-access`, `client.region` | Non-secret FileIO client settings for S3-compatible warehouses |
+
+The `s3.*` and `client.region` keys only take effect when the catalog uses `S3FileIO`. `HiveCatalog`
+defaults to `HadoopFileIO`, which ignores them, so set
+`io-impl=org.apache.iceberg.aws.s3.S3FileIO` in the catalog-level `properties` (it is not accepted
+as a connection property) when you rely on them.
+
+Connection properties take precedence over catalog-level `properties` when the same key is present.
+Credentials must never be placed here; use the Polaris process identity as described above.
+
## Limitations and operational notes
- **Single identity:** Because only `IMPLICIT` authentication is permitted, Polaris cannot mix