You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@polaris.apache.org by yu...@apache.org on 2026/10/26 17:30:12 UTC

(polaris) branch main updated: Apply allow-listed connection properties to HADOOP and HIVE federated catalogs (#5680)

This is an automated email from the ASF dual-hosted git repository.

flyrain pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/polaris.git


The following commit(s) were added to refs/heads/main by this push:
     new 1af3c00b12 Apply allow-listed connection properties to HADOOP and HIVE federated catalogs (#5680)
1af3c00b12 is described below

commit 1af3c00b12b50cb1cf053bfae1b8d3aba5004e84
Author: youngrae kim <16...@users.noreply.github.com>
AuthorDate: Tue Oct 27 02:30:07 2026 +0900

    Apply allow-listed connection properties to HADOOP and HIVE federated catalogs (#5680)
---
 CHANGELOG.md                                       |   5 +
 .../core/connection/ConnectionConfigInfoDpo.java   |  14 +++
 .../hadoop/HadoopConnectionConfigInfoDpo.java      |  17 ++++
 .../hive/HiveConnectionConfigInfoDpo.java          |  22 +++++
 .../IcebergRestConnectionConfigInfoDpo.java        |   6 +-
 .../hadoop/HadoopConnectionConfigInfoDpoTest.java  | 106 +++++++++++++++++++++
 .../hive/HiveConnectionConfigInfoDpoTest.java      | 103 ++++++++++++++++++++
 .../federation/hive-metastore-federation.md        |  22 +++++
 8 files changed, 290 insertions(+), 5 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 96d0023b67..1d2b7f46e5 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -175,6 +175,11 @@ request adding CHANGELOG notes for breaking (!) changes and possibly other secti
 - Catalog federation: `connectionConfigInfo.properties` is now persisted and returned for `HADOOP`
   and `HIVE` connection configurations. It was previously accepted by the management API but
   silently dropped, so `GET /catalogs/{name}` never showed it.
+- Catalog federation: `connectionConfigInfo.properties` is now applied when Polaris initializes
+  the Iceberg client for `HADOOP` and `HIVE` catalogs. As for `ICEBERG_REST`, only an explicit
+  allow-list is forwarded: `suppress-permission-error` (Hadoop), `clients` and `list-all-tables`
+  (Hive), plus the non-secret FileIO settings `s3.endpoint`,
+  `s3.path-style-access` and `client.region` for both. Other keys are stored but ignored.
 - Semantic models: a `datasets` field that is not a JSON array now returns `400 Bad Request`
   instead of being silently skipped, which bypassed every `dataset.source` check.
 - Conditional `loadTable` (`If-None-Match` → HTTP 304) no longer attaches a null
diff --git a/polaris-core/src/main/java/org/apache/polaris/core/connection/ConnectionConfigInfoDpo.java b/polaris-core/src/main/java/org/apache/polaris/core/connection/ConnectionConfigInfoDpo.java
index 747f7103b2..af8d9e5c1c 100644
--- a/polaris-core/src/main/java/org/apache/polaris/core/connection/ConnectionConfigInfoDpo.java
+++ b/polaris-core/src/main/java/org/apache/polaris/core/connection/ConnectionConfigInfoDpo.java
@@ -26,6 +26,7 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo;
 import java.net.MalformedURLException;
 import java.net.URI;
 import java.net.URL;
+import java.util.Collection;
 import java.util.Map;
 import org.apache.polaris.core.admin.model.BigQueryMetastoreConnectionConfigInfo;
 import org.apache.polaris.core.admin.model.ConnectionConfigInfo;
@@ -139,6 +140,19 @@ public abstract class ConnectionConfigInfoDpo implements IcebergCatalogPropertie
     return properties;
   }
 
+  /**
+   * Copies the connection properties whose keys are in {@code allowedKeys} into {@code target}.
+   * Connection properties are set by catalog administrators, so each connection type forwards only
+   * an explicit allow-list to the Iceberg catalog client instead of every key.
+   */
+  protected void copyAllowedProperties(Map<String, String> target, Collection<String> allowedKeys) {
+    for (String key : allowedKeys) {
+      if (properties.containsKey(key)) {
+        target.put(key, properties.get(key));
+      }
+    }
+  }
+
   private static final ObjectMapper DEFAULT_MAPPER;
 
   static {
diff --git a/polaris-core/src/main/java/org/apache/polaris/core/connection/hadoop/HadoopConnectionConfigInfoDpo.java b/polaris-core/src/main/java/org/apache/polaris/core/connection/hadoop/HadoopConnectionConfigInfoDpo.java
index 0de556841d..ca0fcf28a0 100644
--- a/polaris-core/src/main/java/org/apache/polaris/core/connection/hadoop/HadoopConnectionConfigInfoDpo.java
+++ b/polaris-core/src/main/java/org/apache/polaris/core/connection/hadoop/HadoopConnectionConfigInfoDpo.java
@@ -21,9 +21,12 @@ package org.apache.polaris.core.connection.hadoop;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.google.common.base.MoreObjects;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.aws.AwsClientProperties;
+import org.apache.iceberg.aws.s3.S3FileIOProperties;
 import org.apache.polaris.core.admin.model.ConnectionConfigInfo;
 import org.apache.polaris.core.admin.model.HadoopConnectionConfigInfo;
 import org.apache.polaris.core.connection.AuthenticationParametersDpo;
@@ -42,6 +45,19 @@ import org.jspecify.annotations.Nullable;
  */
 public class HadoopConnectionConfigInfoDpo extends ConnectionConfigInfoDpo {
 
+  /**
+   * Connection properties forwarded to the Iceberg {@code HadoopCatalog}: catalog client settings
+   * and non-secret FileIO client settings. Keys that select implementations or carry credentials
+   * are deliberately not forwarded. The FileIO keys only take effect when the catalog is configured
+   * to use {@code S3FileIO} via the catalog-level {@code io-impl} property.
+   */
+  static final List<String> ALLOWED_PROPERTIES =
+      List.of(
+          "suppress-permission-error",
+          S3FileIOProperties.ENDPOINT,
+          S3FileIOProperties.PATH_STYLE_ACCESS,
+          AwsClientProperties.CLIENT_REGION);
+
   private final String warehouse;
 
   public HadoopConnectionConfigInfoDpo(
@@ -88,6 +104,7 @@ public class HadoopConnectionConfigInfoDpo extends ConnectionConfigInfoDpo {
     }
     // Add authentication-specific metadata (non-credential properties)
     properties.putAll(getAuthenticationParameters().asIcebergCatalogProperties(credentialManager));
+    copyAllowedProperties(properties, ALLOWED_PROPERTIES);
     // Add connection credentials from Polaris credential manager
     ConnectionCredentials connectionCredentials = credentialManager.getConnectionCredentials(this);
     properties.putAll(connectionCredentials.credentials());
diff --git a/polaris-core/src/main/java/org/apache/polaris/core/connection/hive/HiveConnectionConfigInfoDpo.java b/polaris-core/src/main/java/org/apache/polaris/core/connection/hive/HiveConnectionConfigInfoDpo.java
index 9ab1f73f53..d295a49f44 100644
--- a/polaris-core/src/main/java/org/apache/polaris/core/connection/hive/HiveConnectionConfigInfoDpo.java
+++ b/polaris-core/src/main/java/org/apache/polaris/core/connection/hive/HiveConnectionConfigInfoDpo.java
@@ -21,9 +21,12 @@ package org.apache.polaris.core.connection.hive;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.google.common.base.MoreObjects;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.aws.AwsClientProperties;
+import org.apache.iceberg.aws.s3.S3FileIOProperties;
 import org.apache.polaris.core.admin.model.ConnectionConfigInfo;
 import org.apache.polaris.core.admin.model.HiveConnectionConfigInfo;
 import org.apache.polaris.core.connection.AuthenticationParametersDpo;
@@ -42,6 +45,20 @@ import org.jspecify.annotations.Nullable;
  */
 public class HiveConnectionConfigInfoDpo extends ConnectionConfigInfoDpo {
 
+  /**
+   * Connection properties forwarded to the Iceberg {@code HiveCatalog}: catalog client settings and
+   * non-secret FileIO client settings. Keys that select implementations or carry credentials are
+   * deliberately not forwarded. The FileIO keys only take effect when the catalog is configured to
+   * use {@code S3FileIO} via the catalog-level {@code io-impl} property.
+   */
+  static final List<String> ALLOWED_PROPERTIES =
+      List.of(
+          CatalogProperties.CLIENT_POOL_SIZE,
+          "list-all-tables",
+          S3FileIOProperties.ENDPOINT,
+          S3FileIOProperties.PATH_STYLE_ACCESS,
+          AwsClientProperties.CLIENT_REGION);
+
   private final String warehouse;
 
   public HiveConnectionConfigInfoDpo(
@@ -85,6 +102,11 @@ public class HiveConnectionConfigInfoDpo extends ConnectionConfigInfoDpo {
       // Add authentication-specific metadata (non-credential properties)
       properties.putAll(
           getAuthenticationParameters().asIcebergCatalogProperties(polarisCredentialManager));
+    }
+    // Same order as the other connection types: allow-listed properties after the authentication
+    // metadata, and credentials last.
+    copyAllowedProperties(properties, ALLOWED_PROPERTIES);
+    if (getAuthenticationParameters() != null) {
       // Add connection credentials from Polaris credential manager
       ConnectionCredentials connectionCredentials =
           polarisCredentialManager.getConnectionCredentials(this);
diff --git a/polaris-core/src/main/java/org/apache/polaris/core/connection/iceberg/IcebergRestConnectionConfigInfoDpo.java b/polaris-core/src/main/java/org/apache/polaris/core/connection/iceberg/IcebergRestConnectionConfigInfoDpo.java
index e6da3cf65e..32cd495c7c 100644
--- a/polaris-core/src/main/java/org/apache/polaris/core/connection/iceberg/IcebergRestConnectionConfigInfoDpo.java
+++ b/polaris-core/src/main/java/org/apache/polaris/core/connection/iceberg/IcebergRestConnectionConfigInfoDpo.java
@@ -86,11 +86,7 @@ public class IcebergRestConnectionConfigInfoDpo extends ConnectionConfigInfoDpo
     // Add authentication-specific metadata (non-credential properties)
     properties.putAll(getAuthenticationParameters().asIcebergCatalogProperties(credentialManager));
 
-    for (String headerKey : ALLOWED_PROPERTIES) {
-      if (getProperties().containsKey(headerKey)) {
-        properties.put(headerKey, getProperties().get(headerKey));
-      }
-    }
+    copyAllowedProperties(properties, ALLOWED_PROPERTIES);
 
     // Add connection credentials from Polaris credential manager
     ConnectionCredentials connectionCredentials = credentialManager.getConnectionCredentials(this);
diff --git a/polaris-core/src/test/java/org/apache/polaris/core/connection/hadoop/HadoopConnectionConfigInfoDpoTest.java b/polaris-core/src/test/java/org/apache/polaris/core/connection/hadoop/HadoopConnectionConfigInfoDpoTest.java
new file mode 100644
index 0000000000..1b1674a135
--- /dev/null
+++ b/polaris-core/src/test/java/org/apache/polaris/core/connection/hadoop/HadoopConnectionConfigInfoDpoTest.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.core.connection.hadoop;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.util.Map;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.aws.AwsClientProperties;
+import org.apache.iceberg.aws.s3.S3FileIOProperties;
+import org.apache.polaris.core.connection.ImplicitAuthenticationParametersDpo;
+import org.apache.polaris.core.credentials.PolarisCredentialManager;
+import org.apache.polaris.core.credentials.connection.ConnectionCredentials;
+import org.junit.jupiter.api.Test;
+
+class HadoopConnectionConfigInfoDpoTest {
+
+  @Test
+  void testAllowedConnectionPropertiesAreForwarded() {
+    HadoopConnectionConfigInfoDpo dpo =
+        createDpo(
+            Map.of(
+                "suppress-permission-error",
+                "true",
+                S3FileIOProperties.ENDPOINT,
+                "http://minio:9000",
+                S3FileIOProperties.PATH_STYLE_ACCESS,
+                "true",
+                AwsClientProperties.CLIENT_REGION,
+                "us-east-1"));
+    Map<String, String> properties = dpo.asIcebergCatalogProperties(mockCredentialManager(dpo));
+    assertThat(properties)
+        .containsEntry("suppress-permission-error", "true")
+        .containsEntry(S3FileIOProperties.ENDPOINT, "http://minio:9000")
+        .containsEntry(S3FileIOProperties.PATH_STYLE_ACCESS, "true")
+        .containsEntry(AwsClientProperties.CLIENT_REGION, "us-east-1");
+  }
+
+  @Test
+  void testOtherConnectionPropertiesAreNotForwarded() {
+    HadoopConnectionConfigInfoDpo dpo =
+        createDpo(
+            Map.of(
+                CatalogProperties.FILE_IO_IMPL,
+                "com.example.Custom",
+                CatalogProperties.LOCK_IMPL,
+                "com.example.Custom",
+                "s3.access-key-id",
+                "AKIA...",
+                "custom-key",
+                "custom-value"));
+
+    Map<String, String> properties = dpo.asIcebergCatalogProperties(mockCredentialManager(dpo));
+
+    assertThat(properties)
+        .doesNotContainKeys(
+            CatalogProperties.FILE_IO_IMPL,
+            CatalogProperties.LOCK_IMPL,
+            "s3.access-key-id",
+            "custom-key");
+  }
+
+  @Test
+  void testNullConnectionPropertiesHandledGracefully() {
+    HadoopConnectionConfigInfoDpo dpo = createDpo(null);
+
+    Map<String, String> properties = dpo.asIcebergCatalogProperties(mockCredentialManager(dpo));
+
+    assertThat(properties).containsEntry(CatalogProperties.URI, dpo.getUri());
+  }
+
+  private static PolarisCredentialManager mockCredentialManager(HadoopConnectionConfigInfoDpo dpo) {
+    PolarisCredentialManager credentialManager = mock(PolarisCredentialManager.class);
+    ConnectionCredentials credentials = mock(ConnectionCredentials.class);
+    when(credentials.credentials()).thenReturn(Map.of());
+    when(credentialManager.getConnectionCredentials(dpo)).thenReturn(credentials);
+    return credentialManager;
+  }
+
+  private static HadoopConnectionConfigInfoDpo createDpo(Map<String, String> properties) {
+    return new HadoopConnectionConfigInfoDpo(
+        "file:///warehouse",
+        new ImplicitAuthenticationParametersDpo(),
+        null,
+        "warehouse",
+        properties);
+  }
+}
diff --git a/polaris-core/src/test/java/org/apache/polaris/core/connection/hive/HiveConnectionConfigInfoDpoTest.java b/polaris-core/src/test/java/org/apache/polaris/core/connection/hive/HiveConnectionConfigInfoDpoTest.java
new file mode 100644
index 0000000000..ba822c4341
--- /dev/null
+++ b/polaris-core/src/test/java/org/apache/polaris/core/connection/hive/HiveConnectionConfigInfoDpoTest.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.core.connection.hive;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.util.Map;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.aws.AwsClientProperties;
+import org.apache.iceberg.aws.s3.S3FileIOProperties;
+import org.apache.polaris.core.connection.ImplicitAuthenticationParametersDpo;
+import org.apache.polaris.core.credentials.PolarisCredentialManager;
+import org.apache.polaris.core.credentials.connection.ConnectionCredentials;
+import org.junit.jupiter.api.Test;
+
+class HiveConnectionConfigInfoDpoTest {
+
+  @Test
+  void testAllowedConnectionPropertiesAreForwarded() {
+    HiveConnectionConfigInfoDpo dpo =
+        createDpo(
+            Map.of(
+                CatalogProperties.CLIENT_POOL_SIZE,
+                "5",
+                "list-all-tables",
+                "true",
+                S3FileIOProperties.ENDPOINT,
+                "http://minio:9000",
+                S3FileIOProperties.PATH_STYLE_ACCESS,
+                "true",
+                AwsClientProperties.CLIENT_REGION,
+                "us-east-1"));
+    Map<String, String> properties = dpo.asIcebergCatalogProperties(mockCredentialManager(dpo));
+    assertThat(properties)
+        .containsEntry(CatalogProperties.CLIENT_POOL_SIZE, "5")
+        .containsEntry("list-all-tables", "true")
+        .containsEntry(S3FileIOProperties.ENDPOINT, "http://minio:9000")
+        .containsEntry(S3FileIOProperties.PATH_STYLE_ACCESS, "true")
+        .containsEntry(AwsClientProperties.CLIENT_REGION, "us-east-1");
+  }
+
+  @Test
+  void testOtherConnectionPropertiesAreNotForwarded() {
+    HiveConnectionConfigInfoDpo dpo =
+        createDpo(
+            Map.of(
+                CatalogProperties.FILE_IO_IMPL,
+                "com.example.Custom",
+                "s3.access-key-id",
+                "AKIA...",
+                "custom-key",
+                "custom-value"));
+
+    Map<String, String> properties = dpo.asIcebergCatalogProperties(mockCredentialManager(dpo));
+
+    assertThat(properties)
+        .doesNotContainKeys(CatalogProperties.FILE_IO_IMPL, "s3.access-key-id", "custom-key");
+  }
+
+  @Test
+  void testNullConnectionPropertiesHandledGracefully() {
+    HiveConnectionConfigInfoDpo dpo = createDpo(null);
+
+    Map<String, String> properties = dpo.asIcebergCatalogProperties(mockCredentialManager(dpo));
+
+    assertThat(properties).containsEntry(CatalogProperties.URI, dpo.getUri());
+  }
+
+  private static PolarisCredentialManager mockCredentialManager(HiveConnectionConfigInfoDpo dpo) {
+    PolarisCredentialManager credentialManager = mock(PolarisCredentialManager.class);
+    ConnectionCredentials credentials = mock(ConnectionCredentials.class);
+    when(credentials.credentials()).thenReturn(Map.of());
+    when(credentialManager.getConnectionCredentials(dpo)).thenReturn(credentials);
+    return credentialManager;
+  }
+
+  private static HiveConnectionConfigInfoDpo createDpo(Map<String, String> properties) {
+    return new HiveConnectionConfigInfoDpo(
+        "thrift://hms:9083",
+        new ImplicitAuthenticationParametersDpo(),
+        "s3://bucket/warehouse",
+        null,
+        properties);
+  }
+}
diff --git a/site/content/in-dev/unreleased/federation/hive-metastore-federation.md b/site/content/in-dev/unreleased/federation/hive-metastore-federation.md
index 06d19b2252..7c84e024f1 100644
--- a/site/content/in-dev/unreleased/federation/hive-metastore-federation.md
+++ b/site/content/in-dev/unreleased/federation/hive-metastore-federation.md
@@ -151,6 +151,28 @@ prefixes. If your IAM trust policy requires an `externalId` or explicit `userArn
 optional fields in `storageConfigInfo`. Polaris persists them and supplies them when assuming the
 role cited by `roleArn` during metadata commits.
 
+## Connection properties
+
+`connectionConfigInfo.properties` holds settings for the Hive client that Polaris creates for the
+federated catalog. Unlike catalog-level `properties`, these values are visible only through the
+Management API and are never returned to catalog clients via `/v1/config`, so they are the right
+place for non-secret client settings. Only the following keys are forwarded to the Iceberg
+`HiveCatalog`; any other key is stored but ignored:
+
+| Key | Purpose |
+|-----|---------|
+| `clients` | Hive metastore client pool size |
+| `list-all-tables` | List non-Iceberg tables as well |
+| `s3.endpoint`, `s3.path-style-access`, `client.region` | Non-secret FileIO client settings for S3-compatible warehouses |
+
+The `s3.*` and `client.region` keys only take effect when the catalog uses `S3FileIO`. `HiveCatalog`
+defaults to `HadoopFileIO`, which ignores them, so set
+`io-impl=org.apache.iceberg.aws.s3.S3FileIO` in the catalog-level `properties` (it is not accepted
+as a connection property) when you rely on them.
+
+Connection properties take precedence over catalog-level `properties` when the same key is present.
+Credentials must never be placed here; use the Polaris process identity as described above.
+
 ## Limitations and operational notes
 
 - **Single identity:** Because only `IMPLICIT` authentication is permitted, Polaris cannot mix