You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@polaris.apache.org by di...@apache.org on 2026/10/01 01:44:16 UTC
(polaris) branch main updated: fix: persist and return connection properties for HADOOP and HIVE catalogs (#5644)
This is an automated email from the ASF dual-hosted git repository.
dimas-b pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/polaris.git
The following commit(s) were added to refs/heads/main by this push:
new 7977527bdd fix: persist and return connection properties for HADOOP and HIVE catalogs (#5644)
7977527bdd is described below
commit 7977527bddc7efe5dd8d9fe7ba000c6e0bfb8d66
Author: youngrae kim <16...@users.noreply.github.com>
AuthorDate: Thu Oct 1 10:44:10 2026 +0900
fix: persist and return connection properties for HADOOP and HIVE catalogs (#5644)
`ConnectionConfigInfo.properties` is defined on the base schema of the
management API, so every connection type accepts it. For `HADOOP` and
`HIVE` the server answered 201 and then dropped the map: the model-to-DPO
conversion never passed it on, both DPO constructors used the
four-argument super constructor that fixes `properties` to an empty map,
and `asConnectionConfigInfoModel` never set the field. `GET
/catalogs/{name}` therefore never showed it, while `ICEBERG_REST` and
`BIGQUERY` round-trip the same field.
Pass the map through the conversion, accept it in both DPO constructors
(and `withServiceIdentity`), and return it from `asConnectionConfigInfoModel`.
How the stored properties feed the Hadoop/Hive catalog initialization is
unchanged and left for a separate discussion.
Tests: `ConnectionConfigInfoDpoTest` gains a HADOOP and a HIVE case that
convert the API model, round-trip the persistence JSON, and convert back
to the API model. Before the fix both failed with
`expected: <{...custom-key=custom-value}> but was: <{}>`.
Fixes #5642
---
CHANGELOG.md | 3 +
.../core/connection/ConnectionConfigInfoDpo.java | 6 +-
.../hadoop/HadoopConnectionConfigInfoDpo.java | 19 ++++-
.../hive/HiveConnectionConfigInfoDpo.java | 11 ++-
.../connection/ConnectionConfigInfoDpoTest.java | 99 ++++++++++++++++++++++
5 files changed, 130 insertions(+), 8 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index ac9565775d..059d3f71d0 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -168,6 +168,9 @@ request adding CHANGELOG notes for breaking (!) changes and possibly other secti
### Fixes
+- Catalog federation: `connectionConfigInfo.properties` is now persisted and returned for `HADOOP`
+ and `HIVE` connection configurations. It was previously accepted by the management API but
+ silently dropped, so `GET /catalogs/{name}` never showed it.
- Semantic models: a `datasets` field that is not a JSON array now returns `400 Bad Request`
instead of being silently skipped, which bypassed every `dataset.source` check.
- Conditional `loadTable` (`If-None-Match` → HTTP 304) no longer attaches a null
diff --git a/polaris-core/src/main/java/org/apache/polaris/core/connection/ConnectionConfigInfoDpo.java b/polaris-core/src/main/java/org/apache/polaris/core/connection/ConnectionConfigInfoDpo.java
index f2e1212519..747f7103b2 100644
--- a/polaris-core/src/main/java/org/apache/polaris/core/connection/ConnectionConfigInfoDpo.java
+++ b/polaris-core/src/main/java/org/apache/polaris/core/connection/ConnectionConfigInfoDpo.java
@@ -215,7 +215,8 @@ public abstract class ConnectionConfigInfoDpo implements IcebergCatalogPropertie
hadoopConfigModel.getUri(),
authenticationParameters,
null /*Service Identity Info*/,
- hadoopConfigModel.getWarehouse());
+ hadoopConfigModel.getWarehouse(),
+ hadoopConfigModel.getProperties());
break;
case HIVE:
HiveConnectionConfigInfo hiveConfigModel =
@@ -228,7 +229,8 @@ public abstract class ConnectionConfigInfoDpo implements IcebergCatalogPropertie
hiveConfigModel.getUri(),
authenticationParameters,
hiveConfigModel.getWarehouse(),
- null /*Service Identity Info*/);
+ null /*Service Identity Info*/,
+ hiveConfigModel.getProperties());
break;
case BIGQUERY:
BigQueryMetastoreConnectionConfigInfo bigqueryConfigModel =
diff --git a/polaris-core/src/main/java/org/apache/polaris/core/connection/hadoop/HadoopConnectionConfigInfoDpo.java b/polaris-core/src/main/java/org/apache/polaris/core/connection/hadoop/HadoopConnectionConfigInfoDpo.java
index f4ce6af8f4..0de556841d 100644
--- a/polaris-core/src/main/java/org/apache/polaris/core/connection/hadoop/HadoopConnectionConfigInfoDpo.java
+++ b/polaris-core/src/main/java/org/apache/polaris/core/connection/hadoop/HadoopConnectionConfigInfoDpo.java
@@ -50,8 +50,15 @@ public class HadoopConnectionConfigInfoDpo extends ConnectionConfigInfoDpo {
@NonNull AuthenticationParametersDpo authenticationParameters,
@JsonProperty(value = "serviceIdentity", required = false)
@Nullable ServiceIdentityInfoDpo serviceIdentityInfo,
- @JsonProperty(value = "warehouse", required = false) @Nullable String remoteCatalogName) {
- super(ConnectionType.HADOOP.getCode(), uri, authenticationParameters, serviceIdentityInfo);
+ @JsonProperty(value = "warehouse", required = false) @Nullable String remoteCatalogName,
+ @JsonProperty(value = "properties", required = false)
+ @Nullable Map<String, String> properties) {
+ super(
+ ConnectionType.HADOOP.getCode(),
+ uri,
+ authenticationParameters,
+ serviceIdentityInfo,
+ properties);
this.warehouse = remoteCatalogName;
}
@@ -67,6 +74,7 @@ public class HadoopConnectionConfigInfoDpo extends ConnectionConfigInfoDpo {
.add("warehouse", getWarehouse())
.add("authenticationParameters", getAuthenticationParameters().toString())
.add("serviceIdentity", getServiceIdentity())
+ .add("properties", getProperties())
.toString();
}
@@ -90,7 +98,11 @@ public class HadoopConnectionConfigInfoDpo extends ConnectionConfigInfoDpo {
public ConnectionConfigInfoDpo withServiceIdentity(
@NonNull ServiceIdentityInfoDpo serviceIdentityInfo) {
return new HadoopConnectionConfigInfoDpo(
- getUri(), getAuthenticationParameters(), serviceIdentityInfo, getWarehouse());
+ getUri(),
+ getAuthenticationParameters(),
+ serviceIdentityInfo,
+ getWarehouse(),
+ getProperties());
}
@Override
@@ -108,6 +120,7 @@ public class HadoopConnectionConfigInfoDpo extends ConnectionConfigInfoDpo {
serviceIdentityInfoDpo ->
serviceIdentityInfoDpo.asServiceIdentityInfoModel(serviceIdentityProvider))
.orElse(null))
+ .setProperties(getProperties())
.build();
}
}
diff --git a/polaris-core/src/main/java/org/apache/polaris/core/connection/hive/HiveConnectionConfigInfoDpo.java b/polaris-core/src/main/java/org/apache/polaris/core/connection/hive/HiveConnectionConfigInfoDpo.java
index a6fa7bfd2f..9ab1f73f53 100644
--- a/polaris-core/src/main/java/org/apache/polaris/core/connection/hive/HiveConnectionConfigInfoDpo.java
+++ b/polaris-core/src/main/java/org/apache/polaris/core/connection/hive/HiveConnectionConfigInfoDpo.java
@@ -50,8 +50,11 @@ public class HiveConnectionConfigInfoDpo extends ConnectionConfigInfoDpo {
@Nullable AuthenticationParametersDpo authenticationParameters,
@JsonProperty(value = "warehouse", required = false) @Nullable String warehouse,
@JsonProperty(value = "serviceIdentity", required = false)
- @Nullable ServiceIdentityInfoDpo serviceIdentity) {
- super(ConnectionType.HIVE.getCode(), uri, authenticationParameters, serviceIdentity);
+ @Nullable ServiceIdentityInfoDpo serviceIdentity,
+ @JsonProperty(value = "properties", required = false)
+ @Nullable Map<String, String> properties) {
+ super(
+ ConnectionType.HIVE.getCode(), uri, authenticationParameters, serviceIdentity, properties);
this.warehouse = warehouse;
}
@@ -66,6 +69,7 @@ public class HiveConnectionConfigInfoDpo extends ConnectionConfigInfoDpo {
.add("uri", getUri())
.add("warehouse", getWarehouse())
.add("authenticationParameters", getAuthenticationParameters().toString())
+ .add("properties", getProperties())
.toString();
}
@@ -93,7 +97,7 @@ public class HiveConnectionConfigInfoDpo extends ConnectionConfigInfoDpo {
public ConnectionConfigInfoDpo withServiceIdentity(
@NonNull ServiceIdentityInfoDpo serviceIdentityInfo) {
return new HiveConnectionConfigInfoDpo(
- getUri(), getAuthenticationParameters(), warehouse, serviceIdentityInfo);
+ getUri(), getAuthenticationParameters(), warehouse, serviceIdentityInfo, getProperties());
}
@Override
@@ -111,6 +115,7 @@ public class HiveConnectionConfigInfoDpo extends ConnectionConfigInfoDpo {
serviceIdentityInfoDpo ->
serviceIdentityInfoDpo.asServiceIdentityInfoModel(serviceIdentityProvider))
.orElse(null))
+ .setProperties(getProperties())
.build();
}
}
diff --git a/polaris-core/src/test/java/org/apache/polaris/core/connection/ConnectionConfigInfoDpoTest.java b/polaris-core/src/test/java/org/apache/polaris/core/connection/ConnectionConfigInfoDpoTest.java
index eae39d7f4d..002403b424 100644
--- a/polaris-core/src/test/java/org/apache/polaris/core/connection/ConnectionConfigInfoDpoTest.java
+++ b/polaris-core/src/test/java/org/apache/polaris/core/connection/ConnectionConfigInfoDpoTest.java
@@ -18,9 +18,14 @@
*/
package org.apache.polaris.core.connection;
+import java.util.Map;
import java.util.Optional;
+import org.apache.polaris.core.admin.model.AuthenticationParameters;
import org.apache.polaris.core.admin.model.AwsIamServiceIdentityInfo;
import org.apache.polaris.core.admin.model.ConnectionConfigInfo;
+import org.apache.polaris.core.admin.model.HadoopConnectionConfigInfo;
+import org.apache.polaris.core.admin.model.HiveConnectionConfigInfo;
+import org.apache.polaris.core.admin.model.ImplicitAuthenticationParameters;
import org.apache.polaris.core.admin.model.ServiceIdentityInfo;
import org.apache.polaris.core.identity.credential.AwsIamServiceIdentityCredential;
import org.apache.polaris.core.identity.provider.ServiceIdentityProvider;
@@ -299,4 +304,98 @@ public class ConnectionConfigInfoDpoTest {
objectMapper.readValue(expectedApiModelJson, ConnectionConfigInfo.class),
connectionConfigInfoApiModel);
}
+
+ @Test
+ void testHadoopConnectionConfigProperties() {
+ Map<String, String> properties = Map.of("hadoop.custom-key", "custom-value");
+
+ // API model -> DPO must keep the connection properties.
+ ConnectionConfigInfo apiModel =
+ HadoopConnectionConfigInfo.builder()
+ .setConnectionType(ConnectionConfigInfo.ConnectionTypeEnum.HADOOP)
+ .setUri("file:///hadoop-catalog/warehouse")
+ .setWarehouse("hadoop-catalog")
+ .setAuthenticationParameters(
+ ImplicitAuthenticationParameters.builder()
+ .setAuthenticationType(AuthenticationParameters.AuthenticationTypeEnum.IMPLICIT)
+ .build())
+ .setProperties(properties)
+ .build();
+ ConnectionConfigInfoDpo fromModel =
+ ConnectionConfigInfoDpo.fromConnectionConfigInfoModelWithSecrets(apiModel, Map.of());
+ Assertions.assertEquals(properties, fromModel.getProperties());
+
+ // Persistence JSON round trip must keep the connection properties.
+ String json =
+ ""
+ + "{"
+ + " \"connectionTypeCode\": 2,"
+ + " \"uri\": \"file:///hadoop-catalog/warehouse\","
+ + " \"warehouse\": \"hadoop-catalog\","
+ + " \"properties\": {"
+ + " \"hadoop.custom-key\": \"custom-value\""
+ + " },"
+ + " \"authenticationParameters\": {"
+ + " \"authenticationTypeCode\": 3"
+ + " }"
+ + "}";
+ ConnectionConfigInfoDpo connectionConfigInfoDpo = ConnectionConfigInfoDpo.deserialize(json);
+ Assertions.assertNotNull(connectionConfigInfoDpo);
+ Assertions.assertEquals(properties, connectionConfigInfoDpo.getProperties());
+ JsonNode tree1 = objectMapper.readTree(json);
+ JsonNode tree2 = objectMapper.readTree(connectionConfigInfoDpo.serialize());
+ Assertions.assertEquals(tree1, tree2);
+
+ // DPO -> API model must expose the connection properties.
+ ConnectionConfigInfo connectionConfigInfoApiModel =
+ connectionConfigInfoDpo.asConnectionConfigInfoModel(serviceIdentityProvider);
+ Assertions.assertEquals(properties, connectionConfigInfoApiModel.getProperties());
+ }
+
+ @Test
+ void testHiveConnectionConfigProperties() {
+ Map<String, String> properties = Map.of("hive.custom-key", "custom-value");
+
+ // API model -> DPO must keep the connection properties.
+ ConnectionConfigInfo apiModel =
+ HiveConnectionConfigInfo.builder()
+ .setConnectionType(ConnectionConfigInfo.ConnectionTypeEnum.HIVE)
+ .setUri("thrift://hms.example.com:9083")
+ .setWarehouse("s3://bucket/warehouse")
+ .setAuthenticationParameters(
+ ImplicitAuthenticationParameters.builder()
+ .setAuthenticationType(AuthenticationParameters.AuthenticationTypeEnum.IMPLICIT)
+ .build())
+ .setProperties(properties)
+ .build();
+ ConnectionConfigInfoDpo fromModel =
+ ConnectionConfigInfoDpo.fromConnectionConfigInfoModelWithSecrets(apiModel, Map.of());
+ Assertions.assertEquals(properties, fromModel.getProperties());
+
+ // Persistence JSON round trip must keep the connection properties.
+ String json =
+ ""
+ + "{"
+ + " \"connectionTypeCode\": 3,"
+ + " \"uri\": \"thrift://hms.example.com:9083\","
+ + " \"warehouse\": \"s3://bucket/warehouse\","
+ + " \"properties\": {"
+ + " \"hive.custom-key\": \"custom-value\""
+ + " },"
+ + " \"authenticationParameters\": {"
+ + " \"authenticationTypeCode\": 3"
+ + " }"
+ + "}";
+ ConnectionConfigInfoDpo connectionConfigInfoDpo = ConnectionConfigInfoDpo.deserialize(json);
+ Assertions.assertNotNull(connectionConfigInfoDpo);
+ Assertions.assertEquals(properties, connectionConfigInfoDpo.getProperties());
+ JsonNode tree1 = objectMapper.readTree(json);
+ JsonNode tree2 = objectMapper.readTree(connectionConfigInfoDpo.serialize());
+ Assertions.assertEquals(tree1, tree2);
+
+ // DPO -> API model must expose the connection properties.
+ ConnectionConfigInfo connectionConfigInfoApiModel =
+ connectionConfigInfoDpo.asConnectionConfigInfoModel(serviceIdentityProvider);
+ Assertions.assertEquals(properties, connectionConfigInfoApiModel.getProperties());
+ }
}