You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@polaris.apache.org by ad...@apache.org on 2026/05/01 07:40:58 UTC
(polaris) branch main updated: Add integration test for Hive Metastore federation (#4315)
This is an automated email from the ASF dual-hosted git repository.
adutra pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/polaris.git
The following commit(s) were added to refs/heads/main by this push:
new e583fbeabf Add integration test for Hive Metastore federation (#4315)
e583fbeabf is described below
commit e583fbeabf7e68cc4870435886552e978b76d186
Author: Alexandre Dutra <ad...@apache.org>
AuthorDate: Fri May 1 09:40:53 2026 +0200
Add integration test for Hive Metastore federation (#4315)
This change adds:
- A new testcontainer with hive metastore, hadoop-aws and AWS SDK
- A new integration test leveraging the testcontainer that verifies that an EXTERNAL Polaris catalog with `connectionType=HIVE` federates read/write to HMS while Polaris still enforces RBAC.
---
gradle/libs.versions.toml | 1 +
gradle/projects.main.properties | 1 +
integration-tests/build.gradle.kts | 1 +
.../test/HiveCatalogFederationIntegrationTest.java | 349 +++++++++++++++++++++
runtime/spark-tests/build.gradle.kts | 21 ++
.../service/spark/it/HiveCatalogFederationIT.java | 108 +++++++
.../spark-tests/src/main/resources/core-site.xml | 53 ++++
tools/hms-testcontainer/build.gradle.kts | 34 ++
.../org/apache/polaris/test/hms/HmsContainer.java | 151 +++++++++
.../apache/polaris/test/hms/Dockerfile-hms-version | 44 +++
10 files changed, 763 insertions(+)
diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml
index c658aa7db1..bf10f1346c 100644
--- a/gradle/libs.versions.toml
+++ b/gradle/libs.versions.toml
@@ -59,6 +59,7 @@ graalvm-polyglot-polyglot = { module = "org.graalvm.polyglot:polyglot", version.
google-cloud-storage-bom = { module = "com.google.cloud:google-cloud-storage-bom", version = "2.67.0" }
guava = { module = "com.google.guava:guava", version = "33.6.0-jre" }
h2 = { module = "com.h2database:h2", version = "2.4.240" }
+hadoop-aws = { module = "org.apache.hadoop:hadoop-aws", version.ref = "hadoop" }
hadoop-client-api = { module = "org.apache.hadoop:hadoop-client-api", version.ref = "hadoop" }
hadoop-client-runtime = { module = "org.apache.hadoop:hadoop-client-runtime", version.ref = "hadoop" }
hadoop-common = { module = "org.apache.hadoop:hadoop-common", version.ref = "hadoop" }
diff --git a/gradle/projects.main.properties b/gradle/projects.main.properties
index 7ae8cd811d..83327ffa08 100644
--- a/gradle/projects.main.properties
+++ b/gradle/projects.main.properties
@@ -39,6 +39,7 @@ polaris-immutables=tools/immutables
polaris-container-spec-helper=tools/container-spec-helper
polaris-minio-testcontainer=tools/minio-testcontainer
polaris-rustfs-testcontainer=tools/rustfs-testcontainer
+polaris-hms-testcontainer=tools/hms-testcontainer
polaris-version=tools/version
polaris-misc-types=tools/misc-types
polaris-extensions-federation-hadoop=extensions/federation/hadoop
diff --git a/integration-tests/build.gradle.kts b/integration-tests/build.gradle.kts
index 2bb985545d..2ea85e98d7 100644
--- a/integration-tests/build.gradle.kts
+++ b/integration-tests/build.gradle.kts
@@ -71,6 +71,7 @@ dependencies {
implementation(project(":polaris-runtime-test-common"))
implementation(project(":polaris-minio-testcontainer"))
implementation(project(":polaris-rustfs-testcontainer"))
+ implementation(project(":polaris-hms-testcontainer"))
}
copiedCodeChecks {
diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/test/HiveCatalogFederationIntegrationTest.java b/integration-tests/src/main/java/org/apache/polaris/service/it/test/HiveCatalogFederationIntegrationTest.java
new file mode 100644
index 0000000000..a623ebdf2e
--- /dev/null
+++ b/integration-tests/src/main/java/org/apache/polaris/service/it/test/HiveCatalogFederationIntegrationTest.java
@@ -0,0 +1,349 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.service.it.test;
+
+import static org.apache.polaris.service.it.env.PolarisClient.polarisClient;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+import java.net.URI;
+import java.nio.file.Path;
+import java.util.List;
+import org.apache.iceberg.exceptions.ForbiddenException;
+import org.apache.polaris.core.admin.model.AuthenticationParameters;
+import org.apache.polaris.core.admin.model.AwsStorageConfigInfo;
+import org.apache.polaris.core.admin.model.Catalog;
+import org.apache.polaris.core.admin.model.CatalogGrant;
+import org.apache.polaris.core.admin.model.CatalogPrivilege;
+import org.apache.polaris.core.admin.model.CatalogProperties;
+import org.apache.polaris.core.admin.model.CatalogRole;
+import org.apache.polaris.core.admin.model.ConnectionConfigInfo;
+import org.apache.polaris.core.admin.model.ExternalCatalog;
+import org.apache.polaris.core.admin.model.GrantResource;
+import org.apache.polaris.core.admin.model.HiveConnectionConfigInfo;
+import org.apache.polaris.core.admin.model.ImplicitAuthenticationParameters;
+import org.apache.polaris.core.admin.model.NamespaceGrant;
+import org.apache.polaris.core.admin.model.NamespacePrivilege;
+import org.apache.polaris.core.admin.model.PrincipalWithCredentials;
+import org.apache.polaris.core.admin.model.StorageConfigInfo;
+import org.apache.polaris.core.config.FeatureConfiguration;
+import org.apache.polaris.service.it.env.ClientCredentials;
+import org.apache.polaris.service.it.env.ManagementApi;
+import org.apache.polaris.service.it.env.PolarisApiEndpoints;
+import org.apache.polaris.service.it.env.PolarisClient;
+import org.apache.polaris.service.it.ext.PolarisIntegrationTestExtension;
+import org.apache.polaris.service.it.ext.SparkSessionBuilder;
+import org.apache.polaris.test.hms.HmsContainer;
+import org.apache.polaris.test.rustfs.RustfsAccess;
+import org.apache.spark.sql.Row;
+import org.apache.spark.sql.SparkSession;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestInstance;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.junit.jupiter.api.io.TempDir;
+
+/**
+ * Integration test for Hive Metastore (HMS) federation. Verifies that an external Polaris catalog
+ * with {@code connectionType=HIVE} can federate read/write operations to a backing HMS, while
+ * Polaris continues to enforce its own RBAC.
+ *
+ * <p>This test requires the following features to be configured:
+ *
+ * <ul>
+ * <li>{@link FeatureConfiguration#ENABLE_CATALOG_FEDERATION} = {@code true}.
+ * <li>{@link FeatureConfiguration#SUPPORTED_CATALOG_CONNECTION_TYPES} must include {@code
+ * "HIVE"}.
+ * <li>{@link FeatureConfiguration#SUPPORTED_EXTERNAL_CATALOG_AUTHENTICATION_TYPES} must include
+ * {@code "IMPLICIT"}; Hive federation only supports {@code IMPLICIT} authentication.
+ * <li>{@link FeatureConfiguration#ENABLE_SUB_CATALOG_RBAC_FOR_FEDERATED_CATALOGS} = {@code true}:
+ * required by tests that exercise RBAC on securables that live in HMS.
+ * <li>{@link FeatureConfiguration#ALLOW_DROPPING_NON_EMPTY_PASSTHROUGH_FACADE_CATALOG} = {@code
+ * true}: used by cleanup code.
+ * <li>{@link FeatureConfiguration#ALLOW_OVERLAPPING_CATALOG_URLS} = {@code true}.
+ * </ul>
+ *
+ * This test also requires a properly-configured {@link RustfsAccess} instance. The {@code
+ * core-site.xml} file in {@code src/main/resources} will be used to configure the S3A filesystem.
+ * Therefore, the test must set the following system properties, which will be substituted into that
+ * file at startup using Hadoop's {@code ${name}} substitution:
+ *
+ * <ul>
+ * <li>{@code polaris.s3.endpoint}
+ * <li>{@code polaris.s3.access-key}
+ * <li>{@code polaris.s3.secret-key}
+ * </ul>
+ *
+ * The values of the above properties must correspond to the values exposed by the RustFS container.
+ */
+@ExtendWith(PolarisIntegrationTestExtension.class)
+@TestInstance(TestInstance.Lifecycle.PER_CLASS)
+public abstract class HiveCatalogFederationIntegrationTest {
+
+ public static final String BUCKET_URI_PREFIX = "/rustfs-test-hive-federation";
+ public static final String RUSTFS_ACCESS_KEY = "test-ak-123-hive-federation";
+ public static final String RUSTFS_SECRET_KEY = "test-sk-123-hive-federation";
+
+ private static final String PRINCIPAL_NAME = "test-hive-federation-user";
+ private static final String PRINCIPAL_ROLE_NAME = "test-hive-federation-user-role";
+ private static final String DIRECT_CATALOG_NAME = "hms_direct";
+
+ private static final CatalogGrant CATALOG_ADMIN_GRANT =
+ CatalogGrant.builder()
+ .setType(GrantResource.TypeEnum.CATALOG)
+ .setPrivilege(CatalogPrivilege.CATALOG_MANAGE_CONTENT)
+ .build();
+
+ /**
+ * Returns the RustFS instance backing the test. The subclass is also responsible for propagating
+ * the endpoint/credentials to the Polaris JVM as system properties so Hadoop's {@code
+ * core-site.xml} can substitute them.
+ */
+ protected abstract RustfsAccess rustfsAccess();
+
+ private PolarisClient client;
+ private ManagementApi managementApi;
+ private PolarisApiEndpoints endpoints;
+ private SparkSession spark;
+ private HmsContainer hmsContainer;
+ private String hmsThriftUri;
+ private URI warehouseLocation;
+ private String s3endpoint;
+
+ @TempDir Path warehouseDir;
+
+ private String federatedCatalogName;
+ private String federatedCatalogRoleName;
+
+ @BeforeAll
+ void setup(PolarisApiEndpoints apiEndpoints, ClientCredentials credentials) {
+ endpoints = apiEndpoints;
+ client = polarisClient(endpoints);
+ String adminToken = client.obtainToken(credentials);
+ managementApi = client.managementApi(adminToken);
+ RustfsAccess rustfs = rustfsAccess();
+ s3endpoint = rustfs.s3endpoint();
+ // s3a:// (not s3://) so the federated HiveCatalog's HadoopFileIO routes to S3AFileSystem
+ // without relying on the legacy fs.s3.impl mapping (also set in core-site.xml as a backstop).
+ warehouseLocation = URI.create("s3a://" + rustfs.bucket() + BUCKET_URI_PREFIX + "/warehouse");
+
+ // HMS validates s3a:// table locations during create-table by talking to the underlying
+ // S3 endpoint. RustFS only accepts requests whose Host header matches the value of its
+ // RUSTFS_SERVER_DOMAINS env var (set to RustFS's fixed host port via the testcontainer).
+ // We therefore connect to RustFS using its exact host:port name from inside HMS, with an
+ // extra /etc/hosts entry pointing that hostname at the Docker host gateway ("host-gateway").
+ String rustfsHost = rustfs.hostPort().substring(0, rustfs.hostPort().indexOf(':'));
+ hmsContainer =
+ new HmsContainer()
+ .withExtraHost(rustfsHost, "host-gateway")
+ .withS3aEndpoint("http://" + rustfs.hostPort(), rustfs.accessKey(), rustfs.secretKey())
+ .withStartupAttempts(3);
+ hmsContainer.start();
+ hmsThriftUri = hmsContainer.thriftUri();
+ }
+
+ @AfterAll
+ void close() throws Exception {
+ if (hmsContainer != null) {
+ hmsContainer.close();
+ }
+ if (client != null) {
+ client.close();
+ }
+ }
+
+ @BeforeEach
+ void before() {
+ federatedCatalogName = "test_catalog_hive_" + System.nanoTime();
+ federatedCatalogRoleName = "test-catalog-role_" + System.nanoTime();
+
+ PrincipalWithCredentials newUserCredentials =
+ managementApi.createPrincipalWithRole(PRINCIPAL_NAME, PRINCIPAL_ROLE_NAME);
+
+ AwsStorageConfigInfo storageConfig =
+ AwsStorageConfigInfo.builder()
+ .setStorageType(StorageConfigInfo.StorageTypeEnum.S3)
+ .setPathStyleAccess(true)
+ .setEndpoint(s3endpoint)
+ .setAllowedLocations(List.of(warehouseLocation.toString()))
+ .build();
+
+ ConnectionConfigInfo connectionConfig =
+ HiveConnectionConfigInfo.builder()
+ .setConnectionType(ConnectionConfigInfo.ConnectionTypeEnum.HIVE)
+ .setUri(hmsThriftUri)
+ .setWarehouse(warehouseLocation.toString())
+ .setAuthenticationParameters(
+ ImplicitAuthenticationParameters.builder()
+ .setAuthenticationType(AuthenticationParameters.AuthenticationTypeEnum.IMPLICIT)
+ .build())
+ .build();
+
+ CatalogProperties catalogProperties = new CatalogProperties(warehouseLocation.toString());
+
+ ExternalCatalog externalCatalog =
+ ExternalCatalog.builder()
+ .setType(Catalog.TypeEnum.EXTERNAL)
+ .setName(federatedCatalogName)
+ .setConnectionConfigInfo(connectionConfig)
+ .setProperties(catalogProperties)
+ .setStorageConfigInfo(storageConfig)
+ .build();
+ managementApi.createCatalog(externalCatalog);
+ managementApi.createCatalogRole(federatedCatalogName, federatedCatalogRoleName);
+ managementApi.addGrant(federatedCatalogName, federatedCatalogRoleName, CATALOG_ADMIN_GRANT);
+ CatalogRole role = managementApi.getCatalogRole(federatedCatalogName, federatedCatalogRoleName);
+ managementApi.grantCatalogRoleToPrincipalRole(PRINCIPAL_ROLE_NAME, federatedCatalogName, role);
+
+ String sparkToken = client.obtainToken(newUserCredentials);
+ spark =
+ SparkSessionBuilder.buildWithTestDefaults()
+ .withWarehouse(warehouseDir.toUri())
+ .addCatalog(
+ federatedCatalogName,
+ "org.apache.iceberg.spark.SparkCatalog",
+ endpoints,
+ sparkToken)
+ // Disable the REST-catalog cache so RBAC changes take effect immediately.
+ .withConfig("spark.sql.catalog." + federatedCatalogName + ".cache-enabled", "false")
+ .withConfig(
+ "spark.sql.catalog." + federatedCatalogName + ".header.X-Iceberg-Access-Delegation",
+ "vended-credentials")
+ // The "direct" catalog talks to HMS directly via Iceberg's HiveCatalog and writes
+ // to the same RustFS bucket that Polaris's federated catalog reads from.
+ .withConfig(
+ "spark.sql.catalog." + DIRECT_CATALOG_NAME, "org.apache.iceberg.spark.SparkCatalog")
+ .withConfig("spark.sql.catalog." + DIRECT_CATALOG_NAME + ".type", "hive")
+ .withConfig("spark.sql.catalog." + DIRECT_CATALOG_NAME + ".uri", hmsThriftUri)
+ .withConfig(
+ "spark.sql.catalog." + DIRECT_CATALOG_NAME + ".warehouse",
+ warehouseLocation.toString())
+ .withConfig(
+ "spark.sql.catalog." + DIRECT_CATALOG_NAME + ".io-impl",
+ "org.apache.iceberg.aws.s3.S3FileIO")
+ .withConfig("spark.sql.catalog." + DIRECT_CATALOG_NAME + ".s3.endpoint", s3endpoint)
+ .withConfig(
+ "spark.sql.catalog." + DIRECT_CATALOG_NAME + ".s3.path-style-access", "true")
+ .withConfig(
+ "spark.sql.catalog." + DIRECT_CATALOG_NAME + ".s3.access-key-id", RUSTFS_ACCESS_KEY)
+ .withConfig(
+ "spark.sql.catalog." + DIRECT_CATALOG_NAME + ".s3.secret-access-key",
+ RUSTFS_SECRET_KEY)
+ .withConfig("spark.sql.catalog." + DIRECT_CATALOG_NAME + ".cache-enabled", "false")
+ .getOrCreate();
+
+ seedHmsTables();
+ }
+
+ @AfterEach
+ void after() {
+ if (spark != null) {
+ // Drop everything we created in HMS via the direct catalog so the embedded Derby DB is
+ // clean for the next test. The built-in HMS "default" namespace cannot be dropped, so we
+ // target ns1/ns2 explicitly.
+ for (String ns : List.of("ns1", "ns2")) {
+ // No PURGE: the Iceberg DeleteReachableFiles action uses Spark beans that need
+ // javax.annotation, which is not on this trimmed-down test classpath.
+ spark.sql(String.format("DROP TABLE IF EXISTS %s.%s.test_table", DIRECT_CATALOG_NAME, ns));
+ spark.sql(String.format("DROP NAMESPACE IF EXISTS %s.%s", DIRECT_CATALOG_NAME, ns));
+ }
+ SparkSession.clearDefaultSession();
+ SparkSession.clearActiveSession();
+ spark.close();
+ }
+ // We don't catalogApi.purge() here: HMS exposes a non-droppable "default" namespace, and
+ // the underlying HMS state has already been cleaned up via the direct catalog above.
+ // ALLOW_DROPPING_NON_EMPTY_PASSTHROUGH_FACADE_CATALOG lets dropCatalog succeed regardless.
+ managementApi.dropCatalog(federatedCatalogName);
+ managementApi.deletePrincipalRole(PRINCIPAL_ROLE_NAME);
+ managementApi.deletePrincipal(PRINCIPAL_NAME);
+ }
+
+ private void seedHmsTables() {
+ spark.sql("USE " + DIRECT_CATALOG_NAME);
+ String warehouse = warehouseLocation.toString();
+ spark.sql("CREATE NAMESPACE IF NOT EXISTS ns1 LOCATION '" + warehouse + "/ns1.db'");
+ spark.sql(
+ "CREATE TABLE IF NOT EXISTS ns1.test_table (id int, name string) USING iceberg "
+ + "LOCATION '"
+ + warehouse
+ + "/ns1/test_table'");
+ spark.sql("INSERT INTO ns1.test_table VALUES (1, 'Alice'), (2, 'Bob')");
+
+ spark.sql("CREATE NAMESPACE IF NOT EXISTS ns2 LOCATION '" + warehouse + "/ns2.db'");
+ spark.sql(
+ "CREATE TABLE IF NOT EXISTS ns2.test_table (id int, name string) USING iceberg "
+ + "LOCATION '"
+ + warehouse
+ + "/ns2/test_table'");
+ spark.sql("INSERT INTO ns2.test_table VALUES (1, 'Apache Polaris'), (2, 'Apache Iceberg')");
+ }
+
+ @Test
+ void testFederatedReadSeesHmsTables() {
+ spark.sql("USE " + federatedCatalogName);
+
+ List<Row> namespaces = spark.sql("SHOW NAMESPACES").collectAsList();
+ assertThat(namespaces).extracting(r -> r.getString(0)).contains("ns1", "ns2");
+
+ List<Row> ns1Data = spark.sql("SELECT * FROM ns1.test_table ORDER BY id").collectAsList();
+ List<Row> ns1Direct =
+ spark
+ .sql("SELECT * FROM " + DIRECT_CATALOG_NAME + ".ns1.test_table ORDER BY id")
+ .collectAsList();
+ assertThat(ns1Data).isEqualTo(ns1Direct).hasSize(2);
+ }
+
+ @Test
+ void testFederatedWriteVisibleInHms() {
+ spark.sql("USE " + federatedCatalogName);
+ spark.sql("INSERT INTO ns1.test_table VALUES (3, 'Charlie')");
+
+ spark.sql(String.format("REFRESH TABLE %s.ns1.test_table", DIRECT_CATALOG_NAME));
+ List<Row> directData =
+ spark
+ .sql("SELECT * FROM " + DIRECT_CATALOG_NAME + ".ns1.test_table ORDER BY id")
+ .collectAsList();
+ assertThat(directData).hasSize(3);
+ assertThat(directData.get(2).getInt(0)).isEqualTo(3);
+ assertThat(directData.get(2).getString(1)).isEqualTo("Charlie");
+ }
+
+ @Test
+ void testFederatedCatalogEnforcesNamespaceRbac() {
+ managementApi.revokeGrant(federatedCatalogName, federatedCatalogRoleName, CATALOG_ADMIN_GRANT);
+ NamespaceGrant ns1Read =
+ NamespaceGrant.builder()
+ .setType(GrantResource.TypeEnum.NAMESPACE)
+ .setPrivilege(NamespacePrivilege.TABLE_READ_DATA)
+ .setNamespace(List.of("ns1"))
+ .build();
+ managementApi.addGrant(federatedCatalogName, federatedCatalogRoleName, ns1Read);
+
+ spark.sql("USE " + federatedCatalogName);
+ assertThat(spark.sql("SELECT * FROM ns1.test_table").collectAsList()).hasSize(2);
+
+ // No grant on ns2: Polaris must reject the federated read.
+ assertThatThrownBy(() -> spark.sql("SELECT * FROM ns2.test_table").collectAsList())
+ .isInstanceOf(ForbiddenException.class);
+ }
+}
diff --git a/runtime/spark-tests/build.gradle.kts b/runtime/spark-tests/build.gradle.kts
index 78d21ed5a6..5b5f4259c2 100644
--- a/runtime/spark-tests/build.gradle.kts
+++ b/runtime/spark-tests/build.gradle.kts
@@ -33,8 +33,29 @@ dependencies {
}
implementation(project(":polaris-runtime-service"))
+ runtimeOnly(project(":polaris-extensions-federation-hive")) {
+ // Brings shaded parquet 1.10 which conflicts with Iceberg's parquet 1.16 in test code.
+ exclude("org.apache.parquet", "parquet-hadoop-bundle")
+ }
+
+ // Adds hadoop-aws to the Polaris JVM classpath so Hive federation's HiveCatalog can use
+ // HadoopFileIO (its default) against an s3a:// warehouse.
+ runtimeOnly(libs.hadoop.aws) {
+ // Exclude the AWS SDK V2 fat-jar `bundle`: hadoop-aws 3.4.3 pulls bundle:2.35.4, which clashes
+ // with Polaris's individual SDK modules. The required modules (s3, sts, etc.) are already on
+ // the classpath via the awssdk-bom.
+ exclude("software.amazon.awssdk", "bundle")
+ exclude("org.slf4j", "slf4j-reload4j")
+ exclude("org.slf4j", "slf4j-log4j12")
+ exclude("ch.qos.reload4j", "reload4j")
+ exclude("log4j", "log4j")
+ }
+ // hadoop-aws references s3-transfer-manager classes but doesn't declare a direct dependency
+ // (the bundle fat-jar provided them); add it explicitly at the BOM-managed version.
+ runtimeOnly("software.amazon.awssdk:s3-transfer-manager")
testImplementation(project(":polaris-tests"))
+ testImplementation(project(":polaris-rustfs-testcontainer"))
testImplementation(testFixtures(project(":polaris-runtime-service")))
testImplementation(project(":polaris-runtime-test-common"))
diff --git a/runtime/spark-tests/src/intTest/java/org/apache/polaris/service/spark/it/HiveCatalogFederationIT.java b/runtime/spark-tests/src/intTest/java/org/apache/polaris/service/spark/it/HiveCatalogFederationIT.java
new file mode 100644
index 0000000000..ccfdc4249e
--- /dev/null
+++ b/runtime/spark-tests/src/intTest/java/org/apache/polaris/service/spark/it/HiveCatalogFederationIT.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.service.spark.it;
+
+import com.google.common.collect.ImmutableMap;
+import io.quarkus.test.common.QuarkusTestResource;
+import io.quarkus.test.common.QuarkusTestResourceLifecycleManager;
+import io.quarkus.test.junit.QuarkusIntegrationTest;
+import io.quarkus.test.junit.QuarkusTestProfile;
+import io.quarkus.test.junit.TestProfile;
+import java.util.Map;
+import org.apache.polaris.service.it.test.HiveCatalogFederationIntegrationTest;
+import org.apache.polaris.test.rustfs.RustfsAccess;
+import org.apache.polaris.test.rustfs.RustfsContainer;
+
+@TestProfile(HiveCatalogFederationIT.HiveFederationProfile.class)
+@QuarkusTestResource(HiveCatalogFederationIT.RustfsResource.class)
+@QuarkusIntegrationTest
+public class HiveCatalogFederationIT extends HiveCatalogFederationIntegrationTest {
+
+ /** Populated by {@link RustfsResource#inject} before {@code @BeforeAll} runs. */
+ private RustfsAccess rustfsAccess;
+
+ @Override
+ protected RustfsAccess rustfsAccess() {
+ return rustfsAccess;
+ }
+
+ public static class HiveFederationProfile implements QuarkusTestProfile {
+ @Override
+ public Map<String, String> getConfigOverrides() {
+ return ImmutableMap.<String, String>builder()
+ .put("polaris.features.\"ENABLE_CATALOG_FEDERATION\"", "true")
+ .put("polaris.features.\"SUPPORTED_CATALOG_CONNECTION_TYPES\"", "[\"HIVE\"]")
+ .put(
+ "polaris.features.\"SUPPORTED_EXTERNAL_CATALOG_AUTHENTICATION_TYPES\"",
+ "[\"IMPLICIT\"]")
+ .put("polaris.features.\"ALLOW_OVERLAPPING_CATALOG_URLS\"", "true")
+ .put("polaris.features.\"ENABLE_SUB_CATALOG_RBAC_FOR_FEDERATED_CATALOGS\"", "true")
+ .put("polaris.features.\"ALLOW_DROPPING_NON_EMPTY_PASSTHROUGH_FACADE_CATALOG\"", "true")
+ .put("polaris.features.\"SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION\"", "false")
+ .put("polaris.features.\"ALLOW_EXTERNAL_CATALOG_CREDENTIAL_VENDING\"", "true")
+ .put("polaris.features.\"ALLOW_FEDERATED_CATALOGS_CREDENTIAL_VENDING\"", "true")
+ .put(
+ "polaris.storage.aws.access-key",
+ HiveCatalogFederationIntegrationTest.RUSTFS_ACCESS_KEY)
+ .put(
+ "polaris.storage.aws.secret-key",
+ HiveCatalogFederationIntegrationTest.RUSTFS_SECRET_KEY)
+ .build();
+ }
+ }
+
+ /**
+ * Starts a RustFS container , exposes its endpoint and credentials, and injects the running
+ * container into {@link HiveCatalogFederationIT#rustfsAccess} on the test instance.
+ */
+ public static class RustfsResource implements QuarkusTestResourceLifecycleManager {
+
+ private RustfsContainer container;
+
+ @Override
+ public Map<String, String> start() {
+ container =
+ new RustfsContainer(
+ null,
+ HiveCatalogFederationIntegrationTest.RUSTFS_ACCESS_KEY,
+ HiveCatalogFederationIntegrationTest.RUSTFS_SECRET_KEY,
+ null,
+ null)
+ .withStartupAttempts(5);
+ container.start();
+ return ImmutableMap.of(
+ "polaris.s3.endpoint", container.s3endpoint(),
+ "polaris.s3.access-key", container.accessKey(),
+ "polaris.s3.secret-key", container.secretKey());
+ }
+
+ @Override
+ public void inject(TestInjector testInjector) {
+ testInjector.injectIntoFields(container, new TestInjector.MatchesType(RustfsAccess.class));
+ }
+
+ @Override
+ public void stop() {
+ if (container != null) {
+ container.close();
+ container = null;
+ }
+ }
+ }
+}
diff --git a/runtime/spark-tests/src/main/resources/core-site.xml b/runtime/spark-tests/src/main/resources/core-site.xml
new file mode 100644
index 0000000000..934051c2a8
--- /dev/null
+++ b/runtime/spark-tests/src/main/resources/core-site.xml
@@ -0,0 +1,53 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+<!--
+ Used only by HiveCatalogFederationIT. Hadoop's Configuration substitutes ${name} from
+ system properties.
+-->
+<configuration>
+ <property>
+ <name>fs.s3.impl</name>
+ <value>org.apache.hadoop.fs.s3a.S3AFileSystem</value>
+ </property>
+ <property>
+ <name>fs.s3a.endpoint</name>
+ <value>${polaris.s3.endpoint}</value>
+ </property>
+ <property>
+ <name>fs.s3a.access.key</name>
+ <value>${polaris.s3.access-key}</value>
+ </property>
+ <property>
+ <name>fs.s3a.secret.key</name>
+ <value>${polaris.s3.secret-key}</value>
+ </property>
+ <property>
+ <name>fs.s3a.path.style.access</name>
+ <value>true</value>
+ </property>
+ <property>
+ <name>fs.s3a.connection.ssl.enabled</name>
+ <value>false</value>
+ </property>
+ <property>
+ <name>fs.s3a.aws.credentials.provider</name>
+ <value>org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider</value>
+ </property>
+</configuration>
diff --git a/tools/hms-testcontainer/build.gradle.kts b/tools/hms-testcontainer/build.gradle.kts
new file mode 100644
index 0000000000..6ef77de859
--- /dev/null
+++ b/tools/hms-testcontainer/build.gradle.kts
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+plugins {
+ id("org.kordamp.gradle.jandex")
+ id("polaris-server")
+}
+
+dependencies {
+ api(platform(libs.testcontainers.bom))
+ api("org.testcontainers:testcontainers")
+
+ implementation(project(":polaris-container-spec-helper"))
+ implementation(libs.guava)
+
+ compileOnly(platform(libs.junit.bom))
+ compileOnly("org.junit.jupiter:junit-jupiter-api")
+}
diff --git a/tools/hms-testcontainer/src/main/java/org/apache/polaris/test/hms/HmsContainer.java b/tools/hms-testcontainer/src/main/java/org/apache/polaris/test/hms/HmsContainer.java
new file mode 100644
index 0000000000..fa5324960f
--- /dev/null
+++ b/tools/hms-testcontainer/src/main/java/org/apache/polaris/test/hms/HmsContainer.java
@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.test.hms;
+
+import com.google.common.base.Preconditions;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.UncheckedIOException;
+import java.nio.charset.StandardCharsets;
+import java.time.Duration;
+import java.util.Objects;
+import org.apache.polaris.containerspec.ContainerSpecHelper;
+import org.slf4j.LoggerFactory;
+import org.testcontainers.containers.GenericContainer;
+import org.testcontainers.containers.output.Slf4jLogConsumer;
+import org.testcontainers.containers.wait.strategy.Wait;
+import org.testcontainers.images.builder.ImageFromDockerfile;
+import org.testcontainers.images.builder.Transferable;
+import org.testcontainers.utility.DockerImageName;
+
+/**
+ * Test container that starts an Apache Hive Metastore (HMS) in standalone mode, backed by an
+ * embedded Derby database.
+ *
+ * <p>The container exposes only the Thrift endpoint (port 9083). The base apache/hive image is
+ * extended at build time with hadoop-aws + aws-java-sdk-bundle (see {@code Dockerfile-hms-version})
+ * so that HMS can recognize {@code s3a://} URIs when validating table locations. Use {@link
+ * #withS3aEndpoint} to point HMS at an S3-compatible test backend (e.g. RustFS / MinIO running in
+ * another container) so location validation succeeds end-to-end.
+ */
+public final class HmsContainer extends GenericContainer<HmsContainer> implements AutoCloseable {
+
+ private static final int THRIFT_PORT = 9083;
+ private static final String DOCKERFILE_RESOURCE = "Dockerfile-hms-version";
+
+ private String hostPort;
+ private String thriftUri;
+ private String s3aEndpoint;
+ private String s3aAccessKey;
+ private String s3aSecretKey;
+
+ @SuppressWarnings("resource")
+ public HmsContainer(String image) {
+ super(buildImage(image));
+ withLogConsumer(new Slf4jLogConsumer(LoggerFactory.getLogger(HmsContainer.class)));
+ addExposedPort(THRIFT_PORT);
+ // The apache/hive image dispatches on this env var:
+ // metastore -> launches HiveMetaStore (Thrift on 9083)
+ // hiveserver2 -> launches HiveServer2
+ withEnv("SERVICE_NAME", "metastore");
+ // Wait until the Thrift port inside the container is listening. The apache/hive
+ // entrypoint runs schema initialization (Derby) before launching the metastore, which
+ // can take ~30s on a cold start.
+ setWaitStrategy(Wait.forListeningPort().withStartupTimeout(Duration.ofMinutes(3)));
+ }
+
+ public HmsContainer() {
+ this(null);
+ }
+
+ /**
+ * Configures HMS with credentials and an endpoint for S3-compatible storage so HMS can validate
+ * {@code s3a://} table locations during {@code create_table}. Must be called before {@link
+ * #start()}.
+ */
+ public HmsContainer withS3aEndpoint(String endpoint, String accessKey, String secretKey) {
+ this.s3aEndpoint = endpoint;
+ this.s3aAccessKey = accessKey;
+ this.s3aSecretKey = secretKey;
+ return this;
+ }
+
+ private static ImageFromDockerfile buildImage(String image) {
+ DockerImageName base =
+ ContainerSpecHelper.containerSpecHelper("hms", HmsContainer.class).dockerImageName(image);
+ String dockerfile =
+ readDockerfile().replaceFirst("(?m)^FROM .*$", "FROM " + base.asCanonicalNameString());
+ return new ImageFromDockerfile("polaris-hms-with-aws", false)
+ .withFileFromString("Dockerfile", dockerfile);
+ }
+
+ private static String readDockerfile() {
+ try (InputStream in = HmsContainer.class.getResourceAsStream(DOCKERFILE_RESOURCE)) {
+ Objects.requireNonNull(in, DOCKERFILE_RESOURCE + " not found on classpath");
+ return new String(in.readAllBytes(), StandardCharsets.UTF_8);
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ }
+
+ public String hostPort() {
+ Preconditions.checkState(hostPort != null, "Container not yet started");
+ return hostPort;
+ }
+
+ public String thriftUri() {
+ Preconditions.checkState(thriftUri != null, "Container not yet started");
+ return thriftUri;
+ }
+
+ @Override
+ public void start() {
+ if (s3aEndpoint != null) {
+ withCopyToContainer(
+ Transferable.of(buildHiveSiteXml(), 0644), "/opt/hive/conf/hive-site.xml");
+ }
+ super.start();
+ this.hostPort = getHost() + ":" + getMappedPort(THRIFT_PORT);
+ this.thriftUri = "thrift://" + this.hostPort;
+ }
+
+ private String buildHiveSiteXml() {
+ return "<?xml version=\"1.0\"?>\n"
+ + "<configuration>\n"
+ + property("fs.s3a.endpoint", s3aEndpoint)
+ + property("fs.s3a.access.key", s3aAccessKey)
+ + property("fs.s3a.secret.key", s3aSecretKey)
+ + property("fs.s3a.path.style.access", "true")
+ + property("fs.s3a.connection.ssl.enabled", "false")
+ // SimpleAWSCredentialsProvider reads the access/secret directly from the keys above.
+ + property(
+ "fs.s3a.aws.credentials.provider",
+ "org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider")
+ + "</configuration>\n";
+ }
+
+ private static String property(String name, String value) {
+ return " <property><name>" + name + "</name><value>" + value + "</value></property>\n";
+ }
+
+ @Override
+ public void close() {
+ stop();
+ }
+}
diff --git a/tools/hms-testcontainer/src/main/resources/org/apache/polaris/test/hms/Dockerfile-hms-version b/tools/hms-testcontainer/src/main/resources/org/apache/polaris/test/hms/Dockerfile-hms-version
new file mode 100644
index 0000000000..514d302b95
--- /dev/null
+++ b/tools/hms-testcontainer/src/main/resources/org/apache/polaris/test/hms/Dockerfile-hms-version
@@ -0,0 +1,44 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# The FROM line is managed by Renovate.
+FROM apache/hive:4.0.1
+
+# The apache/hive:4.x image does not ship hadoop-aws or the AWS SDK in /opt/hive/lib,
+# so HMS cannot resolve s3a:// table locations out of the box. We add them here.
+# The Thrift API is wire-compatible with the older 3.x hive-metastore client we use
+# on the test side (see gradle/libs.versions.toml -> hive-metastore).
+# HADOOP_AWS_VERSION must match the Hadoop version bundled with apache/hive:4.x;
+# AWS_SDK_BUNDLE_VERSION should be pinned for the specific hadoop-aws version.
+ARG HADOOP_AWS_VERSION=3.3.6
+ARG AWS_SDK_BUNDLE_VERSION=1.12.367
+
+USER root
+
+RUN apt-get update && \
+ apt-get install -y --no-install-recommends curl && \
+ rm -rf /var/lib/apt/lists/*
+
+RUN curl -fsSL -o /opt/hive/lib/hadoop-aws.jar \
+ https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/${HADOOP_AWS_VERSION}/hadoop-aws-${HADOOP_AWS_VERSION}.jar
+
+RUN curl -fsSL -o /opt/hive/lib/aws-sdk-bundle.jar \
+ https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-bundle/${AWS_SDK_BUNDLE_VERSION}/aws-java-sdk-bundle-${AWS_SDK_BUNDLE_VERSION}.jar
+
+USER hive