You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@polaris.apache.org by sn...@apache.org on 2026/07/28 11:25:24 UTC

(polaris) branch main updated: Add Floci GCS+ADLS smoke tests to polaris-runtime-service (#5143)

This is an automated email from the ASF dual-hosted git repository.

snazy pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/polaris.git


The following commit(s) were added to refs/heads/main by this push:
     new c339e9072d Add Floci GCS+ADLS smoke tests to polaris-runtime-service (#5143)
c339e9072d is described below

commit c339e9072d546fe82455b9a36fd743ec737c5113
Author: Robert Stupp <sn...@snazy.de>
AuthorDate: Tue Jul 28 13:25:20 2026 +0200

    Add Floci GCS+ADLS smoke tests to polaris-runtime-service (#5143)
---
 runtime/service/build.gradle.kts                   |   2 +
 .../service/it/PolarisRestCatalogFlociAdlsIT.java  | 193 +++++++++++++++++++++
 .../service/it/PolarisRestCatalogFlociGcpIT.java   | 192 ++++++++++++++++++++
 3 files changed, 387 insertions(+)

diff --git a/runtime/service/build.gradle.kts b/runtime/service/build.gradle.kts
index f0efff51de..e85068a982 100644
--- a/runtime/service/build.gradle.kts
+++ b/runtime/service/build.gradle.kts
@@ -157,6 +157,8 @@ dependencies {
 
   testImplementation(platform(libs.testcontainers.bom))
   testImplementation(project(":polaris-floci-aws-testcontainer"))
+  testImplementation(project(":polaris-floci-az-testcontainer"))
+  testImplementation(project(":polaris-floci-gcp-testcontainer"))
   testImplementation(project(":polaris-keycloak-testcontainer"))
 
   testImplementation(project(":polaris-runtime-test-common"))
diff --git a/runtime/service/src/intTest/java/org/apache/polaris/service/it/PolarisRestCatalogFlociAdlsIT.java b/runtime/service/src/intTest/java/org/apache/polaris/service/it/PolarisRestCatalogFlociAdlsIT.java
new file mode 100644
index 0000000000..2f3ea38094
--- /dev/null
+++ b/runtime/service/src/intTest/java/org/apache/polaris/service/it/PolarisRestCatalogFlociAdlsIT.java
@@ -0,0 +1,193 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.service.it;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.iceberg.CatalogProperties.TABLE_DEFAULT_PREFIX;
+import static org.apache.iceberg.types.Types.NestedField.optional;
+import static org.apache.iceberg.types.Types.NestedField.required;
+import static org.apache.polaris.service.it.env.PolarisClient.polarisClient;
+import static org.assertj.core.api.Assertions.assertThat;
+
+import com.google.common.collect.ImmutableMap;
+import io.quarkus.test.common.QuarkusTestResource;
+import io.quarkus.test.common.ResourceArg;
+import io.quarkus.test.junit.QuarkusIntegrationTest;
+import java.io.IOException;
+import java.net.URI;
+import java.util.List;
+import java.util.Map;
+import org.apache.iceberg.DataFiles;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.rest.RESTCatalog;
+import org.apache.polaris.core.admin.model.AzureStorageConfigInfo;
+import org.apache.polaris.core.admin.model.Catalog;
+import org.apache.polaris.core.admin.model.CatalogProperties;
+import org.apache.polaris.core.admin.model.PolarisCatalog;
+import org.apache.polaris.core.admin.model.PrincipalWithCredentials;
+import org.apache.polaris.core.admin.model.StorageConfigInfo;
+import org.apache.polaris.core.config.FeatureConfiguration;
+import org.apache.polaris.service.it.env.ClientCredentials;
+import org.apache.polaris.service.it.env.IcebergHelper;
+import org.apache.polaris.service.it.env.ManagementApi;
+import org.apache.polaris.service.it.env.PolarisApiEndpoints;
+import org.apache.polaris.service.it.env.PolarisClient;
+import org.apache.polaris.service.it.ext.PolarisIntegrationTestExtension;
+import org.apache.polaris.test.floci.az.FlociAzure;
+import org.apache.polaris.test.floci.az.FlociAzureAccess;
+import org.apache.polaris.test.floci.az.FlociAzureTestResource;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestInfo;
+import org.junit.jupiter.api.extension.ExtendWith;
+
+@QuarkusIntegrationTest
+@QuarkusTestResource(
+    value = FlociAzureTestResource.class,
+    initArgs = {
+      @ResourceArg(name = "storageContainer", value = "floci-az-test-polaris"),
+      @ResourceArg(name = "account", value = "polarisfloci"),
+      @ResourceArg(name = "secret", value = "polaris-floci-azure-secret")
+    })
+@ExtendWith(PolarisIntegrationTestExtension.class)
+public class PolarisRestCatalogFlociAdlsIT {
+  private static final Schema SCHEMA =
+      new Schema(
+          required(1, "id", org.apache.iceberg.types.Types.IntegerType.get(), "doc"),
+          optional(2, "data", org.apache.iceberg.types.Types.StringType.get()));
+
+  @FlociAzure static FlociAzureAccess flociAzureAccess;
+
+  private static PolarisClient client;
+  private static ManagementApi managementApi;
+  private static PolarisApiEndpoints endpoints;
+  private static String adminToken;
+
+  private PrincipalWithCredentials principalCredentials;
+  private String catalogName;
+
+  @BeforeAll
+  static void setup(PolarisApiEndpoints apiEndpoints, ClientCredentials adminCredentials) {
+    endpoints = apiEndpoints;
+    client = polarisClient(endpoints);
+    adminToken = client.obtainToken(adminCredentials);
+    managementApi = client.managementApi(adminToken);
+  }
+
+  @AfterAll
+  static void close() throws Exception {
+    client.close();
+  }
+
+  @BeforeEach
+  void before(TestInfo testInfo) {
+    var principalName = client.newEntityName("floci-adls-user");
+    var principalRoleName = client.newEntityName("floci-adls-admin");
+    principalCredentials = managementApi.createPrincipalWithRole(principalName, principalRoleName);
+    catalogName = client.newEntityName(testInfo.getTestMethod().orElseThrow().getName());
+
+    managementApi.createCatalog(principalRoleName, catalog());
+  }
+
+  @AfterEach
+  void cleanUp() {
+    client.cleanUp(adminToken);
+  }
+
+  @Test
+  void createsTableAndWritesData() throws IOException {
+    try (var restCatalog = restCatalog()) {
+      var namespace = Namespace.of("test_ns");
+      restCatalog.createNamespace(namespace);
+
+      var id = TableIdentifier.of(namespace, "t1");
+      var table = restCatalog.createTable(id, SCHEMA);
+      assertThat(restCatalog.tableExists(id)).isTrue();
+
+      @SuppressWarnings("resource")
+      var io = table.io();
+      var location = URI.create(table.locationProvider().newDataLocation("test-file.txt"));
+      var outputFile = io.newOutputFile(location.toString());
+      var fileContent = "Hello Floci ADLS".getBytes(UTF_8);
+      try (var outputStream = outputFile.create()) {
+        outputStream.write(fileContent);
+      }
+
+      try (var inputStream = io.newInputFile(outputFile.location()).newStream()) {
+        assertThat(inputStream.readAllBytes()).isEqualTo(fileContent);
+      }
+
+      var dataFile =
+          DataFiles.builder(PartitionSpec.unpartitioned())
+              .withPath(outputFile.location())
+              .withFormat(FileFormat.PARQUET)
+              .withFileSizeInBytes(fileContent.length)
+              .withRecordCount(1)
+              .build();
+      table.newAppend().appendFile(dataFile).commit();
+
+      assertThat(restCatalog.loadTable(id).currentSnapshot()).isNotNull();
+    }
+  }
+
+  private Catalog catalog() {
+    var baseLocation = flociAzureAccess.location("polaris/" + catalogName);
+    var catalogProps = CatalogProperties.builder(baseLocation);
+    catalogProps.addProperty(FeatureConfiguration.DROP_WITH_PURGE_ENABLED.catalogConfig(), "true");
+    flociAzureAccess
+        .icebergProperties()
+        .forEach((key, value) -> catalogProps.addProperty(TABLE_DEFAULT_PREFIX + key, value));
+
+    return PolarisCatalog.builder()
+        .setType(Catalog.TypeEnum.INTERNAL)
+        .setName(catalogName)
+        .setStorageConfigInfo(
+            AzureStorageConfigInfo.builder()
+                .setStorageType(StorageConfigInfo.StorageTypeEnum.AZURE)
+                .setTenantId(flociAzureAccess.account())
+                .setAllowedLocations(List.of(baseLocation))
+                .setHierarchical(true)
+                .build())
+        .setProperties(catalogProps.build())
+        .build();
+  }
+
+  private RESTCatalog restCatalog() {
+    var authToken = client.obtainToken(principalCredentials);
+    return IcebergHelper.restCatalog(endpoints, catalogName, restCatalogProperties(), authToken);
+  }
+
+  private Map<String, String> restCatalogProperties() {
+    return ImmutableMap.<String, String>builder()
+        .putAll(flociAzureAccess.icebergProperties())
+        .putAll(
+            flociAzureAccess.icebergProperties().entrySet().stream()
+                .collect(
+                    ImmutableMap.toImmutableMap(
+                        entry -> TABLE_DEFAULT_PREFIX + entry.getKey(), Map.Entry::getValue)))
+        .buildKeepingLast();
+  }
+}
diff --git a/runtime/service/src/intTest/java/org/apache/polaris/service/it/PolarisRestCatalogFlociGcpIT.java b/runtime/service/src/intTest/java/org/apache/polaris/service/it/PolarisRestCatalogFlociGcpIT.java
new file mode 100644
index 0000000000..5289328893
--- /dev/null
+++ b/runtime/service/src/intTest/java/org/apache/polaris/service/it/PolarisRestCatalogFlociGcpIT.java
@@ -0,0 +1,192 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.service.it;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.iceberg.CatalogProperties.TABLE_DEFAULT_PREFIX;
+import static org.apache.iceberg.types.Types.NestedField.optional;
+import static org.apache.iceberg.types.Types.NestedField.required;
+import static org.apache.polaris.service.it.env.PolarisClient.polarisClient;
+import static org.assertj.core.api.Assertions.assertThat;
+
+import com.google.common.collect.ImmutableMap;
+import io.quarkus.test.common.QuarkusTestResource;
+import io.quarkus.test.common.ResourceArg;
+import io.quarkus.test.junit.QuarkusIntegrationTest;
+import java.io.IOException;
+import java.net.URI;
+import java.util.List;
+import java.util.Map;
+import org.apache.iceberg.DataFiles;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.rest.RESTCatalog;
+import org.apache.polaris.core.admin.model.Catalog;
+import org.apache.polaris.core.admin.model.CatalogProperties;
+import org.apache.polaris.core.admin.model.GcpStorageConfigInfo;
+import org.apache.polaris.core.admin.model.PolarisCatalog;
+import org.apache.polaris.core.admin.model.PrincipalWithCredentials;
+import org.apache.polaris.core.admin.model.StorageConfigInfo;
+import org.apache.polaris.core.config.FeatureConfiguration;
+import org.apache.polaris.service.it.env.ClientCredentials;
+import org.apache.polaris.service.it.env.IcebergHelper;
+import org.apache.polaris.service.it.env.ManagementApi;
+import org.apache.polaris.service.it.env.PolarisApiEndpoints;
+import org.apache.polaris.service.it.env.PolarisClient;
+import org.apache.polaris.service.it.ext.PolarisIntegrationTestExtension;
+import org.apache.polaris.test.floci.gcp.FlociGcp;
+import org.apache.polaris.test.floci.gcp.FlociGcpAccess;
+import org.apache.polaris.test.floci.gcp.FlociGcpTestResource;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestInfo;
+import org.junit.jupiter.api.extension.ExtendWith;
+
+@QuarkusIntegrationTest
+@QuarkusTestResource(
+    value = FlociGcpTestResource.class,
+    initArgs = {
+      @ResourceArg(name = "bucket", value = "floci-gcp-test-polaris"),
+      @ResourceArg(name = "projectId", value = "polaris-floci-gcp"),
+      @ResourceArg(name = "oauth2Token", value = "polaris-floci-gcp-token")
+    })
+@ExtendWith(PolarisIntegrationTestExtension.class)
+public class PolarisRestCatalogFlociGcpIT {
+  private static final Schema SCHEMA =
+      new Schema(
+          required(1, "id", org.apache.iceberg.types.Types.IntegerType.get(), "doc"),
+          optional(2, "data", org.apache.iceberg.types.Types.StringType.get()));
+
+  @FlociGcp static FlociGcpAccess flociGcpAccess;
+
+  private static PolarisClient client;
+  private static ManagementApi managementApi;
+  private static PolarisApiEndpoints endpoints;
+  private static String adminToken;
+
+  private PrincipalWithCredentials principalCredentials;
+  private String catalogName;
+
+  @BeforeAll
+  static void setup(PolarisApiEndpoints apiEndpoints, ClientCredentials adminCredentials) {
+    endpoints = apiEndpoints;
+    client = polarisClient(endpoints);
+    adminToken = client.obtainToken(adminCredentials);
+    managementApi = client.managementApi(adminToken);
+  }
+
+  @AfterAll
+  static void close() throws Exception {
+    client.close();
+  }
+
+  @BeforeEach
+  void before(TestInfo testInfo) {
+    var principalName = client.newEntityName("floci-gcp-user");
+    var principalRoleName = client.newEntityName("floci-gcp-admin");
+    principalCredentials = managementApi.createPrincipalWithRole(principalName, principalRoleName);
+    catalogName = client.newEntityName(testInfo.getTestMethod().orElseThrow().getName());
+
+    managementApi.createCatalog(principalRoleName, catalog());
+  }
+
+  @AfterEach
+  void cleanUp() {
+    client.cleanUp(adminToken);
+  }
+
+  @Test
+  void createsTableAndWritesData() throws IOException {
+    try (var restCatalog = restCatalog()) {
+      var namespace = Namespace.of("test_ns");
+      restCatalog.createNamespace(namespace);
+
+      var id = TableIdentifier.of(namespace, "t1");
+      var table = restCatalog.createTable(id, SCHEMA);
+      assertThat(restCatalog.tableExists(id)).isTrue();
+
+      @SuppressWarnings("resource")
+      var io = table.io();
+      var location = URI.create(table.locationProvider().newDataLocation("test-file.txt"));
+      var outputFile = io.newOutputFile(location.toString());
+      var fileContent = "Hello Floci GCP".getBytes(UTF_8);
+      try (var outputStream = outputFile.create()) {
+        outputStream.write(fileContent);
+      }
+
+      try (var inputStream = io.newInputFile(outputFile.location()).newStream()) {
+        assertThat(inputStream.readAllBytes()).isEqualTo(fileContent);
+      }
+
+      var dataFile =
+          DataFiles.builder(PartitionSpec.unpartitioned())
+              .withPath(outputFile.location())
+              .withFormat(FileFormat.PARQUET)
+              .withFileSizeInBytes(fileContent.length)
+              .withRecordCount(1)
+              .build();
+      table.newAppend().appendFile(dataFile).commit();
+
+      assertThat(restCatalog.loadTable(id).currentSnapshot()).isNotNull();
+    }
+  }
+
+  private Catalog catalog() {
+    var baseLocation = flociGcpAccess.bucketUri("polaris/" + catalogName).toString();
+    var catalogProps = CatalogProperties.builder(baseLocation);
+    catalogProps.addProperty(FeatureConfiguration.DROP_WITH_PURGE_ENABLED.catalogConfig(), "true");
+    flociGcpAccess
+        .icebergProperties()
+        .forEach((key, value) -> catalogProps.addProperty(TABLE_DEFAULT_PREFIX + key, value));
+
+    return PolarisCatalog.builder()
+        .setType(Catalog.TypeEnum.INTERNAL)
+        .setName(catalogName)
+        .setStorageConfigInfo(
+            GcpStorageConfigInfo.builder()
+                .setGcsServiceAccount(flociGcpAccess.projectId())
+                .setStorageType(StorageConfigInfo.StorageTypeEnum.GCS)
+                .setAllowedLocations(List.of(baseLocation))
+                .build())
+        .setProperties(catalogProps.build())
+        .build();
+  }
+
+  private RESTCatalog restCatalog() {
+    var authToken = client.obtainToken(principalCredentials);
+    return IcebergHelper.restCatalog(endpoints, catalogName, restCatalogProperties(), authToken);
+  }
+
+  private Map<String, String> restCatalogProperties() {
+    return ImmutableMap.<String, String>builder()
+        .putAll(flociGcpAccess.icebergProperties())
+        .putAll(
+            flociGcpAccess.icebergProperties().entrySet().stream()
+                .collect(
+                    ImmutableMap.toImmutableMap(
+                        entry -> TABLE_DEFAULT_PREFIX + entry.getKey(), Map.Entry::getValue)))
+        .buildKeepingLast();
+  }
+}