You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@polaris.apache.org by ru...@apache.org on 2024/09/24 22:04:06 UTC

(polaris) branch main updated: Add SUPPORTED_CATALOG_STORAGE_TYPES to PolarisConfiguration (#165)

This is an automated email from the ASF dual-hosted git repository.

russellspitzer pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/polaris.git


The following commit(s) were added to refs/heads/main by this push:
     new 413eca7  Add SUPPORTED_CATALOG_STORAGE_TYPES to PolarisConfiguration (#165)
413eca7 is described below

commit 413eca7147fc239d7748ae8c3b9a336e6174b956
Author: Eric Maynard <er...@snowflake.com>
AuthorDate: Tue Sep 24 18:04:02 2024 -0400

    Add SUPPORTED_CATALOG_STORAGE_TYPES to PolarisConfiguration (#165)
---
 .../apache/polaris/core/PolarisConfiguration.java  | 17 ++++++++++-
 .../polaris/core/PolarisConfigurationStore.java    |  3 ++
 .../polaris/service/admin/PolarisServiceImpl.java  |  9 ++----
 .../service/catalog/BasePolarisCatalog.java        | 26 +++++++++++-----
 .../service/catalog/BasePolarisCatalogTest.java    | 35 ++++++++++++++++------
 5 files changed, 65 insertions(+), 25 deletions(-)

diff --git a/polaris-core/src/main/java/org/apache/polaris/core/PolarisConfiguration.java b/polaris-core/src/main/java/org/apache/polaris/core/PolarisConfiguration.java
index ea15e9b..397c9af 100644
--- a/polaris-core/src/main/java/org/apache/polaris/core/PolarisConfiguration.java
+++ b/polaris-core/src/main/java/org/apache/polaris/core/PolarisConfiguration.java
@@ -18,7 +18,9 @@
  */
 package org.apache.polaris.core;
 
+import java.util.List;
 import java.util.Optional;
+import org.apache.polaris.core.admin.model.StorageConfigInfo;
 
 public class PolarisConfiguration<T> {
 
@@ -115,7 +117,7 @@ public class PolarisConfiguration<T> {
       PolarisConfiguration.<Boolean>builder()
           .key("ALLOW_NAMESPACE_LOCATION_OVERLAP")
           .description(
-              "If set to true, allow one table's location to reside within another table's location. "
+              "If set to true, allow one namespace's location to reside within another namespace's location. "
                   + "This is only enforced within a parent catalog or namespace.")
           .defaultValue(false)
           .build();
@@ -152,6 +154,19 @@ public class PolarisConfiguration<T> {
           .defaultValue(false)
           .build();
 
+  public static final PolarisConfiguration<List<String>> SUPPORTED_CATALOG_STORAGE_TYPES =
+      PolarisConfiguration.<List<String>>builder()
+          .key("SUPPORTED_CATALOG_STORAGE_TYPES")
+          .catalogConfig("supported.storage.types")
+          .description("The list of supported storage types for a catalog")
+          .defaultValue(
+              List.of(
+                  StorageConfigInfo.StorageTypeEnum.S3.name(),
+                  StorageConfigInfo.StorageTypeEnum.AZURE.name(),
+                  StorageConfigInfo.StorageTypeEnum.GCS.name(),
+                  StorageConfigInfo.StorageTypeEnum.FILE.name()))
+          .build();
+
   public static final PolarisConfiguration<Boolean> CLEANUP_ON_NAMESPACE_DROP =
       PolarisConfiguration.<Boolean>builder()
           .key("CLEANUP_ON_NAMESPACE_DROP")
diff --git a/polaris-core/src/main/java/org/apache/polaris/core/PolarisConfigurationStore.java b/polaris-core/src/main/java/org/apache/polaris/core/PolarisConfigurationStore.java
index a249ed9..f306a91 100644
--- a/polaris-core/src/main/java/org/apache/polaris/core/PolarisConfigurationStore.java
+++ b/polaris-core/src/main/java/org/apache/polaris/core/PolarisConfigurationStore.java
@@ -19,6 +19,7 @@
 package org.apache.polaris.core;
 
 import com.google.common.base.Preconditions;
+import java.util.List;
 import org.apache.polaris.core.entity.CatalogEntity;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
@@ -68,6 +69,8 @@ public interface PolarisConfigurationStore {
 
     if (config.defaultValue instanceof Boolean) {
       return config.cast(Boolean.valueOf(String.valueOf(value)));
+    } else if (config.defaultValue instanceof List<?>) {
+      return config.cast(List.copyOf((List<?>) value));
     } else {
       return config.cast(value);
     }
diff --git a/polaris-service/src/main/java/org/apache/polaris/service/admin/PolarisServiceImpl.java b/polaris-service/src/main/java/org/apache/polaris/service/admin/PolarisServiceImpl.java
index 653b962..0befb0a 100644
--- a/polaris-service/src/main/java/org/apache/polaris/service/admin/PolarisServiceImpl.java
+++ b/polaris-service/src/main/java/org/apache/polaris/service/admin/PolarisServiceImpl.java
@@ -25,6 +25,7 @@ import org.apache.iceberg.catalog.Namespace;
 import org.apache.iceberg.catalog.TableIdentifier;
 import org.apache.iceberg.exceptions.NotAuthorizedException;
 import org.apache.polaris.core.PolarisCallContext;
+import org.apache.polaris.core.PolarisConfiguration;
 import org.apache.polaris.core.admin.model.AddGrantRequest;
 import org.apache.polaris.core.admin.model.Catalog;
 import org.apache.polaris.core.admin.model.CatalogGrant;
@@ -118,13 +119,7 @@ public class PolarisServiceImpl
         polarisCallContext
             .getConfigurationStore()
             .getConfiguration(
-                polarisCallContext,
-                "SUPPORTED_CATALOG_STORAGE_TYPES",
-                List.of(
-                    StorageConfigInfo.StorageTypeEnum.S3.name(),
-                    StorageConfigInfo.StorageTypeEnum.AZURE.name(),
-                    StorageConfigInfo.StorageTypeEnum.GCS.name(),
-                    StorageConfigInfo.StorageTypeEnum.FILE.name()));
+                polarisCallContext, PolarisConfiguration.SUPPORTED_CATALOG_STORAGE_TYPES);
     if (!allowedStorageTypes.contains(storageConfigInfo.getStorageType().name())) {
       LOGGER
           .atWarn()
diff --git a/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java b/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java
index 0df8717..3d926fb 100644
--- a/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java
+++ b/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java
@@ -75,6 +75,7 @@ import org.apache.iceberg.view.ViewOperations;
 import org.apache.iceberg.view.ViewUtil;
 import org.apache.polaris.core.PolarisCallContext;
 import org.apache.polaris.core.PolarisConfiguration;
+import org.apache.polaris.core.admin.model.StorageConfigInfo;
 import org.apache.polaris.core.auth.AuthenticatedPolarisPrincipal;
 import org.apache.polaris.core.catalog.PolarisCatalogHelpers;
 import org.apache.polaris.core.context.CallContext;
@@ -986,14 +987,23 @@ public class BasePolarisCatalog extends BaseMetastoreViewCatalog
           // }
         },
         () -> {
-          List<String> invalidLocations =
-              locations.stream()
-                  .filter(location -> location.startsWith("file:") || location.startsWith("http"))
-                  .collect(Collectors.toList());
-          if (!invalidLocations.isEmpty()) {
-            throw new ForbiddenException(
-                "Invalid locations '%s' for identifier '%s': File locations are not allowed",
-                invalidLocations, identifier);
+          List<String> allowedStorageTypes =
+              callContext
+                  .getPolarisCallContext()
+                  .getConfigurationStore()
+                  .getConfiguration(
+                      callContext.getPolarisCallContext(),
+                      PolarisConfiguration.SUPPORTED_CATALOG_STORAGE_TYPES);
+          if (!allowedStorageTypes.contains(StorageConfigInfo.StorageTypeEnum.FILE.name())) {
+            List<String> invalidLocations =
+                locations.stream()
+                    .filter(location -> location.startsWith("file:") || location.startsWith("http"))
+                    .collect(Collectors.toList());
+            if (!invalidLocations.isEmpty()) {
+              throw new ForbiddenException(
+                  "Invalid locations '%s' for identifier '%s': File locations are not allowed",
+                  invalidLocations, identifier);
+            }
           }
         });
   }
diff --git a/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java b/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java
index c16b93c..63f859b 100644
--- a/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java
+++ b/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java
@@ -688,9 +688,15 @@ public class BasePolarisCatalogTest extends CatalogTests<BasePolarisCatalog> {
         metadataLocation,
         TableMetadataParser.toJson(createSampleTableMetadata(metadataLocation)).getBytes(UTF_8));
 
-    Assertions.assertThatThrownBy(() -> catalog.sendNotification(table, request))
-        .isInstanceOf(ForbiddenException.class)
-        .hasMessageContaining("Invalid location");
+    PolarisCallContext polarisCallContext = callContext.getPolarisCallContext();
+    if (!polarisCallContext
+        .getConfigurationStore()
+        .getConfiguration(polarisCallContext, PolarisConfiguration.SUPPORTED_CATALOG_STORAGE_TYPES)
+        .contains("FILE")) {
+      Assertions.assertThatThrownBy(() -> catalog.sendNotification(table, request))
+          .isInstanceOf(ForbiddenException.class)
+          .hasMessageContaining("Invalid location");
+    }
   }
 
   @Test
@@ -748,9 +754,15 @@ public class BasePolarisCatalogTest extends CatalogTests<BasePolarisCatalog> {
         metadataLocation,
         TableMetadataParser.toJson(createSampleTableMetadata(metadataLocation)).getBytes(UTF_8));
 
-    Assertions.assertThatThrownBy(() -> catalog.sendNotification(table, request))
-        .isInstanceOf(ForbiddenException.class)
-        .hasMessageContaining("Invalid location");
+    PolarisCallContext polarisCallContext = callContext.getPolarisCallContext();
+    if (!polarisCallContext
+        .getConfigurationStore()
+        .getConfiguration(polarisCallContext, PolarisConfiguration.SUPPORTED_CATALOG_STORAGE_TYPES)
+        .contains("FILE")) {
+      Assertions.assertThatThrownBy(() -> catalog.sendNotification(table, request))
+          .isInstanceOf(ForbiddenException.class)
+          .hasMessageContaining("Invalid location");
+    }
 
     // It also fails if we try to use https
     final String httpsMetadataLocation = "https://maliciousdomain.com/metadata.json";
@@ -764,9 +776,14 @@ public class BasePolarisCatalogTest extends CatalogTests<BasePolarisCatalog> {
         httpsMetadataLocation,
         TableMetadataParser.toJson(createSampleTableMetadata(metadataLocation)).getBytes(UTF_8));
 
-    Assertions.assertThatThrownBy(() -> catalog.sendNotification(table, newRequest))
-        .isInstanceOf(ForbiddenException.class)
-        .hasMessageContaining("Invalid location");
+    if (!polarisCallContext
+        .getConfigurationStore()
+        .getConfiguration(polarisCallContext, PolarisConfiguration.SUPPORTED_CATALOG_STORAGE_TYPES)
+        .contains("FILE")) {
+      Assertions.assertThatThrownBy(() -> catalog.sendNotification(table, newRequest))
+          .isInstanceOf(ForbiddenException.class)
+          .hasMessageContaining("Invalid location");
+    }
   }
 
   @Test