You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@polaris.apache.org by sn...@apache.org on 2024/11/29 15:50:53 UTC

(polaris) branch main updated: Remove dropwizard-jackson dep from core (#470)

This is an automated email from the ASF dual-hosted git repository.

snazy pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/polaris.git


The following commit(s) were added to refs/heads/main by this push:
     new a6197bd7 Remove dropwizard-jackson dep from core (#470)
a6197bd7 is described below

commit a6197bd7d8cb5551253fa427e4373897205ecece
Author: Dmitri Bourlatchkov <dm...@dremio.com>
AuthorDate: Fri Nov 29 10:50:47 2024 -0500

    Remove dropwizard-jackson dep from core (#470)
    
    This is a simplification / cleanup. The dependency does not appear to be required in `polaris-core`
    
    Add custom code to `PolarisApplication` find classes directly listed in the `Discoverable`
    service descriptor and register them with the ObjectMapper. This approach to finding sub-types
    is consistent both with the java service descriptors (listed types actually implement the
    service interface) and at the same time allows moving the Dropwizard `Discoverable` dependencies
    to the polaris-service module that actually integrates with Dropwizard.
    
    Move leaf metastore classes to the `Discoverable` service descriptor in their respective module.
    
    Note: this fixes the cross-jar leak of EclipseLinkPolarisMetaStoreManagerFactory in service descriptors.
---
 .../EclipseLinkPolarisMetaStoreManagerFactory.java |  3 +-
 .../services/io.dropwizard.jackson.Discoverable    |  1 -
 polaris-core/build.gradle.kts                      |  4 --
 .../core/persistence/MetaStoreManagerFactory.java  |  3 +-
 polaris-service/build.gradle.kts                   |  2 +-
 .../apache/polaris/service/PolarisApplication.java | 16 +++++
 .../InMemoryPolarisMetaStoreManagerFactory.java    |  3 +-
 .../services/io.dropwizard.jackson.Discoverable    |  2 +-
 .../PolarisApplicationConfigurationTest.java       | 78 ++++++++++++++++++++++
 9 files changed, 101 insertions(+), 11 deletions(-)

diff --git a/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/EclipseLinkPolarisMetaStoreManagerFactory.java b/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/EclipseLinkPolarisMetaStoreManagerFactory.java
index b893c543..03c7afab 100644
--- a/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/EclipseLinkPolarisMetaStoreManagerFactory.java
+++ b/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/EclipseLinkPolarisMetaStoreManagerFactory.java
@@ -20,6 +20,7 @@ package org.apache.polaris.extension.persistence.impl.eclipselink;
 
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.annotation.JsonTypeName;
+import io.dropwizard.jackson.Discoverable;
 import jakarta.annotation.Nonnull;
 import org.apache.polaris.core.PolarisDiagnostics;
 import org.apache.polaris.core.context.RealmContext;
@@ -34,7 +35,7 @@ import org.apache.polaris.core.persistence.PolarisMetaStoreSession;
  */
 @JsonTypeName("eclipse-link")
 public class EclipseLinkPolarisMetaStoreManagerFactory
-    extends LocalPolarisMetaStoreManagerFactory<PolarisEclipseLinkStore> {
+    extends LocalPolarisMetaStoreManagerFactory<PolarisEclipseLinkStore> implements Discoverable {
   @JsonProperty("conf-file")
   private String confFile;
 
diff --git a/polaris-service/src/main/resources/META-INF/services/org.apache.polaris.core.persistence.MetaStoreManagerFactory b/extension/persistence/eclipselink/src/main/resources/META-INF/services/io.dropwizard.jackson.Discoverable
similarity index 91%
rename from polaris-service/src/main/resources/META-INF/services/org.apache.polaris.core.persistence.MetaStoreManagerFactory
rename to extension/persistence/eclipselink/src/main/resources/META-INF/services/io.dropwizard.jackson.Discoverable
index 85ae92ca..58f8891e 100644
--- a/polaris-service/src/main/resources/META-INF/services/org.apache.polaris.core.persistence.MetaStoreManagerFactory
+++ b/extension/persistence/eclipselink/src/main/resources/META-INF/services/io.dropwizard.jackson.Discoverable
@@ -17,5 +17,4 @@
 # under the License.
 #
 
-org.apache.polaris.service.persistence.InMemoryPolarisMetaStoreManagerFactory
 org.apache.polaris.extension.persistence.impl.eclipselink.EclipseLinkPolarisMetaStoreManagerFactory
diff --git a/polaris-core/build.gradle.kts b/polaris-core/build.gradle.kts
index 32dd38f9..de086f65 100644
--- a/polaris-core/build.gradle.kts
+++ b/polaris-core/build.gradle.kts
@@ -33,10 +33,6 @@ dependencies {
   constraints {
     implementation("io.airlift:aircompressor:0.27") { because("Vulnerability detected in 0.25") }
   }
-  // TODO - this is only here for the Discoverable interface
-  // We should use a different mechanism to discover the plugin implementations
-  implementation(platform(libs.dropwizard.bom))
-  implementation("io.dropwizard:dropwizard-jackson")
 
   implementation(platform(libs.jackson.bom))
   implementation("com.fasterxml.jackson.core:jackson-annotations")
diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/MetaStoreManagerFactory.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/MetaStoreManagerFactory.java
index 821d139a..4352a975 100644
--- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/MetaStoreManagerFactory.java
+++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/MetaStoreManagerFactory.java
@@ -19,7 +19,6 @@
 package org.apache.polaris.core.persistence;
 
 import com.fasterxml.jackson.annotation.JsonTypeInfo;
-import io.dropwizard.jackson.Discoverable;
 import java.util.List;
 import java.util.Map;
 import java.util.function.Supplier;
@@ -34,7 +33,7 @@ import org.apache.polaris.core.storage.cache.StorageCredentialCache;
  * configuration
  */
 @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "type")
-public interface MetaStoreManagerFactory extends Discoverable {
+public interface MetaStoreManagerFactory {
 
   PolarisMetaStoreManager getOrCreateMetaStoreManager(RealmContext realmContext);
 
diff --git a/polaris-service/build.gradle.kts b/polaris-service/build.gradle.kts
index 370a5bfc..9d99d49f 100644
--- a/polaris-service/build.gradle.kts
+++ b/polaris-service/build.gradle.kts
@@ -118,7 +118,7 @@ dependencies {
   testImplementation(libs.mockito.core)
   testRuntimeOnly("org.junit.platform:junit-platform-launcher")
 
-  testRuntimeOnly(project(":polaris-eclipselink"))
+  testImplementation(project(":polaris-eclipselink"))
 }
 
 if (project.properties.get("eclipseLink") == "true") {
diff --git a/polaris-service/src/main/java/org/apache/polaris/service/PolarisApplication.java b/polaris-service/src/main/java/org/apache/polaris/service/PolarisApplication.java
index facbe76c..89287a3e 100644
--- a/polaris-service/src/main/java/org/apache/polaris/service/PolarisApplication.java
+++ b/polaris-service/src/main/java/org/apache/polaris/service/PolarisApplication.java
@@ -36,6 +36,8 @@ import io.dropwizard.configuration.SubstitutingSourceProvider;
 import io.dropwizard.core.Application;
 import io.dropwizard.core.setup.Bootstrap;
 import io.dropwizard.core.setup.Environment;
+import io.dropwizard.jackson.Discoverable;
+import io.dropwizard.jackson.DiscoverableSubtypeResolver;
 import io.micrometer.prometheusmetrics.PrometheusConfig;
 import io.micrometer.prometheusmetrics.PrometheusMeterRegistry;
 import io.opentelemetry.api.OpenTelemetry;
@@ -63,6 +65,7 @@ import java.io.InputStream;
 import java.net.URL;
 import java.util.Collections;
 import java.util.EnumSet;
+import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.concurrent.Executors;
@@ -142,6 +145,7 @@ public class PolarisApplication extends Application<PolarisApplicationConfig> {
 
   @Override
   public void initialize(Bootstrap<PolarisApplicationConfig> bootstrap) {
+    registerTypes(bootstrap.getObjectMapper());
     // Enable variable substitution with environment variables
     EnvironmentVariableSubstitutor substitutor = new EnvironmentVariableSubstitutor(false);
     SubstitutingSourceProvider provider =
@@ -152,6 +156,18 @@ public class PolarisApplication extends Application<PolarisApplicationConfig> {
     bootstrap.addCommand(new PurgeRealmsCommand());
   }
 
+  private void registerTypes(ObjectMapper mapper) {
+    // Reuse the DW service discovery method, but unlike the constructor of
+    // DiscoverableSubtypeResolver, use the first level classes from the `Discoverable`
+    // service descriptor and register them with the ObjectMapper.
+    class Discoverer extends DiscoverableSubtypeResolver {
+      List<Class<?>> discover() {
+        return discoverServices(Discoverable.class);
+      }
+    }
+    new Discoverer().discover().forEach(mapper::registerSubtypes);
+  }
+
   @Override
   public void run(PolarisApplicationConfig configuration, Environment environment) {
     MetaStoreManagerFactory metaStoreManagerFactory = configuration.getMetaStoreManagerFactory();
diff --git a/polaris-service/src/main/java/org/apache/polaris/service/persistence/InMemoryPolarisMetaStoreManagerFactory.java b/polaris-service/src/main/java/org/apache/polaris/service/persistence/InMemoryPolarisMetaStoreManagerFactory.java
index 8f1f146e..fc830341 100644
--- a/polaris-service/src/main/java/org/apache/polaris/service/persistence/InMemoryPolarisMetaStoreManagerFactory.java
+++ b/polaris-service/src/main/java/org/apache/polaris/service/persistence/InMemoryPolarisMetaStoreManagerFactory.java
@@ -19,6 +19,7 @@
 package org.apache.polaris.service.persistence;
 
 import com.fasterxml.jackson.annotation.JsonTypeName;
+import io.dropwizard.jackson.Discoverable;
 import jakarta.annotation.Nonnull;
 import java.util.Collections;
 import java.util.HashSet;
@@ -36,7 +37,7 @@ import org.apache.polaris.core.persistence.PolarisTreeMapStore;
 
 @JsonTypeName("in-memory")
 public class InMemoryPolarisMetaStoreManagerFactory
-    extends LocalPolarisMetaStoreManagerFactory<PolarisTreeMapStore> {
+    extends LocalPolarisMetaStoreManagerFactory<PolarisTreeMapStore> implements Discoverable {
   final Set<String> bootstrappedRealms = new HashSet<>();
 
   @Override
diff --git a/polaris-service/src/main/resources/META-INF/services/io.dropwizard.jackson.Discoverable b/polaris-service/src/main/resources/META-INF/services/io.dropwizard.jackson.Discoverable
index 95d1f8ec..4c7ab2ac 100644
--- a/polaris-service/src/main/resources/META-INF/services/io.dropwizard.jackson.Discoverable
+++ b/polaris-service/src/main/resources/META-INF/services/io.dropwizard.jackson.Discoverable
@@ -18,7 +18,7 @@
 #
 
 org.apache.polaris.service.auth.DiscoverableAuthenticator
-org.apache.polaris.core.persistence.MetaStoreManagerFactory
+org.apache.polaris.service.persistence.InMemoryPolarisMetaStoreManagerFactory
 org.apache.polaris.service.config.OAuth2ApiService
 org.apache.polaris.service.context.RealmContextResolver
 org.apache.polaris.service.context.CallContextResolver
diff --git a/polaris-service/src/test/java/org/apache/polaris/service/PolarisApplicationConfigurationTest.java b/polaris-service/src/test/java/org/apache/polaris/service/PolarisApplicationConfigurationTest.java
new file mode 100644
index 00000000..74442e04
--- /dev/null
+++ b/polaris-service/src/test/java/org/apache/polaris/service/PolarisApplicationConfigurationTest.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.service;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import io.dropwizard.testing.ConfigOverride;
+import io.dropwizard.testing.ResourceHelpers;
+import io.dropwizard.testing.junit5.DropwizardAppExtension;
+import io.dropwizard.testing.junit5.DropwizardExtensionsSupport;
+import org.apache.polaris.extension.persistence.impl.eclipselink.EclipseLinkPolarisMetaStoreManagerFactory;
+import org.apache.polaris.service.config.PolarisApplicationConfig;
+import org.apache.polaris.service.persistence.InMemoryPolarisMetaStoreManagerFactory;
+import org.junit.jupiter.api.Nested;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+
+@ExtendWith(DropwizardExtensionsSupport.class)
+public class PolarisApplicationConfigurationTest {
+
+  public static final String CONFIG_PATH =
+      ResourceHelpers.resourceFilePath("polaris-server-integrationtest.yml");
+  // Bind to random ports to support parallelism
+  public static final ConfigOverride RANDOM_APP_PORT =
+      ConfigOverride.config("server.applicationConnectors[0].port", "0");
+  public static final ConfigOverride RANDOM_ADMIN_PORT =
+      ConfigOverride.config("server.adminConnectors[0].port", "0");
+
+  @Nested
+  class DefaultMetastore {
+    private final DropwizardAppExtension<PolarisApplicationConfig> app =
+        new DropwizardAppExtension<>(
+            PolarisApplication.class, CONFIG_PATH, RANDOM_APP_PORT, RANDOM_ADMIN_PORT);
+
+    @Test
+    void testMetastoreType() {
+      assertThat(app.getConfiguration().getMetaStoreManagerFactory())
+          .isInstanceOf(InMemoryPolarisMetaStoreManagerFactory.class);
+    }
+  }
+
+  @Nested
+  class EclipseLinkMetastore {
+    private final DropwizardAppExtension<PolarisApplicationConfig> app =
+        new DropwizardAppExtension<>(
+            PolarisApplication.class,
+            CONFIG_PATH,
+            RANDOM_APP_PORT,
+            RANDOM_ADMIN_PORT,
+            ConfigOverride.config("metaStoreManager.type", "eclipse-link"),
+            ConfigOverride.config("metaStoreManager.persistence-unit", "test-unit"),
+            ConfigOverride.config("metaStoreManager.conf-file", "/test-conf-file"));
+
+    @Test
+    void testMetastoreType() {
+      assertThat(app.getConfiguration().getMetaStoreManagerFactory())
+          .isInstanceOf(EclipseLinkPolarisMetaStoreManagerFactory.class)
+          .extracting("persistenceUnitName", "confFile")
+          .containsExactly("test-unit", "/test-conf-file");
+    }
+  }
+}