You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@polaris.apache.org by sn...@apache.org on 2024/11/29 15:50:53 UTC
(polaris) branch main updated: Remove dropwizard-jackson dep from core (#470)
This is an automated email from the ASF dual-hosted git repository.
snazy pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/polaris.git
The following commit(s) were added to refs/heads/main by this push:
new a6197bd7 Remove dropwizard-jackson dep from core (#470)
a6197bd7 is described below
commit a6197bd7d8cb5551253fa427e4373897205ecece
Author: Dmitri Bourlatchkov <dm...@dremio.com>
AuthorDate: Fri Nov 29 10:50:47 2024 -0500
Remove dropwizard-jackson dep from core (#470)
This is a simplification / cleanup. The dependency does not appear to be required in `polaris-core`
Add custom code to `PolarisApplication` find classes directly listed in the `Discoverable`
service descriptor and register them with the ObjectMapper. This approach to finding sub-types
is consistent both with the java service descriptors (listed types actually implement the
service interface) and at the same time allows moving the Dropwizard `Discoverable` dependencies
to the polaris-service module that actually integrates with Dropwizard.
Move leaf metastore classes to the `Discoverable` service descriptor in their respective module.
Note: this fixes the cross-jar leak of EclipseLinkPolarisMetaStoreManagerFactory in service descriptors.
---
.../EclipseLinkPolarisMetaStoreManagerFactory.java | 3 +-
.../services/io.dropwizard.jackson.Discoverable | 1 -
polaris-core/build.gradle.kts | 4 --
.../core/persistence/MetaStoreManagerFactory.java | 3 +-
polaris-service/build.gradle.kts | 2 +-
.../apache/polaris/service/PolarisApplication.java | 16 +++++
.../InMemoryPolarisMetaStoreManagerFactory.java | 3 +-
.../services/io.dropwizard.jackson.Discoverable | 2 +-
.../PolarisApplicationConfigurationTest.java | 78 ++++++++++++++++++++++
9 files changed, 101 insertions(+), 11 deletions(-)
diff --git a/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/EclipseLinkPolarisMetaStoreManagerFactory.java b/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/EclipseLinkPolarisMetaStoreManagerFactory.java
index b893c543..03c7afab 100644
--- a/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/EclipseLinkPolarisMetaStoreManagerFactory.java
+++ b/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/EclipseLinkPolarisMetaStoreManagerFactory.java
@@ -20,6 +20,7 @@ package org.apache.polaris.extension.persistence.impl.eclipselink;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonTypeName;
+import io.dropwizard.jackson.Discoverable;
import jakarta.annotation.Nonnull;
import org.apache.polaris.core.PolarisDiagnostics;
import org.apache.polaris.core.context.RealmContext;
@@ -34,7 +35,7 @@ import org.apache.polaris.core.persistence.PolarisMetaStoreSession;
*/
@JsonTypeName("eclipse-link")
public class EclipseLinkPolarisMetaStoreManagerFactory
- extends LocalPolarisMetaStoreManagerFactory<PolarisEclipseLinkStore> {
+ extends LocalPolarisMetaStoreManagerFactory<PolarisEclipseLinkStore> implements Discoverable {
@JsonProperty("conf-file")
private String confFile;
diff --git a/polaris-service/src/main/resources/META-INF/services/org.apache.polaris.core.persistence.MetaStoreManagerFactory b/extension/persistence/eclipselink/src/main/resources/META-INF/services/io.dropwizard.jackson.Discoverable
similarity index 91%
rename from polaris-service/src/main/resources/META-INF/services/org.apache.polaris.core.persistence.MetaStoreManagerFactory
rename to extension/persistence/eclipselink/src/main/resources/META-INF/services/io.dropwizard.jackson.Discoverable
index 85ae92ca..58f8891e 100644
--- a/polaris-service/src/main/resources/META-INF/services/org.apache.polaris.core.persistence.MetaStoreManagerFactory
+++ b/extension/persistence/eclipselink/src/main/resources/META-INF/services/io.dropwizard.jackson.Discoverable
@@ -17,5 +17,4 @@
# under the License.
#
-org.apache.polaris.service.persistence.InMemoryPolarisMetaStoreManagerFactory
org.apache.polaris.extension.persistence.impl.eclipselink.EclipseLinkPolarisMetaStoreManagerFactory
diff --git a/polaris-core/build.gradle.kts b/polaris-core/build.gradle.kts
index 32dd38f9..de086f65 100644
--- a/polaris-core/build.gradle.kts
+++ b/polaris-core/build.gradle.kts
@@ -33,10 +33,6 @@ dependencies {
constraints {
implementation("io.airlift:aircompressor:0.27") { because("Vulnerability detected in 0.25") }
}
- // TODO - this is only here for the Discoverable interface
- // We should use a different mechanism to discover the plugin implementations
- implementation(platform(libs.dropwizard.bom))
- implementation("io.dropwizard:dropwizard-jackson")
implementation(platform(libs.jackson.bom))
implementation("com.fasterxml.jackson.core:jackson-annotations")
diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/MetaStoreManagerFactory.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/MetaStoreManagerFactory.java
index 821d139a..4352a975 100644
--- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/MetaStoreManagerFactory.java
+++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/MetaStoreManagerFactory.java
@@ -19,7 +19,6 @@
package org.apache.polaris.core.persistence;
import com.fasterxml.jackson.annotation.JsonTypeInfo;
-import io.dropwizard.jackson.Discoverable;
import java.util.List;
import java.util.Map;
import java.util.function.Supplier;
@@ -34,7 +33,7 @@ import org.apache.polaris.core.storage.cache.StorageCredentialCache;
* configuration
*/
@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "type")
-public interface MetaStoreManagerFactory extends Discoverable {
+public interface MetaStoreManagerFactory {
PolarisMetaStoreManager getOrCreateMetaStoreManager(RealmContext realmContext);
diff --git a/polaris-service/build.gradle.kts b/polaris-service/build.gradle.kts
index 370a5bfc..9d99d49f 100644
--- a/polaris-service/build.gradle.kts
+++ b/polaris-service/build.gradle.kts
@@ -118,7 +118,7 @@ dependencies {
testImplementation(libs.mockito.core)
testRuntimeOnly("org.junit.platform:junit-platform-launcher")
- testRuntimeOnly(project(":polaris-eclipselink"))
+ testImplementation(project(":polaris-eclipselink"))
}
if (project.properties.get("eclipseLink") == "true") {
diff --git a/polaris-service/src/main/java/org/apache/polaris/service/PolarisApplication.java b/polaris-service/src/main/java/org/apache/polaris/service/PolarisApplication.java
index facbe76c..89287a3e 100644
--- a/polaris-service/src/main/java/org/apache/polaris/service/PolarisApplication.java
+++ b/polaris-service/src/main/java/org/apache/polaris/service/PolarisApplication.java
@@ -36,6 +36,8 @@ import io.dropwizard.configuration.SubstitutingSourceProvider;
import io.dropwizard.core.Application;
import io.dropwizard.core.setup.Bootstrap;
import io.dropwizard.core.setup.Environment;
+import io.dropwizard.jackson.Discoverable;
+import io.dropwizard.jackson.DiscoverableSubtypeResolver;
import io.micrometer.prometheusmetrics.PrometheusConfig;
import io.micrometer.prometheusmetrics.PrometheusMeterRegistry;
import io.opentelemetry.api.OpenTelemetry;
@@ -63,6 +65,7 @@ import java.io.InputStream;
import java.net.URL;
import java.util.Collections;
import java.util.EnumSet;
+import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.concurrent.Executors;
@@ -142,6 +145,7 @@ public class PolarisApplication extends Application<PolarisApplicationConfig> {
@Override
public void initialize(Bootstrap<PolarisApplicationConfig> bootstrap) {
+ registerTypes(bootstrap.getObjectMapper());
// Enable variable substitution with environment variables
EnvironmentVariableSubstitutor substitutor = new EnvironmentVariableSubstitutor(false);
SubstitutingSourceProvider provider =
@@ -152,6 +156,18 @@ public class PolarisApplication extends Application<PolarisApplicationConfig> {
bootstrap.addCommand(new PurgeRealmsCommand());
}
+ private void registerTypes(ObjectMapper mapper) {
+ // Reuse the DW service discovery method, but unlike the constructor of
+ // DiscoverableSubtypeResolver, use the first level classes from the `Discoverable`
+ // service descriptor and register them with the ObjectMapper.
+ class Discoverer extends DiscoverableSubtypeResolver {
+ List<Class<?>> discover() {
+ return discoverServices(Discoverable.class);
+ }
+ }
+ new Discoverer().discover().forEach(mapper::registerSubtypes);
+ }
+
@Override
public void run(PolarisApplicationConfig configuration, Environment environment) {
MetaStoreManagerFactory metaStoreManagerFactory = configuration.getMetaStoreManagerFactory();
diff --git a/polaris-service/src/main/java/org/apache/polaris/service/persistence/InMemoryPolarisMetaStoreManagerFactory.java b/polaris-service/src/main/java/org/apache/polaris/service/persistence/InMemoryPolarisMetaStoreManagerFactory.java
index 8f1f146e..fc830341 100644
--- a/polaris-service/src/main/java/org/apache/polaris/service/persistence/InMemoryPolarisMetaStoreManagerFactory.java
+++ b/polaris-service/src/main/java/org/apache/polaris/service/persistence/InMemoryPolarisMetaStoreManagerFactory.java
@@ -19,6 +19,7 @@
package org.apache.polaris.service.persistence;
import com.fasterxml.jackson.annotation.JsonTypeName;
+import io.dropwizard.jackson.Discoverable;
import jakarta.annotation.Nonnull;
import java.util.Collections;
import java.util.HashSet;
@@ -36,7 +37,7 @@ import org.apache.polaris.core.persistence.PolarisTreeMapStore;
@JsonTypeName("in-memory")
public class InMemoryPolarisMetaStoreManagerFactory
- extends LocalPolarisMetaStoreManagerFactory<PolarisTreeMapStore> {
+ extends LocalPolarisMetaStoreManagerFactory<PolarisTreeMapStore> implements Discoverable {
final Set<String> bootstrappedRealms = new HashSet<>();
@Override
diff --git a/polaris-service/src/main/resources/META-INF/services/io.dropwizard.jackson.Discoverable b/polaris-service/src/main/resources/META-INF/services/io.dropwizard.jackson.Discoverable
index 95d1f8ec..4c7ab2ac 100644
--- a/polaris-service/src/main/resources/META-INF/services/io.dropwizard.jackson.Discoverable
+++ b/polaris-service/src/main/resources/META-INF/services/io.dropwizard.jackson.Discoverable
@@ -18,7 +18,7 @@
#
org.apache.polaris.service.auth.DiscoverableAuthenticator
-org.apache.polaris.core.persistence.MetaStoreManagerFactory
+org.apache.polaris.service.persistence.InMemoryPolarisMetaStoreManagerFactory
org.apache.polaris.service.config.OAuth2ApiService
org.apache.polaris.service.context.RealmContextResolver
org.apache.polaris.service.context.CallContextResolver
diff --git a/polaris-service/src/test/java/org/apache/polaris/service/PolarisApplicationConfigurationTest.java b/polaris-service/src/test/java/org/apache/polaris/service/PolarisApplicationConfigurationTest.java
new file mode 100644
index 00000000..74442e04
--- /dev/null
+++ b/polaris-service/src/test/java/org/apache/polaris/service/PolarisApplicationConfigurationTest.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.service;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import io.dropwizard.testing.ConfigOverride;
+import io.dropwizard.testing.ResourceHelpers;
+import io.dropwizard.testing.junit5.DropwizardAppExtension;
+import io.dropwizard.testing.junit5.DropwizardExtensionsSupport;
+import org.apache.polaris.extension.persistence.impl.eclipselink.EclipseLinkPolarisMetaStoreManagerFactory;
+import org.apache.polaris.service.config.PolarisApplicationConfig;
+import org.apache.polaris.service.persistence.InMemoryPolarisMetaStoreManagerFactory;
+import org.junit.jupiter.api.Nested;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+
+@ExtendWith(DropwizardExtensionsSupport.class)
+public class PolarisApplicationConfigurationTest {
+
+ public static final String CONFIG_PATH =
+ ResourceHelpers.resourceFilePath("polaris-server-integrationtest.yml");
+ // Bind to random ports to support parallelism
+ public static final ConfigOverride RANDOM_APP_PORT =
+ ConfigOverride.config("server.applicationConnectors[0].port", "0");
+ public static final ConfigOverride RANDOM_ADMIN_PORT =
+ ConfigOverride.config("server.adminConnectors[0].port", "0");
+
+ @Nested
+ class DefaultMetastore {
+ private final DropwizardAppExtension<PolarisApplicationConfig> app =
+ new DropwizardAppExtension<>(
+ PolarisApplication.class, CONFIG_PATH, RANDOM_APP_PORT, RANDOM_ADMIN_PORT);
+
+ @Test
+ void testMetastoreType() {
+ assertThat(app.getConfiguration().getMetaStoreManagerFactory())
+ .isInstanceOf(InMemoryPolarisMetaStoreManagerFactory.class);
+ }
+ }
+
+ @Nested
+ class EclipseLinkMetastore {
+ private final DropwizardAppExtension<PolarisApplicationConfig> app =
+ new DropwizardAppExtension<>(
+ PolarisApplication.class,
+ CONFIG_PATH,
+ RANDOM_APP_PORT,
+ RANDOM_ADMIN_PORT,
+ ConfigOverride.config("metaStoreManager.type", "eclipse-link"),
+ ConfigOverride.config("metaStoreManager.persistence-unit", "test-unit"),
+ ConfigOverride.config("metaStoreManager.conf-file", "/test-conf-file"));
+
+ @Test
+ void testMetastoreType() {
+ assertThat(app.getConfiguration().getMetaStoreManagerFactory())
+ .isInstanceOf(EclipseLinkPolarisMetaStoreManagerFactory.class)
+ .extracting("persistenceUnitName", "confFile")
+ .containsExactly("test-unit", "/test-conf-file");
+ }
+ }
+}