You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by mm...@apache.org on 2023/01/19 19:19:57 UTC
[pulsar] branch master updated: [improve][broker]Enable custom metadata stores (#19208)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new 86205a9edfa [improve][broker]Enable custom metadata stores (#19208)
86205a9edfa is described below
commit 86205a9edfafb8710b24bdb0d2f1db84c5fc1926
Author: Andras Beni <an...@streamnative.io>
AuthorDate: Thu Jan 19 20:19:48 2023 +0100
[improve][broker]Enable custom metadata stores (#19208)
---
.../pulsar/metadata/api/MetadataStoreProvider.java | 37 +++++++
.../pulsar/metadata/impl/EtcdMetadataStore.java | 17 +++
.../metadata/impl/LocalMemoryMetadataStore.java | 17 +++
.../metadata/impl/MetadataStoreFactoryImpl.java | 67 +++++++----
.../pulsar/metadata/impl/RocksdbMetadataStore.java | 17 +++
.../pulsar/metadata/impl/ZKMetadataStore.java | 20 ++++
.../impl/MetadataStoreFactoryImplTest.java | 123 +++++++++++++++++++++
7 files changed, 279 insertions(+), 19 deletions(-)
diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/api/MetadataStoreProvider.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/api/MetadataStoreProvider.java
new file mode 100644
index 00000000000..9c923aecb6c
--- /dev/null
+++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/api/MetadataStoreProvider.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.metadata.api;
+
+public interface MetadataStoreProvider {
+
+
+ /**
+ * Scheme of the urls that MetadataSores provided by this object can handle.
+ */
+ String urlScheme();
+
+ /**
+ * Creates a new MetadataStore.
+ * @throws MetadataStoreException if any exception happens while creating the metadata store.
+ */
+ MetadataStore create(
+ String metadataURL,
+ MetadataStoreConfig metadataStoreConfig,
+ boolean enableSessionWatcher) throws MetadataStoreException;
+}
diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/EtcdMetadataStore.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/EtcdMetadataStore.java
index 5710477fe06..a7fb7192cb5 100644
--- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/EtcdMetadataStore.java
+++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/EtcdMetadataStore.java
@@ -72,8 +72,10 @@ import lombok.NoArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.pulsar.metadata.api.GetResult;
+import org.apache.pulsar.metadata.api.MetadataStore;
import org.apache.pulsar.metadata.api.MetadataStoreConfig;
import org.apache.pulsar.metadata.api.MetadataStoreException;
+import org.apache.pulsar.metadata.api.MetadataStoreProvider;
import org.apache.pulsar.metadata.api.Notification;
import org.apache.pulsar.metadata.api.NotificationType;
import org.apache.pulsar.metadata.api.Stat;
@@ -89,6 +91,7 @@ import org.apache.pulsar.metadata.impl.batching.OpPut;
@Slf4j
public class EtcdMetadataStore extends AbstractBatchedMetadataStore {
+ static final String ETCD_SCHEME = "etcd";
static final String ETCD_SCHEME_IDENTIFIER = "etcd:";
private final int leaseTTLSeconds;
@@ -498,3 +501,17 @@ class EtcdConfig {
private String authority;
}
+
+class EtcdMetadataStoreProvider implements MetadataStoreProvider {
+
+ @Override
+ public String urlScheme() {
+ return EtcdMetadataStore.ETCD_SCHEME;
+ }
+
+ @Override
+ public MetadataStore create(String metadataURL, MetadataStoreConfig metadataStoreConfig,
+ boolean enableSessionWatcher) throws MetadataStoreException {
+ return new EtcdMetadataStore(metadataURL, metadataStoreConfig, enableSessionWatcher);
+ }
+}
diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/LocalMemoryMetadataStore.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/LocalMemoryMetadataStore.java
index 616d0901c08..94d9a1f8937 100644
--- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/LocalMemoryMetadataStore.java
+++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/LocalMemoryMetadataStore.java
@@ -37,10 +37,12 @@ import org.apache.bookkeeper.common.concurrent.FutureUtils;
import org.apache.pulsar.common.util.FutureUtil;
import org.apache.pulsar.metadata.api.GetResult;
import org.apache.pulsar.metadata.api.MetadataEventSynchronizer;
+import org.apache.pulsar.metadata.api.MetadataStore;
import org.apache.pulsar.metadata.api.MetadataStoreConfig;
import org.apache.pulsar.metadata.api.MetadataStoreException;
import org.apache.pulsar.metadata.api.MetadataStoreException.BadVersionException;
import org.apache.pulsar.metadata.api.MetadataStoreException.NotFoundException;
+import org.apache.pulsar.metadata.api.MetadataStoreProvider;
import org.apache.pulsar.metadata.api.Notification;
import org.apache.pulsar.metadata.api.NotificationType;
import org.apache.pulsar.metadata.api.Stat;
@@ -50,6 +52,7 @@ import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended;
@Slf4j
public class LocalMemoryMetadataStore extends AbstractMetadataStore implements MetadataStoreExtended {
+ static final String MEMORY_SCHEME = "memory";
static final String MEMORY_SCHEME_IDENTIFIER = "memory:";
@Data
@@ -237,3 +240,17 @@ public class LocalMemoryMetadataStore extends AbstractMetadataStore implements M
}
}
}
+
+class MemoryMetadataStoreProvider implements MetadataStoreProvider {
+
+ @Override
+ public String urlScheme() {
+ return LocalMemoryMetadataStore.MEMORY_SCHEME;
+ }
+
+ @Override
+ public MetadataStore create(String metadataURL, MetadataStoreConfig metadataStoreConfig,
+ boolean enableSessionWatcher) throws MetadataStoreException {
+ return new LocalMemoryMetadataStore(metadataURL, metadataStoreConfig);
+ }
+}
diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/MetadataStoreFactoryImpl.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/MetadataStoreFactoryImpl.java
index c64c849f529..dd4df69fc43 100644
--- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/MetadataStoreFactoryImpl.java
+++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/MetadataStoreFactoryImpl.java
@@ -18,13 +18,25 @@
*/
package org.apache.pulsar.metadata.impl;
+import static org.apache.pulsar.metadata.impl.EtcdMetadataStore.ETCD_SCHEME_IDENTIFIER;
+import static org.apache.pulsar.metadata.impl.LocalMemoryMetadataStore.MEMORY_SCHEME_IDENTIFIER;
+import static org.apache.pulsar.metadata.impl.RocksdbMetadataStore.ROCKSDB_SCHEME_IDENTIFIER;
+import static org.apache.pulsar.metadata.impl.ZKMetadataStore.ZK_SCHEME_IDENTIFIER;
+import com.google.common.base.Splitter;
+import java.util.HashMap;
+import java.util.Map;
+import lombok.extern.slf4j.Slf4j;
import org.apache.pulsar.metadata.api.MetadataStore;
import org.apache.pulsar.metadata.api.MetadataStoreConfig;
import org.apache.pulsar.metadata.api.MetadataStoreException;
+import org.apache.pulsar.metadata.api.MetadataStoreProvider;
import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended;
+@Slf4j
public class MetadataStoreFactoryImpl {
+ public static final String METADATASTORE_PROVIDERS_PROPERTY = "pulsar.metadatastore.providers";
+
public static MetadataStore create(String metadataURL, MetadataStoreConfig metadataStoreConfig) throws
MetadataStoreException {
return newInstance(metadataURL, metadataStoreConfig, false);
@@ -45,19 +57,41 @@ public class MetadataStoreFactoryImpl {
private static MetadataStore newInstance(String metadataURL, MetadataStoreConfig metadataStoreConfig,
boolean enableSessionWatcher)
throws MetadataStoreException {
+ MetadataStoreProvider provider = findProvider(metadataURL);
+ return provider.create(metadataURL, metadataStoreConfig, enableSessionWatcher);
+ }
+
+ static Map<String, MetadataStoreProvider> loadProviders() {
+ Map<String, MetadataStoreProvider> providers = new HashMap<>();
+ providers.put(MEMORY_SCHEME_IDENTIFIER, new MemoryMetadataStoreProvider());
+ providers.put(ROCKSDB_SCHEME_IDENTIFIER, new RocksdbMetadataStoreProvider());
+ providers.put(ETCD_SCHEME_IDENTIFIER, new EtcdMetadataStoreProvider());
+ providers.put(ZK_SCHEME_IDENTIFIER, new ZkMetadataStoreProvider());
+
+ String factoryClasses = System.getProperty(METADATASTORE_PROVIDERS_PROPERTY, "");
+
+ for (String className : Splitter.on(',').trimResults().omitEmptyStrings().split(factoryClasses)) {
+ try {
+ Class<? extends MetadataStoreProvider> clazz =
+ (Class<? extends MetadataStoreProvider>) Class.forName(className);
+ MetadataStoreProvider provider = clazz.getConstructor().newInstance();
+ String scheme = provider.urlScheme();
+ providers.put(scheme + ":", provider);
+ } catch (Exception e) {
+ log.warn("Failed to load metadata store provider class for name '{}'", className, e);
+ }
+ }
+ return providers;
+ }
- if (metadataURL.startsWith(LocalMemoryMetadataStore.MEMORY_SCHEME_IDENTIFIER)) {
- return new LocalMemoryMetadataStore(metadataURL, metadataStoreConfig);
- } else if (metadataURL.startsWith(RocksdbMetadataStore.ROCKSDB_SCHEME_IDENTIFIER)) {
- return RocksdbMetadataStore.get(metadataURL, metadataStoreConfig);
- } else if (metadataURL.startsWith(EtcdMetadataStore.ETCD_SCHEME_IDENTIFIER)) {
- return new EtcdMetadataStore(metadataURL, metadataStoreConfig, enableSessionWatcher);
- } else if (metadataURL.startsWith(ZKMetadataStore.ZK_SCHEME_IDENTIFIER)) {
- return new ZKMetadataStore(metadataURL.substring(ZKMetadataStore.ZK_SCHEME_IDENTIFIER.length()),
- metadataStoreConfig, enableSessionWatcher);
- } else {
- return new ZKMetadataStore(metadataURL, metadataStoreConfig, enableSessionWatcher);
+ private static MetadataStoreProvider findProvider(String metadataURL) {
+ Map<String, MetadataStoreProvider> providers = loadProviders();
+ for (Map.Entry<String, MetadataStoreProvider> entry : providers.entrySet()) {
+ if (metadataURL.startsWith(entry.getKey())) {
+ return entry.getValue();
+ }
}
+ return providers.get(ZK_SCHEME_IDENTIFIER);
}
/**
@@ -70,14 +104,9 @@ public class MetadataStoreFactoryImpl {
* @return
*/
public static String removeIdentifierFromMetadataURL(String metadataURL) {
- if (metadataURL.startsWith(LocalMemoryMetadataStore.MEMORY_SCHEME_IDENTIFIER)) {
- return metadataURL.substring(LocalMemoryMetadataStore.MEMORY_SCHEME_IDENTIFIER.length());
- } else if (metadataURL.startsWith(RocksdbMetadataStore.ROCKSDB_SCHEME_IDENTIFIER)) {
- return metadataURL.substring(RocksdbMetadataStore.ROCKSDB_SCHEME_IDENTIFIER.length());
- } else if (metadataURL.startsWith(EtcdMetadataStore.ETCD_SCHEME_IDENTIFIER)) {
- return metadataURL.substring(EtcdMetadataStore.ETCD_SCHEME_IDENTIFIER.length());
- } else if (metadataURL.startsWith(ZKMetadataStore.ZK_SCHEME_IDENTIFIER)) {
- return metadataURL.substring(ZKMetadataStore.ZK_SCHEME_IDENTIFIER.length());
+ MetadataStoreProvider provider = findProvider(metadataURL);
+ if (metadataURL.startsWith(provider.urlScheme() + ":")) {
+ return metadataURL.substring(provider.urlScheme().length() + 1);
}
return metadataURL;
}
diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/RocksdbMetadataStore.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/RocksdbMetadataStore.java
index 5d7e74fadf5..fcec2fcf9c1 100644
--- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/RocksdbMetadataStore.java
+++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/RocksdbMetadataStore.java
@@ -47,8 +47,10 @@ import lombok.extern.slf4j.Slf4j;
import org.apache.pulsar.common.util.FutureUtil;
import org.apache.pulsar.metadata.api.GetResult;
import org.apache.pulsar.metadata.api.MetadataEventSynchronizer;
+import org.apache.pulsar.metadata.api.MetadataStore;
import org.apache.pulsar.metadata.api.MetadataStoreConfig;
import org.apache.pulsar.metadata.api.MetadataStoreException;
+import org.apache.pulsar.metadata.api.MetadataStoreProvider;
import org.apache.pulsar.metadata.api.Notification;
import org.apache.pulsar.metadata.api.NotificationType;
import org.apache.pulsar.metadata.api.Stat;
@@ -75,6 +77,7 @@ import org.rocksdb.WriteOptions;
@Slf4j
public class RocksdbMetadataStore extends AbstractMetadataStore {
+ static final String ROCKSDB_SCHEME = "rocksdb";
static final String ROCKSDB_SCHEME_IDENTIFIER = "rocksdb:";
private static final byte[] SEQUENTIAL_ID_KEY = toBytes("__metadata_sequentialId_key");
@@ -570,3 +573,17 @@ public class RocksdbMetadataStore extends AbstractMetadataStore {
return Optional.ofNullable(synchronizer);
}
}
+
+class RocksdbMetadataStoreProvider implements MetadataStoreProvider {
+
+ @Override
+ public String urlScheme() {
+ return RocksdbMetadataStore.ROCKSDB_SCHEME;
+ }
+
+ @Override
+ public MetadataStore create(String metadataURL, MetadataStoreConfig metadataStoreConfig,
+ boolean enableSessionWatcher) throws MetadataStoreException {
+ return RocksdbMetadataStore.get(metadataURL, metadataStoreConfig);
+ }
+}
diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/ZKMetadataStore.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/ZKMetadataStore.java
index fdd54bd1013..a6d8eb8344c 100644
--- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/ZKMetadataStore.java
+++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/ZKMetadataStore.java
@@ -34,12 +34,14 @@ import lombok.extern.slf4j.Slf4j;
import org.apache.bookkeeper.zookeeper.BoundExponentialBackoffRetryPolicy;
import org.apache.pulsar.common.util.FutureUtil;
import org.apache.pulsar.metadata.api.GetResult;
+import org.apache.pulsar.metadata.api.MetadataStore;
import org.apache.pulsar.metadata.api.MetadataStoreConfig;
import org.apache.pulsar.metadata.api.MetadataStoreException;
import org.apache.pulsar.metadata.api.MetadataStoreException.AlreadyExistsException;
import org.apache.pulsar.metadata.api.MetadataStoreException.BadVersionException;
import org.apache.pulsar.metadata.api.MetadataStoreException.NotFoundException;
import org.apache.pulsar.metadata.api.MetadataStoreLifecycle;
+import org.apache.pulsar.metadata.api.MetadataStoreProvider;
import org.apache.pulsar.metadata.api.Notification;
import org.apache.pulsar.metadata.api.NotificationType;
import org.apache.pulsar.metadata.api.Stat;
@@ -69,6 +71,7 @@ import org.apache.zookeeper.client.ConnectStringParser;
public class ZKMetadataStore extends AbstractBatchedMetadataStore
implements MetadataStoreExtended, MetadataStoreLifecycle {
+ public static final String ZK_SCHEME = "zk";
public static final String ZK_SCHEME_IDENTIFIER = "zk:";
private final String zkConnectString;
@@ -614,3 +617,20 @@ public class ZKMetadataStore extends AbstractBatchedMetadataStore
return rootPath;
}
}
+
+class ZkMetadataStoreProvider implements MetadataStoreProvider {
+
+ @Override
+ public String urlScheme() {
+ return ZKMetadataStore.ZK_SCHEME;
+ }
+
+ @Override
+ public MetadataStore create(String metadataURL, MetadataStoreConfig metadataStoreConfig,
+ boolean enableSessionWatcher) throws MetadataStoreException {
+ if (metadataURL.startsWith(ZKMetadataStore.ZK_SCHEME_IDENTIFIER)) {
+ metadataURL = metadataURL.substring(ZKMetadataStore.ZK_SCHEME_IDENTIFIER.length());
+ }
+ return new ZKMetadataStore(metadataURL, metadataStoreConfig, enableSessionWatcher);
+ }
+}
diff --git a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/impl/MetadataStoreFactoryImplTest.java b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/impl/MetadataStoreFactoryImplTest.java
new file mode 100644
index 00000000000..ba12ddff062
--- /dev/null
+++ b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/impl/MetadataStoreFactoryImplTest.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.metadata.impl;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertTrue;
+import org.apache.pulsar.metadata.api.GetResult;
+import org.apache.pulsar.metadata.api.MetadataStore;
+import org.apache.pulsar.metadata.api.MetadataStoreConfig;
+import org.apache.pulsar.metadata.api.MetadataStoreException;
+import org.apache.pulsar.metadata.api.MetadataStoreProvider;
+import org.apache.pulsar.metadata.api.Stat;
+import org.apache.pulsar.metadata.api.extended.CreateOption;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+import java.util.EnumSet;
+import java.util.List;
+import java.util.Optional;
+import java.util.concurrent.CompletableFuture;
+
+public class MetadataStoreFactoryImplTest {
+
+ private static Object originalProperty;
+
+ @BeforeClass
+ public void setMetadataStoreProperty() {
+ originalProperty = System.getProperties().get(MetadataStoreFactoryImpl.METADATASTORE_PROVIDERS_PROPERTY);
+ System.setProperty(MetadataStoreFactoryImpl.METADATASTORE_PROVIDERS_PROPERTY,
+ MyMetadataStoreProvider.class.getName());
+ }
+
+ @AfterClass
+ public void resetMetadataStoreProperty() {
+ if (originalProperty != null) {
+ System.getProperties().put(MetadataStoreFactoryImpl.METADATASTORE_PROVIDERS_PROPERTY, originalProperty);
+ }
+ }
+
+
+ @Test
+ public void testCreate() throws MetadataStoreException{
+ MetadataStore instance = MetadataStoreFactoryImpl.create(
+ "custom://localhost",
+ MetadataStoreConfig.builder().build());
+ assertTrue(instance instanceof MyMetadataStore);
+ }
+
+
+ @Test
+ public void testRemoveIdentifierFromMetadataURL() {
+ assertEquals(MetadataStoreFactoryImpl.removeIdentifierFromMetadataURL("zk:host:port"), "host:port");
+ assertEquals(MetadataStoreFactoryImpl.removeIdentifierFromMetadataURL("rocksdb:/data/dir"), "/data/dir");
+ assertEquals(MetadataStoreFactoryImpl.removeIdentifierFromMetadataURL("etcd:host:port"), "host:port");
+ assertEquals(MetadataStoreFactoryImpl.removeIdentifierFromMetadataURL("memory:name"), "name");
+ assertEquals(MetadataStoreFactoryImpl.removeIdentifierFromMetadataURL("http://unknown/url/scheme"), "http://unknown/url/scheme");
+ assertEquals(MetadataStoreFactoryImpl.removeIdentifierFromMetadataURL("custom:suffix"), "suffix");
+ }
+
+ public static class MyMetadataStoreProvider implements MetadataStoreProvider {
+
+ @Override
+ public String urlScheme() {
+ return "custom";
+ }
+
+ @Override
+ public MetadataStore create(String metadataURL, MetadataStoreConfig metadataStoreConfig,
+ boolean enableSessionWatcher) throws MetadataStoreException {
+ return new MyMetadataStore();
+ }
+ }
+
+ public static class MyMetadataStore extends AbstractMetadataStore {
+ protected MyMetadataStore() {
+ super("custom");
+ }
+
+ @Override
+ protected CompletableFuture<List<String>> getChildrenFromStore(String path) {
+ return null;
+ }
+
+ @Override
+ protected CompletableFuture<Boolean> existsFromStore(String path) {
+ return null;
+ }
+
+ @Override
+ protected CompletableFuture<Optional<GetResult>> storeGet(String path) {
+ return null;
+ }
+
+ @Override
+ protected CompletableFuture<Void> storeDelete(String path, Optional<Long> expectedVersion) {
+ return null;
+ }
+
+ @Override
+ protected CompletableFuture<Stat> storePut(String path, byte[] data, Optional<Long> optExpectedVersion,
+ EnumSet<CreateOption> options) {
+ return null;
+ }
+ }
+
+
+}