You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dubbo.apache.org by li...@apache.org on 2021/11/04 14:33:52 UTC
[dubbo] branch 3.0-metadata-refactor updated: Introducing metadata
local file cache (#9132)
This is an automated email from the ASF dual-hosted git repository.
liujun pushed a commit to branch 3.0-metadata-refactor
in repository https://gitbox.apache.org/repos/asf/dubbo.git
The following commit(s) were added to refs/heads/3.0-metadata-refactor by this push:
new f1accac Introducing metadata local file cache (#9132)
f1accac is described below
commit f1accac41b900ffd56f5f392c150b5acc9bd0970
Author: ken.lj <ke...@gmail.com>
AuthorDate: Thu Nov 4 22:33:29 2021 +0800
Introducing metadata local file cache (#9132)
---
.../apache/dubbo/common/cache/FileCacheStore.java | 237 +++++++++++++++++++++
.../org/apache/dubbo/common/utils/JsonUtils.java | 27 +++
.../common/utils/{LRUCache.java => LRU2Cache.java} | 8 +-
.../org/apache/dubbo/common/utils/LRUCache.java | 54 +----
.../dubbo/common/cache/FileCacheStoreTest.java | 62 ++++++
.../{LRUCacheTest.java => LRU2CacheTest.java} | 8 +-
.../src/test/resources/test-cache.dubbo.cache | 2 +
.../src/main/resources/spring/dubbo-consumer.xml | 2 +-
.../apache/dubbo/cache/support/lru/LruCache.java | 4 +-
.../apache/dubbo/metadata/MetadataInfoTest.java | 18 ++
.../client/metadata/store/MetaCacheManager.java | 150 +++++++++++++
.../dubbo/registry/support/AbstractRegistry.java | 7 +-
.../metadata/store/MetaCacheManagerTest.java | 112 ++++++++++
.../src/test/resources/test-metadata.dubbo.cache | 1 +
pom.xml | 1 +
15 files changed, 637 insertions(+), 56 deletions(-)
diff --git a/dubbo-common/src/main/java/org/apache/dubbo/common/cache/FileCacheStore.java b/dubbo-common/src/main/java/org/apache/dubbo/common/cache/FileCacheStore.java
new file mode 100644
index 0000000..efe764d
--- /dev/null
+++ b/dubbo-common/src/main/java/org/apache/dubbo/common/cache/FileCacheStore.java
@@ -0,0 +1,237 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dubbo.common.cache;
+
+import org.apache.dubbo.common.logger.Logger;
+import org.apache.dubbo.common.logger.LoggerFactory;
+import org.apache.dubbo.common.utils.CollectionUtils;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.io.RandomAccessFile;
+import java.nio.channels.FileChannel;
+import java.nio.channels.FileLock;
+import java.nio.channels.OverlappingFileLockException;
+import java.nio.charset.StandardCharsets;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Local file interaction class that can back different caches.
+ *
+ * All items in local file are of human friendly format.
+ */
+public class FileCacheStore {
+ private static final Logger logger = LoggerFactory.getLogger(FileCacheStore.class);
+
+ private static final int DEL = 0x7F;
+ private static final char ESCAPE = '%';
+ private static final Set<Character> ILLEGALS = new HashSet<Character>();
+ private static final String SUFFIX = ".dubbo.cache";
+
+ private String fileName;
+ private File basePath;
+ private File cacheFile;
+ private FileLock directoryLock;
+ private File lockFile;
+
+ public FileCacheStore(String basePath, String fileName) throws IOException {
+ if (basePath == null) {
+ basePath = System.getProperty("user.home") + "/.dubbo/";
+ }
+ this.basePath = new File(basePath);
+ this.fileName = fileName;
+
+ this.cacheFile = getFile(fileName, SUFFIX);
+ if (!cacheFile.exists()) {
+ cacheFile.createNewFile();
+ }
+ }
+
+ public Map<String, String> loadCache(int entrySize) throws IOException {
+ Map<String, String> properties = new HashMap<>();
+ try (BufferedReader reader = new BufferedReader(new FileReader(cacheFile))) {
+ int count = 1;
+ String line = reader.readLine();
+ while (line != null && count <= entrySize) {
+ // content has '=' need to be encoded before write
+ if (!line.equals("") && !line.startsWith("#") && line.contains("=")) {
+ String[] pairs = line.split("=");
+ properties.put(pairs[0], pairs[1]);
+ count++;
+ }
+ line = reader.readLine();
+ }
+
+ if (count > entrySize) {
+ logger.warn("Cache file was truncated for exceeding the maximum entry size " + entrySize);
+ }
+ } catch (IOException e) {
+ logger.warn("Load cache failed ", e);
+ throw e;
+ }
+ return properties;
+ }
+
+ public File getFile(String cacheName, String suffix) {
+ cacheName = safeName(cacheName);
+ if (!cacheName.endsWith(suffix)) {
+ cacheName = cacheName + suffix;
+ }
+ return getFile(cacheName);
+ }
+
+ /**
+ * Get a file object for the given name
+ *
+ * @param name the file name
+ * @return a file object
+ */
+ public File getFile(String name) {
+ synchronized (this) {
+ File candidate = basePath;
+ // ensure cache store path exists
+ if (!candidate.isDirectory() && !candidate.mkdirs()) {
+ throw new RuntimeException("Cache store path can't be created: " + candidate);
+ }
+
+ try {
+ tryFileLock(name);
+ } catch (PathNotExclusiveException e) {
+ logger.warn("Path '" + basePath
+ + "' is already used by an existing Dubbo process.\n"
+ + "Please specify another one explicitly.");
+ }
+ }
+
+ File file = new File(basePath, name);
+ for (File parent = file.getParentFile(); parent != null; parent = parent.getParentFile()) {
+ if (basePath.equals(parent)) {
+ return file;
+ }
+ }
+
+ throw new IllegalArgumentException("Attempted to access file outside the dubbo cache path");
+ }
+
+ /**
+ * sanitize a name for valid file or directory name
+ *
+ * @param name
+ * @return sanitized version of name
+ */
+ private static String safeName(String name) {
+ int len = name.length();
+ StringBuilder sb = new StringBuilder(len);
+ for (int i = 0; i < len; i++) {
+ char c = name.charAt(i);
+ if (c <= ' ' || c >= DEL || (c >= 'A' && c <= 'Z') || ILLEGALS.contains(c) || c == ESCAPE) {
+ sb.append(ESCAPE);
+ sb.append(String.format("%04x", (int) c));
+ } else {
+ sb.append(c);
+ }
+ }
+ return sb.toString();
+ }
+
+ private void tryFileLock(String fileName) throws PathNotExclusiveException {
+ lockFile = new File(basePath.getAbsoluteFile(), fileName + ".lock");
+ lockFile.deleteOnExit();
+
+ FileLock dirLock;
+ try {
+ lockFile.createNewFile();
+ if (!lockFile.exists()) {
+ throw new AssertionError("Failed to create lock file " + lockFile);
+ }
+ FileChannel lockFileChannel = new RandomAccessFile(lockFile, "rw").getChannel();
+ dirLock = lockFileChannel.tryLock();
+ } catch (OverlappingFileLockException ofle) {
+ dirLock = null;
+ } catch (IOException ioe) {
+ throw new RuntimeException(ioe);
+ }
+
+ if (dirLock == null) {
+ throw new PathNotExclusiveException(basePath.getAbsolutePath() + " is not exclusive.");
+ }
+
+ this.directoryLock = dirLock;
+ }
+
+ private void unlock() {
+ if (directoryLock != null && directoryLock.isValid()) {
+ try {
+ directoryLock.release();
+ directoryLock.channel().close();
+ deleteFile(lockFile);
+ } catch (IOException e) {
+ throw new RuntimeException("Failed to release cache path's lock file:" + lockFile, e);
+ }
+ }
+ }
+
+ public void refreshCache(Map<String, String> properties, String comment) {
+ if (CollectionUtils.isEmptyMap(properties)) {
+ return;
+ }
+
+ try (BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(cacheFile, false), StandardCharsets.UTF_8))) {
+ bw.write("#" + comment);
+ bw.write("#" + new Date().toString());
+ bw.newLine();
+ for (Map.Entry<String, String> e : properties.entrySet()) {
+ String key = e.getKey();
+ String val = e.getValue();
+ bw.write(key + "=" + val);
+ bw.newLine();
+ }
+ bw.flush();
+ } catch (IOException e) {
+ logger.warn("Update cache error.");
+ }
+ }
+
+ private static void deleteFile(File f) {
+ if (!f.delete()) {
+ logger.debug("Failed to delete file " + f.getAbsolutePath());
+ }
+ }
+
+ private static class PathNotExclusiveException extends Exception {
+ public PathNotExclusiveException() {
+ super();
+ }
+
+ public PathNotExclusiveException(String msg) {
+ super(msg);
+ }
+ }
+
+ public void destroy() {
+ unlock();
+ }
+}
diff --git a/dubbo-common/src/main/java/org/apache/dubbo/common/utils/JsonUtils.java b/dubbo-common/src/main/java/org/apache/dubbo/common/utils/JsonUtils.java
new file mode 100644
index 0000000..7548580
--- /dev/null
+++ b/dubbo-common/src/main/java/org/apache/dubbo/common/utils/JsonUtils.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dubbo.common.utils;
+
+import com.google.gson.Gson;
+
+public class JsonUtils {
+ private static final Gson gson = new Gson();
+
+ public static Gson getGson() {
+ return gson;
+ }
+}
diff --git a/dubbo-common/src/main/java/org/apache/dubbo/common/utils/LRUCache.java b/dubbo-common/src/main/java/org/apache/dubbo/common/utils/LRU2Cache.java
similarity index 95%
copy from dubbo-common/src/main/java/org/apache/dubbo/common/utils/LRUCache.java
copy to dubbo-common/src/main/java/org/apache/dubbo/common/utils/LRU2Cache.java
index 6b39890..b64a2d2 100644
--- a/dubbo-common/src/main/java/org/apache/dubbo/common/utils/LRUCache.java
+++ b/dubbo-common/src/main/java/org/apache/dubbo/common/utils/LRU2Cache.java
@@ -25,8 +25,10 @@ import java.util.concurrent.locks.ReentrantLock;
* </p>
* When the data accessed for the first time, add it to history list. If the size of history list reaches max capacity, eliminate the earliest data (first in first out).
* When the data already exists in the history list, and be accessed for the second time, then it will be put into cache.
+ *
+ * TODO, consider replacing with ConcurrentHashMap to improve performance under concurrency
*/
-public class LRUCache<K, V> extends LinkedHashMap<K, V> {
+public class LRU2Cache<K, V> extends LinkedHashMap<K, V> {
private static final long serialVersionUID = -5167631809472116969L;
@@ -39,11 +41,11 @@ public class LRUCache<K, V> extends LinkedHashMap<K, V> {
// as history list
private PreCache<K, Boolean> preCache;
- public LRUCache() {
+ public LRU2Cache() {
this(DEFAULT_MAX_CAPACITY);
}
- public LRUCache(int maxCapacity) {
+ public LRU2Cache(int maxCapacity) {
super(16, DEFAULT_LOAD_FACTOR, true);
this.maxCapacity = maxCapacity;
this.preCache = new PreCache<>(maxCapacity);
diff --git a/dubbo-common/src/main/java/org/apache/dubbo/common/utils/LRUCache.java b/dubbo-common/src/main/java/org/apache/dubbo/common/utils/LRUCache.java
index 6b39890..90e905a 100644
--- a/dubbo-common/src/main/java/org/apache/dubbo/common/utils/LRUCache.java
+++ b/dubbo-common/src/main/java/org/apache/dubbo/common/utils/LRUCache.java
@@ -20,12 +20,6 @@ import java.util.LinkedHashMap;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
-/**
- * LRU-2
- * </p>
- * When the data accessed for the first time, add it to history list. If the size of history list reaches max capacity, eliminate the earliest data (first in first out).
- * When the data already exists in the history list, and be accessed for the second time, then it will be put into cache.
- */
public class LRUCache<K, V> extends LinkedHashMap<K, V> {
private static final long serialVersionUID = -5167631809472116969L;
@@ -36,9 +30,6 @@ public class LRUCache<K, V> extends LinkedHashMap<K, V> {
private final Lock lock = new ReentrantLock();
private volatile int maxCapacity;
- // as history list
- private PreCache<K, Boolean> preCache;
-
public LRUCache() {
this(DEFAULT_MAX_CAPACITY);
}
@@ -46,7 +37,6 @@ public class LRUCache<K, V> extends LinkedHashMap<K, V> {
public LRUCache(int maxCapacity) {
super(16, DEFAULT_LOAD_FACTOR, true);
this.maxCapacity = maxCapacity;
- this.preCache = new PreCache<>(maxCapacity);
}
@Override
@@ -78,15 +68,7 @@ public class LRUCache<K, V> extends LinkedHashMap<K, V> {
public V put(K key, V value) {
lock.lock();
try {
- if (preCache.containsKey(key)) {
- // add it to cache
- preCache.remove(key);
- return super.put(key, value);
- } else {
- // add it to history list
- preCache.put(key, true);
- return value;
- }
+ return super.put(key, value);
} finally {
lock.unlock();
}
@@ -96,7 +78,6 @@ public class LRUCache<K, V> extends LinkedHashMap<K, V> {
public V remove(Object key) {
lock.lock();
try {
- preCache.remove(key);
return super.remove(key);
} finally {
lock.unlock();
@@ -117,43 +98,26 @@ public class LRUCache<K, V> extends LinkedHashMap<K, V> {
public void clear() {
lock.lock();
try {
- preCache.clear();
super.clear();
} finally {
lock.unlock();
}
}
+ public void lock() {
+ lock.lock();
+ }
+
+ public void releaseLock() {
+ lock.unlock();
+ }
+
public int getMaxCapacity() {
return maxCapacity;
}
public void setMaxCapacity(int maxCapacity) {
- preCache.setMaxCapacity(maxCapacity);
this.maxCapacity = maxCapacity;
}
- static class PreCache<K, V> extends LinkedHashMap<K, V> {
-
- private volatile int maxCapacity;
-
- public PreCache() {
- this(DEFAULT_MAX_CAPACITY);
- }
-
- public PreCache(int maxCapacity) {
- super(16, DEFAULT_LOAD_FACTOR, true);
- this.maxCapacity = maxCapacity;
- }
-
- @Override
- protected boolean removeEldestEntry(java.util.Map.Entry<K, V> eldest) {
- return size() > maxCapacity;
- }
-
- public void setMaxCapacity(int maxCapacity) {
- this.maxCapacity = maxCapacity;
- }
- }
-
}
diff --git a/dubbo-common/src/test/java/org/apache/dubbo/common/cache/FileCacheStoreTest.java b/dubbo-common/src/test/java/org/apache/dubbo/common/cache/FileCacheStoreTest.java
new file mode 100644
index 0000000..e9cc973
--- /dev/null
+++ b/dubbo-common/src/test/java/org/apache/dubbo/common/cache/FileCacheStoreTest.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dubbo.common.cache;
+
+import org.junit.jupiter.api.Test;
+
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.nio.file.Paths;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+public class FileCacheStoreTest {
+ FileCacheStore cacheStore;
+
+ @Test
+ public void testCache() throws Exception {
+ String directoryPath = getDirectoryOfClassPath();
+ String filePath = "test-cache.dubbo.cache";
+ cacheStore = new FileCacheStore(directoryPath, filePath);
+ Map<String, String> properties = cacheStore.loadCache(10);
+ assertEquals(2, properties.size());
+
+ Map<String, String> newProperties = new HashMap<>();
+ newProperties.put("newKey1", "newValue1");
+ newProperties.put("newKey2", "newValue2");
+ newProperties.put("newKey3", "newValue3");
+ newProperties.put("newKey4", "newValue4");
+ cacheStore = new FileCacheStore(directoryPath, "non-exit.dubbo.cache");
+ cacheStore.refreshCache(newProperties, "test refresh cache");
+ Map<String, String> propertiesLimitTo2 = cacheStore.loadCache(2);
+ assertEquals(2, propertiesLimitTo2.size());
+
+ Map<String, String> propertiesLimitTo10 = cacheStore.loadCache(10);
+ assertEquals(4, propertiesLimitTo10.size());
+ }
+
+ private String getDirectoryOfClassPath() throws URISyntaxException {
+ URL resource = this.getClass().getResource("/log4j.xml");
+ String path = Paths.get(resource.toURI()).toFile().getAbsolutePath();
+ int index = path.indexOf("log4j.xml");
+ String directoryPath = path.substring(0, index);
+ return directoryPath;
+ }
+
+}
diff --git a/dubbo-common/src/test/java/org/apache/dubbo/common/utils/LRUCacheTest.java b/dubbo-common/src/test/java/org/apache/dubbo/common/utils/LRU2CacheTest.java
similarity index 93%
rename from dubbo-common/src/test/java/org/apache/dubbo/common/utils/LRUCacheTest.java
rename to dubbo-common/src/test/java/org/apache/dubbo/common/utils/LRU2CacheTest.java
index 78749bb..a616a33 100644
--- a/dubbo-common/src/test/java/org/apache/dubbo/common/utils/LRUCacheTest.java
+++ b/dubbo-common/src/test/java/org/apache/dubbo/common/utils/LRU2CacheTest.java
@@ -19,15 +19,15 @@ package org.apache.dubbo.common.utils;
import org.junit.jupiter.api.Test;
+import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.equalTo;
import static org.junit.jupiter.api.Assertions.assertFalse;
-import static org.hamcrest.MatcherAssert.assertThat;
import static org.junit.jupiter.api.Assertions.assertTrue;
-public class LRUCacheTest {
+public class LRU2CacheTest {
@Test
public void testCache() throws Exception {
- LRUCache<String, Integer> cache = new LRUCache<String, Integer>(3);
+ LRU2Cache<String, Integer> cache = new LRU2Cache<String, Integer>(3);
cache.put("one", 1);
cache.put("two", 2);
cache.put("three", 3);
@@ -63,7 +63,7 @@ public class LRUCacheTest {
@Test
public void testCapacity() throws Exception {
- LRUCache<String, Integer> cache = new LRUCache<String, Integer>();
+ LRU2Cache<String, Integer> cache = new LRU2Cache<String, Integer>();
assertThat(cache.getMaxCapacity(), equalTo(1000));
cache.setMaxCapacity(10);
assertThat(cache.getMaxCapacity(), equalTo(10));
diff --git a/dubbo-common/src/test/resources/test-cache.dubbo.cache b/dubbo-common/src/test/resources/test-cache.dubbo.cache
new file mode 100644
index 0000000..0695eef
--- /dev/null
+++ b/dubbo-common/src/test/resources/test-cache.dubbo.cache
@@ -0,0 +1,2 @@
+111=cache-entry-1
+222=cache-entry-2
diff --git a/dubbo-demo/dubbo-demo-xml/dubbo-demo-xml-consumer/src/main/resources/spring/dubbo-consumer.xml b/dubbo-demo/dubbo-demo-xml/dubbo-demo-xml-consumer/src/main/resources/spring/dubbo-consumer.xml
index be27b4c..88f9253 100644
--- a/dubbo-demo/dubbo-demo-xml/dubbo-demo-xml-consumer/src/main/resources/spring/dubbo-consumer.xml
+++ b/dubbo-demo/dubbo-demo-xml/dubbo-demo-xml-consumer/src/main/resources/spring/dubbo-consumer.xml
@@ -26,7 +26,7 @@
<dubbo:metadata-report address="zookeeper://127.0.0.1:2181"/>
- <dubbo:registry address="zookeeper://127.0.0.1:2181"/>
+ <dubbo:registry address="zookeeper://127.0.0.1:2181?registry-type=service"/>
<dubbo:reference id="demoService" check="false"
interface="org.apache.dubbo.demo.DemoService"/>
diff --git a/dubbo-filter/dubbo-filter-cache/src/main/java/org/apache/dubbo/cache/support/lru/LruCache.java b/dubbo-filter/dubbo-filter-cache/src/main/java/org/apache/dubbo/cache/support/lru/LruCache.java
index 1b8022f..5b7c256 100644
--- a/dubbo-filter/dubbo-filter-cache/src/main/java/org/apache/dubbo/cache/support/lru/LruCache.java
+++ b/dubbo-filter/dubbo-filter-cache/src/main/java/org/apache/dubbo/cache/support/lru/LruCache.java
@@ -18,7 +18,7 @@ package org.apache.dubbo.cache.support.lru;
import org.apache.dubbo.cache.Cache;
import org.apache.dubbo.common.URL;
-import org.apache.dubbo.common.utils.LRUCache;
+import org.apache.dubbo.common.utils.LRU2Cache;
import java.util.Map;
@@ -54,7 +54,7 @@ public class LruCache implements Cache {
*/
public LruCache(URL url) {
final int max = url.getParameter("cache.size", 1000);
- this.store = new LRUCache<>(max);
+ this.store = new LRU2Cache<>(max);
}
/**
diff --git a/dubbo-metadata/dubbo-metadata-api/src/test/java/org/apache/dubbo/metadata/MetadataInfoTest.java b/dubbo-metadata/dubbo-metadata-api/src/test/java/org/apache/dubbo/metadata/MetadataInfoTest.java
index 8b05868..1f78062 100644
--- a/dubbo-metadata/dubbo-metadata-api/src/test/java/org/apache/dubbo/metadata/MetadataInfoTest.java
+++ b/dubbo-metadata/dubbo-metadata-api/src/test/java/org/apache/dubbo/metadata/MetadataInfoTest.java
@@ -18,6 +18,7 @@ package org.apache.dubbo.metadata;
import org.apache.dubbo.common.URL;
+import com.google.gson.Gson;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
@@ -120,4 +121,21 @@ public class MetadataInfoTest {
metadataInfo.removeService(url2);
assertTrue(metadataInfo.updated.get());
}
+
+ @Test
+ public void testJsonFormat() {
+ MetadataInfo metadataInfo = new MetadataInfo("demo");
+
+ // export normal url again
+ metadataInfo.addService(new MetadataInfo.ServiceInfo(url));
+ Gson gson = new Gson();
+ System.out.println(gson.toJson(metadataInfo));
+
+ MetadataInfo metadataInfo2 = new MetadataInfo("demo");
+ // export normal url again
+ metadataInfo2.addService(new MetadataInfo.ServiceInfo(url));
+ metadataInfo2.addService(new MetadataInfo.ServiceInfo(url2));
+ System.out.println(gson.toJson(metadataInfo2));
+
+ }
}
diff --git a/dubbo-registry/dubbo-registry-api/src/main/java/org/apache/dubbo/registry/client/metadata/store/MetaCacheManager.java b/dubbo-registry/dubbo-registry-api/src/main/java/org/apache/dubbo/registry/client/metadata/store/MetaCacheManager.java
new file mode 100644
index 0000000..079c953
--- /dev/null
+++ b/dubbo-registry/dubbo-registry-api/src/main/java/org/apache/dubbo/registry/client/metadata/store/MetaCacheManager.java
@@ -0,0 +1,150 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dubbo.registry.client.metadata.store;
+
+import org.apache.dubbo.common.cache.FileCacheStore;
+import org.apache.dubbo.common.logger.Logger;
+import org.apache.dubbo.common.logger.LoggerFactory;
+import org.apache.dubbo.common.resource.Disposable;
+import org.apache.dubbo.common.utils.JsonUtils;
+import org.apache.dubbo.common.utils.LRUCache;
+import org.apache.dubbo.common.utils.NamedThreadFactory;
+import org.apache.dubbo.common.utils.StringUtils;
+import org.apache.dubbo.metadata.MetadataInfo;
+import org.apache.dubbo.rpc.model.ScopeModel;
+import org.apache.dubbo.rpc.model.ScopeModelAware;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Metadata cache with limited size that uses LRU expiry policy.
+ */
+public class MetaCacheManager implements ScopeModelAware, Disposable {
+ private static final Logger logger = LoggerFactory.getLogger(MetaCacheManager.class);
+ private static final String DEFAULT_FILE_NAME = ".metadata";
+ private static final String SUFFIX = ".dubbo.cache";
+ private static final int DEFAULT_ENTRY_SIZE = 1000;
+
+ private static final long INTERVAL = 60L;
+ private ScheduledExecutorService executorService;
+
+ protected FileCacheStore cacheStore;
+ protected LRUCache<String, MetadataInfo> cache;
+
+ public static MetaCacheManager getInstance(ScopeModel scopeModel) {
+ return scopeModel.getBeanFactory().getOrRegisterBean(MetaCacheManager.class);
+ }
+
+ public MetaCacheManager() {
+ String filePath = System.getProperty("dubbo.meta.cache.filePath");
+ String fileName = System.getProperty("dubbo.meta.cache.fileName");
+ if (StringUtils.isEmpty(fileName)) {
+ fileName = DEFAULT_FILE_NAME;
+ }
+
+ String rawEntrySize = System.getProperty("dubbo.meta.cache.entrySize");
+ int entrySize = StringUtils.parseInteger(rawEntrySize);
+ entrySize = (entrySize == 0 ? DEFAULT_ENTRY_SIZE : entrySize);
+
+ cache = new LRUCache<>(entrySize);
+
+ try {
+ cacheStore = new FileCacheStore(filePath, fileName);
+ Map<String, String> properties = cacheStore.loadCache(entrySize);
+ logger.info("Successfully loaded meta cache from file " + fileName + ", entries " + properties.size());
+ for (Map.Entry<String, String> entry : properties.entrySet()) {
+ String key = (String) entry.getKey();
+ String value = (String) entry.getValue();
+
+ MetadataInfo metadataInfo = JsonUtils.getGson().fromJson(value, MetadataInfo.class);
+ cache.put(key, metadataInfo);
+ }
+
+ executorService = Executors.newSingleThreadScheduledExecutor(new NamedThreadFactory("Dubbo-cache-refresh", true));
+ executorService.scheduleWithFixedDelay(new CacheRefreshTask(cacheStore, cache), 10, INTERVAL, TimeUnit.MINUTES);
+ } catch (Exception e) {
+ logger.error("Load metadata from local cache file error ", e);
+ }
+ }
+
+ public MetadataInfo get(String key) {
+ return cache.get(key);
+ }
+
+ public void put(String key, MetadataInfo metadataInfo) {
+ cache.put(key, metadataInfo);
+ }
+
+ public Map<String, MetadataInfo> getAll() {
+ if (cache.isEmpty()) {
+ return Collections.emptyMap();
+ }
+
+ Map<String, MetadataInfo> copyMap = new HashMap<>();
+ cache.lock();
+ try {
+ for (Map.Entry<String, MetadataInfo> entry : cache.entrySet()) {
+ copyMap.put(entry.getKey(), entry.getValue());
+ }
+ } finally {
+ cache.releaseLock();
+ }
+ return Collections.unmodifiableMap(copyMap);
+ }
+
+ public void update(Map<String, MetadataInfo> revisionToMetadata) {
+ for (Map.Entry<String, MetadataInfo> entry : revisionToMetadata.entrySet()) {
+ cache.put(entry.getKey(), entry.getValue());
+ }
+ }
+
+ public void destroy() {
+ executorService.shutdownNow();
+ }
+
+ protected static class CacheRefreshTask implements Runnable {
+ private final FileCacheStore cacheStore;
+ private final LRUCache<String, MetadataInfo> cache;
+
+ public CacheRefreshTask(FileCacheStore cacheStore, LRUCache<String, MetadataInfo> cache) {
+ this.cacheStore = cacheStore;
+ this.cache = cache;
+ }
+
+ @Override
+ public void run() {
+ Map<String, String> properties = new HashMap<>();
+
+ cache.lock();
+ try {
+ for (Map.Entry<String, MetadataInfo> entry : cache.entrySet()) {
+ properties.put(entry.getKey(), JsonUtils.getGson().toJson(entry.getValue()));
+ }
+ } finally {
+ cache.releaseLock();
+ }
+
+ logger.info("Dumping meta caches, latest entries " + properties.size());
+ cacheStore.refreshCache(properties, "Metadata cache");
+ }
+ }
+}
diff --git a/dubbo-registry/dubbo-registry-api/src/main/java/org/apache/dubbo/registry/support/AbstractRegistry.java b/dubbo-registry/dubbo-registry-api/src/main/java/org/apache/dubbo/registry/support/AbstractRegistry.java
index 4c71d83..176f72b 100644
--- a/dubbo-registry/dubbo-registry-api/src/main/java/org/apache/dubbo/registry/support/AbstractRegistry.java
+++ b/dubbo-registry/dubbo-registry-api/src/main/java/org/apache/dubbo/registry/support/AbstractRegistry.java
@@ -177,8 +177,9 @@ public abstract class AbstractRegistry implements Registry {
return;
}
// Save
+ File lockfile = null;
try {
- File lockfile = new File(file.getAbsolutePath() + ".lock");
+ lockfile = new File(file.getAbsolutePath() + ".lock");
if (!lockfile.exists()) {
lockfile.createNewFile();
}
@@ -214,6 +215,10 @@ public abstract class AbstractRegistry implements Registry {
registryCacheExecutor.execute(new SaveProperties(lastCacheChanged.incrementAndGet()));
}
logger.warn("Failed to save registry cache file, will retry, cause: " + e.getMessage(), e);
+ } finally {
+ if (lockfile != null) {
+ lockfile.delete();
+ }
}
}
diff --git a/dubbo-registry/dubbo-registry-api/src/test/java/org/apache/dubbo/registry/client/metadata/store/MetaCacheManagerTest.java b/dubbo-registry/dubbo-registry-api/src/test/java/org/apache/dubbo/registry/client/metadata/store/MetaCacheManagerTest.java
new file mode 100644
index 0000000..60ecead
--- /dev/null
+++ b/dubbo-registry/dubbo-registry-api/src/test/java/org/apache/dubbo/registry/client/metadata/store/MetaCacheManagerTest.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dubbo.registry.client.metadata.store;
+
+import org.apache.dubbo.common.utils.JsonUtils;
+import org.apache.dubbo.metadata.MetadataInfo;
+
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.nio.file.Paths;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+
+public class MetaCacheManagerTest {
+
+ @BeforeEach
+ public void setup() throws URISyntaxException {
+ String directory = getDirectoryOfClassPath();
+ System.setProperty("dubbo.meta.cache.filePath", directory);
+ System.setProperty("dubbo.meta.cache.fileName", "test-metadata.dubbo.cache");
+ }
+
+ @Test
+ public void testCache() {
+// ScheduledExecutorService cacheRefreshExecutor = Executors.newSingleThreadScheduledExecutor(new NamedThreadFactory("Dubbo-cache-refresh"));
+// ExecutorRepository executorRepository = Mockito.mock(ExecutorRepository.class);
+// when(executorRepository.getCacheRefreshExecutor()).thenReturn(cacheRefreshExecutor);
+// ExtensionAccessor extensionAccessor = Mockito.mock(ExtensionAccessor.class);
+// when(extensionAccessor.getDefaultExtension(ExecutorRepository.class)).thenReturn(executorRepository);
+
+ MetaCacheManager cacheManager = new MetaCacheManager();
+ try {
+// cacheManager.setExtensionAccessor(extensionAccessor);
+
+ MetadataInfo metadataInfo = cacheManager.get("1");
+ assertNotNull(metadataInfo);
+ assertEquals("demo", metadataInfo.getApp());
+ metadataInfo = cacheManager.get("2");
+ assertNull(metadataInfo);
+
+ Map<String, MetadataInfo> newMetadatas = new HashMap<>();
+ MetadataInfo metadataInfo2 = JsonUtils.getGson().fromJson("{\"app\":\"demo2\",\"services\":{\"greeting/org.apache.dubbo.registry.service.DemoService2:1.0.0:dubbo\":{\"name\":\"org.apache.dubbo.registry.service.DemoService2\",\"group\":\"greeting\",\"version\":\"1.0.0\",\"protocol\":\"dubbo\",\"path\":\"org.apache.dubbo.registry.service.DemoService2\",\"params\":{\"application\":\"demo-provider2\",\"sayHello.timeout\":\"7000\",\"version\":\"1.0.0\",\"timeout\":\"5000\",\"group [...]
+ newMetadatas.put("2", metadataInfo2);
+
+ cacheManager.update(newMetadatas);
+ metadataInfo = cacheManager.get("1");
+ assertNotNull(metadataInfo);
+ assertEquals("demo", metadataInfo.getApp());
+ metadataInfo = cacheManager.get("2");
+ assertNotNull(metadataInfo);
+ assertEquals("demo2", metadataInfo.getApp());
+ } finally {
+ cacheManager.destroy();
+ }
+ }
+
+
+ @Test
+ public void testCacheDump() {
+ System.setProperty("dubbo.meta.cache.fileName", "not-exist.dubbo.cache");
+ MetadataInfo metadataInfo3 = JsonUtils.getGson().fromJson("{\"app\":\"demo3\",\"services\":{\"greeting/org.apache.dubbo.registry.service.DemoService2:1.0.0:dubbo\":{\"name\":\"org.apache.dubbo.registry.service.DemoService2\",\"group\":\"greeting\",\"version\":\"1.0.0\",\"protocol\":\"dubbo\",\"path\":\"org.apache.dubbo.registry.service.DemoService2\",\"params\":{\"application\":\"demo-provider2\",\"sayHello.timeout\":\"7000\",\"version\":\"1.0.0\",\"timeout\":\"5000\",\"group\":\ [...]
+ MetaCacheManager cacheManager = new MetaCacheManager();
+ try {
+ cacheManager.put("3", metadataInfo3);
+
+ MetaCacheManager.CacheRefreshTask task = new MetaCacheManager.CacheRefreshTask(cacheManager.cacheStore, cacheManager.cache);
+ task.run();
+
+ MetaCacheManager newCacheManager = null;
+ try {
+ newCacheManager = new MetaCacheManager();
+ MetadataInfo metadataInfo = newCacheManager.get("3");
+ assertNotNull(metadataInfo);
+ assertEquals("demo3", metadataInfo.getApp());
+ } finally {
+ newCacheManager.destroy();
+ }
+ } finally {
+ cacheManager.destroy();
+ }
+ }
+
+
+ private String getDirectoryOfClassPath() throws URISyntaxException {
+ URL resource = this.getClass().getResource("/log4j.xml");
+ String path = Paths.get(resource.toURI()).toFile().getAbsolutePath();
+ int index = path.indexOf("log4j.xml");
+ String directoryPath = path.substring(0, index);
+ return directoryPath;
+ }
+}
diff --git a/dubbo-registry/dubbo-registry-api/src/test/resources/test-metadata.dubbo.cache b/dubbo-registry/dubbo-registry-api/src/test/resources/test-metadata.dubbo.cache
new file mode 100644
index 0000000..e3fa4ae
--- /dev/null
+++ b/dubbo-registry/dubbo-registry-api/src/test/resources/test-metadata.dubbo.cache
@@ -0,0 +1 @@
+1={"app":"demo","services":{"greeting/org.apache.dubbo.registry.service.DemoService2:1.0.0:dubbo":{"name":"org.apache.dubbo.registry.service.DemoService2","group":"greeting","version":"1.0.0","protocol":"dubbo","path":"org.apache.dubbo.registry.service.DemoService2","params":{"application":"demo-provider2","sayHello.timeout":"7000","version":"1.0.0","timeout":"5000","group":"greeting"}}}}
diff --git a/pom.xml b/pom.xml
index e202b5b..0a4cf60 100644
--- a/pom.xml
+++ b/pom.xml
@@ -651,6 +651,7 @@
<exclude>**/generated/**</exclude>
<!-- exclude mockito extensions spi files -->
<exclude>**/mockito-extensions/*</exclude>
+ <exclude>**/*.dubbo.cache</exclude>
</excludes>
</configuration>
</plugin>