You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@dubbo.apache.org by "FinalT (via GitHub)" <gi...@apache.org> on 2023/08/06 02:48:03 UTC
[GitHub] [dubbo-go] FinalT opened a new pull request, #2371: Implement meta cache
FinalT opened a new pull request, #2371:
URL: https://github.com/apache/dubbo-go/pull/2371
(no comment)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@dubbo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: notifications-unsubscribe@dubbo.apache.org
For additional commands, e-mail: notifications-help@dubbo.apache.org
[GitHub] [dubbo-go] zhaoyunxing92 commented on a diff in pull request #2371: Implement meta cache
Posted by "zhaoyunxing92 (via GitHub)" <gi...@apache.org>.
zhaoyunxing92 commented on code in PR #2371:
URL: https://github.com/apache/dubbo-go/pull/2371#discussion_r1287856345
##########
registry/servicediscovery/service_instances_changed_listener_impl.go:
##########
@@ -32,9 +33,24 @@ import (
"dubbo.apache.org/dubbo-go/v3/common/constant"
"dubbo.apache.org/dubbo-go/v3/common/extension"
"dubbo.apache.org/dubbo-go/v3/registry"
+ "dubbo.apache.org/dubbo-go/v3/registry/servicediscovery/store"
"dubbo.apache.org/dubbo-go/v3/remoting"
)
+var (
+ metaCache *store.CacheManager
+)
+
+const (
+ defaultCacheName = "meta"
Review Comment:
建议常量放到公共地方
##########
registry/servicediscovery/service_instances_changed_listener_impl.go:
##########
@@ -32,9 +33,24 @@ import (
"dubbo.apache.org/dubbo-go/v3/common/constant"
"dubbo.apache.org/dubbo-go/v3/common/extension"
"dubbo.apache.org/dubbo-go/v3/registry"
+ "dubbo.apache.org/dubbo-go/v3/registry/servicediscovery/store"
"dubbo.apache.org/dubbo-go/v3/remoting"
)
+var (
+ metaCache *store.CacheManager
+)
+
+const (
+ defaultCacheName = "meta"
Review Comment:
建议常量放到公共地方
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@dubbo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: notifications-unsubscribe@dubbo.apache.org
For additional commands, e-mail: notifications-help@dubbo.apache.org
[GitHub] [dubbo-go] justxuewei commented on a diff in pull request #2371: Implement meta cache
Posted by "justxuewei (via GitHub)" <gi...@apache.org>.
justxuewei commented on code in PR #2371:
URL: https://github.com/apache/dubbo-go/pull/2371#discussion_r1288416835
##########
registry/servicediscovery/store/cache_manager.go:
##########
@@ -0,0 +1,178 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package store
+
+import (
+ "encoding/gob"
+ "os"
+ "sync"
+ "time"
+)
+
+import (
+ "github.com/dubbogo/gost/log/logger"
+
+ "github.com/hashicorp/golang-lru"
+)
+
+type CacheManager struct {
+ name string // The name of the cache manager
+ cacheFile string // The file path where the cache is stored
+ cacheExpired time.Duration // The duration after which the cache expires
+ stop chan struct{} // Channel used to stop the cache expiration routine
+ cache *lru.Cache // The LRU cache implementation
+ lock sync.Mutex
+}
+
+type Item struct {
+ Key string
+ Value interface{}
+}
+
+// NewCacheManager creates a new CacheManager instance.
+// It initializes the cache manager with the provided parameters and starts a routine for cache expiration.
+func NewCacheManager(name, cacheFile string, cacheExpired time.Duration, maxCacheSize int) (*CacheManager, error) {
+ cm := &CacheManager{
+ name: name,
+ cacheFile: cacheFile,
+ cacheExpired: cacheExpired,
+ stop: make(chan struct{}),
+ }
+ cm.cache, _ = lru.New(maxCacheSize)
+
+ // Check if the cache file exists and load the cache if it does
+ if _, err := os.Stat(cacheFile); err == nil {
+ if err = cm.loadCache(); err != nil {
+ logger.Warnf("Failed to load the cache file:[%s].The err is %v", cm.cacheFile, err)
+ }
+ }
+ go cm.RunDumpTask()
+
+ return cm, nil
+}
+
+// Get retrieves the value associated with the given key from the cache.
+func (cm *CacheManager) Get(key string) (interface{}, bool) {
+ return cm.cache.Get(key)
+}
+
+// Set sets the value associated with the given key in the cache.
+func (cm *CacheManager) Set(key string, value interface{}) {
+ cm.cache.Add(key, value)
+}
+
+// Delete removes the value associated with the given key from the cache.
+func (cm *CacheManager) Delete(key string) {
+ cm.cache.Remove(key)
+}
+
+// GetAll returns all the key-value pairs in the cache.
+func (cm *CacheManager) GetAll() map[string]interface{} {
+ keys := cm.cache.Keys()
+
+ result := make(map[string]interface{})
+ for _, k := range keys {
+ result[k.(string)], _ = cm.cache.Get(k)
+ }
+
+ return result
+}
+
+// loadCache loads the cache from the cache file.
+func (cm *CacheManager) loadCache() error {
+ cf, err := os.Open(cm.cacheFile)
+ if err != nil {
+ return err
+ }
+
+ decoder := gob.NewDecoder(cf)
+ for {
+ var it Item
+ err = decoder.Decode(&it)
+ if err != nil {
+ if err.Error() == "EOF" {
+ break // Reached end of file
+ }
+ return err
+ }
+ // Add the loaded keys to the front of the LRU list
+ cm.cache.Add(it.Key, it.Value)
+ }
+
+ return cf.Close()
+}
+
+// dumpCache dumps the cache to the cache file.
+func (cm *CacheManager) dumpCache() error {
+
+ cm.lock.Lock()
+ defer cm.lock.Unlock()
+
+ items := cm.GetAll()
+
+ file, err := os.Create(cm.cacheFile)
+ if err != nil {
+ return err
+ }
+
+ encoder := gob.NewEncoder(file)
+ for k, v := range items {
+ gob.Register(v)
+ err = encoder.Encode(&Item{
+ Key: k,
+ Value: v,
+ })
+ if err != nil {
+ return err
+ }
+ }
+ return file.Close()
+
+}
+
+func (cm *CacheManager) RunDumpTask() {
+ ticker := time.NewTicker(cm.cacheExpired)
+ for {
+ select {
+ case <-ticker.C:
+ // Dump the cache to the file
+ if err := cm.dumpCache(); err != nil {
+ // Handle error
+ logger.Warnf("Failed to dump cache,the err is %v", err)
+ } else {
+ logger.Infof("Dumping [%s] caches, latest entries %d", cm.name, cm.cache.Len())
+ }
+ case <-cm.stop:
+ ticker.Stop()
+ return
+ }
+ }
+}
Review Comment:
IIUC, you want to save the data to a file over a period of time. Then, `cacheExpired` is a confused name for me. Some names, like `dumpInterval`, are better than the current name.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@dubbo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: notifications-unsubscribe@dubbo.apache.org
For additional commands, e-mail: notifications-help@dubbo.apache.org
[GitHub] [dubbo-go] justxuewei commented on a diff in pull request #2371: Implement meta cache
Posted by "justxuewei (via GitHub)" <gi...@apache.org>.
justxuewei commented on code in PR #2371:
URL: https://github.com/apache/dubbo-go/pull/2371#discussion_r1289454428
##########
registry/servicediscovery/service_instances_changed_listener_impl.go:
##########
@@ -104,6 +122,10 @@ func (lstn *ServiceInstancesChangedListenerImpl) OnEvent(e observer.Event) error
newRevisionToMetadata[revision] = metadataInfo
}
lstn.revisionToMetadata = newRevisionToMetadata
+ for revision, metadataInfo := range newRevisionToMetadata {
+ metaCache.Set(revision, metadataInfo)
+ }
+ metaCache.DumpCache()
Review Comment:
I see the `DumpCache()` is called every time metadata is updated in your lastest patch. I'm worried about performance if the matadata is update frequently. How much persistent metadata improves performance? I am thinking it not costs too much resources to reload the metadata from remotes once the matadata is lost. Wdyt? /cc @chickenlj
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@dubbo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: notifications-unsubscribe@dubbo.apache.org
For additional commands, e-mail: notifications-help@dubbo.apache.org
[GitHub] [dubbo-go] justxuewei merged pull request #2371: Implement meta cache
Posted by "justxuewei (via GitHub)" <gi...@apache.org>.
justxuewei merged PR #2371:
URL: https://github.com/apache/dubbo-go/pull/2371
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@dubbo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: notifications-unsubscribe@dubbo.apache.org
For additional commands, e-mail: notifications-help@dubbo.apache.org
[GitHub] [dubbo-go] justxuewei commented on a diff in pull request #2371: Implement meta cache
Posted by "justxuewei (via GitHub)" <gi...@apache.org>.
justxuewei commented on code in PR #2371:
URL: https://github.com/apache/dubbo-go/pull/2371#discussion_r1288416835
##########
registry/servicediscovery/store/cache_manager.go:
##########
@@ -0,0 +1,178 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package store
+
+import (
+ "encoding/gob"
+ "os"
+ "sync"
+ "time"
+)
+
+import (
+ "github.com/dubbogo/gost/log/logger"
+
+ "github.com/hashicorp/golang-lru"
+)
+
+type CacheManager struct {
+ name string // The name of the cache manager
+ cacheFile string // The file path where the cache is stored
+ cacheExpired time.Duration // The duration after which the cache expires
+ stop chan struct{} // Channel used to stop the cache expiration routine
+ cache *lru.Cache // The LRU cache implementation
+ lock sync.Mutex
+}
+
+type Item struct {
+ Key string
+ Value interface{}
+}
+
+// NewCacheManager creates a new CacheManager instance.
+// It initializes the cache manager with the provided parameters and starts a routine for cache expiration.
+func NewCacheManager(name, cacheFile string, cacheExpired time.Duration, maxCacheSize int) (*CacheManager, error) {
+ cm := &CacheManager{
+ name: name,
+ cacheFile: cacheFile,
+ cacheExpired: cacheExpired,
+ stop: make(chan struct{}),
+ }
+ cm.cache, _ = lru.New(maxCacheSize)
+
+ // Check if the cache file exists and load the cache if it does
+ if _, err := os.Stat(cacheFile); err == nil {
+ if err = cm.loadCache(); err != nil {
+ logger.Warnf("Failed to load the cache file:[%s].The err is %v", cm.cacheFile, err)
+ }
+ }
+ go cm.RunDumpTask()
+
+ return cm, nil
+}
+
+// Get retrieves the value associated with the given key from the cache.
+func (cm *CacheManager) Get(key string) (interface{}, bool) {
+ return cm.cache.Get(key)
+}
+
+// Set sets the value associated with the given key in the cache.
+func (cm *CacheManager) Set(key string, value interface{}) {
+ cm.cache.Add(key, value)
+}
+
+// Delete removes the value associated with the given key from the cache.
+func (cm *CacheManager) Delete(key string) {
+ cm.cache.Remove(key)
+}
+
+// GetAll returns all the key-value pairs in the cache.
+func (cm *CacheManager) GetAll() map[string]interface{} {
+ keys := cm.cache.Keys()
+
+ result := make(map[string]interface{})
+ for _, k := range keys {
+ result[k.(string)], _ = cm.cache.Get(k)
+ }
+
+ return result
+}
+
+// loadCache loads the cache from the cache file.
+func (cm *CacheManager) loadCache() error {
+ cf, err := os.Open(cm.cacheFile)
+ if err != nil {
+ return err
+ }
+
+ decoder := gob.NewDecoder(cf)
+ for {
+ var it Item
+ err = decoder.Decode(&it)
+ if err != nil {
+ if err.Error() == "EOF" {
+ break // Reached end of file
+ }
+ return err
+ }
+ // Add the loaded keys to the front of the LRU list
+ cm.cache.Add(it.Key, it.Value)
+ }
+
+ return cf.Close()
+}
+
+// dumpCache dumps the cache to the cache file.
+func (cm *CacheManager) dumpCache() error {
+
+ cm.lock.Lock()
+ defer cm.lock.Unlock()
+
+ items := cm.GetAll()
+
+ file, err := os.Create(cm.cacheFile)
+ if err != nil {
+ return err
+ }
+
+ encoder := gob.NewEncoder(file)
+ for k, v := range items {
+ gob.Register(v)
+ err = encoder.Encode(&Item{
+ Key: k,
+ Value: v,
+ })
+ if err != nil {
+ return err
+ }
+ }
+ return file.Close()
+
+}
+
+func (cm *CacheManager) RunDumpTask() {
+ ticker := time.NewTicker(cm.cacheExpired)
+ for {
+ select {
+ case <-ticker.C:
+ // Dump the cache to the file
+ if err := cm.dumpCache(); err != nil {
+ // Handle error
+ logger.Warnf("Failed to dump cache,the err is %v", err)
+ } else {
+ logger.Infof("Dumping [%s] caches, latest entries %d", cm.name, cm.cache.Len())
+ }
+ case <-cm.stop:
+ ticker.Stop()
+ return
+ }
+ }
+}
Review Comment:
IIUR, you want to save the data to file over a period of time. Then, `cacheExpired` is a confused name for me. Some names, like `dumpInterval`, are better than the current name.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@dubbo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: notifications-unsubscribe@dubbo.apache.org
For additional commands, e-mail: notifications-help@dubbo.apache.org
[GitHub] [dubbo-go] justxuewei commented on a diff in pull request #2371: Implement meta cache
Posted by "justxuewei (via GitHub)" <gi...@apache.org>.
justxuewei commented on code in PR #2371:
URL: https://github.com/apache/dubbo-go/pull/2371#discussion_r1289454428
##########
registry/servicediscovery/service_instances_changed_listener_impl.go:
##########
@@ -104,6 +122,10 @@ func (lstn *ServiceInstancesChangedListenerImpl) OnEvent(e observer.Event) error
newRevisionToMetadata[revision] = metadataInfo
}
lstn.revisionToMetadata = newRevisionToMetadata
+ for revision, metadataInfo := range newRevisionToMetadata {
+ metaCache.Set(revision, metadataInfo)
+ }
+ metaCache.DumpCache()
Review Comment:
I see the `DumpCache()` is called every time metadata is updated in your lastest patch. I'm worried about performance if the matadata is update frequently. How much persisting metadata improves performance? I am thinking it not costs too much resources to pull the metadata from remotes once the metadata is lost. Wdyt? /cc @chickenlj
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@dubbo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: notifications-unsubscribe@dubbo.apache.org
For additional commands, e-mail: notifications-help@dubbo.apache.org
[GitHub] [dubbo-go] sonarcloud[bot] commented on pull request #2371: Implement meta cache
Posted by "sonarcloud[bot] (via GitHub)" <gi...@apache.org>.
sonarcloud[bot] commented on PR #2371:
URL: https://github.com/apache/dubbo-go/pull/2371#issuecomment-1672846396
Kudos, SonarCloud Quality Gate passed! [](https://sonarcloud.io/dashboard?id=apache_dubbo-go&pullRequest=2371)
[](https://sonarcloud.io/project/issues?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=BUG) [](https://sonarcloud.io/project/issues?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=BUG) [0 Bugs](https://sonarcloud.io/project/issues?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=BUG)
[](https://sonarcloud.io/project/issues?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=VULNERABILITY) [](https://sonarcloud.io/project/issues?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=VULNERABILITY) [0 Vulnerabilities](https://sonarcloud.io/project/issues?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=VULNERABILITY)
[](https://sonarcloud.io/project/security_hotspots?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=SECURITY_HOTSPOT) [](https://sonarcloud.io/project/security_hotspots?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=SECURITY_HOTSPOT) [0 Security Hotspots](https://sonarcloud.io/project/security_hotspots?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=SECURITY_HOTSPOT)
[](https://sonarcloud.io/project/issues?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=CODE_SMELL) [](https://sonarcloud.io/project/issues?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=CODE_SMELL) [2 Code Smells](https://sonarcloud.io/project/issues?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=CODE_SMELL)
[](https://sonarcloud.io/component_measures?id=apache_dubbo-go&pullRequest=2371) No Coverage information
[](https://sonarcloud.io/component_measures?id=apache_dubbo-go&pullRequest=2371&metric=new_duplicated_lines_density&view=list) [9.7% Duplication](https://sonarcloud.io/component_measures?id=apache_dubbo-go&pullRequest=2371&metric=new_duplicated_lines_density&view=list)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@dubbo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: notifications-unsubscribe@dubbo.apache.org
For additional commands, e-mail: notifications-help@dubbo.apache.org
[GitHub] [dubbo-go] sonarcloud[bot] commented on pull request #2371: Implement meta cache
Posted by "sonarcloud[bot] (via GitHub)" <gi...@apache.org>.
sonarcloud[bot] commented on PR #2371:
URL: https://github.com/apache/dubbo-go/pull/2371#issuecomment-1672412504
Kudos, SonarCloud Quality Gate passed! [](https://sonarcloud.io/dashboard?id=apache_dubbo-go&pullRequest=2371)
[](https://sonarcloud.io/project/issues?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=BUG) [](https://sonarcloud.io/project/issues?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=BUG) [0 Bugs](https://sonarcloud.io/project/issues?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=BUG)
[](https://sonarcloud.io/project/issues?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=VULNERABILITY) [](https://sonarcloud.io/project/issues?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=VULNERABILITY) [0 Vulnerabilities](https://sonarcloud.io/project/issues?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=VULNERABILITY)
[](https://sonarcloud.io/project/security_hotspots?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=SECURITY_HOTSPOT) [](https://sonarcloud.io/project/security_hotspots?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=SECURITY_HOTSPOT) [0 Security Hotspots](https://sonarcloud.io/project/security_hotspots?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=SECURITY_HOTSPOT)
[](https://sonarcloud.io/project/issues?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=CODE_SMELL) [](https://sonarcloud.io/project/issues?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=CODE_SMELL) [2 Code Smells](https://sonarcloud.io/project/issues?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=CODE_SMELL)
[](https://sonarcloud.io/component_measures?id=apache_dubbo-go&pullRequest=2371) No Coverage information
[](https://sonarcloud.io/component_measures?id=apache_dubbo-go&pullRequest=2371&metric=new_duplicated_lines_density&view=list) [10.4% Duplication](https://sonarcloud.io/component_measures?id=apache_dubbo-go&pullRequest=2371&metric=new_duplicated_lines_density&view=list)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@dubbo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: notifications-unsubscribe@dubbo.apache.org
For additional commands, e-mail: notifications-help@dubbo.apache.org
[GitHub] [dubbo-go] sonarcloud[bot] commented on pull request #2371: Implement meta cache
Posted by "sonarcloud[bot] (via GitHub)" <gi...@apache.org>.
sonarcloud[bot] commented on PR #2371:
URL: https://github.com/apache/dubbo-go/pull/2371#issuecomment-1666689679
Kudos, SonarCloud Quality Gate passed! [](https://sonarcloud.io/dashboard?id=apache_dubbo-go&pullRequest=2371)
[](https://sonarcloud.io/project/issues?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=BUG) [](https://sonarcloud.io/project/issues?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=BUG) [0 Bugs](https://sonarcloud.io/project/issues?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=BUG)
[](https://sonarcloud.io/project/issues?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=VULNERABILITY) [](https://sonarcloud.io/project/issues?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=VULNERABILITY) [0 Vulnerabilities](https://sonarcloud.io/project/issues?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=VULNERABILITY)
[](https://sonarcloud.io/project/security_hotspots?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=SECURITY_HOTSPOT) [](https://sonarcloud.io/project/security_hotspots?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=SECURITY_HOTSPOT) [0 Security Hotspots](https://sonarcloud.io/project/security_hotspots?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=SECURITY_HOTSPOT)
[](https://sonarcloud.io/project/issues?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=CODE_SMELL) [](https://sonarcloud.io/project/issues?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=CODE_SMELL) [2 Code Smells](https://sonarcloud.io/project/issues?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=CODE_SMELL)
[](https://sonarcloud.io/component_measures?id=apache_dubbo-go&pullRequest=2371) No Coverage information
[](https://sonarcloud.io/component_measures?id=apache_dubbo-go&pullRequest=2371&metric=new_duplicated_lines_density&view=list) [9.6% Duplication](https://sonarcloud.io/component_measures?id=apache_dubbo-go&pullRequest=2371&metric=new_duplicated_lines_density&view=list)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@dubbo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: notifications-unsubscribe@dubbo.apache.org
For additional commands, e-mail: notifications-help@dubbo.apache.org
[GitHub] [dubbo-go] Lvnszn commented on a diff in pull request #2371: Implement meta cache
Posted by "Lvnszn (via GitHub)" <gi...@apache.org>.
Lvnszn commented on code in PR #2371:
URL: https://github.com/apache/dubbo-go/pull/2371#discussion_r1288709638
##########
registry/servicediscovery/store/cache_manager.go:
##########
@@ -0,0 +1,178 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package store
+
+import (
+ "encoding/gob"
+ "os"
+ "sync"
+ "time"
+)
+
+import (
+ "github.com/dubbogo/gost/log/logger"
+
+ "github.com/hashicorp/golang-lru"
+)
+
+type CacheManager struct {
+ name string // The name of the cache manager
+ cacheFile string // The file path where the cache is stored
+ cacheExpired time.Duration // The duration after which the cache expires
+ stop chan struct{} // Channel used to stop the cache expiration routine
+ cache *lru.Cache // The LRU cache implementation
+ lock sync.Mutex
+}
+
+type Item struct {
+ Key string
+ Value interface{}
+}
+
+// NewCacheManager creates a new CacheManager instance.
+// It initializes the cache manager with the provided parameters and starts a routine for cache expiration.
+func NewCacheManager(name, cacheFile string, cacheExpired time.Duration, maxCacheSize int) (*CacheManager, error) {
+ cm := &CacheManager{
+ name: name,
+ cacheFile: cacheFile,
+ cacheExpired: cacheExpired,
+ stop: make(chan struct{}),
+ }
+ cm.cache, _ = lru.New(maxCacheSize)
+
+ // Check if the cache file exists and load the cache if it does
+ if _, err := os.Stat(cacheFile); err == nil {
+ if err = cm.loadCache(); err != nil {
+ logger.Warnf("Failed to load the cache file:[%s].The err is %v", cm.cacheFile, err)
+ }
+ }
+ go cm.RunDumpTask()
+
+ return cm, nil
+}
+
+// Get retrieves the value associated with the given key from the cache.
+func (cm *CacheManager) Get(key string) (interface{}, bool) {
+ return cm.cache.Get(key)
+}
+
+// Set sets the value associated with the given key in the cache.
+func (cm *CacheManager) Set(key string, value interface{}) {
+ cm.cache.Add(key, value)
+}
+
+// Delete removes the value associated with the given key from the cache.
+func (cm *CacheManager) Delete(key string) {
+ cm.cache.Remove(key)
+}
+
+// GetAll returns all the key-value pairs in the cache.
+func (cm *CacheManager) GetAll() map[string]interface{} {
+ keys := cm.cache.Keys()
+
+ result := make(map[string]interface{})
+ for _, k := range keys {
+ result[k.(string)], _ = cm.cache.Get(k)
+ }
+
+ return result
+}
+
+// loadCache loads the cache from the cache file.
+func (cm *CacheManager) loadCache() error {
+ cf, err := os.Open(cm.cacheFile)
+ if err != nil {
+ return err
+ }
+
+ decoder := gob.NewDecoder(cf)
+ for {
+ var it Item
+ err = decoder.Decode(&it)
+ if err != nil {
+ if err.Error() == "EOF" {
+ break // Reached end of file
+ }
+ return err
+ }
+ // Add the loaded keys to the front of the LRU list
+ cm.cache.Add(it.Key, it.Value)
+ }
+
+ return cf.Close()
+}
+
+// dumpCache dumps the cache to the cache file.
+func (cm *CacheManager) dumpCache() error {
+
+ cm.lock.Lock()
+ defer cm.lock.Unlock()
+
+ items := cm.GetAll()
+
+ file, err := os.Create(cm.cacheFile)
+ if err != nil {
+ return err
+ }
+
+ encoder := gob.NewEncoder(file)
+ for k, v := range items {
+ gob.Register(v)
+ err = encoder.Encode(&Item{
+ Key: k,
+ Value: v,
+ })
+ if err != nil {
+ return err
+ }
+ }
+ return file.Close()
+
+}
+
+func (cm *CacheManager) RunDumpTask() {
+ ticker := time.NewTicker(cm.cacheExpired)
Review Comment:
这个缓存过期是用来清理缓存的么？但是看着使用好像是 dump 到文件里
##########
registry/servicediscovery/store/cache_manager.go:
##########
@@ -0,0 +1,178 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package store
+
+import (
+ "encoding/gob"
+ "os"
+ "sync"
+ "time"
+)
+
+import (
+ "github.com/dubbogo/gost/log/logger"
+
+ "github.com/hashicorp/golang-lru"
+)
+
+type CacheManager struct {
+ name string // The name of the cache manager
+ cacheFile string // The file path where the cache is stored
+ cacheExpired time.Duration // The duration after which the cache expires
+ stop chan struct{} // Channel used to stop the cache expiration routine
+ cache *lru.Cache // The LRU cache implementation
+ lock sync.Mutex
+}
+
+type Item struct {
+ Key string
+ Value interface{}
+}
+
+// NewCacheManager creates a new CacheManager instance.
+// It initializes the cache manager with the provided parameters and starts a routine for cache expiration.
+func NewCacheManager(name, cacheFile string, cacheExpired time.Duration, maxCacheSize int) (*CacheManager, error) {
+ cm := &CacheManager{
+ name: name,
+ cacheFile: cacheFile,
+ cacheExpired: cacheExpired,
+ stop: make(chan struct{}),
+ }
+ cm.cache, _ = lru.New(maxCacheSize)
+
+ // Check if the cache file exists and load the cache if it does
+ if _, err := os.Stat(cacheFile); err == nil {
+ if err = cm.loadCache(); err != nil {
+ logger.Warnf("Failed to load the cache file:[%s].The err is %v", cm.cacheFile, err)
+ }
+ }
+ go cm.RunDumpTask()
+
+ return cm, nil
+}
+
+// Get retrieves the value associated with the given key from the cache.
+func (cm *CacheManager) Get(key string) (interface{}, bool) {
+ return cm.cache.Get(key)
+}
+
+// Set sets the value associated with the given key in the cache.
+func (cm *CacheManager) Set(key string, value interface{}) {
+ cm.cache.Add(key, value)
+}
+
+// Delete removes the value associated with the given key from the cache.
+func (cm *CacheManager) Delete(key string) {
+ cm.cache.Remove(key)
+}
+
+// GetAll returns all the key-value pairs in the cache.
+func (cm *CacheManager) GetAll() map[string]interface{} {
+ keys := cm.cache.Keys()
+
+ result := make(map[string]interface{})
+ for _, k := range keys {
+ result[k.(string)], _ = cm.cache.Get(k)
+ }
+
+ return result
+}
+
+// loadCache loads the cache from the cache file.
+func (cm *CacheManager) loadCache() error {
+ cf, err := os.Open(cm.cacheFile)
+ if err != nil {
+ return err
+ }
+
+ decoder := gob.NewDecoder(cf)
+ for {
+ var it Item
+ err = decoder.Decode(&it)
+ if err != nil {
+ if err.Error() == "EOF" {
+ break // Reached end of file
+ }
+ return err
+ }
+ // Add the loaded keys to the front of the LRU list
+ cm.cache.Add(it.Key, it.Value)
+ }
+
+ return cf.Close()
+}
+
+// dumpCache dumps the cache to the cache file.
+func (cm *CacheManager) dumpCache() error {
+
+ cm.lock.Lock()
+ defer cm.lock.Unlock()
+
+ items := cm.GetAll()
+
+ file, err := os.Create(cm.cacheFile)
+ if err != nil {
+ return err
+ }
+
+ encoder := gob.NewEncoder(file)
+ for k, v := range items {
+ gob.Register(v)
+ err = encoder.Encode(&Item{
+ Key: k,
+ Value: v,
+ })
+ if err != nil {
+ return err
+ }
+ }
+ return file.Close()
+
+}
+
+func (cm *CacheManager) RunDumpTask() {
+ ticker := time.NewTicker(cm.cacheExpired)
Review Comment:
这个缓存过期是用来清理缓存的么？但是看着使用好像是 dump 到文件里
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@dubbo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: notifications-unsubscribe@dubbo.apache.org
For additional commands, e-mail: notifications-help@dubbo.apache.org
[GitHub] [dubbo-go] sonarcloud[bot] commented on pull request #2371: Implement meta cache
Posted by "sonarcloud[bot] (via GitHub)" <gi...@apache.org>.
sonarcloud[bot] commented on PR #2371:
URL: https://github.com/apache/dubbo-go/pull/2371#issuecomment-1666687783
Kudos, SonarCloud Quality Gate passed! [](https://sonarcloud.io/dashboard?id=apache_dubbo-go&pullRequest=2371)
[](https://sonarcloud.io/project/issues?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=BUG) [](https://sonarcloud.io/project/issues?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=BUG) [0 Bugs](https://sonarcloud.io/project/issues?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=BUG)
[](https://sonarcloud.io/project/issues?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=VULNERABILITY) [](https://sonarcloud.io/project/issues?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=VULNERABILITY) [0 Vulnerabilities](https://sonarcloud.io/project/issues?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=VULNERABILITY)
[](https://sonarcloud.io/project/security_hotspots?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=SECURITY_HOTSPOT) [](https://sonarcloud.io/project/security_hotspots?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=SECURITY_HOTSPOT) [0 Security Hotspots](https://sonarcloud.io/project/security_hotspots?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=SECURITY_HOTSPOT)
[](https://sonarcloud.io/project/issues?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=CODE_SMELL) [](https://sonarcloud.io/project/issues?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=CODE_SMELL) [2 Code Smells](https://sonarcloud.io/project/issues?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=CODE_SMELL)
[](https://sonarcloud.io/component_measures?id=apache_dubbo-go&pullRequest=2371) No Coverage information
[](https://sonarcloud.io/component_measures?id=apache_dubbo-go&pullRequest=2371&metric=new_duplicated_lines_density&view=list) [9.4% Duplication](https://sonarcloud.io/component_measures?id=apache_dubbo-go&pullRequest=2371&metric=new_duplicated_lines_density&view=list)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@dubbo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: notifications-unsubscribe@dubbo.apache.org
For additional commands, e-mail: notifications-help@dubbo.apache.org
[GitHub] [dubbo-go] justxuewei commented on a diff in pull request #2371: Implement meta cache
Posted by "justxuewei (via GitHub)" <gi...@apache.org>.
justxuewei commented on code in PR #2371:
URL: https://github.com/apache/dubbo-go/pull/2371#discussion_r1289454428
##########
registry/servicediscovery/service_instances_changed_listener_impl.go:
##########
@@ -104,6 +122,10 @@ func (lstn *ServiceInstancesChangedListenerImpl) OnEvent(e observer.Event) error
newRevisionToMetadata[revision] = metadataInfo
}
lstn.revisionToMetadata = newRevisionToMetadata
+ for revision, metadataInfo := range newRevisionToMetadata {
+ metaCache.Set(revision, metadataInfo)
+ }
+ metaCache.DumpCache()
Review Comment:
I see the `DumpCache()` is called every time metadata is updated in your lastest patch. I'm worried about performance if the matadata is update frequently. How much persistent metadata improves performance? I am thinking it not costs too much resources to pull the metadata from remotes once the matadata is lost. Wdyt? /cc @chickenlj
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@dubbo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: notifications-unsubscribe@dubbo.apache.org
For additional commands, e-mail: notifications-help@dubbo.apache.org
[GitHub] [dubbo-go] codecov-commenter commented on pull request #2371: Implement meta cache
Posted by "codecov-commenter (via GitHub)" <gi...@apache.org>.
codecov-commenter commented on PR #2371:
URL: https://github.com/apache/dubbo-go/pull/2371#issuecomment-1666798122
## [Codecov](https://app.codecov.io/gh/apache/dubbo-go/pull/2371?src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=apache) Report
> Merging [#2371](https://app.codecov.io/gh/apache/dubbo-go/pull/2371?src=pr&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=apache) (e340a3e) into [main](https://app.codecov.io/gh/apache/dubbo-go/commit/8b431355ac2022cde863c145e8ca76d9e5a2e136?el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=apache) (8b43135) will **increase** coverage by `0.07%`.
> The diff coverage is `72.41%`.
> :exclamation: Current head e340a3e differs from pull request most recent head 235a354. Consider uploading reports for the commit 235a354 to get more accurate results
```diff
@@ Coverage Diff @@
## main #2371 +/- ##
==========================================
+ Coverage 44.27% 44.34% +0.07%
==========================================
Files 304 305 +1
Lines 18528 18612 +84
==========================================
+ Hits 8203 8254 +51
- Misses 9467 9492 +25
- Partials 858 866 +8
```
| [Files Changed](https://app.codecov.io/gh/apache/dubbo-go/pull/2371?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=apache) | Coverage Δ | |
|---|---|---|
| [...scovery/service\_instances\_changed\_listener\_impl.go](https://app.codecov.io/gh/apache/dubbo-go/pull/2371?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=apache#diff-cmVnaXN0cnkvc2VydmljZWRpc2NvdmVyeS9zZXJ2aWNlX2luc3RhbmNlc19jaGFuZ2VkX2xpc3RlbmVyX2ltcGwuZ28=) | `1.60% <14.28%> (+1.60%)` | :arrow_up: |
| [registry/servicediscovery/store/cache\_manager.go](https://app.codecov.io/gh/apache/dubbo-go/pull/2371?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=apache#diff-cmVnaXN0cnkvc2VydmljZWRpc2NvdmVyeS9zdG9yZS9jYWNoZV9tYW5hZ2VyLmdv) | `83.56% <83.56%> (ø)` | |
... and [5 files with indirect coverage changes](https://app.codecov.io/gh/apache/dubbo-go/pull/2371/indirect-changes?src=pr&el=tree-more&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=apache)
:mega: We’re building smart automated test selection to slash your CI/CD build times. [Learn more](https://about.codecov.io/iterative-testing/?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=apache)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@dubbo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: notifications-unsubscribe@dubbo.apache.org
For additional commands, e-mail: notifications-help@dubbo.apache.org
[GitHub] [dubbo-go] sonarcloud[bot] commented on pull request #2371: Implement meta cache
Posted by "sonarcloud[bot] (via GitHub)" <gi...@apache.org>.
sonarcloud[bot] commented on PR #2371:
URL: https://github.com/apache/dubbo-go/pull/2371#issuecomment-1670574076
Kudos, SonarCloud Quality Gate passed! [](https://sonarcloud.io/dashboard?id=apache_dubbo-go&pullRequest=2371)
[](https://sonarcloud.io/project/issues?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=BUG) [](https://sonarcloud.io/project/issues?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=BUG) [0 Bugs](https://sonarcloud.io/project/issues?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=BUG)
[](https://sonarcloud.io/project/issues?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=VULNERABILITY) [](https://sonarcloud.io/project/issues?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=VULNERABILITY) [0 Vulnerabilities](https://sonarcloud.io/project/issues?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=VULNERABILITY)
[](https://sonarcloud.io/project/security_hotspots?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=SECURITY_HOTSPOT) [](https://sonarcloud.io/project/security_hotspots?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=SECURITY_HOTSPOT) [0 Security Hotspots](https://sonarcloud.io/project/security_hotspots?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=SECURITY_HOTSPOT)
[](https://sonarcloud.io/project/issues?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=CODE_SMELL) [](https://sonarcloud.io/project/issues?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=CODE_SMELL) [2 Code Smells](https://sonarcloud.io/project/issues?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=CODE_SMELL)
[](https://sonarcloud.io/component_measures?id=apache_dubbo-go&pullRequest=2371) No Coverage information
[](https://sonarcloud.io/component_measures?id=apache_dubbo-go&pullRequest=2371&metric=new_duplicated_lines_density&view=list) [9.5% Duplication](https://sonarcloud.io/component_measures?id=apache_dubbo-go&pullRequest=2371&metric=new_duplicated_lines_density&view=list)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@dubbo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: notifications-unsubscribe@dubbo.apache.org
For additional commands, e-mail: notifications-help@dubbo.apache.org
[GitHub] [dubbo-go] sonarcloud[bot] commented on pull request #2371: Implement meta cache
Posted by "sonarcloud[bot] (via GitHub)" <gi...@apache.org>.
sonarcloud[bot] commented on PR #2371:
URL: https://github.com/apache/dubbo-go/pull/2371#issuecomment-1666688469
Kudos, SonarCloud Quality Gate passed! [](https://sonarcloud.io/dashboard?id=apache_dubbo-go&pullRequest=2371)
[](https://sonarcloud.io/project/issues?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=BUG) [](https://sonarcloud.io/project/issues?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=BUG) [0 Bugs](https://sonarcloud.io/project/issues?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=BUG)
[](https://sonarcloud.io/project/issues?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=VULNERABILITY) [](https://sonarcloud.io/project/issues?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=VULNERABILITY) [0 Vulnerabilities](https://sonarcloud.io/project/issues?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=VULNERABILITY)
[](https://sonarcloud.io/project/security_hotspots?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=SECURITY_HOTSPOT) [](https://sonarcloud.io/project/security_hotspots?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=SECURITY_HOTSPOT) [0 Security Hotspots](https://sonarcloud.io/project/security_hotspots?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=SECURITY_HOTSPOT)
[](https://sonarcloud.io/project/issues?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=CODE_SMELL) [](https://sonarcloud.io/project/issues?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=CODE_SMELL) [2 Code Smells](https://sonarcloud.io/project/issues?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=CODE_SMELL)
[](https://sonarcloud.io/component_measures?id=apache_dubbo-go&pullRequest=2371) No Coverage information
[](https://sonarcloud.io/component_measures?id=apache_dubbo-go&pullRequest=2371&metric=new_duplicated_lines_density&view=list) [9.5% Duplication](https://sonarcloud.io/component_measures?id=apache_dubbo-go&pullRequest=2371&metric=new_duplicated_lines_density&view=list)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@dubbo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: notifications-unsubscribe@dubbo.apache.org
For additional commands, e-mail: notifications-help@dubbo.apache.org
[GitHub] [dubbo-go] sonarcloud[bot] commented on pull request #2371: Implement meta cache
Posted by "sonarcloud[bot] (via GitHub)" <gi...@apache.org>.
sonarcloud[bot] commented on PR #2371:
URL: https://github.com/apache/dubbo-go/pull/2371#issuecomment-1666694279
Kudos, SonarCloud Quality Gate passed! [](https://sonarcloud.io/dashboard?id=apache_dubbo-go&pullRequest=2371)
[](https://sonarcloud.io/project/issues?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=BUG) [](https://sonarcloud.io/project/issues?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=BUG) [0 Bugs](https://sonarcloud.io/project/issues?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=BUG)
[](https://sonarcloud.io/project/issues?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=VULNERABILITY) [](https://sonarcloud.io/project/issues?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=VULNERABILITY) [0 Vulnerabilities](https://sonarcloud.io/project/issues?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=VULNERABILITY)
[](https://sonarcloud.io/project/security_hotspots?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=SECURITY_HOTSPOT) [](https://sonarcloud.io/project/security_hotspots?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=SECURITY_HOTSPOT) [0 Security Hotspots](https://sonarcloud.io/project/security_hotspots?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=SECURITY_HOTSPOT)
[](https://sonarcloud.io/project/issues?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=CODE_SMELL) [](https://sonarcloud.io/project/issues?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=CODE_SMELL) [2 Code Smells](https://sonarcloud.io/project/issues?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=CODE_SMELL)
[](https://sonarcloud.io/component_measures?id=apache_dubbo-go&pullRequest=2371) No Coverage information
[](https://sonarcloud.io/component_measures?id=apache_dubbo-go&pullRequest=2371&metric=new_duplicated_lines_density&view=list) [9.4% Duplication](https://sonarcloud.io/component_measures?id=apache_dubbo-go&pullRequest=2371&metric=new_duplicated_lines_density&view=list)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@dubbo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: notifications-unsubscribe@dubbo.apache.org
For additional commands, e-mail: notifications-help@dubbo.apache.org
[GitHub] [dubbo-go] justxuewei commented on a diff in pull request #2371: Implement meta cache
Posted by "justxuewei (via GitHub)" <gi...@apache.org>.
justxuewei commented on code in PR #2371:
URL: https://github.com/apache/dubbo-go/pull/2371#discussion_r1288388055
##########
common/constant/key.go:
##########
@@ -416,3 +416,10 @@ const (
MetricsMetadata = "dubbo.metrics.metadata"
MetricApp = "dubbo.metrics.app"
)
+
+// default meta cache config
+const (
+ DefaultMetaCacheName = "meta"
+ DefaultMetaFileName = ".matadata"
Review Comment:
This name is too common to use. Considering adding a prefix, `dubbogo.metadata` to avoid name conflict. Btw, there is a typo:
```suggestion
DefaultMetaFileName = ".metadata"
```
##########
registry/servicediscovery/store/cache_manager.go:
##########
@@ -0,0 +1,178 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package store
+
+import (
+ "encoding/gob"
+ "os"
+ "sync"
+ "time"
+)
+
+import (
+ "github.com/dubbogo/gost/log/logger"
+
+ "github.com/hashicorp/golang-lru"
+)
+
+type CacheManager struct {
+ name string // The name of the cache manager
+ cacheFile string // The file path where the cache is stored
+ cacheExpired time.Duration // The duration after which the cache expires
+ stop chan struct{} // Channel used to stop the cache expiration routine
+ cache *lru.Cache // The LRU cache implementation
+ lock sync.Mutex
+}
+
+type Item struct {
+ Key string
+ Value interface{}
+}
+
+// NewCacheManager creates a new CacheManager instance.
+// It initializes the cache manager with the provided parameters and starts a routine for cache expiration.
+func NewCacheManager(name, cacheFile string, cacheExpired time.Duration, maxCacheSize int) (*CacheManager, error) {
+ cm := &CacheManager{
+ name: name,
+ cacheFile: cacheFile,
+ cacheExpired: cacheExpired,
+ stop: make(chan struct{}),
+ }
+ cm.cache, _ = lru.New(maxCacheSize)
Review Comment:
Please do not ingore the error, although the current implemented lru returns `nil`.
##########
registry/servicediscovery/store/cache_manager.go:
##########
@@ -0,0 +1,178 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package store
+
+import (
+ "encoding/gob"
+ "os"
+ "sync"
+ "time"
+)
+
+import (
+ "github.com/dubbogo/gost/log/logger"
+
+ "github.com/hashicorp/golang-lru"
+)
+
+type CacheManager struct {
+ name string // The name of the cache manager
+ cacheFile string // The file path where the cache is stored
+ cacheExpired time.Duration // The duration after which the cache expires
+ stop chan struct{} // Channel used to stop the cache expiration routine
+ cache *lru.Cache // The LRU cache implementation
+ lock sync.Mutex
+}
+
+type Item struct {
+ Key string
+ Value interface{}
+}
+
+// NewCacheManager creates a new CacheManager instance.
+// It initializes the cache manager with the provided parameters and starts a routine for cache expiration.
+func NewCacheManager(name, cacheFile string, cacheExpired time.Duration, maxCacheSize int) (*CacheManager, error) {
+ cm := &CacheManager{
+ name: name,
+ cacheFile: cacheFile,
+ cacheExpired: cacheExpired,
+ stop: make(chan struct{}),
+ }
+ cm.cache, _ = lru.New(maxCacheSize)
+
+ // Check if the cache file exists and load the cache if it does
+ if _, err := os.Stat(cacheFile); err == nil {
+ if err = cm.loadCache(); err != nil {
+ logger.Warnf("Failed to load the cache file:[%s].The err is %v", cm.cacheFile, err)
+ }
+ }
+ go cm.RunDumpTask()
+
+ return cm, nil
+}
+
+// Get retrieves the value associated with the given key from the cache.
+func (cm *CacheManager) Get(key string) (interface{}, bool) {
+ return cm.cache.Get(key)
+}
+
+// Set sets the value associated with the given key in the cache.
+func (cm *CacheManager) Set(key string, value interface{}) {
+ cm.cache.Add(key, value)
+}
+
+// Delete removes the value associated with the given key from the cache.
+func (cm *CacheManager) Delete(key string) {
+ cm.cache.Remove(key)
+}
Review Comment:
Is there a data race if we don't protect reading/writing `cm.cache` by the lock? Further, if dumping from cache is in process and no lock protects `cm.cache`, it might have some dirty data.
##########
registry/servicediscovery/service_instances_changed_listener_impl.go:
##########
@@ -88,9 +98,14 @@ func (lstn *ServiceInstancesChangedListenerImpl) OnEvent(e observer.Event) error
revisionToInstances[revision] = append(subInstances, instance)
metadataInfo := lstn.revisionToMetadata[revision]
if metadataInfo == nil {
- metadataInfo, err = GetMetadataInfo(instance, revision)
- if err != nil {
- return err
+ if val, ok := metaCache.Get(revision); ok {
+ metadataInfo = val.(*common.MetadataInfo)
+ } else {
+ metadataInfo, err = GetMetadataInfo(instance, revision)
+ if err != nil {
+ return err
+ }
+ metaCache.Set(revision, metadataInfo)
Review Comment:
You update the metadata cache at line 122-124. Is here necessary to update cache here?
##########
registry/servicediscovery/store/cache_manager.go:
##########
@@ -0,0 +1,178 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package store
+
+import (
+ "encoding/gob"
+ "os"
+ "sync"
+ "time"
+)
+
+import (
+ "github.com/dubbogo/gost/log/logger"
+
+ "github.com/hashicorp/golang-lru"
+)
+
+type CacheManager struct {
+ name string // The name of the cache manager
+ cacheFile string // The file path where the cache is stored
+ cacheExpired time.Duration // The duration after which the cache expires
+ stop chan struct{} // Channel used to stop the cache expiration routine
+ cache *lru.Cache // The LRU cache implementation
+ lock sync.Mutex
+}
+
+type Item struct {
+ Key string
+ Value interface{}
+}
+
+// NewCacheManager creates a new CacheManager instance.
+// It initializes the cache manager with the provided parameters and starts a routine for cache expiration.
+func NewCacheManager(name, cacheFile string, cacheExpired time.Duration, maxCacheSize int) (*CacheManager, error) {
+ cm := &CacheManager{
+ name: name,
+ cacheFile: cacheFile,
+ cacheExpired: cacheExpired,
+ stop: make(chan struct{}),
+ }
+ cm.cache, _ = lru.New(maxCacheSize)
+
+ // Check if the cache file exists and load the cache if it does
+ if _, err := os.Stat(cacheFile); err == nil {
+ if err = cm.loadCache(); err != nil {
+ logger.Warnf("Failed to load the cache file:[%s].The err is %v", cm.cacheFile, err)
+ }
+ }
+ go cm.RunDumpTask()
+
+ return cm, nil
+}
+
+// Get retrieves the value associated with the given key from the cache.
+func (cm *CacheManager) Get(key string) (interface{}, bool) {
+ return cm.cache.Get(key)
+}
+
+// Set sets the value associated with the given key in the cache.
+func (cm *CacheManager) Set(key string, value interface{}) {
+ cm.cache.Add(key, value)
+}
+
+// Delete removes the value associated with the given key from the cache.
+func (cm *CacheManager) Delete(key string) {
+ cm.cache.Remove(key)
+}
+
+// GetAll returns all the key-value pairs in the cache.
+func (cm *CacheManager) GetAll() map[string]interface{} {
+ keys := cm.cache.Keys()
+
+ result := make(map[string]interface{})
+ for _, k := range keys {
+ result[k.(string)], _ = cm.cache.Get(k)
+ }
+
+ return result
+}
+
+// loadCache loads the cache from the cache file.
+func (cm *CacheManager) loadCache() error {
+ cf, err := os.Open(cm.cacheFile)
+ if err != nil {
+ return err
+ }
+
+ decoder := gob.NewDecoder(cf)
+ for {
+ var it Item
+ err = decoder.Decode(&it)
+ if err != nil {
+ if err.Error() == "EOF" {
+ break // Reached end of file
+ }
+ return err
+ }
+ // Add the loaded keys to the front of the LRU list
+ cm.cache.Add(it.Key, it.Value)
+ }
+
+ return cf.Close()
+}
+
+// dumpCache dumps the cache to the cache file.
+func (cm *CacheManager) dumpCache() error {
+
+ cm.lock.Lock()
+ defer cm.lock.Unlock()
+
+ items := cm.GetAll()
+
+ file, err := os.Create(cm.cacheFile)
+ if err != nil {
+ return err
+ }
+
+ encoder := gob.NewEncoder(file)
+ for k, v := range items {
+ gob.Register(v)
+ err = encoder.Encode(&Item{
+ Key: k,
+ Value: v,
+ })
+ if err != nil {
+ return err
+ }
+ }
+ return file.Close()
+
+}
+
+func (cm *CacheManager) RunDumpTask() {
+ ticker := time.NewTicker(cm.cacheExpired)
+ for {
+ select {
+ case <-ticker.C:
+ // Dump the cache to the file
+ if err := cm.dumpCache(); err != nil {
+ // Handle error
+ logger.Warnf("Failed to dump cache,the err is %v", err)
+ } else {
+ logger.Infof("Dumping [%s] caches, latest entries %d", cm.name, cm.cache.Len())
+ }
+ case <-cm.stop:
+ ticker.Stop()
+ return
+ }
+ }
+}
Review Comment:
I am wondering why we need to write the cache to file. Is it worth saving outdated cache here?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@dubbo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: notifications-unsubscribe@dubbo.apache.org
For additional commands, e-mail: notifications-help@dubbo.apache.org
[GitHub] [dubbo-go] justxuewei commented on a diff in pull request #2371: Implement meta cache
Posted by "justxuewei (via GitHub)" <gi...@apache.org>.
justxuewei commented on code in PR #2371:
URL: https://github.com/apache/dubbo-go/pull/2371#discussion_r1289454428
##########
registry/servicediscovery/service_instances_changed_listener_impl.go:
##########
@@ -104,6 +122,10 @@ func (lstn *ServiceInstancesChangedListenerImpl) OnEvent(e observer.Event) error
newRevisionToMetadata[revision] = metadataInfo
}
lstn.revisionToMetadata = newRevisionToMetadata
+ for revision, metadataInfo := range newRevisionToMetadata {
+ metaCache.Set(revision, metadataInfo)
+ }
+ metaCache.DumpCache()
Review Comment:
I see the `DumpCache()` is called every time metadata is updated in your lastest patch. I'm worried about performance if the matadata is update frequently. How much persisting metadata improves performance? I am thinking it not costs too much resources to pull the metadata from remotes once the matadata is lost. Wdyt? /cc @chickenlj
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@dubbo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: notifications-unsubscribe@dubbo.apache.org
For additional commands, e-mail: notifications-help@dubbo.apache.org
[GitHub] [dubbo-go] sonarcloud[bot] commented on pull request #2371: Implement meta cache
Posted by "sonarcloud[bot] (via GitHub)" <gi...@apache.org>.
sonarcloud[bot] commented on PR #2371:
URL: https://github.com/apache/dubbo-go/pull/2371#issuecomment-1666784669
Kudos, SonarCloud Quality Gate passed! [](https://sonarcloud.io/dashboard?id=apache_dubbo-go&pullRequest=2371)
[](https://sonarcloud.io/project/issues?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=BUG) [](https://sonarcloud.io/project/issues?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=BUG) [0 Bugs](https://sonarcloud.io/project/issues?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=BUG)
[](https://sonarcloud.io/project/issues?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=VULNERABILITY) [](https://sonarcloud.io/project/issues?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=VULNERABILITY) [0 Vulnerabilities](https://sonarcloud.io/project/issues?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=VULNERABILITY)
[](https://sonarcloud.io/project/security_hotspots?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=SECURITY_HOTSPOT) [](https://sonarcloud.io/project/security_hotspots?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=SECURITY_HOTSPOT) [0 Security Hotspots](https://sonarcloud.io/project/security_hotspots?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=SECURITY_HOTSPOT)
[](https://sonarcloud.io/project/issues?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=CODE_SMELL) [](https://sonarcloud.io/project/issues?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=CODE_SMELL) [2 Code Smells](https://sonarcloud.io/project/issues?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=CODE_SMELL)
[](https://sonarcloud.io/component_measures?id=apache_dubbo-go&pullRequest=2371) No Coverage information
[](https://sonarcloud.io/component_measures?id=apache_dubbo-go&pullRequest=2371&metric=new_duplicated_lines_density&view=list) [9.5% Duplication](https://sonarcloud.io/component_measures?id=apache_dubbo-go&pullRequest=2371&metric=new_duplicated_lines_density&view=list)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@dubbo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: notifications-unsubscribe@dubbo.apache.org
For additional commands, e-mail: notifications-help@dubbo.apache.org
[GitHub] [dubbo-go] FinalT commented on pull request #2371: Implement meta cache
Posted by "FinalT (via GitHub)" <gi...@apache.org>.
FinalT commented on PR #2371:
URL: https://github.com/apache/dubbo-go/pull/2371#issuecomment-1672435704
@justxuewei please rerun ci
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@dubbo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: notifications-unsubscribe@dubbo.apache.org
For additional commands, e-mail: notifications-help@dubbo.apache.org
[GitHub] [dubbo-go] Lvnszn commented on a diff in pull request #2371: Implement meta cache
Posted by "Lvnszn (via GitHub)" <gi...@apache.org>.
Lvnszn commented on code in PR #2371:
URL: https://github.com/apache/dubbo-go/pull/2371#discussion_r1288707631
##########
registry/servicediscovery/store/cache_manager.go:
##########
@@ -0,0 +1,178 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package store
+
+import (
+ "encoding/gob"
+ "os"
+ "sync"
+ "time"
+)
+
+import (
+ "github.com/dubbogo/gost/log/logger"
+
+ "github.com/hashicorp/golang-lru"
+)
+
+type CacheManager struct {
+ name string // The name of the cache manager
+ cacheFile string // The file path where the cache is stored
+ cacheExpired time.Duration // The duration after which the cache expires
+ stop chan struct{} // Channel used to stop the cache expiration routine
+ cache *lru.Cache // The LRU cache implementation
+ lock sync.Mutex
+}
+
+type Item struct {
+ Key string
+ Value interface{}
+}
+
+// NewCacheManager creates a new CacheManager instance.
+// It initializes the cache manager with the provided parameters and starts a routine for cache expiration.
+func NewCacheManager(name, cacheFile string, cacheExpired time.Duration, maxCacheSize int) (*CacheManager, error) {
+ cm := &CacheManager{
+ name: name,
+ cacheFile: cacheFile,
+ cacheExpired: cacheExpired,
+ stop: make(chan struct{}),
+ }
+ cm.cache, _ = lru.New(maxCacheSize)
+
+ // Check if the cache file exists and load the cache if it does
+ if _, err := os.Stat(cacheFile); err == nil {
+ if err = cm.loadCache(); err != nil {
+ logger.Warnf("Failed to load the cache file:[%s].The err is %v", cm.cacheFile, err)
+ }
+ }
+ go cm.RunDumpTask()
+
+ return cm, nil
+}
+
+// Get retrieves the value associated with the given key from the cache.
+func (cm *CacheManager) Get(key string) (interface{}, bool) {
+ return cm.cache.Get(key)
+}
+
+// Set sets the value associated with the given key in the cache.
+func (cm *CacheManager) Set(key string, value interface{}) {
+ cm.cache.Add(key, value)
+}
+
+// Delete removes the value associated with the given key from the cache.
+func (cm *CacheManager) Delete(key string) {
+ cm.cache.Remove(key)
+}
+
+// GetAll returns all the key-value pairs in the cache.
+func (cm *CacheManager) GetAll() map[string]interface{} {
+ keys := cm.cache.Keys()
+
+ result := make(map[string]interface{})
+ for _, k := range keys {
+ result[k.(string)], _ = cm.cache.Get(k)
+ }
+
+ return result
+}
+
+// loadCache loads the cache from the cache file.
+func (cm *CacheManager) loadCache() error {
+ cf, err := os.Open(cm.cacheFile)
+ if err != nil {
+ return err
+ }
+
+ decoder := gob.NewDecoder(cf)
+ for {
+ var it Item
+ err = decoder.Decode(&it)
+ if err != nil {
+ if err.Error() == "EOF" {
+ break // Reached end of file
+ }
+ return err
+ }
+ // Add the loaded keys to the front of the LRU list
+ cm.cache.Add(it.Key, it.Value)
+ }
+
+ return cf.Close()
+}
+
+// dumpCache dumps the cache to the cache file.
+func (cm *CacheManager) dumpCache() error {
+
+ cm.lock.Lock()
+ defer cm.lock.Unlock()
+
+ items := cm.GetAll()
+
+ file, err := os.Create(cm.cacheFile)
+ if err != nil {
+ return err
+ }
+
+ encoder := gob.NewEncoder(file)
+ for k, v := range items {
+ gob.Register(v)
+ err = encoder.Encode(&Item{
+ Key: k,
+ Value: v,
+ })
+ if err != nil {
+ return err
+ }
+ }
+ return file.Close()
+
+}
+
+func (cm *CacheManager) RunDumpTask() {
+ ticker := time.NewTicker(cm.cacheExpired)
+ for {
+ select {
+ case <-ticker.C:
+ // Dump the cache to the file
+ if err := cm.dumpCache(); err != nil {
+ // Handle error
+ logger.Warnf("Failed to dump cache,the err is %v", err)
+ } else {
+ logger.Infof("Dumping [%s] caches, latest entries %d", cm.name, cm.cache.Len())
+ }
+ case <-cm.stop:
+ ticker.Stop()
+ return
+ }
+ }
+}
+
+// destroy stops the cache expiration routine, clears the cache and removes the cache file.
+func (cm *CacheManager) destroy() {
Review Comment:
这块逻辑看着没有地方使用，是否需要？
##########
registry/servicediscovery/store/cache_manager.go:
##########
@@ -0,0 +1,178 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package store
+
+import (
+ "encoding/gob"
+ "os"
+ "sync"
+ "time"
+)
+
+import (
+ "github.com/dubbogo/gost/log/logger"
+
+ "github.com/hashicorp/golang-lru"
+)
+
+type CacheManager struct {
+ name string // The name of the cache manager
+ cacheFile string // The file path where the cache is stored
+ cacheExpired time.Duration // The duration after which the cache expires
+ stop chan struct{} // Channel used to stop the cache expiration routine
+ cache *lru.Cache // The LRU cache implementation
+ lock sync.Mutex
+}
+
+type Item struct {
+ Key string
+ Value interface{}
+}
+
+// NewCacheManager creates a new CacheManager instance.
+// It initializes the cache manager with the provided parameters and starts a routine for cache expiration.
+func NewCacheManager(name, cacheFile string, cacheExpired time.Duration, maxCacheSize int) (*CacheManager, error) {
+ cm := &CacheManager{
+ name: name,
+ cacheFile: cacheFile,
+ cacheExpired: cacheExpired,
+ stop: make(chan struct{}),
+ }
+ cm.cache, _ = lru.New(maxCacheSize)
+
+ // Check if the cache file exists and load the cache if it does
+ if _, err := os.Stat(cacheFile); err == nil {
+ if err = cm.loadCache(); err != nil {
+ logger.Warnf("Failed to load the cache file:[%s].The err is %v", cm.cacheFile, err)
+ }
+ }
+ go cm.RunDumpTask()
+
+ return cm, nil
+}
+
+// Get retrieves the value associated with the given key from the cache.
+func (cm *CacheManager) Get(key string) (interface{}, bool) {
+ return cm.cache.Get(key)
+}
+
+// Set sets the value associated with the given key in the cache.
+func (cm *CacheManager) Set(key string, value interface{}) {
+ cm.cache.Add(key, value)
+}
+
+// Delete removes the value associated with the given key from the cache.
+func (cm *CacheManager) Delete(key string) {
+ cm.cache.Remove(key)
+}
+
+// GetAll returns all the key-value pairs in the cache.
+func (cm *CacheManager) GetAll() map[string]interface{} {
+ keys := cm.cache.Keys()
+
+ result := make(map[string]interface{})
+ for _, k := range keys {
+ result[k.(string)], _ = cm.cache.Get(k)
+ }
+
+ return result
+}
+
+// loadCache loads the cache from the cache file.
+func (cm *CacheManager) loadCache() error {
+ cf, err := os.Open(cm.cacheFile)
+ if err != nil {
+ return err
+ }
+
+ decoder := gob.NewDecoder(cf)
+ for {
+ var it Item
+ err = decoder.Decode(&it)
+ if err != nil {
+ if err.Error() == "EOF" {
+ break // Reached end of file
+ }
+ return err
+ }
+ // Add the loaded keys to the front of the LRU list
+ cm.cache.Add(it.Key, it.Value)
+ }
+
+ return cf.Close()
+}
+
+// dumpCache dumps the cache to the cache file.
+func (cm *CacheManager) dumpCache() error {
+
+ cm.lock.Lock()
+ defer cm.lock.Unlock()
+
+ items := cm.GetAll()
+
+ file, err := os.Create(cm.cacheFile)
+ if err != nil {
+ return err
+ }
+
+ encoder := gob.NewEncoder(file)
+ for k, v := range items {
+ gob.Register(v)
+ err = encoder.Encode(&Item{
+ Key: k,
+ Value: v,
+ })
+ if err != nil {
+ return err
+ }
+ }
+ return file.Close()
+
+}
+
+func (cm *CacheManager) RunDumpTask() {
+ ticker := time.NewTicker(cm.cacheExpired)
+ for {
+ select {
+ case <-ticker.C:
+ // Dump the cache to the file
+ if err := cm.dumpCache(); err != nil {
+ // Handle error
+ logger.Warnf("Failed to dump cache,the err is %v", err)
+ } else {
+ logger.Infof("Dumping [%s] caches, latest entries %d", cm.name, cm.cache.Len())
+ }
+ case <-cm.stop:
+ ticker.Stop()
+ return
+ }
+ }
+}
+
+// destroy stops the cache expiration routine, clears the cache and removes the cache file.
+func (cm *CacheManager) destroy() {
Review Comment:
这块逻辑看着没有地方使用，是否需要？
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@dubbo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: notifications-unsubscribe@dubbo.apache.org
For additional commands, e-mail: notifications-help@dubbo.apache.org
[GitHub] [dubbo-go] justxuewei commented on a diff in pull request #2371: Implement meta cache
Posted by "justxuewei (via GitHub)" <gi...@apache.org>.
justxuewei commented on code in PR #2371:
URL: https://github.com/apache/dubbo-go/pull/2371#discussion_r1288416835
##########
registry/servicediscovery/store/cache_manager.go:
##########
@@ -0,0 +1,178 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package store
+
+import (
+ "encoding/gob"
+ "os"
+ "sync"
+ "time"
+)
+
+import (
+ "github.com/dubbogo/gost/log/logger"
+
+ "github.com/hashicorp/golang-lru"
+)
+
+type CacheManager struct {
+ name string // The name of the cache manager
+ cacheFile string // The file path where the cache is stored
+ cacheExpired time.Duration // The duration after which the cache expires
+ stop chan struct{} // Channel used to stop the cache expiration routine
+ cache *lru.Cache // The LRU cache implementation
+ lock sync.Mutex
+}
+
+type Item struct {
+ Key string
+ Value interface{}
+}
+
+// NewCacheManager creates a new CacheManager instance.
+// It initializes the cache manager with the provided parameters and starts a routine for cache expiration.
+func NewCacheManager(name, cacheFile string, cacheExpired time.Duration, maxCacheSize int) (*CacheManager, error) {
+ cm := &CacheManager{
+ name: name,
+ cacheFile: cacheFile,
+ cacheExpired: cacheExpired,
+ stop: make(chan struct{}),
+ }
+ cm.cache, _ = lru.New(maxCacheSize)
+
+ // Check if the cache file exists and load the cache if it does
+ if _, err := os.Stat(cacheFile); err == nil {
+ if err = cm.loadCache(); err != nil {
+ logger.Warnf("Failed to load the cache file:[%s].The err is %v", cm.cacheFile, err)
+ }
+ }
+ go cm.RunDumpTask()
+
+ return cm, nil
+}
+
+// Get retrieves the value associated with the given key from the cache.
+func (cm *CacheManager) Get(key string) (interface{}, bool) {
+ return cm.cache.Get(key)
+}
+
+// Set sets the value associated with the given key in the cache.
+func (cm *CacheManager) Set(key string, value interface{}) {
+ cm.cache.Add(key, value)
+}
+
+// Delete removes the value associated with the given key from the cache.
+func (cm *CacheManager) Delete(key string) {
+ cm.cache.Remove(key)
+}
+
+// GetAll returns all the key-value pairs in the cache.
+func (cm *CacheManager) GetAll() map[string]interface{} {
+ keys := cm.cache.Keys()
+
+ result := make(map[string]interface{})
+ for _, k := range keys {
+ result[k.(string)], _ = cm.cache.Get(k)
+ }
+
+ return result
+}
+
+// loadCache loads the cache from the cache file.
+func (cm *CacheManager) loadCache() error {
+ cf, err := os.Open(cm.cacheFile)
+ if err != nil {
+ return err
+ }
+
+ decoder := gob.NewDecoder(cf)
+ for {
+ var it Item
+ err = decoder.Decode(&it)
+ if err != nil {
+ if err.Error() == "EOF" {
+ break // Reached end of file
+ }
+ return err
+ }
+ // Add the loaded keys to the front of the LRU list
+ cm.cache.Add(it.Key, it.Value)
+ }
+
+ return cf.Close()
+}
+
+// dumpCache dumps the cache to the cache file.
+func (cm *CacheManager) dumpCache() error {
+
+ cm.lock.Lock()
+ defer cm.lock.Unlock()
+
+ items := cm.GetAll()
+
+ file, err := os.Create(cm.cacheFile)
+ if err != nil {
+ return err
+ }
+
+ encoder := gob.NewEncoder(file)
+ for k, v := range items {
+ gob.Register(v)
+ err = encoder.Encode(&Item{
+ Key: k,
+ Value: v,
+ })
+ if err != nil {
+ return err
+ }
+ }
+ return file.Close()
+
+}
+
+func (cm *CacheManager) RunDumpTask() {
+ ticker := time.NewTicker(cm.cacheExpired)
+ for {
+ select {
+ case <-ticker.C:
+ // Dump the cache to the file
+ if err := cm.dumpCache(); err != nil {
+ // Handle error
+ logger.Warnf("Failed to dump cache,the err is %v", err)
+ } else {
+ logger.Infof("Dumping [%s] caches, latest entries %d", cm.name, cm.cache.Len())
+ }
+ case <-cm.stop:
+ ticker.Stop()
+ return
+ }
+ }
+}
Review Comment:
IIUR, you want to save the data to a file over a period of time. Then, `cacheExpired` is a confused name for me. Some names, like `dumpInterval`, are better than the current name.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@dubbo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: notifications-unsubscribe@dubbo.apache.org
For additional commands, e-mail: notifications-help@dubbo.apache.org
[GitHub] [dubbo-go] sonarcloud[bot] commented on pull request #2371: Implement meta cache
Posted by "sonarcloud[bot] (via GitHub)" <gi...@apache.org>.
sonarcloud[bot] commented on PR #2371:
URL: https://github.com/apache/dubbo-go/pull/2371#issuecomment-1666743774
Kudos, SonarCloud Quality Gate passed! [](https://sonarcloud.io/dashboard?id=apache_dubbo-go&pullRequest=2371)
[](https://sonarcloud.io/project/issues?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=BUG) [](https://sonarcloud.io/project/issues?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=BUG) [0 Bugs](https://sonarcloud.io/project/issues?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=BUG)
[](https://sonarcloud.io/project/issues?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=VULNERABILITY) [](https://sonarcloud.io/project/issues?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=VULNERABILITY) [0 Vulnerabilities](https://sonarcloud.io/project/issues?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=VULNERABILITY)
[](https://sonarcloud.io/project/security_hotspots?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=SECURITY_HOTSPOT) [](https://sonarcloud.io/project/security_hotspots?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=SECURITY_HOTSPOT) [0 Security Hotspots](https://sonarcloud.io/project/security_hotspots?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=SECURITY_HOTSPOT)
[](https://sonarcloud.io/project/issues?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=CODE_SMELL) [](https://sonarcloud.io/project/issues?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=CODE_SMELL) [2 Code Smells](https://sonarcloud.io/project/issues?id=apache_dubbo-go&pullRequest=2371&resolved=false&types=CODE_SMELL)
[](https://sonarcloud.io/component_measures?id=apache_dubbo-go&pullRequest=2371) No Coverage information
[](https://sonarcloud.io/component_measures?id=apache_dubbo-go&pullRequest=2371&metric=new_duplicated_lines_density&view=list) [9.2% Duplication](https://sonarcloud.io/component_measures?id=apache_dubbo-go&pullRequest=2371&metric=new_duplicated_lines_density&view=list)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@dubbo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: notifications-unsubscribe@dubbo.apache.org
For additional commands, e-mail: notifications-help@dubbo.apache.org