You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@servicecomb.apache.org by li...@apache.org on 2024/01/08 13:22:58 UTC
(servicecomb-kie) branch dev updated (93bbb89 -> 7bead9e)
This is an automated email from the ASF dual-hosted git repository.
littlecui pushed a change to branch dev
in repository https://gitbox.apache.org/repos/asf/servicecomb-kie.git
from 93bbb89 add value filter in ListKV API (#302)
new c8f01b1 fix the concurrent bug of KvIdCache
new cf90eaf fix the bug of do not report the error which occured in action of get kvdocs from etcd
new 9861c09 fix the bug of do not report the error which occured in action of get kvdocs from etcd
new 7bead9e resolve conflicts in master
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
server/datasource/etcd/kv/kv_cache.go | 62 ++++++++++++++++++++---------------
server/datasource/etcd/kv/kv_dao.go | 7 ++--
2 files changed, 41 insertions(+), 28 deletions(-)
(servicecomb-kie) 02/04: fix the bug of do not report the error which occured in action of get kvdocs from etcd
Posted by li...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
littlecui pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/servicecomb-kie.git
commit cf90eaf67d615ccd083fc1b3dd01baa8c7737458
Author: songshiyuan 00649746 <so...@huawei.com>
AuthorDate: Tue Dec 12 23:57:24 2023 +0800
fix the bug of do not report the error which occured in action of get kvdocs from etcd
---
server/datasource/etcd/kv/kv_cache.go | 17 +++++++++++++----
1 file changed, 13 insertions(+), 4 deletions(-)
diff --git a/server/datasource/etcd/kv/kv_cache.go b/server/datasource/etcd/kv/kv_cache.go
index fe165b8..ba7b346 100644
--- a/server/datasource/etcd/kv/kv_cache.go
+++ b/server/datasource/etcd/kv/kv_cache.go
@@ -248,7 +248,10 @@ func Search(ctx context.Context, req *CacheSearchReq) (*model.KVResponse, bool)
}
return true
})
- tpData := kvCache.getKvFromEtcd(ctx, req, kvIdsLeft)
+ tpData, err := kvCache.getKvFromEtcd(ctx, req, kvIdsLeft)
+ if err != nil {
+ return nil, true, err
+ }
docs = append(docs, tpData...)
for _, doc := range docs {
@@ -261,14 +264,15 @@ func Search(ctx context.Context, req *CacheSearchReq) (*model.KVResponse, bool)
return result, true
}
-func (kc *Cache) getKvFromEtcd(ctx context.Context, req *CacheSearchReq, kvIdsLeft []string) []*model.KVDoc {
+func (kc *Cache) getKvFromEtcd(ctx context.Context, req *CacheSearchReq, kvIdsLeft []string) ([]*model.KVDoc, error) {
if len(kvIdsLeft) == 0 {
- return nil
+ return nil, nil
}
openlog.Debug("get kv from etcd by kvId")
wg := sync.WaitGroup{}
docs := make([]*model.KVDoc, len(kvIdsLeft))
+ var Err error
for i, kvID := range kvIdsLeft {
wg.Add(1)
go func(kvID string, cnt int) {
@@ -278,12 +282,14 @@ func (kc *Cache) getKvFromEtcd(ctx context.Context, req *CacheSearchReq, kvIdsLe
kv, err := etcdadpt.Get(ctx, docKey)
if err != nil {
openlog.Error(fmt.Sprintf("failed to get kv from etcd, err %v", err))
+ Err = err
return
}
doc, err := kc.GetKvDoc(kv)
if err != nil {
openlog.Error(fmt.Sprintf("failed to unmarshal kv, err %v", err))
+ Err = err
return
}
@@ -292,7 +298,10 @@ func (kc *Cache) getKvFromEtcd(ctx context.Context, req *CacheSearchReq, kvIdsLe
}(kvID, i)
}
wg.Wait()
- return docs
+ if Err != nil {
+ return nil, Err
+ }
+ return docs, nil
}
func isMatch(req *CacheSearchReq, doc *model.KVDoc) bool {
(servicecomb-kie) 03/04: fix the bug of do not report the error which occured in action of get kvdocs from etcd
Posted by li...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
littlecui pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/servicecomb-kie.git
commit 9861c093d90cde4389a1bb45f0f0241f1865b68d
Author: songshiyuan 00649746 <so...@huawei.com>
AuthorDate: Wed Dec 13 00:26:48 2023 +0800
fix the bug of do not report the error which occured in action of get kvdocs from etcd
---
server/datasource/etcd/kv/kv_cache.go | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
diff --git a/server/datasource/etcd/kv/kv_cache.go b/server/datasource/etcd/kv/kv_cache.go
index ba7b346..74012cf 100644
--- a/server/datasource/etcd/kv/kv_cache.go
+++ b/server/datasource/etcd/kv/kv_cache.go
@@ -272,7 +272,7 @@ func (kc *Cache) getKvFromEtcd(ctx context.Context, req *CacheSearchReq, kvIdsLe
openlog.Debug("get kv from etcd by kvId")
wg := sync.WaitGroup{}
docs := make([]*model.KVDoc, len(kvIdsLeft))
- var Err error
+ var getKvErr error
for i, kvID := range kvIdsLeft {
wg.Add(1)
go func(kvID string, cnt int) {
@@ -282,14 +282,14 @@ func (kc *Cache) getKvFromEtcd(ctx context.Context, req *CacheSearchReq, kvIdsLe
kv, err := etcdadpt.Get(ctx, docKey)
if err != nil {
openlog.Error(fmt.Sprintf("failed to get kv from etcd, err %v", err))
- Err = err
+ getKvErr = err
return
}
doc, err := kc.GetKvDoc(kv)
if err != nil {
openlog.Error(fmt.Sprintf("failed to unmarshal kv, err %v", err))
- Err = err
+ getKvErr = err
return
}
@@ -298,8 +298,8 @@ func (kc *Cache) getKvFromEtcd(ctx context.Context, req *CacheSearchReq, kvIdsLe
}(kvID, i)
}
wg.Wait()
- if Err != nil {
- return nil, Err
+ if getKvErr != nil {
+ return nil, getKvErr
}
return docs, nil
}
(servicecomb-kie) 01/04: fix the concurrent bug of KvIdCache
Posted by li...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
littlecui pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/servicecomb-kie.git
commit c8f01b12f76e8d1737d42222a1cfa6ee0436ea11
Author: songshiyuan 00649746 <so...@huawei.com>
AuthorDate: Thu Dec 7 21:46:35 2023 +0800
fix the concurrent bug of KvIdCache
---
server/datasource/etcd/kv/kv_cache.go | 41 ++++++++++++++++++-----------------
1 file changed, 21 insertions(+), 20 deletions(-)
diff --git a/server/datasource/etcd/kv/kv_cache.go b/server/datasource/etcd/kv/kv_cache.go
index 61d017b..fe165b8 100644
--- a/server/datasource/etcd/kv/kv_cache.go
+++ b/server/datasource/etcd/kv/kv_cache.go
@@ -9,15 +9,16 @@ import (
"sync"
"time"
- "github.com/apache/servicecomb-kie/pkg/model"
- "github.com/apache/servicecomb-kie/pkg/stringutil"
- "github.com/apache/servicecomb-kie/server/datasource"
- "github.com/apache/servicecomb-kie/server/datasource/etcd/key"
"github.com/go-chassis/foundation/backoff"
"github.com/go-chassis/openlog"
"github.com/little-cui/etcdadpt"
goCache "github.com/patrickmn/go-cache"
"go.etcd.io/etcd/api/v3/mvccpb"
+
+ "github.com/apache/servicecomb-kie/pkg/model"
+ "github.com/apache/servicecomb-kie/pkg/stringutil"
+ "github.com/apache/servicecomb-kie/server/datasource"
+ "github.com/apache/servicecomb-kie/server/datasource/etcd/key"
)
func Init() {
@@ -35,8 +36,6 @@ const (
backOffMinInterval = 5 * time.Second
)
-type IDSet map[string]struct{}
-
type Cache struct {
timeOut time.Duration
client etcdadpt.Client
@@ -158,11 +157,13 @@ func (kc *Cache) cachePut(rsp *etcdadpt.Response) {
cacheKey := kc.GetCacheKey(kvDoc.Domain, kvDoc.Project, kvDoc.Labels)
m, ok := kc.LoadKvIDSet(cacheKey)
if !ok {
- kc.StoreKvIDSet(cacheKey, IDSet{kvDoc.ID: struct{}{}})
+ z := &sync.Map{}
+ z.Store(kvDoc.ID, struct{}{})
+ kc.StoreKvIDSet(cacheKey, z)
openlog.Info("cacheKey " + cacheKey + "not exists")
continue
}
- m[kvDoc.ID] = struct{}{}
+ m.Store(kvDoc.ID, struct{}{})
}
}
@@ -180,23 +181,23 @@ func (kc *Cache) cacheDelete(rsp *etcdadpt.Response) {
openlog.Error("cacheKey " + cacheKey + "not exists")
continue
}
- delete(m, kvDoc.ID)
+ m.Delete(kvDoc.ID)
}
}
-func (kc *Cache) LoadKvIDSet(cacheKey string) (IDSet, bool) {
+func (kc *Cache) LoadKvIDSet(cacheKey string) (*sync.Map, bool) {
val, ok := kc.kvIDCache.Load(cacheKey)
if !ok {
return nil, false
}
- kvIds, ok := val.(IDSet)
+ kvIds, ok := val.(*sync.Map)
if !ok {
return nil, false
}
return kvIds, true
}
-func (kc *Cache) StoreKvIDSet(cacheKey string, kvIds IDSet) {
+func (kc *Cache) StoreKvIDSet(cacheKey string, kvIds *sync.Map) {
kc.kvIDCache.Store(cacheKey, kvIds)
}
@@ -232,21 +233,21 @@ func Search(ctx context.Context, req *CacheSearchReq) (*model.KVResponse, bool)
cacheKey := kvCache.GetCacheKey(req.Domain, req.Project, req.Opts.Labels)
kvIds, ok := kvCache.LoadKvIDSet(cacheKey)
if !ok {
- kvCache.StoreKvIDSet(cacheKey, IDSet{})
- return result, true
+ kvCache.StoreKvIDSet(cacheKey, &sync.Map{})
+ return result, true, nil
}
var docs []*model.KVDoc
var kvIdsLeft []string
- for kvID := range kvIds {
- if doc, ok := kvCache.LoadKvDoc(kvID); ok {
+ kvIds.Range(func(kvID, value any) bool {
+ if doc, ok := kvCache.LoadKvDoc(kvID.(string)); ok {
docs = append(docs, doc)
- continue
+ } else {
+ kvIdsLeft = append(kvIdsLeft, kvID.(string))
}
- kvIdsLeft = append(kvIdsLeft, kvID)
- }
-
+ return true
+ })
tpData := kvCache.getKvFromEtcd(ctx, req, kvIdsLeft)
docs = append(docs, tpData...)
(servicecomb-kie) 04/04: resolve conflicts in master
Posted by li...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
littlecui pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/servicecomb-kie.git
commit 7bead9e662d322f9aba3cabe7d8cdcdfe9953614
Author: tornado-ssy <16...@qq.com>
AuthorDate: Mon Jan 8 21:22:33 2024 +0800
resolve conflicts in master
---
server/datasource/etcd/kv/kv_cache.go | 6 +++---
server/datasource/etcd/kv/kv_dao.go | 7 +++++--
2 files changed, 8 insertions(+), 5 deletions(-)
diff --git a/server/datasource/etcd/kv/kv_cache.go b/server/datasource/etcd/kv/kv_cache.go
index 74012cf..7cf6529 100644
--- a/server/datasource/etcd/kv/kv_cache.go
+++ b/server/datasource/etcd/kv/kv_cache.go
@@ -221,9 +221,9 @@ func (kc *Cache) DeleteKvDoc(kvID string) {
kc.kvDocCache.Delete(kvID)
}
-func Search(ctx context.Context, req *CacheSearchReq) (*model.KVResponse, bool) {
+func Search(ctx context.Context, req *CacheSearchReq) (*model.KVResponse, bool, error) {
if !req.Opts.ExactLabels {
- return nil, false
+ return nil, false, nil
}
openlog.Debug(fmt.Sprintf("using cache to search kv, domain %v, project %v, opts %+v", req.Domain, req.Project, *req.Opts))
@@ -261,7 +261,7 @@ func Search(ctx context.Context, req *CacheSearchReq) (*model.KVResponse, bool)
}
}
result.Total = len(result.Data)
- return result, true
+ return result, true, nil
}
func (kc *Cache) getKvFromEtcd(ctx context.Context, req *CacheSearchReq, kvIdsLeft []string) ([]*model.KVDoc, error) {
diff --git a/server/datasource/etcd/kv/kv_dao.go b/server/datasource/etcd/kv/kv_dao.go
index 2332b58..d7260df 100644
--- a/server/datasource/etcd/kv/kv_dao.go
+++ b/server/datasource/etcd/kv/kv_dao.go
@@ -524,15 +524,18 @@ func (s *Dao) listData(ctx context.Context, project, domain string, options ...d
}
if Enabled() {
- result, useCache := Search(ctx, &CacheSearchReq{
+ result, useCache, err := Search(ctx, &CacheSearchReq{
Domain: domain,
Project: project,
Opts: &opts,
Regex: regex,
})
- if useCache {
+ if useCache && err == nil {
return result, opts, nil
}
+ if useCache && err != nil {
+ openlog.Error("using cache to search kv failed: " + err.Error())
+ }
}
result, err := matchLabelsSearch(ctx, domain, project, regex, opts)