You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@servicecomb.apache.org by li...@apache.org on 2021/04/27 02:39:59 UTC
[servicecomb-service-center] branch page created (now 7b6f99d)
This is an automated email from the ASF dual-hosted git repository.
littlecui pushed a change to branch page
in repository https://gitbox.apache.org/repos/asf/servicecomb-service-center.git.
at 7b6f99d SCB-2176 Fix: raise err 'etcdserver: request too large' when list 'dep-queue'
This branch includes the following new commits:
new 7b6f99d SCB-2176 Fix: raise err 'etcdserver: request too large' when list 'dep-queue'
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[servicecomb-service-center] 01/01: SCB-2176 Fix: raise err
'etcdserver: request too large' when list 'dep-queue'
Posted by li...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
littlecui pushed a commit to branch page
in repository https://gitbox.apache.org/repos/asf/servicecomb-service-center.git
commit 7b6f99db17becbf072d5be15835f00a05e7b0200
Author: little-cui <su...@qq.com>
AuthorDate: Tue Apr 27 10:39:41 2021 +0800
SCB-2176 Fix: raise err 'etcdserver: request too large' when list 'dep-queue'
---
server/core/backend/common.go | 2 +-
server/plugin/discovery/common.go | 2 +
server/plugin/discovery/config.go | 19 ++-
server/plugin/discovery/config_test.go | 3 +-
server/plugin/discovery/etcd/adaptor.go | 2 +-
server/plugin/discovery/etcd/cacher_kv.go | 2 +-
server/plugin/discovery/etcd/indexer_cache.go | 2 +-
server/plugin/discovery/etcd/indexer_etcd.go | 21 +--
server/plugin/discovery/etcd/indexer_test.go | 16 ++-
server/plugin/discovery/etcd/listwatch.go | 10 +-
server/plugin/discovery/etcd/listwatch_inner.go | 10 +-
server/plugin/discovery/etcd/listwatch_test.go | 19 +--
server/plugin/discovery/etcd/watcher_test.go | 11 +-
server/plugin/registry/etcd/etcd.go | 2 +-
server/plugin/registry/etcd/etcd_test.go | 12 +-
server/plugin/registry/option.go | 8 +-
server/service/auth/datasource.go | 35 -----
server/service/auth/etcd/etcd.go | 163 ------------------------
server/service/auth/etcd/etcd_test.go | 121 ------------------
server/service/auth/etcd/key_generator.go | 78 ------------
server/service/auth/etcd/key_generator_test.go | 37 ------
server/service/auth/manager.go | 58 ---------
server/service/auth/manager_test.go | 48 -------
server/service/auth/options.go | 22 ----
server/service/auth/types.go | 18 ---
25 files changed, 87 insertions(+), 634 deletions(-)
diff --git a/server/core/backend/common.go b/server/core/backend/common.go
index 53fa826..60009b3 100644
--- a/server/core/backend/common.go
+++ b/server/core/backend/common.go
@@ -90,7 +90,7 @@ func registerInnerTypes() {
WithInitSize(100).WithParser(proto.DependencyRuleParser)))
DependencyQueue = Store().MustInstall(NewAddOn("DEPENDENCY_QUEUE",
discovery.Configure().WithPrefix(core.GetServiceDependencyQueueRootKey("")).
- WithInitSize(100).WithParser(proto.DependencyQueueParser)))
+ WithInitSize(100).WithPageSize(1500).WithParser(proto.DependencyQueueParser)))
PROJECT = Store().MustInstall(NewAddOn("PROJECT",
discovery.Configure().WithPrefix(core.GetProjectRootKey("")).
WithInitSize(100).WithParser(proto.StringParser)))
diff --git a/server/plugin/discovery/common.go b/server/plugin/discovery/common.go
index 39232aa..85ed1e6 100644
--- a/server/plugin/discovery/common.go
+++ b/server/plugin/discovery/common.go
@@ -23,4 +23,6 @@ import (
const (
DefaultTimeout = 30 * time.Second
DefaultCacheInitSize = 100
+ // grpc does not allow to transport a large body more then 4MB in a request
+ DefaultPageSize = 4096
)
diff --git a/server/plugin/discovery/config.go b/server/plugin/discovery/config.go
index d0e53f0..ba488e7 100644
--- a/server/plugin/discovery/config.go
+++ b/server/plugin/discovery/config.go
@@ -24,8 +24,11 @@ import (
type Config struct {
// Key is the prefix to unique specify resource type
- Key string
- InitSize int
+ Key string
+ // InitSize the init cache size, disable cache when InitSize = 0
+ InitSize int
+ // PageSize try to batch query backend to get all KVs
+ PageSize int64
Timeout time.Duration
Period time.Duration
DeferHandler DeferHandler
@@ -35,8 +38,8 @@ type Config struct {
}
func (cfg *Config) String() string {
- return fmt.Sprintf("{key: %s, timeout: %s, period: %s, lease: %v}",
- cfg.Key, cfg.Timeout, cfg.Period, cfg.Lease)
+ return fmt.Sprintf("{key: %s, timeout: %s, period: %s, lease: %v, initSize: %v, pageSize: %v}",
+ cfg.Key, cfg.Timeout, cfg.Period, cfg.Lease, cfg.InitSize, cfg.PageSize)
}
func (cfg *Config) WithPrefix(key string) *Config {
@@ -49,6 +52,11 @@ func (cfg *Config) WithInitSize(size int) *Config {
return cfg
}
+func (cfg *Config) WithPageSize(size int64) *Config {
+ cfg.PageSize = size
+ return cfg
+}
+
func (cfg *Config) WithTimeout(ot time.Duration) *Config {
cfg.Timeout = ot
return cfg
@@ -94,9 +102,10 @@ func (cfg *Config) WithLease() *Config {
func Configure() *Config {
return &Config{
Key: "/",
+ InitSize: DefaultCacheInitSize,
+ PageSize: DefaultPageSize,
Timeout: DefaultTimeout,
Period: time.Second,
- InitSize: DefaultCacheInitSize,
Parser: proto.BytesParser,
}
}
diff --git a/server/plugin/discovery/config_test.go b/server/plugin/discovery/config_test.go
index 4a39dc3..83f2caa 100644
--- a/server/plugin/discovery/config_test.go
+++ b/server/plugin/discovery/config_test.go
@@ -17,6 +17,7 @@ package discovery
import (
"github.com/apache/servicecomb-service-center/server/core/proto"
+ "strings"
"testing"
"time"
)
@@ -74,7 +75,7 @@ func TestConfigure(t *testing.T) {
if cfg.Parser != proto.MapParser {
t.Fatalf("TestConfigure failed")
}
- if cfg.String() != "{key: /test, timeout: 2s, period: 3s, lease: false}" {
+ if !strings.HasPrefix(cfg.String(), "{key:") {
t.Fatalf("TestConfigure failed")
}
}
diff --git a/server/plugin/discovery/etcd/adaptor.go b/server/plugin/discovery/etcd/adaptor.go
index fcc5302..6933c1f 100644
--- a/server/plugin/discovery/etcd/adaptor.go
+++ b/server/plugin/discovery/etcd/adaptor.go
@@ -60,7 +60,7 @@ func NewEtcdAdaptor(name string, cfg *discovery.Config) *Adaptor {
"core will not cache '%s' and ignore all events of it, cache enabled: %v, init size: %d",
name, core.ServerInfo.Config.EnableCache, cfg.InitSize)
adaptor.Cacher = discovery.NullCacher
- adaptor.Indexer = NewEtcdIndexer(cfg.Key, cfg.Parser)
+ adaptor.Indexer = NewEtcdIndexer(cfg)
}
return &adaptor
}
diff --git a/server/plugin/discovery/etcd/cacher_kv.go b/server/plugin/discovery/etcd/cacher_kv.go
index a61d1e3..af1dda8 100644
--- a/server/plugin/discovery/etcd/cacher_kv.go
+++ b/server/plugin/discovery/etcd/cacher_kv.go
@@ -129,7 +129,7 @@ func (c *KvCacher) ListAndWatch(ctx context.Context) error {
defer log.Recover() // ensure ListAndWatch never raise panic
cfg := ListWatchConfig{
- Timeout: c.Cfg.Timeout,
+ Config: c.Cfg,
Context: ctx,
}
diff --git a/server/plugin/discovery/etcd/indexer_cache.go b/server/plugin/discovery/etcd/indexer_cache.go
index 4afa847..8f8242d 100644
--- a/server/plugin/discovery/etcd/indexer_cache.go
+++ b/server/plugin/discovery/etcd/indexer_cache.go
@@ -64,7 +64,7 @@ func (i *CacheIndexer) Creditable() bool {
func NewCacheIndexer(cfg *discovery.Config, cache discovery.Cache) *CacheIndexer {
return &CacheIndexer{
- Indexer: NewEtcdIndexer(cfg.Key, cfg.Parser),
+ Indexer: NewEtcdIndexer(cfg),
CacheIndexer: discovery.NewCacheIndexer(cache),
}
}
diff --git a/server/plugin/discovery/etcd/indexer_etcd.go b/server/plugin/discovery/etcd/indexer_etcd.go
index 3f5c8ff..aaaa6b1 100644
--- a/server/plugin/discovery/etcd/indexer_etcd.go
+++ b/server/plugin/discovery/etcd/indexer_etcd.go
@@ -18,7 +18,6 @@ package etcd
import (
"fmt"
- "github.com/apache/servicecomb-service-center/server/core/proto"
"strings"
"github.com/apache/servicecomb-service-center/pkg/log"
@@ -33,20 +32,19 @@ import (
// Indexer implements discovery.Indexer.
// Indexer searches data from etcd server.
type Indexer struct {
+ Cfg *discovery.Config
Client registry.Registry
- Parser proto.Parser
- Root string
}
func (i *Indexer) CheckPrefix(key string) error {
- if strings.Index(key, i.Root) != 0 {
- return fmt.Errorf("search '%s' mismatch pattern %s", key, i.Root)
+ if strings.Index(key, i.Cfg.Key) != 0 {
+ return fmt.Errorf("search '%s' mismatch pattern %s", key, i.Cfg.Key)
}
return nil
}
func (i *Indexer) Search(ctx context.Context, opts ...registry.PluginOpOption) (r *discovery.Response, err error) {
- op := registry.OpGet(opts...)
+ op := i.toGetOpts(opts)
key := util.BytesToStringWithNoCopy(op.Key)
log.Debugf("search '%s' match special options, request etcd server, opts: %s", key, op)
@@ -66,7 +64,7 @@ func (i *Indexer) Search(ctx context.Context, opts ...registry.PluginOpOption) (
return
}
- p := i.Parser
+ p := i.Cfg.Parser
if op.KeyOnly {
p = nil
}
@@ -83,11 +81,16 @@ func (i *Indexer) Search(ctx context.Context, opts ...registry.PluginOpOption) (
return
}
+func (i *Indexer) toGetOpts(opts []registry.PluginOpOption) registry.PluginOp {
+ op := registry.OpGet(append([]registry.PluginOpOption{registry.WithLimit(i.Cfg.PageSize)}, opts...)...)
+ return op
+}
+
// Creditable implements discovery.Indexer.Creditable.
func (i *Indexer) Creditable() bool {
return true
}
-func NewEtcdIndexer(root string, p proto.Parser) (indexer *Indexer) {
- return &Indexer{Client: backend.Registry(), Parser: p, Root: root}
+func NewEtcdIndexer(cfg *discovery.Config) (indexer *Indexer) {
+ return &Indexer{Cfg: cfg, Client: backend.Registry()}
}
diff --git a/server/plugin/discovery/etcd/indexer_test.go b/server/plugin/discovery/etcd/indexer_test.go
index 57cd156..fc79ea3 100644
--- a/server/plugin/discovery/etcd/indexer_test.go
+++ b/server/plugin/discovery/etcd/indexer_test.go
@@ -28,7 +28,10 @@ import (
func TestEtcdIndexer_Search(t *testing.T) {
data := ®istry.PluginResponse{Revision: 1}
c := &mockRegistry{}
- i := &Indexer{Client: c, Root: "/", Parser: proto.BytesParser}
+ cfg := &discovery.Config{
+ Key: "/", Parser: proto.BytesParser,
+ }
+ i := &Indexer{Cfg: cfg, Client: c}
// case: key does not contain prefix
resp, err := i.Search(context.Background(), registry.WithStrKey("a"))
@@ -52,14 +55,14 @@ func TestEtcdIndexer_Search(t *testing.T) {
// case: parse error
data.Count = 2
data.Kvs = []*mvccpb.KeyValue{{Key: []byte("/a/b"), Value: []byte("abc")}, {Key: []byte("/a/c"), Value: []byte("{}")}}
- old := i.Parser
- i.Parser = proto.MapParser
+ old := i.Cfg.Parser
+ i.Cfg.Parser = proto.MapParser
c.Response = data
resp, err = i.Search(context.Background(), registry.WithStrKey("/a"))
if err != nil || resp == nil || resp.Count != 2 || len(resp.Kvs) != 1 {
t.Fatalf("TestEtcdIndexer_Search failed, %v, %v", err, resp)
}
- i.Parser = old
+ i.Cfg.Parser = old
// case: normal
data.Count = 2
@@ -81,6 +84,9 @@ func TestEtcdIndexer_Search(t *testing.T) {
func TestCacheIndexer_Search(t *testing.T) {
c := &mockCache{}
+ cfg := &discovery.Config{
+ Key: "/",
+ }
cli := &mockRegistry{Response: ®istry.PluginResponse{
Revision: 1,
Kvs: []*mvccpb.KeyValue{{Key: []byte("/a/b"), Value: []byte("abc")}},
@@ -88,7 +94,7 @@ func TestCacheIndexer_Search(t *testing.T) {
}}
i := &CacheIndexer{
Indexer: &Indexer{
- Root: "/",
+ Cfg: cfg,
Client: cli,
},
CacheIndexer: discovery.NewCacheIndexer(c),
diff --git a/server/plugin/discovery/etcd/listwatch.go b/server/plugin/discovery/etcd/listwatch.go
index b7f7470..818d040 100644
--- a/server/plugin/discovery/etcd/listwatch.go
+++ b/server/plugin/discovery/etcd/listwatch.go
@@ -19,23 +19,23 @@ package etcd
import (
"context"
"fmt"
+ "github.com/apache/servicecomb-service-center/server/plugin/discovery"
"github.com/apache/servicecomb-service-center/server/plugin/registry"
- "time"
)
type ListWatchConfig struct {
- Timeout time.Duration
+ *discovery.Config
Context context.Context
}
func (lo *ListWatchConfig) String() string {
- return fmt.Sprintf("{timeout: %s}", lo.Timeout)
+ return fmt.Sprintf("{Config: %v, timeout: %s}", lo.Config, lo.Timeout)
}
type ListWatch interface {
- List(op ListWatchConfig) (*registry.PluginResponse, error)
+ List(cfg ListWatchConfig) (*registry.PluginResponse, error)
// not support new multiple watchers
- Watch(op ListWatchConfig) Watcher
+ Watch(cfg ListWatchConfig) Watcher
//
DoWatch(ctx context.Context, f func(*registry.PluginResponse)) error
Revision() int64
diff --git a/server/plugin/discovery/etcd/listwatch_inner.go b/server/plugin/discovery/etcd/listwatch_inner.go
index e7aa7f8..3853249 100644
--- a/server/plugin/discovery/etcd/listwatch_inner.go
+++ b/server/plugin/discovery/etcd/listwatch_inner.go
@@ -30,10 +30,10 @@ type innerListWatch struct {
rev int64
}
-func (lw *innerListWatch) List(op ListWatchConfig) (*registry.PluginResponse, error) {
- otCtx, cancel := context.WithTimeout(op.Context, op.Timeout)
+func (lw *innerListWatch) List(cfg ListWatchConfig) (*registry.PluginResponse, error) {
+ otCtx, cancel := context.WithTimeout(cfg.Context, cfg.Timeout)
defer cancel()
- resp, err := lw.Client.Do(otCtx, registry.WatchPrefixOpOptions(lw.Prefix)...)
+ resp, err := lw.Client.Do(otCtx, lw.toGetOpts(cfg)...)
if err != nil {
log.Errorf(err, "list prefix %s failed, current rev: %d", lw.Prefix, lw.Revision())
return nil, err
@@ -42,6 +42,10 @@ func (lw *innerListWatch) List(op ListWatchConfig) (*registry.PluginResponse, er
return resp, nil
}
+func (lw *innerListWatch) toGetOpts(cfg ListWatchConfig) []registry.PluginOpOption {
+ return append([]registry.PluginOpOption{registry.WithLimit(cfg.PageSize)}, registry.WatchPrefixOpOptions(lw.Prefix)...)
+}
+
func (lw *innerListWatch) Revision() int64 {
return lw.rev
}
diff --git a/server/plugin/discovery/etcd/listwatch_test.go b/server/plugin/discovery/etcd/listwatch_test.go
index c1e1b83..099ff59 100644
--- a/server/plugin/discovery/etcd/listwatch_test.go
+++ b/server/plugin/discovery/etcd/listwatch_test.go
@@ -19,9 +19,11 @@ package etcd
import (
"context"
"fmt"
+ "github.com/apache/servicecomb-service-center/server/plugin/discovery"
"github.com/apache/servicecomb-service-center/server/plugin/registry"
"github.com/apache/servicecomb-service-center/server/plugin/registry/buildin"
"github.com/coreos/etcd/mvcc/mvccpb"
+ "strings"
"testing"
"time"
)
@@ -61,11 +63,12 @@ func TestPrefixListWatch(t *testing.T) {
Prefix: "a",
rev: 1,
}
- resp, err := lw.List(ListWatchConfig{Timeout: time.Second, Context: context.Background()})
+ cfg := &discovery.Config{Timeout: time.Second}
+ resp, err := lw.List(ListWatchConfig{Config: cfg, Context: context.Background()})
if resp != nil || err == nil || lw.Revision() != 1 {
t.Fatalf("TestPrefixListWatch failed")
}
- w := lw.Watch(ListWatchConfig{Timeout: time.Second, Context: context.Background()})
+ w := lw.Watch(ListWatchConfig{Config: cfg, Context: context.Background()})
resp = <-w.EventBus()
if resp != nil || lw.Revision() != 0 {
t.Fatalf("TestPrefixListWatch failed")
@@ -80,11 +83,11 @@ func TestPrefixListWatch(t *testing.T) {
Prefix: "a",
rev: 1,
}
- resp, err = lw.List(ListWatchConfig{Timeout: time.Second, Context: context.Background()})
+ resp, err = lw.List(ListWatchConfig{Config: cfg, Context: context.Background()})
if resp == nil || err != nil || lw.Revision() != 2 {
t.Fatalf("TestPrefixListWatch failed")
}
- w = lw.Watch(ListWatchConfig{Timeout: time.Second, Context: context.Background()})
+ w = lw.Watch(ListWatchConfig{Config: cfg, Context: context.Background()})
resp = <-w.EventBus()
if resp != nil || lw.Revision() != 0 {
t.Fatalf("TestPrefixListWatch failed")
@@ -100,11 +103,11 @@ func TestPrefixListWatch(t *testing.T) {
Prefix: "a",
rev: 1,
}
- resp, err = lw.List(ListWatchConfig{Timeout: time.Second, Context: context.Background()})
+ resp, err = lw.List(ListWatchConfig{Config: cfg, Context: context.Background()})
if resp == nil || err != nil || lw.Revision() != 4 {
t.Fatalf("TestPrefixListWatch failed")
}
- w = lw.Watch(ListWatchConfig{Timeout: time.Second, Context: context.Background()})
+ w = lw.Watch(ListWatchConfig{Config: cfg, Context: context.Background()})
resp = <-w.EventBus()
if resp == nil || lw.Revision() != 3 {
t.Fatalf("TestPrefixListWatch failed")
@@ -113,8 +116,8 @@ func TestPrefixListWatch(t *testing.T) {
}
func TestListWatchConfig_String(t *testing.T) {
- lw := ListWatchConfig{Timeout: time.Second, Context: context.Background()}
- if lw.String() != "{timeout: 1s}" {
+ lw := ListWatchConfig{Config: &discovery.Config{Timeout: time.Second}, Context: context.Background()}
+ if strings.Contains(lw.String(), "timeout: 1s") {
t.Fatalf("TestListWatchConfig_String failed")
}
}
diff --git a/server/plugin/discovery/etcd/watcher_test.go b/server/plugin/discovery/etcd/watcher_test.go
index 2576193..9161738 100644
--- a/server/plugin/discovery/etcd/watcher_test.go
+++ b/server/plugin/discovery/etcd/watcher_test.go
@@ -19,6 +19,7 @@ package etcd
import (
"context"
"fmt"
+ "github.com/apache/servicecomb-service-center/server/plugin/discovery"
"github.com/apache/servicecomb-service-center/server/plugin/registry"
"testing"
"time"
@@ -41,7 +42,7 @@ type mockListWatch struct {
Rev int64
}
-func (lw *mockListWatch) List(op ListWatchConfig) (*registry.PluginResponse, error) {
+func (lw *mockListWatch) List(_ ListWatchConfig) (*registry.PluginResponse, error) {
if lw.ListResponse == nil {
return nil, fmt.Errorf("error")
}
@@ -59,7 +60,7 @@ func (lw *mockListWatch) DoWatch(ctx context.Context, f func(*registry.PluginRes
<-ctx.Done()
return nil
}
-func (lw *mockListWatch) Watch(op ListWatchConfig) Watcher {
+func (lw *mockListWatch) Watch(_ ListWatchConfig) Watcher {
return lw.Watcher
}
func (lw *mockListWatch) Revision() int64 {
@@ -67,7 +68,8 @@ func (lw *mockListWatch) Revision() int64 {
}
func TestInnerWatcher_EventBus(t *testing.T) {
- w := newInnerWatcher(&mockListWatch{}, ListWatchConfig{Timeout: time.Second, Context: context.Background()})
+ w := newInnerWatcher(&mockListWatch{}, ListWatchConfig{
+ Config: &discovery.Config{Timeout: time.Second}, Context: context.Background()})
resp := <-w.EventBus()
if resp != nil {
t.Fatalf("TestInnerWatcher_EventBus failed")
@@ -77,7 +79,8 @@ func TestInnerWatcher_EventBus(t *testing.T) {
test := ®istry.PluginResponse{
Action: registry.Put,
}
- w = newInnerWatcher(&mockListWatch{ListResponse: test}, ListWatchConfig{Timeout: time.Second, Context: context.Background()})
+ w = newInnerWatcher(&mockListWatch{ListResponse: test}, ListWatchConfig{
+ Config: &discovery.Config{Timeout: time.Second}, Context: context.Background()})
resp = <-w.EventBus()
if resp == nil || resp.Action != registry.Put {
t.Fatalf("TestInnerWatcher_EventBus failed")
diff --git a/server/plugin/registry/etcd/etcd.go b/server/plugin/registry/etcd/etcd.go
index 159204f..6ba4305 100644
--- a/server/plugin/registry/etcd/etcd.go
+++ b/server/plugin/registry/etcd/etcd.go
@@ -420,7 +420,7 @@ func (c *Client) paging(ctx context.Context, op registry.PluginOp) (*clientv3.Ge
limit++
start = 1
}
- ops := append(baseOps, clientv3.WithLimit(int64(limit)))
+ ops := append(baseOps, clientv3.WithLimit(limit))
recordResp, err := c.Client.Get(ctx, nextKey, ops...)
if err != nil {
return nil, err
diff --git a/server/plugin/registry/etcd/etcd_test.go b/server/plugin/registry/etcd/etcd_test.go
index bfdee00..bdc4393 100644
--- a/server/plugin/registry/etcd/etcd_test.go
+++ b/server/plugin/registry/etcd/etcd_test.go
@@ -357,7 +357,7 @@ func TestEtcdClient(t *testing.T) {
// large data
var wg sync.WaitGroup
- for i := 0; i < registry.DefaultPageCount+1; i++ {
+ for i := 0; i < registry.DefaultPageSize+1; i++ {
wg.Add(1)
v := strconv.Itoa(i)
go func() {
@@ -373,14 +373,14 @@ func TestEtcdClient(t *testing.T) {
resp, err = etcdc.Do(context.Background(), registry.GET,
registry.WithStrKey("/test_page/"),
registry.WithStrEndKey("/test_page/9999"))
- if err != nil || !resp.Succeeded || resp.Count != registry.DefaultPageCount+1 ||
- len(resp.Kvs) != registry.DefaultPageCount+1 {
+ if err != nil || !resp.Succeeded || resp.Count != registry.DefaultPageSize+1 ||
+ len(resp.Kvs) != registry.DefaultPageSize+1 {
t.Fatalf("TestEtcdClient_Do failed, %#v", err)
}
resp, err = etcdc.Do(context.Background(), registry.GET,
registry.WithStrKey("/test_page/"), registry.WithPrefix(), registry.WithDescendOrder())
- if err != nil || !resp.Succeeded || resp.Count != registry.DefaultPageCount+1 ||
- len(resp.Kvs) != registry.DefaultPageCount+1 ||
+ if err != nil || !resp.Succeeded || resp.Count != registry.DefaultPageSize+1 ||
+ len(resp.Kvs) != registry.DefaultPageSize+1 ||
string(resp.Kvs[0].Key) != "/test_page/999" {
t.Fatalf("TestEtcdClient_Do failed, %#v", err)
}
@@ -826,7 +826,7 @@ func TestEtcdClient_paging(t *testing.T) {
op := registry.PluginOp{
Offset: -1,
- Limit: registry.DefaultPageCount,
+ Limit: registry.DefaultPageSize,
}
r, err := c.paging(context2.Background(), op)
if err != nil {
diff --git a/server/plugin/registry/option.go b/server/plugin/registry/option.go
index f137a1e..bbc6869 100644
--- a/server/plugin/registry/option.go
+++ b/server/plugin/registry/option.go
@@ -35,9 +35,11 @@ type PluginOp struct {
IgnoreLease bool
Mode CacheMode
WatchCallback WatchCallback
- Offset int64
- Limit int64
- Global bool
+ // Offset >= 0 will paging the results, -1 will return all results.
+ Offset int64
+ // Limit is page size. not work if Offset = -1
+ Limit int64
+ Global bool
}
func (op PluginOp) String() string {
diff --git a/server/service/auth/datasource.go b/server/service/auth/datasource.go
deleted file mode 100644
index 3f081ed..0000000
--- a/server/service/auth/datasource.go
+++ /dev/null
@@ -1,35 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more
-// contributor license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright ownership.
-// The ASF licenses this file to You under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance with
-// the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package auth
-
-import (
- "context"
- "github.com/apache/servicecomb-service-center/pkg/rbacframe"
-)
-
-type DataSource interface {
- AccountExist(ctx context.Context, key string) (bool, error)
- GetAccount(ctx context.Context, key string) (*rbacframe.Account, error)
- ListAccount(ctx context.Context, key string) ([]*rbacframe.Account, int64, error)
- DeleteAccount(ctx context.Context, key string) (bool, error)
- UpdateAccount(ctx context.Context, key string, account *rbacframe.Account) error
-
- AddDomain(ctx context.Context, domain string) (bool, error)
- DomainExist(ctx context.Context, domain string) (bool, error)
-
- AddProject(ctx context.Context, domain, project string) (bool, error)
- ProjectExist(ctx context.Context, domain, project string) (bool, error)
-}
diff --git a/server/service/auth/etcd/etcd.go b/server/service/auth/etcd/etcd.go
deleted file mode 100644
index 89d3420..0000000
--- a/server/service/auth/etcd/etcd.go
+++ /dev/null
@@ -1,163 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more
-// contributor license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright ownership.
-// The ASF licenses this file to You under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance with
-// the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package etcd
-
-import (
- "context"
- "encoding/json"
- "errors"
- "github.com/apache/servicecomb-service-center/pkg/log"
- "github.com/apache/servicecomb-service-center/pkg/rbacframe"
- "github.com/apache/servicecomb-service-center/server/core/backend"
- "github.com/apache/servicecomb-service-center/server/plugin/registry"
- utils "github.com/apache/servicecomb-service-center/server/service/util"
-)
-
-// TODO: define error with names here
-
-var ErrNotUnique = errors.New("kv result is not unique")
-
-func init() {
- // TODO: set logger
- // TODO: register storage plugin to plugin manager
-}
-
-type DataSource struct{}
-
-func NewDataSource() *DataSource {
- // TODO: construct a reasonable DataSource instance
- log.Warnf("auth data source enable etcd mode")
-
- inst := &DataSource{}
- // TODO: deal with exception
- if err := inst.initialize(); err != nil {
- return inst
- }
- return inst
-}
-
-func (ds *DataSource) initialize() error {
- // TODO: init DataSource members
- return nil
-}
-
-func (ds *DataSource) AccountExist(ctx context.Context, key string) (bool, error) {
- resp, err := backend.Registry().Do(ctx, registry.GET,
- registry.WithStrKey(GenerateETCDAccountKey(key)))
- if err != nil {
- return false, err
- }
- if resp.Count == 0 {
- return false, nil
- }
- return true, nil
-}
-
-func (ds *DataSource) GetAccount(ctx context.Context, key string) (*rbacframe.Account, error) {
- resp, err := backend.Registry().Do(ctx, registry.GET,
- registry.WithStrKey(GenerateETCDAccountKey(key)))
- if err != nil {
- return nil, err
- }
- if resp.Count != 1 {
- return nil, ErrNotUnique
- }
- account := &rbacframe.Account{}
- err = json.Unmarshal(resp.Kvs[0].Value, account)
- if err != nil {
- log.Errorf(err, "account info format invalid")
- return nil, err
- }
- return account, nil
-}
-func (ds *DataSource) ListAccount(ctx context.Context, key string) ([]*rbacframe.Account, int64, error) {
- resp, err := backend.Registry().Do(ctx, registry.GET,
- registry.WithStrKey(GenerateETCDAccountKey(key)), registry.WithPrefix())
- if err != nil {
- return nil, 0, err
- }
- accounts := make([]*rbacframe.Account, 0, resp.Count)
- for _, v := range resp.Kvs {
- a := &rbacframe.Account{}
- err = json.Unmarshal(v.Value, a)
- if err != nil {
- log.Error("account info format invalid:", err)
- continue //do not fail if some account is invalid
- }
- a.Password = ""
- accounts = append(accounts, a)
- }
- return accounts, resp.Count, nil
-}
-func (ds *DataSource) DeleteAccount(ctx context.Context, key string) (bool, error) {
- resp, err := backend.Registry().Do(ctx, registry.DEL,
- registry.WithStrKey(GenerateETCDAccountKey(key)))
- if err != nil {
- return false, err
- }
- return resp.Count != 0, nil
-}
-func (ds *DataSource) UpdateAccount(ctx context.Context, key string, account *rbacframe.Account) error {
- value, err := json.Marshal(account)
- if err != nil {
- log.Errorf(err, "account info is invalid")
- return err
- }
- _, err = backend.Registry().Do(ctx, registry.PUT,
- registry.WithStrKey(GenerateETCDAccountKey(key)),
- registry.WithValue(value))
- return err
-}
-
-func (ds *DataSource) AddDomain(ctx context.Context, domain string) (bool, error) {
- ok, err := backend.Registry().PutNoOverride(ctx,
- registry.WithStrKey(GenerateETCDDomainKey(domain)))
- if err != nil {
- return false, err
- }
- return ok, nil
-}
-
-func (ds *DataSource) DomainExist(ctx context.Context, domain string) (bool, error) {
- opts := append(utils.FromContext(ctx),
- registry.WithStrKey(GenerateETCDDomainKey(domain)),
- registry.WithCountOnly())
- rsp, err := backend.Store().Domain().Search(ctx, opts...)
- if err != nil {
- return false, err
- }
- return rsp.Count > 0, nil
-}
-
-func (ds *DataSource) AddProject(ctx context.Context, domain, project string) (bool, error) {
- ok, err := backend.Registry().PutNoOverride(ctx,
- registry.WithStrKey(GenerateETCDProjectKey(domain, project)))
- if err != nil {
- return ok, err
- }
- return ok, nil
-}
-
-func (ds *DataSource) ProjectExist(ctx context.Context, domain, project string) (bool, error) {
- opts := append(utils.FromContext(ctx),
- registry.WithStrKey(GenerateETCDProjectKey(domain, project)),
- registry.WithCountOnly())
- rsp, err := backend.Store().Project().Search(ctx, opts...)
- if err != nil {
- return false, err
- }
- return rsp.Count > 0, nil
-}
diff --git a/server/service/auth/etcd/etcd_test.go b/server/service/auth/etcd/etcd_test.go
deleted file mode 100644
index 7e2b418..0000000
--- a/server/service/auth/etcd/etcd_test.go
+++ /dev/null
@@ -1,121 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package etcd
-
-import (
- "context"
- "github.com/apache/servicecomb-service-center/pkg/rbacframe"
- mgr "github.com/apache/servicecomb-service-center/server/plugin"
- "github.com/apache/servicecomb-service-center/server/plugin/discovery/etcd"
- etcd2 "github.com/apache/servicecomb-service-center/server/plugin/registry/etcd"
- "github.com/apache/servicecomb-service-center/server/plugin/tracing/pzipkin"
- "github.com/apache/servicecomb-service-center/server/service/auth"
- "github.com/astaxie/beego"
- "github.com/stretchr/testify/assert"
- "testing"
-)
-
-var (
- a1 = rbacframe.Account{
- ID: "11111-22222-33333",
- Name: "test-account1",
- Password: "tnuocca-tset",
- Role: "admin",
- TokenExpirationTime: "2020-12-30",
- CurrentPassword: "tnuocca-tset1",
- }
- a2 = rbacframe.Account{
- ID: "11111-22222-33333-44444",
- Name: "test-account2",
- Password: "tnuocca-tset",
- Role: "admin",
- TokenExpirationTime: "2020-12-30",
- CurrentPassword: "tnuocca-tset1",
- }
-)
-
-func init() {
- beego.AppConfig.Set("registry_plugin", "etcd")
- mgr.RegisterPlugin(mgr.Plugin{PName: mgr.REGISTRY, Name: "etcd", New: etcd2.NewRegistry})
- mgr.RegisterPlugin(mgr.Plugin{PName: mgr.DISCOVERY, Name: "buildin", New: etcd.NewRepository})
- mgr.RegisterPlugin(mgr.Plugin{PName: mgr.DISCOVERY, Name: "etcd", New: etcd.NewRepository})
- mgr.RegisterPlugin(mgr.Plugin{PName: mgr.TRACING, Name: "buildin", New: pzipkin.New})
- auth.Install("etcd", func(opts auth.Options) (auth.DataSource, error) {
- return NewDataSource(), nil
- })
- err := auth.Init(auth.Options{
- Endpoint: "",
- PluginImplName: "etcd",
- })
- if err != nil {
- panic("failed to register etcd auth plugin")
- }
-}
-
-func TestAccount(t *testing.T) {
- t.Run("add and get account", func(t *testing.T) {
- err := auth.Auth().UpdateAccount(context.Background(), "test-account-key", &a1)
- assert.NoError(t, err)
- r, err := auth.Auth().GetAccount(context.Background(), "test-account-key")
- assert.NoError(t, err)
- assert.Equal(t, a1, *r)
- })
- t.Run("account should exist", func(t *testing.T) {
- exist, err := auth.Auth().AccountExist(context.Background(), "test-account-key")
- assert.NoError(t, err)
- assert.True(t, exist)
- })
- t.Run("delete account", func(t *testing.T) {
- err := auth.Auth().UpdateAccount(context.Background(), "test-account-key222", &a1)
- assert.NoError(t, err)
- _, err = auth.Auth().DeleteAccount(context.Background(), "test-account-key222")
- assert.NoError(t, err)
- })
- t.Run("add two accounts and list", func(t *testing.T) {
- err := auth.Auth().UpdateAccount(context.Background(), "key1", &a1)
- assert.NoError(t, err)
- err = auth.Auth().UpdateAccount(context.Background(), "key2", &a2)
- assert.NoError(t, err)
- accs, n, err := auth.Auth().ListAccount(context.Background(), "key")
- assert.NoError(t, err)
- assert.Equal(t, int64(2), n)
- t.Log(accs)
- })
-}
-
-func TestDomain(t *testing.T) {
- t.Run("test domain", func(t *testing.T) {
- _, err := auth.Auth().AddDomain(context.Background(), "test-domain")
- assert.NoError(t, err)
- r, err := auth.Auth().DomainExist(context.Background(), "test-domain")
- assert.NoError(t, err)
- assert.Equal(t, true, r)
- })
-}
-
-func TestProject(t *testing.T) {
- t.Run("test project", func(t *testing.T) {
- _, err := auth.Auth().AddProject(context.Background(), "test-domain", "test-project")
- assert.NoError(t, err)
- r, err := auth.Auth().ProjectExist(context.Background(), "test-domain", "test-project")
- assert.NoError(t, err)
- assert.Equal(t, true, r)
- })
-}
diff --git a/server/service/auth/etcd/key_generator.go b/server/service/auth/etcd/key_generator.go
deleted file mode 100644
index d2c83bb..0000000
--- a/server/service/auth/etcd/key_generator.go
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package etcd
-
-import "github.com/apache/servicecomb-service-center/pkg/util"
-
-const (
- RegistryRootKey = "cse-sr"
- RegistryProjectKey = "projects"
- SPLIT = "/"
- RegistryDomainKey = "domains"
-)
-
-func GetRootKey() string {
- return SPLIT + RegistryRootKey
-}
-
-func (ds *DataSource) GenerateAccountKey(name string) string {
- return util.StringJoin([]string{
- GetRootKey(),
- "accounts",
- name,
- }, SPLIT)
-}
-
-func GenerateETCDAccountKey(name string) string {
- return util.StringJoin([]string{
- GetRootKey(),
- "accounts",
- name,
- }, SPLIT)
-}
-
-func GenerateETCDProjectKey(domain, project string) string {
- return util.StringJoin([]string{
- GetProjectRootKey(domain),
- project,
- }, SPLIT)
-}
-
-func GetProjectRootKey(domain string) string {
- return util.StringJoin([]string{
- GetRootKey(),
- RegistryProjectKey,
- domain,
- }, SPLIT)
-}
-
-func GenerateETCDDomainKey(domain string) string {
- return util.StringJoin([]string{
- GetDomainRootKey(),
- domain,
- }, SPLIT)
-}
-
-func GetDomainRootKey() string {
- return util.StringJoin([]string{
- GetRootKey(),
- RegistryDomainKey,
- }, SPLIT)
-}
diff --git a/server/service/auth/etcd/key_generator_test.go b/server/service/auth/etcd/key_generator_test.go
deleted file mode 100644
index 80d4a3a..0000000
--- a/server/service/auth/etcd/key_generator_test.go
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package etcd
-
-import (
- "github.com/stretchr/testify/assert"
- "testing"
-)
-
-func TestGenerateETCDAccountKey(t *testing.T) {
- assert.Equal(t, "/cse-sr/accounts/admin", GenerateETCDAccountKey("admin"))
-}
-
-func TestGenerateETCDProjectKey(t *testing.T) {
- assert.Equal(t, "/cse-sr/projects/domain/project", GenerateETCDProjectKey("domain", "project"))
-}
-
-func TestGenerateETCDDomainKey(t *testing.T) {
- assert.Equal(t, "/cse-sr/domains/domain", GenerateETCDDomainKey("domain"))
-}
diff --git a/server/service/auth/manager.go b/server/service/auth/manager.go
deleted file mode 100644
index c772e23..0000000
--- a/server/service/auth/manager.go
+++ /dev/null
@@ -1,58 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more
-// contributor license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright ownership.
-// The ASF licenses this file to You under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance with
-// the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package auth
-
-import (
- "fmt"
- "github.com/apache/servicecomb-service-center/pkg/log"
-)
-
-type dataSourceEngine func(opts Options) (DataSource, error)
-
-var (
- plugins = make(map[ImplName]dataSourceEngine)
- authDataSourceInst DataSource
-)
-
-// load plugins configuration into plugins
-func Install(pluginImplName string, engineFunc dataSourceEngine) {
- plugins[ImplName(pluginImplName)] = engineFunc
-}
-
-// construct storage plugin instance
-// invoked by sc main process
-func Init(opts Options) error {
- if opts.PluginImplName == "" {
- return nil
- }
-
- authDataSourceEngine, ok := plugins[opts.PluginImplName]
- if !ok {
- return fmt.Errorf("plugin implement not supported [%s]", opts.PluginImplName)
- }
- var err error
- authDataSourceInst, err = authDataSourceEngine(opts)
- if err != nil {
- return err
- }
- log.Info(fmt.Sprintf("storage shim plugin [%s@%s] enabled", opts.PluginImplName, opts.Endpoint))
- return nil
-}
-
-// usage: auth.Auth().CreateAccount()
-func Auth() DataSource {
- return authDataSourceInst
-}
diff --git a/server/service/auth/manager_test.go b/server/service/auth/manager_test.go
deleted file mode 100644
index 40c77a9..0000000
--- a/server/service/auth/manager_test.go
+++ /dev/null
@@ -1,48 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more
-// contributor license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright ownership.
-// The ASF licenses this file to You under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance with
-// the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package auth
-
-import (
- "github.com/apache/servicecomb-service-center/server/service/auth/etcd"
- "github.com/go-chassis/go-archaius"
- "github.com/stretchr/testify/assert"
- "testing"
-)
-
-func TestInit(t *testing.T) {
- _ = archaius.Init(archaius.WithMemorySource())
- _ = archaius.Set("servicecomb.auth.name", "etcd")
- t.Run("circuit datasource auth", func(t *testing.T) {
- err := Init(Options{
- Endpoint: "",
- PluginImplName: "",
- })
- assert.NoError(t, err)
- })
- t.Run("install and init", func(t *testing.T) {
- Install("etcd",
- func(opts Options) (DataSource, error) {
- return etcd.NewDataSource(), nil
- })
-
- // sc main function initialize step
- err := Init(Options{
- Endpoint: "",
- PluginImplName: ImplName(archaius.GetString("servicecomb.auth.name", "etcd")),
- })
- assert.NoError(t, err)
- })
-}
diff --git a/server/service/auth/options.go b/server/service/auth/options.go
deleted file mode 100644
index 3166411..0000000
--- a/server/service/auth/options.go
+++ /dev/null
@@ -1,22 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more
-// contributor license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright ownership.
-// The ASF licenses this file to You under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance with
-// the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package auth
-
-//Options contains configuration for plugins
-type Options struct {
- Endpoint string
- PluginImplName ImplName
-}
diff --git a/server/service/auth/types.go b/server/service/auth/types.go
deleted file mode 100644
index 01355e3..0000000
--- a/server/service/auth/types.go
+++ /dev/null
@@ -1,18 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more
-// contributor license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright ownership.
-// The ASF licenses this file to You under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance with
-// the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package auth
-
-type ImplName string