You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@servicecomb.apache.org by ti...@apache.org on 2021/08/23 01:07:56 UTC
[servicecomb-kie] branch master updated: Fix: Can not page the kvs
(#204)
This is an automated email from the ASF dual-hosted git repository.
tianxiaoliang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/servicecomb-kie.git
The following commit(s) were added to refs/heads/master by this push:
new 718a04e Fix: Can not page the kvs (#204)
718a04e is described below
commit 718a04ec27c231ec26a1098c27e4421011485424
Author: little-cui <su...@qq.com>
AuthorDate: Mon Aug 23 09:07:51 2021 +0800
Fix: Can not page the kvs (#204)
---
server/datasource/etcd/history/history_dao.go | 7 ++-
server/datasource/etcd/kv/kv_dao.go | 5 +-
server/datasource/history_dao_test.go | 44 +++++++++++---
server/datasource/kv_dao_test.go | 84 ++++++++++++++++++++++++++
server/datasource/mongo/history/history_dao.go | 4 +-
server/datasource/mongo/kv/kv_dao.go | 2 +-
server/datasource/options.go | 6 +-
server/service/kv/kv_svc_test.go | 22 +++----
8 files changed, 144 insertions(+), 30 deletions(-)
diff --git a/server/datasource/etcd/history/history_dao.go b/server/datasource/etcd/history/history_dao.go
index 603b4f0..d4c6d83 100644
--- a/server/datasource/etcd/history/history_dao.go
+++ b/server/datasource/etcd/history/history_dao.go
@@ -39,7 +39,8 @@ func (s *Dao) GetHistory(ctx context.Context, kvID, project, domain string, opti
for _, o := range options {
o(&opts)
}
- kvs, _, err := etcdadpt.List(ctx, key.HisList(domain, project, kvID), etcdadpt.WithOrderByCreate(), etcdadpt.WithDescendOrder())
+ kvs, _, err := etcdadpt.List(ctx, key.HisList(domain, project, kvID),
+ etcdadpt.WithOrderByCreate(), etcdadpt.WithDescendOrder())
if err != nil {
openlog.Error(err.Error())
return nil, err
@@ -52,11 +53,11 @@ func (s *Dao) GetHistory(ctx context.Context, kvID, project, domain string, opti
func pagingResult(kvs []*mvccpb.KeyValue, offset, limit int64) []*model.KVDoc {
total := int64(len(kvs))
- end := offset + limit
- if offset != 0 && limit != 0 {
+ if limit != 0 {
if offset >= total {
return []*model.KVDoc{}
}
+ end := offset + limit
if end > total {
end = total
}
diff --git a/server/datasource/etcd/kv/kv_dao.go b/server/datasource/etcd/kv/kv_dao.go
index 39590a6..be7a297 100644
--- a/server/datasource/etcd/kv/kv_dao.go
+++ b/server/datasource/etcd/kv/kv_dao.go
@@ -216,7 +216,8 @@ func (s *Dao) List(ctx context.Context, project, domain string, options ...datas
return nil, err
}
// TODO may be OOM
- kvs, _, err := etcdadpt.List(ctx, key.KVList(domain, project))
+ kvs, _, err := etcdadpt.List(ctx, key.KVList(domain, project),
+ etcdadpt.WithOrderByCreate(), etcdadpt.WithAscendOrder())
if err != nil {
openlog.Error("list kv failed: " + err.Error())
return nil, err
@@ -276,7 +277,7 @@ func toRegex(opts datasource.FindOptions) (*regexp.Regexp, error) {
}
func pagingResult(result *model.KVResponse, opts datasource.FindOptions) *model.KVResponse {
- if opts.Offset == 0 || opts.Limit == 0 {
+ if opts.Limit == 0 {
return result
}
total := int64(result.Total)
diff --git a/server/datasource/history_dao_test.go b/server/datasource/history_dao_test.go
index 99ece08..3cb9650 100644
--- a/server/datasource/history_dao_test.go
+++ b/server/datasource/history_dao_test.go
@@ -19,22 +19,23 @@ package datasource_test
import (
"context"
- kvsvc "github.com/apache/servicecomb-kie/server/service/kv"
"testing"
- common2 "github.com/apache/servicecomb-kie/pkg/common"
+ _ "github.com/apache/servicecomb-kie/test"
+
+ "github.com/apache/servicecomb-kie/pkg/common"
"github.com/apache/servicecomb-kie/pkg/model"
"github.com/apache/servicecomb-kie/server/datasource"
-
- _ "github.com/apache/servicecomb-kie/test"
+ kvsvc "github.com/apache/servicecomb-kie/server/service/kv"
"github.com/stretchr/testify/assert"
)
func TestGetHistory(t *testing.T) {
- kv, err := kvsvc.Create(context.TODO(), &model.KVDoc{
- Key: "history",
+ ctx := context.TODO()
+ kv, err := kvsvc.Create(ctx, &model.KVDoc{
+ Key: "TestGetHistory",
Value: "2s",
- Status: common2.StatusEnabled,
+ Status: common.StatusEnabled,
Labels: map[string]string{
"app": "mall",
"service": "cart",
@@ -44,10 +45,35 @@ func TestGetHistory(t *testing.T) {
})
assert.Nil(t, err)
assert.NotEmpty(t, kv.ID)
+ defer kvsvc.FindOneAndDelete(ctx, kv.ID, "kv-test", "default")
+
+ _, uErr := kvsvc.Update(ctx, &model.UpdateKVRequest{
+ ID: kv.ID,
+ Value: "3s",
+ Domain: "default",
+ Project: "kv-test",
+ })
+ assert.NoError(t, uErr)
+
t.Run("after create kv, should has history", func(t *testing.T) {
- h, err := datasource.GetBroker().GetHistoryDao().GetHistory(context.TODO(), kv.ID, "kv-test", "default")
+ h, err := datasource.GetBroker().GetHistoryDao().GetHistory(ctx, kv.ID, "kv-test", "default")
assert.NoError(t, err)
- assert.GreaterOrEqual(t, h.Total, 1)
+ assert.Equal(t, 2, h.Total)
+ assert.Equal(t, 2, len(h.Data))
})
+ t.Run("test paging, should pass", func(t *testing.T) {
+ resp, err := datasource.GetBroker().GetHistoryDao().GetHistory(ctx, kv.ID, "kv-test", "default",
+ datasource.WithOffset(0), datasource.WithLimit(1))
+ assert.NoError(t, err)
+ assert.Equal(t, 2, resp.Total)
+ assert.Equal(t, 1, len(resp.Data))
+ assert.Equal(t, "3s", resp.Data[0].Value)
+ resp, err = datasource.GetBroker().GetHistoryDao().GetHistory(ctx, kv.ID, "kv-test", "default",
+ datasource.WithOffset(1), datasource.WithLimit(1))
+ assert.NoError(t, err)
+ assert.Equal(t, 2, resp.Total)
+ assert.Equal(t, 1, len(resp.Data))
+ assert.Equal(t, "2s", resp.Data[0].Value)
+ })
}
diff --git a/server/datasource/kv_dao_test.go b/server/datasource/kv_dao_test.go
new file mode 100644
index 0000000..9228739
--- /dev/null
+++ b/server/datasource/kv_dao_test.go
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package datasource_test
+
+import (
+ "context"
+ "testing"
+
+ "github.com/apache/servicecomb-kie/pkg/common"
+ "github.com/apache/servicecomb-kie/pkg/model"
+ "github.com/apache/servicecomb-kie/server/datasource"
+ kvsvc "github.com/apache/servicecomb-kie/server/service/kv"
+ "github.com/stretchr/testify/assert"
+)
+
+func TestList(t *testing.T) {
+ ctx := context.TODO()
+ kv1, err := kvsvc.Create(ctx, &model.KVDoc{
+ Key: "TestList1",
+ Value: "2s",
+ Status: common.StatusEnabled,
+ Labels: map[string]string{
+ "app": "mall",
+ "service": "cart",
+ },
+ Domain: "default",
+ Project: "kv-list-test",
+ })
+ assert.Nil(t, err)
+ assert.NotEmpty(t, kv1.ID)
+ defer kvsvc.FindOneAndDelete(ctx, kv1.ID, "kv-list-test", "default")
+
+ kv2, err := kvsvc.Create(ctx, &model.KVDoc{
+ Key: "TestList2",
+ Value: "3s",
+ Status: common.StatusEnabled,
+ Labels: map[string]string{
+ "app": "mall",
+ "service": "cart",
+ },
+ Domain: "default",
+ Project: "kv-list-test",
+ })
+ assert.Nil(t, err)
+ assert.NotEmpty(t, kv2.ID)
+ defer kvsvc.FindOneAndDelete(ctx, kv2.ID, "kv-list-test", "default")
+
+ t.Run("after create kv, should list results", func(t *testing.T) {
+ h, err := datasource.GetBroker().GetKVDao().List(ctx, "kv-list-test", "default")
+ assert.NoError(t, err)
+ assert.Equal(t, 2, h.Total)
+ assert.Equal(t, 2, len(h.Data))
+ })
+ t.Run("test paging, should pass", func(t *testing.T) {
+ resp, err := datasource.GetBroker().GetKVDao().List(ctx, "kv-list-test", "default",
+ datasource.WithOffset(0), datasource.WithLimit(1))
+ assert.NoError(t, err)
+ assert.Equal(t, 2, resp.Total)
+ assert.Equal(t, 1, len(resp.Data))
+ assert.Equal(t, "2s", resp.Data[0].Value)
+
+ resp, err = datasource.GetBroker().GetKVDao().List(ctx, "kv-list-test", "default",
+ datasource.WithOffset(1), datasource.WithLimit(1))
+ assert.NoError(t, err)
+ assert.Equal(t, 2, resp.Total)
+ assert.Equal(t, 1, len(resp.Data))
+ assert.Equal(t, "3s", resp.Data[0].Value)
+ })
+}
diff --git a/server/datasource/mongo/history/history_dao.go b/server/datasource/mongo/history/history_dao.go
index 52acf40..10a3b01 100644
--- a/server/datasource/mongo/history/history_dao.go
+++ b/server/datasource/mongo/history/history_dao.go
@@ -54,9 +54,9 @@ func (s *Dao) GetHistory(ctx context.Context, kvID, project, domain string, opti
func getHistoryByKeyID(ctx context.Context, filter bson.M, offset, limit int64) (*model.KVResponse, error) {
collection := session.GetDB().Collection(session.CollectionKVRevision)
opt := options.Find().SetSort(map[string]interface{}{
- "revision": -1,
+ "update_revision": -1,
})
- if offset != 0 && limit != 0 {
+ if limit > 0 {
opt = opt.SetLimit(limit)
opt = opt.SetSkip(offset)
}
diff --git a/server/datasource/mongo/kv/kv_dao.go b/server/datasource/mongo/kv/kv_dao.go
index 17f622d..45060f6 100644
--- a/server/datasource/mongo/kv/kv_dao.go
+++ b/server/datasource/mongo/kv/kv_dao.go
@@ -107,7 +107,7 @@ func findKV(ctx context.Context, domain string, project string, opts datasource.
}
}
opt := options.Find()
- if opts.Offset != 0 && opts.Limit != 0 {
+ if opts.Limit > 0 {
opt = opt.SetLimit(opts.Limit)
opt = opt.SetSkip(opts.Offset)
}
diff --git a/server/datasource/options.go b/server/datasource/options.go
index 05f2653..c735357 100644
--- a/server/datasource/options.go
+++ b/server/datasource/options.go
@@ -51,8 +51,10 @@ type FindOptions struct {
LabelFormat string
ClearLabel bool
Timeout time.Duration
- Offset int64
- Limit int64
+ // Offset the offset of the response, start at 0
+ Offset int64
+ // Limit the page size of the response, dot not paging if limit=0
+ Limit int64
}
//FindOption is functional option to find key value
diff --git a/server/service/kv/kv_svc_test.go b/server/service/kv/kv_svc_test.go
index bc23ccf..7401d8d 100644
--- a/server/service/kv/kv_svc_test.go
+++ b/server/service/kv/kv_svc_test.go
@@ -18,16 +18,16 @@
package kv_test
import (
- "github.com/apache/servicecomb-kie/server/datasource"
- kvsvc "github.com/apache/servicecomb-kie/server/service/kv"
- _ "github.com/apache/servicecomb-kie/test"
- "github.com/go-chassis/cari/config"
-
"context"
"testing"
- common2 "github.com/apache/servicecomb-kie/pkg/common"
+ _ "github.com/apache/servicecomb-kie/test"
+
+ "github.com/apache/servicecomb-kie/pkg/common"
"github.com/apache/servicecomb-kie/pkg/model"
+ "github.com/apache/servicecomb-kie/server/datasource"
+ kvsvc "github.com/apache/servicecomb-kie/server/service/kv"
+ "github.com/go-chassis/cari/config"
"github.com/go-chassis/openlog"
log "github.com/go-chassis/seclog"
"github.com/stretchr/testify/assert"
@@ -52,7 +52,7 @@ func TestService_CreateOrUpdate(t *testing.T) {
kv, err := kvsvc.Create(context.TODO(), &model.KVDoc{
Key: "timeout",
Value: "2s",
- Status: common2.StatusEnabled,
+ Status: common.StatusEnabled,
Labels: map[string]string{
"app": "mall",
"service": "cart",
@@ -67,7 +67,7 @@ func TestService_CreateOrUpdate(t *testing.T) {
kv, err := kvsvc.Create(context.TODO(), &model.KVDoc{
Key: "timeout",
Value: "2s",
- Status: common2.StatusEnabled,
+ Status: common.StatusEnabled,
Labels: map[string]string{
"app": "mall",
"service": "cart",
@@ -90,7 +90,7 @@ func TestService_CreateOrUpdate(t *testing.T) {
beforeKV, err := kvsvc.Create(context.Background(), &model.KVDoc{
Key: "timeout",
Value: "1s",
- Status: common2.StatusEnabled,
+ Status: common.StatusEnabled,
Labels: map[string]string{
"app": "mall",
},
@@ -120,7 +120,7 @@ func TestService_Create(t *testing.T) {
result, err := kvsvc.Create(context.TODO(), &model.KVDoc{
Key: "timeout",
Value: "2s",
- Status: common2.StatusEnabled,
+ Status: common.StatusEnabled,
Labels: map[string]string{
"app": "mall",
"service": "utCart",
@@ -137,7 +137,7 @@ func TestService_Create(t *testing.T) {
_, err := kvsvc.Create(context.TODO(), &model.KVDoc{
Key: "timeout",
Value: "2s",
- Status: common2.StatusEnabled,
+ Status: common.StatusEnabled,
Labels: map[string]string{
"app": "mall",
"service": "utCart",