You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@servicecomb.apache.org by ti...@apache.org on 2020/02/13 06:02:56 UTC
[servicecomb-kie] branch master updated: #92 replace limit/offset
by pageNum/pageSize / add param:total (#93)
This is an automated email from the ASF dual-hosted git repository.
tianxiaoliang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/servicecomb-kie.git
The following commit(s) were added to refs/heads/master by this push:
new f10641e #92 replace limit/offset by pageNum/pageSize / add param:total (#93)
f10641e is described below
commit f10641eab31449864c3a51ce116d1d29078ac2d7
Author: GuoYL <53...@users.noreply.github.com>
AuthorDate: Thu Feb 13 14:02:50 2020 +0800
#92 replace limit/offset by pageNum/pageSize / add param:total (#93)
* replace limit/offset by pageNum/pageSize / add param:total
* modify ut
* fix bug
* modify as comment
---
pkg/model/kv.go | 2 --
server/resource/v1/common.go | 34 ++++++++++++++------------
server/resource/v1/history_resource.go | 12 +++++-----
server/resource/v1/kv_resource.go | 40 +++++++++++++++----------------
server/resource/v1/kv_resource_test.go | 19 +--------------
server/service/mongo/history/dao.go | 20 +++++++++-------
server/service/mongo/history/service.go | 4 ++--
server/service/mongo/kv/kv_dao.go | 24 ++++++++++++-------
server/service/mongo/kv/kv_service.go | 15 ++++--------
server/service/mongo/view/view_service.go | 16 +++++--------
server/service/options.go | 16 ++++++-------
server/service/service.go | 2 +-
12 files changed, 94 insertions(+), 110 deletions(-)
diff --git a/pkg/model/kv.go b/pkg/model/kv.go
index 0db6875..ac92854 100644
--- a/pkg/model/kv.go
+++ b/pkg/model/kv.go
@@ -29,8 +29,6 @@ type KVRequest struct {
//KVResponse represents the key value list
type KVResponse struct {
LabelDoc *LabelDocResponse `json:"label,omitempty"`
- PageNum int `json:"num,omitempty"`
- Size int `json:"size,omitempty"`
Total int `json:"total,omitempty"`
Data []*KVDoc `json:"data,omitempty"`
}
diff --git a/server/resource/v1/common.go b/server/resource/v1/common.go
index 99722a8..b99dc80 100644
--- a/server/resource/v1/common.go
+++ b/server/resource/v1/common.go
@@ -36,6 +36,8 @@ import (
"gopkg.in/yaml.v2"
)
+//const of server
+
//err
var (
ErrInvalidRev = errors.New(common.MsgInvalidRev)
@@ -174,29 +176,31 @@ func eventHappened(rctx *restful.Context, waitStr string, topic *pubsub.Topic) (
}
return happened, nil
}
-func checkPagination(limitStr, offsetStr string) (int64, int64, error) {
+
+// size from 1 to start
+func checkPagination(pageNum, pageSize string) (int64, int64, error) {
var err error
- var limit, offset int64
- if limitStr != "" {
- limit, err = strconv.ParseInt(limitStr, 10, 64)
+ var num, size int64
+ if pageNum != "" {
+ num, err = strconv.ParseInt(pageNum, 10, 64)
if err != nil {
return 0, 0, err
}
- if limit < 1 || limit > 50 {
- return 0, 0, errors.New("invalid limit number")
+ if num < 1 {
+ return 0, 0, errors.New("invalid pageNum number")
}
}
- if offsetStr != "" {
- offset, err = strconv.ParseInt(offsetStr, 10, 64)
+ if pageSize != "" {
+ size, err = strconv.ParseInt(pageSize, 10, 64)
if err != nil {
- return 0, 0, errors.New("invalid offset number")
+ return 0, 0, errors.New("invalid pageSize number")
}
- if offset < 0 {
- return 0, 0, errors.New("invalid offset number")
+ if size < 1 || size > 100 {
+ return 0, 0, errors.New("invalid pageSize number")
}
}
- return limit, offset, err
+ return num, size, err
}
func checkStatus(status string) (string, error) {
@@ -209,13 +213,13 @@ func checkStatus(status string) (string, error) {
}
func queryAndResponse(rctx *restful.Context,
- domain interface{}, project string, key string, labels map[string]string, limit, offset int64, status string) {
+ domain interface{}, project string, key string, labels map[string]string, pageNum, pageSize int64, status string) {
m := getMatchPattern(rctx)
opts := []service.FindOption{
service.WithKey(key),
service.WithLabels(labels),
- service.WithLimit(limit),
- service.WithOffset(offset),
+ service.WithPageNum(pageNum),
+ service.WithPageSize(pageSize),
}
if m == common.PatternExact {
opts = append(opts, service.WithExactLabels())
diff --git a/server/resource/v1/history_resource.go b/server/resource/v1/history_resource.go
index 09d60e6..73ea442 100644
--- a/server/resource/v1/history_resource.go
+++ b/server/resource/v1/history_resource.go
@@ -37,9 +37,9 @@ type HistoryResource struct {
func (r *HistoryResource) GetRevisions(context *restful.Context) {
var err error
keyID := context.ReadPathParameter("key_id")
- limitStr := context.ReadQueryParameter("limit")
- offsetStr := context.ReadQueryParameter("offset")
- limit, offset, err := checkPagination(limitStr, offsetStr)
+ pageNumStr := context.ReadQueryParameter("pageNum")
+ pageSizeStr := context.ReadQueryParameter("pageSize")
+ pageNum, pageSize, err := checkPagination(pageNumStr, pageSizeStr)
if err != nil {
WriteErrResponse(context, http.StatusBadRequest, err.Error(), common.ContentTypeText)
return
@@ -50,10 +50,10 @@ func (r *HistoryResource) GetRevisions(context *restful.Context) {
return
}
key := context.ReadQueryParameter("key")
- revisions, err := service.HistoryService.GetHistory(context.Ctx, keyID,
+ revisions, _, err := service.HistoryService.GetHistory(context.Ctx, keyID,
service.WithKey(key),
- service.WithLimit(limit),
- service.WithOffset(offset))
+ service.WithPageSize(pageSize),
+ service.WithPageNum(pageNum))
if err != nil {
if err == service.ErrRevisionNotExist {
WriteErrResponse(context, http.StatusNotFound, err.Error(), common.ContentTypeText)
diff --git a/server/resource/v1/kv_resource.go b/server/resource/v1/kv_resource.go
index b006a31..0054025 100644
--- a/server/resource/v1/kv_resource.go
+++ b/server/resource/v1/kv_resource.go
@@ -101,9 +101,9 @@ func (r *KVResource) GetByKey(rctx *restful.Context) {
WriteErrResponse(rctx, http.StatusInternalServerError, common.MsgDomainMustNotBeEmpty, common.ContentTypeText)
return
}
- limitStr := rctx.ReadQueryParameter("limit")
- offsetStr := rctx.ReadQueryParameter("offset")
- limit, offset, err := checkPagination(limitStr, offsetStr)
+ pageNumStr := rctx.ReadQueryParameter("pageNum")
+ pageSizeStr := rctx.ReadQueryParameter("pageSize")
+ pageNum, pageSize, err := checkPagination(pageNumStr, pageSizeStr)
if err != nil {
WriteErrResponse(rctx, http.StatusBadRequest, err.Error(), common.ContentTypeText)
return
@@ -114,7 +114,7 @@ func (r *KVResource) GetByKey(rctx *restful.Context) {
WriteErrResponse(rctx, http.StatusBadRequest, err.Error(), common.ContentTypeText)
return
}
- returnData(rctx, domain, project, labels, limit, offset, status)
+ returnData(rctx, domain, project, labels, pageNum, pageSize, status)
}
//List response kv list
@@ -131,9 +131,9 @@ func (r *KVResource) List(rctx *restful.Context) {
WriteErrResponse(rctx, http.StatusBadRequest, err.Error(), common.ContentTypeText)
return
}
- limitStr := rctx.ReadQueryParameter("limit")
- offsetStr := rctx.ReadQueryParameter("offset")
- limit, offset, err := checkPagination(limitStr, offsetStr)
+ pageNumStr := rctx.ReadQueryParameter("pageNum")
+ pageSizeStr := rctx.ReadQueryParameter("pageSize")
+ pageNum, pageSize, err := checkPagination(pageNumStr, pageSizeStr)
if err != nil {
WriteErrResponse(rctx, http.StatusBadRequest, err.Error(), common.ContentTypeText)
return
@@ -144,15 +144,15 @@ func (r *KVResource) List(rctx *restful.Context) {
WriteErrResponse(rctx, http.StatusBadRequest, err.Error(), common.ContentTypeText)
return
}
- returnData(rctx, domain, project, labels, limit, offset, status)
+ returnData(rctx, domain, project, labels, pageNum, pageSize, status)
}
-func returnData(rctx *restful.Context, domain interface{}, project string, labels map[string]string, limit, offset int64, status string) {
+func returnData(rctx *restful.Context, domain interface{}, project string, labels map[string]string, pageNum, pageSize int64, status string) {
revStr := rctx.ReadQueryParameter(common.QueryParamRev)
wait := rctx.ReadQueryParameter(common.QueryParamWait)
if revStr == "" {
if wait == "" {
- queryAndResponse(rctx, domain, project, "", labels, limit, offset, status)
+ queryAndResponse(rctx, domain, project, "", labels, pageNum, pageSize, status)
return
}
changed, err := eventHappened(rctx, wait, &pubsub.Topic{
@@ -166,7 +166,7 @@ func returnData(rctx *restful.Context, domain interface{}, project string, label
return
}
if changed {
- queryAndResponse(rctx, domain, project, "", labels, limit, offset, status)
+ queryAndResponse(rctx, domain, project, "", labels, pageNum, pageSize, status)
return
}
rctx.WriteHeader(http.StatusNotModified)
@@ -181,7 +181,7 @@ func returnData(rctx *restful.Context, domain interface{}, project string, label
return
}
if revised {
- queryAndResponse(rctx, domain, project, "", labels, limit, offset, status)
+ queryAndResponse(rctx, domain, project, "", labels, pageNum, pageSize, status)
return
} else if wait != "" {
changed, err := eventHappened(rctx, wait, &pubsub.Topic{
@@ -195,7 +195,7 @@ func returnData(rctx *restful.Context, domain interface{}, project string, label
return
}
if changed {
- queryAndResponse(rctx, domain, project, "", labels, limit, offset, status)
+ queryAndResponse(rctx, domain, project, "", labels, pageNum, pageSize, status)
return
}
rctx.WriteHeader(http.StatusNotModified)
@@ -221,17 +221,17 @@ func (r *KVResource) Search(context *restful.Context) {
return
}
var kvs []*model.KVResponse
- limitStr := context.ReadQueryParameter("limit")
- offsetStr := context.ReadQueryParameter("offset")
- limit, offset, err := checkPagination(limitStr, offsetStr)
+ pageNumStr := context.ReadQueryParameter("pageNum")
+ pageSizeStr := context.ReadQueryParameter("pageSize")
+ pageNum, pageSize, err := checkPagination(pageNumStr, pageSizeStr)
if err != nil {
WriteErrResponse(context, http.StatusBadRequest, err.Error(), common.ContentTypeText)
return
}
if labelCombinations == nil {
result, err := service.KVService.FindKV(context.Ctx, domain.(string), project,
- service.WithLimit(limit),
- service.WithOffset(offset))
+ service.WithPageNum(pageNum),
+ service.WithPageSize(pageSize))
if err != nil {
openlogging.Error("can not find by labels", openlogging.WithTags(openlogging.Tags{
"err": err.Error(),
@@ -247,8 +247,8 @@ func (r *KVResource) Search(context *restful.Context) {
}))
result, err := service.KVService.FindKV(context.Ctx, domain.(string), project,
service.WithLabels(labels),
- service.WithLimit(limit),
- service.WithOffset(offset))
+ service.WithPageNum(pageNum),
+ service.WithPageSize(pageSize))
if err != nil {
if err == service.ErrKeyNotExists {
continue
diff --git a/server/resource/v1/kv_resource_test.go b/server/resource/v1/kv_resource_test.go
index 7765464..b0370b2 100644
--- a/server/resource/v1/kv_resource_test.go
+++ b/server/resource/v1/kv_resource_test.go
@@ -222,24 +222,7 @@ func TestKVResource_List(t *testing.T) {
t.Log(duration)
})
t.Run("list kv by service label offset, should return 1kv", func(t *testing.T) {
- r, _ := http.NewRequest("GET", "/v1/test/kie/kv?label=service:utService&offset=1", nil)
- noopH := &handler2.NoopAuthHandler{}
- chain, _ := handler.CreateChain(common.Provider, "testchain1", noopH.Name())
- r.Header.Set("Content-Type", "application/json")
- kvr := &v1.KVResource{}
- c, err := restfultest.New(kvr, chain)
- assert.NoError(t, err)
- resp := httptest.NewRecorder()
- c.ServeHTTP(resp, r)
- body, err := ioutil.ReadAll(resp.Body)
- assert.NoError(t, err)
- result := &model.KVResponse{}
- err = json.Unmarshal(body, result)
- assert.NoError(t, err)
- assert.Equal(t, 1, len(result.Data))
- })
- t.Run("list kv by service label limit, should return 1kv", func(t *testing.T) {
- r, _ := http.NewRequest("GET", "/v1/test/kie/kv?label=service:utService&limit=1", nil)
+ r, _ := http.NewRequest("GET", "/v1/test/kie/kv?label=service:utService&pageNum=1&pageSize=1", nil)
noopH := &handler2.NoopAuthHandler{}
chain, _ := handler.CreateChain(common.Provider, "testchain1", noopH.Name())
r.Header.Set("Content-Type", "application/json")
diff --git a/server/service/mongo/history/dao.go b/server/service/mongo/history/dao.go
index 0057bbc..0c6ce73 100644
--- a/server/service/mongo/history/dao.go
+++ b/server/service/mongo/history/dao.go
@@ -27,20 +27,22 @@ import (
"go.mongodb.org/mongo-driver/mongo/options"
)
-func getHistoryByKeyID(ctx context.Context, filter bson.M, limit, offset int64) ([]*model.KVDoc, error) {
+func getHistoryByKeyID(ctx context.Context, filter bson.M, pageNum, pageSize int64) ([]*model.KVDoc, int, error) {
collection := session.GetDB().Collection(session.CollectionKVRevision)
opt := options.Find().SetSort(map[string]interface{}{
"revision": -1,
})
- if limit != 0 {
- opt = opt.SetLimit(limit)
+ if pageNum != 0 && pageSize != 0 {
+ opt = opt.SetLimit(pageSize)
+ opt = opt.SetSkip(pageSize * (pageNum - 1))
}
- if offset != 0 {
- opt = opt.SetSkip(offset)
+ curTotal, err := collection.CountDocuments(ctx, filter)
+ if err != nil {
+ return nil, 0, err
}
cur, err := collection.Find(ctx, filter, opt)
if err != nil {
- return nil, err
+ return nil, 0, err
}
kvs := make([]*model.KVDoc, 0)
var exist bool
@@ -49,15 +51,15 @@ func getHistoryByKeyID(ctx context.Context, filter bson.M, limit, offset int64)
err := cur.Decode(&elem)
if err != nil {
openlogging.Error("decode error: " + err.Error())
- return nil, err
+ return nil, 0, err
}
exist = true
kvs = append(kvs, &elem)
}
if !exist {
- return nil, service.ErrRevisionNotExist
+ return nil, 0, service.ErrRevisionNotExist
}
- return kvs, nil
+ return kvs, int(curTotal), nil
}
//AddHistory add kv history
diff --git a/server/service/mongo/history/service.go b/server/service/mongo/history/service.go
index 9502e75..8de1918 100644
--- a/server/service/mongo/history/service.go
+++ b/server/service/mongo/history/service.go
@@ -30,7 +30,7 @@ type Service struct {
}
//GetHistory get all history by label id
-func (s *Service) GetHistory(ctx context.Context, kvID string, options ...service.FindOption) ([]*model.KVDoc, error) {
+func (s *Service) GetHistory(ctx context.Context, kvID string, options ...service.FindOption) ([]*model.KVDoc, int, error) {
var filter primitive.M
opts := service.FindOptions{}
for _, o := range options {
@@ -40,5 +40,5 @@ func (s *Service) GetHistory(ctx context.Context, kvID string, options ...servic
"id": kvID,
}
- return getHistoryByKeyID(ctx, filter, opts.Limit, opts.Offset)
+ return getHistoryByKeyID(ctx, filter, opts.PageNum, opts.PageSize)
}
diff --git a/server/service/mongo/kv/kv_dao.go b/server/service/mongo/kv/kv_dao.go
index 97cf51f..837a67b 100644
--- a/server/service/mongo/kv/kv_dao.go
+++ b/server/service/mongo/kv/kv_dao.go
@@ -104,7 +104,7 @@ func updateKeyValue(ctx context.Context, kv *model.KVDoc) error {
}
-func findKV(ctx context.Context, domain string, project string, opts service.FindOptions) (*mongo.Cursor, error) {
+func findKV(ctx context.Context, domain string, project string, opts service.FindOptions) (*mongo.Cursor, int, error) {
collection := session.GetDB().Collection(session.CollectionKV)
ctx, _ = context.WithTimeout(ctx, opts.Timeout)
filter := bson.M{"domain": domain, "project": project}
@@ -117,11 +117,19 @@ func findKV(ctx context.Context, domain string, project string, opts service.Fin
}
}
opt := options.Find()
- if opts.Limit != 0 {
- opt = opt.SetLimit(opts.Limit)
+ if opts.PageSize != 0 && opts.PageNum != 0 {
+ opt = opt.SetLimit(opts.PageSize)
+ opt = opt.SetSkip(opts.PageSize * (opts.PageNum - 1))
}
- if opts.Offset != 0 {
- opt = opt.SetSkip(opts.Offset)
+ curTotal, err := collection.CountDocuments(ctx, filter)
+ if err != nil {
+ if err.Error() == context.DeadlineExceeded.Error() {
+ openlogging.Error("find kv failed, deadline exceeded", openlogging.WithTags(openlogging.Tags{
+ "timeout": opts.Timeout,
+ }))
+ return nil, 0, fmt.Errorf("can not find kv in %s", opts.Timeout)
+ }
+ return nil, 0, err
}
if opts.Status != "" {
filter["status"] = opts.Status
@@ -132,11 +140,11 @@ func findKV(ctx context.Context, domain string, project string, opts service.Fin
openlogging.Error("find kv failed, deadline exceeded", openlogging.WithTags(openlogging.Tags{
"timeout": opts.Timeout,
}))
- return nil, fmt.Errorf("can not find kv in %s", opts.Timeout)
+ return nil, 0, fmt.Errorf("can not find kv in %s", opts.Timeout)
}
- return nil, err
+ return nil, 0, err
}
- return cur, err
+ return cur, int(curTotal), err
}
func findOneKey(ctx context.Context, filter bson.M) ([]*model.KVDoc, error) {
collection := session.GetDB().Collection(session.CollectionKV)
diff --git a/server/service/mongo/kv/kv_service.go b/server/service/mongo/kv/kv_service.go
index d0b3393..9ecfb61 100644
--- a/server/service/mongo/kv/kv_service.go
+++ b/server/service/mongo/kv/kv_service.go
@@ -31,12 +31,6 @@ import (
"github.com/go-mesh/openlogging"
)
-//const
-const (
- existKvLimit = 2
- existKvOffset = 0
-)
-
//Service operate data in mongodb
type Service struct {
timeout time.Duration
@@ -123,9 +117,7 @@ func (s *Service) Exist(ctx context.Context, domain, key string, project string,
kvs, err := s.FindKV(ctx, domain, project,
service.WithExactLabels(),
service.WithLabels(opts.Labels),
- service.WithKey(key),
- service.WithLimit(existKvLimit),
- service.WithOffset(existKvOffset))
+ service.WithKey(key))
if err != nil {
openlogging.Error(err.Error())
return nil, err
@@ -166,7 +158,7 @@ func (s *Service) List(ctx context.Context, domain, project string, options ...s
for _, o := range options {
o(&opts)
}
- cur, err := findKV(ctx, domain, project, opts)
+ cur, total, err := findKV(ctx, domain, project, opts)
if err != nil {
return nil, err
}
@@ -186,6 +178,7 @@ func (s *Service) List(ctx context.Context, domain, project string, options ...s
clearPart(curKV)
result.Data = append(result.Data, curKV)
}
+ result.Total = total
if len(result.Data) == 0 {
return nil, service.ErrKeyNotExists
}
@@ -210,7 +203,7 @@ func (s *Service) FindKV(ctx context.Context, domain string, project string, opt
return nil, session.ErrMissingProject
}
- cur, err := findKV(ctx, domain, project, opts)
+ cur, _, err := findKV(ctx, domain, project, opts)
if err != nil {
return nil, err
}
diff --git a/server/service/mongo/view/view_service.go b/server/service/mongo/view/view_service.go
index 9b706fb..5ca6fdf 100644
--- a/server/service/mongo/view/view_service.go
+++ b/server/service/mongo/view/view_service.go
@@ -120,11 +120,9 @@ func (s *Service) List(ctx context.Context, domain, project string, opts ...serv
collection := session.GetDB().Collection(session.CollectionView)
filter := bson.M{"domain": domain, "project": project}
mOpt := options.Find()
- if option.Limit != 0 {
- mOpt = mOpt.SetLimit(option.Limit)
- }
- if option.Offset != 0 {
- mOpt = mOpt.SetSkip(option.Offset)
+ if option.PageNum != 0 && option.PageSize != 0 {
+ mOpt = mOpt.SetLimit(option.PageSize)
+ mOpt = mOpt.SetSkip(option.PageSize * (option.PageNum - 1))
}
cur, err := collection.Find(ctx, filter, mOpt)
if err != nil {
@@ -150,11 +148,9 @@ func (s *Service) GetContent(ctx context.Context, id, domain, project string, op
o(&option)
}
mOpt := options.Find()
- if option.Limit != 0 {
- mOpt = mOpt.SetLimit(option.Limit)
- }
- if option.Offset != 0 {
- mOpt = mOpt.SetSkip(option.Offset)
+ if option.PageNum != 0 && option.PageSize != 0 {
+ mOpt = mOpt.SetLimit(option.PageSize)
+ mOpt = mOpt.SetSkip(option.PageSize * (option.PageNum - 1))
}
collection := session.GetDB().Collection(generateViewName(id, domain, project))
cur, err := collection.Find(ctx, bson.D{}, mOpt)
diff --git a/server/service/options.go b/server/service/options.go
index 5022d9f..bca3a90 100644
--- a/server/service/options.go
+++ b/server/service/options.go
@@ -39,8 +39,8 @@ type FindOptions struct {
LabelID string
ClearLabel bool
Timeout time.Duration
- Limit int64
- Offset int64
+ PageNum int64
+ PageSize int64
}
//FindOption is functional option to find key value
@@ -102,16 +102,16 @@ func WithOutLabelField() FindOption {
}
}
-//WithLimit tells service paging limit
-func WithLimit(l int64) FindOption {
+//WithPageNum tells service paging limit
+func WithPageNum(l int64) FindOption {
return func(o *FindOptions) {
- o.Limit = l
+ o.PageNum = l
}
}
-//WithOffset tells service paging offset
-func WithOffset(os int64) FindOption {
+//WithPageSize tells service paging offset
+func WithPageSize(os int64) FindOption {
return func(o *FindOptions) {
- o.Offset = os
+ o.PageSize = os
}
}
diff --git a/server/service/service.go b/server/service/service.go
index 8d06331..c1451d5 100644
--- a/server/service/service.go
+++ b/server/service/service.go
@@ -51,7 +51,7 @@ type KV interface {
//History provide api of History entity
type History interface {
- GetHistory(ctx context.Context, keyID string, options ...FindOption) ([]*model.KVDoc, error)
+ GetHistory(ctx context.Context, keyID string, options ...FindOption) ([]*model.KVDoc, int, error)
}
//Revision is global revision number management