You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@servicecomb.apache.org by as...@apache.org on 2019/06/27 11:53:26 UTC
[servicecomb-kie] 15/29: add license, unify response kv struct
This is an automated email from the ASF dual-hosted git repository.
asifdxtreme pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/servicecomb-kie.git
commit 95972504161e9920fead916288bf817f22d9eada
Author: tian <xi...@gmail.com>
AuthorDate: Thu Jun 6 17:27:05 2019 +0800
add license, unify response kv struct
---
pkg/common/common.go | 1 +
pkg/model/kv.go | 6 ++---
pkg/model/mongodb_doc.go | 2 +-
proxy.sh | 15 ++++++++++-
scripts/start.sh | 18 ++++++++++++-
server/dao/kie_api.go | 53 ++++++++++++++++++++++++++++++++++-----
server/dao/kv.go | 2 +-
server/dao/kv_test.go | 20 ++++++++++++---
server/dao/options.go | 8 ++++++
server/resource/v1/common.go | 16 +++++++++++-
server/resource/v1/kv_resource.go | 26 +++++++++++++------
11 files changed, 141 insertions(+), 26 deletions(-)
diff --git a/pkg/common/common.go b/pkg/common/common.go
index 47e224e..345086a 100644
--- a/pkg/common/common.go
+++ b/pkg/common/common.go
@@ -24,4 +24,5 @@ const (
const (
HeaderMatch = "X-Match"
+ HeaderDepth = "X-Depth"
)
diff --git a/pkg/model/kv.go b/pkg/model/kv.go
index be9830e..332c074 100644
--- a/pkg/model/kv.go
+++ b/pkg/model/kv.go
@@ -17,9 +17,7 @@
package model
-
-
-
-
type KVResponse struct {
+ LabelDoc *LabelDoc `json:"labels"`
+ Data []*KVDoc `json:"data"`
}
diff --git a/pkg/model/mongodb_doc.go b/pkg/model/mongodb_doc.go
index a298ca2..7645a14 100644
--- a/pkg/model/mongodb_doc.go
+++ b/pkg/model/mongodb_doc.go
@@ -22,7 +22,7 @@ import "go.mongodb.org/mongo-driver/bson/primitive"
type LabelDoc struct {
ID primitive.ObjectID `json:"_id,omitempty" bson:"_id,omitempty"`
Labels map[string]string `json:"labels,omitempty"`
- Revision int `json:"revision"`
+ Revision int `json:"revision,omitempty"`
Domain string `json:"domain,omitempty"` //tenant info
}
type KVDoc struct {
diff --git a/proxy.sh b/proxy.sh
index da4430b..5fcc641 100755
--- a/proxy.sh
+++ b/proxy.sh
@@ -1,3 +1,16 @@
#!/usr/bin/env bash
-
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
export GOPROXY=https://goproxy.io
diff --git a/scripts/start.sh b/scripts/start.sh
index 35e62ea..682634c 100755
--- a/scripts/start.sh
+++ b/scripts/start.sh
@@ -19,7 +19,9 @@
root_dir=/opt/servicecomb-kie
net_name=$(ip -o -4 route show to default | awk '{print $5}')
listen_addr=$(ifconfig ${net_name} | grep -E 'inet\W' | grep -o -E [0-9]+.[0-9]+.[0-9]+.[0-9]+ | head -n 1)
-
+if [ -z "${LOG_LEVEL}" ]; then
+ export LOG_LEVEL="DEBUG"
+fi
writeConfig(){
echo "write template config..."
@@ -38,7 +40,21 @@ cse:
Provider:
default: auth-handler,ratelimiter-provider
EOM
+cat <<EOM > ${root_dir}/conf/lager.yaml
+logger_level: ${LOG_LEVEL}
+
+logger_file: log/chassis.log
+
+log_format_text: true
+
+rollingPolicy: size
+log_rotate_date: 1
+
+log_rotate_size: 10
+
+log_backup_count: 7
+EOM
cat <<EOM > /etc/servicecomb-kie/kie-conf.yaml
db:
uri: mongodb://${MONGODB_USER}:${MONGODB_PWD}@${MONGODB_ADDR}/kie
diff --git a/server/dao/kie_api.go b/server/dao/kie_api.go
index c5ae9bd..aaf613f 100644
--- a/server/dao/kie_api.go
+++ b/server/dao/kie_api.go
@@ -26,6 +26,7 @@ import (
"go.mongodb.org/mongo-driver/bson/primitive"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
+ "reflect"
"time"
)
@@ -184,7 +185,7 @@ func (s *MongodbService) FindKVByLabelID(ctx context.Context, domain, labelID, k
//FindKV get kvs by key, labels
//because labels has a a lot of combination,
//you can use WithExactLabels to return only one kv which's labels exactly match the criteria
-func (s *MongodbService) FindKV(ctx context.Context, domain string, options ...FindOption) ([]*model.KVDoc, error) {
+func (s *MongodbService) FindKV(ctx context.Context, domain string, options ...FindOption) ([]*model.KVResponse, error) {
opts := FindOptions{}
for _, o := range options {
o(&opts)
@@ -213,9 +214,11 @@ func (s *MongodbService) FindKV(ctx context.Context, domain string, options ...F
if cur.Err() != nil {
return nil, err
}
+ kvResp := make([]*model.KVResponse, 0)
if opts.ExactLabels {
openlogging.Debug(fmt.Sprintf("find one [%s] with lables [%s] in [%s]", opts.Key, opts.Labels, domain))
curKV := &model.KVDoc{} //reuse this pointer to reduce GC, only clear label
+
//check label length to get the exact match
for cur.Next(ctx) { //although complexity is O(n), but there won't be so much labels for one key
curKV.Labels = nil
@@ -226,27 +229,65 @@ func (s *MongodbService) FindKV(ctx context.Context, domain string, options ...F
}
if len(curKV.Labels) == len(opts.Labels) {
openlogging.Debug("hit exact labels")
+ curKV.Domain = ""
curKV.Labels = nil //exact match don't need to return labels
- return []*model.KVDoc{curKV}, nil
+ labelGroup := &model.KVResponse{
+ LabelDoc: &model.LabelDoc{
+ Labels: opts.Labels,
+ ID: primitive.NilObjectID,
+ },
+ Data: make([]*model.KVDoc, 0),
+ }
+ labelGroup.Data = append(labelGroup.Data, curKV)
+ kvResp = append(kvResp, labelGroup)
+ return kvResp, nil
}
}
return nil, ErrKeyNotExists
} else {
- kvs := make([]*model.KVDoc, 0)
+ if opts.Depth == 0 {
+ opts.Depth = 1
+ }
for cur.Next(ctx) {
curKV := &model.KVDoc{}
+
if err := cur.Decode(curKV); err != nil {
openlogging.Error("decode to KVs error: " + err.Error())
return nil, err
}
- kvs = append(kvs, curKV)
+ if (len(curKV.Labels) - len(opts.Labels)) > opts.Depth {
+ //because it is query by labels, so result can not be minus
+ //so many labels,then continue
+ openlogging.Debug("so deep, skip this key")
+ continue
+ }
+ openlogging.Info(fmt.Sprintf("%v", curKV))
+ var groupExist bool
+ var labelGroup *model.KVResponse
+ for _, labelGroup = range kvResp {
+ if reflect.DeepEqual(labelGroup.LabelDoc.Labels, curKV.Labels) {
+ groupExist = true
+ labelGroup.Data = append(labelGroup.Data, curKV)
+ break
+ }
+ }
+ if !groupExist {
+ labelGroup = &model.KVResponse{
+ LabelDoc: &model.LabelDoc{
+ Labels: curKV.Labels,
+ },
+ Data: []*model.KVDoc{curKV},
+ }
+ openlogging.Debug("add new label group")
+ }
+ kvResp = append(kvResp, labelGroup)
}
- if len(kvs) == 0 {
+ if len(kvResp) == 0 {
return nil, ErrKeyNotExists
}
- return kvs, nil
+ return kvResp, nil
}
}
diff --git a/server/dao/kv.go b/server/dao/kv.go
index 1f7eccf..c48774b 100644
--- a/server/dao/kv.go
+++ b/server/dao/kv.go
@@ -99,7 +99,7 @@ func (s *MongodbService) KVExist(ctx context.Context, domain, key string, option
return primitive.NilObjectID, ErrTooMany
}
- return kvs[0].ID, nil
+ return kvs[0].Data[0].ID, nil
}
}
diff --git a/server/dao/kv_test.go b/server/dao/kv_test.go
index 7e2a5eb..efbe6d4 100644
--- a/server/dao/kv_test.go
+++ b/server/dao/kv_test.go
@@ -96,7 +96,7 @@ var _ = Describe("Kv mongodb service", func() {
"app": "mall",
}), dao.WithExactLabels())
It("should be 1s", func() {
- Expect(kvs1[0].Value).Should(Equal(beforeKV.Value))
+ Expect(kvs1[0].Data[0].Value).Should(Equal(beforeKV.Value))
})
afterKV, err := s.CreateOrUpdate(context.Background(), "default", &model.KVDoc{
Key: "timeout",
@@ -119,19 +119,33 @@ var _ = Describe("Kv mongodb service", func() {
"app": "mall",
}), dao.WithExactLabels())
It("should be 3s", func() {
- Expect(kvs[0].Value).Should(Equal(afterKV.Value))
+ Expect(kvs[0].Data[0].Value).Should(Equal(afterKV.Value))
})
})
})
Describe("greedy find by kv and labels", func() {
- Context("with labels app ", func() {
+ Context("with labels app,depth is 1 ", func() {
kvs, err := s.FindKV(context.Background(), "default", dao.WithKey("timeout"), dao.WithLabels(map[string]string{
"app": "mall",
}))
It("should not return err", func() {
Expect(err).Should(BeNil())
})
+ It("should has 2 records", func() {
+ Expect(len(kvs)).Should(Equal(2))
+ })
+
+ })
+ Context("with labels app,depth is 2 ", func() {
+ kvs, err := s.FindKV(context.Background(), "default", dao.WithKey("timeout"),
+ dao.WithLabels(map[string]string{
+ "app": "mall",
+ }),
+ dao.WithDepth(2))
+ It("should not return err", func() {
+ Expect(err).Should(BeNil())
+ })
It("should has 3 records", func() {
Expect(len(kvs)).Should(Equal(3))
})
diff --git a/server/dao/options.go b/server/dao/options.go
index e1af384..7e33798 100644
--- a/server/dao/options.go
+++ b/server/dao/options.go
@@ -19,6 +19,7 @@ package dao
type FindOptions struct {
ExactLabels bool
+ Depth int
Key string
Labels map[string]string
LabelID string
@@ -55,6 +56,13 @@ func WithLabelID(label string) FindOption {
}
}
+//WithDepth if you use greedy match this can specify the match depth
+func WithDepth(d int) FindOption {
+ return func(o *FindOptions) {
+ o.Depth = d
+ }
+}
+
//WithOutLabelField will clear all labels attributes in kv doc
func WithOutLabelField() FindOption {
return func(o *FindOptions) {
diff --git a/server/resource/v1/common.go b/server/resource/v1/common.go
index 2293b72..eca9fef 100644
--- a/server/resource/v1/common.go
+++ b/server/resource/v1/common.go
@@ -24,22 +24,36 @@ import (
"github.com/apache/servicecomb-kie/pkg/model"
"github.com/go-chassis/go-chassis/server/restful"
"github.com/go-mesh/openlogging"
+ "strconv"
)
const (
- TenantHeaderParam = "X-Domain-Name"
+ HeaderTenant = "X-Domain-Name"
+
FindExact = "exact"
FindMany = "greedy"
MsgDomainMustNotBeEmpty = "domain must not be empty"
MsgIllegalFindPolicy = "value of header " + common.HeaderMatch + " can be greedy or exact"
MsgIllegalLabels = "label's value can not be empty, " +
"label can not be duplicated, please check your query parameters"
+ MsgIllegalDepth = "X-Depth must be number"
ErrIDMustNotEmpty = "must supply id if you want to remove key"
)
func ReadDomain(context *restful.Context) interface{} {
return context.ReadRestfulRequest().Attribute("domain")
}
+func ReadFindDepth(context *restful.Context) (int, error) {
+ d := context.ReadRestfulRequest().HeaderParameter(common.HeaderDepth)
+ if d == "" {
+ return 1, nil
+ }
+ depth, err := strconv.Atoi(d)
+ if err != nil {
+ return 0, err
+ }
+ return depth, nil
+}
func ReadMatchPolicy(context *restful.Context) string {
policy := context.ReadRestfulRequest().HeaderParameter(common.HeaderMatch)
if policy == "" {
diff --git a/server/resource/v1/kv_resource.go b/server/resource/v1/kv_resource.go
index db2d2b6..7832f23 100644
--- a/server/resource/v1/kv_resource.go
+++ b/server/resource/v1/kv_resource.go
@@ -91,10 +91,15 @@ func (r *KVResource) FindWithKey(context *restful.Context) {
return
}
policy := ReadMatchPolicy(context)
- var kvs []*model.KVDoc
+ d, err := ReadFindDepth(context)
+ if err != nil {
+ WriteErrResponse(context, http.StatusBadRequest, MsgIllegalDepth)
+ return
+ }
+ var kvs []*model.KVResponse
switch policy {
case common.MatchGreedy:
- kvs, err = s.FindKV(context.Ctx, domain.(string), dao.WithKey(key), dao.WithLabels(labels))
+ kvs, err = s.FindKV(context.Ctx, domain.(string), dao.WithKey(key), dao.WithLabels(labels), dao.WithDepth(d))
case common.MatchExact:
kvs, err = s.FindKV(context.Ctx, domain.(string), dao.WithKey(key), dao.WithLabels(labels),
dao.WithExactLabels())
@@ -138,10 +143,15 @@ func (r *KVResource) FindByLabels(context *restful.Context) {
return
}
policy := ReadMatchPolicy(context)
- var kvs []*model.KVDoc
+ d, err := ReadFindDepth(context)
+ if err != nil {
+ WriteErrResponse(context, http.StatusBadRequest, MsgIllegalDepth)
+ return
+ }
+ var kvs []*model.KVResponse
switch policy {
case common.MatchGreedy:
- kvs, err = s.FindKV(context.Ctx, domain.(string), dao.WithLabels(labels))
+ kvs, err = s.FindKV(context.Ctx, domain.(string), dao.WithLabels(labels), dao.WithDepth(d))
case common.MatchExact:
kvs, err = s.FindKV(context.Ctx, domain.(string), dao.WithLabels(labels),
dao.WithExactLabels())
@@ -199,7 +209,7 @@ func (r *KVResource) URLPatterns() []restful.Route {
ParamType: goRestful.PathParameterKind,
}, {
DataType: "string",
- Name: TenantHeaderParam,
+ Name: HeaderTenant,
ParamType: goRestful.HeaderParameterKind,
Desc: "set kv to other tenant",
}, {
@@ -230,7 +240,7 @@ func (r *KVResource) URLPatterns() []restful.Route {
ParamType: goRestful.PathParameterKind,
}, {
DataType: "string",
- Name: TenantHeaderParam,
+ Name: HeaderTenant,
ParamType: goRestful.HeaderParameterKind,
}, {
DataType: "string",
@@ -257,7 +267,7 @@ func (r *KVResource) URLPatterns() []restful.Route {
Parameters: []*restful.Parameters{
{
DataType: "string",
- Name: TenantHeaderParam,
+ Name: HeaderTenant,
ParamType: goRestful.HeaderParameterKind,
}, {
DataType: "string",
@@ -283,7 +293,7 @@ func (r *KVResource) URLPatterns() []restful.Route {
Parameters: []*restful.Parameters{
{
DataType: "string",
- Name: TenantHeaderParam,
+ Name: HeaderTenant,
ParamType: goRestful.HeaderParameterKind,
}, {
DataType: "string",