You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@servicecomb.apache.org by ti...@apache.org on 2020/03/20 02:02:35 UTC
[servicecomb-kie] branch master updated: #79 add get interface for
polling data (#98)
This is an automated email from the ASF dual-hosted git repository.
tianxiaoliang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/servicecomb-kie.git
The following commit(s) were added to refs/heads/master by this push:
new c898a7e #79 add get interface for polling data (#98)
c898a7e is described below
commit c898a7e5412becb4b22b38f8e85980794369bd59
Author: GuoYL <53...@users.noreply.github.com>
AuthorDate: Fri Mar 20 10:02:26 2020 +0800
#79 add get interface for polling data (#98)
* add get interface for polling data
* change header
* db modify
* fix update failed
* modify as comment
* modify as comment
* go checker
* add handler to test
* modify
* when header is empty don't record
---
deployments/db.js | 16 +++---
pkg/common/common.go | 5 ++
pkg/model/db_schema.go | 14 +++++
pkg/model/kv.go | 20 +++----
server/handler/track_handler.go | 17 +++---
server/resource/v1/common.go | 2 +-
server/resource/v1/doc_struct.go | 25 +++++++++
server/resource/v1/history_resource.go | 62 ++++++++++++++++++++++
server/resource/v1/history_resource_test.go | 39 ++++++++++++++
server/resource/v1/kv_resource_test.go | 3 --
.../mongo/{record => track}/polling_detail_dao.go | 46 +++++++++++++++-
server/service/service.go | 1 +
12 files changed, 209 insertions(+), 41 deletions(-)
diff --git a/deployments/db.js b/deployments/db.js
index 0c8acb2..a3304df 100644
--- a/deployments/db.js
+++ b/deployments/db.js
@@ -98,21 +98,20 @@ db.createCollection( "view", {
} );
db.createCollection( "polling_detail", {
+ capped: true,
+ max: 100,
validator: { $jsonSchema: {
bsonType: "object",
- required: [ "id","params","session_id","url_path" ],
+ required: [ "id","session_id","domain","url_path" ],
properties: {
id: {
bsonType: "string",
},
- session_id: {
- bsonType: "string",
- },
domain: {
bsonType: "string",
},
params: {
- bsonType: "string"
+ bsonType: "object"
},
ip: {
bsonType: "string"
@@ -120,9 +119,6 @@ db.createCollection( "polling_detail", {
user_agent: {
bsonType: "string"
},
- url_path: {
- bsonType: "string"
- },
response_body: {
bsonType: "object"
},
@@ -130,7 +126,7 @@ db.createCollection( "polling_detail", {
bsonType: "object"
},
response_code: {
- bsonType: "string"
+ bsonType: "number"
}
}
} }
@@ -142,7 +138,7 @@ db.kv.createIndex({key: 1, label_id: 1,domain:1,project:1},{ unique: true });
db.label.createIndex({"id": 1}, { unique: true } );
db.label.createIndex({format: 1,domain:1,project:1},{ unique: true });
db.polling_detail.createIndex({"id": 1}, { unique: true } );
-db.polling_detail.createIndex({session:1,domain:1}, { unique: true } );
+db.polling_detail.createIndex({session_id:1,domain:1}, { unique: true } );
db.view.createIndex({"id": 1}, { unique: true } );
db.view.createIndex({display:1,domain:1,project:1},{ unique: true });
//db config
diff --git a/pkg/common/common.go b/pkg/common/common.go
index 191d627..06f1118 100644
--- a/pkg/common/common.go
+++ b/pkg/common/common.go
@@ -31,6 +31,11 @@ const (
QueryParamStatus = "status"
QueryParamOffset = "offset"
QueryParamLimit = "limit"
+ //polling data
+ QueryParamSessionID = "sessionId"
+ QueryParamIP = "ip"
+ QueryParamURLPath = "urlPath"
+ QueryParamUserAgent = "userAgent"
)
//http headers
diff --git a/pkg/model/db_schema.go b/pkg/model/db_schema.go
index 950e0cd..e786a85 100644
--- a/pkg/model/db_schema.go
+++ b/pkg/model/db_schema.go
@@ -55,3 +55,17 @@ type ViewDoc struct {
Domain string `json:"domain,omitempty" yaml:"domain,omitempty"`
Criteria string `json:"criteria,omitempty" yaml:"criteria,omitempty"`
}
+
+//PollingDetail is db struct, it record operation history
+type PollingDetail struct {
+ ID string `json:"id,omitempty" yaml:"id,omitempty"`
+ SessionID string `json:"session_id,omitempty" bson:"session_id," yaml:"session_id,omitempty"`
+ Domain string `json:"domain,omitempty" yaml:"domain,omitempty"`
+ PollingData map[string]interface{} `json:"params,omitempty" yaml:"params,omitempty"`
+ IP string `json:"ip,omitempty" yaml:"ip,omitempty"`
+ UserAgent string `json:"user_agent,omitempty" bson:"user_agent," yaml:"user_agent,omitempty"`
+ URLPath string `json:"url_path,omitempty" bson:"url_path," yaml:"url_path,omitempty"`
+ ResponseBody interface{} `json:"response_body,omitempty" bson:"response_body," yaml:"response_body,omitempty"`
+ ResponseHeader map[string][]string `json:"response_header,omitempty" bson:"response_header," yaml:"response_header,omitempty"`
+ ResponseCode int `json:"response_code,omitempty" bson:"response_code," yaml:"response_code,omitempty"`
+}
diff --git a/pkg/model/kv.go b/pkg/model/kv.go
index b002037..71d20cd 100644
--- a/pkg/model/kv.go
+++ b/pkg/model/kv.go
@@ -53,20 +53,6 @@ type ViewResponse struct {
Data []*ViewDoc `json:"data,omitempty"`
}
-//PollingDetail record operation history
-type PollingDetail struct {
- ID string `json:"id,omitempty" yaml:"id,omitempty"`
- SessionID string `json:"session_id,omitempty" yaml:"session_id,omitempty"`
- Domain string `json:"domain,omitempty" yaml:"domain,omitempty"`
- PollingData map[string]interface{} `json:"params,omitempty" yaml:"params,omitempty"`
- IP string `json:"ip,omitempty" yaml:"ip,omitempty"`
- UserAgent string `json:"user_agent,omitempty" yaml:"user_agent,omitempty"`
- URLPath string `json:"url_path,omitempty" yaml:"url_path,omitempty"`
- ResponseBody interface{} `json:"response_body,omitempty" yaml:"response_body,omitempty"`
- ResponseHeader map[string][]string `json:"response_header,omitempty" yaml:"response_header,omitempty"`
- ResponseCode int `json:"response_code,omitempty" yaml:"response_code,omitempty"`
-}
-
//DocResponseSingleKey is response doc
type DocResponseSingleKey struct {
CreateRevision int64 `json:"create_revision"`
@@ -87,6 +73,12 @@ type DocResponseGetKey struct {
Total int64 `json:"total"`
}
+//PollingDataResponse is response doc
+type PollingDataResponse struct {
+ Data []*PollingDetail `json:"data"`
+ Total int `json:"total"`
+}
+
//DocHealthCheck is response doc
type DocHealthCheck struct {
Version string `json:"version"`
diff --git a/server/handler/track_handler.go b/server/handler/track_handler.go
index f85df93..3d6979d 100644
--- a/server/handler/track_handler.go
+++ b/server/handler/track_handler.go
@@ -22,7 +22,7 @@ import (
"github.com/apache/servicecomb-kie/pkg/iputil"
"github.com/apache/servicecomb-kie/pkg/model"
"github.com/apache/servicecomb-kie/server/resource/v1"
- "github.com/apache/servicecomb-kie/server/service/mongo/record"
+ "github.com/apache/servicecomb-kie/server/service/mongo/track"
"github.com/emicklei/go-restful"
"github.com/go-chassis/go-chassis/core/handler"
"github.com/go-chassis/go-chassis/core/invocation"
@@ -60,14 +60,7 @@ func (h *TrackHandler) Handle(chain *handler.Chain, inv *invocation.Invocation,
return
}
chain.Next(inv, func(ir *invocation.Response) error {
- resp, ok := ir.Result.(*restful.Response)
- if !ok {
- err := cb(ir)
- if err != nil {
- return err
- }
- return nil
- }
+ resp, _ := ir.Result.(*restful.Response)
revStr := req.QueryParameter(common.QueryParamRev)
wait := req.QueryParameter(common.QueryParamWait)
data := &model.PollingDetail{}
@@ -78,14 +71,16 @@ func (h *TrackHandler) Handle(chain *handler.Chain, inv *invocation.Invocation,
data.IP = iputil.ClientIP(req.Request)
data.ResponseBody = inv.Ctx.Value(common.RespBodyContextKey)
data.ResponseCode = ir.Status
- data.ResponseHeader = resp.Header()
+ if resp != nil {
+ data.ResponseHeader = resp.Header()
+ }
data.PollingData = map[string]interface{}{
"revStr": revStr,
"wait": wait,
"project": req.HeaderParameter(v1.PathParameterProject),
"labels": req.QueryParameter("label"),
}
- _, err := record.CreateOrUpdate(inv.Ctx, data)
+ _, err := track.CreateOrUpdate(inv.Ctx, data)
if err != nil {
openlogging.Warn("record polling detail failed" + err.Error())
err := cb(ir)
diff --git a/server/resource/v1/common.go b/server/resource/v1/common.go
index 957308f..2d48b79 100644
--- a/server/resource/v1/common.go
+++ b/server/resource/v1/common.go
@@ -40,7 +40,7 @@ import (
//const of server
const (
HeaderUserAgent = "User-Agent"
- HeaderSessionID = "sessionID"
+ HeaderSessionID = "X-Session-Id"
PathParameterProject = "project"
PathParameterKey = "key"
AttributeDomainKey = "domain"
diff --git a/server/resource/v1/doc_struct.go b/server/resource/v1/doc_struct.go
index 5c3e35c..1a5aebb 100644
--- a/server/resource/v1/doc_struct.go
+++ b/server/resource/v1/doc_struct.go
@@ -101,6 +101,31 @@ var (
ParamType: goRestful.QueryParameterKind,
Desc: "pagination",
}
+ //polling data
+ DocQuerySessionIDParameters = &restful.Parameters{
+ DataType: "string",
+ Name: common.QueryParamSessionID,
+ ParamType: goRestful.QueryParameterKind,
+ Desc: "sessionId is the Unique identification of the client",
+ }
+ DocQueryIPParameters = &restful.Parameters{
+ DataType: "string",
+ Name: common.QueryParamIP,
+ ParamType: goRestful.QueryParameterKind,
+ Desc: "client ip",
+ }
+ DocQueryURLPathParameters = &restful.Parameters{
+ DataType: "string",
+ Name: common.QueryParamURLPath,
+ ParamType: goRestful.QueryParameterKind,
+ Desc: "address of the call",
+ }
+ DocQueryUserAgentParameters = &restful.Parameters{
+ DataType: "string",
+ Name: common.QueryParamUserAgent,
+ ParamType: goRestful.QueryParameterKind,
+ Desc: "user agent of the call",
+ }
)
//swagger doc path params
diff --git a/server/resource/v1/history_resource.go b/server/resource/v1/history_resource.go
index 32376c1..9b6269a 100644
--- a/server/resource/v1/history_resource.go
+++ b/server/resource/v1/history_resource.go
@@ -19,6 +19,7 @@ package v1
import (
"github.com/apache/servicecomb-kie/pkg/model"
+ "github.com/apache/servicecomb-kie/server/service/mongo/track"
"github.com/go-chassis/go-chassis/pkg/runtime"
"net/http"
"strconv"
@@ -75,6 +76,49 @@ func (r *HistoryResource) GetRevisions(context *restful.Context) {
}
}
+//GetPollingData get the record of the get or list history
+func (r *HistoryResource) GetPollingData(context *restful.Context) {
+ query := &model.PollingDetail{}
+ sessionID := context.ReadQueryParameter(common.QueryParamSessionID)
+ if sessionID != "" {
+ query.SessionID = sessionID
+ }
+ ip := context.ReadQueryParameter(common.QueryParamIP)
+ if ip != "" {
+ query.IP = ip
+ }
+ urlPath := context.ReadQueryParameter(common.QueryParamURLPath)
+ if urlPath != "" {
+ query.URLPath = urlPath
+ }
+ userAgent := context.ReadQueryParameter(common.QueryParamUserAgent)
+ if userAgent != "" {
+ query.UserAgent = userAgent
+ }
+ domain := ReadDomain(context)
+ if domain == nil {
+ WriteErrResponse(context, http.StatusInternalServerError, common.MsgDomainMustNotBeEmpty, common.ContentTypeText)
+ return
+ }
+ query.Domain = domain.(string)
+ records, err := track.Get(context.Ctx, query)
+ if err != nil {
+ if err == service.ErrRecordNotExists {
+ WriteErrResponse(context, http.StatusNotFound, err.Error(), common.ContentTypeText)
+ return
+ }
+ WriteErrResponse(context, http.StatusInternalServerError, err.Error(), common.ContentTypeText)
+ return
+ }
+ resp := &model.PollingDataResponse{}
+ resp.Data = records
+ resp.Total = len(records)
+ err = writeResponse(context, resp)
+ if err != nil {
+ openlogging.Error(err.Error())
+ }
+}
+
//HealthCheck provider version info and time info
func (r *HistoryResource) HealthCheck(context *restful.Context) {
domain := ReadDomain(context)
@@ -128,5 +172,23 @@ func (r *HistoryResource) URLPatterns() []restful.Route {
Consumes: []string{goRestful.MIME_JSON, common.ContentTypeYaml},
Produces: []string{goRestful.MIME_JSON, common.ContentTypeYaml},
},
+ {
+ Method: http.MethodGet,
+ Path: "/v1/{project}/kie/track",
+ ResourceFunc: r.GetPollingData,
+ FuncDesc: "get polling tracks of clients of kie server",
+ Parameters: []*restful.Parameters{
+ DocPathProject, DocQuerySessionIDParameters, DocQueryIPParameters, DocQueryURLPathParameters, DocQueryUserAgentParameters,
+ },
+ Returns: []*restful.Returns{
+ {
+ Code: http.StatusOK,
+ Message: "true",
+ Model: []model.PollingDataResponse{},
+ },
+ },
+ Consumes: []string{goRestful.MIME_JSON, common.ContentTypeYaml},
+ Produces: []string{goRestful.MIME_JSON, common.ContentTypeYaml},
+ },
}
}
diff --git a/server/resource/v1/history_resource_test.go b/server/resource/v1/history_resource_test.go
index 2a05a56..e7f9ed8 100644
--- a/server/resource/v1/history_resource_test.go
+++ b/server/resource/v1/history_resource_test.go
@@ -85,6 +85,45 @@ func TestHistoryResource_GetRevisions(t *testing.T) {
}
+func TestHistoryResource_GetPollingData(t *testing.T) {
+ t.Run("list kv by service label, to create a polling data", func(t *testing.T) {
+ r, _ := http.NewRequest("GET", "/v1/test/kie/kv", nil)
+ noopH := &handler2.NoopAuthHandler{}
+ noopH2 := &handler2.TrackHandler{}
+ chain, _ := handler.CreateChain(common.Provider, "testchain3", noopH.Name(), noopH2.Name())
+ r.Header.Set("Content-Type", "application/json")
+ r.Header.Set("X-Session-Id", "test")
+ kvr := &v1.KVResource{}
+ c, err := restfultest.New(kvr, chain)
+ assert.NoError(t, err)
+ resp := httptest.NewRecorder()
+ c.ServeHTTP(resp, r)
+ body, err := ioutil.ReadAll(resp.Body)
+ assert.NoError(t, err)
+ result := &model.KVResponse{}
+ err = json.Unmarshal(body, result)
+ assert.NoError(t, err)
+ })
+ t.Run("get polling data", func(t *testing.T) {
+ r, _ := http.NewRequest("GET", "/v1/test/kie/track?sessionId=test", nil)
+ noopH := &handler2.NoopAuthHandler{}
+ chain, _ := handler.CreateChain(common.Provider, "testchain3", noopH.Name())
+ r.Header.Set("Content-Type", "application/json")
+ revision := &v1.HistoryResource{}
+ c, err := restfultest.New(revision, chain)
+ assert.NoError(t, err)
+ resp := httptest.NewRecorder()
+ c.ServeHTTP(resp, r)
+ body, err := ioutil.ReadAll(resp.Body)
+ assert.NoError(t, err)
+ result := &model.PollingDataResponse{}
+ err = json.Unmarshal(body, result)
+ assert.NoError(t, err)
+ assert.NotEmpty(t, result.Data)
+ })
+
+}
+
func Test_HeathCheck(t *testing.T) {
path := fmt.Sprintf("/v1/health")
r, _ := http.NewRequest("GET", path, nil)
diff --git a/server/resource/v1/kv_resource_test.go b/server/resource/v1/kv_resource_test.go
index 35aa46f..e0450d7 100644
--- a/server/resource/v1/kv_resource_test.go
+++ b/server/resource/v1/kv_resource_test.go
@@ -76,7 +76,6 @@ func TestKVResource_Put(t *testing.T) {
noopH := &handler2.NoopAuthHandler{}
chain, _ := handler.CreateChain(common.Provider, "testchain1", noopH.Name())
r.Header.Set("Content-Type", "application/json")
- r.Header.Set("sessionID", "test")
kvr := &v1.KVResource{}
c, _ := restfultest.New(kvr, chain)
resp := httptest.NewRecorder()
@@ -101,7 +100,6 @@ func TestKVResource_Put(t *testing.T) {
noopH := &handler2.NoopAuthHandler{}
chain, _ := handler.CreateChain(common.Provider, "testchain1", noopH.Name())
r.Header.Set("Content-Type", "application/json")
- r.Header.Set("sessionID", "test")
kvr := &v1.KVResource{}
c, _ := restfultest.New(kvr, chain)
resp := httptest.NewRecorder()
@@ -127,7 +125,6 @@ func TestKVResource_Put(t *testing.T) {
noopH := &handler2.NoopAuthHandler{}
chain, _ := handler.CreateChain(common.Provider, "testchain1", noopH.Name())
r.Header.Set("Content-Type", "application/json")
- r.Header.Set("sessionID", "test")
kvr := &v1.KVResource{}
c, _ := restfultest.New(kvr, chain)
resp := httptest.NewRecorder()
diff --git a/server/service/mongo/record/polling_detail_dao.go b/server/service/mongo/track/polling_detail_dao.go
similarity index 57%
rename from server/service/mongo/record/polling_detail_dao.go
rename to server/service/mongo/track/polling_detail_dao.go
index 6cefe5c..998931d 100644
--- a/server/service/mongo/record/polling_detail_dao.go
+++ b/server/service/mongo/track/polling_detail_dao.go
@@ -15,12 +15,14 @@
* limitations under the License.
*/
-package record
+package track
import (
"context"
"github.com/apache/servicecomb-kie/pkg/model"
+ "github.com/apache/servicecomb-kie/server/service"
"github.com/apache/servicecomb-kie/server/service/mongo/session"
+ "github.com/go-mesh/openlogging"
uuid "github.com/satori/go.uuid"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
@@ -42,9 +44,49 @@ func CreateOrUpdate(ctx context.Context, detail *model.PollingDetail) (*model.Po
}
return nil, res.Err()
}
- _, err := collection.UpdateOne(ctx, queryFilter, detail)
+ _, err := collection.UpdateOne(ctx, queryFilter, bson.D{{"$set", detail}})
if err != nil {
return nil, err
}
return detail, nil
}
+
+//Get is to get a
+func Get(ctx context.Context, detail *model.PollingDetail) ([]*model.PollingDetail, error) {
+ collection := session.GetDB().Collection(session.CollectionPollingDetail)
+ queryFilter := bson.M{"domain": detail.Domain}
+ if detail.SessionID != "" {
+ queryFilter["session_id"] = detail.SessionID
+ }
+ if detail.IP != "" {
+ queryFilter["ip"] = detail.IP
+ }
+ if detail.UserAgent != "" {
+ queryFilter["user_agent"] = detail.UserAgent
+ }
+ if detail.URLPath != "" {
+ queryFilter["url_path"] = detail.URLPath
+ }
+ cur, err := collection.Find(ctx, queryFilter)
+ if err != nil {
+ return nil, err
+ }
+ defer cur.Close(ctx)
+ if cur.Err() != nil {
+ return nil, err
+ }
+ records := make([]*model.PollingDetail, 0)
+ for cur.Next(ctx) {
+ curRecord := &model.PollingDetail{}
+ if err := cur.Decode(curRecord); err != nil {
+ openlogging.Error("decode to KVs error: " + err.Error())
+ return nil, err
+ }
+ curRecord.Domain = ""
+ records = append(records, curRecord)
+ }
+ if len(records) == 0 {
+ return nil, service.ErrRecordNotExists
+ }
+ return records, nil
+}
diff --git a/server/service/service.go b/server/service/service.go
index c1451d5..5720a6c 100644
--- a/server/service/service.go
+++ b/server/service/service.go
@@ -35,6 +35,7 @@ var (
//db errors
var (
ErrKeyNotExists = errors.New("can not find any key value")
+ ErrRecordNotExists = errors.New("can not find any polling data")
ErrRevisionNotExist = errors.New("revision does not exist")
ErrAliasNotGiven = errors.New("label alias not given")
)