You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by kl...@apache.org on 2023/05/12 08:33:17 UTC
[incubator-devlake] branch main updated: [feat-4762] part1: Minimal support for deleting scopes (#5153)
This is an automated email from the ASF dual-hosted git repository.
klesh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 70ea96de2 [feat-4762] part1: Minimal support for deleting scopes (#5153)
70ea96de2 is described below
commit 70ea96de2fb7673caafcd1608dcee4c67780f65f
Author: Keon Amini <ke...@merico.dev>
AuthorDate: Fri May 12 01:33:11 2023 -0700
[feat-4762] part1: Minimal support for deleting scopes (#5153)
* feat: Minimal support for deleting scopes
* feat: pagerduty adapted to use Delete Scopes
* fix: fixes added per PR feedback
---
backend/core/models/blueprint.go | 89 ++++++
backend/go.mod | 4 +-
backend/go.sum | 15 -
backend/helpers/pluginhelper/api/api_collector.go | 6 +-
.../helpers/pluginhelper/api/api_collector_test.go | 17 +-
.../pluginhelper/api/api_collector_with_state.go | 7 +-
backend/helpers/pluginhelper/api/api_extractor.go | 1 -
backend/helpers/pluginhelper/api/api_rawdata.go | 26 +-
.../helpers/pluginhelper/api/remote_api_helper.go | 12 +-
backend/helpers/pluginhelper/api/scope_helper.go | 356 ++++++++++++++++++---
.../helpers/pluginhelper/api/scope_helper_test.go | 4 +-
.../pluginhelper}/services/blueprint_helper.go | 115 +++++--
backend/plugins/pagerduty/api/scope.go | 21 +-
.../e2e/raw_tables/_raw_pagerduty_incidents.csv | 6 +-
.../_tool_pagerduty_assignments.csv | 6 +-
.../snapshot_tables/_tool_pagerduty_incidents.csv | 6 +-
.../e2e/snapshot_tables/_tool_pagerduty_users.csv | 4 +-
backend/plugins/pagerduty/impl/impl.go | 17 +-
.../plugins/pagerduty/tasks/incidents_collector.go | 8 +-
.../plugins/pagerduty/tasks/incidents_converter.go | 8 +-
.../plugins/pagerduty/tasks/incidents_extractor.go | 8 +-
backend/plugins/pagerduty/tasks/task_data.go | 8 +
backend/server/api/router.go | 7 +-
backend/server/services/blueprint.go | 30 +-
backend/server/services/init.go | 5 +-
backend/test/e2e/remote/helper.go | 22 +-
backend/test/e2e/remote/python_plugin_test.go | 41 ++-
backend/test/helper/api.go | 46 ++-
backend/test/helper/json_helper.go | 12 +
backend/test/helper/models.go | 7 +
30 files changed, 727 insertions(+), 187 deletions(-)
diff --git a/backend/core/models/blueprint.go b/backend/core/models/blueprint.go
index 2f2643c98..f09b39d29 100644
--- a/backend/core/models/blueprint.go
+++ b/backend/core/models/blueprint.go
@@ -55,6 +55,36 @@ type BlueprintSettings struct {
AfterPlan json.RawMessage `json:"after_plan"`
}
+// UpdateConnections unmarshals the connections on this BlueprintSettings
+func (bps *BlueprintSettings) UnmarshalConnections() ([]*plugin.BlueprintConnectionV200, errors.Error) {
+ var connections []*plugin.BlueprintConnectionV200
+ err := json.Unmarshal(bps.Connections, &connections)
+ if err != nil {
+ return nil, errors.Default.Wrap(err, `unmarshal connections fail`)
+ }
+ return connections, nil
+}
+
+// UpdateConnections updates the connections on this BlueprintSettings reference according to the updater function
+func (bps *BlueprintSettings) UpdateConnections(updater func(c *plugin.BlueprintConnectionV200) errors.Error) errors.Error {
+ conns, err := bps.UnmarshalConnections()
+ if err != nil {
+ return err
+ }
+ for i, conn := range conns {
+ err = updater(conn)
+ if err != nil {
+ return err
+ }
+ conns[i] = conn
+ }
+ bps.Connections, err = errors.Convert01(json.Marshal(&conns))
+ if err != nil {
+ return err
+ }
+ return nil
+}
+
// UnmarshalPlan unmarshals Plan in JSON to strong-typed plugin.PipelinePlan
func (bp *Blueprint) UnmarshalPlan() (plugin.PipelinePlan, errors.Error) {
var plan plugin.PipelinePlan
@@ -65,6 +95,65 @@ func (bp *Blueprint) UnmarshalPlan() (plugin.PipelinePlan, errors.Error) {
return plan, nil
}
+// UnmarshalSettings unmarshals the BlueprintSettings on the Blueprint
+func (bp *Blueprint) UnmarshalSettings() (BlueprintSettings, errors.Error) {
+ var settings BlueprintSettings
+ err := errors.Convert(json.Unmarshal(bp.Settings, &settings))
+ if err != nil {
+ return settings, errors.Default.Wrap(err, `unmarshal settings fail`)
+ }
+ return settings, nil
+}
+
+// GetConnections Gets all the blueprint connections for this blueprint
+func (bp *Blueprint) GetConnections() ([]*plugin.BlueprintConnectionV200, errors.Error) {
+ settings, err := bp.UnmarshalSettings()
+ if err != nil {
+ return nil, err
+ }
+ conns, err := settings.UnmarshalConnections()
+ if err != nil {
+ return nil, err
+ }
+ return conns, nil
+}
+
+// UpdateSettings updates the blueprint instance with this settings reference
+func (bp *Blueprint) UpdateSettings(settings *BlueprintSettings) errors.Error {
+ if settings.Connections == nil {
+ bp.Settings = nil
+ } else {
+ settingsRaw, err := errors.Convert01(json.Marshal(settings))
+ if err != nil {
+ return err
+ }
+ bp.Settings = settingsRaw
+ }
+ return nil
+}
+
+// GetScopes Gets all the scopes across all the connections for this blueprint
+func (bp *Blueprint) GetScopes(connectionId uint64) ([]*plugin.BlueprintScopeV200, errors.Error) {
+ conns, err := bp.GetConnections()
+ if err != nil {
+ return nil, err
+ }
+ visited := map[string]any{}
+ var result []*plugin.BlueprintScopeV200
+ for _, conn := range conns {
+ if conn.ConnectionId != connectionId {
+ continue
+ }
+ for _, scope := range conn.Scopes {
+ if _, ok := visited[scope.Id]; !ok {
+ result = append(result, scope)
+ visited[scope.Id] = true
+ }
+ }
+ }
+ return result, nil
+}
+
func (Blueprint) TableName() string {
return "_devlake_blueprints"
}
diff --git a/backend/go.mod b/backend/go.mod
index 34a700b92..416f8fe91 100644
--- a/backend/go.mod
+++ b/backend/go.mod
@@ -47,9 +47,7 @@ require (
gorm.io/gorm v1.24.1-0.20221019064659-5dd2bb482755
)
-require (
- github.com/jmespath/go-jmespath v0.4.0 // indirect
-)
+require github.com/jmespath/go-jmespath v0.4.0 // indirect
require (
github.com/KyleBanks/depth v1.2.1 // indirect
diff --git a/backend/go.sum b/backend/go.sum
index 352a9013c..0b792721f 100644
--- a/backend/go.sum
+++ b/backend/go.sum
@@ -128,8 +128,6 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc=
-github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.0-20210816181553-5444fa50b93d/go.mod h1:tmAIfUFEirG/Y8jhZ9M+h36obRZAk/1fcSpXwAVlfqE=
github.com/denisenkom/go-mssqldb v0.9.0 h1:RSohk2RsiZqLZ0zCjtfn3S4Gp4exhpBWHyQ7D0yGjAk=
github.com/denisenkom/go-mssqldb v0.9.0/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU=
github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4=
@@ -234,9 +232,6 @@ github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6Wezm
github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM=
github.com/gocarina/gocsv v0.0.0-20220707092902-b9da1f06c77e h1:GMIV+S6grz+vlIaUsP+fedQ6L+FovyMPMY26WO8dwQE=
github.com/gocarina/gocsv v0.0.0-20220707092902-b9da1f06c77e/go.mod h1:5YoVOkjYAQumqlV356Hj3xeYh4BdZuLE0/nRkf2NKkI=
-github.com/goccy/go-json v0.9.7/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
-github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
-github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw=
@@ -505,15 +500,6 @@ github.com/leodido/go-urn v1.1.0/go.mod h1:+cyI34gQWZcE1eQU7NVgKkkzdXDQHr1dBMtdA
github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w=
github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY=
-github.com/lestrrat-go/backoff/v2 v2.0.8/go.mod h1:rHP/q/r9aT27n24JQLa7JhSQZCKBBOiM/uP402WwN8Y=
-github.com/lestrrat-go/blackmagic v1.0.0/go.mod h1:TNgH//0vYSs8VXDCfkZLgIrVTTXQELZffUV0tz3MtdQ=
-github.com/lestrrat-go/httpcc v1.0.1/go.mod h1:qiltp3Mt56+55GPVCbTdM9MlqhvzyuL6W/NMDA8vA5E=
-github.com/lestrrat-go/iter v1.0.1/go.mod h1:zIdgO1mRKhn8l9vrZJZz9TUMMFbQbLeTsbqPDrJ/OJc=
-github.com/lestrrat-go/jwx v1.2.25 h1:tAx93jN2SdPvFn08fHNAhqFJazn5mBBOB8Zli0g0otA=
-github.com/lestrrat-go/jwx v1.2.25/go.mod h1:zoNuZymNl5lgdcu6P7K6ie2QRll5HVfF4xwxBBK1NxY=
-github.com/lestrrat-go/option v1.0.0/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I=
-github.com/lestrrat-go/option v1.0.1 h1:oAzP2fvZGQKWkvHa1/SAcFolBEca1oN+mQ7eooNBEYU=
-github.com/lestrrat-go/option v1.0.1/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I=
github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
@@ -801,7 +787,6 @@ golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm
golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.8.0 h1:pd9TJtTueMTVQXzk8E2XESSMQDj/U7OUu0PqJqPXQjQ=
golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE=
diff --git a/backend/helpers/pluginhelper/api/api_collector.go b/backend/helpers/pluginhelper/api/api_collector.go
index 517025dea..051008e3a 100644
--- a/backend/helpers/pluginhelper/api/api_collector.go
+++ b/backend/helpers/pluginhelper/api/api_collector.go
@@ -354,10 +354,14 @@ func (collector *ApiCollector) fetchPagesUndetermined(reqData *RequestData) {
}
func (collector *ApiCollector) generateUrl(pager *Pager, input interface{}) (string, errors.Error) {
+ params := collector.args.Params
+ if collector.args.Options != nil {
+ params = collector.args.Options.GetParams()
+ }
var buf bytes.Buffer
err := collector.urlTemplate.Execute(&buf, &RequestData{
Pager: pager,
- Params: collector.args.Params,
+ Params: params,
Input: input,
})
if err != nil {
diff --git a/backend/helpers/pluginhelper/api/api_collector_test.go b/backend/helpers/pluginhelper/api/api_collector_test.go
index 8b4c6d187..e90e4b11b 100644
--- a/backend/helpers/pluginhelper/api/api_collector_test.go
+++ b/backend/helpers/pluginhelper/api/api_collector_test.go
@@ -33,6 +33,14 @@ import (
"github.com/stretchr/testify/mock"
)
+type TestOpts struct{}
+
+func (t TestOpts) GetParams() any {
+ return struct {
+ Name string
+ }{Name: "testparams"}
+}
+
func TestFetchPageUndetermined(t *testing.T) {
mockDal := new(mockdal.Dal)
mockDal.On("AutoMigrate", mock.Anything, mock.Anything).Return(nil).Once()
@@ -76,16 +84,13 @@ func TestFetchPageUndetermined(t *testing.T) {
mockApi.On("WaitAsync").Return(nil)
mockApi.On("GetAfterFunction", mock.Anything).Return(nil)
mockApi.On("SetAfterFunction", mock.Anything).Return()
- params := struct {
- Name string
- }{Name: "testparams"}
mockApi.On("Release").Return()
collector, err := NewApiCollector(ApiCollectorArgs{
RawDataSubTaskArgs: RawDataSubTaskArgs{
- Ctx: mockCtx,
- Table: "whatever rawtable",
- Params: params,
+ Ctx: mockCtx,
+ Table: "whatever rawtable",
+ Options: &TestOpts{},
},
ApiClient: mockApi,
Input: mockInput,
diff --git a/backend/helpers/pluginhelper/api/api_collector_with_state.go b/backend/helpers/pluginhelper/api/api_collector_with_state.go
index f62d6b365..65d4e2176 100644
--- a/backend/helpers/pluginhelper/api/api_collector_with_state.go
+++ b/backend/helpers/pluginhelper/api/api_collector_with_state.go
@@ -145,9 +145,10 @@ func (m *ApiCollectorStateManager) Execute() errors.Error {
func NewStatefulApiCollectorForFinalizableEntity(args FinalizableApiCollectorArgs) (plugin.SubTask, errors.Error) {
// create a manager which could execute multiple collector but acts as a single subtask to callers
manager, err := NewStatefulApiCollector(RawDataSubTaskArgs{
- Ctx: args.Ctx,
- Params: args.Params,
- Table: args.Table,
+ Ctx: args.Ctx,
+ Options: args.Options,
+ Params: args.Params,
+ Table: args.Table,
}, args.TimeAfter)
if err != nil {
return nil, err
diff --git a/backend/helpers/pluginhelper/api/api_extractor.go b/backend/helpers/pluginhelper/api/api_extractor.go
index 2b4f86e5d..2a4c89ba6 100644
--- a/backend/helpers/pluginhelper/api/api_extractor.go
+++ b/backend/helpers/pluginhelper/api/api_extractor.go
@@ -106,7 +106,6 @@ func (extractor *ApiExtractor) Execute() errors.Error {
if err != nil {
return errors.Default.Wrap(err, "error calling plugin Extract implementation")
}
-
for _, result := range results {
// get the batch operator for the specific type
batch, err := divider.ForType(reflect.TypeOf(result))
diff --git a/backend/helpers/pluginhelper/api/api_rawdata.go b/backend/helpers/pluginhelper/api/api_rawdata.go
index 24199c35c..d743cd66d 100644
--- a/backend/helpers/pluginhelper/api/api_rawdata.go
+++ b/backend/helpers/pluginhelper/api/api_rawdata.go
@@ -22,6 +22,7 @@ import (
"fmt"
"github.com/apache/incubator-devlake/core/errors"
plugin "github.com/apache/incubator-devlake/core/plugin"
+ "reflect"
"time"
"gorm.io/datatypes"
@@ -37,6 +38,10 @@ type RawData struct {
CreatedAt time.Time
}
+type TaskOptions interface {
+ GetParams() any
+}
+
// RawDataSubTaskArgs FIXME ...
type RawDataSubTaskArgs struct {
Ctx plugin.SubTaskContext
@@ -44,9 +49,12 @@ type RawDataSubTaskArgs struct {
// Table store raw data
Table string `comment:"Raw data table name"`
- // This struct will be JSONEncoded and stored into database along with raw data itself, to identity minimal
- // set of data to be process, for example, we process JiraIssues by Board
- Params interface{} `comment:"To identify a set of records with same UrlTemplate, i.e. {ConnectionId, BoardId} for jira entities"`
+ // Deprecated: Use Options instead
+ // This struct will be JSONEncoded and stored into database along with raw data itself, to identity minimal set of
+ // data to be processed, for example, we process JiraIssues by Board
+ Params any `comment:"To identify a set of records with same UrlTemplate, i.e. {ConnectionId, BoardId} for jira entities"`
+
+ Options TaskOptions `comment:"To identify a set of records with same UrlTemplate, i.e. {ConnectionId, BoardId} for jira entities"`
}
// RawDataSubTask is Common features for raw data sub-tasks
@@ -64,12 +72,18 @@ func NewRawDataSubTask(args RawDataSubTaskArgs) (*RawDataSubTask, errors.Error)
if args.Table == "" {
return nil, errors.Default.New("Table is required for RawDataSubTask")
}
+ var params any
+ if args.Options != nil {
+ params = args.Options.GetParams()
+ } else { // fallback to old way
+ params = args.Params
+ }
paramsString := ""
- if args.Params == nil {
- args.Ctx.GetLogger().Warn(nil, "Missing `Params` for raw data subtask %s", args.Ctx.GetName())
+ if params == nil || reflect.ValueOf(params).IsZero() {
+ args.Ctx.GetLogger().Warn(nil, fmt.Sprintf("Missing `Params` for raw data subtask %s", args.Ctx.GetName()))
} else {
// TODO: maybe sort it to make it consistent
- paramsBytes, err := json.Marshal(args.Params)
+ paramsBytes, err := json.Marshal(params)
if err != nil {
return nil, errors.Default.Wrap(err, "unable to serialize subtask parameters")
}
diff --git a/backend/helpers/pluginhelper/api/remote_api_helper.go b/backend/helpers/pluginhelper/api/remote_api_helper.go
index a34457e38..798801a20 100644
--- a/backend/helpers/pluginhelper/api/remote_api_helper.go
+++ b/backend/helpers/pluginhelper/api/remote_api_helper.go
@@ -135,13 +135,13 @@ func (r *RemoteApiHelper[Conn, Scope, ApiScope, Group]) GetScopesFromRemote(inpu
getGroup func(basicRes coreContext.BasicRes, gid string, queryData *RemoteQueryData, connection Conn) ([]Group, errors.Error),
getScope func(basicRes coreContext.BasicRes, gid string, queryData *RemoteQueryData, connection Conn) ([]ApiScope, errors.Error),
) (*plugin.ApiResourceOutput, errors.Error) {
- connectionId, _ := extractFromReqParam(input.Params)
- if connectionId == 0 {
+ connectionId, err := errors.Convert01(strconv.ParseUint(input.Params["connectionId"], 10, 64))
+ if err != nil || connectionId == 0 {
return nil, errors.BadInput.New("invalid connectionId")
}
var connection Conn
- err := r.connHelper.First(&connection, input.Params)
+ err = r.connHelper.First(&connection, input.Params)
if err != nil {
return nil, err
}
@@ -245,13 +245,13 @@ func (r *RemoteApiHelper[Conn, Scope, ApiScope, Group]) GetScopesFromRemote(inpu
}
func (r *RemoteApiHelper[Conn, Scope, ApiScope, Group]) SearchRemoteScopes(input *plugin.ApiResourceInput, searchScope func(basicRes coreContext.BasicRes, queryData *RemoteQueryData, connection Conn) ([]ApiScope, errors.Error)) (*plugin.ApiResourceOutput, errors.Error) {
- connectionId, _ := extractFromReqParam(input.Params)
- if connectionId == 0 {
+ connectionId, err := errors.Convert01(strconv.ParseUint(input.Params["connectionId"], 10, 64))
+ if err != nil || connectionId == 0 {
return nil, errors.BadInput.New("invalid connectionId")
}
var connection Conn
- err := r.connHelper.First(&connection, input.Params)
+ err = r.connHelper.First(&connection, input.Params)
if err != nil {
return nil, err
}
diff --git a/backend/helpers/pluginhelper/api/scope_helper.go b/backend/helpers/pluginhelper/api/scope_helper.go
index 10cdc37c5..a2edbcf7c 100644
--- a/backend/helpers/pluginhelper/api/scope_helper.go
+++ b/backend/helpers/pluginhelper/api/scope_helper.go
@@ -20,9 +20,13 @@ package api
import (
"encoding/json"
"fmt"
+ "github.com/apache/incubator-devlake/core/models"
+ "github.com/apache/incubator-devlake/core/models/domainlayer/domaininfo"
+ serviceHelper "github.com/apache/incubator-devlake/helpers/pluginhelper/services"
"net/http"
"strconv"
"strings"
+ "sync"
"time"
"github.com/apache/incubator-devlake/core/context"
@@ -37,6 +41,11 @@ import (
"reflect"
)
+var (
+ tablesCache []string // these cached vars can probably be moved somewhere more centralized later
+ tablesCacheLoader = new(sync.Once)
+)
+
type NoTransformation struct{}
// ScopeApiHelper is used to write the CURD of scopes
@@ -44,9 +53,27 @@ type ScopeApiHelper[Conn any, Scope any, Tr any] struct {
log log.Logger
db dal.Dal
validator *validator.Validate
+ bpManager *serviceHelper.BlueprintManager
connHelper *ConnectionApiHelper
}
+type (
+ requestParams struct {
+ connectionId uint64
+ scopeId string
+ plugin string
+ }
+ deleteRequestParams struct {
+ requestParams
+ deleteDataOnly bool
+ }
+
+ getRequestParams struct {
+ requestParams
+ loadBlueprints bool
+ }
+)
+
// NewScopeHelper creates a ScopeHelper for scopes management
func NewScopeHelper[Conn any, Scope any, Tr any](
basicRes context.BasicRes,
@@ -59,10 +86,18 @@ func NewScopeHelper[Conn any, Scope any, Tr any](
if connHelper == nil {
return nil
}
+ tablesCacheLoader.Do(func() {
+ var err errors.Error
+ tablesCache, err = basicRes.GetDal().AllTables()
+ if err != nil {
+ panic(err)
+ }
+ })
return &ScopeApiHelper[Conn, Scope, Tr]{
log: basicRes.GetLogger(),
db: basicRes.GetDal(),
validator: vld,
+ bpManager: serviceHelper.NewBlueprintManager(basicRes.GetDal()),
connHelper: connHelper,
}
}
@@ -70,6 +105,7 @@ func NewScopeHelper[Conn any, Scope any, Tr any](
type ScopeRes[T any] struct {
Scope T `mapstructure:",squash"`
TransformationRuleName string `mapstructure:"transformationRuleName,omitempty"`
+ Blueprints []*models.Blueprint
}
type ScopeReq[T any] struct {
@@ -87,12 +123,11 @@ func (c *ScopeApiHelper[Conn, Scope, Tr]) Put(input *plugin.ApiResourceInput) (*
if err != nil {
return nil, errors.BadInput.Wrap(err, "decoding scope error")
}
- // Extract the connection ID from the input.Params map
- connectionId, _ := extractFromReqParam(input.Params)
- if connectionId == 0 {
+ params := c.extractFromReqParam(input)
+ if params.connectionId == 0 {
return nil, errors.BadInput.New("invalid connectionId")
}
- err = c.VerifyConnection(connectionId)
+ err = c.VerifyConnection(params.connectionId)
if err != nil {
return nil, err
}
@@ -111,7 +146,7 @@ func (c *ScopeApiHelper[Conn, Scope, Tr]) Put(input *plugin.ApiResourceInput) (*
}
// Set the connection ID, CreatedDate, and UpdatedDate fields
- setScopeFields(v, connectionId, &now, &now)
+ setScopeFields(v, params.connectionId, &now, &now)
// Verify that the primary key value is valid
err = VerifyScope(v, c.validator)
@@ -136,20 +171,19 @@ func (c *ScopeApiHelper[Conn, Scope, Tr]) Put(input *plugin.ApiResourceInput) (*
}
func (c *ScopeApiHelper[Conn, Scope, Tr]) Update(input *plugin.ApiResourceInput, fieldName string) (*plugin.ApiResourceOutput, errors.Error) {
- connectionId, scopeId := extractFromReqParam(input.Params)
-
- if connectionId == 0 {
- return &plugin.ApiResourceOutput{Body: nil, Status: http.StatusInternalServerError}, errors.BadInput.New("invalid connectionId")
+ params := c.extractFromReqParam(input)
+ if params.connectionId == 0 {
+ return nil, errors.BadInput.New("invalid connectionId")
}
- if len(scopeId) == 0 || scopeId == "0" {
- return &plugin.ApiResourceOutput{Body: nil, Status: http.StatusInternalServerError}, errors.BadInput.New("invalid scopeId")
+ if len(params.scopeId) == 0 {
+ return nil, errors.BadInput.New("invalid scopeId")
}
- err := c.VerifyConnection(connectionId)
+ err := c.VerifyConnection(params.connectionId)
if err != nil {
return &plugin.ApiResourceOutput{Body: nil, Status: http.StatusInternalServerError}, err
}
var scope Scope
- err = c.db.First(&scope, dal.Where(fmt.Sprintf("connection_id = ? AND %s = ?", fieldName), connectionId, scopeId))
+ err = c.db.First(&scope, dal.Where(fmt.Sprintf("connection_id = ? AND %s = ?", fieldName), params.connectionId, params.scopeId))
if err != nil {
return &plugin.ApiResourceOutput{Body: nil, Status: http.StatusInternalServerError}, errors.Default.New("getting Scope error")
}
@@ -178,7 +212,9 @@ func (c *ScopeApiHelper[Conn, Scope, Tr]) Update(input *plugin.ApiResourceInput,
return nil, errors.NotFound.New("transformationRule not found")
}
}
- scopeRes := &ScopeRes[Scope]{scope, reflect.ValueOf(rule).FieldByName("Name").String()}
+ scopeRes := &ScopeRes[Scope]{
+ Scope: scope,
+ TransformationRuleName: reflect.ValueOf(rule).FieldByName("Name").String()}
return &plugin.ApiResourceOutput{Body: scopeRes, Status: http.StatusOK}, nil
}
@@ -186,19 +222,19 @@ func (c *ScopeApiHelper[Conn, Scope, Tr]) Update(input *plugin.ApiResourceInput,
// GetScopeList returns a list of scopes. It expects a fieldName argument, which is used
// to extract the connection ID from the input.Params map.
-func (c *ScopeApiHelper[Conn, Scope, Tr]) GetScopeList(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
+func (c *ScopeApiHelper[Conn, Scope, Tr]) GetScopeList(input *plugin.ApiResourceInput, scopeIdFieldName ...string) (*plugin.ApiResourceOutput, errors.Error) {
// Extract the connection ID from the input.Params map
- connectionId, _ := extractFromReqParam(input.Params)
- if connectionId == 0 {
+ params := c.extractFromGetReqParam(input)
+ if params.connectionId == 0 {
return nil, errors.BadInput.New("invalid path params: \"connectionId\" not set")
}
- err := c.VerifyConnection(connectionId)
+ err := c.VerifyConnection(params.connectionId)
if err != nil {
return nil, err
}
limit, offset := GetLimitOffset(input.Query, "pageSize", "page")
var scopes []*Scope
- err = c.db.All(&scopes, dal.Where("connection_id = ?", connectionId), dal.Limit(limit), dal.Offset(offset))
+ err = c.db.All(&scopes, dal.Where("connection_id = ?", params.connectionId), dal.Limit(limit), dal.Offset(offset))
if err != nil {
return nil, err
}
@@ -207,24 +243,53 @@ func (c *ScopeApiHelper[Conn, Scope, Tr]) GetScopeList(input *plugin.ApiResource
if err != nil {
return nil, err
}
+ if params.loadBlueprints {
+ if len(scopeIdFieldName) == 0 {
+ return nil, errors.Default.New("scope Id field name is not known") //temporary, limited solution until I properly refactor all of this in another PR
+ }
+ scopesById := c.mapByScopeId(apiScopes, scopeIdFieldName[0])
+ var scopeIds []string
+ for id := range scopesById {
+ scopeIds = append(scopeIds, id)
+ }
+ blueprintMap, err := c.bpManager.GetBlueprintsByScopes(params.connectionId, scopeIds...)
+ if err != nil {
+ return nil, errors.Default.Wrap(err, fmt.Sprintf("error getting blueprints for scopes from connection %d", params.connectionId))
+ }
+ apiScopes = nil
+ for scopeId, scope := range scopesById {
+ if bps, ok := blueprintMap[scopeId]; ok {
+ scope.Blueprints = bps
+ delete(blueprintMap, scopeId)
+ }
+ apiScopes = append(apiScopes, scope)
+ }
+ if len(blueprintMap) > 0 {
+ var danglingIds []string
+ for bpId := range blueprintMap {
+ danglingIds = append(danglingIds, bpId)
+ }
+ c.log.Warn(nil, "The following dangling scopes were found: %v", danglingIds)
+ }
+ }
return &plugin.ApiResourceOutput{Body: apiScopes, Status: http.StatusOK}, nil
}
-func (c *ScopeApiHelper[Conn, Scope, Tr]) GetScope(input *plugin.ApiResourceInput, fieldName string) (*plugin.ApiResourceOutput, errors.Error) {
- connectionId, scopeId := extractFromReqParam(input.Params)
- if connectionId == 0 {
+func (c *ScopeApiHelper[Conn, Scope, Tr]) GetScope(input *plugin.ApiResourceInput, scopeIdColumnName string) (*plugin.ApiResourceOutput, errors.Error) {
+ params := c.extractFromGetReqParam(input)
+ if params == nil || params.connectionId == 0 {
return nil, errors.BadInput.New("invalid path params: \"connectionId\" not set")
}
- if len(scopeId) == 0 || scopeId == "0" {
+ if len(params.scopeId) == 0 || params.scopeId == "0" {
return nil, errors.BadInput.New("invalid path params: \"scopeId\" not set/invalid")
}
- err := c.VerifyConnection(connectionId)
+ err := c.VerifyConnection(params.connectionId)
if err != nil {
return nil, err
}
db := c.db
- query := dal.Where(fmt.Sprintf("connection_id = ? AND %s = ?", fieldName), connectionId, scopeId)
+ query := dal.Where(fmt.Sprintf("connection_id = ? AND %s = ?", scopeIdColumnName), params.connectionId, params.scopeId)
var scope Scope
err = db.First(&scope, query)
if db.IsErrorNotFound(err) {
@@ -245,9 +310,95 @@ func (c *ScopeApiHelper[Conn, Scope, Tr]) GetScope(input *plugin.ApiResourceInpu
return nil, errors.NotFound.New("transformationRule not found")
}
}
- scopeRes := &ScopeRes[Scope]{scope, reflect.ValueOf(rule).FieldByName("Name").String()}
+ scopeRes := &ScopeRes[Scope]{
+ Scope: scope,
+ TransformationRuleName: reflect.ValueOf(rule).FieldByName("Name").String(),
+ }
return &plugin.ApiResourceOutput{Body: scopeRes, Status: http.StatusOK}, nil
}
+func (c *ScopeApiHelper[Conn, Scope, Tr]) DeleteScope(input *plugin.ApiResourceInput, scopeIdFieldName string, rawScopeParamName string,
+ getScopeParamValue func(db dal.Dal, scopeId string) (string, errors.Error)) (*plugin.ApiResourceOutput, errors.Error) {
+ params := c.extractFromDeleteReqParam(input)
+ if params == nil || params.connectionId == 0 {
+ return nil, errors.BadInput.New("invalid path params: \"connectionId\" not set")
+ }
+ if len(params.scopeId) == 0 || params.scopeId == "0" {
+ return nil, errors.BadInput.New("invalid path params: \"scopeId\" not set/invalid")
+ }
+ err := c.VerifyConnection(params.connectionId)
+ if err != nil {
+ return nil, errors.Default.Wrap(err, fmt.Sprintf("error verifying connection for connection ID %d", params.connectionId))
+ }
+ db := c.db
+ blueprintsMap, err := c.bpManager.GetBlueprintsByScopes(params.connectionId, params.scopeId)
+ if err != nil {
+ return nil, errors.Default.Wrap(err, fmt.Sprintf("error retrieving scope with scope ID %s", params.scopeId))
+ }
+ blueprints := blueprintsMap[params.scopeId]
+ // find all tables for this plugin
+ tables, err := getAffectedTables(params.plugin)
+ if err != nil {
+ return nil, errors.Default.Wrap(err, fmt.Sprintf("error getting database tables managed by plugin %s", params.plugin))
+ }
+ // delete all the plugin records referencing this scope
+ if rawScopeParamName != "" {
+ scopeParamValue := params.scopeId
+ if getScopeParamValue != nil {
+ scopeParamValue, err = getScopeParamValue(c.db, params.scopeId) // this function is optional - use it if API data params stores a value different to the scope id (e.g. github plugin)
+ if err != nil {
+ return nil, errors.Default.Wrap(err, fmt.Sprintf("error extracting scope parameter name for scope %s", params.scopeId))
+ }
+ }
+ for _, table := range tables {
+ err = db.Exec(createDeleteQuery(table, rawScopeParamName, scopeParamValue))
+ if err != nil {
+ return nil, errors.Default.Wrap(err, fmt.Sprintf("error deleting data bound to scope %s for plugin %s", params.scopeId, params.plugin))
+ }
+ }
+ }
+ var impactedBlueprints []*models.Blueprint
+ if !params.deleteDataOnly {
+ // Delete the scope itself
+ scope := new(Scope)
+ err = c.db.Delete(&scope, dal.Where(fmt.Sprintf("connection_id = ? AND %s = ?", scopeIdFieldName),
+ params.connectionId, params.scopeId))
+ if err != nil {
+ return nil, errors.Default.Wrap(err, fmt.Sprintf("error deleting scope %s", params.scopeId))
+ }
+ // update the blueprints (remove scope reference from them)
+ for _, blueprint := range blueprints {
+ settings, _ := blueprint.UnmarshalSettings()
+ var changed bool
+ err = settings.UpdateConnections(func(c *plugin.BlueprintConnectionV200) errors.Error {
+ var retainedScopes []*plugin.BlueprintScopeV200
+ for _, bpScope := range c.Scopes {
+ if bpScope.Id == params.scopeId { // we'll be removing this one
+ changed = true
+ } else {
+ retainedScopes = append(retainedScopes, bpScope)
+ }
+ }
+ c.Scopes = retainedScopes
+ return nil
+ })
+ if err != nil {
+ return nil, errors.Default.Wrap(err, fmt.Sprintf("error removing scope %s from blueprint %d", params.scopeId, blueprint.ID))
+ }
+ if changed {
+ err = blueprint.UpdateSettings(&settings)
+ if err != nil {
+ return nil, errors.Default.Wrap(err, fmt.Sprintf("error writing new settings into blueprint %s", blueprint.Name))
+ }
+ err = c.bpManager.SaveDbBlueprint(blueprint)
+ if err != nil {
+ return nil, errors.Default.Wrap(err, fmt.Sprintf("error saving the updated blueprint %s", blueprint.Name))
+ }
+ impactedBlueprints = append(impactedBlueprints, blueprint)
+ }
+ }
+ }
+ return &plugin.ApiResourceOutput{Body: impactedBlueprints, Status: http.StatusOK}, nil
+}
func (c *ScopeApiHelper[Conn, Scope, Tr]) VerifyConnection(connId uint64) errors.Error {
var conn Conn
@@ -261,10 +412,10 @@ func (c *ScopeApiHelper[Conn, Scope, Tr]) VerifyConnection(connId uint64) errors
return nil
}
-func (c *ScopeApiHelper[Conn, Scope, Tr]) addTransformationName(scopes []*Scope) ([]ScopeRes[Scope], errors.Error) {
+func (c *ScopeApiHelper[Conn, Scope, Tr]) addTransformationName(scopes []*Scope) ([]*ScopeRes[Scope], errors.Error) {
var ruleIds []uint64
- apiScopes := make([]ScopeRes[Scope], 0)
+ apiScopes := make([]*ScopeRes[Scope], 0)
for _, scope := range scopes {
valueRepoRuleId := reflect.ValueOf(scope).Elem().FieldByName("TransformationRuleId")
if !valueRepoRuleId.IsValid() {
@@ -291,9 +442,12 @@ func (c *ScopeApiHelper[Conn, Scope, Tr]) addTransformationName(scopes []*Scope)
for _, scope := range scopes {
field := reflect.ValueOf(scope).Elem().FieldByName("TransformationRuleId")
if field.IsValid() {
- apiScopes = append(apiScopes, ScopeRes[Scope]{*scope, names[field.Uint()]})
+ apiScopes = append(apiScopes, &ScopeRes[Scope]{
+ Scope: *scope,
+ TransformationRuleName: names[field.Uint()],
+ })
} else {
- apiScopes = append(apiScopes, ScopeRes[Scope]{Scope: *scope, TransformationRuleName: ""})
+ apiScopes = append(apiScopes, &ScopeRes[Scope]{Scope: *scope, TransformationRuleName: ""})
}
}
@@ -312,13 +466,65 @@ func (c *ScopeApiHelper[Conn, Scope, Tr]) save(scope interface{}) errors.Error {
return nil
}
-func extractFromReqParam(params map[string]string) (uint64, string) {
- connectionId, err := strconv.ParseUint(params["connectionId"], 10, 64)
- if err != nil {
- return 0, ""
+func (c *ScopeApiHelper[Conn, Scope, Tr]) mapByScopeId(scopes []*ScopeRes[Scope], scopeIdFieldName string) map[string]*ScopeRes[Scope] {
+ scopeMap := map[string]*ScopeRes[Scope]{}
+ for _, scope := range scopes {
+ scopeId := fmt.Sprintf("%v", reflectField(scope.Scope, scopeIdFieldName).Interface())
+ scopeMap[scopeId] = scope
+ }
+ return scopeMap
+}
+
+func (c *ScopeApiHelper[Conn, Scope, Tr]) extractFromReqParam(input *plugin.ApiResourceInput) *requestParams {
+ connectionId, err := strconv.ParseUint(input.Params["connectionId"], 10, 64)
+ if err != nil || connectionId == 0 {
+ connectionId = 0
+ }
+ scopeId := input.Params["scopeId"]
+ pluginName := input.Params["plugin"]
+ return &requestParams{
+ connectionId: connectionId,
+ scopeId: scopeId,
+ plugin: pluginName,
+ }
+}
+
+func (c *ScopeApiHelper[Conn, Scope, Tr]) extractFromDeleteReqParam(input *plugin.ApiResourceInput) *deleteRequestParams {
+ params := c.extractFromReqParam(input)
+ var err errors.Error
+ var deleteDataOnly bool
+ {
+ ddo, ok := input.Query["delete_data_only"]
+ if ok {
+ deleteDataOnly, err = errors.Convert01(strconv.ParseBool(ddo[0]))
+ }
+ if err != nil {
+ deleteDataOnly = false
+ }
+ }
+ return &deleteRequestParams{
+ requestParams: *params,
+ deleteDataOnly: deleteDataOnly,
+ }
+}
+
+func (c *ScopeApiHelper[Conn, Scope, Tr]) extractFromGetReqParam(input *plugin.ApiResourceInput) *getRequestParams {
+ params := c.extractFromReqParam(input)
+ var err errors.Error
+ var loadBlueprints bool
+ {
+ lbps, ok := input.Query["blueprints"]
+ if ok {
+ loadBlueprints, err = errors.Convert01(strconv.ParseBool(lbps[0]))
+ }
+ if err != nil {
+ loadBlueprints = false
+ }
+ }
+ return &getRequestParams{
+ requestParams: *params,
+ loadBlueprints: loadBlueprints,
}
- scopeId := params["scopeId"]
- return connectionId, scopeId
}
func setScopeFields(p interface{}, connectionId uint64, createdDate *time.Time, updatedDate *time.Time) {
@@ -410,3 +616,81 @@ func (sr *ScopeRes[T]) MarshalJSON() ([]byte, error) {
return result, nil
}
+
+func createDeleteQuery(tableName string, scopeIdKey string, scopeId string) string {
+ column := "_raw_data_params"
+ if tableName == (models.CollectorLatestState{}.TableName()) {
+ column = "raw_data_params"
+ } else if strings.HasPrefix(tableName, "_raw_") {
+ column = "params"
+ }
+ query := `DELETE FROM ` + tableName + ` WHERE ` + column + ` LIKE '%"` + scopeIdKey + `":"` + scopeId + `"%'`
+ return query
+}
+
+func getAffectedTables(pluginName string) ([]string, errors.Error) {
+ var tables []string
+ meta, err := plugin.GetPlugin(pluginName)
+ if err != nil {
+ return nil, err
+ }
+ if pluginModel, ok := meta.(plugin.PluginModel); !ok {
+ return nil, errors.Default.New(fmt.Sprintf("plugin \"%s\" does not implement listing its tables", pluginName))
+ } else {
+ // collect raw tables
+ for _, table := range tablesCache {
+ if strings.HasPrefix(table, "_raw_"+pluginName) {
+ tables = append(tables, table)
+ }
+ }
+ // collect tool tables
+ tablesInfo := pluginModel.GetTablesInfo()
+ for _, table := range tablesInfo {
+ // we only care about tables with RawOrigin
+ ok = hasField(table, "RawDataParams")
+ if ok {
+ tables = append(tables, table.TableName())
+ }
+ }
+ // collect domain tables
+ for _, domainTable := range domaininfo.GetDomainTablesInfo() {
+ // we only care about tables with RawOrigin
+ ok = hasField(domainTable, "RawDataParams")
+ if ok {
+ tables = append(tables, domainTable.TableName())
+ }
+ }
+ // additional tables
+ tables = append(tables, models.CollectorLatestState{}.TableName())
+ }
+ return tables, nil
+}
+
+func reflectField(obj any, fieldName string) reflect.Value {
+ return reflectValue(obj).FieldByName(fieldName)
+}
+
+func hasField(obj any, fieldName string) bool {
+ _, ok := reflectType(obj).FieldByName(fieldName)
+ return ok
+}
+
+func reflectValue(obj any) reflect.Value {
+ val := reflect.ValueOf(obj)
+ kind := val.Kind()
+ for kind == reflect.Ptr || kind == reflect.Interface {
+ val = val.Elem()
+ kind = val.Kind()
+ }
+ return val
+}
+
+func reflectType(obj any) reflect.Type {
+ typ := reflect.TypeOf(obj)
+ kind := typ.Kind()
+ for kind == reflect.Ptr {
+ typ = typ.Elem()
+ kind = typ.Kind()
+ }
+ return typ
+}
diff --git a/backend/helpers/pluginhelper/api/scope_helper_test.go b/backend/helpers/pluginhelper/api/scope_helper_test.go
index f77acb079..a7f405267 100644
--- a/backend/helpers/pluginhelper/api/scope_helper_test.go
+++ b/backend/helpers/pluginhelper/api/scope_helper_test.go
@@ -257,6 +257,7 @@ func TestScopeApiHelper_Put(t *testing.T) {
mockDal.On("CreateOrUpdate", mock.Anything, mock.Anything).Return(nil)
mockDal.On("First", mock.Anything, mock.Anything).Return(nil)
mockDal.On("All", mock.Anything, mock.Anything).Return(nil)
+ mockDal.On("AllTables").Return(nil, nil)
connHelper := NewConnectionHelper(mockRes, nil)
@@ -293,9 +294,8 @@ func TestScopeApiHelper_Put(t *testing.T) {
"updatedAt": "string",
"updatedDate": "string",
}}}}
-
// create a mock ScopeApiHelper with a mock database connection
- apiHelper := &ScopeApiHelper[TestConnection, TestRepo, TestTransformationRule]{db: mockDal, connHelper: connHelper}
+ apiHelper := NewScopeHelper[TestConnection, TestRepo, TestTransformationRule](mockRes, nil, connHelper)
// test a successful call to Put
_, err := apiHelper.Put(input)
assert.NoError(t, err)
diff --git a/backend/server/services/blueprint_helper.go b/backend/helpers/pluginhelper/services/blueprint_helper.go
similarity index 53%
rename from backend/server/services/blueprint_helper.go
rename to backend/helpers/pluginhelper/services/blueprint_helper.go
index adf5f6f39..d784be53c 100644
--- a/backend/server/services/blueprint_helper.go
+++ b/backend/helpers/pluginhelper/services/blueprint_helper.go
@@ -22,20 +22,39 @@ import (
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models"
+ "github.com/apache/incubator-devlake/core/plugin"
)
+type BlueprintManager struct {
+ db dal.Dal
+}
+
+type GetBlueprintQuery struct {
+ Enable *bool
+ IsManual *bool
+ Label string
+ SkipRecords int
+ PageSize int
+}
+
+func NewBlueprintManager(db dal.Dal) *BlueprintManager {
+ return &BlueprintManager{
+ db: db,
+ }
+}
+
// SaveDbBlueprint accepts a Blueprint instance and upsert it to database
-func SaveDbBlueprint(blueprint *models.Blueprint) errors.Error {
+func (b *BlueprintManager) SaveDbBlueprint(blueprint *models.Blueprint) errors.Error {
var err error
if blueprint.ID != 0 {
- err = db.Update(&blueprint)
+ err = b.db.Update(&blueprint)
} else {
- err = db.Create(&blueprint)
+ err = b.db.Create(&blueprint)
}
if err != nil {
return errors.Default.Wrap(err, "error creating DB blueprint")
}
- err = db.Delete(&models.DbBlueprintLabel{}, dal.Where(`blueprint_id = ?`, blueprint.ID))
+ err = b.db.Delete(&models.DbBlueprintLabel{}, dal.Where(`blueprint_id = ?`, blueprint.ID))
if err != nil {
return errors.Default.Wrap(err, "error delete DB blueprint's old labelModels")
}
@@ -47,7 +66,7 @@ func SaveDbBlueprint(blueprint *models.Blueprint) errors.Error {
Name: blueprint.Labels[i],
})
}
- err = db.Create(&blueprintLabels)
+ err = b.db.Create(&blueprintLabels)
if err != nil {
return errors.Default.Wrap(err, "error creating DB blueprint's labelModels")
}
@@ -56,7 +75,7 @@ func SaveDbBlueprint(blueprint *models.Blueprint) errors.Error {
}
// GetDbBlueprints returns a paginated list of Blueprints based on `query`
-func GetDbBlueprints(query *BlueprintQuery) ([]*models.Blueprint, int64, errors.Error) {
+func (b *BlueprintManager) GetDbBlueprints(query *GetBlueprintQuery) ([]*models.Blueprint, int64, errors.Error) {
// process query parameters
clauses := []dal.Clause{dal.From(&models.Blueprint{})}
if query.Enable != nil {
@@ -73,26 +92,27 @@ func GetDbBlueprints(query *BlueprintQuery) ([]*models.Blueprint, int64, errors.
}
// count total records
- count, err := db.Count(clauses...)
+ count, err := b.db.Count(clauses...)
if err != nil {
return nil, 0, err
}
-
+ clauses = append(clauses, dal.Orderby("id DESC"))
// load paginated blueprints from database
- clauses = append(clauses,
- dal.Orderby("id DESC"),
- dal.Offset(query.GetSkip()),
- dal.Limit(query.GetPageSize()),
- )
+ if query.SkipRecords != 0 {
+ clauses = append(clauses, dal.Offset(query.SkipRecords))
+ }
+ if query.PageSize != 0 {
+ clauses = append(clauses, dal.Limit(query.PageSize))
+ }
dbBlueprints := make([]*models.Blueprint, 0)
- err = db.All(&dbBlueprints, clauses...)
+ err = b.db.All(&dbBlueprints, clauses...)
if err != nil {
return nil, 0, errors.Default.Wrap(err, "error getting DB count of blueprints")
}
// load labels for blueprints
for _, dbBlueprint := range dbBlueprints {
- err = fillBlueprintDetail(dbBlueprint)
+ err = b.fillBlueprintDetail(dbBlueprint)
if err != nil {
return nil, 0, err
}
@@ -102,43 +122,88 @@ func GetDbBlueprints(query *BlueprintQuery) ([]*models.Blueprint, int64, errors.
}
// GetDbBlueprint returns the detail of a given Blueprint ID
-func GetDbBlueprint(blueprintId uint64) (*models.Blueprint, errors.Error) {
+func (b *BlueprintManager) GetDbBlueprint(blueprintId uint64) (*models.Blueprint, errors.Error) {
blueprint := &models.Blueprint{}
- err := db.First(blueprint, dal.Where("id = ?", blueprintId))
+ err := b.db.First(blueprint, dal.Where("id = ?", blueprintId))
if err != nil {
- if db.IsErrorNotFound(err) {
+ if b.db.IsErrorNotFound(err) {
return nil, errors.NotFound.Wrap(err, "could not find blueprint in DB")
}
return nil, errors.Default.Wrap(err, "error getting blueprint from DB")
}
- err = fillBlueprintDetail(blueprint)
+ err = b.fillBlueprintDetail(blueprint)
if err != nil {
return nil, err
}
return blueprint, nil
}
+// GetBlueprintsByScopes returns all blueprints that have these scopeIds
+func (b *BlueprintManager) GetBlueprintsByScopes(connectionId uint64, scopeIds ...string) (map[string][]*models.Blueprint, errors.Error) {
+ bps, _, err := b.GetDbBlueprints(&GetBlueprintQuery{})
+ if err != nil {
+ return nil, err
+ }
+ scopeMap := map[string][]*models.Blueprint{}
+ for _, bp := range bps {
+ scopes, err := bp.GetScopes(connectionId)
+ if err != nil {
+ return nil, err
+ }
+ for _, scope := range scopes {
+ if contains(scopeIds, scope.Id) {
+ if inserted, ok := scopeMap[scope.Id]; !ok {
+ scopeMap[scope.Id] = []*models.Blueprint{bp}
+ } else {
+ inserted = append(inserted, bp)
+ scopeMap[scope.Id] = inserted
+ }
+ break
+ }
+ }
+ }
+ return scopeMap, nil
+}
+
+// GetBlueprintConnections returns the connections associated with this blueprint Id
+func (b *BlueprintManager) GetBlueprintConnections(blueprintId uint64) ([]*plugin.BlueprintConnectionV200, errors.Error) {
+ bp, err := b.GetDbBlueprint(blueprintId)
+ if err != nil {
+ return nil, err
+ }
+ return bp.GetConnections()
+}
+
// GetDbBlueprintByProjectName returns the detail of a given projectName
-func GetDbBlueprintByProjectName(projectName string) (*models.Blueprint, errors.Error) {
+func (b *BlueprintManager) GetDbBlueprintByProjectName(projectName string) (*models.Blueprint, errors.Error) {
dbBlueprint := &models.Blueprint{}
- err := db.First(dbBlueprint, dal.Where("project_name = ?", projectName))
+ err := b.db.First(dbBlueprint, dal.Where("project_name = ?", projectName))
if err != nil {
- if db.IsErrorNotFound(err) {
+ if b.db.IsErrorNotFound(err) {
return nil, errors.NotFound.Wrap(err, fmt.Sprintf("could not find blueprint in DB by projectName %s", projectName))
}
return nil, errors.Default.Wrap(err, fmt.Sprintf("error getting blueprint from DB by projectName %s", projectName))
}
- err = fillBlueprintDetail(dbBlueprint)
+ err = b.fillBlueprintDetail(dbBlueprint)
if err != nil {
return nil, err
}
return dbBlueprint, nil
}
-func fillBlueprintDetail(blueprint *models.Blueprint) errors.Error {
- err := db.Pluck("name", &blueprint.Labels, dal.From(&models.DbBlueprintLabel{}), dal.Where("blueprint_id = ?", blueprint.ID))
+func (b *BlueprintManager) fillBlueprintDetail(blueprint *models.Blueprint) errors.Error {
+ err := b.db.Pluck("name", &blueprint.Labels, dal.From(&models.DbBlueprintLabel{}), dal.Where("blueprint_id = ?", blueprint.ID))
if err != nil {
return errors.Internal.Wrap(err, "error getting the blueprint labels from database")
}
return nil
}
+
+func contains(list []string, target string) bool {
+ for _, t := range list {
+ if t == target {
+ return true
+ }
+ }
+ return false
+}
diff --git a/backend/plugins/pagerduty/api/scope.go b/backend/plugins/pagerduty/api/scope.go
index 59fae3b9a..ec5996c77 100644
--- a/backend/plugins/pagerduty/api/scope.go
+++ b/backend/plugins/pagerduty/api/scope.go
@@ -58,7 +58,7 @@ func PutScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors
// @Failure 500 {object} shared.ApiBody "Internal Error"
// @Router /plugins/pagerduty/connections/{connectionId}/scopes/{serviceId} [PATCH]
func UpdateScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
- return scopeHelper.Update(input, "id")
+ return scopeHelper.Update(input, "")
}
// GetScopeList get PagerDuty repos
@@ -68,12 +68,13 @@ func UpdateScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, err
// @Param connectionId path int true "connection ID"
// @Param pageSize query int false "page size, default 50"
// @Param page query int false "page size, default 1"
+// @Param blueprints query bool false "also return blueprints using these scopes as part of the payload"
// @Success 200 {object} []ScopeRes
// @Failure 400 {object} shared.ApiBody "Bad Request"
// @Failure 500 {object} shared.ApiBody "Internal Error"
// @Router /plugins/pagerduty/connections/{connectionId}/scopes/ [GET]
func GetScopeList(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
- return scopeHelper.GetScopeList(input)
+ return scopeHelper.GetScopeList(input, "Id")
}
// GetScope get one PagerDuty service
@@ -82,6 +83,7 @@ func GetScopeList(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, er
// @Tags plugins/pagerduty
// @Param connectionId path int true "connection ID"
// @Param serviceId path int true "service ID"
+// @Param blueprints query bool false "also return blueprints using this scope as part of the payload"
// @Success 200 {object} ScopeRes
// @Failure 400 {object} shared.ApiBody "Bad Request"
// @Failure 500 {object} shared.ApiBody "Internal Error"
@@ -89,3 +91,18 @@ func GetScopeList(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, er
func GetScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
return scopeHelper.GetScope(input, "id")
}
+
+// DeleteScope delete plugin data associated with the scope and optionally the scope itself
+// @Summary delete plugin data associated with the scope and optionally the scope itself
+// @Description delete data associated with plugin scope
+// @Tags plugins/pagerduty
+// @Param connectionId path int true "connection ID"
+// @Param serviceId path int true "service ID"
+// @Param delete_data_only query bool false "Only delete the scope data, not the scope itself"
+// @Success 200 {object} []models.Blueprint "list of blueprints impacted by the deletion"
+// @Failure 400 {object} shared.ApiBody "Bad Request"
+// @Failure 500 {object} shared.ApiBody "Internal Error"
+// @Router /plugins/pagerduty/connections/{connectionId}/scopes/{serviceId} [DELETE]
+func DeleteScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
+ return scopeHelper.DeleteScope(input, "Id", "ScopeId", nil)
+}
diff --git a/backend/plugins/pagerduty/e2e/raw_tables/_raw_pagerduty_incidents.csv b/backend/plugins/pagerduty/e2e/raw_tables/_raw_pagerduty_incidents.csv
index 8cda8d9c8..65e43ff14 100644
--- a/backend/plugins/pagerduty/e2e/raw_tables/_raw_pagerduty_incidents.csv
+++ b/backend/plugins/pagerduty/e2e/raw_tables/_raw_pagerduty_incidents.csv
@@ -1,4 +1,4 @@
id,params,data,url,input,created_at
-1,"{""ConnectionId"":1}","{""incident_number"": 4, ""title"": ""Crash reported"", ""created_at"": ""2022-11-03T06:23:06.000000Z"", ""status"": ""triggered"", ""incident_key"": ""bb60942875634ee6a7fe94ddb51c3a09"", ""service"": {""id"": ""PIKL83L"", ""type"": ""service_reference"", ""summary"": ""DevService"", ""self"": ""https://api.pagerduty.com/services/PIKL83L"", ""html_url"": ""https://keon-test.pagerduty.com/service-directory/PIKL83L""}, ""assignments"": [{""at"": ""2022-11-03T06:23 [...]
-2,"{""ConnectionId"":1}","{""incident_number"": 5, ""title"": ""Slow startup"", ""created_at"": ""2022-11-03T06:44:28.000000Z"", ""status"": ""acknowledged"", ""incident_key"": ""d7bc6d39c37e4af8b206a12ff6b05793"", ""service"": {""id"": ""PIKL83L"", ""type"": ""service_reference"", ""summary"": ""DevService"", ""self"": ""https://api.pagerduty.com/services/PIKL83L"", ""html_url"": ""https://keon-test.pagerduty.com/service-directory/PIKL83L""}, ""assignments"": [{""at"": ""2022-11-03T06:4 [...]
-3,"{""ConnectionId"":1}","{""incident_number"": 6, ""title"": ""Spamming logs"", ""created_at"": ""2022-11-03T06:45:36.000000Z"", ""status"": ""resolved"", ""incident_key"": ""9f5acd07975e4c57bc717d8d9e066785"", ""service"": {""id"": ""PIKL83L"", ""type"": ""service_reference"", ""summary"": ""DevService"", ""self"": ""https://api.pagerduty.com/services/PIKL83L"", ""html_url"": ""https://keon-test.pagerduty.com/service-directory/PIKL83L""}, ""assignments"": [], ""last_status_change_at"": [...]
+1,"{""ConnectionId"":1,""ScopeId"":""PIKL83L""}","{""incident_number"": 4, ""title"": ""Crash reported"", ""created_at"": ""2022-11-03T06:23:06.000000Z"", ""status"": ""triggered"", ""incident_key"": ""bb60942875634ee6a7fe94ddb51c3a09"", ""service"": {""id"": ""PIKL83L"", ""type"": ""service_reference"", ""summary"": ""DevService"", ""self"": ""https://api.pagerduty.com/services/PIKL83L"", ""html_url"": ""https://keon-test.pagerduty.com/service-directory/PIKL83L""}, ""assignments"": [{"" [...]
+2,"{""ConnectionId"":1,""ScopeId"":""PIKL83L""}","{""incident_number"": 5, ""title"": ""Slow startup"", ""created_at"": ""2022-11-03T06:44:28.000000Z"", ""status"": ""acknowledged"", ""incident_key"": ""d7bc6d39c37e4af8b206a12ff6b05793"", ""service"": {""id"": ""PIKL83L"", ""type"": ""service_reference"", ""summary"": ""DevService"", ""self"": ""https://api.pagerduty.com/services/PIKL83L"", ""html_url"": ""https://keon-test.pagerduty.com/service-directory/PIKL83L""}, ""assignments"": [{" [...]
+3,"{""ConnectionId"":1,""ScopeId"":""PIKL83L""}","{""incident_number"": 6, ""title"": ""Spamming logs"", ""created_at"": ""2022-11-03T06:45:36.000000Z"", ""status"": ""resolved"", ""incident_key"": ""9f5acd07975e4c57bc717d8d9e066785"", ""service"": {""id"": ""PIKL83L"", ""type"": ""service_reference"", ""summary"": ""DevService"", ""self"": ""https://api.pagerduty.com/services/PIKL83L"", ""html_url"": ""https://keon-test.pagerduty.com/service-directory/PIKL83L""}, ""assignments"": [], "" [...]
diff --git a/backend/plugins/pagerduty/e2e/snapshot_tables/_tool_pagerduty_assignments.csv b/backend/plugins/pagerduty/e2e/snapshot_tables/_tool_pagerduty_assignments.csv
index a1b1bc317..3201f177f 100644
--- a/backend/plugins/pagerduty/e2e/snapshot_tables/_tool_pagerduty_assignments.csv
+++ b/backend/plugins/pagerduty/e2e/snapshot_tables/_tool_pagerduty_assignments.csv
@@ -1,4 +1,4 @@
incident_number,user_id,created_at,updated_at,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark,connection_id,assigned_at
-4,P25K520,2022-11-03T07:11:37.415+00:00,2022-11-03T07:11:37.415+00:00,"{""ConnectionId"":1}",_raw_pagerduty_incidents,1,,1,2022-11-03T07:02:36.000+00:00
-4,PQYACO3,2022-11-03T07:11:37.415+00:00,2022-11-03T07:11:37.415+00:00,"{""ConnectionId"":1}",_raw_pagerduty_incidents,1,,1,2022-11-03T06:23:06.000+00:00
-5,PQYACO3,2022-11-03T07:11:37.415+00:00,2022-11-03T07:11:37.415+00:00,"{""ConnectionId"":1}",_raw_pagerduty_incidents,2,,1,2022-11-03T06:44:37.000+00:00
+4,P25K520,2022-11-03T07:11:37.415+00:00,2022-11-03T07:11:37.415+00:00,"{""ConnectionId"":1,""ScopeId"":""PIKL83L""}",_raw_pagerduty_incidents,1,,1,2022-11-03T07:02:36.000+00:00
+4,PQYACO3,2022-11-03T07:11:37.415+00:00,2022-11-03T07:11:37.415+00:00,"{""ConnectionId"":1,""ScopeId"":""PIKL83L""}",_raw_pagerduty_incidents,1,,1,2022-11-03T06:23:06.000+00:00
+5,PQYACO3,2022-11-03T07:11:37.415+00:00,2022-11-03T07:11:37.415+00:00,"{""ConnectionId"":1,""ScopeId"":""PIKL83L""}",_raw_pagerduty_incidents,2,,1,2022-11-03T06:44:37.000+00:00
diff --git a/backend/plugins/pagerduty/e2e/snapshot_tables/_tool_pagerduty_incidents.csv b/backend/plugins/pagerduty/e2e/snapshot_tables/_tool_pagerduty_incidents.csv
index de1a59010..539209e20 100644
--- a/backend/plugins/pagerduty/e2e/snapshot_tables/_tool_pagerduty_incidents.csv
+++ b/backend/plugins/pagerduty/e2e/snapshot_tables/_tool_pagerduty_incidents.csv
@@ -1,4 +1,4 @@
connection_id,number,created_at,updated_at,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark,url,service_id,summary,status,urgency,created_date,updated_date
-1,4,2022-11-03T07:11:37.422+00:00,2022-11-03T07:11:37.422+00:00,"{""ConnectionId"":1}",_raw_pagerduty_incidents,1,,https://keon-test.pagerduty.com/incidents/Q3YON8WNWTZMRQ,PIKL83L,[#4] Crash reported,triggered,high,2022-11-03T06:23:06.000+00:00,2022-11-03T07:02:36.000+00:00
-1,5,2022-11-03T07:11:37.422+00:00,2022-11-03T07:11:37.422+00:00,"{""ConnectionId"":1}",_raw_pagerduty_incidents,2,,https://keon-test.pagerduty.com/incidents/Q3CZAU7Q4008QD,PIKL83L,[#5] Slow startup,acknowledged,high,2022-11-03T06:44:28.000+00:00,2022-11-03T06:44:37.000+00:00
-1,6,2022-11-03T07:11:37.422+00:00,2022-11-03T07:11:37.422+00:00,"{""ConnectionId"":1}",_raw_pagerduty_incidents,3,,https://keon-test.pagerduty.com/incidents/Q1OHFWFP3GPXOG,PIKL83L,[#6] Spamming logs,resolved,low,2022-11-03T06:45:36.000+00:00,2022-11-03T06:51:44.000+00:00
+1,4,2022-11-03T07:11:37.422+00:00,2022-11-03T07:11:37.422+00:00,"{""ConnectionId"":1,""ScopeId"":""PIKL83L""}",_raw_pagerduty_incidents,1,,https://keon-test.pagerduty.com/incidents/Q3YON8WNWTZMRQ,PIKL83L,[#4] Crash reported,triggered,high,2022-11-03T06:23:06.000+00:00,2022-11-03T07:02:36.000+00:00
+1,5,2022-11-03T07:11:37.422+00:00,2022-11-03T07:11:37.422+00:00,"{""ConnectionId"":1,""ScopeId"":""PIKL83L""}",_raw_pagerduty_incidents,2,,https://keon-test.pagerduty.com/incidents/Q3CZAU7Q4008QD,PIKL83L,[#5] Slow startup,acknowledged,high,2022-11-03T06:44:28.000+00:00,2022-11-03T06:44:37.000+00:00
+1,6,2022-11-03T07:11:37.422+00:00,2022-11-03T07:11:37.422+00:00,"{""ConnectionId"":1,""ScopeId"":""PIKL83L""}",_raw_pagerduty_incidents,3,,https://keon-test.pagerduty.com/incidents/Q1OHFWFP3GPXOG,PIKL83L,[#6] Spamming logs,resolved,low,2022-11-03T06:45:36.000+00:00,2022-11-03T06:51:44.000+00:00
diff --git a/backend/plugins/pagerduty/e2e/snapshot_tables/_tool_pagerduty_users.csv b/backend/plugins/pagerduty/e2e/snapshot_tables/_tool_pagerduty_users.csv
index a47f8b668..a679b64e0 100644
--- a/backend/plugins/pagerduty/e2e/snapshot_tables/_tool_pagerduty_users.csv
+++ b/backend/plugins/pagerduty/e2e/snapshot_tables/_tool_pagerduty_users.csv
@@ -1,3 +1,3 @@
connection_id,id,created_at,updated_at,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark,url,name
-1,P25K520,2022-11-03T07:11:37.418+00:00,2022-11-03T07:11:37.418+00:00,"{""ConnectionId"":1}",_raw_pagerduty_incidents,1,,https://keon-test.pagerduty.com/users/P25K520,Kian Amini
-1,PQYACO3,2022-11-03T07:11:37.418+00:00,2022-11-03T07:11:37.418+00:00,"{""ConnectionId"":1}",_raw_pagerduty_incidents,2,,https://keon-test.pagerduty.com/users/PQYACO3,Keon Amini
+1,P25K520,2022-11-03T07:11:37.418+00:00,2022-11-03T07:11:37.418+00:00,"{""ConnectionId"":1,""ScopeId"":""PIKL83L""}",_raw_pagerduty_incidents,1,,https://keon-test.pagerduty.com/users/P25K520,Kian Amini
+1,PQYACO3,2022-11-03T07:11:37.418+00:00,2022-11-03T07:11:37.418+00:00,"{""ConnectionId"":1,""ScopeId"":""PIKL83L""}",_raw_pagerduty_incidents,2,,https://keon-test.pagerduty.com/users/PQYACO3,Keon Amini
diff --git a/backend/plugins/pagerduty/impl/impl.go b/backend/plugins/pagerduty/impl/impl.go
index d669e7e82..01b11285b 100644
--- a/backend/plugins/pagerduty/impl/impl.go
+++ b/backend/plugins/pagerduty/impl/impl.go
@@ -20,6 +20,7 @@ package impl
import (
"fmt"
"github.com/apache/incubator-devlake/core/context"
+ "github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
@@ -35,6 +36,8 @@ var _ plugin.PluginMeta = (*PagerDuty)(nil)
var _ plugin.PluginInit = (*PagerDuty)(nil)
var _ plugin.PluginTask = (*PagerDuty)(nil)
var _ plugin.PluginApi = (*PagerDuty)(nil)
+
+var _ plugin.PluginModel = (*PagerDuty)(nil)
var _ plugin.PluginBlueprintV100 = (*PagerDuty)(nil)
var _ plugin.CloseablePluginTask = (*PagerDuty)(nil)
@@ -57,6 +60,15 @@ func (p PagerDuty) SubTaskMetas() []plugin.SubTaskMeta {
}
}
+func (p PagerDuty) GetTablesInfo() []dal.Tabler {
+ return []dal.Tabler{
+ models.Service{},
+ models.Incident{},
+ models.User{},
+ models.Assignment{},
+ }
+}
+
func (p PagerDuty) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]interface{}) (interface{}, errors.Error) {
op, err := tasks.DecodeAndValidateTaskOptions(options)
if err != nil {
@@ -130,8 +142,9 @@ func (p PagerDuty) ApiResources() map[string]map[string]plugin.ApiResourceHandle
"PUT": api.PutScope,
},
"connections/:connectionId/scopes/:scopeId": {
- "GET": api.GetScope,
- "PATCH": api.UpdateScope,
+ "GET": api.GetScope,
+ "PATCH": api.UpdateScope,
+ "DELETE": api.DeleteScope,
},
"connections/:connectionId/transformation_rules": {
"POST": api.CreateTransformationRule,
diff --git a/backend/plugins/pagerduty/tasks/incidents_collector.go b/backend/plugins/pagerduty/tasks/incidents_collector.go
index 883f071bf..65b6b1cc8 100644
--- a/backend/plugins/pagerduty/tasks/incidents_collector.go
+++ b/backend/plugins/pagerduty/tasks/incidents_collector.go
@@ -61,11 +61,9 @@ func CollectIncidents(taskCtx plugin.SubTaskContext) errors.Error {
data := taskCtx.GetData().(*PagerDutyTaskData)
db := taskCtx.GetDal()
args := api.RawDataSubTaskArgs{
- Ctx: taskCtx,
- Params: PagerDutyParams{
- ConnectionId: data.Options.ConnectionId,
- },
- Table: RAW_INCIDENTS_TABLE,
+ Ctx: taskCtx,
+ Options: data.Options,
+ Table: RAW_INCIDENTS_TABLE,
}
collector, err := api.NewStatefulApiCollectorForFinalizableEntity(api.FinalizableApiCollectorArgs{
RawDataSubTaskArgs: args,
diff --git a/backend/plugins/pagerduty/tasks/incidents_converter.go b/backend/plugins/pagerduty/tasks/incidents_converter.go
index 4f1d4bc9a..dda345389 100644
--- a/backend/plugins/pagerduty/tasks/incidents_converter.go
+++ b/backend/plugins/pagerduty/tasks/incidents_converter.go
@@ -68,11 +68,9 @@ func ConvertIncidents(taskCtx plugin.SubTaskContext) errors.Error {
idGen := didgen.NewDomainIdGenerator(&models.Incident{})
converter, err := api.NewDataConverter(api.DataConverterArgs{
RawDataSubTaskArgs: api.RawDataSubTaskArgs{
- Ctx: taskCtx,
- Params: PagerDutyParams{
- ConnectionId: data.Options.ConnectionId,
- },
- Table: RAW_INCIDENTS_TABLE,
+ Ctx: taskCtx,
+ Options: data.Options,
+ Table: RAW_INCIDENTS_TABLE,
},
InputRowType: reflect.TypeOf(IncidentWithUser{}),
Input: cursor,
diff --git a/backend/plugins/pagerduty/tasks/incidents_extractor.go b/backend/plugins/pagerduty/tasks/incidents_extractor.go
index d5ba7eb26..6c83830b9 100644
--- a/backend/plugins/pagerduty/tasks/incidents_extractor.go
+++ b/backend/plugins/pagerduty/tasks/incidents_extractor.go
@@ -32,11 +32,9 @@ func ExtractIncidents(taskCtx plugin.SubTaskContext) errors.Error {
data := taskCtx.GetData().(*PagerDutyTaskData)
extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{
RawDataSubTaskArgs: api.RawDataSubTaskArgs{
- Ctx: taskCtx,
- Params: PagerDutyParams{
- ConnectionId: data.Options.ConnectionId,
- },
- Table: RAW_INCIDENTS_TABLE,
+ Ctx: taskCtx,
+ Options: data.Options,
+ Table: RAW_INCIDENTS_TABLE,
},
Extract: func(row *api.RawData) ([]interface{}, errors.Error) {
incidentRaw := &raw.Incidents{}
diff --git a/backend/plugins/pagerduty/tasks/task_data.go b/backend/plugins/pagerduty/tasks/task_data.go
index 91ba00329..b223a167e 100644
--- a/backend/plugins/pagerduty/tasks/task_data.go
+++ b/backend/plugins/pagerduty/tasks/task_data.go
@@ -41,6 +41,14 @@ type PagerDutyTaskData struct {
type PagerDutyParams struct {
ConnectionId uint64
+ ScopeId string
+}
+
+func (p *PagerDutyOptions) GetParams() any {
+ return PagerDutyParams{
+ ConnectionId: p.ConnectionId,
+ ScopeId: p.ServiceId,
+ }
}
func DecodeAndValidateTaskOptions(options map[string]interface{}) (*PagerDutyOptions, errors.Error) {
diff --git a/backend/server/api/router.go b/backend/server/api/router.go
index 26114da9b..73e21856d 100644
--- a/backend/server/api/router.go
+++ b/backend/server/api/router.go
@@ -88,22 +88,23 @@ func registerPluginEndpoints(r *gin.Engine, pluginName string, apiResources map[
r.Handle(
method,
fmt.Sprintf("/plugins/%s/%s", pluginName, resourcePath),
- handlePluginCall(h),
+ handlePluginCall(pluginName, h),
)
}
}
}
-func handlePluginCall(handler plugin.ApiResourceHandler) func(c *gin.Context) {
+func handlePluginCall(pluginName string, handler plugin.ApiResourceHandler) func(c *gin.Context) {
return func(c *gin.Context) {
var err error
input := &plugin.ApiResourceInput{}
+ input.Params = make(map[string]string)
if len(c.Params) > 0 {
- input.Params = make(map[string]string)
for _, param := range c.Params {
input.Params[param.Key] = param.Value
}
}
+ input.Params["plugin"] = pluginName
input.Query = c.Request.URL.Query()
if c.Request.Body != nil {
if strings.HasPrefix(c.Request.Header.Get("Content-Type"), "multipart/form-data;") {
diff --git a/backend/server/services/blueprint.go b/backend/server/services/blueprint.go
index aec8a22cc..0e8d5c176 100644
--- a/backend/server/services/blueprint.go
+++ b/backend/server/services/blueprint.go
@@ -20,6 +20,7 @@ package services
import (
"encoding/json"
"fmt"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/services"
"strings"
"github.com/apache/incubator-devlake/core/dal"
@@ -31,6 +32,10 @@ import (
"github.com/robfig/cron/v3"
)
+var (
+ blueprintLog = logruslog.Global.Nested("blueprint")
+)
+
// BlueprintQuery is a query for GetBlueprints
type BlueprintQuery struct {
Pagination
@@ -39,10 +44,6 @@ type BlueprintQuery struct {
Label string `form:"label"`
}
-var (
- blueprintLog = logruslog.Global.Nested("blueprint")
-)
-
type BlueprintJob struct {
Blueprint *models.Blueprint
}
@@ -63,7 +64,7 @@ func CreateBlueprint(blueprint *models.Blueprint) errors.Error {
if err != nil {
return err
}
- err = SaveDbBlueprint(blueprint)
+ err = bpManager.SaveDbBlueprint(blueprint)
if err != nil {
return err
}
@@ -76,7 +77,13 @@ func CreateBlueprint(blueprint *models.Blueprint) errors.Error {
// GetBlueprints returns a paginated list of Blueprints based on `query`
func GetBlueprints(query *BlueprintQuery) ([]*models.Blueprint, int64, errors.Error) {
- blueprints, count, err := GetDbBlueprints(query)
+ blueprints, count, err := bpManager.GetDbBlueprints(&services.GetBlueprintQuery{
+ Enable: query.Enable,
+ IsManual: query.IsManual,
+ Label: query.Label,
+ SkipRecords: query.GetSkip(),
+ PageSize: query.GetPageSize(),
+ })
if err != nil {
return nil, 0, errors.Convert(err)
}
@@ -85,7 +92,7 @@ func GetBlueprints(query *BlueprintQuery) ([]*models.Blueprint, int64, errors.Er
// GetBlueprint returns the detail of a given Blueprint ID
func GetBlueprint(blueprintId uint64) (*models.Blueprint, errors.Error) {
- blueprint, err := GetDbBlueprint(blueprintId)
+ blueprint, err := bpManager.GetDbBlueprint(blueprintId)
if err != nil {
if db.IsErrorNotFound(err) {
return nil, errors.NotFound.New("blueprint not found")
@@ -100,7 +107,7 @@ func GetBlueprintByProjectName(projectName string) (*models.Blueprint, errors.Er
if projectName == "" {
return nil, errors.Internal.New("can not use the empty projectName to search the unique blueprint")
}
- blueprint, err := GetDbBlueprintByProjectName(projectName)
+ blueprint, err := bpManager.GetDbBlueprintByProjectName(projectName)
if err != nil {
// Allow specific projectName to fail to find the corresponding blueprint
if db.IsErrorNotFound(err) {
@@ -177,7 +184,7 @@ func saveBlueprint(blueprint *models.Blueprint) (*models.Blueprint, errors.Error
if err != nil {
return nil, errors.BadInput.WrapRaw(err)
}
- err = SaveDbBlueprint(blueprint)
+ err = bpManager.SaveDbBlueprint(blueprint)
if err != nil {
return nil, err
}
@@ -221,7 +228,10 @@ func PatchBlueprint(id uint64, body map[string]interface{}) (*models.Blueprint,
func ReloadBlueprints(c *cron.Cron) errors.Error {
enable := true
isManual := false
- blueprints, _, err := GetDbBlueprints(&BlueprintQuery{Enable: &enable, IsManual: &isManual})
+ blueprints, _, err := bpManager.GetDbBlueprints(&services.GetBlueprintQuery{
+ Enable: &enable,
+ IsManual: &isManual,
+ })
if err != nil {
return err
}
diff --git a/backend/server/services/init.go b/backend/server/services/init.go
index d1eead9fa..e4f9c065b 100644
--- a/backend/server/services/init.go
+++ b/backend/server/services/init.go
@@ -29,6 +29,7 @@ import (
"github.com/apache/incubator-devlake/core/models/migrationscripts"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/core/runner"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/services"
"github.com/apache/incubator-devlake/impls/dalgorm"
"github.com/apache/incubator-devlake/impls/logruslog"
"github.com/apache/incubator-devlake/server/services/auth"
@@ -39,6 +40,8 @@ import (
var cfg config.ConfigReader
var logger log.Logger
var db dal.Dal
+
+var bpManager *services.BlueprintManager
var basicRes context.BasicRes
var migrator plugin.Migrator
var cronManager *cron.Cron
@@ -57,7 +60,7 @@ func InitResources() {
cfg = basicRes.GetConfigReader()
logger = basicRes.GetLogger()
db = basicRes.GetDal()
-
+ bpManager = services.NewBlueprintManager(db)
// initialize db migrator
migrator, err = runner.InitMigrator(basicRes)
if err != nil {
diff --git a/backend/test/e2e/remote/helper.go b/backend/test/e2e/remote/helper.go
index e62cbef1b..e47be0613 100644
--- a/backend/test/e2e/remote/helper.go
+++ b/backend/test/e2e/remote/helper.go
@@ -78,23 +78,21 @@ func CreateTestConnection(client *helper.DevlakeClient) *helper.Connection {
return connection
}
-func CreateTestScope(client *helper.DevlakeClient, connectionId uint64) any {
- res := client.CreateTransformationRule(PLUGIN_NAME, connectionId, FakeTxRule{Name: "Tx rule", Env: "test env"})
- rule, ok := res.(map[string]interface{})
- if !ok {
- panic("Cannot cast transform rule")
- }
- ruleId := uint64(rule["id"].(float64))
-
- scope := client.CreateScope(PLUGIN_NAME,
+func CreateTestScope(client *helper.DevlakeClient, rule *FakeTxRule, connectionId uint64) *FakeProject {
+ scopes := helper.Cast[[]FakeProject](client.CreateScope(PLUGIN_NAME,
connectionId,
FakeProject{
Id: "p1",
Name: "Project 1",
ConnectionId: connectionId,
Url: "http://fake.org/api/project/p1",
- TransformationRuleId: ruleId,
+ TransformationRuleId: rule.Id,
},
- )
- return scope
+ ))
+ return &scopes[0]
+}
+
+func CreateTestTransformationRule(client *helper.DevlakeClient, connectionId uint64) *FakeTxRule {
+ rule := helper.Cast[FakeTxRule](client.CreateTransformationRule(PLUGIN_NAME, connectionId, FakeTxRule{Name: "Tx rule", Env: "test env"}))
+ return &rule
}
diff --git a/backend/test/e2e/remote/python_plugin_test.go b/backend/test/e2e/remote/python_plugin_test.go
index 8c7d385e4..e20ca493b 100644
--- a/backend/test/e2e/remote/python_plugin_test.go
+++ b/backend/test/e2e/remote/python_plugin_test.go
@@ -58,13 +58,11 @@ func TestRemoteScopeGroups(t *testing.T) {
func TestRemoteScopes(t *testing.T) {
client := CreateClient(t)
connection := CreateTestConnection(client)
-
output := client.RemoteScopes(helper.RemoteScopesQuery{
PluginName: PLUGIN_NAME,
ConnectionId: connection.ID,
GroupId: "group1",
})
-
scopes := output.Children
require.Equal(t, 1, len(scopes))
scope := scopes[0]
@@ -82,25 +80,28 @@ func TestRemoteScopes(t *testing.T) {
func TestCreateScope(t *testing.T) {
client := CreateClient(t)
- var connectionId uint64 = 1
-
- CreateTestScope(client, connectionId)
+ conn := CreateTestConnection(client)
+ rule := CreateTestTransformationRule(client, conn.ID)
+ scope := CreateTestScope(client, rule, conn.ID)
- scopes := client.ListScopes(PLUGIN_NAME, connectionId)
+ scopes := client.ListScopes(PLUGIN_NAME, conn.ID, false)
require.Equal(t, 1, len(scopes))
- cicd_scope := helper.Cast[FakeProject](scopes[0])
- require.Equal(t, connectionId, cicd_scope.ConnectionId)
- require.Equal(t, "p1", cicd_scope.Id)
- require.Equal(t, "Project 1", cicd_scope.Name)
- require.Equal(t, "http://fake.org/api/project/p1", cicd_scope.Url)
+
+ cicdScope := helper.Cast[FakeProject](scopes[0].Scope)
+ require.Equal(t, conn.ID, cicdScope.ConnectionId)
+ require.Equal(t, "p1", cicdScope.Id)
+ require.Equal(t, "Project 1", cicdScope.Name)
+ require.Equal(t, "http://fake.org/api/project/p1", cicdScope.Url)
+
+ cicdScope.Name = "scope-name-2"
+ client.UpdateScope(PLUGIN_NAME, conn.ID, cicdScope.Id, scope)
}
func TestRunPipeline(t *testing.T) {
client := CreateClient(t)
conn := CreateTestConnection(client)
-
- CreateTestScope(client, conn.ID)
-
+ rule := CreateTestTransformationRule(client, conn.ID)
+ scope := CreateTestScope(client, rule, conn.ID)
pipeline := client.RunPipeline(models.NewPipeline{
Name: "remote_test",
Plan: []plugin.PipelineStage{
@@ -110,13 +111,12 @@ func TestRunPipeline(t *testing.T) {
Subtasks: nil,
Options: map[string]interface{}{
"connectionId": conn.ID,
- "scopeId": "p1",
+ "scopeId": scope.Id,
},
},
},
},
})
-
require.Equal(t, models.TASK_COMPLETED, pipeline.Status)
require.Equal(t, 1, pipeline.FinishedTasks)
require.Equal(t, "", pipeline.ErrorName)
@@ -129,8 +129,8 @@ func TestBlueprintV200(t *testing.T) {
client.CreateProject(&helper.ProjectConfig{
ProjectName: projectName,
})
- CreateTestScope(client, connection.ID)
-
+ rule := CreateTestTransformationRule(client, connection.ID)
+ scope := CreateTestScope(client, rule, connection.ID)
blueprint := client.CreateBasicBlueprintV2(
"Test blueprint",
&helper.BlueprintV2Config{
@@ -139,7 +139,7 @@ func TestBlueprintV200(t *testing.T) {
ConnectionId: connection.ID,
Scopes: []*plugin.BlueprintScopeV200{
{
- Id: "p1",
+ Id: scope.Id,
Name: "Test scope",
Entities: []string{
plugin.DOMAIN_TYPE_CICD,
@@ -158,8 +158,7 @@ func TestBlueprintV200(t *testing.T) {
project := client.GetProject(projectName)
require.Equal(t, blueprint.Name, project.Blueprint.Name)
- pipeline := client.TriggerBlueprint(blueprint.ID)
- require.Equal(t, pipeline.Status, models.TASK_COMPLETED)
+ client.TriggerBlueprint(blueprint.ID)
}
func TestCreateTxRule(t *testing.T) {
diff --git a/backend/test/helper/api.go b/backend/test/helper/api.go
index 9c8fb18c0..ed6d5c0e4 100644
--- a/backend/test/helper/api.go
+++ b/backend/test/helper/api.go
@@ -26,6 +26,8 @@ import (
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models"
"github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "github.com/apache/incubator-devlake/server/api/blueprints"
apiProject "github.com/apache/incubator-devlake/server/api/project"
"github.com/stretchr/testify/require"
)
@@ -88,6 +90,20 @@ func (d *DevlakeClient) CreateBasicBlueprintV2(name string, config *BlueprintV2C
return blueprint
}
+func (d *DevlakeClient) ListBlueprints() blueprints.PaginatedBlueprint {
+ return sendHttpRequest[blueprints.PaginatedBlueprint](d.testCtx, d.timeout, debugInfo{
+ print: true,
+ inlineJson: false,
+ }, http.MethodGet, fmt.Sprintf("%s/blueprints", d.Endpoint), nil, nil)
+}
+
+func (d *DevlakeClient) GetBlueprint(blueprintId uint64) models.Blueprint {
+ return sendHttpRequest[models.Blueprint](d.testCtx, d.timeout, debugInfo{
+ print: true,
+ inlineJson: false,
+ }, http.MethodGet, fmt.Sprintf("%s/blueprint/%d", d.Endpoint, blueprintId), nil, nil)
+}
+
func (d *DevlakeClient) CreateProject(project *ProjectConfig) models.ApiOutputProject {
var metrics []models.BaseMetric
doraSeen := false
@@ -152,18 +168,30 @@ func (d *DevlakeClient) UpdateScope(pluginName string, connectionId uint64, scop
}, http.MethodPatch, fmt.Sprintf("%s/plugins/%s/connections/%d/scopes/%s", d.Endpoint, pluginName, connectionId, scopeId), nil, scope)
}
-func (d *DevlakeClient) ListScopes(pluginName string, connectionId uint64) []any {
- return sendHttpRequest[[]any](d.testCtx, d.timeout, debugInfo{
+func (d *DevlakeClient) ListScopes(pluginName string, connectionId uint64, listBlueprints bool) []ScopeResponse {
+ scopesRaw := sendHttpRequest[[]map[string]any](d.testCtx, d.timeout, debugInfo{
print: true,
inlineJson: false,
- }, http.MethodGet, fmt.Sprintf("%s/plugins/%s/connections/%d/scopes", d.Endpoint, pluginName, connectionId), nil, nil)
+ }, http.MethodGet, fmt.Sprintf("%s/plugins/%s/connections/%d/scopes?blueprints=%v", d.Endpoint, pluginName, connectionId, listBlueprints), nil, nil)
+ var responses []ScopeResponse
+ for _, scopeRaw := range scopesRaw {
+ responses = append(responses, getScopeResponse(scopeRaw))
+ }
+ return responses
}
-func (d *DevlakeClient) GetScope(pluginName string, connectionId uint64, scopeId string) any {
- return sendHttpRequest[any](d.testCtx, d.timeout, debugInfo{
+func (d *DevlakeClient) GetScope(pluginName string, connectionId uint64, scopeId string, listBlueprints bool) any {
+ return sendHttpRequest[api.ScopeRes[any]](d.testCtx, d.timeout, debugInfo{
+ print: true,
+ inlineJson: false,
+ }, http.MethodGet, fmt.Sprintf("%s/plugins/%s/connections/%d/scopes/%s?blueprints=%v", d.Endpoint, pluginName, connectionId, scopeId, listBlueprints), nil, nil)
+}
+
+func (d *DevlakeClient) DeleteScope(pluginName string, connectionId uint64, scopeId string, deleteDataOnly bool) []models.Blueprint {
+ return sendHttpRequest[[]models.Blueprint](d.testCtx, d.timeout, debugInfo{
print: true,
inlineJson: false,
- }, http.MethodGet, fmt.Sprintf("%s/plugins/%s/connections/%d/scopes/%s", d.Endpoint, pluginName, connectionId, scopeId), nil, nil)
+ }, http.MethodDelete, fmt.Sprintf("%s/plugins/%s/connections/%d/scopes/%s?delete_data_only=%v", d.Endpoint, pluginName, connectionId, scopeId, deleteDataOnly), nil, nil)
}
func (d *DevlakeClient) CreateTransformationRule(pluginName string, connectionId uint64, rules any) any {
@@ -316,3 +344,9 @@ func (d *DevlakeClient) monitorPipeline(id uint64) models.Pipeline {
}))
return pipelineResult
}
+
+func getScopeResponse(scopeRaw map[string]any) ScopeResponse {
+ response := Cast[ScopeResponse](scopeRaw)
+ response.Scope = scopeRaw
+ return response
+}
diff --git a/backend/test/helper/json_helper.go b/backend/test/helper/json_helper.go
index 780cdf0aa..3a771230a 100644
--- a/backend/test/helper/json_helper.go
+++ b/backend/test/helper/json_helper.go
@@ -44,6 +44,18 @@ func ToMap(x any) map[string]any {
return m
}
+func FromMap[T any](x map[string]any) *T {
+ b, err := json.Marshal(x)
+ if err != nil {
+ panic(err)
+ }
+ t := new(T)
+ if err = json.Unmarshal(b, &t); err != nil {
+ panic(err)
+ }
+ return t
+}
+
// ToCleanJson FIXME
func ToCleanJson(inline bool, x any) json.RawMessage {
j, err := json.Marshal(x)
diff --git a/backend/test/helper/models.go b/backend/test/helper/models.go
index 5c3541fa8..a362f4ce7 100644
--- a/backend/test/helper/models.go
+++ b/backend/test/helper/models.go
@@ -18,6 +18,7 @@ limitations under the License.
package helper
import (
+ "github.com/apache/incubator-devlake/core/models"
"time"
"github.com/apache/incubator-devlake/core/plugin"
@@ -35,6 +36,12 @@ type (
EnableDora bool
MetricPlugins []ProjectPlugin
}
+
+ ScopeResponse struct {
+ Scope any
+ TransformationRuleName string
+ Blueprints []*models.Blueprint
+ }
)
type Connection struct {