You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by kl...@apache.org on 2024/04/22 05:49:36 UTC
(incubator-devlake) 01/01: refactor: remote plugins adopts dshelpers
This is an automated email from the ASF dual-hosted git repository.
klesh pushed a commit to branch kw-5519-remoteapi-dshelper
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit c2d64b5412b5d23477df8129bcf1fab6926fd322
Author: Klesh Wong <zh...@merico.dev>
AuthorDate: Mon Apr 22 13:49:02 2024 +0800
refactor: remote plugins adopts dshelpers
---
backend/core/dal/dal.go | 4 +-
backend/core/models/dynamic_tabler.go | 5 +
backend/core/runner/loader.go | 2 +
.../pluginhelper/api/ds_connection_api_helper.go | 56 +-
backend/helpers/pluginhelper/api/ds_helper.go | 17 +-
...{model_api_helper.go => ds_model_api_helper.go} | 136 +++--
.../pluginhelper/api/ds_remote_api_proxy_api.go | 4 +-
.../pluginhelper/api/ds_scope_api_helper.go | 57 +-
.../pluginhelper/api/ds_scope_config_api_helper.go | 42 +-
.../pluginhelper/api/scope_config_helper.go | 184 ------
.../helpers/pluginhelper/api/scope_db_helper.go | 148 -----
.../pluginhelper/api/scope_generic_helper.go | 669 ---------------------
backend/helpers/pluginhelper/api/scope_helper.go | 108 ----
.../helpers/pluginhelper/api/scope_helper_test.go | 356 -----------
..._helper.go => any_connection_service_helper.go} | 50 +-
...rvice_helper.go => any_model_service_helper.go} | 66 +-
...elper.go => any_scope_config_service_helper.go} | 47 +-
...rvice_helper.go => any_scope_service_helper.go} | 128 ++--
.../helpers/srvhelper/connection_service_helper.go | 57 +-
backend/helpers/srvhelper/model_service_helper.go | 188 ++----
.../srvhelper/scope_config_service_helper.go | 48 +-
backend/helpers/srvhelper/scope_service_helper.go | 315 ++--------
backend/impls/dalgorm/dalgorm.go | 12 +-
.../plugins/azuredevops_go/api/blueprint_v200.go | 2 +-
.../plugins/azuredevops_go/api/connection_api.go | 3 +-
backend/plugins/gitlab/api/blueprint_v200.go | 2 +-
backend/python/DevelopmentSetup.md | 3 +-
backend/python/plugins/azuredevops/run.sh | 3 +
backend/server/services/remote/models/models.go | 25 +-
.../services/remote/plugin/connection_api.go | 87 +--
.../server/services/remote/plugin/default_api.go | 26 +-
backend/server/services/remote/plugin/init.go | 3 -
.../services/remote/plugin/plugin_extensions.go | 3 +-
.../server/services/remote/plugin/plugin_impl.go | 32 +-
.../services/remote/plugin/remote_scope_api.go | 6 +-
backend/server/services/remote/plugin/scope_api.go | 108 +---
.../{scope_db_helper.go => scope_db_helper.go.bak} | 0
37 files changed, 545 insertions(+), 2457 deletions(-)
diff --git a/backend/core/dal/dal.go b/backend/core/dal/dal.go
index 48cc36010..6cd5e9a26 100644
--- a/backend/core/dal/dal.go
+++ b/backend/core/dal/dal.go
@@ -162,7 +162,7 @@ type Dal interface {
// RenameTable renames table name
RenameTable(oldName, newName string) errors.Error
// GetColumns returns table columns in database
- GetColumns(dst Tabler, filter func(columnMeta ColumnMeta) bool) (cms []ColumnMeta, err errors.Error)
+ GetColumns(dst interface{}, filter func(columnMeta ColumnMeta) bool) (cms []ColumnMeta, err errors.Error)
// GetPrimaryKeyFields get the PrimaryKey from `gorm` tag
GetPrimaryKeyFields(t reflect.Type) []reflect.StructField
// RenameColumn renames column name for specified table
@@ -252,7 +252,7 @@ func GetColumnNames(d Dal, dst Tabler, filter func(columnMeta ColumnMeta) bool)
}
// GetPrimarykeyColumns get returns PrimaryKey table Meta in database
-func GetPrimarykeyColumns(d Dal, dst Tabler) ([]ColumnMeta, errors.Error) {
+func GetPrimarykeyColumns(d Dal, dst interface{}) ([]ColumnMeta, errors.Error) {
return d.GetColumns(dst, func(columnMeta ColumnMeta) bool {
isPrimaryKey, ok := columnMeta.PrimaryKey()
return isPrimaryKey && ok
diff --git a/backend/core/models/dynamic_tabler.go b/backend/core/models/dynamic_tabler.go
index bd3c6c444..dc0e2aa39 100644
--- a/backend/core/models/dynamic_tabler.go
+++ b/backend/core/models/dynamic_tabler.go
@@ -33,6 +33,7 @@ type DynamicTabler interface {
dal.Tabler
json.Marshaler
json.Unmarshaler
+ ModelName() string
NewValue() any
New() DynamicTabler
NewSlice() DynamicTabler
@@ -57,6 +58,10 @@ func NewDynamicTabler(tableName string, objType reflect.Type) DynamicTabler {
}
}
+func (d *DynamicTablerImpl) ModelName() string {
+ return d.table
+}
+
func (d *DynamicTablerImpl) NewValue() any {
return reflect.New(d.objType).Interface()
}
diff --git a/backend/core/runner/loader.go b/backend/core/runner/loader.go
index 6e435a172..400e04d41 100644
--- a/backend/core/runner/loader.go
+++ b/backend/core/runner/loader.go
@@ -81,6 +81,7 @@ func LoadGoPlugins(basicRes context.BasicRes) errors.Error {
}
func LoadRemotePlugins(basicRes context.BasicRes) errors.Error {
+ logger := basicRes.GetLogger().Nested("remote-plugins")
remotePluginDir := basicRes.GetConfig("REMOTE_PLUGIN_DIR")
if remotePluginDir != "" {
basicRes.GetLogger().Info("Loading remote plugins")
@@ -94,6 +95,7 @@ func LoadRemotePlugins(basicRes context.BasicRes) errors.Error {
invoker := bridge.NewCmdInvoker(path)
result := invoker.Call("plugin-info", bridge.DefaultContext)
if result.Err != nil {
+ logger.Error(result.Err, "Error calling plugin-info")
return errors.Default.Wrap(result.Err, "Error calling plugin-info")
}
pluginInfo := &models.PluginInfo{}
diff --git a/backend/helpers/pluginhelper/api/ds_connection_api_helper.go b/backend/helpers/pluginhelper/api/ds_connection_api_helper.go
index 27eeb4383..bd033ae36 100644
--- a/backend/helpers/pluginhelper/api/ds_connection_api_helper.go
+++ b/backend/helpers/pluginhelper/api/ds_connection_api_helper.go
@@ -23,33 +23,31 @@ import (
"github.com/apache/incubator-devlake/server/api/shared"
"github.com/apache/incubator-devlake/core/context"
+ "github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/srvhelper"
)
-// DsConnectionApiHelper
-type DsConnectionApiHelper[C plugin.ToolLayerConnection, S plugin.ToolLayerScope, SC plugin.ToolLayerScopeConfig] struct {
- *ModelApiHelper[C]
- *srvhelper.ConnectionSrvHelper[C, S, SC]
+// DsAnyConnectionApiHelper
+type DsAnyConnectionApiHelper struct {
+ *AnyModelApiHelper
+ *srvhelper.AnyConnectionSrvHelper
}
-func NewDsConnectionApiHelper[
- C plugin.ToolLayerConnection,
- S plugin.ToolLayerScope,
- SC plugin.ToolLayerScopeConfig](
+func NewAnyDsConnectionApiHelper(
basicRes context.BasicRes,
- connSrvHelper *srvhelper.ConnectionSrvHelper[C, S, SC],
- sterilizer func(c C) C,
-) *DsConnectionApiHelper[C, S, SC] {
- return &DsConnectionApiHelper[C, S, SC]{
- ModelApiHelper: NewModelApiHelper[C](basicRes, connSrvHelper.ModelSrvHelper, []string{"connectionId"}, sterilizer),
- ConnectionSrvHelper: connSrvHelper,
+ connSrvHelper *srvhelper.AnyConnectionSrvHelper,
+ sterilizer func(c any) any,
+) *DsAnyConnectionApiHelper {
+ return &DsAnyConnectionApiHelper{
+ AnyModelApiHelper: NewAnyModelApiHelper(basicRes, connSrvHelper.AnyModelSrvHelper, []string{"connectionId"}, sterilizer),
+ AnyConnectionSrvHelper: connSrvHelper,
}
}
-func (connApi *DsConnectionApiHelper[C, S, SC]) GetMergedConnection(input *plugin.ApiResourceInput) (*C, errors.Error) {
- connection, err := connApi.FindByPk(input)
+func (connApi *DsAnyConnectionApiHelper) GetMergedConnectionAny(input *plugin.ApiResourceInput) (any, errors.Error) {
+ connection, err := connApi.FindByPkAny(input)
if err != nil {
return nil, errors.BadInput.Wrap(err, "find connection from db")
}
@@ -61,13 +59,12 @@ func (connApi *DsConnectionApiHelper[C, S, SC]) GetMergedConnection(input *plugi
return connection, nil
}
-func (connApi *DsConnectionApiHelper[C, S, SC]) Delete(input *plugin.ApiResourceInput) (out *plugin.ApiResourceOutput, err errors.Error) {
- var conn *C
- conn, err = connApi.FindByPk(input)
+func (connApi *DsAnyConnectionApiHelper) Delete(input *plugin.ApiResourceInput) (out *plugin.ApiResourceOutput, err errors.Error) {
+ conn, err := connApi.FindByPkAny(input)
if err != nil {
return nil, err
}
- refs, err := connApi.ConnectionSrvHelper.DeleteConnection(conn)
+ refs, err := connApi.DeleteConnectionAny(conn)
if err != nil {
return &plugin.ApiResourceOutput{Body: &shared.ApiBody{
Success: false,
@@ -92,3 +89,22 @@ func extractConnectionId(input *plugin.ApiResourceInput) (uint64, errors.Error)
}
return id, nil
}
+
+type DsConnectionApiHelper[C dal.Tabler] struct {
+ *DsAnyConnectionApiHelper
+ *ModelApiHelper[C]
+}
+
+func NewDsConnectionApiHelper[C dal.Tabler](
+ anyConnectionApiHelper *DsAnyConnectionApiHelper,
+) *DsConnectionApiHelper[C] {
+ return &DsConnectionApiHelper[C]{
+ DsAnyConnectionApiHelper: anyConnectionApiHelper,
+ ModelApiHelper: NewModelApiHelper[C](anyConnectionApiHelper.AnyModelApiHelper),
+ }
+}
+
+func (connApi *DsConnectionApiHelper[C]) GetMergedConnection(input *plugin.ApiResourceInput) (*C, errors.Error) {
+ connection, err := connApi.GetMergedConnectionAny(input)
+ return connection.(*C), err
+}
diff --git a/backend/helpers/pluginhelper/api/ds_helper.go b/backend/helpers/pluginhelper/api/ds_helper.go
index 0b14356e3..a4ce9a08d 100644
--- a/backend/helpers/pluginhelper/api/ds_helper.go
+++ b/backend/helpers/pluginhelper/api/ds_helper.go
@@ -33,11 +33,11 @@ type DsHelper[
SC plugin.ToolLayerScopeConfig,
] struct {
ConnSrv *srvhelper.ConnectionSrvHelper[C, S, SC]
- ConnApi *DsConnectionApiHelper[C, S, SC]
+ ConnApi *DsConnectionApiHelper[C]
ScopeSrv *srvhelper.ScopeSrvHelper[C, S, SC]
- ScopeApi *DsScopeApiHelper[C, S, SC]
+ ScopeApi *DsScopeApiHelper
ScopeConfigSrv *srvhelper.ScopeConfigSrvHelper[C, S, SC]
- ScopeConfigApi *DsScopeConfigApiHelper[C, S, SC]
+ ScopeConfigApi *DsScopeConfigApiHelper
}
func NewDataSourceHelper[
@@ -53,16 +53,19 @@ func NewDataSourceHelper[
scopeConfigSterilizer func(s SC) SC,
) *DsHelper[C, S, SC] {
connSrv := srvhelper.NewConnectionSrvHelper[C, S, SC](basicRes, pluginName)
- connApi := NewDsConnectionApiHelper[C, S, SC](basicRes, connSrv, connectionSterilizer)
+ connApiAny := NewAnyDsConnectionApiHelper(basicRes, connSrv.AnyConnectionSrvHelper, func(c any) any {
+ return connectionSterilizer(c.(C))
+ })
+ connApi := NewDsConnectionApiHelper[C](connApiAny)
scopeSrv := srvhelper.NewScopeSrvHelper[C, S, SC](basicRes, pluginName, scopeSearchColumns)
- scopeApi := NewDsScopeApiHelper[C, S, SC](basicRes, scopeSrv, scopeSterilizer)
+ scopeApi := NewDsScopeApiHelper(basicRes, scopeSrv.AnyScopeSrvHelper)
var scSrv *srvhelper.ScopeConfigSrvHelper[C, S, SC]
- var scApi *DsScopeConfigApiHelper[C, S, SC]
+ var scApi *DsScopeConfigApiHelper
scType := reflect.TypeOf(new(SC))
if scType != noScopeConfig {
scSrv = srvhelper.NewScopeConfigSrvHelper[C, S, SC](basicRes, scopeSearchColumns)
- scApi = NewDsScopeConfigApiHelper[C, S, SC](basicRes, scSrv, scopeConfigSterilizer)
+ scApi = NewDsScopeConfigApiHelper(basicRes, scSrv.AnyScopeConfigSrvHelper)
}
return &DsHelper[C, S, SC]{
ConnSrv: connSrv,
diff --git a/backend/helpers/pluginhelper/api/model_api_helper.go b/backend/helpers/pluginhelper/api/ds_model_api_helper.go
similarity index 55%
rename from backend/helpers/pluginhelper/api/model_api_helper.go
rename to backend/helpers/pluginhelper/api/ds_model_api_helper.go
index dc41c69e4..813199664 100644
--- a/backend/helpers/pluginhelper/api/model_api_helper.go
+++ b/backend/helpers/pluginhelper/api/ds_model_api_helper.go
@@ -20,6 +20,7 @@ package api
import (
"fmt"
"net/http"
+ "reflect"
"github.com/apache/incubator-devlake/core/context"
"github.com/apache/incubator-devlake/core/dal"
@@ -33,56 +34,56 @@ import (
var vld = validator.New()
-type ModelApiHelper[M dal.Tabler] struct {
+type AnyModelApiHelper struct {
+ *srvhelper.AnyModelSrvHelper
basicRes context.BasicRes
- dalHelper *srvhelper.ModelSrvHelper[M]
log log.Logger
modelName string
pkPathVarNames []string
- sterilizers []func(m M) M
+ sterilizers []func(m any) any
}
-func NewModelApiHelper[M dal.Tabler](
+func NewAnyModelApiHelper(
basicRes context.BasicRes,
- dalHelper *srvhelper.ModelSrvHelper[M],
+ srvHelper *srvhelper.AnyModelSrvHelper,
pkPathVarNames []string, // path variable names of primary key
- sterilizer func(m M) M,
-) *ModelApiHelper[M] {
- m := new(M)
+ sterilizer func(m any) any,
+) *AnyModelApiHelper {
+ m := srvHelper.New()
modelName := fmt.Sprintf("%T", m)
- modelApiHelper := &ModelApiHelper[M]{
- basicRes: basicRes,
- dalHelper: dalHelper,
- log: basicRes.GetLogger().Nested(fmt.Sprintf("%s_dal", modelName)),
- modelName: modelName,
- pkPathVarNames: pkPathVarNames,
+ modelApiHelper := &AnyModelApiHelper{
+ AnyModelSrvHelper: srvHelper,
+ basicRes: basicRes,
+ log: basicRes.GetLogger().Nested(fmt.Sprintf("%s_dal", modelName)),
+ modelName: modelName,
+ pkPathVarNames: pkPathVarNames,
}
if sterilizer != nil {
- modelApiHelper.sterilizers = []func(m M) M{sterilizer}
+ modelApiHelper.sterilizers = []func(m any) any{sterilizer}
}
return modelApiHelper
}
-func (self *ModelApiHelper[M]) Post(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
- model := new(M)
+func (modelApi *AnyModelApiHelper) Post(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
+ model := modelApi.New()
err := utils.DecodeMapStruct(input.Body, model, false)
if err != nil {
return nil, err
}
- err = self.dalHelper.Create(model)
+ err = modelApi.CreateAny(model)
if err != nil {
return nil, err
}
- model = self.Sanitize(model)
+ model = modelApi.Sanitize(model)
return &plugin.ApiResourceOutput{
Status: http.StatusCreated,
Body: model,
}, nil
}
-func (self *ModelApiHelper[M]) ExtractPkValues(input *plugin.ApiResourceInput) ([]interface{}, errors.Error) {
- pkv := make([]interface{}, len(self.pkPathVarNames))
- for i, pkn := range self.pkPathVarNames {
+func (modelApi *AnyModelApiHelper) ExtractPkValues(input *plugin.ApiResourceInput) ([]interface{}, errors.Error) {
+ pkv := make([]interface{}, len(modelApi.pkPathVarNames))
+ for i, pkn := range modelApi.pkPathVarNames {
var ok bool
pkv[i], ok = input.Params[pkn]
if !ok {
@@ -92,107 +93,108 @@ func (self *ModelApiHelper[M]) ExtractPkValues(input *plugin.ApiResourceInput) (
return pkv, nil
}
-func (self *ModelApiHelper[M]) FindByPk(input *plugin.ApiResourceInput) (*M, errors.Error) {
- pkv, err := self.ExtractPkValues(input)
+func (modelApi *AnyModelApiHelper) FindByPkAny(input *plugin.ApiResourceInput) (any, errors.Error) {
+ pkv, err := modelApi.ExtractPkValues(input)
if err != nil {
return nil, err
}
- return self.dalHelper.FindByPk(pkv...)
+ return modelApi.AnyModelSrvHelper.FindByPkAny(pkv...)
}
-func (self *ModelApiHelper[M]) GetDetail(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
- model, err := self.FindByPk(input)
+func (modelApi *AnyModelApiHelper) GetDetail(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
+ model, err := modelApi.FindByPkAny(input)
if err != nil {
return nil, err
}
- model = self.Sanitize(model)
+ model = modelApi.Sanitize(model)
return &plugin.ApiResourceOutput{
Body: model,
}, nil
}
-func (self *ModelApiHelper[M]) Sanitize(model *M) *M {
- if self.sterilizers != nil {
- for _, sterilizer := range self.sterilizers {
- sanitizedModel := sterilizer(*model)
+func (modelApi *AnyModelApiHelper) Sanitize(model any) any {
+ if modelApi.sterilizers != nil {
+ for _, sterilizer := range modelApi.sterilizers {
+ sanitizedModel := sterilizer(model)
model = &sanitizedModel
}
}
return model
}
-func (self *ModelApiHelper[M]) BatchSanitize(models []*M) []*M {
- for idx, m := range models {
- model := *m
- models[idx] = self.Sanitize(&model)
+func (modelApi *AnyModelApiHelper) BatchSanitize(models any) any {
+ array := reflect.ValueOf(models)
+ for i := 0; i < array.Len(); i++ {
+ model := array.Index(i)
+ model.Set(reflect.ValueOf(modelApi.Sanitize(model.Interface())))
}
return models
}
-type CustomMerge[M dal.Tabler] interface {
- MergeFromRequest(target *M, body map[string]interface{}) error
+type CustomMerge interface {
+ MergeFromRequest(target any, body map[string]interface{}) error
}
-// PatchModel will get an "M" from database and try to merge update from request body
+// PatchModelAny will get an "M" from database and try to merge update from request body
// zeroFields decides whether "M" will be zeroed if "M" doesn't implement CustomMerge.
-func (self *ModelApiHelper[M]) PatchModel(input *plugin.ApiResourceInput, zeroFields bool) (*M, error) {
- model, err := self.FindByPk(input)
+func (modelApi *AnyModelApiHelper) PatchModel(input *plugin.ApiResourceInput, zeroFields bool) (any, error) {
+ model, err := modelApi.FindByPkAny(input)
if err != nil {
return nil, err
}
- if v, ok := (interface{}(model)).(CustomMerge[M]); ok {
+ if v, ok := (model).(CustomMerge); ok {
if err := v.MergeFromRequest(model, input.Body); err != nil {
return nil, err
}
} else {
err = utils.DecodeMapStruct(input.Body, model, zeroFields)
if err != nil {
- return nil, errors.BadInput.Wrap(err, fmt.Sprintf("faled to patch %s", self.modelName))
+ return nil, errors.BadInput.Wrap(err, fmt.Sprintf("faled to patch %s", modelApi.modelName))
}
}
return model, nil
}
-func (self *ModelApiHelper[M]) Patch(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
- model, err := self.PatchModel(input, true)
+func (modelApi *AnyModelApiHelper) Patch(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
+ model, err := modelApi.PatchModel(input, true)
if err != nil {
return nil, errors.Convert(err)
}
- if err := self.dalHelper.Update(model); err != nil {
+ if err := modelApi.UpdateAny(model); err != nil {
return nil, err
}
- model = self.Sanitize(model)
+ model = modelApi.Sanitize(model)
return &plugin.ApiResourceOutput{
Body: model,
}, nil
}
-func (self *ModelApiHelper[M]) Delete(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
- model, err := self.FindByPk(input)
+func (modelApi *AnyModelApiHelper) Delete(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
+ model, err := modelApi.FindByPkAny(input)
if err != nil {
return nil, err
}
- err = self.dalHelper.DeleteModel(model)
+ err = modelApi.DeleteModelAny(model)
if err != nil {
return nil, err
}
- model = self.Sanitize(model)
+ model = modelApi.Sanitize(model)
return &plugin.ApiResourceOutput{
Body: model,
}, nil
}
-func (self *ModelApiHelper[M]) GetAll(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
- all, err := self.dalHelper.GetAll()
- all = self.BatchSanitize(all)
+func (modelApi *AnyModelApiHelper) GetAll(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
+ all, err := modelApi.QueryAllAny()
+ all = modelApi.BatchSanitize(all)
return &plugin.ApiResourceOutput{
Body: all,
}, err
}
-func (self *ModelApiHelper[M]) PutMultipleCb(input *plugin.ApiResourceInput, beforeSave func(*M) errors.Error) (*plugin.ApiResourceOutput, errors.Error) {
+func (modelApi *AnyModelApiHelper) PutMultipleCb(input *plugin.ApiResourceInput, beforeSave func(any) errors.Error) (*plugin.ApiResourceOutput, errors.Error) {
var req struct {
- Data []*M `json:"data"`
+ Data []any `json:"data"`
}
err := utils.DecodeMapStruct(input.Body, &req, false)
if err != nil {
@@ -205,12 +207,13 @@ func (self *ModelApiHelper[M]) PutMultipleCb(input *plugin.ApiResourceInput, bef
return nil, err
}
}
- err := self.dalHelper.CreateOrUpdate(item)
+ err := modelApi.CreateOrUpdateAny(item)
if err != nil {
return nil, errors.BadInput.Wrap(err, fmt.Sprintf("failed to save item %d", i))
}
}
- req.Data = self.BatchSanitize(req.Data)
+ // TODO
+ // req.Data = modelApi.BatchSanitize(req.Data)
return &plugin.ApiResourceOutput{
Body: req.Data,
}, nil
@@ -237,3 +240,20 @@ func parsePagination[P any](input *plugin.ApiResourceInput) (*P, errors.Error) {
}
return pagination, nil
}
+
+type ModelApiHelper[M dal.Tabler] struct {
+ apiHelper *AnyModelApiHelper
+}
+
+func NewModelApiHelper[M dal.Tabler](
+ anyModelApiHelper *AnyModelApiHelper,
+) *ModelApiHelper[M] {
+ return &ModelApiHelper[M]{
+ apiHelper: anyModelApiHelper,
+ }
+}
+
+func (modelApi *ModelApiHelper[M]) FindByPk(input *plugin.ApiResourceInput) (*M, errors.Error) {
+ model, err := modelApi.apiHelper.FindByPkAny(input)
+ return model.(*M), err
+}
diff --git a/backend/helpers/pluginhelper/api/ds_remote_api_proxy_api.go b/backend/helpers/pluginhelper/api/ds_remote_api_proxy_api.go
index a0ec27f39..48e1650c8 100644
--- a/backend/helpers/pluginhelper/api/ds_remote_api_proxy_api.go
+++ b/backend/helpers/pluginhelper/api/ds_remote_api_proxy_api.go
@@ -44,7 +44,7 @@ func NewDsRemoteApiProxyHelper[
) *DsRemoteApiProxyHelper[C] {
return &DsRemoteApiProxyHelper[C]{
ModelApiHelper: modelApiHelper,
- logger: modelApiHelper.basicRes.GetLogger().Nested("remote_api_helper"),
+ logger: modelApiHelper.apiHelper.basicRes.GetLogger().Nested("remote_api_helper"),
httpClientCache: make(map[string]*ApiClient),
}
}
@@ -73,7 +73,7 @@ func (rap *DsRemoteApiProxyHelper[C]) getApiClient(connection *C) (*ApiClient, e
return rap.httpClientCache[key], nil
}
// create new client if cache missed
- client, err := NewApiClientFromConnection(gocontext.TODO(), rap.basicRes, c.(plugin.ApiConnection))
+ client, err := NewApiClientFromConnection(gocontext.TODO(), rap.apiHelper.basicRes, c.(plugin.ApiConnection))
if err != nil {
return nil, err
}
diff --git a/backend/helpers/pluginhelper/api/ds_scope_api_helper.go b/backend/helpers/pluginhelper/api/ds_scope_api_helper.go
index eb0b8af51..fdca51b40 100644
--- a/backend/helpers/pluginhelper/api/ds_scope_api_helper.go
+++ b/backend/helpers/pluginhelper/api/ds_scope_api_helper.go
@@ -22,6 +22,7 @@ import (
"github.com/apache/incubator-devlake/core/context"
"github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/models"
"github.com/apache/incubator-devlake/core/models/common"
"github.com/apache/incubator-devlake/core/plugin"
serviceHelper "github.com/apache/incubator-devlake/helpers/pluginhelper/services"
@@ -29,39 +30,38 @@ import (
"github.com/apache/incubator-devlake/server/api/shared"
)
+// for documentation purposes
type ScopeRefDoc = serviceHelper.BlueprintProjectPairs
-
type PutScopesReqBody[T any] struct {
Data []*T `json:"data"`
}
+type ScopeDetail[S plugin.ToolLayerScope, SC plugin.ToolLayerScopeConfig] struct {
+ Scope S `json:"scope"`
+ ScopeConfig *SC `json:"scopeConfig,omitempty"`
+ Blueprints []*models.Blueprint `json:"blueprints,omitempty"`
+}
-type ScopeDetail[S plugin.ToolLayerScope, SC plugin.ToolLayerScopeConfig] srvhelper.ScopeDetail[S, SC]
-
-type DsScopeApiHelper[C plugin.ToolLayerConnection, S plugin.ToolLayerScope, SC plugin.ToolLayerScopeConfig] struct {
- *ModelApiHelper[S]
- *srvhelper.ScopeSrvHelper[C, S, SC]
+type DsScopeApiHelper struct {
+ *AnyModelApiHelper
+ *srvhelper.AnyScopeSrvHelper
}
-func NewDsScopeApiHelper[
- C plugin.ToolLayerConnection,
- S plugin.ToolLayerScope,
- SC plugin.ToolLayerScopeConfig](
+func NewDsScopeApiHelper(
basicRes context.BasicRes,
- srvHelper *srvhelper.ScopeSrvHelper[C, S, SC],
- sterilizer func(s S) S,
-) *DsScopeApiHelper[C, S, SC] {
- return &DsScopeApiHelper[C, S, SC]{
- ModelApiHelper: NewModelApiHelper[S](basicRes, srvHelper.ModelSrvHelper, []string{"connectionId", "scopeId"}, sterilizer),
- ScopeSrvHelper: srvHelper,
+ srvHelper *srvhelper.AnyScopeSrvHelper,
+) *DsScopeApiHelper {
+ return &DsScopeApiHelper{
+ AnyModelApiHelper: NewAnyModelApiHelper(basicRes, srvHelper.AnyModelSrvHelper, []string{"connectionId", "scopeId"}, nil),
+ AnyScopeSrvHelper: srvHelper,
}
}
-func (scopeApi *DsScopeApiHelper[C, S, SC]) GetPage(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
+func (scopeApi *DsScopeApiHelper) GetPage(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
pagination, err := parsePagination[srvhelper.ScopePagination](input)
if err != nil {
return nil, errors.BadInput.Wrap(err, "failed to decode pathvars into pagination")
}
- scopes, count, err := scopeApi.ScopeSrvHelper.GetScopesPage(pagination)
+ scopes, count, err := scopeApi.GetScopesPageAny(pagination)
if err != nil {
return nil, err
}
@@ -73,12 +73,12 @@ func (scopeApi *DsScopeApiHelper[C, S, SC]) GetPage(input *plugin.ApiResourceInp
}, nil
}
-func (scopeApi *DsScopeApiHelper[C, S, SC]) GetScopeDetail(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
+func (scopeApi *DsScopeApiHelper) GetScopeDetail(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
pkv, err := scopeApi.ExtractPkValues(input)
if err != nil {
return nil, err
}
- scopeDetail, err := scopeApi.ScopeSrvHelper.GetScopeDetail(input.Query.Get("blueprints") == "true", pkv...)
+ scopeDetail, err := scopeApi.GetScopeDetailAny(input.Query.Get("blueprints") == "true", pkv...)
if err != nil {
return nil, err
}
@@ -87,12 +87,12 @@ func (scopeApi *DsScopeApiHelper[C, S, SC]) GetScopeDetail(input *plugin.ApiReso
}, nil
}
-func (scopeApi *DsScopeApiHelper[C, S, SC]) GetScopeLatestSyncState(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
+func (scopeApi *DsScopeApiHelper) GetScopeLatestSyncState(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
pkv, err := scopeApi.ExtractPkValues(input)
if err != nil {
return nil, err
}
- scopeLatestSyncStates, err := scopeApi.ScopeSrvHelper.GetScopeLatestSyncState(pkv...)
+ scopeLatestSyncStates, err := scopeApi.GetScopeLatestSyncStateAny(pkv...)
if err != nil {
return nil, err
}
@@ -101,7 +101,7 @@ func (scopeApi *DsScopeApiHelper[C, S, SC]) GetScopeLatestSyncState(input *plugi
}, nil
}
-func (scopeApi *DsScopeApiHelper[C, S, SC]) PutMultiple(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
+func (scopeApi *DsScopeApiHelper) PutMultiple(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
// fix data[].connectionId
connectionId, err := extractConnectionId(input)
if err != nil {
@@ -118,10 +118,10 @@ func (scopeApi *DsScopeApiHelper[C, S, SC]) PutMultiple(input *plugin.ApiResourc
}
dict["connectionId"] = connectionId
}
- return scopeApi.ModelApiHelper.PutMultipleCb(input, func(m *S) errors.Error {
+ return scopeApi.PutMultipleCb(input, func(m any) errors.Error {
ok := setRawDataOrigin(m, common.RawDataOrigin{
RawDataTable: fmt.Sprintf("_raw_%s_scopes", scopeApi.GetPluginName()),
- RawDataParams: plugin.MarshalScopeParams((*m).ScopeParams()),
+ RawDataParams: plugin.MarshalScopeParams(scopeApi.GetScopeParams(m)),
})
if !ok {
panic("set raw data origin failed")
@@ -130,15 +130,14 @@ func (scopeApi *DsScopeApiHelper[C, S, SC]) PutMultiple(input *plugin.ApiResourc
})
}
-func (scopeApi *DsScopeApiHelper[C, S, SC]) Delete(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
- var scope *S
- scope, err := scopeApi.FindByPk(input)
+func (scopeApi *DsScopeApiHelper) Delete(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
+ scope, err := scopeApi.FindByPkAny(input)
if err != nil {
return nil, err
}
// time.Sleep(1 * time.Minute) # uncomment this line if you were to verify pipelines get blocked while deleting data
// check referencing blueprints
- refs, err := scopeApi.ScopeSrvHelper.DeleteScope(scope, input.Query.Get("delete_data_only") == "true")
+ refs, err := scopeApi.DeleteScopeAny(scope, input.Query.Get("delete_data_only") == "true")
if err != nil {
return &plugin.ApiResourceOutput{Body: &shared.ApiBody{
Success: false,
diff --git a/backend/helpers/pluginhelper/api/ds_scope_config_api_helper.go b/backend/helpers/pluginhelper/api/ds_scope_config_api_helper.go
index 6e76341ef..9a834c588 100644
--- a/backend/helpers/pluginhelper/api/ds_scope_config_api_helper.go
+++ b/backend/helpers/pluginhelper/api/ds_scope_config_api_helper.go
@@ -26,64 +26,58 @@ import (
)
// DsScopeConfigApiHelper
-type DsScopeConfigApiHelper[C plugin.ToolLayerConnection, S plugin.ToolLayerScope, SC plugin.ToolLayerScopeConfig] struct {
- *ModelApiHelper[SC]
- *srvhelper.ScopeConfigSrvHelper[C, S, SC]
+type DsScopeConfigApiHelper struct {
+ *AnyModelApiHelper
+ *srvhelper.AnyScopeConfigSrvHelper
}
-func NewDsScopeConfigApiHelper[
- C plugin.ToolLayerConnection,
- S plugin.ToolLayerScope,
- SC plugin.ToolLayerScopeConfig,
-](
+func NewDsScopeConfigApiHelper(
basicRes context.BasicRes,
- dalHelper *srvhelper.ScopeConfigSrvHelper[C, S, SC],
- sterilizer func(sc SC) SC,
-) *DsScopeConfigApiHelper[C, S, SC] {
- return &DsScopeConfigApiHelper[C, S, SC]{
- ModelApiHelper: NewModelApiHelper[SC](basicRes, dalHelper.ModelSrvHelper, []string{"scopeConfigId"}, sterilizer),
- ScopeConfigSrvHelper: dalHelper,
+ srvHelper *srvhelper.AnyScopeConfigSrvHelper,
+) *DsScopeConfigApiHelper {
+ return &DsScopeConfigApiHelper{
+ AnyModelApiHelper: NewAnyModelApiHelper(basicRes, srvHelper.AnyModelSrvHelper, []string{"connectionId", "scopeId"}, nil),
+ AnyScopeConfigSrvHelper: srvHelper,
}
}
-func (connApi *DsScopeConfigApiHelper[C, S, SC]) GetAll(input *plugin.ApiResourceInput) (out *plugin.ApiResourceOutput, err errors.Error) {
+func (scopeConfigApi *DsScopeConfigApiHelper) GetAll(input *plugin.ApiResourceInput) (out *plugin.ApiResourceOutput, err errors.Error) {
connectionId, err := extractConnectionId(input)
if err != nil {
return nil, err
}
- scopeConfigs := errors.Must1(connApi.ScopeConfigSrvHelper.GetAllByConnectionId(connectionId))
+ scopeConfigs := errors.Must1(scopeConfigApi.GetAllByConnectionIdAny(connectionId))
return &plugin.ApiResourceOutput{
Body: scopeConfigs,
}, nil
}
-func (connApi *DsScopeConfigApiHelper[C, S, SC]) Post(input *plugin.ApiResourceInput) (out *plugin.ApiResourceOutput, err errors.Error) {
+func (scopeConfigApi *DsScopeConfigApiHelper) Post(input *plugin.ApiResourceInput) (out *plugin.ApiResourceOutput, err errors.Error) {
// fix connectionId
connectionId, err := extractConnectionId(input)
if err != nil {
return nil, err
}
input.Body["connectionId"] = connectionId
- return connApi.ModelApiHelper.Post(input)
+ return scopeConfigApi.AnyModelApiHelper.Post(input)
}
-func (connApi *DsScopeConfigApiHelper[C, S, SC]) Patch(input *plugin.ApiResourceInput) (out *plugin.ApiResourceOutput, err errors.Error) {
+func (scopeConfigApi *DsScopeConfigApiHelper) Patch(input *plugin.ApiResourceInput) (out *plugin.ApiResourceOutput, err errors.Error) {
// fix connectionId
connectionId, err := extractConnectionId(input)
if err != nil {
return nil, err
}
input.Body["connectionId"] = connectionId
- return connApi.ModelApiHelper.Patch(input)
+ return scopeConfigApi.AnyModelApiHelper.Patch(input)
}
-func (connApi *DsScopeConfigApiHelper[C, S, SC]) Delete(input *plugin.ApiResourceInput) (out *plugin.ApiResourceOutput, err errors.Error) {
- var scopeConfig *SC
- scopeConfig, err = connApi.FindByPk(input)
+func (scopeConfigApi *DsScopeConfigApiHelper) Delete(input *plugin.ApiResourceInput) (out *plugin.ApiResourceOutput, err errors.Error) {
+ scopeConfig, err := scopeConfigApi.FindByPkAny(input)
if err != nil {
return nil, err
}
- refs, err := connApi.ScopeConfigSrvHelper.DeleteScopeConfig(scopeConfig)
+ refs, err := scopeConfigApi.DeleteScopeConfigAny(scopeConfig)
if err != nil {
return &plugin.ApiResourceOutput{Body: &shared.ApiBody{
Success: false,
diff --git a/backend/helpers/pluginhelper/api/scope_config_helper.go b/backend/helpers/pluginhelper/api/scope_config_helper.go
deleted file mode 100644
index e1da762f5..000000000
--- a/backend/helpers/pluginhelper/api/scope_config_helper.go
+++ /dev/null
@@ -1,184 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package api
-
-import (
- "github.com/apache/incubator-devlake/core/models/common"
- "net/http"
- "reflect"
- "strconv"
-
- "github.com/apache/incubator-devlake/core/context"
- "github.com/apache/incubator-devlake/core/dal"
- "github.com/apache/incubator-devlake/core/errors"
- "github.com/apache/incubator-devlake/core/log"
- "github.com/apache/incubator-devlake/core/plugin"
- "github.com/apache/incubator-devlake/helpers/pluginhelper/api/apihelperabstract"
- "github.com/go-playground/validator/v10"
-)
-
-// ScopeConfigHelper is used to write the CURD of scope config
-type ScopeConfigHelper[ScopeConfig dal.Tabler, ScopeConfigOperator common.ScopeConfigOperator[ScopeConfig]] struct {
- log log.Logger
- db dal.Dal
- validator *validator.Validate
- pluginName string
-}
-
-// NewScopeConfigHelper creates a ScopeConfigHelper for scope config management
-func NewScopeConfigHelper[Tr dal.Tabler, ScopeConfigOperator common.ScopeConfigOperator[Tr]](
- basicRes context.BasicRes,
- vld *validator.Validate,
- pluginName string,
-) *ScopeConfigHelper[Tr, ScopeConfigOperator] {
- if vld == nil {
- vld = validator.New()
- }
- return &ScopeConfigHelper[Tr, ScopeConfigOperator]{
- log: basicRes.GetLogger(),
- db: basicRes.GetDal(),
- validator: vld,
- pluginName: pluginName,
- }
-}
-
-func (t ScopeConfigHelper[ScopeConfig, ScopeConfigOperator]) Create(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
- connectionId, e := strconv.ParseUint(input.Params["connectionId"], 10, 64)
- if e != nil || connectionId == 0 {
- return nil, errors.Default.Wrap(e, "the connection ID should be an non-zero integer")
- }
- config := new(ScopeConfig)
- if err := DecodeMapStruct(input.Body, config, false); err != nil {
- return nil, errors.Default.Wrap(err, "error in decoding scope config")
- }
- var scopeConfigOperator ScopeConfigOperator = new(ScopeConfig)
- scopeConfigOperator.SetConnectionId(config, connectionId)
- if t.validator != nil {
- if err := t.validator.Struct(config); err != nil {
- return nil, errors.Default.Wrap(err, "error validating scope config")
- }
- }
- if cv, ok := interface{}(config).(apihelperabstract.ComplexValidate); ok {
- err := cv.Validate()
- if err != nil {
- return nil, err
- }
- }
- valueConnectionId := reflect.ValueOf(config).Elem().FieldByName("ConnectionId")
- if valueConnectionId.IsValid() {
- valueConnectionId.SetUint(connectionId)
- }
-
- if err := t.db.Create(config); err != nil {
- if t.db.IsDuplicationError(err) {
- return nil, errors.BadInput.New("there was a scope config with the same name, please choose another name")
- }
- return nil, errors.BadInput.Wrap(err, "error on saving ScopeConfig")
- }
- return &plugin.ApiResourceOutput{Body: config, Status: http.StatusOK}, nil
-}
-
-func (t ScopeConfigHelper[ScopeConfig, ScopeConfigOperator]) Update(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
- scopeConfigId, e := strconv.ParseUint(input.Params["id"], 10, 64)
- if e != nil {
- return nil, errors.Default.Wrap(e, "the scope config ID should be an integer")
- }
- config := new(ScopeConfig)
- err := t.db.First(config, dal.Where("id = ?", scopeConfigId))
- if err != nil {
- return nil, errors.Default.Wrap(err, "error on saving ScopeConfig")
- }
- err = DecodeMapStruct(input.Body, config, true)
- if err != nil {
- return nil, errors.Default.Wrap(err, "error decoding map into scopeConfig")
- }
- if cv, ok := interface{}(config).(apihelperabstract.ComplexValidate); ok {
- err := cv.Validate()
- if err != nil {
- return nil, err
- }
- }
- err = t.db.Update(config, dal.Where("id = ?", scopeConfigId))
- if err != nil {
- if t.db.IsDuplicationError(err) {
- return nil, errors.BadInput.New("there was a scope config with the same name, please choose another name")
- }
- return nil, errors.BadInput.Wrap(err, "error on saving ScopeConfig")
- }
- return &plugin.ApiResourceOutput{Body: config, Status: http.StatusOK}, nil
-}
-
-func (t ScopeConfigHelper[ScopeConfig, ScopeConfigOperator]) Get(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
- scopeConfigId, err := strconv.ParseUint(input.Params["id"], 10, 64)
- if err != nil {
- return nil, errors.Default.Wrap(err, "the scope config ID should be an integer")
- }
- var config ScopeConfig
- err = t.db.First(&config, dal.Where("id = ?", scopeConfigId))
- if err != nil {
- return nil, errors.Default.Wrap(err, "error on get ScopeConfig")
- }
- return &plugin.ApiResourceOutput{Body: config, Status: http.StatusOK}, nil
-}
-
-func (t ScopeConfigHelper[ScopeConfig, ScopeConfigOperator]) List(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
- connectionId, e := strconv.ParseUint(input.Params["connectionId"], 10, 64)
- if e != nil || connectionId == 0 {
- return nil, errors.Default.Wrap(e, "the connection ID should be an non-zero integer")
- }
- var configs []ScopeConfig
- limit, offset := GetLimitOffset(input.Query, "pageSize", "page")
- err := t.db.All(&configs, dal.Where("connection_id = ?", connectionId), dal.Limit(limit), dal.Offset(offset))
- if err != nil {
- return nil, errors.Default.Wrap(err, "error on get ScopeConfig list")
- }
- return &plugin.ApiResourceOutput{Body: configs, Status: http.StatusOK}, nil
-}
-
-func (t ScopeConfigHelper[ScopeConfig, ScopeConfigOperator]) Delete(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
- scopeConfigId, err := errors.Convert01(strconv.ParseUint(input.Params["id"], 10, 64))
- if err != nil {
- return nil, errors.Default.Wrap(err, "the scope config ID should be an integer")
- }
- connectionId, err := errors.Convert01(strconv.ParseUint(input.Params["connectionId"], 10, 64))
- if err != nil {
- return nil, errors.Default.Wrap(err, "the scope connection ID should be an integer")
- }
- var config ScopeConfig
- err = t.db.Delete(&config, dal.Where("id = ? AND connection_id = ?", scopeConfigId, connectionId))
- if err != nil {
- return nil, errors.Default.Wrap(err, "error deleting ScopeConfig")
- }
- err = t.nullOutScopeReferences(scopeConfigId)
- if err != nil {
- return nil, err
- }
- return &plugin.ApiResourceOutput{Body: nil, Status: http.StatusOK}, nil
-}
-func (t ScopeConfigHelper[ScopeConfig, ScopeConfigOperator]) nullOutScopeReferences(scopeConfigId uint64) errors.Error {
- p, _ := plugin.GetPlugin(t.pluginName)
- pluginSrc, ok := p.(plugin.PluginSource)
- if !ok {
- return errors.Default.New("plugin doesn't implement PluginSource")
- }
- scopeModel := pluginSrc.Scope()
- if scopeModel == nil {
- return nil
- }
- return t.db.UpdateColumn(scopeModel.TableName(), "scope_config_id", nil, dal.Where("scope_config_id = ?", scopeConfigId))
-}
diff --git a/backend/helpers/pluginhelper/api/scope_db_helper.go b/backend/helpers/pluginhelper/api/scope_db_helper.go
deleted file mode 100644
index 0a3d0e2b8..000000000
--- a/backend/helpers/pluginhelper/api/scope_db_helper.go
+++ /dev/null
@@ -1,148 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package api
-
-import (
- "fmt"
- "reflect"
-
- "github.com/apache/incubator-devlake/core/context"
- "github.com/apache/incubator-devlake/core/dal"
- "github.com/apache/incubator-devlake/core/errors"
- "github.com/apache/incubator-devlake/core/plugin"
-)
-
-type ScopeDatabaseHelper[Conn any, Scope plugin.ToolLayerScope, Tr any] interface {
- VerifyConnection(connectionId uint64) errors.Error
- SaveScope(scopes []*Scope) errors.Error
- UpdateScope(scope *Scope) errors.Error
- GetScope(connectionId uint64, scopeId string) (*Scope, errors.Error)
- ListScopes(input *plugin.ApiResourceInput, connectionId uint64) ([]*Scope, int64, errors.Error)
-
- DeleteScope(scope *Scope) errors.Error
- GetScopeConfig(ruleId uint64) (*Tr, errors.Error)
- ListScopeConfigs(ruleIds []uint64) ([]*Tr, errors.Error)
- GetScopeAndConfig(connectionId uint64, scopeId string) (*Scope, *Tr, errors.Error)
-}
-
-type ScopeDatabaseHelperImpl[Conn any, Scope plugin.ToolLayerScope, Tr any] struct {
- ScopeDatabaseHelper[Conn, Scope, Tr]
- db dal.Dal
- connHelper *ConnectionApiHelper
- params *ReflectionParameters
-}
-
-func NewScopeDatabaseHelperImpl[Conn any, Scope plugin.ToolLayerScope, Tr any](
- basicRes context.BasicRes, connHelper *ConnectionApiHelper, params *ReflectionParameters) *ScopeDatabaseHelperImpl[Conn, Scope, Tr] {
- return &ScopeDatabaseHelperImpl[Conn, Scope, Tr]{
- db: basicRes.GetDal(),
- connHelper: connHelper,
- params: params,
- }
-}
-
-func (s *ScopeDatabaseHelperImpl[Conn, Scope, Tr]) VerifyConnection(connectionId uint64) errors.Error {
- var conn Conn
- err := s.connHelper.FirstById(&conn, connectionId)
- if err != nil {
- if s.db.IsErrorNotFound(err) {
- return errors.BadInput.New("Invalid Connection Id")
- }
- return err
- }
- return nil
-}
-
-func (s *ScopeDatabaseHelperImpl[Conn, Scope, Tr]) SaveScope(scopes []*Scope) errors.Error {
- err := s.db.CreateOrUpdate(&scopes)
- if err != nil {
- if s.db.IsDuplicationError(err) {
- return errors.BadInput.New("the scope already exists")
- }
- return err
- }
- return nil
-}
-
-func (s *ScopeDatabaseHelperImpl[Conn, Scope, Tr]) UpdateScope(scope *Scope) errors.Error {
- return s.db.Update(&scope)
-}
-
-func (s *ScopeDatabaseHelperImpl[Conn, Scope, Tr]) GetScope(connectionId uint64, scopeId string) (*Scope, errors.Error) {
- query := dal.Where(fmt.Sprintf("connection_id = ? AND %s = ?", s.params.ScopeIdColumnName), connectionId, scopeId)
- scope := new(Scope)
- err := s.db.First(scope, query)
- return scope, err
-}
-
-func (s *ScopeDatabaseHelperImpl[Conn, Scope, Tr]) GetScopeAndConfig(connectionId uint64, scopeId string) (*Scope, *Tr, errors.Error) {
- scope, err := s.GetScope(connectionId, scopeId)
- if err != nil {
- return nil, nil, err
- }
- scopeConfig := new(Tr)
- scIdField := reflectField(scope, "ScopeConfigId")
- if scIdField.IsValid() {
- if scIdField.Uint() != 0 {
- err = s.db.First(scopeConfig, dal.Where("id = ?", scIdField.Uint()))
- if err != nil {
- return nil, nil, err
- }
- }
- }
- entitiesField := reflectField(scopeConfig, "Entities")
- if entitiesField.IsValid() && entitiesField.IsNil() {
- entitiesField.Set(reflect.ValueOf(plugin.DOMAIN_TYPES))
- }
- return scope, scopeConfig, nil
-}
-
-func (s *ScopeDatabaseHelperImpl[Conn, Scope, Tr]) ListScopes(input *plugin.ApiResourceInput, connectionId uint64) ([]*Scope, int64, errors.Error) {
- searchTerm := input.Query.Get("searchTerm")
- query := dal.Where("connection_id = ?", connectionId)
- if searchTerm != "" && s.params.SearchScopeParamName != "" {
- query = dal.Where(fmt.Sprintf("connection_id = ? AND %s LIKE ?", s.params.SearchScopeParamName), connectionId, "%"+searchTerm+"%")
- }
- limit, offset := GetLimitOffset(input.Query, "pageSize", "page")
- var scopes []*Scope
- count, err := s.db.Count(dal.From(new(Scope)), query)
- if err != nil {
- return nil, 0, err
- }
- err = s.db.All(&scopes, query, dal.Limit(limit), dal.Offset(offset))
- return scopes, count, err
-}
-
-func (s *ScopeDatabaseHelperImpl[Conn, Scope, Tr]) DeleteScope(scope *Scope) errors.Error {
- err := s.db.Delete(&scope)
- return err
-}
-
-func (s *ScopeDatabaseHelperImpl[Conn, Scope, Tr]) GetScopeConfig(ruleId uint64) (*Tr, errors.Error) {
- var rule Tr
- err := s.db.First(&rule, dal.Where("id = ?", ruleId))
- return &rule, err
-}
-
-func (s *ScopeDatabaseHelperImpl[Conn, Scope, Tr]) ListScopeConfigs(ruleIds []uint64) ([]*Tr, errors.Error) {
- var rules []*Tr
- err := s.db.All(&rules, dal.Where("id IN (?)", ruleIds))
- return rules, err
-}
-
-var _ ScopeDatabaseHelper[any, plugin.ToolLayerScope, any] = &ScopeDatabaseHelperImpl[any, plugin.ToolLayerScope, any]{}
diff --git a/backend/helpers/pluginhelper/api/scope_generic_helper.go b/backend/helpers/pluginhelper/api/scope_generic_helper.go
deleted file mode 100644
index a782b4e09..000000000
--- a/backend/helpers/pluginhelper/api/scope_generic_helper.go
+++ /dev/null
@@ -1,669 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package api
-
-import (
- "fmt"
- "reflect"
- "strconv"
- "strings"
- "time"
-
- "github.com/apache/incubator-devlake/core/context"
- "github.com/apache/incubator-devlake/core/dal"
- "github.com/apache/incubator-devlake/core/errors"
- "github.com/apache/incubator-devlake/core/log"
- "github.com/apache/incubator-devlake/core/models"
- "github.com/apache/incubator-devlake/core/models/common"
- "github.com/apache/incubator-devlake/core/models/domainlayer/domaininfo"
- "github.com/apache/incubator-devlake/core/plugin"
- "github.com/apache/incubator-devlake/helpers/dbhelper"
- serviceHelper "github.com/apache/incubator-devlake/helpers/pluginhelper/services"
- "github.com/go-playground/validator/v10"
-)
-
-type NoScopeConfig struct{}
-
-type (
- GenericScopeApiHelper[Conn any, Scope plugin.ToolLayerScope, ScopeConfig any] struct {
- basicRes context.BasicRes
- log log.Logger
- db dal.Dal
- validator *validator.Validate
- reflectionParams *ReflectionParameters
- dbHelper ScopeDatabaseHelper[Conn, Scope, ScopeConfig]
- bpManager *serviceHelper.BlueprintManager
- connHelper *ConnectionApiHelper
- opts *ScopeHelperOptions
- plugin string
- }
- // as of golang v1.20, embedding generic fields is not supported
- // let's divide the struct into two parts for swagger doc to work
- // https://stackoverflow.com/questions/66118867/go-generics-is-it-possible-to-embed-generic-structs
- ScopeResDoc[ScopeConfig any] struct {
- ScopeConfig *ScopeConfig `mapstructure:"scopeConfig,omitempty" json:"scopeConfig"`
- Blueprints []*models.Blueprint `mapstructure:"blueprints,omitempty" json:"blueprints"`
- }
- // Alias, for swagger purposes
- ScopeRes[Scope plugin.ToolLayerScope, ScopeConfig any] struct {
- Scope Scope `mapstructure:"scope,omitempty" json:"scope,omitempty"`
- ScopeConfig *ScopeConfig `mapstructure:"scopeConfig,omitempty" json:"scopeConfig,omitempty"`
- Blueprints []*models.Blueprint `mapstructure:"blueprints,omitempty" json:"blueprints,omitempty"`
- }
- ScopeListRes[Scope plugin.ToolLayerScope, ScopeConfig any] struct {
- Scopes []*ScopeRes[Scope, ScopeConfig] `mapstructure:"scopes" json:"scopes"`
- Count int64 `mapstructure:"count" json:"count"`
- }
- ReflectionParameters struct {
- // This corresponds to the struct field of the scope struct's ID field
- ScopeIdFieldName string `validate:"required"`
- // This corresponds to the database column name of the scope struct's ID (typically primary key) field
- ScopeIdColumnName string `validate:"required"`
- // This corresponds to the scope field on the ApiParams struct of a plugin.
- RawScopeParamName string `validate:"required"`
- // This corresponds to the scope field for allowing data scope search.
- SearchScopeParamName string
- }
- ScopeHelperOptions struct {
- // Define this if the raw params doesn't store the ScopeId but a different attribute of the Scope (e.g. Name)
- GetScopeParamValue func(db dal.Dal, scopeId string) (string, errors.Error)
- IsRemote bool
- }
-)
-
-type (
- requestParams struct {
- connectionId uint64
- scopeId string
- plugin string
- }
- deleteRequestParams struct {
- requestParams
- deleteDataOnly bool
- }
-
- getRequestParams struct {
- requestParams
- loadBlueprints bool
- }
-)
-
-func NewGenericScopeHelper[Conn any, Scope plugin.ToolLayerScope, ScopeConfig any](
- basicRes context.BasicRes,
- vld *validator.Validate,
- connHelper *ConnectionApiHelper,
- dbHelper ScopeDatabaseHelper[Conn, Scope, ScopeConfig],
- params *ReflectionParameters,
- opts *ScopeHelperOptions,
-) *GenericScopeApiHelper[Conn, Scope, ScopeConfig] {
- if connHelper == nil {
- panic("nil connHelper")
- }
- if params == nil {
- panic("reflection params not provided")
- }
- err := vld.Struct(params)
- if err != nil {
- panic(err)
- }
- if opts == nil {
- opts = &ScopeHelperOptions{}
- }
- return &GenericScopeApiHelper[Conn, Scope, ScopeConfig]{
- basicRes: basicRes,
- log: basicRes.GetLogger(),
- db: basicRes.GetDal(),
- validator: vld,
- reflectionParams: params,
- dbHelper: dbHelper,
- bpManager: serviceHelper.NewBlueprintManager(basicRes.GetDal()),
- connHelper: connHelper,
- opts: opts,
- plugin: connHelper.pluginName,
- }
-}
-
-func (gs *GenericScopeApiHelper[Conn, Scope, ScopeConfig]) DbHelper() ScopeDatabaseHelper[Conn, Scope, ScopeConfig] {
- return gs.dbHelper
-}
-
-// hacky, temporary solution
-func (gs *GenericScopeApiHelper[Conn, Scope, ScopeConfig]) setRawDataOrigin(scopes ...*Scope) {
- for _, scope := range scopes {
- if !setRawDataOrigin(scope, common.RawDataOrigin{
- RawDataTable: fmt.Sprintf("_raw_%s_scopes", gs.plugin),
- RawDataParams: plugin.MarshalScopeParams((*scope).ScopeParams()),
- }) {
- panic("RawDataOrigin could not be set")
- }
- }
-}
-
-func (gs *GenericScopeApiHelper[Conn, Scope, ScopeConfig]) PutScopes(input *plugin.ApiResourceInput, scopes []*Scope) ([]*ScopeRes[Scope, ScopeConfig], errors.Error) {
- params, err := gs.extractFromReqParam(input, false)
- if err != nil {
- return nil, err
- }
- err = gs.dbHelper.VerifyConnection(params.connectionId)
- if err != nil {
- return nil, errors.Default.Wrap(err, fmt.Sprintf("error verifying connection for connection ID %d", params.connectionId))
- }
- if len(scopes) == 0 {
- return nil, nil
- }
- err = gs.validatePrimaryKeys(scopes)
- if err != nil {
- return nil, err
- }
- now := time.Now()
- for _, scope := range scopes {
- // Set the connection ID, CreatedAt, and UpdatedAt fields
- gs.setScopeFields(scope, params.connectionId, &now, &now)
- err = gs.verifyScope(scope, gs.validator)
- if err != nil {
- return nil, errors.Default.Wrap(err, "error verifying scope")
- }
- }
- gs.setRawDataOrigin(scopes...)
- if err != nil {
- return nil, errors.Default.Wrap(err, "error saving scope")
- }
- err = gs.dbHelper.SaveScope(scopes)
- if err != nil {
- return nil, errors.Default.Wrap(err, "error saving scope")
- }
- apiScopes, err := gs.addScopeConfig(scopes...)
- if err != nil {
- return nil, errors.Default.Wrap(err, "error associating scope config to scope")
- }
- return apiScopes, nil
-}
-
-func (gs *GenericScopeApiHelper[Conn, Scope, ScopeConfig]) UpdateScope(input *plugin.ApiResourceInput) (*ScopeRes[Scope, ScopeConfig], errors.Error) {
- params, err := gs.extractFromReqParam(input, true)
- if err != nil {
- return nil, err
- }
- err = gs.dbHelper.VerifyConnection(params.connectionId)
- if err != nil {
- return nil, err
- }
- scope, err := gs.dbHelper.GetScope(params.connectionId, params.scopeId)
- if err != nil {
- return nil, err
- }
- err = DecodeMapStruct(input.Body, scope, true)
- if err != nil {
- return nil, errors.Default.Wrap(err, "patch scope error")
- }
- err = gs.verifyScope(scope, gs.validator)
- if err != nil {
- return nil, errors.Default.Wrap(err, "Invalid scope")
- }
- gs.setRawDataOrigin(scope)
- err = gs.dbHelper.UpdateScope(scope)
- if err != nil {
- return nil, errors.Default.Wrap(err, "error on saving Scope")
- }
- scopeRes, err := gs.addScopeConfig(scope)
- if err != nil {
- return nil, err
- }
- return scopeRes[0], nil
-}
-
-func (gs *GenericScopeApiHelper[Conn, Scope, ScopeConfig]) GetScopes(input *plugin.ApiResourceInput) (*ScopeListRes[Scope, ScopeConfig], errors.Error) {
- params, err := gs.extractFromGetReqParam(input, false)
- if err != nil {
- return nil, errors.BadInput.New("invalid path params: \"connectionId\" not set")
- }
- err = gs.dbHelper.VerifyConnection(params.connectionId)
- if err != nil {
- return nil, errors.Default.Wrap(err, fmt.Sprintf("error verifying connection for connection ID %d", params.connectionId))
- }
- scopes, count, err := gs.dbHelper.ListScopes(input, params.connectionId)
- if err != nil {
- return nil, err
- }
- apiScopes, err := gs.addScopeConfig(scopes...)
- if err != nil {
- return nil, errors.Default.Wrap(err, "error associating scope configs with scopes")
- }
- // return empty array rather than nil in case of no scopes
- if len(apiScopes) > 0 && params.loadBlueprints {
- for _, apiScope := range apiScopes {
- apiScope.Blueprints = gs.bpManager.GetBlueprintsByScopeId(params.connectionId, params.plugin, apiScope.Scope.ScopeId())
- }
- }
- return &ScopeListRes[Scope, ScopeConfig]{
- Scopes: apiScopes,
- Count: count,
- }, nil
-}
-
-func (gs *GenericScopeApiHelper[Conn, Scope, ScopeConfig]) GetScope(input *plugin.ApiResourceInput) (*ScopeRes[Scope, ScopeConfig], errors.Error) {
- params, err := gs.extractFromGetReqParam(input, true)
- if err != nil {
- return nil, err
- }
- err = gs.dbHelper.VerifyConnection(params.connectionId)
- if err != nil {
- return nil, errors.Default.Wrap(err, fmt.Sprintf("error verifying connection for connection ID %d", params.connectionId))
- }
- scope, err := gs.dbHelper.GetScope(params.connectionId, params.scopeId)
- if err != nil {
- return nil, errors.Default.Wrap(err, fmt.Sprintf("error retrieving scope with scope ID %s", params.scopeId))
- }
- apiScopes, err := gs.addScopeConfig(scope)
- if err != nil {
- return nil, errors.Default.Wrap(err, fmt.Sprintf("error associating scope config with scope %s", params.scopeId))
- }
- scopeRes := apiScopes[0]
- if params.loadBlueprints {
- scopeRes.Blueprints = gs.bpManager.GetBlueprintsByScopeId(params.connectionId, params.plugin, params.scopeId)
- }
- return scopeRes, nil
-}
-
-func (gs *GenericScopeApiHelper[Conn, Scope, ScopeConfig]) DeleteScope(input *plugin.ApiResourceInput) (refs *serviceHelper.BlueprintProjectPairs, err errors.Error) {
- txHelper := dbhelper.NewTxHelper(gs.basicRes, &err)
- defer txHelper.End()
- tx := txHelper.Begin()
- err = txHelper.LockTablesTimeout(2*time.Second, dal.LockTables{{Table: "_devlake_pipelines"}})
- if err != nil {
- err = errors.Conflict.Wrap(err, "This data scope cannot be deleted due to a table lock error. There might be running pipeline(s) or other deletion operations in progress.")
- return
- }
- count := errors.Must1(tx.Count(
- dal.From("_devlake_pipelines"),
- dal.Where("status = ?", models.TASK_RUNNING),
- ))
- if count > 0 {
- err = errors.Conflict.New("This data scope cannot be deleted because a pipeline is running. Please try again after you cancel the pipeline or wait for it to finish.")
- return
- }
- // time.Sleep(1 * time.Minute) # uncomment this line if you were to verify pipelines get blocked while deleting data
- params, err := gs.extractFromDeleteReqParam(input)
- if err != nil {
- return nil, err
- }
- err = gs.dbHelper.VerifyConnection(params.connectionId)
- if err != nil {
- return nil, errors.BadInput.Wrap(err, fmt.Sprintf("error verifying connection for connection ID %d", params.connectionId))
- }
- scope, err := gs.dbHelper.GetScope(params.connectionId, params.scopeId)
- if err != nil {
- return nil, err
- }
-
- if !params.deleteDataOnly {
- blueprints := gs.bpManager.GetBlueprintsByScopeId(params.connectionId, params.plugin, params.scopeId)
- if len(blueprints) > 0 {
- refs = &serviceHelper.BlueprintProjectPairs{}
- for _, bp := range blueprints {
- refs.Blueprints = append(refs.Blueprints, bp.Name)
- refs.Projects = append(refs.Projects, bp.ProjectName)
- }
- return refs, errors.Conflict.New("Found one or more references to this scope")
- }
- }
- if err = gs.deleteScopeData(*scope); err != nil {
- return nil, err
- }
- if !params.deleteDataOnly {
- // Delete the scope itself
- errors.Must(gs.dbHelper.DeleteScope(scope))
- }
- return nil, nil
-}
-
-func (gs *GenericScopeApiHelper[Conn, Scope, ScopeConfig]) addScopeConfig(scopes ...*Scope) ([]*ScopeRes[Scope, ScopeConfig], errors.Error) {
- apiScopes := make([]*ScopeRes[Scope, ScopeConfig], len(scopes))
- for i, scope := range scopes {
- apiScopes[i] = &ScopeRes[Scope, ScopeConfig]{
- Scope: *scope,
- }
- scIdField := reflectField(scope, "ScopeConfigId")
- if scIdField.IsValid() && scIdField.Uint() > 0 {
- scopeConfig, err := gs.dbHelper.GetScopeConfig(scIdField.Uint())
- if err != nil {
- return nil, err
- }
- apiScopes[i].ScopeConfig = scopeConfig
- }
- }
- return apiScopes, nil
-}
-
-func (gs *GenericScopeApiHelper[Conn, Scope, ScopeConfig]) extractFromReqParam(input *plugin.ApiResourceInput, withScopeId bool) (*requestParams, errors.Error) {
- connectionId, err := strconv.ParseUint(input.Params["connectionId"], 10, 64)
- if err != nil {
- return nil, errors.BadInput.Wrap(err, "Invalid \"connectionId\"")
- }
- if connectionId == 0 {
- return nil, errors.BadInput.New("\"connectionId\" cannot be 0")
- }
- var scopeId string
- if withScopeId {
- scopeId = input.Params["scopeId"]
- // Path params that use `/*param` handlers instead of `/:param` start with a /, so remove it
- if scopeId[0] == '/' {
- scopeId = scopeId[1:]
- }
- }
- return &requestParams{
- connectionId: connectionId,
- plugin: gs.plugin,
- scopeId: scopeId,
- }, nil
-}
-
-func (gs *GenericScopeApiHelper[Conn, Scope, ScopeConfig]) extractFromDeleteReqParam(input *plugin.ApiResourceInput) (*deleteRequestParams, errors.Error) {
- params, err := gs.extractFromReqParam(input, true)
- if err != nil {
- return nil, err
- }
- var deleteDataOnly bool
- {
- ddo, ok := input.Query["delete_data_only"]
- if ok {
- deleteDataOnly, err = errors.Convert01(strconv.ParseBool(ddo[0]))
- if err != nil {
- deleteDataOnly = false
- }
- }
- }
- return &deleteRequestParams{
- requestParams: *params,
- deleteDataOnly: deleteDataOnly,
- }, nil
-}
-
-func (gs *GenericScopeApiHelper[Conn, Scope, ScopeConfig]) extractFromGetReqParam(input *plugin.ApiResourceInput, withScopeId bool) (*getRequestParams, errors.Error) {
- params, err := gs.extractFromReqParam(input, withScopeId)
- if err != nil {
- return nil, err
- }
- var loadBlueprints bool
- {
- lbps, ok := input.Query["blueprints"]
- if ok {
- loadBlueprints, err = errors.Convert01(strconv.ParseBool(lbps[0]))
- if err != nil {
- loadBlueprints = false
- }
- }
- }
- return &getRequestParams{
- requestParams: *params,
- loadBlueprints: loadBlueprints,
- }, nil
-}
-
-func (gs *GenericScopeApiHelper[Conn, Scope, ScopeConfig]) createRawParams(connectionId uint64, scopeId any) string {
- // TODO for future: have ScopeParams expose a constructor so we pass the variables to that instead of this hack
- paramsMap := map[string]any{
- "ConnectionId": connectionId,
- gs.reflectionParams.RawScopeParamName: scopeId,
- }
- return plugin.MarshalScopeParams(paramsMap)
-}
-
-func (gs *GenericScopeApiHelper[Conn, Scope, ScopeConfig]) setScopeFields(p interface{}, connectionId uint64, createdAt *time.Time, updatedAt *time.Time) {
- pType := reflect.TypeOf(p)
- if pType.Kind() != reflect.Ptr {
- panic("expected a pointer to a struct")
- }
- pValue := reflectValue(p)
- // set connectionId
- connIdField := pValue.FieldByName("ConnectionId")
- connIdField.SetUint(connectionId)
-
- // set raw params
- rawParams := pValue.FieldByName("RawDataParams")
- if !rawParams.IsValid() {
- panic("scope is missing the field \"RawDataParams\"")
- }
- scopeIdField := pValue.FieldByName(gs.reflectionParams.ScopeIdFieldName)
- rawParams.Set(reflect.ValueOf(gs.createRawParams(connectionId, scopeIdField.Interface())))
-
- // set CreatedAt
- createdAtField := pValue.FieldByName("CreatedAt")
- if createdAtField.IsValid() && createdAtField.Type().AssignableTo(reflect.TypeOf(createdAt)) {
- createdAtField.Set(reflect.ValueOf(createdAt))
- }
-
- // set UpdatedAt
- updatedAtField := pValue.FieldByName("UpdatedAt")
- if !updatedAtField.IsValid() || (updatedAt != nil && !updatedAtField.Type().AssignableTo(reflect.TypeOf(updatedAt))) {
- return
- }
- if updatedAt == nil {
- // if updatedAt is nil, set UpdatedAt to be nil
- updatedAtField.Set(reflect.Zero(updatedAtField.Type()))
- } else {
- // if updatedAt is not nil, set UpdatedAt to be the value
- updatedAtFieldValue := reflect.ValueOf(updatedAt)
- updatedAtField.Set(updatedAtFieldValue)
- }
-}
-
-// returnPrimaryKeyValue returns a string containing the primary key value(s) of a struct, concatenated with "-" between them.
-// This function receives an interface{} type argument p, which can be a pointer to any struct.
-// The function uses reflection to iterate through the fields of the struct, and checks if each field is tagged as "primaryKey".
-func returnPrimaryKeyValue(p interface{}) string {
- result := ""
- // get the type and value of the input interface using reflection
- t := reflectType(p)
- v := reflectValue(p)
- // iterate over each field in the struct type
- for i := 0; i < t.NumField(); i++ {
- // get the i-th field
- field := t.Field(i)
-
- // check if the field is marked as "primaryKey" in the struct tag
- if strings.Contains(string(field.Tag), "primaryKey") {
- // if this is the first primaryKey field encountered, set the result to be its value
- if result == "" {
- result = fmt.Sprintf("%v", v.Field(i).Interface())
- } else {
- // if this is not the first primaryKey field, append its value to the result with a "-" separator
- result = fmt.Sprintf("%s-%v", result, v.Field(i).Interface())
- }
- }
- }
-
- // return the final primary key value as a string
- return result
-}
-
-func (gs *GenericScopeApiHelper[Conn, Scope, ScopeConfig]) verifyScope(scope interface{}, vld *validator.Validate) errors.Error {
- if gs.opts.IsRemote {
- return nil
- }
- pType := reflect.TypeOf(scope)
- if pType.Kind() != reflect.Ptr {
- panic("expected a pointer to a struct")
- }
- if err := vld.Struct(scope); err != nil {
- return errors.Default.Wrap(err, "error validating target")
- }
- return nil
-}
-
-func (gs *GenericScopeApiHelper[Conn, Scope, ScopeConfig]) validatePrimaryKeys(scopes []*Scope) errors.Error {
- if gs.opts.IsRemote {
- return nil
- }
- keeper := make(map[string]struct{})
- for _, scope := range scopes {
- // Ensure that the primary key value is unique
- primaryValueStr := returnPrimaryKeyValue(scope)
- if _, ok := keeper[primaryValueStr]; ok {
- return errors.BadInput.New("duplicate scope was requested")
- } else {
- keeper[primaryValueStr] = struct{}{}
- }
- }
- return nil
-}
-
-func (gs *GenericScopeApiHelper[Conn, Scope, ScopeConfig]) deleteScopeData(scope plugin.ToolLayerScope) errors.Error {
- // find all tables for this plugin
- tables, err := gs.getAffectedTables(gs.plugin)
- if err != nil {
- return errors.Default.Wrap(err, fmt.Sprintf("error getting database tables managed by plugin %s", gs.plugin))
- }
- scopeParams := plugin.MarshalScopeParams(scope.ScopeParams())
- err = gs.transactionalDelete(tables, scopeParams)
- if err != nil {
- return errors.Default.Wrap(err, fmt.Sprintf("error deleting data bound to scope %s for plugin %s", scopeParams, gs.plugin))
- }
- return nil
-}
-
-func (gs *GenericScopeApiHelper[Conn, Scope, ScopeConfig]) transactionalDelete(tables []string, rawDataParams string) errors.Error {
- generateWhereClause := func(table string) (string, []any) {
- var where string
- var params []interface{}
- if strings.HasPrefix(table, "_raw_") {
- // raw table: should check connection and scope
- where = "params = ?"
- params = []interface{}{rawDataParams}
- } else if strings.HasPrefix(table, "_tool_") {
- // tool layer table: should check connection and scope
- where = "_raw_data_params = ?"
- params = []interface{}{rawDataParams}
- } else {
- // framework tables: should check plugin, connection and scope
- if table == (models.CollectorLatestState{}.TableName()) {
- // diff sync state
- where = "raw_data_table LIKE ? AND raw_data_params = ?"
- } else {
- // domain layer table
- where = "_raw_data_table LIKE ? AND _raw_data_params = ?"
- }
- rawDataTablePrefix := fmt.Sprintf("_raw_%s%%", gs.plugin)
- params = []interface{}{rawDataTablePrefix, rawDataParams}
- }
- return where, params
- }
- tx := gs.db.Begin()
- for _, table := range tables {
- where, params := generateWhereClause(table)
- gs.log.Info("deleting data from table %s with WHERE \"%s\" and params: \"%v\"", table, where, params)
- sql := fmt.Sprintf("DELETE FROM %s WHERE %s", table, where)
- err := tx.Exec(sql, params...)
- if err != nil {
- err2 := tx.Rollback()
- if err2 != nil {
- gs.log.Warn(err2, fmt.Sprintf("error rolling back table data deletion transaction. sql: %s params: %v", sql, params))
- }
- return err
- }
- }
- err := tx.Commit()
- if err != nil {
- return errors.Default.Wrap(err, "error committing delete transaction for plugin tables")
- }
- // validate everything was deleted
- var failedTables []string
- for _, table := range tables {
- where, params := generateWhereClause(table)
- count, err := gs.db.Count(dal.From(table), dal.Where(where, params...))
- if err != nil {
- return err
- }
- if count > 0 {
- failedTables = append(failedTables, table)
- }
- }
- if len(failedTables) > 0 {
- return errors.Default.New(fmt.Sprintf("Failed to delete all expected rows from the following table(s): %v", failedTables))
- }
- return nil
-}
-
-// GetScopeLatestSyncState only works for remote plugins.
-// Make sure all remote plugins save their state in table `_devlake_collector_latest_state`.
-// For golang version plugin, use `dsHelper.ScopeApi.GetScopeLatestSyncState` instead.
-func (gs *GenericScopeApiHelper[Conn, Scope, ScopeConfig]) GetScopeLatestSyncState(input *plugin.ApiResourceInput) ([]*models.LatestSyncState, errors.Error) {
- scope, err := gs.GetScope(input)
- if err != nil {
- return nil, err
- }
- params := plugin.MarshalScopeParams(scope.Scope.ScopeParams())
- scopeSyncStates := []*models.LatestSyncState{}
- if err := gs.db.All(
- &scopeSyncStates,
- dal.Select("raw_data_table, latest_success_start, raw_data_params"),
- dal.From("_devlake_collector_latest_state"),
- dal.Where("raw_data_params = ?", params),
- ); err != nil {
- return nil, err
- }
- return scopeSyncStates, nil
-}
-
-func (gs *GenericScopeApiHelper[Conn, Scope, ScopeConfig]) getAffectedTables(pluginName string) ([]string, errors.Error) {
- var tables []string
- meta, err := plugin.GetPlugin(pluginName)
- if err != nil {
- return nil, err
- }
- if pluginModel, ok := meta.(plugin.PluginModel); !ok {
- return nil, errors.Default.New(fmt.Sprintf("plugin \"%s\" does not implement listing its tables", pluginName))
- } else {
- // Unfortunately, can't cache the tables because Python creates some tables on a per-demand basis, so such a cache would possibly get outdated.
- // It's a rare scenario in practice, but might as well play it safe and sacrifice some performance here
- var allTables []string
- if allTables, err = gs.db.AllTables(); err != nil {
- return nil, err
- }
- // collect raw tables
- for _, table := range allTables {
- if strings.HasPrefix(table, "_raw_"+pluginName) {
- tables = append(tables, table)
- }
- }
- // collect tool tables
- toolModels := pluginModel.GetTablesInfo()
- for _, toolModel := range toolModels {
- if !isScopeModel(toolModel) && hasField(toolModel, "RawDataParams") {
- tables = append(tables, toolModel.TableName())
- }
- }
- // collect domain tables
- for _, domainModel := range domaininfo.GetDomainTablesInfo() {
- // we only care about tables with RawOrigin
- ok = hasField(domainModel, "RawDataParams")
- if ok {
- tables = append(tables, domainModel.TableName())
- }
- }
- // additional tables
- tables = append(tables, models.CollectorLatestState{}.TableName())
- }
- gs.log.Debug("Discovered %d tables used by plugin \"%s\": %v", len(tables), pluginName, tables)
- return tables, nil
-}
-
-func isScopeModel(obj dal.Tabler) bool {
- _, ok := obj.(plugin.ToolLayerScope)
- return ok
-}
diff --git a/backend/helpers/pluginhelper/api/scope_helper.go b/backend/helpers/pluginhelper/api/scope_helper.go
deleted file mode 100644
index 935280e06..000000000
--- a/backend/helpers/pluginhelper/api/scope_helper.go
+++ /dev/null
@@ -1,108 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package api
-
-import (
- "net/http"
-
- "github.com/apache/incubator-devlake/core/context"
- "github.com/apache/incubator-devlake/core/errors"
- "github.com/apache/incubator-devlake/core/plugin"
- "github.com/apache/incubator-devlake/server/api/shared"
- "github.com/go-playground/validator/v10"
-)
-
-type (
- // ScopeApiHelper is used to write the CURD of scopes
- ScopeApiHelper[Conn any, Scope plugin.ToolLayerScope, Tr any] struct {
- *GenericScopeApiHelper[Conn, Scope, Tr]
- }
- ScopeReq[T any] struct {
- Data []*T `json:"data"`
- }
-)
-
-// NewScopeHelper creates a ScopeHelper for scopes management
-func NewScopeHelper[Conn any, Scope plugin.ToolLayerScope, Tr any](
- basicRes context.BasicRes,
- vld *validator.Validate,
- connHelper *ConnectionApiHelper,
- dbHelper ScopeDatabaseHelper[Conn, Scope, Tr],
- params *ReflectionParameters,
- opts *ScopeHelperOptions,
-) *ScopeApiHelper[Conn, Scope, Tr] {
- return &ScopeApiHelper[Conn, Scope, Tr]{
- NewGenericScopeHelper[Conn, Scope, Tr](
- basicRes, vld, connHelper, dbHelper, params, opts),
- }
-}
-
-// Put saves the given scopes to the database. It expects a slice of struct pointers
-// as the scopes argument. It also expects a fieldName argument, which is used to extract
-// the connection ID from the input.Params map.
-func (c *ScopeApiHelper[Conn, Scope, Tr]) Put(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
- var req struct {
- Data []*Scope `json:"data"`
- }
- err := errors.Convert(DecodeMapStruct(input.Body, &req, true))
- if err != nil {
- return nil, errors.BadInput.Wrap(err, "decoding scope error")
- }
- // Extract the connection ID from the input.Params map
- apiScopes, err := c.PutScopes(input, req.Data)
- if err != nil {
- return nil, err
- }
- return &plugin.ApiResourceOutput{Body: apiScopes, Status: http.StatusOK}, nil
-}
-
-func (c *ScopeApiHelper[Conn, Scope, Tr]) Update(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
- apiScope, err := c.GenericScopeApiHelper.UpdateScope(input)
- if err != nil {
- return nil, err
- }
- return &plugin.ApiResourceOutput{Body: apiScope, Status: http.StatusOK}, nil
-}
-
-func (c *ScopeApiHelper[Conn, Scope, Tr]) GetScopeList(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
- scopes, err := c.GetScopes(input)
- if err != nil {
- return nil, err
- }
- return &plugin.ApiResourceOutput{Body: scopes, Status: http.StatusOK}, nil
-}
-
-func (c *ScopeApiHelper[Conn, Scope, Tr]) GetScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
- scope, err := c.GenericScopeApiHelper.GetScope(input)
- if err != nil {
- return nil, err
- }
- return &plugin.ApiResourceOutput{Body: scope, Status: http.StatusOK}, nil
-}
-
-func (c *ScopeApiHelper[Conn, Scope, Tr]) Delete(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
- refs, err := c.DeleteScope(input)
- if err != nil {
- return &plugin.ApiResourceOutput{Body: &shared.ApiBody{
- Success: false,
- Message: err.Error(),
- Data: refs,
- }, Status: err.GetType().GetHttpCode()}, err
- }
- return &plugin.ApiResourceOutput{Body: nil, Status: http.StatusOK}, nil
-}
diff --git a/backend/helpers/pluginhelper/api/scope_helper_test.go b/backend/helpers/pluginhelper/api/scope_helper_test.go
deleted file mode 100644
index 8956ad09b..000000000
--- a/backend/helpers/pluginhelper/api/scope_helper_test.go
+++ /dev/null
@@ -1,356 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package api
-
-import (
- "encoding/json"
- "fmt"
- "reflect"
- "testing"
- "time"
-
- "github.com/apache/incubator-devlake/core/models/common"
- "github.com/apache/incubator-devlake/core/plugin"
- "github.com/apache/incubator-devlake/helpers/unithelper"
- mockcontext "github.com/apache/incubator-devlake/mocks/core/context"
- mockdal "github.com/apache/incubator-devlake/mocks/core/dal"
- "github.com/go-playground/validator/v10"
- "github.com/stretchr/testify/assert"
- "github.com/stretchr/testify/mock"
- "gorm.io/gorm"
-)
-
-type TestModel struct {
- ID uint `gorm:"primaryKey" validate:"required"`
- Name string `gorm:"primaryKey;type:BIGINT NOT NULL"`
-}
-
-type TestFakeGitlabRepo struct {
- common.Scope
- GitlabId int `json:"gitlabId" mapstructure:"gitlabId" gorm:"primaryKey"`
- CreatedDate *time.Time `json:"createdDate" mapstructure:"-"`
- UpdatedDate *time.Time `json:"updatedDate" mapstructure:"-"`
-}
-
-func (t TestFakeGitlabRepo) ScopeId() string {
- return fmt.Sprintf("%d", t.GitlabId)
-}
-
-func (t TestFakeGitlabRepo) ScopeName() string {
- return ""
-}
-
-func (t TestFakeGitlabRepo) ScopeFullName() string {
- return ""
-}
-
-func (t TestFakeGitlabRepo) TableName() string {
- return ""
-}
-
-func (t TestFakeGitlabRepo) ScopeParams() interface{} {
- return nil
-}
-
-func (t TestFakeGitlabRepo) ScopeConnectionId() uint64 {
- return 0
-}
-
-type TestFakeGithubRepo struct {
- common.Scope
- GithubId int `json:"githubId" gorm:"primaryKey" mapstructure:"githubId"`
- Name string `json:"name" gorm:"type:varchar(255)" mapstructure:"name,omitempty"`
- HTMLUrl string `json:"HTMLUrl" gorm:"type:varchar(255)" mapstructure:"HTMLUrl,omitempty"`
- Description string `json:"description" mapstructure:"description,omitempty"`
- OwnerId int `json:"ownerId" mapstructure:"ownerId,omitempty"`
- Language string `json:"language" gorm:"type:varchar(255)" mapstructure:"language,omitempty"`
- ParentGithubId int `json:"parentId" mapstructure:"parentGithubId,omitempty"`
- ParentHTMLUrl string `json:"parentHtmlUrl" mapstructure:"parentHtmlUrl,omitempty"`
- CloneUrl string `json:"cloneUrl" gorm:"type:varchar(255)" mapstructure:"cloneUrl,omitempty"`
- CreatedDate *time.Time `json:"createdDate" mapstructure:"-"`
- UpdatedDate *time.Time `json:"updatedDate" mapstructure:"-"`
-}
-
-func (r TestFakeGithubRepo) ScopeId() string {
- return fmt.Sprintf("%d", r.GithubId)
-}
-
-func (r TestFakeGithubRepo) ScopeName() string {
- return r.Name
-}
-
-func (r TestFakeGithubRepo) ScopeFullName() string {
- return r.Name
-}
-
-func (r TestFakeGithubRepo) ScopeParams() interface{} {
- return nil
-}
-
-func (TestFakeGithubRepo) ScopeConnectionId() uint64 {
- return 0
-}
-
-func (TestFakeGithubRepo) TableName() string {
- return "_tool_github_repos"
-}
-
-type TestConnection struct {
- common.Model
- Name string `gorm:"type:varchar(100);uniqueIndex" json:"name" validate:"required"`
- Endpoint string `mapstructure:"endpoint" env:"GITHUB_ENDPOINT" validate:"required"`
- Proxy string `mapstructure:"proxy" env:"GITHUB_PROXY"`
- RateLimitPerHour int `comment:"api request rate limit per hour"`
- Token string `mapstructure:"token" env:"GITHUB_AUTH" validate:"required" encrypt:"yes"`
-}
-
-func (TestConnection) TableName() string {
- return "_tool_github_connections"
-}
-
-func TestVerifyScope(t *testing.T) {
- apiHelper := createMockScopeHelper[TestFakeGithubRepo]("GithubId")
- testCases := []struct {
- name string
- model TestModel
- wantErr bool
- }{
- {
- name: "valid case",
- model: TestModel{
- ID: 1,
- Name: "test name",
- },
- wantErr: false,
- },
- {
- name: "zero value",
- model: TestModel{
- ID: 0,
- Name: "test name",
- },
- wantErr: true,
- },
- {
- name: "nil value",
- model: TestModel{
- ID: 1,
- Name: "",
- },
- wantErr: false,
- },
- }
-
- for _, tc := range testCases {
- err := apiHelper.verifyScope(&tc.model, validator.New())
- if (err != nil) != tc.wantErr {
- t.Errorf("unexpected error value - got: %v, want: %v", err, tc.wantErr)
- }
-
- }
-}
-
-type TestScopeConfig struct {
- common.Model `mapstructure:"-"`
- Name string `mapstructure:"name" json:"name" gorm:"type:varchar(255);index:idx_name_github,unique" validate:"required"`
- PrType string `mapstructure:"prType,omitempty" json:"prType" gorm:"type:varchar(255)"`
- PrComponent string `mapstructure:"prComponent,omitempty" json:"prComponent" gorm:"type:varchar(255)"`
- PrBodyClosePattern string `mapstructure:"prBodyClosePattern,omitempty" json:"prBodyClosePattern" gorm:"type:varchar(255)"`
- IssueSeverity string `mapstructure:"issueSeverity,omitempty" json:"issueSeverity" gorm:"type:varchar(255)"`
- IssuePriority string `mapstructure:"issuePriority,omitempty" json:"issuePriority" gorm:"type:varchar(255)"`
- IssueComponent string `mapstructure:"issueComponent,omitempty" json:"issueComponent" gorm:"type:varchar(255)"`
- IssueTypeBug string `mapstructure:"issueTypeBug,omitempty" json:"issueTypeBug" gorm:"type:varchar(255)"`
- IssueTypeIncident string `mapstructure:"issueTypeIncident,omitempty" json:"issueTypeIncident" gorm:"type:varchar(255)"`
- IssueTypeRequirement string `mapstructure:"issueTypeRequirement,omitempty" json:"issueTypeRequirement" gorm:"type:varchar(255)"`
- DeploymentPattern string `mapstructure:"deploymentPattern,omitempty" json:"deploymentPattern" gorm:"type:varchar(255)"`
- ProductionPattern string `mapstructure:"productionPattern,omitempty" json:"productionPattern" gorm:"type:varchar(255)"`
- Refdiff json.RawMessage `mapstructure:"refdiff,omitempty" json:"refdiff" swaggertype:"object" format:"json"`
-}
-
-func (TestScopeConfig) TableName() string {
- return "_tool_github_scope_configs"
-}
-
-func TestSetScopeFields(t *testing.T) {
- // create a struct
- type P struct {
- ConnectionId uint64 `json:"connectionId" mapstructure:"connectionId" gorm:"primaryKey"`
- GitlabId int `json:"gitlabId" mapstructure:"gitlabId" gorm:"primaryKey"`
- CreatedAt *time.Time `json:"createdAt" mapstructure:"-"`
- UpdatedAt *time.Time `json:"updatedAt" mapstructure:"-"`
- common.NoPKModel `json:"-" mapstructure:"-"`
- }
- p := P{}
- apiHelper := createMockScopeHelper[TestFakeGitlabRepo]("GitlabId")
-
- // call setScopeFields to assign value
- connectionId := uint64(123)
- createdAt := time.Now()
- updatedAt := &createdAt
- apiHelper.setScopeFields(&p, connectionId, &createdAt, updatedAt)
-
- // verify fields
- if p.ConnectionId != connectionId {
- t.Errorf("ConnectionId not set correctly, expected: %v, got: %v", connectionId, p.ConnectionId)
- }
-
- if !p.CreatedAt.Equal(createdAt) {
- t.Errorf("CreatedAt not set correctly, expected: %v, got: %v", createdAt, p.CreatedAt)
- }
-
- if p.UpdatedAt == nil {
- t.Errorf("UpdatedDate not set correctly, expected: %v, got: %v", updatedAt, p.UpdatedAt)
- } else if !p.UpdatedAt.Equal(*updatedAt) {
- t.Errorf("UpdatedDate not set correctly, expected: %v, got: %v", updatedAt, p.UpdatedAt)
- }
-
- apiHelper.setScopeFields(&p, connectionId, &createdAt, nil)
-
- // verify fields
- if p.ConnectionId != connectionId {
- t.Errorf("ConnectionId not set correctly, expected: %v, got: %v", connectionId, p.ConnectionId)
- }
-
- if !p.CreatedAt.Equal(createdAt) {
- t.Errorf("CreatedDate not set correctly, expected: %v, got: %v", createdAt, p.CreatedAt)
- }
-
- if p.UpdatedAt != nil {
- t.Errorf("UpdatedDate not set correctly, expected: %v, got: %v", nil, p.UpdatedAt)
- }
-}
-
-func TestReturnPrimaryKeyValue(t *testing.T) {
- // Define a test struct with the primaryKey tag on one of its fields.
- type TestStruct struct {
- ConnectionId int `json:"connectionId" gorm:"primaryKey"`
- Id int `json:"id" gorm:"primaryKey"`
- Name string `json:"name"`
- CreatedAt time.Time
- UpdatedAt time.Time
- DeletedAt gorm.DeletedAt `gorm:"index"`
- }
-
- // Create an instance of the test struct.
- test := TestStruct{
- ConnectionId: 1,
- Id: 123,
- Name: "Test",
- CreatedAt: time.Now(),
- }
-
- // Call the function and check if it returns the correct primary key value.
- result := returnPrimaryKeyValue(test)
- expected := "1-123"
- if result != expected {
- t.Errorf("returnPrimaryKeyValue returned %s, expected %s", result, expected)
- }
-
- // Test with a different struct that has no field with primaryKey tag.
- type TestStruct2 struct {
- Id int `json:"id"`
- Name string `json:"name"`
- CreatedAt time.Time
- UpdatedAt time.Time
- }
-
- test2 := TestStruct2{
- Id: 456,
- Name: "Test 2",
- CreatedAt: time.Now(),
- }
-
- result2 := returnPrimaryKeyValue(test2)
- expected2 := ""
- if result2 != expected2 {
- t.Errorf("returnPrimaryKeyValue returned %s, expected %s", result2, expected2)
- }
-}
-
-func TestScopeApiHelper_Put(t *testing.T) {
- apiHelper := createMockScopeHelper[TestFakeGithubRepo]("GithubId")
- // create a mock input, scopes, and connection
- input := &plugin.ApiResourceInput{Params: map[string]string{"connectionId": "123"}, Body: map[string]interface{}{
- "data": []map[string]interface{}{
- {
- "HTMLUrl": "string",
- "githubId": 1,
- "cloneUrl": "string",
- "connectionId": 1,
- "createdAt": "string",
- "createdDate": "string",
- "description": "string",
- "language": "string",
- "name": "string",
- "owner": "string",
- "scopeConfigId": 0,
- "updatedAt": "string",
- "updatedDate": "string",
- },
- {
- "HTMLUrl": "11",
- "githubId": 2,
- "cloneUrl": "string",
- "connectionId": 1,
- "createdAt": "string",
- "createdDate": "string",
- "description": "string",
- "language": "string",
- "name": "string",
- "owner": "string",
- "scopeConfigId": 0,
- "updatedAt": "string",
- "updatedDate": "string",
- }}}}
- // test a successful call to Put
- _, err := apiHelper.Put(input)
- assert.NoError(t, err)
-}
-
-func createMockScopeHelper[Repo plugin.ToolLayerScope](scopeIdFieldName string) *ScopeApiHelper[TestConnection, Repo, TestScopeConfig] {
- mockDal := new(mockdal.Dal)
- mockLogger := unithelper.DummyLogger()
- mockRes := new(mockcontext.BasicRes)
-
- mockRes.On("GetDal").Return(mockDal)
- mockRes.On("GetConfig", mock.Anything).Return("")
- mockRes.On("GetLogger").Return(mockLogger)
-
- // we expect total 2 deletion calls after all code got carried out
- mockDal.On("Delete", mock.Anything, mock.Anything).Return(nil).Twice()
- mockDal.On("GetPrimaryKeyFields", mock.Anything).Return(
- []reflect.StructField{
- {Name: "ID", Type: reflect.TypeOf("")},
- },
- )
- mockDal.On("CreateOrUpdate", mock.Anything, mock.Anything).Return(nil)
- mockDal.On("First", mock.Anything, mock.Anything).Return(nil)
- mockDal.On("All", mock.Anything, mock.Anything).Return(nil)
- mockDal.On("AllTables").Return(nil, nil)
-
- connHelper := NewConnectionHelper(mockRes, nil, "dummy_plugin")
-
- params := &ReflectionParameters{
- ScopeIdFieldName: scopeIdFieldName,
- ScopeIdColumnName: "scope_id",
- RawScopeParamName: "ScopeId",
- }
- dbHelper := NewScopeDatabaseHelperImpl[TestConnection, Repo, TestScopeConfig](mockRes, connHelper, params)
- // create a mock ScopeApiHelper with a mock database connection
- return NewScopeHelper[TestConnection, Repo, TestScopeConfig](mockRes, validator.New(), connHelper, dbHelper, params, nil)
-}
diff --git a/backend/helpers/srvhelper/connection_service_helper.go b/backend/helpers/srvhelper/any_connection_service_helper.go
similarity index 58%
copy from backend/helpers/srvhelper/connection_service_helper.go
copy to backend/helpers/srvhelper/any_connection_service_helper.go
index 7244e2724..47fcb2bb5 100644
--- a/backend/helpers/srvhelper/connection_service_helper.go
+++ b/backend/helpers/srvhelper/any_connection_service_helper.go
@@ -24,52 +24,62 @@ import (
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models"
- "github.com/apache/incubator-devlake/core/plugin"
)
+type ConnectionModelInfo interface {
+ ModelInfo
+ GetConnectionId(any) uint64
+}
+
// ConnectionSrvHelper
-type ConnectionSrvHelper[C plugin.ToolLayerConnection, S plugin.ToolLayerScope, SC plugin.ToolLayerScopeConfig] struct {
- *ModelSrvHelper[C]
- pluginName string
+type AnyConnectionSrvHelper struct {
+ *AnyModelSrvHelper
+ connModelInfo ConnectionModelInfo
+ scopeModelInfo ScopeModelInfo
+ scopeConfigModelInfo ScopeConfigModelInfo
+ pluginName string
}
// NewConnectionSrvHelper creates a ConnectionDalHelper for connection management
-func NewConnectionSrvHelper[
- C plugin.ToolLayerConnection,
- S plugin.ToolLayerScope,
- SC plugin.ToolLayerScopeConfig,
-](
+func NewAnyConnectionSrvHelper(
basicRes context.BasicRes,
pluginName string,
-) *ConnectionSrvHelper[C, S, SC] {
- return &ConnectionSrvHelper[C, S, SC]{
- ModelSrvHelper: NewModelSrvHelper[C](basicRes, nil),
- pluginName: pluginName,
+ connModelInfo ConnectionModelInfo,
+ scopeModelInfo ScopeModelInfo,
+ scopeConfigModelInfo ScopeConfigModelInfo,
+) *AnyConnectionSrvHelper {
+ return &AnyConnectionSrvHelper{
+ AnyModelSrvHelper: NewAnyModelSrvHelper(basicRes, connModelInfo, nil),
+ connModelInfo: connModelInfo,
+ scopeModelInfo: scopeModelInfo,
+ scopeConfigModelInfo: scopeConfigModelInfo,
+ pluginName: pluginName,
}
}
-func (connSrv *ConnectionSrvHelper[C, S, SC]) DeleteConnection(connection *C) (refs *DsRefs, err errors.Error) {
- err = connSrv.ModelSrvHelper.NoRunningPipeline(func(tx dal.Transaction) errors.Error {
+func (connSrv *AnyConnectionSrvHelper) DeleteConnectionAny(connection any) (refs *DsRefs, err errors.Error) {
+ err = connSrv.NoRunningPipeline(func(tx dal.Transaction) errors.Error {
// make sure no blueprint is using the connection
- connectionId := (*connection).ConnectionId()
+ connectionId := connSrv.connModelInfo.GetConnectionId(connection)
refs = toDsRefs(connSrv.getAllBlueprinsByConnection(connectionId))
if refs != nil {
return errors.Conflict.New("Cannot delete the connection because it is referenced by blueprints")
}
- scopeCount := errors.Must1(connSrv.db.Count(dal.From(new(S)), dal.Where("connection_id = ?", connectionId)))
+ scopeCount := errors.Must1(connSrv.db.Count(dal.From(connSrv.scopeModelInfo.TableName()), dal.Where("connection_id = ?", connectionId)))
if scopeCount > 0 {
return errors.Conflict.New("Please delete all data scope(s) before you delete this Data Connection.")
}
errors.Must(tx.Delete(connection))
- if reflect.TypeOf(new(SC)) != reflect.TypeOf(new(NoScopeConfig)) {
- errors.Must(connSrv.db.Delete(new(SC), dal.Where("connection_id = ?", connectionId)))
+ sc := connSrv.scopeConfigModelInfo.New()
+ if reflect.TypeOf(sc) != reflect.TypeOf(new(NoScopeConfig)) {
+ errors.Must(connSrv.db.Delete(sc, dal.Where("connection_id = ?", connectionId)))
}
return nil
})
return
}
-func (connSrv *ConnectionSrvHelper[C, S, SC]) getAllBlueprinsByConnection(connectionId uint64) []*models.Blueprint {
+func (connSrv *AnyConnectionSrvHelper) getAllBlueprinsByConnection(connectionId uint64) []*models.Blueprint {
blueprints := make([]*models.Blueprint, 0)
errors.Must(connSrv.db.All(
&blueprints,
diff --git a/backend/helpers/srvhelper/model_service_helper.go b/backend/helpers/srvhelper/any_model_service_helper.go
similarity index 78%
copy from backend/helpers/srvhelper/model_service_helper.go
copy to backend/helpers/srvhelper/any_model_service_helper.go
index 3ec9abe39..91be8fa25 100644
--- a/backend/helpers/srvhelper/model_service_helper.go
+++ b/backend/helpers/srvhelper/any_model_service_helper.go
@@ -34,26 +34,31 @@ type CustomValidator interface {
CustomValidate(entity interface{}, validate *validator.Validate) errors.Error
}
-type ModelSrvHelper[M dal.Tabler] struct {
+type ModelInfo interface {
+ New() any
+ NewSlice() any
+ ModelName() string
+ TableName() string
+}
+
+type AnyModelSrvHelper struct {
+ ModelInfo
basicRes context.BasicRes
log log.Logger
db dal.Dal
validator *validator.Validate
- modelName string
pk []dal.ColumnMeta
pkWhere string
pkCount int
searchColumns []string
}
-func NewModelSrvHelper[M dal.Tabler](basicRes context.BasicRes, searchColumns []string) *ModelSrvHelper[M] {
- m := new(M)
- modelName := fmt.Sprintf("%T", m)
+func NewAnyModelSrvHelper(basicRes context.BasicRes, modelInfo ModelInfo, searchColumns []string) *AnyModelSrvHelper {
db := basicRes.GetDal()
if db == nil {
db = basicRes.GetDal()
}
- pk := errors.Must1(dal.GetPrimarykeyColumns(db, *m))
+ pk := errors.Must1(dal.GetPrimarykeyColumns(db, modelInfo.TableName()))
pkWhere := ""
for _, col := range pk {
if pkWhere != "" {
@@ -62,12 +67,12 @@ func NewModelSrvHelper[M dal.Tabler](basicRes context.BasicRes, searchColumns []
pkWhere += fmt.Sprintf("%s = ? ", col.Name())
}
- return &ModelSrvHelper[M]{
+ return &AnyModelSrvHelper{
+ ModelInfo: modelInfo,
basicRes: basicRes,
- log: basicRes.GetLogger().Nested(fmt.Sprintf("%s_dal", modelName)),
+ log: basicRes.GetLogger().Nested(fmt.Sprintf("%s_dal", modelInfo.ModelName())),
db: db,
validator: validator.New(),
- modelName: modelName,
pk: pk,
pkWhere: pkWhere,
pkCount: len(pk),
@@ -75,14 +80,7 @@ func NewModelSrvHelper[M dal.Tabler](basicRes context.BasicRes, searchColumns []
}
}
-func (srv *ModelSrvHelper[M]) NewTx(tx dal.Transaction) *ModelSrvHelper[M] {
- helper := new(ModelSrvHelper[M])
- *helper = *srv
- helper.db = tx
- return helper
-}
-
-func (srv *ModelSrvHelper[M]) ValidateModel(model *M) errors.Error {
+func (srv *AnyModelSrvHelper) ValidateModel(model any) errors.Error {
// the model can validate itself
if customValidator, ok := (interface{}(model)).(CustomValidator); ok {
return customValidator.CustomValidate(model, srv.validator)
@@ -95,7 +93,7 @@ func (srv *ModelSrvHelper[M]) ValidateModel(model *M) errors.Error {
}
// Create validates given model and insert it into database if validation passed
-func (srv *ModelSrvHelper[M]) Create(model *M) errors.Error {
+func (srv *AnyModelSrvHelper) CreateAny(model any) errors.Error {
err := srv.ValidateModel(model)
if err != nil {
return err
@@ -103,7 +101,7 @@ func (srv *ModelSrvHelper[M]) Create(model *M) errors.Error {
err = srv.db.Create(model)
if err != nil {
if srv.db.IsDuplicationError(err) {
- return errors.Conflict.Wrap(err, fmt.Sprintf("%s already exists", srv.modelName))
+ return errors.Conflict.Wrap(err, fmt.Sprintf("%s already exists", srv.ModelName()))
}
return err
}
@@ -111,11 +109,11 @@ func (srv *ModelSrvHelper[M]) Create(model *M) errors.Error {
}
// Update validates given model and update it into database if validation passed
-func (srv *ModelSrvHelper[M]) Update(model *M) errors.Error {
+func (srv *AnyModelSrvHelper) UpdateAny(model any) errors.Error {
err := srv.ValidateModel(model)
if err != nil {
if srv.db.IsDuplicationError(err) {
- return errors.Conflict.Wrap(err, fmt.Sprintf("%s already exists", srv.modelName))
+ return errors.Conflict.Wrap(err, fmt.Sprintf("%s already exists", srv.ModelName()))
}
return err
}
@@ -123,7 +121,7 @@ func (srv *ModelSrvHelper[M]) Update(model *M) errors.Error {
}
// CreateOrUpdate validates given model and insert or update it into database if validation passed
-func (srv *ModelSrvHelper[M]) CreateOrUpdate(model *M) errors.Error {
+func (srv *AnyModelSrvHelper) CreateOrUpdateAny(model any) errors.Error {
err := srv.ValidateModel(model)
if err != nil {
return err
@@ -132,20 +130,20 @@ func (srv *ModelSrvHelper[M]) CreateOrUpdate(model *M) errors.Error {
}
// DeleteModel deletes given model from database
-func (srv *ModelSrvHelper[M]) DeleteModel(model *M) errors.Error {
+func (srv *AnyModelSrvHelper) DeleteModelAny(model any) errors.Error {
return srv.db.Delete(model)
}
// FindByPk returns model with given primary key from database
-func (srv *ModelSrvHelper[M]) FindByPk(pk ...interface{}) (*M, errors.Error) {
+func (srv *AnyModelSrvHelper) FindByPkAny(pk ...interface{}) (any, errors.Error) {
if len(pk) != srv.pkCount {
return nil, errors.BadInput.New("invalid primary key")
}
- model := new(M)
- err := srv.db.First(model, dal.Where(srv.pkWhere, pk...))
+ model := srv.New()
+ err := srv.db.First(model, dal.From(srv.TableName()), dal.Where(srv.pkWhere, pk...))
if err != nil {
if srv.db.IsErrorNotFound(err) {
- return nil, errors.NotFound.Wrap(err, fmt.Sprintf("%s not found", srv.modelName))
+ return nil, errors.NotFound.Wrap(err, fmt.Sprintf("%s not found", srv.ModelName()))
}
return nil, err
}
@@ -153,13 +151,13 @@ func (srv *ModelSrvHelper[M]) FindByPk(pk ...interface{}) (*M, errors.Error) {
}
// GetAll returns all models from database
-func (srv *ModelSrvHelper[M]) GetAll() ([]*M, errors.Error) {
- array := make([]*M, 0)
+func (srv *AnyModelSrvHelper) QueryAllAny() (any, errors.Error) {
+ array := srv.NewSlice()
return array, srv.db.All(&array)
}
-func (srv *ModelSrvHelper[M]) GetPage(pagination *Pagination, query ...dal.Clause) ([]*M, int64, errors.Error) {
- query = append(query, dal.From(new(M)))
+func (srv *AnyModelSrvHelper) QueryPageAny(pagination *Pagination, query ...dal.Clause) (any, int64, errors.Error) {
+ query = append(query, dal.From(srv.TableName()))
// process keyword
searchTerm := pagination.SearchTerm
if searchTerm != "" && len(srv.searchColumns) > 0 {
@@ -183,11 +181,11 @@ func (srv *ModelSrvHelper[M]) GetPage(pagination *Pagination, query ...dal.Claus
return nil, 0, err
}
query = append(query, dal.Limit(pagination.GetLimit()), dal.Offset(pagination.GetOffset()))
- var scopes []*M
- return scopes, count, srv.db.All(&scopes, query...)
+ array := srv.NewSlice()
+ return array, count, srv.db.All(&array, query...)
}
-func (srv *ModelSrvHelper[M]) NoRunningPipeline(fn func(tx dal.Transaction) errors.Error, tablesToLock ...*dal.LockTable) (err errors.Error) {
+func (srv *AnyModelSrvHelper) NoRunningPipeline(fn func(tx dal.Transaction) errors.Error, tablesToLock ...*dal.LockTable) (err errors.Error) {
// make sure no pipeline is running
tablesToLock = append(tablesToLock, &dal.LockTable{Table: "_devlake_pipelines", Exclusive: true})
txHelper := dbhelper.NewTxHelper(srv.basicRes, &err)
diff --git a/backend/helpers/srvhelper/scope_config_service_helper.go b/backend/helpers/srvhelper/any_scope_config_service_helper.go
similarity index 56%
copy from backend/helpers/srvhelper/scope_config_service_helper.go
copy to backend/helpers/srvhelper/any_scope_config_service_helper.go
index 796ba29a5..bcc2e5e79 100644
--- a/backend/helpers/srvhelper/scope_config_service_helper.go
+++ b/backend/helpers/srvhelper/any_scope_config_service_helper.go
@@ -18,10 +18,11 @@ limitations under the License.
package srvhelper
import (
+ "reflect"
+
"github.com/apache/incubator-devlake/core/context"
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
- "github.com/apache/incubator-devlake/core/plugin"
)
// NoScopeConfig is a placeholder for plugins that don't have any scope configuration yet
@@ -31,23 +32,33 @@ func (NoScopeConfig) TableName() string { return "" }
func (NoScopeConfig) ScopeConfigId() uint64 { return 0 }
func (NoScopeConfig) ScopeConfigConnectionId() uint64 { return 0 }
+type ScopeConfigModelInfo interface {
+ ModelInfo
+ GetConnectionId(any) uint64
+ GetScopeConfigId(any) uint64
+}
+
// ScopeConfigSrvHelper
-type ScopeConfigSrvHelper[C plugin.ToolLayerConnection, S plugin.ToolLayerScope, SC plugin.ToolLayerScopeConfig] struct {
- *ModelSrvHelper[SC]
+type AnyScopeConfigSrvHelper struct {
+ ScopeConfigModelInfo
+ ScopeModelInfo
+ *AnyModelSrvHelper
}
-func NewScopeConfigSrvHelper[
- C plugin.ToolLayerConnection,
- S plugin.ToolLayerScope,
- SC plugin.ToolLayerScopeConfig,
-](basicRes context.BasicRes, searchColumns []string) *ScopeConfigSrvHelper[C, S, SC] {
- return &ScopeConfigSrvHelper[C, S, SC]{
- ModelSrvHelper: NewModelSrvHelper[SC](basicRes, searchColumns),
+func NewAnyScopeConfigSrvHelper(
+ basicRes context.BasicRes,
+ scopeConfigModelInfo ScopeConfigModelInfo,
+ scopeModelInfo ScopeModelInfo,
+) *AnyScopeConfigSrvHelper {
+ return &AnyScopeConfigSrvHelper{
+ ScopeConfigModelInfo: scopeConfigModelInfo,
+ ScopeModelInfo: scopeModelInfo,
+ AnyModelSrvHelper: NewAnyModelSrvHelper(basicRes, scopeConfigModelInfo, nil),
}
}
-func (scopeConfigSrv *ScopeConfigSrvHelper[C, S, SC]) GetAllByConnectionId(connectionId uint64) ([]*SC, errors.Error) {
- var scopeConfigs []*SC
+func (scopeConfigSrv *AnyScopeConfigSrvHelper) GetAllByConnectionIdAny(connectionId uint64) (any, errors.Error) {
+ scopeConfigs := scopeConfigSrv.ScopeConfigModelInfo.NewSlice()
err := scopeConfigSrv.db.All(&scopeConfigs,
dal.Where("connection_id = ?", connectionId),
dal.Orderby("id DESC"),
@@ -55,15 +66,17 @@ func (scopeConfigSrv *ScopeConfigSrvHelper[C, S, SC]) GetAllByConnectionId(conne
return scopeConfigs, err
}
-func (scopeConfigSrv *ScopeConfigSrvHelper[C, S, SC]) DeleteScopeConfig(scopeConfig *SC) (refs []*S, err errors.Error) {
- err = scopeConfigSrv.ModelSrvHelper.NoRunningPipeline(func(tx dal.Transaction) errors.Error {
+func (scopeConfigSrv *AnyScopeConfigSrvHelper) DeleteScopeConfigAny(scopeConfig any) (refs any, err errors.Error) {
+ err = scopeConfigSrv.NoRunningPipeline(func(tx dal.Transaction) errors.Error {
// make sure no scope is using the scopeConfig
- sc := *scopeConfig
+ connectionId := scopeConfigSrv.ScopeConfigModelInfo.GetConnectionId(scopeConfig)
+ scopeConfigId := scopeConfigSrv.ScopeConfigModelInfo.GetScopeConfigId(scopeConfig)
+ refs = scopeConfigSrv.ScopeModelInfo.NewSlice()
errors.Must(tx.All(
&refs,
- dal.Where("connection_id = ? AND scope_config_id = ?", sc.ScopeConfigConnectionId(), sc.ScopeConfigId()),
+ dal.Where("connection_id = ? AND scope_config_id = ?", connectionId, scopeConfigId),
))
- if len(refs) > 0 {
+ if reflect.ValueOf(refs).Len() > 0 {
return errors.Conflict.New("Please delete all data scope(s) before you delete this ScopeConfig.")
}
errors.Must(tx.Delete(scopeConfig))
diff --git a/backend/helpers/srvhelper/scope_service_helper.go b/backend/helpers/srvhelper/any_scope_service_helper.go
similarity index 65%
copy from backend/helpers/srvhelper/scope_service_helper.go
copy to backend/helpers/srvhelper/any_scope_service_helper.go
index deda7c25e..3b86534c7 100644
--- a/backend/helpers/srvhelper/scope_service_helper.go
+++ b/backend/helpers/srvhelper/any_scope_service_helper.go
@@ -37,75 +37,85 @@ type ScopePagination struct {
Blueprints bool `json:"blueprints" mapstructure:"blueprints"`
}
-type ScopeDetail[S plugin.ToolLayerScope, SC plugin.ToolLayerScopeConfig] struct {
- Scope S `json:"scope"`
- ScopeConfig *SC `json:"scopeConfig,omitempty"`
+type AnyScopeDetail struct {
+ Scope any `json:"scope"`
+ ScopeConfig any `json:"scopeConfig,omitempty"`
Blueprints []*models.Blueprint `json:"blueprints,omitempty"`
}
-type ScopeSrvHelper[C plugin.ToolLayerConnection, S plugin.ToolLayerScope, SC plugin.ToolLayerScopeConfig] struct {
- *ModelSrvHelper[S]
+type ScopeModelInfo interface {
+ ModelInfo
+ GetScopeId(any) string
+ GetConnectionId(any) uint64
+ GetScopeConfigId(any) uint64
+ GetScopeParams(any) interface{}
+}
+
+type AnyScopeSrvHelper struct {
+ ScopeModelInfo
+ ScopeConfigModelInfo
+ *AnyModelSrvHelper
pluginName string
}
-// NewScopeSrvHelper creates a ScopeDalHelper for scope management
-func NewScopeSrvHelper[
- C plugin.ToolLayerConnection,
- S plugin.ToolLayerScope,
- SC plugin.ToolLayerScopeConfig,
-](
+func NewAnyScopeSrvHelper(
basicRes context.BasicRes,
+ scopeModelInfo ScopeModelInfo,
+ scopeConfigModelInfo ScopeConfigModelInfo,
pluginName string,
searchColumns []string,
-) *ScopeSrvHelper[C, S, SC] {
- return &ScopeSrvHelper[C, S, SC]{
- ModelSrvHelper: NewModelSrvHelper[S](basicRes, searchColumns),
- pluginName: pluginName,
+) *AnyScopeSrvHelper {
+ return &AnyScopeSrvHelper{
+ AnyModelSrvHelper: NewAnyModelSrvHelper(basicRes, scopeModelInfo, searchColumns),
+ pluginName: pluginName,
+ ScopeModelInfo: scopeModelInfo,
+ ScopeConfigModelInfo: scopeConfigModelInfo,
}
}
-func (scopeSrv *ScopeSrvHelper[C, S, SC]) GetPluginName() string {
+func (scopeSrv *AnyScopeSrvHelper) GetPluginName() string {
return scopeSrv.pluginName
}
-func (scopeSrv *ScopeSrvHelper[C, S, SC]) Validate(scope *S) errors.Error {
- connectionId := (*scope).ScopeConnectionId()
- connectionCount := errors.Must1(scopeSrv.db.Count(dal.From(new(SC)), dal.Where("id = ?", connectionId)))
+func (scopeSrv *AnyScopeSrvHelper) Validate(scope any) errors.Error {
+ connectionId := scopeSrv.ScopeModelInfo.GetConnectionId(scope)
+ connectionCount := errors.Must1(scopeSrv.db.Count(dal.From(scopeSrv.ScopeModelInfo.TableName()), dal.Where("id = ?", connectionId)))
if connectionCount == 0 {
return errors.BadInput.New("connectionId is invalid")
}
- scopeConfigId := (*scope).ScopeScopeConfigId()
- scopeConfigCount := errors.Must1(scopeSrv.db.Count(dal.From(new(SC)), dal.Where("id = ?", scopeConfigId)))
+ scopeConfigId := scopeSrv.ScopeModelInfo.GetScopeConfigId(scope)
+ scopeConfigCount := errors.Must1(scopeSrv.db.Count(dal.From(scopeSrv.ScopeModelInfo.TableName()), dal.Where("id = ?", scopeConfigId)))
if scopeConfigCount == 0 {
return errors.BadInput.New("scopeConfigId is invalid")
}
return nil
}
-func (scopeSrv *ScopeSrvHelper[C, S, SC]) GetScopeDetail(includeBlueprints bool, pkv ...interface{}) (*ScopeDetail[S, SC], errors.Error) {
- scope, err := scopeSrv.ModelSrvHelper.FindByPk(pkv...)
+func (scopeSrv *AnyScopeSrvHelper) GetScopeDetailAny(includeBlueprints bool, pkv ...interface{}) (*AnyScopeDetail, errors.Error) {
+ scope, err := scopeSrv.FindByPkAny(pkv...)
if err != nil {
return nil, err
}
- s := *scope
- scopeDetail := &ScopeDetail[S, SC]{
- Scope: s,
- ScopeConfig: scopeSrv.getScopeConfig(s.ScopeScopeConfigId()),
+ scopeConfigId := scopeSrv.ScopeModelInfo.GetScopeConfigId(scope)
+ scopeDetail := &AnyScopeDetail{
+ Scope: scope,
+ ScopeConfig: scopeSrv.getScopeConfig(scopeConfigId),
}
if includeBlueprints {
- scopeDetail.Blueprints = scopeSrv.getAllBlueprinsByScope(s.ScopeConnectionId(), s.ScopeId())
+ connectionId := scopeSrv.ScopeModelInfo.GetConnectionId(scope)
+ scopeId := scopeSrv.ScopeModelInfo.GetScopeId(scope)
+ scopeDetail.Blueprints = scopeSrv.getAllBlueprinsByScope(connectionId, scopeId)
}
return scopeDetail, nil
}
-func (scopeSrv *ScopeSrvHelper[C, S, SC]) GetScopeLatestSyncState(pkv ...interface{}) ([]*models.LatestSyncState, errors.Error) {
- scope, err := scopeSrv.ModelSrvHelper.FindByPk(pkv...)
+func (scopeSrv *AnyScopeSrvHelper) GetScopeLatestSyncStateAny(pkv ...interface{}) ([]*models.LatestSyncState, errors.Error) {
+ scope, err := scopeSrv.FindByPkAny(pkv...)
if err != nil {
return nil, err
}
- s := *scope
- params := plugin.MarshalScopeParams(s.ScopeParams())
- scopeSrv.log.Debug("scope: %#+v, params: %+v", s, params)
+ params := plugin.MarshalScopeParams(scopeSrv.ScopeModelInfo.GetScopeParams(scope))
+ scopeSrv.log.Debug("scope: %#+v, params: %+v", scope, params)
scopeSyncStates := []*models.LatestSyncState{}
if err := scopeSrv.db.All(
&scopeSyncStates,
@@ -126,73 +136,77 @@ func (scopeSrv *ScopeSrvHelper[C, S, SC]) GetScopeLatestSyncState(pkv ...interfa
}
// MapScopeDetails returns scope details (scope and scopeConfig) for the given blueprint scopes
-func (scopeSrv *ScopeSrvHelper[C, S, SC]) MapScopeDetails(connectionId uint64, bpScopes []*models.BlueprintScope) ([]*ScopeDetail[S, SC], errors.Error) {
+func (scopeSrv *AnyScopeSrvHelper) MapScopeDetailsAny(connectionId uint64, bpScopes []*models.BlueprintScope) ([]*AnyScopeDetail, errors.Error) {
var err errors.Error
- scopeDetails := make([]*ScopeDetail[S, SC], len(bpScopes))
+ scopeDetails := make([]*AnyScopeDetail, len(bpScopes))
for i, bpScope := range bpScopes {
- scopeDetails[i], err = scopeSrv.GetScopeDetail(false, connectionId, bpScope.ScopeId)
+ scopeDetails[i], err = scopeSrv.GetScopeDetailAny(false, connectionId, bpScope.ScopeId)
if err != nil {
return nil, err
}
if scopeDetails[i].ScopeConfig == nil {
- scopeDetails[i].ScopeConfig = new(SC)
+ scopeDetails[i].ScopeConfig = scopeSrv.ScopeConfigModelInfo.New()
}
setDefaultEntities(scopeDetails[i].ScopeConfig)
}
return scopeDetails, nil
}
-func (scopeSrv *ScopeSrvHelper[C, S, SC]) GetScopesPage(pagination *ScopePagination) ([]*ScopeDetail[S, SC], int64, errors.Error) {
+func (scopeSrv *AnyScopeSrvHelper) GetScopesPageAny(pagination *ScopePagination) ([]*AnyScopeDetail, int64, errors.Error) {
if pagination.ConnectionId < 1 {
return nil, 0, errors.BadInput.New("connectionId is required")
}
- scopes, count, err := scopeSrv.ModelSrvHelper.GetPage(
+ scopes, count, err := scopeSrv.QueryPageAny(
&pagination.Pagination,
dal.Where("connection_id = ?", pagination.ConnectionId),
)
if err != nil {
return nil, 0, err
}
-
- data := make([]*ScopeDetail[S, SC], len(scopes))
- for i, s := range scopes {
+ slice := reflect.ValueOf(scopes)
+ data := make([]*AnyScopeDetail, slice.Len())
+ for i := 0; i < slice.Len(); i++ {
// load blueprints
- scope := *s
- scopeDetail := &ScopeDetail[S, SC]{
+ scope := slice.Index(i).Interface()
+ scopeConfigId := scopeSrv.ScopeModelInfo.GetScopeConfigId(scope)
+ scopeDetail := &AnyScopeDetail{
Scope: scope,
- ScopeConfig: scopeSrv.getScopeConfig(scope.ScopeScopeConfigId()),
+ ScopeConfig: scopeSrv.getScopeConfig(scopeConfigId),
}
if pagination.Blueprints {
- scopeDetail.Blueprints = scopeSrv.getAllBlueprinsByScope(scope.ScopeConnectionId(), scope.ScopeId())
+ connectionId := scopeSrv.ScopeModelInfo.GetConnectionId(scope)
+ scopeId := scopeSrv.ScopeModelInfo.GetScopeId(scope)
+ scopeDetail.Blueprints = scopeSrv.getAllBlueprinsByScope(connectionId, scopeId)
}
data[i] = scopeDetail
}
return data, count, nil
}
-func (scopeSrv *ScopeSrvHelper[C, S, SC]) DeleteScope(scope *S, dataOnly bool) (refs *DsRefs, err errors.Error) {
- err = scopeSrv.ModelSrvHelper.NoRunningPipeline(func(tx dal.Transaction) errors.Error {
- s := *scope
+func (scopeSrv *AnyScopeSrvHelper) DeleteScopeAny(scope any, dataOnly bool) (refs *DsRefs, err errors.Error) {
+ err = scopeSrv.NoRunningPipeline(func(tx dal.Transaction) errors.Error {
// check referencing blueprints
if !dataOnly {
- refs = toDsRefs(scopeSrv.getAllBlueprinsByScope(s.ScopeConnectionId(), s.ScopeId()))
+ connectionId := scopeSrv.ScopeModelInfo.GetConnectionId(scope)
+ scopeId := scopeSrv.ScopeModelInfo.GetScopeId(scope)
+ refs = toDsRefs(scopeSrv.getAllBlueprinsByScope(connectionId, scopeId))
if refs != nil {
return errors.Conflict.New("Cannot delete the scope because it is referenced by blueprints")
}
errors.Must(tx.Delete(scope))
}
// delete data
- scopeSrv.deleteScopeData(s, tx)
+ scopeSrv.deleteScopeData(scope, tx)
return nil
})
return
}
-func (scopeSrv *ScopeSrvHelper[C, S, SC]) getScopeConfig(scopeConfigId uint64) *SC {
+func (scopeSrv *AnyScopeSrvHelper) getScopeConfig(scopeConfigId uint64) any {
if scopeConfigId < 1 {
return nil
}
- var scopeConfig SC
+ scopeConfig := scopeSrv.ScopeConfigModelInfo.New()
err := scopeSrv.db.First(
&scopeConfig,
dal.Where(
@@ -206,7 +220,7 @@ func (scopeSrv *ScopeSrvHelper[C, S, SC]) getScopeConfig(scopeConfigId uint64) *
return &scopeConfig
}
-func (scopeSrv *ScopeSrvHelper[C, S, SC]) getAllBlueprinsByScope(connectionId uint64, scopeId string) []*models.Blueprint {
+func (scopeSrv *AnyScopeSrvHelper) getAllBlueprinsByScope(connectionId uint64, scopeId string) []*models.Blueprint {
blueprints := make([]*models.Blueprint, 0)
errors.Must(scopeSrv.db.All(
&blueprints,
@@ -223,8 +237,8 @@ func (scopeSrv *ScopeSrvHelper[C, S, SC]) getAllBlueprinsByScope(connectionId ui
return blueprints
}
-func (scopeSrv *ScopeSrvHelper[C, S, SC]) deleteScopeData(scope plugin.ToolLayerScope, tx dal.Transaction) {
- rawDataParams := plugin.MarshalScopeParams(scope.ScopeParams())
+func (scopeSrv *AnyScopeSrvHelper) deleteScopeData(scope any, tx dal.Transaction) {
+ rawDataParams := plugin.MarshalScopeParams(scopeSrv.ScopeModelInfo.GetScopeParams(scope))
generateWhereClause := func(table string) (string, []any) {
var where string
var params []interface{}
@@ -259,7 +273,7 @@ func (scopeSrv *ScopeSrvHelper[C, S, SC]) deleteScopeData(scope plugin.ToolLayer
}
}
-func (scopeSrv *ScopeSrvHelper[C, S, SC]) getAffectedTables() ([]string, errors.Error) {
+func (scopeSrv *AnyScopeSrvHelper) getAffectedTables() ([]string, errors.Error) {
var tables []string
meta, err := plugin.GetPlugin(scopeSrv.pluginName)
if err != nil {
diff --git a/backend/helpers/srvhelper/connection_service_helper.go b/backend/helpers/srvhelper/connection_service_helper.go
index 7244e2724..6320dfa50 100644
--- a/backend/helpers/srvhelper/connection_service_helper.go
+++ b/backend/helpers/srvhelper/connection_service_helper.go
@@ -18,19 +18,23 @@ limitations under the License.
package srvhelper
import (
- "reflect"
-
"github.com/apache/incubator-devlake/core/context"
- "github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
- "github.com/apache/incubator-devlake/core/models"
"github.com/apache/incubator-devlake/core/plugin"
)
+type GenericConnectionModelInfo[C plugin.ToolLayerConnection] struct {
+ *GenericModelInfo[C]
+}
+
+func (*GenericConnectionModelInfo[C]) GetConnectionId(connection any) uint64 {
+ return connection.(plugin.ToolLayerConnection).ConnectionId()
+}
+
// ConnectionSrvHelper
type ConnectionSrvHelper[C plugin.ToolLayerConnection, S plugin.ToolLayerScope, SC plugin.ToolLayerScopeConfig] struct {
+ *AnyConnectionSrvHelper
*ModelSrvHelper[C]
- pluginName string
}
// NewConnectionSrvHelper creates a ConnectionDalHelper for connection management
@@ -43,44 +47,17 @@ func NewConnectionSrvHelper[
pluginName string,
) *ConnectionSrvHelper[C, S, SC] {
return &ConnectionSrvHelper[C, S, SC]{
+ AnyConnectionSrvHelper: NewAnyConnectionSrvHelper(
+ basicRes,
+ pluginName,
+ &GenericConnectionModelInfo[C]{},
+ &GenericScopeModelInfo[S]{},
+ &GenericScopeConfigModelInfo[SC]{},
+ ),
ModelSrvHelper: NewModelSrvHelper[C](basicRes, nil),
- pluginName: pluginName,
}
}
func (connSrv *ConnectionSrvHelper[C, S, SC]) DeleteConnection(connection *C) (refs *DsRefs, err errors.Error) {
- err = connSrv.ModelSrvHelper.NoRunningPipeline(func(tx dal.Transaction) errors.Error {
- // make sure no blueprint is using the connection
- connectionId := (*connection).ConnectionId()
- refs = toDsRefs(connSrv.getAllBlueprinsByConnection(connectionId))
- if refs != nil {
- return errors.Conflict.New("Cannot delete the connection because it is referenced by blueprints")
- }
- scopeCount := errors.Must1(connSrv.db.Count(dal.From(new(S)), dal.Where("connection_id = ?", connectionId)))
- if scopeCount > 0 {
- return errors.Conflict.New("Please delete all data scope(s) before you delete this Data Connection.")
- }
- errors.Must(tx.Delete(connection))
- if reflect.TypeOf(new(SC)) != reflect.TypeOf(new(NoScopeConfig)) {
- errors.Must(connSrv.db.Delete(new(SC), dal.Where("connection_id = ?", connectionId)))
- }
- return nil
- })
- return
-}
-
-func (connSrv *ConnectionSrvHelper[C, S, SC]) getAllBlueprinsByConnection(connectionId uint64) []*models.Blueprint {
- blueprints := make([]*models.Blueprint, 0)
- errors.Must(connSrv.db.All(
- &blueprints,
- dal.From("_devlake_blueprints bp"),
- dal.Join("JOIN _devlake_blueprint_connections cn ON cn.blueprint_id = bp.id"),
- dal.Where(
- "mode = ? AND cn.connection_id = ? AND cn.plugin_name = ?",
- "NORMAL",
- connectionId,
- connSrv.pluginName,
- ),
- ))
- return blueprints
+ return connSrv.DeleteConnectionAny(connection)
}
diff --git a/backend/helpers/srvhelper/model_service_helper.go b/backend/helpers/srvhelper/model_service_helper.go
index 3ec9abe39..7ffd53600 100644
--- a/backend/helpers/srvhelper/model_service_helper.go
+++ b/backend/helpers/srvhelper/model_service_helper.go
@@ -19,198 +19,84 @@ package srvhelper
import (
"fmt"
- "time"
"github.com/apache/incubator-devlake/core/context"
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
- "github.com/apache/incubator-devlake/core/log"
- "github.com/apache/incubator-devlake/core/models"
- "github.com/apache/incubator-devlake/helpers/dbhelper"
- "github.com/go-playground/validator/v10"
)
-type CustomValidator interface {
- CustomValidate(entity interface{}, validate *validator.Validate) errors.Error
+type GenericModelInfo[M dal.Tabler] struct {
+ modelName string
+ tableName string
}
-type ModelSrvHelper[M dal.Tabler] struct {
- basicRes context.BasicRes
- log log.Logger
- db dal.Dal
- validator *validator.Validate
- modelName string
- pk []dal.ColumnMeta
- pkWhere string
- pkCount int
- searchColumns []string
+func NewGenericModelInfo[M dal.Tabler]() *GenericModelInfo[M] {
+ var m M
+ return &GenericModelInfo[M]{
+ modelName: fmt.Sprintf("%T", m),
+ tableName: m.TableName(),
+ }
}
-func NewModelSrvHelper[M dal.Tabler](basicRes context.BasicRes, searchColumns []string) *ModelSrvHelper[M] {
- m := new(M)
- modelName := fmt.Sprintf("%T", m)
- db := basicRes.GetDal()
- if db == nil {
- db = basicRes.GetDal()
- }
- pk := errors.Must1(dal.GetPrimarykeyColumns(db, *m))
- pkWhere := ""
- for _, col := range pk {
- if pkWhere != "" {
- pkWhere += " AND "
- }
- pkWhere += fmt.Sprintf("%s = ? ", col.Name())
- }
+func (info *GenericModelInfo[M]) ModelName() string {
+ return info.modelName
+}
- return &ModelSrvHelper[M]{
- basicRes: basicRes,
- log: basicRes.GetLogger().Nested(fmt.Sprintf("%s_dal", modelName)),
- db: db,
- validator: validator.New(),
- modelName: modelName,
- pk: pk,
- pkWhere: pkWhere,
- pkCount: len(pk),
- searchColumns: searchColumns,
- }
+func (info *GenericModelInfo[M]) TableName() string {
+ return info.tableName
}
-func (srv *ModelSrvHelper[M]) NewTx(tx dal.Transaction) *ModelSrvHelper[M] {
- helper := new(ModelSrvHelper[M])
- *helper = *srv
- helper.db = tx
- return helper
+func (info *GenericModelInfo[M]) New() interface{} {
+ return new(M)
}
-func (srv *ModelSrvHelper[M]) ValidateModel(model *M) errors.Error {
- // the model can validate itself
- if customValidator, ok := (interface{}(model)).(CustomValidator); ok {
- return customValidator.CustomValidate(model, srv.validator)
- }
- // basic validator
- if e := srv.validator.Struct(model); e != nil {
- return errors.BadInput.Wrap(e, "validation faild")
+func (info *GenericModelInfo[M]) NewSlice() interface{} {
+ return new([]*M)
+}
+
+type ModelSrvHelper[M dal.Tabler] struct {
+ *AnyModelSrvHelper
+}
+
+func NewModelSrvHelper[M dal.Tabler](basicRes context.BasicRes, searchColumns []string) *ModelSrvHelper[M] {
+ return &ModelSrvHelper[M]{
+ NewAnyModelSrvHelper(basicRes, NewGenericModelInfo[M](), searchColumns),
}
- return nil
}
// Create validates given model and insert it into database if validation passed
func (srv *ModelSrvHelper[M]) Create(model *M) errors.Error {
- err := srv.ValidateModel(model)
- if err != nil {
- return err
- }
- err = srv.db.Create(model)
- if err != nil {
- if srv.db.IsDuplicationError(err) {
- return errors.Conflict.Wrap(err, fmt.Sprintf("%s already exists", srv.modelName))
- }
- return err
- }
- return err
+ return srv.CreateAny(model)
}
// Update validates given model and update it into database if validation passed
func (srv *ModelSrvHelper[M]) Update(model *M) errors.Error {
- err := srv.ValidateModel(model)
- if err != nil {
- if srv.db.IsDuplicationError(err) {
- return errors.Conflict.Wrap(err, fmt.Sprintf("%s already exists", srv.modelName))
- }
- return err
- }
- return srv.db.Update(model)
+ return srv.UpdateAny(model)
}
// CreateOrUpdate validates given model and insert or update it into database if validation passed
func (srv *ModelSrvHelper[M]) CreateOrUpdate(model *M) errors.Error {
- err := srv.ValidateModel(model)
- if err != nil {
- return err
- }
- return srv.db.CreateOrUpdate(model)
+ return srv.CreateOrUpdateAny(model)
}
// DeleteModel deletes given model from database
func (srv *ModelSrvHelper[M]) DeleteModel(model *M) errors.Error {
- return srv.db.Delete(model)
+ return srv.DeleteModelAny(model)
}
// FindByPk returns model with given primary key from database
func (srv *ModelSrvHelper[M]) FindByPk(pk ...interface{}) (*M, errors.Error) {
- if len(pk) != srv.pkCount {
- return nil, errors.BadInput.New("invalid primary key")
- }
- model := new(M)
- err := srv.db.First(model, dal.Where(srv.pkWhere, pk...))
- if err != nil {
- if srv.db.IsErrorNotFound(err) {
- return nil, errors.NotFound.Wrap(err, fmt.Sprintf("%s not found", srv.modelName))
- }
- return nil, err
- }
- return model, nil
+ model, err := srv.FindByPkAny(pk...)
+ return model.(*M), err
}
// GetAll returns all models from database
func (srv *ModelSrvHelper[M]) GetAll() ([]*M, errors.Error) {
- array := make([]*M, 0)
- return array, srv.db.All(&array)
+ array, err := srv.QueryAllAny()
+ return array.([]*M), err
}
func (srv *ModelSrvHelper[M]) GetPage(pagination *Pagination, query ...dal.Clause) ([]*M, int64, errors.Error) {
- query = append(query, dal.From(new(M)))
- // process keyword
- searchTerm := pagination.SearchTerm
- if searchTerm != "" && len(srv.searchColumns) > 0 {
- sql := ""
- value := "%" + searchTerm + "%"
- values := make([]interface{}, len(srv.searchColumns))
- for i, field := range srv.searchColumns {
- if sql != "" {
- sql += " OR "
- }
- sql += fmt.Sprintf("%s LIKE ?", field)
- values[i] = value
- }
- sql = fmt.Sprintf("(%s)", sql)
- query = append(query,
- dal.Where(sql, values...),
- )
- }
- count, err := srv.db.Count(query...)
- if err != nil {
- return nil, 0, err
- }
- query = append(query, dal.Limit(pagination.GetLimit()), dal.Offset(pagination.GetOffset()))
- var scopes []*M
- return scopes, count, srv.db.All(&scopes, query...)
-}
-
-func (srv *ModelSrvHelper[M]) NoRunningPipeline(fn func(tx dal.Transaction) errors.Error, tablesToLock ...*dal.LockTable) (err errors.Error) {
- // make sure no pipeline is running
- tablesToLock = append(tablesToLock, &dal.LockTable{Table: "_devlake_pipelines", Exclusive: true})
- txHelper := dbhelper.NewTxHelper(srv.basicRes, &err)
- defer txHelper.End()
- tx := txHelper.Begin()
- err = txHelper.LockTablesTimeout(2*time.Second, tablesToLock)
- if err != nil {
- err = errors.Conflict.Wrap(err, "lock pipelines table timedout")
- return
- }
- count := errors.Must1(tx.Count(
- dal.From("_devlake_pipelines"),
- dal.Where("status = ?", models.TASK_RUNNING),
- ))
- if count > 0 {
- err = errors.Conflict.New("at least one pipeline is running")
- return
- }
- // time.Sleep(1 * time.Minute) # uncomment this line if you were to verify pipelines get blocked while deleting data
- // creating a nested transaction to avoid mysql complaining about table(s) NOT being locked
- nextedTxHelper := dbhelper.NewTxHelper(srv.basicRes, &err)
- defer nextedTxHelper.End()
- nestedTX := nextedTxHelper.Begin()
- err = fn(nestedTX)
- return
+ array, count, err := srv.QueryPageAny(pagination, query...)
+ return array.([]*M), count, err
}
diff --git a/backend/helpers/srvhelper/scope_config_service_helper.go b/backend/helpers/srvhelper/scope_config_service_helper.go
index 796ba29a5..cc77b7dfd 100644
--- a/backend/helpers/srvhelper/scope_config_service_helper.go
+++ b/backend/helpers/srvhelper/scope_config_service_helper.go
@@ -19,21 +19,25 @@ package srvhelper
import (
"github.com/apache/incubator-devlake/core/context"
- "github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
)
-// NoScopeConfig is a placeholder for plugins that don't have any scope configuration yet
-type NoScopeConfig struct{}
+type GenericScopeConfigModelInfo[SC plugin.ToolLayerScopeConfig] struct {
+ *GenericModelInfo[SC]
+}
+
+func (*GenericScopeConfigModelInfo[SC]) GetConnectionId(scopeConfig any) uint64 {
+ return scopeConfig.(plugin.ToolLayerScopeConfig).ScopeConfigConnectionId()
+}
-func (NoScopeConfig) TableName() string { return "" }
-func (NoScopeConfig) ScopeConfigId() uint64 { return 0 }
-func (NoScopeConfig) ScopeConfigConnectionId() uint64 { return 0 }
+func (*GenericScopeConfigModelInfo[SC]) GetScopeConfigId(scopeConfig any) uint64 {
+ return scopeConfig.(plugin.ToolLayerScopeConfig).ScopeConfigId()
+}
// ScopeConfigSrvHelper
type ScopeConfigSrvHelper[C plugin.ToolLayerConnection, S plugin.ToolLayerScope, SC plugin.ToolLayerScopeConfig] struct {
- *ModelSrvHelper[SC]
+ *AnyScopeConfigSrvHelper
}
func NewScopeConfigSrvHelper[
@@ -42,32 +46,20 @@ func NewScopeConfigSrvHelper[
SC plugin.ToolLayerScopeConfig,
](basicRes context.BasicRes, searchColumns []string) *ScopeConfigSrvHelper[C, S, SC] {
return &ScopeConfigSrvHelper[C, S, SC]{
- ModelSrvHelper: NewModelSrvHelper[SC](basicRes, searchColumns),
+ AnyScopeConfigSrvHelper: NewAnyScopeConfigSrvHelper(
+ basicRes,
+ &GenericScopeConfigModelInfo[SC]{},
+ &GenericScopeModelInfo[S]{},
+ ),
}
}
func (scopeConfigSrv *ScopeConfigSrvHelper[C, S, SC]) GetAllByConnectionId(connectionId uint64) ([]*SC, errors.Error) {
- var scopeConfigs []*SC
- err := scopeConfigSrv.db.All(&scopeConfigs,
- dal.Where("connection_id = ?", connectionId),
- dal.Orderby("id DESC"),
- )
- return scopeConfigs, err
+ all, err := scopeConfigSrv.GetAllByConnectionIdAny(connectionId)
+ return all.([]*SC), err
}
func (scopeConfigSrv *ScopeConfigSrvHelper[C, S, SC]) DeleteScopeConfig(scopeConfig *SC) (refs []*S, err errors.Error) {
- err = scopeConfigSrv.ModelSrvHelper.NoRunningPipeline(func(tx dal.Transaction) errors.Error {
- // make sure no scope is using the scopeConfig
- sc := *scopeConfig
- errors.Must(tx.All(
- &refs,
- dal.Where("connection_id = ? AND scope_config_id = ?", sc.ScopeConfigConnectionId(), sc.ScopeConfigId()),
- ))
- if len(refs) > 0 {
- return errors.Conflict.New("Please delete all data scope(s) before you delete this ScopeConfig.")
- }
- errors.Must(tx.Delete(scopeConfig))
- return nil
- })
- return
+ all, err := scopeConfigSrv.DeleteScopeConfigAny(scopeConfig)
+ return all.([]*S), err
}
diff --git a/backend/helpers/srvhelper/scope_service_helper.go b/backend/helpers/srvhelper/scope_service_helper.go
index deda7c25e..7263d5a32 100644
--- a/backend/helpers/srvhelper/scope_service_helper.go
+++ b/backend/helpers/srvhelper/scope_service_helper.go
@@ -18,23 +18,30 @@ limitations under the License.
package srvhelper
import (
- "fmt"
- "reflect"
- "sort"
- "strings"
-
"github.com/apache/incubator-devlake/core/context"
- "github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models"
- "github.com/apache/incubator-devlake/core/models/domainlayer/domaininfo"
"github.com/apache/incubator-devlake/core/plugin"
)
-type ScopePagination struct {
- Pagination `mapstructure:",squash"`
- ConnectionId uint64 `json:"connectionId" mapstructure:"connectionId" validate:"required"`
- Blueprints bool `json:"blueprints" mapstructure:"blueprints"`
+type GenericScopeModelInfo[S plugin.ToolLayerScope] struct {
+ *GenericModelInfo[S]
+}
+
+func (*GenericScopeModelInfo[S]) GetScopeId(scope any) string {
+ return scope.(plugin.ToolLayerScope).ScopeId()
+}
+
+func (*GenericScopeModelInfo[S]) GetConnectionId(scope any) uint64 {
+ return scope.(plugin.ToolLayerScope).ScopeConnectionId()
+}
+
+func (*GenericScopeModelInfo[S]) GetScopeConfigId(scope any) uint64 {
+ return scope.(plugin.ToolLayerScope).ScopeScopeConfigId()
+}
+
+func (*GenericScopeModelInfo[S]) GetScopeParams(scope any) any {
+ return scope.(plugin.ToolLayerScope).ScopeScopeConfigId()
}
type ScopeDetail[S plugin.ToolLayerScope, SC plugin.ToolLayerScopeConfig] struct {
@@ -44,8 +51,7 @@ type ScopeDetail[S plugin.ToolLayerScope, SC plugin.ToolLayerScopeConfig] struct
}
type ScopeSrvHelper[C plugin.ToolLayerConnection, S plugin.ToolLayerScope, SC plugin.ToolLayerScopeConfig] struct {
- *ModelSrvHelper[S]
- pluginName string
+ *AnyScopeSrvHelper
}
// NewScopeSrvHelper creates a ScopeDalHelper for scope management
@@ -59,284 +65,55 @@ func NewScopeSrvHelper[
searchColumns []string,
) *ScopeSrvHelper[C, S, SC] {
return &ScopeSrvHelper[C, S, SC]{
- ModelSrvHelper: NewModelSrvHelper[S](basicRes, searchColumns),
- pluginName: pluginName,
- }
-}
-
-func (scopeSrv *ScopeSrvHelper[C, S, SC]) GetPluginName() string {
- return scopeSrv.pluginName
-}
-
-func (scopeSrv *ScopeSrvHelper[C, S, SC]) Validate(scope *S) errors.Error {
- connectionId := (*scope).ScopeConnectionId()
- connectionCount := errors.Must1(scopeSrv.db.Count(dal.From(new(SC)), dal.Where("id = ?", connectionId)))
- if connectionCount == 0 {
- return errors.BadInput.New("connectionId is invalid")
- }
- scopeConfigId := (*scope).ScopeScopeConfigId()
- scopeConfigCount := errors.Must1(scopeSrv.db.Count(dal.From(new(SC)), dal.Where("id = ?", scopeConfigId)))
- if scopeConfigCount == 0 {
- return errors.BadInput.New("scopeConfigId is invalid")
+ AnyScopeSrvHelper: NewAnyScopeSrvHelper(
+ basicRes,
+ &GenericScopeModelInfo[S]{},
+ &GenericScopeConfigModelInfo[SC]{},
+ pluginName,
+ searchColumns,
+ ),
}
- return nil
}
func (scopeSrv *ScopeSrvHelper[C, S, SC]) GetScopeDetail(includeBlueprints bool, pkv ...interface{}) (*ScopeDetail[S, SC], errors.Error) {
- scope, err := scopeSrv.ModelSrvHelper.FindByPk(pkv...)
- if err != nil {
- return nil, err
- }
- s := *scope
- scopeDetail := &ScopeDetail[S, SC]{
- Scope: s,
- ScopeConfig: scopeSrv.getScopeConfig(s.ScopeScopeConfigId()),
- }
- if includeBlueprints {
- scopeDetail.Blueprints = scopeSrv.getAllBlueprinsByScope(s.ScopeConnectionId(), s.ScopeId())
- }
- return scopeDetail, nil
-}
-
-func (scopeSrv *ScopeSrvHelper[C, S, SC]) GetScopeLatestSyncState(pkv ...interface{}) ([]*models.LatestSyncState, errors.Error) {
- scope, err := scopeSrv.ModelSrvHelper.FindByPk(pkv...)
- if err != nil {
- return nil, err
- }
- s := *scope
- params := plugin.MarshalScopeParams(s.ScopeParams())
- scopeSrv.log.Debug("scope: %#+v, params: %+v", s, params)
- scopeSyncStates := []*models.LatestSyncState{}
- if err := scopeSrv.db.All(
- &scopeSyncStates,
- dal.Select("raw_data_table, latest_success_start, raw_data_params"),
- dal.From("_devlake_collector_latest_state"),
- dal.Where("raw_data_params = ?", params),
- ); err != nil {
- return nil, err
- }
- scopeSrv.log.Debug("param: %+v, resp: %+v", scopeSyncStates)
- sort.Slice(scopeSyncStates, func(i, j int) bool {
- if scopeSyncStates[i].LatestSuccessStart != nil && scopeSyncStates[j].LatestSuccessStart != nil {
- return scopeSyncStates[i].LatestSuccessStart.After(*scopeSyncStates[j].LatestSuccessStart)
- }
- return false
- })
- return scopeSyncStates, nil
+ return scopeSrv.mapScopeDetail(scopeSrv.GetScopeDetailAny(includeBlueprints, pkv...))
}
// MapScopeDetails returns scope details (scope and scopeConfig) for the given blueprint scopes
func (scopeSrv *ScopeSrvHelper[C, S, SC]) MapScopeDetails(connectionId uint64, bpScopes []*models.BlueprintScope) ([]*ScopeDetail[S, SC], errors.Error) {
- var err errors.Error
- scopeDetails := make([]*ScopeDetail[S, SC], len(bpScopes))
- for i, bpScope := range bpScopes {
- scopeDetails[i], err = scopeSrv.GetScopeDetail(false, connectionId, bpScope.ScopeId)
- if err != nil {
- return nil, err
- }
- if scopeDetails[i].ScopeConfig == nil {
- scopeDetails[i].ScopeConfig = new(SC)
- }
- setDefaultEntities(scopeDetails[i].ScopeConfig)
- }
- return scopeDetails, nil
+ return scopeSrv.mapScopeDetails(scopeSrv.MapScopeDetailsAny(connectionId, bpScopes))
}
func (scopeSrv *ScopeSrvHelper[C, S, SC]) GetScopesPage(pagination *ScopePagination) ([]*ScopeDetail[S, SC], int64, errors.Error) {
- if pagination.ConnectionId < 1 {
- return nil, 0, errors.BadInput.New("connectionId is required")
- }
- scopes, count, err := scopeSrv.ModelSrvHelper.GetPage(
- &pagination.Pagination,
- dal.Where("connection_id = ?", pagination.ConnectionId),
- )
- if err != nil {
- return nil, 0, err
- }
-
- data := make([]*ScopeDetail[S, SC], len(scopes))
- for i, s := range scopes {
- // load blueprints
- scope := *s
- scopeDetail := &ScopeDetail[S, SC]{
- Scope: scope,
- ScopeConfig: scopeSrv.getScopeConfig(scope.ScopeScopeConfigId()),
- }
- if pagination.Blueprints {
- scopeDetail.Blueprints = scopeSrv.getAllBlueprinsByScope(scope.ScopeConnectionId(), scope.ScopeId())
- }
- data[i] = scopeDetail
- }
- return data, count, nil
-}
-
-func (scopeSrv *ScopeSrvHelper[C, S, SC]) DeleteScope(scope *S, dataOnly bool) (refs *DsRefs, err errors.Error) {
- err = scopeSrv.ModelSrvHelper.NoRunningPipeline(func(tx dal.Transaction) errors.Error {
- s := *scope
- // check referencing blueprints
- if !dataOnly {
- refs = toDsRefs(scopeSrv.getAllBlueprinsByScope(s.ScopeConnectionId(), s.ScopeId()))
- if refs != nil {
- return errors.Conflict.New("Cannot delete the scope because it is referenced by blueprints")
- }
- errors.Must(tx.Delete(scope))
- }
- // delete data
- scopeSrv.deleteScopeData(s, tx)
- return nil
- })
- return
+ scopes, count, err := scopeSrv.GetScopesPageAny(pagination)
+ scopeDetails, err := scopeSrv.mapScopeDetails(scopes, err)
+ return scopeDetails, count, err
}
-func (scopeSrv *ScopeSrvHelper[C, S, SC]) getScopeConfig(scopeConfigId uint64) *SC {
- if scopeConfigId < 1 {
- return nil
- }
- var scopeConfig SC
- err := scopeSrv.db.First(
- &scopeConfig,
- dal.Where(
- "id = ?",
- scopeConfigId,
- ),
- )
+func (scopeSrv *ScopeSrvHelper[C, S, SC]) mapScopeDetail(detail *AnyScopeDetail, err errors.Error) (*ScopeDetail[S, SC], errors.Error) {
if err != nil {
- return nil
- }
- return &scopeConfig
-}
-
-func (scopeSrv *ScopeSrvHelper[C, S, SC]) getAllBlueprinsByScope(connectionId uint64, scopeId string) []*models.Blueprint {
- blueprints := make([]*models.Blueprint, 0)
- errors.Must(scopeSrv.db.All(
- &blueprints,
- dal.From("_devlake_blueprints bp"),
- dal.Join("JOIN _devlake_blueprint_scopes sc ON sc.blueprint_id = bp.id"),
- dal.Where(
- "mode = ? AND sc.connection_id = ? AND sc.plugin_name = ? AND sc.scope_id = ?",
- "NORMAL",
- connectionId,
- scopeSrv.pluginName,
- scopeId,
- ),
- ))
- return blueprints
-}
-
-func (scopeSrv *ScopeSrvHelper[C, S, SC]) deleteScopeData(scope plugin.ToolLayerScope, tx dal.Transaction) {
- rawDataParams := plugin.MarshalScopeParams(scope.ScopeParams())
- generateWhereClause := func(table string) (string, []any) {
- var where string
- var params []interface{}
- if strings.HasPrefix(table, "_raw_") {
- // raw table: should check connection and scope
- where = "params = ?"
- params = []interface{}{rawDataParams}
- } else if strings.HasPrefix(table, "_tool_") {
- // tool layer table: should check connection and scope
- where = "_raw_data_params = ?"
- params = []interface{}{rawDataParams}
- } else {
- // framework tables: should check plugin, connection and scope
- if table == (models.CollectorLatestState{}.TableName()) {
- // diff sync state
- where = "raw_data_table LIKE ? AND raw_data_params = ?"
- } else {
- // domain layer table
- where = "_raw_data_table LIKE ? AND _raw_data_params = ?"
- }
- rawDataTablePrefix := fmt.Sprintf("_raw_%s%%", scopeSrv.pluginName)
- params = []interface{}{rawDataTablePrefix, rawDataParams}
- }
- return where, params
- }
- tables := errors.Must1(scopeSrv.getAffectedTables())
- for _, table := range tables {
- where, params := generateWhereClause(table)
- scopeSrv.log.Info("deleting data from table %s with WHERE \"%s\" and params: \"%v\"", table, where, params)
- sql := fmt.Sprintf("DELETE FROM %s WHERE %s", table, where)
- errors.Must(tx.Exec(sql, params...))
+ return nil, err
}
+ return &ScopeDetail[S, SC]{
+ Scope: detail.Scope.(S),
+ ScopeConfig: detail.ScopeConfig.(*SC),
+ }, nil
}
-func (scopeSrv *ScopeSrvHelper[C, S, SC]) getAffectedTables() ([]string, errors.Error) {
- var tables []string
- meta, err := plugin.GetPlugin(scopeSrv.pluginName)
+func (scopeSrv *ScopeSrvHelper[C, S, SC]) mapScopeDetails(details []*AnyScopeDetail, err errors.Error) ([]*ScopeDetail[S, SC], errors.Error) {
if err != nil {
return nil, err
}
- if pluginModel, ok := meta.(plugin.PluginModel); !ok {
- panic(errors.Default.New(fmt.Sprintf("plugin \"%s\" does not implement listing its tables", scopeSrv.pluginName)))
- } else {
- // Unfortunately, can't cache the tables because Python creates some tables on a per-demand basis, so such a cache would possibly get outdated.
- // It's a rare scenario in practice, but might as well play it safe and sacrifice some performance here
- var allTables []string
- if allTables, err = scopeSrv.db.AllTables(); err != nil {
- return nil, err
- }
- // collect raw tables
- for _, table := range allTables {
- if strings.HasPrefix(table, "_raw_"+scopeSrv.pluginName) {
- tables = append(tables, table)
- }
- }
- // collect tool tables
- toolModels := pluginModel.GetTablesInfo()
- for _, toolModel := range toolModels {
- if !isScopeModel(toolModel) && hasField(toolModel, "RawDataParams") {
- tables = append(tables, toolModel.TableName())
- }
+ scopeDetails := make([]*ScopeDetail[S, SC], len(details))
+ for i, detail := range details {
+ scopeDetails[i] = &ScopeDetail[S, SC]{
+ Scope: detail.Scope.(S),
+ ScopeConfig: detail.ScopeConfig.(*SC),
}
- // collect domain tables
- for _, domainModel := range domaininfo.GetDomainTablesInfo() {
- // we only care about tables with RawOrigin
- ok = hasField(domainModel, "RawDataParams")
- if ok {
- tables = append(tables, domainModel.TableName())
- }
- }
- // additional tables
- tables = append(tables, models.CollectorLatestState{}.TableName())
}
- scopeSrv.log.Debug("Discovered %d tables used by plugin \"%s\": %v", len(tables), scopeSrv.pluginName, tables)
- return tables, nil
-}
-
-// TODO: sort out the follow functions
-func isScopeModel(obj dal.Tabler) bool {
- _, ok := obj.(plugin.ToolLayerScope)
- return ok
+ return scopeDetails, err
}
-func hasField(obj any, fieldName string) bool {
- obj = models.UnwrapObject(obj)
- _, ok := reflectType(obj).FieldByName(fieldName)
- return ok
-}
-
-func reflectType(obj any) reflect.Type {
- obj = models.UnwrapObject(obj)
- typ := reflect.TypeOf(obj)
- kind := typ.Kind()
- for kind == reflect.Ptr {
- typ = typ.Elem()
- kind = typ.Kind()
- }
- return typ
-}
-
-func setDefaultEntities(sc interface{}) {
- v := reflect.ValueOf(sc)
- if v.Kind() != reflect.Pointer {
- panic(fmt.Errorf("sc must be a pointer"))
- }
- entities := v.Elem().FieldByName("Entities")
- if !entities.IsValid() ||
- !(entities.Kind() == reflect.Array || entities.Kind() == reflect.Slice) ||
- entities.Type().Elem().Kind() != reflect.String {
- return
- }
- if entities.IsNil() || entities.Len() == 0 {
- entities.Set(reflect.ValueOf(plugin.DOMAIN_TYPES))
- }
+func (scopeSrv *ScopeSrvHelper[C, S, SC]) DeleteScope(scope *S, dataOnly bool) (refs *DsRefs, err errors.Error) {
+ return scopeSrv.DeleteScopeAny(scope, dataOnly)
}
diff --git a/backend/impls/dalgorm/dalgorm.go b/backend/impls/dalgorm/dalgorm.go
index a3de41123..080098d77 100644
--- a/backend/impls/dalgorm/dalgorm.go
+++ b/backend/impls/dalgorm/dalgorm.go
@@ -294,8 +294,16 @@ func (d *Dalgorm) UpdateAllColumn(entity interface{}, clauses ...dal.Clause) err
}
// GetColumns FIXME ...
-func (d *Dalgorm) GetColumns(dst dal.Tabler, filter func(columnMeta dal.ColumnMeta) bool) (cms []dal.ColumnMeta, _ errors.Error) {
- columnTypes, err := d.db.Migrator().ColumnTypes(dst.TableName())
+func (d *Dalgorm) GetColumns(dst interface{}, filter func(columnMeta dal.ColumnMeta) bool) (cms []dal.ColumnMeta, _ errors.Error) {
+ tableName := ""
+ if tabler, ok := dst.(dal.Tabler); ok {
+ tableName = tabler.TableName()
+ } else if tn, ok := dst.(string); ok {
+ tableName = tn
+ } else {
+ panic(fmt.Errorf("invalid dst: %v", dst))
+ }
+ columnTypes, err := d.db.Migrator().ColumnTypes(tableName)
if err != nil {
return nil, d.convertGormError(err)
}
diff --git a/backend/plugins/azuredevops_go/api/blueprint_v200.go b/backend/plugins/azuredevops_go/api/blueprint_v200.go
index a9d178e47..4adf19347 100644
--- a/backend/plugins/azuredevops_go/api/blueprint_v200.go
+++ b/backend/plugins/azuredevops_go/api/blueprint_v200.go
@@ -44,7 +44,7 @@ func MakePipelinePlanV200(
if err != nil {
return nil, nil, err
}
- scopeDetails, err := dsHelper.ScopeApi.MapScopeDetails(connectionId, bpScopes)
+ scopeDetails, err := dsHelper.ScopeSrv.MapScopeDetails(connectionId, bpScopes)
if err != nil {
return nil, nil, err
}
diff --git a/backend/plugins/azuredevops_go/api/connection_api.go b/backend/plugins/azuredevops_go/api/connection_api.go
index b096890d9..45e6be6e8 100644
--- a/backend/plugins/azuredevops_go/api/connection_api.go
+++ b/backend/plugins/azuredevops_go/api/connection_api.go
@@ -18,12 +18,13 @@ limitations under the License.
package api
import (
+ "net/http"
+
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/azuredevops_go/models"
"github.com/apache/incubator-devlake/server/api/shared"
- "net/http"
)
type AzuredevopsTestConnResponse struct {
diff --git a/backend/plugins/gitlab/api/blueprint_v200.go b/backend/plugins/gitlab/api/blueprint_v200.go
index 2596c4ba1..4e048e85f 100644
--- a/backend/plugins/gitlab/api/blueprint_v200.go
+++ b/backend/plugins/gitlab/api/blueprint_v200.go
@@ -50,7 +50,7 @@ func MakePipelinePlanV200(
if err != nil {
return nil, nil, err
}
- scopeDetails, err := dsHelper.ScopeApi.MapScopeDetails(connectionId, bpScopes)
+ scopeDetails, err := dsHelper.ScopeSrv.MapScopeDetails(connectionId, bpScopes)
if err != nil {
return nil, nil, err
}
diff --git a/backend/python/DevelopmentSetup.md b/backend/python/DevelopmentSetup.md
index f026fe760..c5a662a79 100644
--- a/backend/python/DevelopmentSetup.md
+++ b/backend/python/DevelopmentSetup.md
@@ -21,7 +21,8 @@ limitations under the License.
- Try `deadsnakes` if you are using Ubuntu 22.04 or above, the `python3.9-dev` is required.
- Use `virtualenv` if you are having multiple python versions. `virtualenv -p python3.9 path/to/venv` and `source path/to/venv/bin/activate.sh` should do the trick
3. both `mysql-client` and `postgresql` are required.
- - `postgresql` is required for `psycopg2` to work
+ - (macOS?) `postgresql` is required for `psycopg2` to work
+ - (Ubuntu) `libmysqlclient-dev` and `libpq-dev` is required for `psycopg2` to work
4. [poetry](https://python-poetry.org/) is required.
- run `cd backend/python/pydevlake && poetry install`
- run `cd backend/python/plugins/azuredevops && poetry install`
diff --git a/backend/python/plugins/azuredevops/run.sh b/backend/python/plugins/azuredevops/run.sh
index e1ea5dd32..b5c7472bb 100755
--- a/backend/python/plugins/azuredevops/run.sh
+++ b/backend/python/plugins/azuredevops/run.sh
@@ -17,4 +17,7 @@
#
cd "$(dirname "$0")"
+env >&2
+# [ -n "$VIRTUAL_ENV" ] && echo "Using virtualenv: $VIRTUAL_ENV" >&2 && . "$VIRTUAL_ENV/bin/activate"
+
poetry run python azuredevops/main.py "$@"
diff --git a/backend/server/services/remote/models/models.go b/backend/server/services/remote/models/models.go
index 58cd39ce5..b6ae623fe 100644
--- a/backend/server/services/remote/models/models.go
+++ b/backend/server/services/remote/models/models.go
@@ -47,12 +47,6 @@ type PluginInfo struct {
Extension PluginExtension `json:"extension"`
}
-// Type aliases used by the API helper for better readability
-type (
- RemoteScopeConfig any
- RemoteConnection any
-)
-
type DynamicModelInfo struct {
JsonSchema map[string]any `json:"json_schema" validate:"required"`
TableName string `json:"table_name" validate:"required"`
@@ -75,6 +69,13 @@ type ApiParams struct {
ScopeId string
}
+type RemoteConnection any
+
+func (d RemoteConnection) ConnectionId() uint64 {
+ return 0
+ // return reflect.ValueOf(d.DynamicTabler.Unwrap()).Elem().FieldByName("Id").Uint()
+}
+
type DynamicScopeModel struct {
models.DynamicTabler
}
@@ -158,3 +159,15 @@ type ToolModel struct {
}
var _ plugin.ToolLayerScope = (*DynamicScopeModel)(nil)
+
+type RemoteScopeConfig struct {
+ models.DynamicTabler
+}
+
+func (d RemoteScopeConfig) ScopeConfigConnectionId() uint64 {
+ return reflect.ValueOf(d.DynamicTabler.Unwrap()).Elem().FieldByName("ConnectionId").Uint()
+}
+
+func (d RemoteScopeConfig) ScopeConfigId() uint64 {
+ return reflect.ValueOf(d.DynamicTabler.Unwrap()).Elem().FieldByName("ScopeConfigId").Uint()
+}
diff --git a/backend/server/services/remote/plugin/connection_api.go b/backend/server/services/remote/plugin/connection_api.go
index 38c832ee7..999e6e3a9 100644
--- a/backend/server/services/remote/plugin/connection_api.go
+++ b/backend/server/services/remote/plugin/connection_api.go
@@ -20,13 +20,13 @@ package plugin
import (
"encoding/json"
"fmt"
+
"github.com/spf13/cast"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/server/api/shared"
"github.com/apache/incubator-devlake/server/services/remote/bridge"
- "net/http"
)
type TestConnectionResult struct {
@@ -35,34 +35,6 @@ type TestConnectionResult struct {
Status int `json:"status"`
}
-func sanitizeConnection(connection interface{}) (map[string]interface{}, error) {
- data, err := json.Marshal(connection)
- if err != nil {
- return nil, err
- }
- result := make(map[string]interface{})
-
- if err := json.Unmarshal(data, &result); err != nil {
- return nil, err
- }
- if _, ok := result["token"]; ok {
- result["token"] = ""
- }
- return result, nil
-}
-
-func multiSanitizeConnections(connections []interface{}) ([]map[string]interface{}, error) {
- var results []map[string]interface{}
- for _, c := range connections {
- result, err := sanitizeConnection(c)
- if err != nil {
- return nil, err
- }
- results = append(results, result)
- }
- return results, nil
-}
-
func (pa *pluginAPI) TestConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
var result TestConnectionResult
err := pa.invoker.Call("test-connection", bridge.DefaultContext, input.Body).Get(&result)
@@ -82,8 +54,7 @@ func (pa *pluginAPI) TestConnection(input *plugin.ApiResourceInput) (*plugin.Api
}
func (pa *pluginAPI) TestExistingConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
- connection := pa.connType.New()
- err := pa.connhelper.First(connection, input.Params)
+ connection, err := pa.dsHelper.ConnApi.GetMergedConnection(input)
if err != nil {
return nil, err
}
@@ -119,65 +90,21 @@ func (pa *pluginAPI) TestExistingConnection(input *plugin.ApiResourceInput) (*pl
}
func (pa *pluginAPI) PostConnections(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
- connection := pa.connType.New()
- err := pa.connhelper.Create(connection, input)
- if err != nil {
- return nil, err
- }
- conn := connection.Unwrap()
- result, sanitizeErr := sanitizeConnection(conn)
- if sanitizeErr != nil {
- return nil, errors.Convert(sanitizeErr)
- }
- return &plugin.ApiResourceOutput{Body: result, Status: http.StatusOK}, nil
+ return pa.dsHelper.ConnApi.Post(input)
}
func (pa *pluginAPI) ListConnections(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
- connections := pa.connType.NewSlice()
- err := pa.connhelper.List(connections)
- if err != nil {
- return nil, err
- }
- conns := connections.UnwrapSlice()
- if len(conns) == 0 {
- conns = []interface{}{}
- return &plugin.ApiResourceOutput{Body: conns}, nil
- }
- results, sanitizeErr := multiSanitizeConnections(conns)
- if sanitizeErr != nil {
- return nil, errors.Convert(sanitizeErr)
- }
- return &plugin.ApiResourceOutput{Body: results}, nil
+ return pa.dsHelper.ConnApi.GetAll(input)
}
func (pa *pluginAPI) GetConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
- connection := pa.connType.New()
- err := pa.connhelper.First(connection, input.Params)
- if err != nil {
- return nil, err
- }
- conn := connection.Unwrap()
- result, sanitizeErr := sanitizeConnection(conn)
- if sanitizeErr != nil {
- return nil, errors.Convert(sanitizeErr)
- }
- return &plugin.ApiResourceOutput{Body: result}, nil
+ return pa.dsHelper.ConnApi.GetDetail(input)
}
func (pa *pluginAPI) PatchConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
- connection := pa.connType.New()
- err := pa.connhelper.Patch(connection, input)
- if err != nil {
- return nil, err
- }
- conn := connection.Unwrap()
- result, sanitizeErr := sanitizeConnection(conn)
- if sanitizeErr != nil {
- return nil, errors.Convert(sanitizeErr)
- }
- return &plugin.ApiResourceOutput{Body: result, Status: http.StatusOK}, nil
+ return pa.dsHelper.ConnApi.Patch(input)
}
func (pa *pluginAPI) DeleteConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
- return pa.connhelper.Delete(pa.connType.New(), input)
+ return pa.dsHelper.ConnApi.Delete(input)
}
diff --git a/backend/server/services/remote/plugin/default_api.go b/backend/server/services/remote/plugin/default_api.go
index f891a5a20..465799ec7 100644
--- a/backend/server/services/remote/plugin/default_api.go
+++ b/backend/server/services/remote/plugin/default_api.go
@@ -30,8 +30,7 @@ type pluginAPI struct {
connType models.DynamicTabler
scopeType models.DynamicTabler
scopeConfigType models.DynamicTabler
- connhelper *api.ConnectionApiHelper
- scopeHelper *api.GenericScopeApiHelper[remoteModel.RemoteConnection, remoteModel.DynamicScopeModel, remoteModel.RemoteScopeConfig]
+ dsHelper *api.DsHelper[remoteModel.RemoteConnection, remoteModel.DynamicScopeModel, remoteModel.RemoteScopeConfig]
}
func GetDefaultAPI(
@@ -39,14 +38,14 @@ func GetDefaultAPI(
connType models.DynamicTabler,
scopeConfigType models.DynamicTabler,
scopeType models.DynamicTabler,
- connHelper *api.ConnectionApiHelper,
+ dsHelper *api.DsHelper[remoteModel.RemoteConnection, remoteModel.DynamicScopeModel, remoteModel.RemoteScopeConfig],
) map[string]map[string]plugin.ApiResourceHandler {
papi := &pluginAPI{
invoker: invoker,
connType: connType,
scopeConfigType: scopeConfigType,
scopeType: scopeType,
- connhelper: connHelper,
+ dsHelper: dsHelper,
}
resources := map[string]map[string]plugin.ApiResourceHandler{
"test": {
@@ -94,24 +93,5 @@ func GetDefaultAPI(
"GET": papi.SearchRemoteScopes,
},
}
- papi.createScopeHelper()
return resources
}
-
-func (pa *pluginAPI) createScopeHelper() {
- params := &api.ReflectionParameters{
- ScopeIdFieldName: "Id",
- ScopeIdColumnName: "id",
- RawScopeParamName: "ScopeId",
- }
- pa.scopeHelper = api.NewGenericScopeHelper[remoteModel.RemoteConnection, remoteModel.DynamicScopeModel, remoteModel.RemoteScopeConfig](
- basicRes,
- vld,
- pa.connhelper,
- NewScopeDatabaseHelperImpl(pa, basicRes, params),
- params,
- &api.ScopeHelperOptions{
- IsRemote: true,
- },
- )
-}
diff --git a/backend/server/services/remote/plugin/init.go b/backend/server/services/remote/plugin/init.go
index 12a2fa9a8..615e577ad 100644
--- a/backend/server/services/remote/plugin/init.go
+++ b/backend/server/services/remote/plugin/init.go
@@ -22,16 +22,13 @@ import (
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/server/services/remote/bridge"
"github.com/apache/incubator-devlake/server/services/remote/models"
- "github.com/go-playground/validator/v10"
)
var (
basicRes context.BasicRes
- vld *validator.Validate
)
func Init(br context.BasicRes) {
- vld = validator.New()
basicRes = br
}
diff --git a/backend/server/services/remote/plugin/plugin_extensions.go b/backend/server/services/remote/plugin/plugin_extensions.go
index 5f1f4612c..fea3e0c61 100644
--- a/backend/server/services/remote/plugin/plugin_extensions.go
+++ b/backend/server/services/remote/plugin/plugin_extensions.go
@@ -44,8 +44,7 @@ func (p remoteDatasourcePlugin) MakeDataSourcePipelinePlanV200(
connectionId uint64,
bpScopes []*coreModels.BlueprintScope,
) (coreModels.PipelinePlan, []plugin.Scope, errors.Error) {
- connection := p.connectionTabler.New()
- err := p.connHelper.FirstById(connection, connectionId)
+ connection, err := p.dsHelper.ConnSrv.FindByPk(connectionId)
if err != nil {
return nil, nil, err
}
diff --git a/backend/server/services/remote/plugin/plugin_impl.go b/backend/server/services/remote/plugin/plugin_impl.go
index 6242b1600..dd61e4730 100644
--- a/backend/server/services/remote/plugin/plugin_impl.go
+++ b/backend/server/services/remote/plugin/plugin_impl.go
@@ -19,7 +19,9 @@ package plugin
import (
"fmt"
+ "reflect"
+ "github.com/apache/incubator-devlake/core/context"
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
coreModels "github.com/apache/incubator-devlake/core/models"
@@ -28,6 +30,7 @@ import (
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/server/services/remote/bridge"
"github.com/apache/incubator-devlake/server/services/remote/models"
+ remoteModel "github.com/apache/incubator-devlake/server/services/remote/models"
"github.com/apache/incubator-devlake/server/services/remote/models/migrationscripts"
"github.com/apache/incubator-devlake/server/services/remote/plugin/doc"
)
@@ -46,7 +49,7 @@ type (
migrationScripts []plugin.MigrationScript
resources map[string]map[string]plugin.ApiResourceHandler
openApiSpec string
- connHelper *api.ConnectionApiHelper
+ dsHelper *api.DsHelper[remoteModel.RemoteConnection, remoteModel.DynamicScopeModel, remoteModel.RemoteScopeConfig]
}
RemotePluginTaskData struct {
DbUrl string `json:"db_url"`
@@ -91,12 +94,6 @@ func newPlugin(info *models.PluginInfo, invoker bridge.Invoker) (*remotePluginIm
script := script
scripts = append(scripts, &script)
}
- connectionHelper := api.NewConnectionHelper(
- basicRes,
- vld,
- info.Name,
- )
- apiResources := GetDefaultAPI(invoker, connectionTabler, scopeConfigTabler, scopeTabler, connectionHelper)
p := remotePluginImpl{
name: info.Name,
invoker: invoker,
@@ -107,9 +104,7 @@ func newPlugin(info *models.PluginInfo, invoker bridge.Invoker) (*remotePluginIm
scopeConfigTabler: scopeConfigTabler,
toolModelTablers: toolModelTablers,
migrationScripts: scripts,
- resources: apiResources,
openApiSpec: *openApiSpec,
- connHelper: connectionHelper,
}
remoteBridge := bridge.NewBridge(invoker)
for _, subtask := range info.SubtaskMetas {
@@ -125,6 +120,25 @@ func newPlugin(info *models.PluginInfo, invoker bridge.Invoker) (*remotePluginIm
return &p, nil
}
+func (p *remotePluginImpl) Init(basicRes context.BasicRes) errors.Error {
+ p.dsHelper = api.NewDataSourceHelper[remoteModel.RemoteConnection, remoteModel.DynamicScopeModel, remoteModel.RemoteScopeConfig](
+ basicRes,
+ p.Name(),
+ []string{"name"},
+ func(c models.RemoteConnection) models.RemoteConnection {
+ reflect.ValueOf(c.DynamicTabler.Unwrap()).Elem().FieldByName("token").SetString("")
+ return c
+ },
+ nil,
+ nil,
+ p.connectionTabler,
+ p.scopeTabler,
+ p.scopeConfigTabler,
+ )
+ p.resources = GetDefaultAPI(p.invoker, p.connectionTabler, p.scopeConfigTabler, p.scopeTabler, p.dsHelper)
+ return nil
+}
+
func (p *remotePluginImpl) SubTaskMetas() []plugin.SubTaskMeta {
return p.subtaskMetas
}
diff --git a/backend/server/services/remote/plugin/remote_scope_api.go b/backend/server/services/remote/plugin/remote_scope_api.go
index 200449b38..503a3e487 100644
--- a/backend/server/services/remote/plugin/remote_scope_api.go
+++ b/backend/server/services/remote/plugin/remote_scope_api.go
@@ -18,10 +18,11 @@ limitations under the License.
package plugin
import (
- "github.com/apache/incubator-devlake/core/models"
"net/http"
"strconv"
+ "github.com/apache/incubator-devlake/core/models"
+
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/server/services/remote/bridge"
@@ -44,8 +45,7 @@ func (pa *pluginAPI) GetRemoteScopes(input *plugin.ApiResourceInput) (*plugin.Ap
if connectionId == 0 {
return nil, errors.BadInput.New("invalid connectionId")
}
- connection := pa.connType.New()
- err := pa.connhelper.First(connection, input.Params)
+ connection, err := pa.dsHelper.ConnApi.FindByPk(input)
if err != nil {
return nil, err
}
diff --git a/backend/server/services/remote/plugin/scope_api.go b/backend/server/services/remote/plugin/scope_api.go
index 9450422e7..77da2f268 100644
--- a/backend/server/services/remote/plugin/scope_api.go
+++ b/backend/server/services/remote/plugin/scope_api.go
@@ -18,16 +18,10 @@ limitations under the License.
package plugin
import (
- "net/http"
- "sort"
"strings"
- "github.com/apache/incubator-devlake/server/services/remote/models"
- "github.com/mitchellh/mapstructure"
-
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
- "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
)
type request struct {
@@ -35,60 +29,15 @@ type request struct {
}
func (pa *pluginAPI) PutScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
- var scopes request
- err := errors.Convert(mapstructure.Decode(input.Body, &scopes))
- if err != nil {
- return nil, errors.BadInput.Wrap(err, "decoding scope error")
- }
- var slice []*models.DynamicScopeModel
- for _, scope := range scopes.Data {
- obj := models.NewDynamicScopeModel(pa.scopeType)
- err = models.MapTo(scope, obj)
- if err != nil {
- return nil, err
- }
- slice = append(slice, obj)
- }
- apiScopes, err := pa.scopeHelper.PutScopes(input, slice)
- if err != nil {
- return nil, err
- }
- response, err := convertScopeResponse(apiScopes...)
- if err != nil {
- return nil, err
- }
- return &plugin.ApiResourceOutput{Body: response, Status: http.StatusOK}, nil
+ return pa.dsHelper.ScopeApi.PutMultiple(input)
}
func (pa *pluginAPI) UpdateScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
- apiScopes, err := pa.scopeHelper.UpdateScope(input)
- if err != nil {
- return nil, err
- }
- response, err := convertScopeResponse(apiScopes)
- if err != nil {
- return nil, err
- }
- return &plugin.ApiResourceOutput{Body: response[0], Status: http.StatusOK}, nil
+ return pa.dsHelper.ScopeApi.Patch(input)
}
func (pa *pluginAPI) ListScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
- paginated, err := pa.scopeHelper.GetScopes(input)
- if err != nil {
- return nil, err
- }
- scopes, err := convertScopeResponse(paginated.Scopes...)
- if err != nil {
- return nil, err
- }
- body := &struct {
- Scopes []map[string]any `json:"scopes"`
- Count int64 `json:"count"`
- }{
- Scopes: scopes,
- Count: paginated.Count,
- }
- return &plugin.ApiResourceOutput{Body: body, Status: http.StatusOK}, nil
+ return pa.dsHelper.ScopeApi.GetPage(input)
}
func (pa *pluginAPI) GetScopeDispatcher(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
@@ -101,58 +50,13 @@ func (pa *pluginAPI) GetScopeDispatcher(input *plugin.ApiResourceInput) (*plugin
}
func (pa *pluginAPI) GetScopeLatestSyncState(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
- scopeSyncStates, err := pa.scopeHelper.GetScopeLatestSyncState(input)
- if err != nil {
- return nil, err
- }
- sort.Slice(scopeSyncStates, func(i, j int) bool {
- if scopeSyncStates[i].LatestSuccessStart != nil && scopeSyncStates[j].LatestSuccessStart != nil {
- return scopeSyncStates[i].LatestSuccessStart.After(*scopeSyncStates[j].LatestSuccessStart)
- }
- return false
- })
- return &plugin.ApiResourceOutput{Body: scopeSyncStates, Status: http.StatusOK}, nil
+ return pa.dsHelper.ScopeApi.GetScopeLatestSyncState(input)
}
func (pa *pluginAPI) GetScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
- scope, err := pa.scopeHelper.GetScope(input)
- if err != nil {
- return nil, err
- }
- response, err := convertScopeResponse(scope)
- if err != nil {
- return nil, err
- }
- return &plugin.ApiResourceOutput{Body: response[0], Status: http.StatusOK}, nil
+ return pa.dsHelper.ScopeApi.GetDetail(input)
}
func (pa *pluginAPI) DeleteScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
- refs, err := pa.scopeHelper.DeleteScope(input)
- if err != nil {
- return &plugin.ApiResourceOutput{Body: refs, Status: err.GetType().GetHttpCode()}, err
- }
- return &plugin.ApiResourceOutput{Body: nil, Status: http.StatusOK}, nil
-}
-
-// convertScopeResponse adapt the "remote" scopes to a serializable api.ScopeRes. This code is needed because squashed mapstructure don't work
-// with dynamic/runtime structs used by remote plugins
-func convertScopeResponse(scopes ...*api.ScopeRes[models.DynamicScopeModel, models.RemoteScopeConfig]) ([]map[string]any, errors.Error) {
- responses := make([]map[string]any, len(scopes))
- for i, scope := range scopes {
- resMap := map[string]any{}
- err := models.MapTo(scope, &resMap)
- if err != nil {
- return nil, err
- }
- scopeMap := map[string]map[string]any{}
- err = models.MapTo(scope.Scope, &scopeMap)
- if err != nil {
- return nil, err
- }
- for k, v := range scopeMap["DynamicTabler"] {
- resMap[k] = v
- }
- responses[i] = resMap
- }
- return responses, nil
+ return pa.dsHelper.ScopeApi.Delete(input)
}
diff --git a/backend/server/services/remote/plugin/scope_db_helper.go b/backend/server/services/remote/plugin/scope_db_helper.go.bak
similarity index 100%
rename from backend/server/services/remote/plugin/scope_db_helper.go
rename to backend/server/services/remote/plugin/scope_db_helper.go.bak