You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by zh...@apache.org on 2023/05/31 04:07:14 UTC
[incubator-devlake] branch main updated: refactor: rename transformation rule to scope config - gitlab (#5313)
This is an automated email from the ASF dual-hosted git repository.
zhangliang2022 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 4b6820f89 refactor: rename transformation rule to scope config - gitlab (#5313)
4b6820f89 is described below
commit 4b6820f89cb2c56db37c8c835a2e27e4a7b2676f
Author: Klesh Wong <zh...@merico.dev>
AuthorDate: Wed May 31 12:07:10 2023 +0800
refactor: rename transformation rule to scope config - gitlab (#5313)
---
backend/plugins/gitlab/api/blueprint.go | 217 ---------------------
backend/plugins/gitlab/api/blueprint_V200_test.go | 48 ++---
backend/plugins/gitlab/api/blueprint_test.go | 136 -------------
backend/plugins/gitlab/api/blueprint_v200.go | 47 +++--
backend/plugins/gitlab/api/connection.go | 22 ---
backend/plugins/gitlab/api/init.go | 10 +-
backend/plugins/gitlab/api/scope.go | 2 +-
backend/plugins/gitlab/api/scope_config.go | 83 ++++++++
backend/plugins/gitlab/api/swagger.go | 42 +---
backend/plugins/gitlab/api/transformation_rule.go | 83 --------
backend/plugins/gitlab/e2e/account_test.go | 9 +-
backend/plugins/gitlab/e2e/issues_test.go | 9 +-
backend/plugins/gitlab/e2e/mr_commits_test.go | 9 +-
backend/plugins/gitlab/e2e/mr_detail_test .go | 6 +-
backend/plugins/gitlab/e2e/mr_enrich_test.go | 6 +-
backend/plugins/gitlab/e2e/mr_notes_test.go | 9 +-
backend/plugins/gitlab/e2e/mr_test.go | 6 +-
.../plugins/gitlab/e2e/pipelines_detail_test.go | 6 +-
backend/plugins/gitlab/e2e/pipelines_test.go | 6 +-
backend/plugins/gitlab/e2e/project_test.go | 9 +-
backend/plugins/gitlab/gitlab.go | 2 +-
backend/plugins/gitlab/impl/impl.go | 37 ++--
...22_add_connection_id_to_transformation_rules.go | 23 ++-
.../migrationscripts/20230529_scope_config.go | 56 ++++++
.../gitlab/models/migrationscripts/register.go | 1 +
backend/plugins/gitlab/models/project.go | 2 +-
.../{transformation_rule.go => scope_config.go} | 8 +-
backend/plugins/gitlab/tasks/issue_extractor.go | 5 +-
.../plugins/gitlab/tasks/mr_detail_extractor.go | 2 +-
backend/plugins/gitlab/tasks/mr_extractor.go | 2 +-
backend/plugins/gitlab/tasks/task_data.go | 15 +-
31 files changed, 297 insertions(+), 621 deletions(-)
diff --git a/backend/plugins/gitlab/api/blueprint.go b/backend/plugins/gitlab/api/blueprint.go
deleted file mode 100644
index ffa3fc600..000000000
--- a/backend/plugins/gitlab/api/blueprint.go
+++ /dev/null
@@ -1,217 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package api
-
-import (
- "context"
- "encoding/json"
- "fmt"
- "io"
- "net/http"
- "net/url"
-
- "github.com/apache/incubator-devlake/core/errors"
- "github.com/apache/incubator-devlake/core/models/domainlayer/didgen"
- "github.com/apache/incubator-devlake/core/plugin"
- "github.com/apache/incubator-devlake/core/utils"
- "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
- aha "github.com/apache/incubator-devlake/helpers/pluginhelper/api/apihelperabstract"
- "github.com/apache/incubator-devlake/plugins/gitlab/models"
- "github.com/apache/incubator-devlake/plugins/gitlab/tasks"
-)
-
-func MakePipelinePlan(subtaskMetas []plugin.SubTaskMeta, connectionId uint64, scope []*plugin.BlueprintScopeV100) (plugin.PipelinePlan, errors.Error) {
- var err errors.Error
- connection := new(models.GitlabConnection)
- err = connectionHelper.FirstById(connection, connectionId)
- if err != nil {
- return nil, err
- }
-
- apiClient, err := api.NewApiClientFromConnection(
- context.TODO(),
- basicRes,
- connection,
- )
- if err != nil {
- return nil, err
- }
-
- plan, err := makePipelinePlan(subtaskMetas, scope, apiClient, connection)
- if err != nil {
- return nil, err
- }
- return plan, nil
-}
-
-func makePipelinePlan(
- subtaskMetas []plugin.SubTaskMeta,
- scope []*plugin.BlueprintScopeV100,
- apiClient aha.ApiClientAbstract,
- connection *models.GitlabConnection,
-) (plugin.PipelinePlan, errors.Error) {
- var err errors.Error
- var repo *tasks.GitlabApiProject
- plan := make(plugin.PipelinePlan, len(scope))
- for i, scopeElem := range scope {
- // handle taskOptions and transformationRules, by dumping them to taskOptions
- transformationRules := make(map[string]interface{})
- if len(scopeElem.Transformation) > 0 {
- err = errors.Convert(json.Unmarshal(scopeElem.Transformation, &transformationRules))
- if err != nil {
- return nil, err
- }
- }
- // construct task options for github
- options := make(map[string]interface{})
- err = errors.Convert(json.Unmarshal(scopeElem.Options, &options))
- if err != nil {
- return nil, err
- }
- options["connectionId"] = connection.ID
- options["transformationRules"] = transformationRules
- // make sure task options is valid
- op, err := tasks.DecodeAndValidateTaskOptions(options)
- if err != nil {
- return nil, err
- }
-
- memorizedGetApiRepo := func() (*tasks.GitlabApiProject, errors.Error) {
- if repo == nil {
- repo, err = getApiRepo(op, apiClient)
- }
- return repo, err
- }
-
- // refdiff
- if refdiffRules, ok := transformationRules["refdiff"]; ok && refdiffRules != nil {
- // add a new task to next stage
- j := i + 1
- if j == len(plan) {
- plan = append(plan, nil)
- }
- repo, err = memorizedGetApiRepo()
- if err != nil {
- return nil, err
- }
- ops := refdiffRules.(map[string]interface{})
- ops["repoId"] = didgen.NewDomainIdGenerator(&models.GitlabProject{}).Generate(connection.ID, repo.GitlabId)
- plan[j] = plugin.PipelineStage{
- {
- Plugin: "refdiff",
- Options: ops,
- },
- }
- // remove it from github transformationRules
- delete(transformationRules, "refdiff")
- }
-
- // construct subtasks
- subtasks, err := api.MakePipelinePlanSubtasks(subtaskMetas, scopeElem.Entities)
- if err != nil {
- return nil, err
- }
- stage := plan[i]
- if stage == nil {
- stage = plugin.PipelineStage{}
- }
- stage = append(stage, &plugin.PipelineTask{
- Plugin: "gitlab",
- Subtasks: subtasks,
- Options: options,
- })
- // collect git data by gitextractor if CODE was requested
- if utils.StringsContains(scopeElem.Entities, plugin.DOMAIN_TYPE_CODE) {
- // here is the tricky part, we have to obtain the repo id beforehand
- repo, err = memorizedGetApiRepo()
- if err != nil {
- return nil, err
- }
- cloneUrl, err := errors.Convert01(url.Parse(repo.HttpUrlToRepo))
- if err != nil {
- return nil, err
- }
- cloneUrl.User = url.UserPassword("git", connection.Token)
- stage = append(stage, &plugin.PipelineTask{
- Plugin: "gitextractor",
- Options: map[string]interface{}{
- "url": cloneUrl.String(),
- "repoId": didgen.NewDomainIdGenerator(&models.GitlabProject{}).Generate(connection.ID, repo.GitlabId),
- "proxy": connection.Proxy,
- },
- })
- }
- // This is just to add a dora subtask, then we can add another two subtasks at the end of plans
- // The only purpose is to adapt old blueprints
- // DEPRECATED, will be removed in v0.17
- // dora
- if productionPattern, ok := transformationRules["productionPattern"]; ok && productionPattern != nil {
- j := i + 1
- // add a new task to next stage
- if j == len(plan) {
- plan = append(plan, nil)
- }
- if plan[j] != nil {
- j++
- }
- if j == len(plan) {
- plan = append(plan, nil)
- }
- repo, err = memorizedGetApiRepo()
- if err != nil {
- return nil, err
- }
- plan[j] = plugin.PipelineStage{
- {
- Plugin: "dora",
- // This is just to add a dora subtask, then we can add another two subtasks at the end of plans
- // The only purpose is to adapt old blueprints
- Subtasks: []string{"EnrichTaskEnv"},
- Options: map[string]interface{}{},
- },
- }
- }
- plan[i] = stage
- repo = nil
- }
- return plan, nil
-}
-
-func getApiRepo(
- op *tasks.GitlabOptions,
- apiClient aha.ApiClientAbstract,
-) (*tasks.GitlabApiProject, errors.Error) {
- apiRepo := &tasks.GitlabApiProject{}
- res, err := apiClient.Get(fmt.Sprintf("projects/%d", op.ProjectId), nil, nil)
- if err != nil {
- return nil, err
- }
- defer res.Body.Close()
- if res.StatusCode != http.StatusOK {
- return nil, errors.HttpStatus(res.StatusCode).New(fmt.Sprintf("unexpected status code when requesting repo detail from %s", res.Request.URL.String()))
- }
- body, err := errors.Convert01(io.ReadAll(res.Body))
- if err != nil {
- return nil, err
- }
- err = errors.Convert(json.Unmarshal(body, apiRepo))
- if err != nil {
- return nil, err
- }
- return apiRepo, nil
-}
diff --git a/backend/plugins/gitlab/api/blueprint_V200_test.go b/backend/plugins/gitlab/api/blueprint_V200_test.go
index 0c0720fe2..c0534a87b 100644
--- a/backend/plugins/gitlab/api/blueprint_V200_test.go
+++ b/backend/plugins/gitlab/api/blueprint_V200_test.go
@@ -42,21 +42,20 @@ import (
func TestMakeDataSourcePipelinePlanV200(t *testing.T) {
const testConnectionID uint64 = 1
- const testTransformationRuleId uint64 = 2
+ const testScopeConfigId uint64 = 2
const testID int = 37
const testGitlabEndPoint string = "https://gitlab.com/api/v4/"
const testHttpUrlToRepo string = "https://this_is_cloneUrl"
const testToken string = "nddtf"
const testName string = "gitlab-test"
- const testTransformationRuleName string = "github transformation rule"
+ const testScopeConfigName string = "gitlab scope config"
const testProxy string = ""
syncPolicy := &plugin.BlueprintSyncPolicy{}
bpScopes := []*plugin.BlueprintScopeV200{
{
- Entities: []string{plugin.DOMAIN_TYPE_CODE, plugin.DOMAIN_TYPE_TICKET, plugin.DOMAIN_TYPE_CICD},
- Id: strconv.Itoa(testID),
- Name: testName,
+ Id: strconv.Itoa(testID),
+ Name: testName,
},
}
@@ -65,16 +64,19 @@ func TestMakeDataSourcePipelinePlanV200(t *testing.T) {
GitlabId: testID,
Name: testName,
- TransformationRuleId: testTransformationRuleId,
- CreatedDate: &time.Time{},
- HttpUrlToRepo: testHttpUrlToRepo,
+ ScopeConfigId: testScopeConfigId,
+ CreatedDate: &time.Time{},
+ HttpUrlToRepo: testHttpUrlToRepo,
}
- var testTransformationRule = &models.GitlabTransformationRule{
- Model: common.Model{
- ID: testTransformationRuleId,
+ var testScopeConfig = &models.GitlabScopeConfig{
+ ScopeConfig: common.ScopeConfig{
+ Model: common.Model{
+ ID: testScopeConfigId,
+ },
+ Entities: []string{plugin.DOMAIN_TYPE_CODE, plugin.DOMAIN_TYPE_TICKET, plugin.DOMAIN_TYPE_CICD},
},
- Name: testTransformationRuleName,
+ Name: testScopeConfigName,
PrType: "hey,man,wasup",
Refdiff: map[string]interface{}{
"tagsPattern": "pattern",
@@ -132,9 +134,9 @@ func TestMakeDataSourcePipelinePlanV200(t *testing.T) {
tasks.ExtractApiPipelinesMeta.Name,
},
Options: map[string]interface{}{
- "connectionId": uint64(1),
- "projectId": testID,
- "transformationRuleId": testTransformationRuleId,
+ "connectionId": uint64(1),
+ "projectId": testID,
+ "scopeConfigId": testScopeConfigId,
},
},
{
@@ -180,20 +182,20 @@ func TestMakeDataSourcePipelinePlanV200(t *testing.T) {
// Refresh Global Variables and set the sql mock
basicRes = unithelper.DummyBasicRes(func(mockDal *mockdal.Dal) {
- mockDal.On("First", mock.Anything, mock.Anything).Run(func(args mock.Arguments) {
+ mockDal.On("First", mock.AnythingOfType("*models.GitlabConnection"), mock.Anything).Run(func(args mock.Arguments) {
dst := args.Get(0).(*models.GitlabConnection)
*dst = *testGitlabConnection
- }).Return(nil).Once()
+ }).Return(nil)
- mockDal.On("First", mock.Anything, mock.Anything).Run(func(args mock.Arguments) {
+ mockDal.On("First", mock.AnythingOfType("*models.GitlabProject"), mock.Anything).Run(func(args mock.Arguments) {
dst := args.Get(0).(*models.GitlabProject)
*dst = *testGitlabProject
- }).Return(nil).Twice()
+ }).Return(nil)
- mockDal.On("First", mock.Anything, mock.Anything).Run(func(args mock.Arguments) {
- dst := args.Get(0).(*models.GitlabTransformationRule)
- *dst = *testTransformationRule
- }).Return(nil).Once()
+ mockDal.On("First", mock.AnythingOfType("*models.GitlabScopeConfig"), mock.Anything).Run(func(args mock.Arguments) {
+ dst := args.Get(0).(*models.GitlabScopeConfig)
+ *dst = *testScopeConfig
+ }).Return(nil)
})
connectionHelper = helper.NewConnectionHelper(
basicRes,
diff --git a/backend/plugins/gitlab/api/blueprint_test.go b/backend/plugins/gitlab/api/blueprint_test.go
deleted file mode 100644
index 1d7ed10c9..000000000
--- a/backend/plugins/gitlab/api/blueprint_test.go
+++ /dev/null
@@ -1,136 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package api
-
-import (
- "bytes"
- "encoding/json"
- "io"
- "net/http"
- "testing"
-
- "github.com/apache/incubator-devlake/core/models/common"
- "github.com/apache/incubator-devlake/core/plugin"
- helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
- mockplugin "github.com/apache/incubator-devlake/mocks/core/plugin"
- mockaha "github.com/apache/incubator-devlake/mocks/helpers/pluginhelper/api/apihelperabstract"
- "github.com/apache/incubator-devlake/plugins/gitlab/models"
- "github.com/apache/incubator-devlake/plugins/gitlab/tasks"
- "github.com/stretchr/testify/assert"
- "github.com/stretchr/testify/mock"
-)
-
-func TestProcessScope(t *testing.T) {
- connection := &models.GitlabConnection{
- BaseConnection: helper.BaseConnection{
- Name: "gitlab-test",
- Model: common.Model{
- ID: 1,
- },
- },
- GitlabConn: models.GitlabConn{
- RestConnection: helper.RestConnection{
- Endpoint: "https://gitlab.com/api/v4/",
- Proxy: "",
- RateLimitPerHour: 0,
- },
- AccessToken: helper.AccessToken{
- Token: "123",
- },
- },
- }
- mockApiCLient := mockaha.NewApiClientAbstract(t)
- repo := &tasks.GitlabApiProject{
- GitlabId: 12345,
- HttpUrlToRepo: "https://this_is_HttpUrlToRepo",
- }
- js, err := json.Marshal(repo)
- assert.Nil(t, err)
- res := &http.Response{}
- res.Body = io.NopCloser(bytes.NewBuffer(js))
- res.StatusCode = http.StatusOK
- mockApiCLient.On("Get", "projects/12345", mock.Anything, mock.Anything).Return(res, nil)
-
- mockMeta := mockplugin.NewPluginMeta(t)
- mockMeta.On("RootPkgPath").Return("github.com/apache/incubator-devlake/plugins/gitlab")
- err = plugin.RegisterPlugin("gitlab", mockMeta)
- assert.Nil(t, err)
- bs := &plugin.BlueprintScopeV100{
- Entities: []string{"CODE"},
- Options: json.RawMessage(`{
- "projectId": 12345
- }`),
- Transformation: json.RawMessage(`{
- "prType": "hey,man,wasup",
- "refdiff": {
- "tagsPattern": "pattern",
- "tagsLimit": 10,
- "tagsOrder": "reverse semver"
- },
- "productionPattern": "xxxx"
- }`),
- }
- scopes := make([]*plugin.BlueprintScopeV100, 0)
- scopes = append(scopes, bs)
- plan, err := makePipelinePlan(nil, scopes, mockApiCLient, connection)
- assert.Nil(t, err)
-
- expectPlan := plugin.PipelinePlan{
- plugin.PipelineStage{
- {
- Plugin: "gitlab",
- Subtasks: []string{},
- Options: map[string]interface{}{
- "connectionId": uint64(1),
- "projectId": float64(12345),
- "transformationRules": map[string]interface{}{
- "prType": "hey,man,wasup",
- "productionPattern": "xxxx",
- },
- },
- },
- {
- Plugin: "gitextractor",
- Options: map[string]interface{}{
- "proxy": "",
- "repoId": "gitlab:GitlabProject:1:12345",
- "url": "https://git:123@this_is_HttpUrlToRepo",
- },
- },
- },
- plugin.PipelineStage{
- {
- Plugin: "refdiff",
- Options: map[string]interface{}{
- "repoId": "gitlab:GitlabProject:1:12345",
- "tagsLimit": float64(10),
- "tagsOrder": "reverse semver",
- "tagsPattern": "pattern",
- },
- },
- },
- plugin.PipelineStage{
- {
- Plugin: "dora",
- Subtasks: []string{"EnrichTaskEnv"},
- Options: map[string]interface{}{},
- },
- },
- }
- assert.Equal(t, expectPlan, plan)
-}
diff --git a/backend/plugins/gitlab/api/blueprint_v200.go b/backend/plugins/gitlab/api/blueprint_v200.go
index 6950ae772..4a493d78a 100644
--- a/backend/plugins/gitlab/api/blueprint_v200.go
+++ b/backend/plugins/gitlab/api/blueprint_v200.go
@@ -84,8 +84,13 @@ func makeScopeV200(connectionId uint64, scopes []*plugin.BlueprintScopeV200) ([]
return nil, err
}
- if utils.StringsContains(scope.Entities, plugin.DOMAIN_TYPE_CODE_REVIEW) ||
- utils.StringsContains(scope.Entities, plugin.DOMAIN_TYPE_CODE) {
+ scopeConfig, err := GetScopeConfigByRepo(gitlabProject)
+ if err != nil {
+ return nil, err
+ }
+
+ if utils.StringsContains(scopeConfig.Entities, plugin.DOMAIN_TYPE_CODE_REVIEW) ||
+ utils.StringsContains(scopeConfig.Entities, plugin.DOMAIN_TYPE_CODE) {
// if we don't need to collect gitex, we need to add repo to scopes here
scopeRepo := code.NewRepo(id, gitlabProject.Name)
@@ -96,14 +101,14 @@ func makeScopeV200(connectionId uint64, scopes []*plugin.BlueprintScopeV200) ([]
}
// add cicd_scope to scopes
- if utils.StringsContains(scope.Entities, plugin.DOMAIN_TYPE_CICD) {
+ if utils.StringsContains(scopeConfig.Entities, plugin.DOMAIN_TYPE_CICD) {
scopeCICD := devops.NewCicdScope(id, gitlabProject.Name)
sc = append(sc, scopeCICD)
}
// add board to scopes
- if utils.StringsContains(scope.Entities, plugin.DOMAIN_TYPE_TICKET) {
+ if utils.StringsContains(scopeConfig.Entities, plugin.DOMAIN_TYPE_TICKET) {
scopeTicket := ticket.NewBoard(id, gitlabProject.Name)
sc = append(sc, scopeTicket)
@@ -128,8 +133,8 @@ func makePipelinePlanV200(
return nil, err
}
- // get transformationRuleId
- transformationRules, err := GetTransformationRuleByRepo(repo)
+ // get scopeConfigId
+ scopeConfig, err := GetScopeConfigByRepo(repo)
if err != nil {
return nil, err
}
@@ -144,13 +149,13 @@ func makePipelinePlanV200(
options := make(map[string]interface{})
options["connectionId"] = connection.ID
options["projectId"] = intScopeId
- options["transformationRuleId"] = transformationRules.ID
+ options["scopeConfigId"] = scopeConfig.ID
if syncPolicy.TimeAfter != nil {
options["timeAfter"] = syncPolicy.TimeAfter.Format(time.RFC3339)
}
// construct subtasks
- subtasks, err := helper.MakePipelinePlanSubtasks(subtaskMetas, scope.Entities)
+ subtasks, err := helper.MakePipelinePlanSubtasks(subtaskMetas, scopeConfig.Entities)
if err != nil {
return nil, err
}
@@ -162,7 +167,7 @@ func makePipelinePlanV200(
})
// collect git data by gitextractor if CODE was requested
- if utils.StringsContains(scope.Entities, plugin.DOMAIN_TYPE_CODE) {
+ if utils.StringsContains(scopeConfig.Entities, plugin.DOMAIN_TYPE_CODE) {
cloneUrl, err := errors.Convert01(url.Parse(repo.HttpUrlToRepo))
if err != nil {
return nil, err
@@ -181,10 +186,10 @@ func makePipelinePlanV200(
plans = append(plans, stage)
// refdiff part
- if transformationRules.Refdiff != nil {
+ if scopeConfig.Refdiff != nil {
task := &plugin.PipelineTask{
Plugin: "refdiff",
- Options: transformationRules.Refdiff,
+ Options: scopeConfig.Refdiff,
}
plans = append(plans, plugin.PipelineStage{task})
}
@@ -211,24 +216,24 @@ func GetRepoByConnectionIdAndscopeId(connectionId uint64, scopeId string) (*mode
return repo, nil
}
-// GetTransformationRuleByRepo get the GetTransformationRule by Repo
-func GetTransformationRuleByRepo(repo *models.GitlabProject) (*models.GitlabTransformationRule, errors.Error) {
- transformationRules := &models.GitlabTransformationRule{}
- transformationRuleId := repo.TransformationRuleId
- if transformationRuleId != 0 {
+// GetScopeConfigByRepo get the GetScopeConfig by Repo
+func GetScopeConfigByRepo(repo *models.GitlabProject) (*models.GitlabScopeConfig, errors.Error) {
+ scopeConfigs := &models.GitlabScopeConfig{}
+ scopeConfigId := repo.ScopeConfigId
+ if scopeConfigId != 0 {
db := basicRes.GetDal()
- err := db.First(transformationRules, dal.Where("id = ?", transformationRuleId))
+ err := db.First(scopeConfigs, dal.Where("id = ?", scopeConfigId))
if err != nil {
if db.IsErrorNotFound(err) {
- return nil, errors.Default.Wrap(err, fmt.Sprintf("can not find transformationRules by transformationRuleId [%d]", transformationRuleId))
+ return nil, errors.Default.Wrap(err, fmt.Sprintf("can not find scopeConfig by id [%d]", scopeConfigId))
}
- return nil, errors.Default.Wrap(err, fmt.Sprintf("fail to find transformationRules by transformationRuleId [%d]", transformationRuleId))
+ return nil, errors.Default.Wrap(err, fmt.Sprintf("fail to find scopeConfig by id [%d]", scopeConfigId))
}
} else {
- transformationRules.ID = 0
+ scopeConfigs.ID = 0
}
- return transformationRules, nil
+ return scopeConfigs, nil
}
func GetApiProject(
diff --git a/backend/plugins/gitlab/api/connection.go b/backend/plugins/gitlab/api/connection.go
index 48685b4f5..d23d9dc3e 100644
--- a/backend/plugins/gitlab/api/connection.go
+++ b/backend/plugins/gitlab/api/connection.go
@@ -164,25 +164,3 @@ func GetConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, e
}
return &plugin.ApiResourceOutput{Body: connection}, err
}
-
-// @Summary pipelines plan for gitlab
-// @Description pipelines plan for gitlab
-// @Tags plugins/gitlab
-// @Accept application/json
-// @Param blueprint body GitlabPipelinePlan true "json"
-// @Router /pipelines/gitlab/pipeline-plan [post]
-func PostGitlabPipeline(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
- blueprint := &GitlabPipelinePlan{}
- return &plugin.ApiResourceOutput{Body: blueprint, Status: http.StatusOK}, nil
-}
-
-type GitlabPipelinePlan [][]struct {
- Plugin string `json:"plugin"`
- Subtasks []string `json:"subtasks"`
- Options struct {
- ConnectionID int `json:"connectionId"`
- ProjectId int `json:"projectId"`
- Since string
- Transformation models.GitlabTransformationRule `json:"transformation"`
- } `json:"options"`
-}
diff --git a/backend/plugins/gitlab/api/init.go b/backend/plugins/gitlab/api/init.go
index cbc336966..302cc5198 100644
--- a/backend/plugins/gitlab/api/init.go
+++ b/backend/plugins/gitlab/api/init.go
@@ -26,10 +26,10 @@ import (
var vld *validator.Validate
var connectionHelper *api.ConnectionApiHelper
-var scopeHelper *api.ScopeApiHelper[models.GitlabConnection, models.GitlabProject, models.GitlabTransformationRule]
+var scopeHelper *api.ScopeApiHelper[models.GitlabConnection, models.GitlabProject, models.GitlabScopeConfig]
var remoteHelper *api.RemoteApiHelper[models.GitlabConnection, models.GitlabProject, models.GitlabApiProject, models.GroupResponse]
var basicRes context.BasicRes
-var trHelper *api.TransformationRuleHelper[models.GitlabTransformationRule]
+var scHelper *api.ScopeConfigHelper[models.GitlabScopeConfig]
func Init(br context.BasicRes) {
basicRes = br
@@ -42,11 +42,11 @@ func Init(br context.BasicRes) {
ScopeIdFieldName: "GitlabId",
ScopeIdColumnName: "gitlab_id",
}
- scopeHelper = api.NewScopeHelper[models.GitlabConnection, models.GitlabProject, models.GitlabTransformationRule](
+ scopeHelper = api.NewScopeHelper[models.GitlabConnection, models.GitlabProject, models.GitlabScopeConfig](
basicRes,
vld,
connectionHelper,
- api.NewScopeDatabaseHelperImpl[models.GitlabConnection, models.GitlabProject, models.GitlabTransformationRule](
+ api.NewScopeDatabaseHelperImpl[models.GitlabConnection, models.GitlabProject, models.GitlabScopeConfig](
basicRes, connectionHelper, params),
params,
nil,
@@ -56,7 +56,7 @@ func Init(br context.BasicRes) {
vld,
connectionHelper,
)
- trHelper = api.NewTransformationRuleHelper[models.GitlabTransformationRule](
+ scHelper = api.NewScopeConfigHelper[models.GitlabScopeConfig](
basicRes,
vld,
)
diff --git a/backend/plugins/gitlab/api/scope.go b/backend/plugins/gitlab/api/scope.go
index 32927e559..27abf3b2c 100644
--- a/backend/plugins/gitlab/api/scope.go
+++ b/backend/plugins/gitlab/api/scope.go
@@ -26,7 +26,7 @@ import (
type ScopeRes struct {
models.GitlabProject
- TransformationRuleName string `json:"transformationRuleName,omitempty"`
+ ScopeConfigName string `json:"scopeConfigName,omitempty"`
}
type ScopeReq api.ScopeReq[models.GitlabProject]
diff --git a/backend/plugins/gitlab/api/scope_config.go b/backend/plugins/gitlab/api/scope_config.go
new file mode 100644
index 000000000..9ecf74196
--- /dev/null
+++ b/backend/plugins/gitlab/api/scope_config.go
@@ -0,0 +1,83 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package api
+
+import (
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
+)
+
+// CreateScopeConfig create scope config for Gitlab
+// @Summary create scope config for Gitlab
+// @Description create scope config for Gitlab
+// @Tags plugins/gitlab
+// @Accept application/json
+// @Param connectionId path int true "connectionId"
+// @Param scopeConfig body models.GitlabScopeConfig true "scope config"
+// @Success 200 {object} models.GitlabScopeConfig
+// @Failure 400 {object} shared.ApiBody "Bad Request"
+// @Failure 500 {object} shared.ApiBody "Internal Error"
+// @Router /plugins/gitlab/connections/{connectionId}/scope_configs [POST]
+func CreateScopeConfig(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
+ return scHelper.Create(input)
+}
+
+// UpdateScopeConfig update scope config for Gitlab
+// @Summary update scope config for Gitlab
+// @Description update scope config for Gitlab
+// @Tags plugins/gitlab
+// @Accept application/json
+// @Param id path int true "id"
+// @Param connectionId path int true "connectionId"
+// @Param scopeConfig body models.GitlabScopeConfig true "scope config"
+// @Success 200 {object} models.GitlabScopeConfig
+// @Failure 400 {object} shared.ApiBody "Bad Request"
+// @Failure 500 {object} shared.ApiBody "Internal Error"
+// @Router /plugins/gitlab/connections/{connectionId}/scope_configs/{id} [PATCH]
+func UpdateScopeConfig(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
+ return scHelper.Update(input)
+}
+
+// GetScopeConfig return one scope config
+// @Summary return one scope config
+// @Description return one scope config
+// @Tags plugins/gitlab
+// @Param id path int true "id"
+// @Param connectionId path int true "connectionId"
+// @Success 200 {object} models.GitlabScopeConfig
+// @Failure 400 {object} shared.ApiBody "Bad Request"
+// @Failure 500 {object} shared.ApiBody "Internal Error"
+// @Router /plugins/gitlab/connections/{connectionId}/scope_configs/{id} [GET]
+func GetScopeConfig(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
+ return scHelper.Get(input)
+}
+
+// GetScopeConfigList return all scope configs
+// @Summary return all scope configs
+// @Description return all scope configs
+// @Tags plugins/gitlab
+// @Param connectionId path int true "connectionId"
+// @Param pageSize query int false "page size, default 50"
+// @Param page query int false "page size, default 1"
+// @Success 200 {object} []models.GitlabScopeConfig
+// @Failure 400 {object} shared.ApiBody "Bad Request"
+// @Failure 500 {object} shared.ApiBody "Internal Error"
+// @Router /plugins/gitlab/connections/{connectionId}/scope_configs [GET]
+func GetScopeConfigList(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
+ return scHelper.List(input)
+}
diff --git a/backend/plugins/gitlab/api/swagger.go b/backend/plugins/gitlab/api/swagger.go
index d62ee36bc..73b146ca8 100644
--- a/backend/plugins/gitlab/api/swagger.go
+++ b/backend/plugins/gitlab/api/swagger.go
@@ -17,38 +17,16 @@ limitations under the License.
package api
-// @Summary blueprints setting for gitlab
-// @Description blueprint setting for gitlab
+import (
+ "github.com/apache/incubator-devlake/plugins/gitlab/tasks"
+)
+
+type GitlabTaskOptions tasks.GitlabOptions
+
+// @Summary gitlab task options for pipelines
+// @Description This is a dummy API to demonstrate the available task options for gitlab pipelines
// @Tags plugins/gitlab
// @Accept application/json
-// @Param blueprint body GitlabBlueprintSetting true "json"
-// @Router /blueprints/gitlab/blueprint-setting [post]
+// @Param pipeline body GitlabTaskOptions true "json"
+// @Router /pipelines/gitlab/pipeline-task [post]
func _() {}
-
-type GitlabBlueprintSetting []struct {
- Version string `json:"version"`
- Connections []struct {
- Plugin string `json:"plugin"`
- ConnectionID int `json:"connectionId"`
- Scope []struct {
- Transformation CodeTransformationRules `json:"transformation"`
- Options struct {
- ProjectId int `json:"projectId"`
- Since string
- } `json:"options"`
- Entities []string `json:"entities"`
- } `json:"scopes"`
- } `json:"connections"`
-}
-
-type CodeTransformationRules struct {
- PrType string `mapstructure:"prType" json:"prType"`
- PrComponent string `mapstructure:"prComponent" json:"prComponent"`
- PrBodyClosePattern string `mapstructure:"prBodyClosePattern" json:"prBodyClosePattern"`
- IssueSeverity string `mapstructure:"issueSeverity" json:"issueSeverity"`
- IssuePriority string `mapstructure:"issuePriority" json:"issuePriority"`
- IssueComponent string `mapstructure:"issueComponent" json:"issueComponent"`
- IssueTypeBug string `mapstructure:"issueTypeBug" json:"issueTypeBug"`
- IssueTypeIncident string `mapstructure:"issueTypeIncident" json:"issueTypeIncident"`
- IssueTypeRequirement string `mapstructure:"issueTypeRequirement" json:"issueTypeRequirement"`
-}
diff --git a/backend/plugins/gitlab/api/transformation_rule.go b/backend/plugins/gitlab/api/transformation_rule.go
deleted file mode 100644
index 73de6c296..000000000
--- a/backend/plugins/gitlab/api/transformation_rule.go
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package api
-
-import (
- "github.com/apache/incubator-devlake/core/errors"
- "github.com/apache/incubator-devlake/core/plugin"
-)
-
-// CreateTransformationRule create transformation rule for Gitlab
-// @Summary create transformation rule for Gitlab
-// @Description create transformation rule for Gitlab
-// @Tags plugins/gitlab
-// @Accept application/json
-// @Param connectionId path int true "connectionId"
-// @Param transformationRule body models.GitlabTransformationRule true "transformation rule"
-// @Success 200 {object} models.GitlabTransformationRule
-// @Failure 400 {object} shared.ApiBody "Bad Request"
-// @Failure 500 {object} shared.ApiBody "Internal Error"
-// @Router /plugins/gitlab/connections/{connectionId}/transformation_rules [POST]
-func CreateTransformationRule(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
- return trHelper.Create(input)
-}
-
-// UpdateTransformationRule update transformation rule for Gitlab
-// @Summary update transformation rule for Gitlab
-// @Description update transformation rule for Gitlab
-// @Tags plugins/gitlab
-// @Accept application/json
-// @Param id path int true "id"
-// @Param connectionId path int true "connectionId"
-// @Param transformationRule body models.GitlabTransformationRule true "transformation rule"
-// @Success 200 {object} models.GitlabTransformationRule
-// @Failure 400 {object} shared.ApiBody "Bad Request"
-// @Failure 500 {object} shared.ApiBody "Internal Error"
-// @Router /plugins/gitlab/connections/{connectionId}/transformation_rules/{id} [PATCH]
-func UpdateTransformationRule(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
- return trHelper.Update(input)
-}
-
-// GetTransformationRule return one transformation rule
-// @Summary return one transformation rule
-// @Description return one transformation rule
-// @Tags plugins/gitlab
-// @Param id path int true "id"
-// @Param connectionId path int true "connectionId"
-// @Success 200 {object} models.GitlabTransformationRule
-// @Failure 400 {object} shared.ApiBody "Bad Request"
-// @Failure 500 {object} shared.ApiBody "Internal Error"
-// @Router /plugins/gitlab/connections/{connectionId}/transformation_rules/{id} [GET]
-func GetTransformationRule(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
- return trHelper.Get(input)
-}
-
-// GetTransformationRuleList return all transformation rules
-// @Summary return all transformation rules
-// @Description return all transformation rules
-// @Tags plugins/gitlab
-// @Param connectionId path int true "connectionId"
-// @Param pageSize query int false "page size, default 50"
-// @Param page query int false "page size, default 1"
-// @Success 200 {object} []models.GitlabTransformationRule
-// @Failure 400 {object} shared.ApiBody "Bad Request"
-// @Failure 500 {object} shared.ApiBody "Internal Error"
-// @Router /plugins/gitlab/connections/{connectionId}/transformation_rules [GET]
-func GetTransformationRuleList(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
- return trHelper.List(input)
-}
diff --git a/backend/plugins/gitlab/e2e/account_test.go b/backend/plugins/gitlab/e2e/account_test.go
index 7743e19cd..8b1f48b79 100644
--- a/backend/plugins/gitlab/e2e/account_test.go
+++ b/backend/plugins/gitlab/e2e/account_test.go
@@ -18,12 +18,13 @@ limitations under the License.
package e2e
import (
+ "testing"
+
"github.com/apache/incubator-devlake/core/models/domainlayer/crossdomain"
"github.com/apache/incubator-devlake/helpers/e2ehelper"
"github.com/apache/incubator-devlake/plugins/gitlab/impl"
"github.com/apache/incubator-devlake/plugins/gitlab/models"
"github.com/apache/incubator-devlake/plugins/gitlab/tasks"
- "testing"
)
func TestGitlabAccountDataFlow(t *testing.T) {
@@ -33,9 +34,9 @@ func TestGitlabAccountDataFlow(t *testing.T) {
taskData := &tasks.GitlabTaskData{
Options: &tasks.GitlabOptions{
- ConnectionId: 1,
- ProjectId: 12345678,
- GitlabTransformationRule: new(models.GitlabTransformationRule),
+ ConnectionId: 1,
+ ProjectId: 12345678,
+ ScopeConfig: new(models.GitlabScopeConfig),
},
}
diff --git a/backend/plugins/gitlab/e2e/issues_test.go b/backend/plugins/gitlab/e2e/issues_test.go
index 374f21d9b..4f106742b 100644
--- a/backend/plugins/gitlab/e2e/issues_test.go
+++ b/backend/plugins/gitlab/e2e/issues_test.go
@@ -18,12 +18,13 @@ limitations under the License.
package e2e
import (
+ "testing"
+
"github.com/apache/incubator-devlake/core/models/domainlayer/ticket"
"github.com/apache/incubator-devlake/helpers/e2ehelper"
"github.com/apache/incubator-devlake/plugins/gitlab/impl"
"github.com/apache/incubator-devlake/plugins/gitlab/models"
"github.com/apache/incubator-devlake/plugins/gitlab/tasks"
- "testing"
)
func TestGitlabIssueDataFlow(t *testing.T) {
@@ -33,9 +34,9 @@ func TestGitlabIssueDataFlow(t *testing.T) {
taskData := &tasks.GitlabTaskData{
Options: &tasks.GitlabOptions{
- ConnectionId: 1,
- ProjectId: 12345678,
- GitlabTransformationRule: new(models.GitlabTransformationRule),
+ ConnectionId: 1,
+ ProjectId: 12345678,
+ ScopeConfig: new(models.GitlabScopeConfig),
},
}
// import raw data table
diff --git a/backend/plugins/gitlab/e2e/mr_commits_test.go b/backend/plugins/gitlab/e2e/mr_commits_test.go
index 521eb207e..897cf9c0d 100644
--- a/backend/plugins/gitlab/e2e/mr_commits_test.go
+++ b/backend/plugins/gitlab/e2e/mr_commits_test.go
@@ -18,13 +18,14 @@ limitations under the License.
package e2e
import (
+ "testing"
+
"github.com/apache/incubator-devlake/core/models/common"
"github.com/apache/incubator-devlake/core/models/domainlayer/code"
"github.com/apache/incubator-devlake/helpers/e2ehelper"
"github.com/apache/incubator-devlake/plugins/gitlab/impl"
"github.com/apache/incubator-devlake/plugins/gitlab/models"
"github.com/apache/incubator-devlake/plugins/gitlab/tasks"
- "testing"
)
func TestGitlabMrCommitDataFlow(t *testing.T) {
@@ -34,9 +35,9 @@ func TestGitlabMrCommitDataFlow(t *testing.T) {
taskData := &tasks.GitlabTaskData{
Options: &tasks.GitlabOptions{
- ConnectionId: 1,
- ProjectId: 12345678,
- GitlabTransformationRule: new(models.GitlabTransformationRule),
+ ConnectionId: 1,
+ ProjectId: 12345678,
+ ScopeConfig: new(models.GitlabScopeConfig),
},
}
diff --git a/backend/plugins/gitlab/e2e/mr_detail_test .go b/backend/plugins/gitlab/e2e/mr_detail_test .go
index 176cfe8c6..c92356676 100644
--- a/backend/plugins/gitlab/e2e/mr_detail_test .go
+++ b/backend/plugins/gitlab/e2e/mr_detail_test .go
@@ -35,9 +35,9 @@ func TestGitlabMrDetailDataFlow(t *testing.T) {
taskData := &tasks.GitlabTaskData{
Options: &tasks.GitlabOptions{
- ConnectionId: 1,
- ProjectId: 12345678,
- GitlabTransformationRule: new(models.GitlabTransformationRule),
+ ConnectionId: 1,
+ ProjectId: 12345678,
+ ScopeConfig: new(models.GitlabScopeConfig),
},
}
// import raw data table
diff --git a/backend/plugins/gitlab/e2e/mr_enrich_test.go b/backend/plugins/gitlab/e2e/mr_enrich_test.go
index a803ffd54..fd18003cc 100644
--- a/backend/plugins/gitlab/e2e/mr_enrich_test.go
+++ b/backend/plugins/gitlab/e2e/mr_enrich_test.go
@@ -33,9 +33,9 @@ func TestGitlabMrEnrichDataFlow(t *testing.T) {
taskData := &tasks.GitlabTaskData{
Options: &tasks.GitlabOptions{
- ConnectionId: 1,
- ProjectId: 12345678,
- GitlabTransformationRule: new(models.GitlabTransformationRule),
+ ConnectionId: 1,
+ ProjectId: 12345678,
+ ScopeConfig: new(models.GitlabScopeConfig),
},
}
diff --git a/backend/plugins/gitlab/e2e/mr_notes_test.go b/backend/plugins/gitlab/e2e/mr_notes_test.go
index 9058a69fb..13555a1c5 100644
--- a/backend/plugins/gitlab/e2e/mr_notes_test.go
+++ b/backend/plugins/gitlab/e2e/mr_notes_test.go
@@ -18,12 +18,13 @@ limitations under the License.
package e2e
import (
+ "testing"
+
"github.com/apache/incubator-devlake/core/models/domainlayer/code"
"github.com/apache/incubator-devlake/helpers/e2ehelper"
"github.com/apache/incubator-devlake/plugins/gitlab/impl"
"github.com/apache/incubator-devlake/plugins/gitlab/models"
"github.com/apache/incubator-devlake/plugins/gitlab/tasks"
- "testing"
)
func TestGitlabMrNoteDataFlow(t *testing.T) {
@@ -33,9 +34,9 @@ func TestGitlabMrNoteDataFlow(t *testing.T) {
taskData := &tasks.GitlabTaskData{
Options: &tasks.GitlabOptions{
- ConnectionId: 1,
- ProjectId: 12345678,
- GitlabTransformationRule: new(models.GitlabTransformationRule),
+ ConnectionId: 1,
+ ProjectId: 12345678,
+ ScopeConfig: new(models.GitlabScopeConfig),
},
}
// import raw data table
diff --git a/backend/plugins/gitlab/e2e/mr_test.go b/backend/plugins/gitlab/e2e/mr_test.go
index 5fe825fbe..e95e3de04 100644
--- a/backend/plugins/gitlab/e2e/mr_test.go
+++ b/backend/plugins/gitlab/e2e/mr_test.go
@@ -35,9 +35,9 @@ func TestGitlabMrDataFlow(t *testing.T) {
taskData := &tasks.GitlabTaskData{
Options: &tasks.GitlabOptions{
- ConnectionId: 1,
- ProjectId: 12345678,
- GitlabTransformationRule: new(models.GitlabTransformationRule),
+ ConnectionId: 1,
+ ProjectId: 12345678,
+ ScopeConfig: new(models.GitlabScopeConfig),
},
}
// import raw data table
diff --git a/backend/plugins/gitlab/e2e/pipelines_detail_test.go b/backend/plugins/gitlab/e2e/pipelines_detail_test.go
index 1488b76d4..9d1d68830 100644
--- a/backend/plugins/gitlab/e2e/pipelines_detail_test.go
+++ b/backend/plugins/gitlab/e2e/pipelines_detail_test.go
@@ -40,9 +40,9 @@ func TestGitlabPipelineDetailDataFlow(t *testing.T) {
taskData := &tasks.GitlabTaskData{
Options: &tasks.GitlabOptions{
- ConnectionId: 1,
- ProjectId: 12345678,
- GitlabTransformationRule: new(models.GitlabTransformationRule),
+ ConnectionId: 1,
+ ProjectId: 12345678,
+ ScopeConfig: new(models.GitlabScopeConfig),
},
RegexEnricher: regexEnricher,
}
diff --git a/backend/plugins/gitlab/e2e/pipelines_test.go b/backend/plugins/gitlab/e2e/pipelines_test.go
index 9c0bb8b70..620a11d5b 100644
--- a/backend/plugins/gitlab/e2e/pipelines_test.go
+++ b/backend/plugins/gitlab/e2e/pipelines_test.go
@@ -39,9 +39,9 @@ func TestGitlabPipelineDataFlow(t *testing.T) {
taskData := &tasks.GitlabTaskData{
Options: &tasks.GitlabOptions{
- ConnectionId: 1,
- ProjectId: 12345678,
- GitlabTransformationRule: new(models.GitlabTransformationRule),
+ ConnectionId: 1,
+ ProjectId: 12345678,
+ ScopeConfig: new(models.GitlabScopeConfig),
},
RegexEnricher: regexEnricher,
}
diff --git a/backend/plugins/gitlab/e2e/project_test.go b/backend/plugins/gitlab/e2e/project_test.go
index 2632537e2..e0c2a13e2 100644
--- a/backend/plugins/gitlab/e2e/project_test.go
+++ b/backend/plugins/gitlab/e2e/project_test.go
@@ -18,9 +18,10 @@ limitations under the License.
package e2e
import (
+ "testing"
+
"github.com/apache/incubator-devlake/core/models/common"
"github.com/apache/incubator-devlake/core/models/domainlayer/devops"
- "testing"
"github.com/apache/incubator-devlake/core/models/domainlayer/crossdomain"
"github.com/apache/incubator-devlake/core/models/domainlayer/ticket"
@@ -39,9 +40,9 @@ func TestGitlabProjectDataFlow(t *testing.T) {
taskData := &tasks.GitlabTaskData{
Options: &tasks.GitlabOptions{
- ConnectionId: 1,
- ProjectId: 12345678,
- GitlabTransformationRule: new(models.GitlabTransformationRule),
+ ConnectionId: 1,
+ ProjectId: 12345678,
+ ScopeConfig: new(models.GitlabScopeConfig),
},
}
diff --git a/backend/plugins/gitlab/gitlab.go b/backend/plugins/gitlab/gitlab.go
index c44faa4da..5a6d2701c 100644
--- a/backend/plugins/gitlab/gitlab.go
+++ b/backend/plugins/gitlab/gitlab.go
@@ -51,7 +51,7 @@ func main() {
"projectId": *projectId,
"connectionId": *connectionId,
"timeAfter": *timeAfter,
- "transformationRules": map[string]interface{}{
+ "scopeConfig": map[string]interface{}{
"prType": *prType,
"prComponent": *prComponent,
"prBodyClosePattern": *prBodyClosePattern,
diff --git a/backend/plugins/gitlab/impl/impl.go b/backend/plugins/gitlab/impl/impl.go
index 2966c5b28..4343b7a55 100644
--- a/backend/plugins/gitlab/impl/impl.go
+++ b/backend/plugins/gitlab/impl/impl.go
@@ -39,7 +39,6 @@ var _ interface {
plugin.PluginTask
plugin.PluginModel
plugin.PluginMigration
- plugin.PluginBlueprintV100
plugin.DataSourcePluginBlueprintV200
plugin.CloseablePluginTask
plugin.PluginSource
@@ -60,8 +59,8 @@ func (p Gitlab) Scope() interface{} {
return &models.GitlabProject{}
}
-func (p Gitlab) TransformationRule() interface{} {
- return &models.GitlabTransformationRule{}
+func (p Gitlab) ScopeConfig() interface{} {
+ return &models.GitlabScopeConfig{}
}
func (p Gitlab) MakeDataSourcePipelinePlanV200(connectionId uint64, scopes []*plugin.BlueprintScopeV200, syncPolicy plugin.BlueprintSyncPolicy) (plugin.PipelinePlan, []plugin.Scope, errors.Error) {
@@ -197,24 +196,24 @@ func (p Gitlab) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]i
}
}
- if op.GitlabTransformationRule == nil && op.TransformationRuleId != 0 {
- var transformationRule models.GitlabTransformationRule
+ if op.ScopeConfig == nil && op.ScopeConfigId != 0 {
+ var scopeConfig models.GitlabScopeConfig
db := taskCtx.GetDal()
- err = db.First(&transformationRule, dal.Where("id = ?", op.TransformationRuleId))
+ err = db.First(&scopeConfig, dal.Where("id = ?", op.ScopeConfigId))
if err != nil {
if db.IsErrorNotFound(err) {
- return nil, errors.Default.Wrap(err, fmt.Sprintf("can not find transformationRules by transformationRuleId [%d]", op.TransformationRuleId))
+ return nil, errors.Default.Wrap(err, fmt.Sprintf("can not find scopeConfigs by scopeConfigId [%d]", op.ScopeConfigId))
}
- return nil, errors.Default.Wrap(err, fmt.Sprintf("fail to find transformationRules by transformationRuleId [%d]", op.TransformationRuleId))
+ return nil, errors.Default.Wrap(err, fmt.Sprintf("fail to find scopeConfigs by scopeConfigId [%d]", op.ScopeConfigId))
}
- op.GitlabTransformationRule = &transformationRule
+ op.ScopeConfig = &scopeConfig
}
regexEnricher := helper.NewRegexEnricher()
- if err := regexEnricher.TryAdd(devops.DEPLOYMENT, op.DeploymentPattern); err != nil {
+ if err := regexEnricher.TryAdd(devops.DEPLOYMENT, op.ScopeConfig.DeploymentPattern); err != nil {
return nil, errors.BadInput.Wrap(err, "invalid value for `deploymentPattern`")
}
- if err := regexEnricher.TryAdd(devops.PRODUCTION, op.ProductionPattern); err != nil {
+ if err := regexEnricher.TryAdd(devops.PRODUCTION, op.ScopeConfig.ProductionPattern); err != nil {
return nil, errors.BadInput.Wrap(err, "invalid value for `productionPattern`")
}
@@ -239,10 +238,6 @@ func (p Gitlab) MigrationScripts() []plugin.MigrationScript {
return migrationscripts.All()
}
-func (p Gitlab) MakePipelinePlan(connectionId uint64, scope []*plugin.BlueprintScopeV100) (plugin.PipelinePlan, errors.Error) {
- return api.MakePipelinePlan(p.SubTaskMetas(), connectionId, scope)
-}
-
func (p Gitlab) ApiResources() map[string]map[string]plugin.ApiResourceHandler {
return map[string]map[string]plugin.ApiResourceHandler{
"test": {
@@ -272,13 +267,13 @@ func (p Gitlab) ApiResources() map[string]map[string]plugin.ApiResourceHandler {
"GET": api.GetScopeList,
"PUT": api.PutScope,
},
- "connections/:connectionId/transformation_rules": {
- "POST": api.CreateTransformationRule,
- "GET": api.GetTransformationRuleList,
+ "connections/:connectionId/scope_configs": {
+ "POST": api.CreateScopeConfig,
+ "GET": api.GetScopeConfigList,
},
- "connections/:connectionId/transformation_rules/:id": {
- "PATCH": api.UpdateTransformationRule,
- "GET": api.GetTransformationRule,
+ "connections/:connectionId/scope_configs/:id": {
+ "PATCH": api.UpdateScopeConfig,
+ "GET": api.GetScopeConfig,
},
"connections/:connectionId/proxy/rest/*path": {
"GET": api.Proxy,
diff --git a/backend/plugins/gitlab/models/migrationscripts/20230322_add_connection_id_to_transformation_rules.go b/backend/plugins/gitlab/models/migrationscripts/20230322_add_connection_id_to_transformation_rules.go
index 4f710c71a..47328b0ec 100644
--- a/backend/plugins/gitlab/models/migrationscripts/20230322_add_connection_id_to_transformation_rules.go
+++ b/backend/plugins/gitlab/models/migrationscripts/20230322_add_connection_id_to_transformation_rules.go
@@ -22,25 +22,34 @@ import (
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/helpers/migrationhelper"
- "github.com/apache/incubator-devlake/plugins/gitlab/models"
)
type addConnectionIdToTransformationRule struct{}
-type transformationRule20220322 struct {
+type project20230322 struct {
+ ConnectionId uint64 `gorm:"primaryKey"`
+ GitlabId int `gorm:"primaryKey"`
+ TransformationRuleId uint64
+}
+
+func (project20230322) TableName() string {
+ return "_tool_gitlab_projects"
+}
+
+type transformationRule20230322 struct {
ConnectionId uint64
}
-func (transformationRule20220322) TableName() string {
+func (transformationRule20230322) TableName() string {
return "_tool_gitlab_transformation_rules"
}
func (u *addConnectionIdToTransformationRule) Up(baseRes context.BasicRes) errors.Error {
- err := migrationhelper.AutoMigrateTables(baseRes, &transformationRule20220322{})
+ err := migrationhelper.AutoMigrateTables(baseRes, &transformationRule20230322{})
if err != nil {
return err
}
- var scopes []models.GitlabProject
+ var scopes []project20230322
err = baseRes.GetDal().All(&scopes)
if err != nil {
return err
@@ -55,14 +64,14 @@ func (u *addConnectionIdToTransformationRule) Up(baseRes context.BasicRes) error
// set connection_id for rules
for trId, cId := range idMap {
err = baseRes.GetDal().UpdateColumn(
- &models.GitlabTransformationRule{}, "connection_id", cId,
+ &transformationRule20230322{}, "connection_id", cId,
dal.Where("id = ?", trId))
if err != nil {
return err
}
}
// delete all rules that are not referenced.
- return baseRes.GetDal().Delete(&models.GitlabTransformationRule{}, dal.Where("connection_id IS NULL OR connection_id = 0"))
+ return baseRes.GetDal().Delete(&transformationRule20230322{}, dal.Where("connection_id IS NULL OR connection_id = 0"))
}
func (*addConnectionIdToTransformationRule) Version() uint64 {
diff --git a/backend/plugins/gitlab/models/migrationscripts/20230529_scope_config.go b/backend/plugins/gitlab/models/migrationscripts/20230529_scope_config.go
new file mode 100644
index 000000000..a4a3a4e62
--- /dev/null
+++ b/backend/plugins/gitlab/models/migrationscripts/20230529_scope_config.go
@@ -0,0 +1,56 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+ "github.com/apache/incubator-devlake/core/context"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/helpers/migrationhelper"
+)
+
+type renameTr2ScopeConfig struct {
+}
+
+type scopeConfig20230529 struct {
+ Entities []string `gorm:"type:json" json:"entities"`
+}
+
+func (scopeConfig20230529) TableName() string {
+ return "_tool_gitlab_scope_configs"
+}
+
+func (u *renameTr2ScopeConfig) Up(baseRes context.BasicRes) errors.Error {
+ db := baseRes.GetDal()
+ err := db.RenameColumn("_tool_gitlab_projects", "transformation_rule_id", "scope_config_id")
+ if err != nil {
+ return err
+ }
+ err = db.RenameTable("_tool_gitlab_transformation_rules", "_tool_gitlab_scope_configs")
+ if err != nil {
+ return err
+ }
+ return migrationhelper.AutoMigrateTables(baseRes, &scopeConfig20230529{})
+}
+
+func (*renameTr2ScopeConfig) Version() uint64 {
+ return 20230529173435
+}
+
+func (*renameTr2ScopeConfig) Name() string {
+ return "rename transformation rule to scope config for gitlab"
+}
diff --git a/backend/plugins/gitlab/models/migrationscripts/register.go b/backend/plugins/gitlab/models/migrationscripts/register.go
index 6be1f60fe..b39b8bba3 100644
--- a/backend/plugins/gitlab/models/migrationscripts/register.go
+++ b/backend/plugins/gitlab/models/migrationscripts/register.go
@@ -35,5 +35,6 @@ func All() []plugin.MigrationScript {
new(addConnectionIdToTransformationRule),
new(addGitlabCommitAuthorInfo),
new(addTypeEnvToPipeline),
+ new(renameTr2ScopeConfig),
}
}
diff --git a/backend/plugins/gitlab/models/project.go b/backend/plugins/gitlab/models/project.go
index 7368a9ab6..b5133421e 100644
--- a/backend/plugins/gitlab/models/project.go
+++ b/backend/plugins/gitlab/models/project.go
@@ -31,7 +31,7 @@ var _ plugin.ToolLayerScope = (*GitlabProject)(nil)
type GitlabProject struct {
ConnectionId uint64 `json:"connectionId" mapstructure:"connectionId" validate:"required" gorm:"primaryKey"`
- TransformationRuleId uint64 `json:"transformationRuleId,omitempty" mapstructure:"transformationRuleId"`
+ ScopeConfigId uint64 `json:"scopeConfigId,omitempty" mapstructure:"scopeConfigId"`
GitlabId int `json:"gitlabId" mapstructure:"gitlabId" validate:"required" gorm:"primaryKey"`
Name string `json:"name" mapstructure:"name" gorm:"type:varchar(255)"`
Description string `json:"description" mapstructure:"description"`
diff --git a/backend/plugins/gitlab/models/transformation_rule.go b/backend/plugins/gitlab/models/scope_config.go
similarity index 92%
rename from backend/plugins/gitlab/models/transformation_rule.go
rename to backend/plugins/gitlab/models/scope_config.go
index 255f30860..23bc35c34 100644
--- a/backend/plugins/gitlab/models/transformation_rule.go
+++ b/backend/plugins/gitlab/models/scope_config.go
@@ -22,8 +22,8 @@ import (
"gorm.io/datatypes"
)
-type GitlabTransformationRule struct {
- common.Model
+type GitlabScopeConfig struct {
+ common.ScopeConfig `mapstructure:",squash" json:",inline" gorm:"embedded"`
ConnectionId uint64 `mapstructure:"connectionId" json:"connectionId"`
Name string `gorm:"type:varchar(255);index:idx_name_gitlab,unique" validate:"required" mapstructure:"name" json:"name"`
PrType string `mapstructure:"prType" json:"prType"`
@@ -40,6 +40,6 @@ type GitlabTransformationRule struct {
Refdiff datatypes.JSONMap `mapstructure:"refdiff,omitempty" json:"refdiff" swaggertype:"object" format:"json"`
}
-func (t GitlabTransformationRule) TableName() string {
- return "_tool_gitlab_transformation_rules"
+func (t GitlabScopeConfig) TableName() string {
+ return "_tool_gitlab_scope_configs"
}
diff --git a/backend/plugins/gitlab/tasks/issue_extractor.go b/backend/plugins/gitlab/tasks/issue_extractor.go
index f3e35b58c..315200089 100644
--- a/backend/plugins/gitlab/tasks/issue_extractor.go
+++ b/backend/plugins/gitlab/tasks/issue_extractor.go
@@ -19,12 +19,13 @@ package tasks
import (
"encoding/json"
+ "regexp"
+
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models/domainlayer/ticket"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/gitlab/models"
- "regexp"
)
var ExtractApiIssuesMeta = plugin.SubTaskMeta{
@@ -127,7 +128,7 @@ type IssuesResponse struct {
func ExtractApiIssues(taskCtx plugin.SubTaskContext) errors.Error {
rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_ISSUE_TABLE)
- config := data.Options.GitlabTransformationRule
+ config := data.Options.ScopeConfig
var issueSeverityRegex *regexp.Regexp
var issueComponentRegex *regexp.Regexp
var issuePriorityRegex *regexp.Regexp
diff --git a/backend/plugins/gitlab/tasks/mr_detail_extractor.go b/backend/plugins/gitlab/tasks/mr_detail_extractor.go
index 3808b6473..e48d88fb1 100644
--- a/backend/plugins/gitlab/tasks/mr_detail_extractor.go
+++ b/backend/plugins/gitlab/tasks/mr_detail_extractor.go
@@ -37,7 +37,7 @@ var ExtractApiMergeRequestDetailsMeta = plugin.SubTaskMeta{
func ExtractApiMergeRequestDetails(taskCtx plugin.SubTaskContext) errors.Error {
rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_MERGE_REQUEST_DETAIL_TABLE)
- config := data.Options.GitlabTransformationRule
+ config := data.Options.ScopeConfig
var labelTypeRegex *regexp.Regexp
var labelComponentRegex *regexp.Regexp
var prType = config.PrType
diff --git a/backend/plugins/gitlab/tasks/mr_extractor.go b/backend/plugins/gitlab/tasks/mr_extractor.go
index a7e39c230..bb22570b5 100644
--- a/backend/plugins/gitlab/tasks/mr_extractor.go
+++ b/backend/plugins/gitlab/tasks/mr_extractor.go
@@ -79,7 +79,7 @@ var ExtractApiMergeRequestsMeta = plugin.SubTaskMeta{
func ExtractApiMergeRequests(taskCtx plugin.SubTaskContext) errors.Error {
rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_MERGE_REQUEST_TABLE)
- config := data.Options.GitlabTransformationRule
+ config := data.Options.ScopeConfig
var labelTypeRegex *regexp.Regexp
var labelComponentRegex *regexp.Regexp
var prType = config.PrType
diff --git a/backend/plugins/gitlab/tasks/task_data.go b/backend/plugins/gitlab/tasks/task_data.go
index 4170a4547..a49d11389 100644
--- a/backend/plugins/gitlab/tasks/task_data.go
+++ b/backend/plugins/gitlab/tasks/task_data.go
@@ -26,12 +26,11 @@ import (
)
type GitlabOptions struct {
- ConnectionId uint64 `mapstructure:"connectionId" json:"connectionId"`
- ProjectId int `mapstructure:"projectId" json:"projectId"`
- TransformationRuleId uint64 `mapstructure:"transformationRuleId" json:"transformationRuleId"`
- Tasks []string `mapstructure:"tasks" json:"tasks,omitempty"`
- TimeAfter string
- *models.GitlabTransformationRule `mapstructure:"transformationRules" json:"transformationRules"`
+ ConnectionId uint64 `mapstructure:"connectionId" json:"connectionId"`
+ ProjectId int `mapstructure:"projectId" json:"projectId"`
+ ScopeConfigId uint64 `mapstructure:"scopeConfigId" json:"scopeConfigId"`
+ TimeAfter string `mapstructure:"timeAfter" json:"timeAfter"`
+ ScopeConfig *models.GitlabScopeConfig `mapstructure:"scopeConfig" json:"scopeConfig"`
}
type GitlabTaskData struct {
@@ -54,8 +53,8 @@ func DecodeAndValidateTaskOptions(options map[string]interface{}) (*GitlabOption
if op.ConnectionId == 0 {
return nil, errors.BadInput.New("connectionId is invalid")
}
- if op.GitlabTransformationRule == nil && op.TransformationRuleId == 0 {
- op.GitlabTransformationRule = new(models.GitlabTransformationRule)
+ if op.ScopeConfig == nil && op.ScopeConfigId == 0 {
+ op.ScopeConfig = new(models.GitlabScopeConfig)
}
return &op, nil
}