You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by kl...@apache.org on 2022/12/12 12:55:52 UTC
[incubator-devlake] branch main updated: feat: gitlab plan v200 (#3847)
This is an automated email from the ASF dual-hosted git repository.
klesh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 1acd677c4 feat: gitlab plan v200 (#3847)
1acd677c4 is described below
commit 1acd677c48214339770b4d550b74ac05c27f7f30
Author: mappjzc <zh...@merico.dev>
AuthorDate: Mon Dec 12 20:55:48 2022 +0800
feat: gitlab plan v200 (#3847)
* feat: gitlab plan v200
Add gitlab scope and TransformationRule for plan v200.
Add TestMakeDataSourcePipelinePlanV200
Nddtfjiang <zh...@merico.dev>
* fix: fix for review
fix for review
Nddtfjiang <zh...@merico.dev>
* fix: fix for review
fix for reivew
Nddtfjiang <zh...@merico.dev>
* fix: fix for refactor
fix for refactor
Nddtfjiang <zh...@merico.dev>
* fix: fix for review and blueprint debug
fix for review and blueprint debug.
Nddtfjiang <zh...@merico.dev>
* fix: fix blueprint save scope failed
fix blueprint save scope failed.
Nddtfjiang <zh...@merico.dev>
---
.../unithelper/dummy_baesres.go | 20 +-
helpers/unithelper/dummy_logger.go | 2 +-
impl/dalgorm/dalgorm.go | 5 +
models/common/base.go | 8 +
models/domainlayer/code/repo.go | 12 ++
models/domainlayer/devops/cicd_scope.go | 15 +-
models/domainlayer/domainlayer.go | 7 +
models/domainlayer/ticket/board.go | 11 +
plugins/core/dal/dal.go | 2 +
plugins/gitlab/api/blueprint_V200_test.go | 228 +++++++++++++++++++++
plugins/gitlab/api/blueprint_v200.go | 197 +++++++++++++++---
plugins/gitlab/impl/impl.go | 22 +-
plugins/gitlab/models/project.go | 1 +
plugins/gitlab/models/transformation_rule.go | 28 +--
plugins/gitlab/tasks/job_convertor.go | 2 +-
plugins/gitlab/tasks/pipeline_commit_convertor.go | 5 +-
plugins/gitlab/tasks/pipeline_convertor.go | 5 +-
services/blueprint_makeplan_v200.go | 5 +-
18 files changed, 506 insertions(+), 69 deletions(-)
diff --git a/models/domainlayer/domainlayer.go b/helpers/unithelper/dummy_baesres.go
similarity index 63%
copy from models/domainlayer/domainlayer.go
copy to helpers/unithelper/dummy_baesres.go
index 33e0b606a..2155ff054 100644
--- a/models/domainlayer/domainlayer.go
+++ b/helpers/unithelper/dummy_baesres.go
@@ -15,13 +15,23 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
-package domainlayer
+package unithelper
import (
- "github.com/apache/incubator-devlake/models/common"
+ "github.com/apache/incubator-devlake/mocks"
+ "github.com/stretchr/testify/mock"
)
-type DomainEntity struct {
- Id string `json:"id" gorm:"primaryKey;type:varchar(255);comment:This key is generated based on details from the original plugin"` // format: <Plugin>:<Entity>:<PK0>:<PK1>
- common.NoPKModel
+// DummyBasicRes FIXME ...
+func DummyBasicRes(callback func(mockDal *mocks.Dal)) *mocks.BasicRes {
+ mockRes := new(mocks.BasicRes)
+ mockLog := DummyLogger()
+ mockDal := new(mocks.Dal)
+
+ callback(mockDal)
+
+ mockRes.On("GetDal").Return(mockDal)
+ mockRes.On("GetLogger").Return(mockLog)
+ mockRes.On("GetConfig", mock.Anything).Return("")
+ return mockRes
}
diff --git a/helpers/unithelper/dummy_logger.go b/helpers/unithelper/dummy_logger.go
index 7258e5286..97c7e21c6 100644
--- a/helpers/unithelper/dummy_logger.go
+++ b/helpers/unithelper/dummy_logger.go
@@ -31,7 +31,7 @@ func DummyLogger() *mocks.Logger {
logger.On("Debug", mock.Anything, mock.Anything).Maybe()
logger.On("Info", mock.Anything, mock.Anything).Maybe()
logger.On("Warn", mock.Anything, mock.Anything).Maybe()
- logger.On("Error", mock.Anything, mock.Anything).Maybe()
+ logger.On("Error", mock.Anything, mock.Anything, mock.Anything).Maybe()
logger.On("Nested", mock.Anything).Return(logger).Maybe()
return logger
}
diff --git a/impl/dalgorm/dalgorm.go b/impl/dalgorm/dalgorm.go
index 8b0887bca..b239456fb 100644
--- a/impl/dalgorm/dalgorm.go
+++ b/impl/dalgorm/dalgorm.go
@@ -355,6 +355,11 @@ func (d *Dalgorm) Begin() dal.Transaction {
return newTransaction(d)
}
+// IsErrorNotFound checking if the sql error is not found.
+func (d *Dalgorm) IsErrorNotFound(err errors.Error) bool {
+ return errors.Is(err, gorm.ErrRecordNotFound)
+}
+
// NewDalgorm creates a *Dalgorm
func NewDalgorm(db *gorm.DB) *Dalgorm {
return &Dalgorm{db}
diff --git a/models/common/base.go b/models/common/base.go
index 5efa2133d..4b9a69544 100644
--- a/models/common/base.go
+++ b/models/common/base.go
@@ -45,6 +45,14 @@ type RawDataOrigin struct {
RawDataRemark string `gorm:"column:_raw_data_remark" json:"_raw_data_remark"`
}
+func NewNoPKModel() NoPKModel {
+ now := time.Now()
+ return NoPKModel{
+ CreatedAt: now,
+ UpdatedAt: now,
+ }
+}
+
var (
DUPLICATE_REGEX = regexp.MustCompile(`(?i)\bduplicate\b`)
)
diff --git a/models/domainlayer/code/repo.go b/models/domainlayer/code/repo.go
index ffc1038a1..54d317ef1 100644
--- a/models/domainlayer/code/repo.go
+++ b/models/domainlayer/code/repo.go
@@ -60,3 +60,15 @@ func (r *Repo) ScopeId() string {
func (r *Repo) ScopeName() string {
return r.Name
}
+
+func NewRepo(id string, name string) *Repo {
+ repo := &Repo{
+ DomainEntity: domainlayer.NewDomainEntity(id),
+ }
+
+ repo.Name = name
+ repo.CreatedDate = &repo.CreatedAt
+ repo.UpdatedDate = &repo.UpdatedAt
+
+ return repo
+}
diff --git a/models/domainlayer/devops/cicd_scope.go b/models/domainlayer/devops/cicd_scope.go
index fcb3646e3..65bc7b775 100644
--- a/models/domainlayer/devops/cicd_scope.go
+++ b/models/domainlayer/devops/cicd_scope.go
@@ -18,9 +18,10 @@ limitations under the License.
package devops
import (
+ "time"
+
"github.com/apache/incubator-devlake/models/domainlayer"
"github.com/apache/incubator-devlake/plugins/core"
- "time"
)
var _ core.Scope = (*CicdScope)(nil)
@@ -45,3 +46,15 @@ func (r *CicdScope) ScopeId() string {
func (r *CicdScope) ScopeName() string {
return r.Name
}
+
+func NewCicdScope(id string, name string) *CicdScope {
+ cicdScope := &CicdScope{
+ DomainEntity: domainlayer.NewDomainEntity(id),
+ }
+
+ cicdScope.Name = name
+ cicdScope.CreatedDate = &cicdScope.CreatedAt
+ cicdScope.UpdatedDate = &cicdScope.UpdatedAt
+
+ return cicdScope
+}
diff --git a/models/domainlayer/domainlayer.go b/models/domainlayer/domainlayer.go
index 33e0b606a..94e703ce1 100644
--- a/models/domainlayer/domainlayer.go
+++ b/models/domainlayer/domainlayer.go
@@ -25,3 +25,10 @@ type DomainEntity struct {
Id string `json:"id" gorm:"primaryKey;type:varchar(255);comment:This key is generated based on details from the original plugin"` // format: <Plugin>:<Entity>:<PK0>:<PK1>
common.NoPKModel
}
+
+func NewDomainEntity(id string) DomainEntity {
+ return DomainEntity{
+ Id: id,
+ NoPKModel: common.NewNoPKModel(),
+ }
+}
diff --git a/models/domainlayer/ticket/board.go b/models/domainlayer/ticket/board.go
index 2f090d52e..5fb6f6f00 100644
--- a/models/domainlayer/ticket/board.go
+++ b/models/domainlayer/ticket/board.go
@@ -49,6 +49,17 @@ func (r *Board) ScopeName() string {
return r.Name
}
+func NewBoard(id string, name string) *Board {
+ board := &Board{
+ DomainEntity: domainlayer.NewDomainEntity(id),
+ }
+
+ board.Name = name
+ board.CreatedDate = &board.CreatedAt
+
+ return board
+}
+
type BoardSprint struct {
common.NoPKModel
BoardId string `gorm:"primaryKey;type:varchar(255)"`
diff --git a/plugins/core/dal/dal.go b/plugins/core/dal/dal.go
index d5fa37e9c..6b1d80214 100644
--- a/plugins/core/dal/dal.go
+++ b/plugins/core/dal/dal.go
@@ -140,6 +140,8 @@ type Dal interface {
Session(config SessionConfig) Dal
// Begin create a new transaction
Begin() Transaction
+ // checking if the sql error is not found.
+ IsErrorNotFound(err errors.Error) bool
}
type Transaction interface {
diff --git a/plugins/gitlab/api/blueprint_V200_test.go b/plugins/gitlab/api/blueprint_V200_test.go
new file mode 100644
index 000000000..dcbdbf615
--- /dev/null
+++ b/plugins/gitlab/api/blueprint_V200_test.go
@@ -0,0 +1,228 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package api
+
+import (
+ "strconv"
+ "testing"
+ "time"
+
+ "github.com/apache/incubator-devlake/errors"
+ "github.com/apache/incubator-devlake/helpers/unithelper"
+ "github.com/apache/incubator-devlake/mocks"
+ "github.com/apache/incubator-devlake/models/common"
+ "github.com/apache/incubator-devlake/models/domainlayer/code"
+ "github.com/apache/incubator-devlake/models/domainlayer/devops"
+ "github.com/apache/incubator-devlake/models/domainlayer/ticket"
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/gitlab/models"
+ "github.com/apache/incubator-devlake/plugins/gitlab/tasks"
+ "github.com/apache/incubator-devlake/plugins/helper"
+ "github.com/go-playground/validator/v10"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/mock"
+)
+
+func TestMakeDataSourcePipelinePlanV200(t *testing.T) {
+ const testConnectionID uint64 = 1
+ const testTransformationRuleId uint64 = 2
+ const testID int = 37
+ const testGitlabEndPoint string = "https://gitlab.com/api/v4/"
+ const testHttpUrlToRepo string = "https://this_is_cloneUrl"
+ const testToken string = "nddtf"
+ const testName string = "gitlab-test"
+ const testTransformationRuleName string = "github transformation rule"
+ const testProxy string = ""
+
+ bpScopes := []*core.BlueprintScopeV200{
+ {
+ Entities: []string{core.DOMAIN_TYPE_CODE, core.DOMAIN_TYPE_TICKET, core.DOMAIN_TYPE_CICD},
+ Id: strconv.Itoa(testID),
+ Name: testName,
+ },
+ }
+
+ var testGitlabProject *models.GitlabProject = &models.GitlabProject{
+ ConnectionId: testConnectionID,
+ GitlabId: testID,
+ Name: testName,
+
+ TransformationRuleId: testTransformationRuleId,
+ CreatedDate: time.Time{},
+ HttpUrlToRepo: testHttpUrlToRepo,
+ }
+
+ var testTransformationRule *models.GitlabTransformationRule = &models.GitlabTransformationRule{
+ Model: common.Model{
+ ID: testTransformationRuleId,
+ },
+ Name: testTransformationRuleName,
+ PrType: "hey,man,wasup",
+ Refdiff: map[string]interface{}{
+ "tagsPattern": "pattern",
+ "tagsLimit": 10,
+ "tagsOrder": "reverse semver",
+ },
+ }
+
+ var testGitlabConnection *models.GitlabConnection = &models.GitlabConnection{
+ RestConnection: helper.RestConnection{
+ BaseConnection: helper.BaseConnection{
+ Name: testName,
+ Model: common.Model{
+ ID: testConnectionID,
+ },
+ },
+ Endpoint: testGitlabEndPoint,
+ Proxy: testProxy,
+ RateLimitPerHour: 0,
+ },
+ AccessToken: helper.AccessToken{
+ Token: testToken,
+ },
+ }
+
+ var expectRepoId = "gitlab:GitlabProject:1:37"
+
+ var testSubTaskMeta = []core.SubTaskMeta{
+ tasks.ConvertProjectMeta,
+ tasks.CollectApiIssuesMeta,
+ tasks.ExtractApiIssuesMeta,
+ tasks.ConvertIssuesMeta,
+ tasks.ConvertIssueLabelsMeta,
+ tasks.CollectApiJobsMeta,
+ tasks.ExtractApiJobsMeta,
+ tasks.CollectApiPipelinesMeta,
+ tasks.ExtractApiPipelinesMeta,
+ }
+
+ var expectPlans core.PipelinePlan = core.PipelinePlan{
+ {
+ {
+ Plugin: "gitlab",
+ Subtasks: []string{
+ tasks.ConvertProjectMeta.Name,
+ tasks.CollectApiIssuesMeta.Name,
+ tasks.ExtractApiIssuesMeta.Name,
+ tasks.ConvertIssuesMeta.Name,
+ tasks.ConvertIssueLabelsMeta.Name,
+ tasks.CollectApiJobsMeta.Name,
+ tasks.ExtractApiJobsMeta.Name,
+ tasks.CollectApiPipelinesMeta.Name,
+ tasks.ExtractApiPipelinesMeta.Name,
+ },
+ SkipOnFail: false,
+ Options: map[string]interface{}{
+ "connectionId": uint64(1),
+ "projectId": testID,
+ "entities": bpScopes[0].Entities,
+ },
+ },
+ {
+ Plugin: "gitextractor",
+ SkipOnFail: false,
+ Options: map[string]interface{}{
+ "proxy": "",
+ "repoId": expectRepoId,
+ "url": "https://git:nddtf@this_is_cloneUrl",
+ },
+ },
+ },
+ {
+ {
+ Plugin: "refdiff",
+ SkipOnFail: false,
+ Options: map[string]interface{}{
+ "tagsLimit": 10,
+ "tagsOrder": "reverse semver",
+ "tagsPattern": "pattern",
+ },
+ },
+ },
+ }
+
+ expectRepo := code.NewRepo(expectRepoId, testName)
+ expectRepo.ForkedFrom = testGitlabProject.ForkedFromProjectWebUrl
+
+ expectCicdScope := devops.NewCicdScope(expectRepoId, testName)
+ expectCicdScope.Description = ""
+ expectCicdScope.Url = ""
+
+ expectBoard := ticket.NewBoard(expectRepoId, testName)
+ expectBoard.Description = ""
+ expectBoard.Url = ""
+ expectBoard.Type = ""
+
+ var err errors.Error
+
+ // register gitlab plugin for NewDomainIdGenerator
+ mockMeta := mocks.NewPluginMeta(t)
+ mockMeta.On("RootPkgPath").Return("github.com/apache/incubator-devlake/plugins/gitlab")
+ err = core.RegisterPlugin("gitlab", mockMeta)
+ assert.Equal(t, err, nil)
+
+ // Refresh Global Variables and set the sql mock
+ BasicRes = unithelper.DummyBasicRes(func(mockDal *mocks.Dal) {
+ mockDal.On("First", mock.Anything, mock.Anything).Run(func(args mock.Arguments) {
+ dst := args.Get(0).(*models.GitlabConnection)
+ *dst = *testGitlabConnection
+ }).Return(nil).Once()
+
+ mockDal.On("First", mock.Anything, mock.Anything).Run(func(args mock.Arguments) {
+ dst := args.Get(0).(*models.GitlabProject)
+ *dst = *testGitlabProject
+ }).Return(nil).Twice()
+
+ mockDal.On("First", mock.Anything, mock.Anything).Run(func(args mock.Arguments) {
+ dst := args.Get(0).(*models.GitlabTransformationRule)
+ *dst = *testTransformationRule
+ }).Return(nil).Once()
+ })
+ connectionHelper = helper.NewConnectionHelper(
+ BasicRes,
+ validator.New(),
+ )
+
+ plans, scopes, err := MakePipelinePlanV200(testSubTaskMeta, testConnectionID, bpScopes)
+ assert.Equal(t, err, nil)
+
+ assert.Equal(t, expectPlans, plans)
+
+ // ignore CreatedDate UpdatedDate CreatedAt UpdatedAt checking
+ expectRepo.CreatedDate = scopes[0].(*code.Repo).CreatedDate
+ expectRepo.UpdatedDate = scopes[0].(*code.Repo).UpdatedDate
+ expectRepo.CreatedAt = scopes[0].(*code.Repo).CreatedAt
+ expectRepo.UpdatedAt = scopes[0].(*code.Repo).UpdatedAt
+
+ expectCicdScope.CreatedDate = scopes[1].(*devops.CicdScope).CreatedDate
+ expectCicdScope.UpdatedDate = scopes[1].(*devops.CicdScope).UpdatedDate
+ expectCicdScope.CreatedAt = scopes[1].(*devops.CicdScope).CreatedAt
+ expectCicdScope.UpdatedAt = scopes[1].(*devops.CicdScope).UpdatedAt
+
+ expectBoard.CreatedDate = scopes[2].(*ticket.Board).CreatedDate
+ expectBoard.CreatedAt = scopes[2].(*ticket.Board).CreatedAt
+ expectBoard.UpdatedAt = scopes[2].(*ticket.Board).UpdatedAt
+
+ var expectScopes []core.Scope = []core.Scope{
+ expectRepo,
+ expectCicdScope,
+ expectBoard,
+ }
+
+ assert.Equal(t, expectScopes, scopes)
+}
diff --git a/plugins/gitlab/api/blueprint_v200.go b/plugins/gitlab/api/blueprint_v200.go
index 3109c5994..81e929903 100644
--- a/plugins/gitlab/api/blueprint_v200.go
+++ b/plugins/gitlab/api/blueprint_v200.go
@@ -18,63 +18,192 @@ limitations under the License.
package api
import (
+ "fmt"
+ "net/url"
+ "strconv"
+
"github.com/apache/incubator-devlake/errors"
- "github.com/go-playground/validator/v10"
+ "github.com/apache/incubator-devlake/utils"
"github.com/apache/incubator-devlake/models/domainlayer/code"
+ "github.com/apache/incubator-devlake/models/domainlayer/devops"
"github.com/apache/incubator-devlake/models/domainlayer/didgen"
"github.com/apache/incubator-devlake/models/domainlayer/ticket"
"github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/core/dal"
"github.com/apache/incubator-devlake/plugins/gitlab/models"
"github.com/apache/incubator-devlake/plugins/helper"
)
-func MakeDataSourcePipelinePlanV200(connectionId uint64, scopes []*core.BlueprintScopeV200) (pp core.PipelinePlan, sc []core.Scope, err errors.Error) {
- pp = make(core.PipelinePlan, 0, 1)
- sc = make([]core.Scope, 0, 3*len(scopes))
- err = nil
-
- connectionHelper := helper.NewConnectionHelper(BasicRes, validator.New())
+func MakePipelinePlanV200(subtaskMetas []core.SubTaskMeta, connectionId uint64, scope []*core.BlueprintScopeV200) (core.PipelinePlan, []core.Scope, errors.Error) {
+ var err errors.Error
+ connection := new(models.GitlabConnection)
+ err1 := connectionHelper.FirstById(connection, connectionId)
+ if err1 != nil {
+ return nil, nil, errors.Default.Wrap(err1, fmt.Sprintf("error on get connection by id[%d]", connectionId))
+ }
- // get the connection info for url
- connection := &models.GitlabConnection{}
- err = connectionHelper.FirstById(connection, connectionId)
+ sc, err := makeScopeV200(connectionId, scope)
if err != nil {
return nil, nil, err
}
- ps := make(core.PipelineStage, 0, len(scopes))
- for _, scope := range scopes {
- var board ticket.Board
- var repo code.Repo
+ pp, err := makePipelinePlanV200(subtaskMetas, scope, connection)
+ if err != nil {
+ return nil, nil, err
+ }
- id := didgen.NewDomainIdGenerator(&models.GitlabProject{}).Generate(connectionId, scope.Id)
+ return pp, sc, nil
+}
- repo.Id = id
- repo.Name = scope.Name
+func makeScopeV200(connectionId uint64, scopes []*core.BlueprintScopeV200) ([]core.Scope, errors.Error) {
+ sc := make([]core.Scope, 0, 3*len(scopes))
- board.Id = id
- board.Name = scope.Name
+ for _, scope := range scopes {
+ intScopeId, err1 := strconv.Atoi(scope.Id)
+ if err1 != nil {
+ return nil, errors.Default.Wrap(err1, fmt.Sprintf("Failed to strconv.Atoi for scope.Id [%s]", scope.Id))
+ }
+ id := didgen.NewDomainIdGenerator(&models.GitlabProject{}).Generate(connectionId, intScopeId)
+
+ // get repo from db
+ gitlabProject, err := GetRepoByConnectionIdAndscopeId(connectionId, scope.Id)
+ if err != nil {
+ return nil, err
+ }
+
+ if utils.StringsContains(scope.Entities, core.DOMAIN_TYPE_CODE_REVIEW) ||
+ utils.StringsContains(scope.Entities, core.DOMAIN_TYPE_CODE) {
+ // if we don't need to collect gitex, we need to add repo to scopes here
+ scopeRepo := code.NewRepo(id, gitlabProject.Name)
+
+ if gitlabProject.ForkedFromProjectWebUrl != "" {
+ scopeRepo.ForkedFrom = gitlabProject.ForkedFromProjectWebUrl
+ }
+ sc = append(sc, scopeRepo)
+ }
+
+ // add cicd_scope to scopes
+ if utils.StringsContains(scope.Entities, core.DOMAIN_TYPE_CICD) {
+ scopeCICD := devops.NewCicdScope(id, gitlabProject.Name)
+
+ sc = append(sc, scopeCICD)
+ }
+
+ // add board to scopes
+ if utils.StringsContains(scope.Entities, core.DOMAIN_TYPE_TICKET) {
+ scopeTicket := ticket.NewBoard(id, gitlabProject.Name)
+
+ sc = append(sc, scopeTicket)
+ }
+ }
- sc = append(sc, &repo)
- sc = append(sc, &board)
+ return sc, nil
+}
- ps = append(ps, &core.PipelineTask{
- Plugin: "gitlab",
- Options: map[string]interface{}{
- "name": scope.Name,
- },
+func makePipelinePlanV200(subtaskMetas []core.SubTaskMeta, scopes []*core.BlueprintScopeV200, connection *models.GitlabConnection) (core.PipelinePlan, errors.Error) {
+ plans := make(core.PipelinePlan, 0, 3*len(scopes))
+ for _, scope := range scopes {
+ var stage core.PipelineStage
+ var err errors.Error
+ // get repo
+ repo, err := GetRepoByConnectionIdAndscopeId(connection.ID, scope.Id)
+ if err != nil {
+ return nil, err
+ }
+
+ // get transformationRuleId
+ transformationRules, err := GetTransformationRuleByRepo(repo)
+ if err != nil {
+ return nil, err
+ }
+
+ // get int scopeId
+ intScopeId, err1 := strconv.Atoi(scope.Id)
+ if err != nil {
+ return nil, errors.Default.Wrap(err1, fmt.Sprintf("Failed to strconv.Atoi for scope.Id [%s]", scope.Id))
+ }
+
+ // gitlab main part
+ options := make(map[string]interface{})
+ options["connectionId"] = connection.ID
+ options["projectId"] = intScopeId
+ options["entities"] = scope.Entities
+
+ // construct subtasks
+ subtasks, err := helper.MakePipelinePlanSubtasks(subtaskMetas, scope.Entities)
+ if err != nil {
+ return nil, err
+ }
+
+ stage = append(stage, &core.PipelineTask{
+ Plugin: "gitlab",
+ Subtasks: subtasks,
+ Options: options,
})
- ps = append(ps, &core.PipelineTask{
- Plugin: "gitextractor",
- Options: map[string]interface{}{
- "url": connection.Endpoint + scope.Name,
- },
- })
+ // collect git data by gitextractor if CODE was requested
+ if utils.StringsContains(scope.Entities, core.DOMAIN_TYPE_CODE) {
+ cloneUrl, err := errors.Convert01(url.Parse(repo.HttpUrlToRepo))
+ if err != nil {
+ return nil, err
+ }
+ cloneUrl.User = url.UserPassword("git", connection.Token)
+ stage = append(stage, &core.PipelineTask{
+ Plugin: "gitextractor",
+ Options: map[string]interface{}{
+ "url": cloneUrl.String(),
+ "repoId": didgen.NewDomainIdGenerator(&models.GitlabProject{}).Generate(connection.ID, repo.GitlabId),
+ "proxy": connection.Proxy,
+ },
+ })
+ }
+
+ plans = append(plans, stage)
+
+ // refdiff part
+ if transformationRules.Refdiff != nil {
+ task := &core.PipelineTask{
+ Plugin: "refdiff",
+ Options: transformationRules.Refdiff,
+ }
+ plans = append(plans, core.PipelineStage{task})
+ }
}
+ return plans, nil
+}
- pp = append(pp, ps)
+// GetRepoByConnectionIdAndscopeId get tbe repo by the connectionId and the scopeId
+func GetRepoByConnectionIdAndscopeId(connectionId uint64, scopeId string) (*models.GitlabProject, errors.Error) {
+ repo := &models.GitlabProject{}
+ db := BasicRes.GetDal()
+ err := db.First(repo, dal.Where("connection_id = ? AND gitlab_id = ?", connectionId, scopeId))
+ if err != nil {
+ if db.IsErrorNotFound(err) {
+ return nil, errors.Default.Wrap(err, fmt.Sprintf("can not find repo by connection [%d] scope [%s]", connectionId, scopeId))
+ }
+ return nil, errors.Default.Wrap(err, fmt.Sprintf("fail to find repo by connection [%d] scope [%s]", connectionId, scopeId))
+ }
- return pp, sc, nil
+ return repo, nil
+}
+
+// GetTransformationRuleByRepo get the GetTransformationRule by Repo
+func GetTransformationRuleByRepo(repo *models.GitlabProject) (*models.GitlabTransformationRule, errors.Error) {
+ transformationRules := &models.GitlabTransformationRule{}
+ transformationRuleId := repo.TransformationRuleId
+ if transformationRuleId != 0 {
+ db := BasicRes.GetDal()
+ err := db.First(transformationRules, dal.Where("id = ?", transformationRuleId))
+ if err != nil {
+ if db.IsErrorNotFound(err) {
+ return nil, errors.Default.Wrap(err, fmt.Sprintf("can not find transformationRules by transformationRuleId [%d]", transformationRuleId))
+ }
+ return nil, errors.Default.Wrap(err, fmt.Sprintf("fail to find transformationRules by transformationRuleId [%d]", transformationRuleId))
+ }
+ } else {
+ transformationRules.ID = 0
+ }
+
+ return transformationRules, nil
}
diff --git a/plugins/gitlab/impl/impl.go b/plugins/gitlab/impl/impl.go
index 395930cc9..b779b498f 100644
--- a/plugins/gitlab/impl/impl.go
+++ b/plugins/gitlab/impl/impl.go
@@ -19,11 +19,11 @@ package impl
import (
"fmt"
+ "strconv"
"time"
"github.com/apache/incubator-devlake/errors"
"github.com/apache/incubator-devlake/plugins/core"
- "github.com/apache/incubator-devlake/plugins/core/dal"
"github.com/apache/incubator-devlake/plugins/gitlab/api"
"github.com/apache/incubator-devlake/plugins/gitlab/models"
"github.com/apache/incubator-devlake/plugins/gitlab/models/migrationscripts"
@@ -65,8 +65,8 @@ func (plugin Gitlab) TransformationRule() interface{} {
return &models.GitlabTransformationRule{}
}
-func (plugin Gitlab) MakeDataSourcePipelinePlanV200(connectionId uint64, scopes []*core.BlueprintScopeV200) (pp core.PipelinePlan, sc []core.Scope, err errors.Error) {
- return api.MakeDataSourcePipelinePlanV200(connectionId, scopes)
+func (plugin Gitlab) MakeDataSourcePipelinePlanV200(connectionId uint64, scopes []*core.BlueprintScopeV200) (core.PipelinePlan, []core.Scope, errors.Error) {
+ return api.MakePipelinePlanV200(plugin.SubTaskMetas(), connectionId, scopes)
}
func (plugin Gitlab) GetTablesInfo() []core.Tabler {
@@ -162,19 +162,23 @@ func (plugin Gitlab) PrepareTaskData(taskCtx core.TaskContext, options map[strin
}
}
- if op.GitlabTransformationRule == nil && op.TransformationRuleId != 0 {
- var transformationRule models.GitlabTransformationRule
- err = taskCtx.GetDal().First(&transformationRule, dal.Where("id = ?", op.TransformationRuleId))
+ if op.GitlabTransformationRule == nil && op.ProjectId != 0 {
+ repo, err := api.GetRepoByConnectionIdAndscopeId(op.ConnectionId, strconv.Itoa(op.ProjectId))
if err != nil {
- return nil, errors.BadInput.Wrap(err, "fail to get transformationRule")
+ return nil, err
}
- op.GitlabTransformationRule = &transformationRule
+ transformationRule, err := api.GetTransformationRuleByRepo(repo)
+ if err != nil {
+ return nil, err
+ }
+ op.GitlabTransformationRule = transformationRule
}
taskData := tasks.GitlabTaskData{
Options: op,
ApiClient: apiClient,
}
+
if !createdDateAfter.IsZero() {
taskData.CreatedDateAfter = &createdDateAfter
logger.Debug("collect data updated createdDateAfter %s", createdDateAfter)
@@ -210,11 +214,11 @@ func (plugin Gitlab) ApiResources() map[string]map[string]core.ApiResourceHandle
},
"connections/:connectionId/scopes/:projectId": {
"GET": api.GetScope,
+ "PUT": api.PutScope,
"PATCH": api.UpdateScope,
},
"connections/:connectionId/scopes": {
"GET": api.GetScopeList,
- "PUT": api.PutScope,
},
"transformation_rules": {
"POST": api.CreateTransformationRule,
diff --git a/plugins/gitlab/models/project.go b/plugins/gitlab/models/project.go
index 2a2fa69ba..313f904be 100644
--- a/plugins/gitlab/models/project.go
+++ b/plugins/gitlab/models/project.go
@@ -38,6 +38,7 @@ type GitlabProject struct {
StarCount int `json:"starCount" mapstructure:"StarCount"`
ForkedFromProjectId int `json:"forkedFromProjectId" mapstructure:"forkedFromProjectId"`
ForkedFromProjectWebUrl string `json:"forkedFromProjectWebUrl" mapstructure:"forkedFromProjectWebUrl" gorm:"type:varchar(255)"`
+ HttpUrlToRepo string `json:"http_url_to_repo" gorm:"varchar(255)"`
CreatedDate time.Time `json:"createdDate" mapstructure:"-"`
UpdatedDate *time.Time `json:"updatedDate" mapstructure:"-"`
diff --git a/plugins/gitlab/models/transformation_rule.go b/plugins/gitlab/models/transformation_rule.go
index 962e62434..7ab0fa144 100644
--- a/plugins/gitlab/models/transformation_rule.go
+++ b/plugins/gitlab/models/transformation_rule.go
@@ -17,21 +17,25 @@ limitations under the License.
package models
-import "github.com/apache/incubator-devlake/models/common"
+import (
+ "github.com/apache/incubator-devlake/models/common"
+ "gorm.io/datatypes"
+)
type GitlabTransformationRule struct {
common.Model
- Name string `gorm:"type:varchar(255)"`
- PrType string `mapstructure:"prType" json:"prType"`
- PrComponent string `mapstructure:"prComponent" json:"prComponent"`
- PrBodyClosePattern string `mapstructure:"prBodyClosePattern" json:"prBodyClosePattern"`
- IssueSeverity string `mapstructure:"issueSeverity" json:"issueSeverity"`
- IssuePriority string `mapstructure:"issuePriority" json:"issuePriority"`
- IssueComponent string `mapstructure:"issueComponent" json:"issueComponent"`
- IssueTypeBug string `mapstructure:"issueTypeBug" json:"issueTypeBug"`
- IssueTypeIncident string `mapstructure:"issueTypeIncident" json:"issueTypeIncident"`
- IssueTypeRequirement string `mapstructure:"issueTypeRequirement" json:"issueTypeRequirement"`
- DeploymentPattern string `mapstructure:"deploymentPattern" json:"deploymentPattern"`
+ Name string `gorm:"type:varchar(255)"`
+ PrType string `mapstructure:"prType" json:"prType"`
+ PrComponent string `mapstructure:"prComponent" json:"prComponent"`
+ PrBodyClosePattern string `mapstructure:"prBodyClosePattern" json:"prBodyClosePattern"`
+ IssueSeverity string `mapstructure:"issueSeverity" json:"issueSeverity"`
+ IssuePriority string `mapstructure:"issuePriority" json:"issuePriority"`
+ IssueComponent string `mapstructure:"issueComponent" json:"issueComponent"`
+ IssueTypeBug string `mapstructure:"issueTypeBug" json:"issueTypeBug"`
+ IssueTypeIncident string `mapstructure:"issueTypeIncident" json:"issueTypeIncident"`
+ IssueTypeRequirement string `mapstructure:"issueTypeRequirement" json:"issueTypeRequirement"`
+ DeploymentPattern string `mapstructure:"deploymentPattern" json:"deploymentPattern"`
+ Refdiff datatypes.JSONMap `mapstructure:"refdiff,omitempty" json:"refdiff" swaggertype:"object" format:"json"`
}
func (t GitlabTransformationRule) TableName() string {
diff --git a/plugins/gitlab/tasks/job_convertor.go b/plugins/gitlab/tasks/job_convertor.go
index 917284bbd..1c95f0a46 100644
--- a/plugins/gitlab/tasks/job_convertor.go
+++ b/plugins/gitlab/tasks/job_convertor.go
@@ -36,7 +36,7 @@ var ConvertJobMeta = core.SubTaskMeta{
EntryPoint: ConvertJobs,
EnabledByDefault: true,
Description: "Convert tool layer table gitlab_job into domain layer table job",
- DomainTypes: []string{core.DOMAIN_TYPE_CROSS},
+ DomainTypes: []string{core.DOMAIN_TYPE_CICD},
}
func ConvertJobs(taskCtx core.SubTaskContext) (err errors.Error) {
diff --git a/plugins/gitlab/tasks/pipeline_commit_convertor.go b/plugins/gitlab/tasks/pipeline_commit_convertor.go
index 923ec8264..17c90b027 100644
--- a/plugins/gitlab/tasks/pipeline_commit_convertor.go
+++ b/plugins/gitlab/tasks/pipeline_commit_convertor.go
@@ -18,6 +18,8 @@ limitations under the License.
package tasks
import (
+ "reflect"
+
"github.com/apache/incubator-devlake/errors"
"github.com/apache/incubator-devlake/models/domainlayer/devops"
"github.com/apache/incubator-devlake/models/domainlayer/didgen"
@@ -25,7 +27,6 @@ import (
"github.com/apache/incubator-devlake/plugins/core/dal"
gitlabModels "github.com/apache/incubator-devlake/plugins/gitlab/models"
"github.com/apache/incubator-devlake/plugins/helper"
- "reflect"
)
var ConvertPipelineCommitMeta = core.SubTaskMeta{
@@ -33,7 +34,7 @@ var ConvertPipelineCommitMeta = core.SubTaskMeta{
EntryPoint: ConvertPipelineCommits,
EnabledByDefault: true,
Description: "Convert tool layer table gitlab_pipeline_project into domain layer table pipeline",
- DomainTypes: []string{core.DOMAIN_TYPE_CROSS},
+ DomainTypes: []string{core.DOMAIN_TYPE_CICD},
}
func ConvertPipelineCommits(taskCtx core.SubTaskContext) errors.Error {
diff --git a/plugins/gitlab/tasks/pipeline_convertor.go b/plugins/gitlab/tasks/pipeline_convertor.go
index 70dd9a856..2b90aa795 100644
--- a/plugins/gitlab/tasks/pipeline_convertor.go
+++ b/plugins/gitlab/tasks/pipeline_convertor.go
@@ -18,10 +18,11 @@ limitations under the License.
package tasks
import (
- "github.com/apache/incubator-devlake/errors"
"reflect"
"time"
+ "github.com/apache/incubator-devlake/errors"
+
"github.com/apache/incubator-devlake/models/domainlayer"
"github.com/apache/incubator-devlake/models/domainlayer/devops"
"github.com/apache/incubator-devlake/models/domainlayer/didgen"
@@ -36,7 +37,7 @@ var ConvertPipelineMeta = core.SubTaskMeta{
EntryPoint: ConvertPipelines,
EnabledByDefault: true,
Description: "Convert tool layer table gitlab_pipeline into domain layer table pipeline",
- DomainTypes: []string{core.DOMAIN_TYPE_CROSS},
+ DomainTypes: []string{core.DOMAIN_TYPE_CICD},
}
func ConvertPipelines(taskCtx core.SubTaskContext) errors.Error {
diff --git a/services/blueprint_makeplan_v200.go b/services/blueprint_makeplan_v200.go
index 448d42530..d28b22d78 100644
--- a/services/blueprint_makeplan_v200.go
+++ b/services/blueprint_makeplan_v200.go
@@ -42,12 +42,13 @@ func GeneratePlanJsonV200(
if len(scopes) > 0 {
e := db.Where("project_name = ?", projectName).Delete(&crossdomain.ProjectMapping{}).Error
if e != nil {
- return nil, errors.Convert(err)
+ return nil, errors.Default.Wrap(e, fmt.Sprintf("projectName:[%s]", projectName))
}
for _, scope := range scopes {
e = basicRes.GetDal().CreateOrUpdate(scope)
if e != nil {
- return nil, errors.Convert(err)
+ scopeInfo := fmt.Sprintf("[Id:%s][Name:%s][TableName:%s]", scope.ScopeId(), scope.ScopeName(), scope.TableName())
+ return nil, errors.Default.Wrap(e, fmt.Sprintf("failed to create scopes:[%s]", scopeInfo))
}
}
}