You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by li...@apache.org on 2022/12/12 13:09:25 UTC
[incubator-devlake] branch main updated: feat(jenkins): update bp v200 (#3913)
This is an automated email from the ASF dual-hosted git repository.
likyh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 74b1714e8 feat(jenkins): update bp v200 (#3913)
74b1714e8 is described below
commit 74b1714e88f4806bcb34fab62924f714709dca8a
Author: Warren Chen <yi...@merico.dev>
AuthorDate: Mon Dec 12 21:09:20 2022 +0800
feat(jenkins): update bp v200 (#3913)
---
plugins/azure/api/blueprint.go | 6 ++-
plugins/jenkins/api/blueprint_v100.go | 6 ++-
plugins/jenkins/api/blueprint_v200.go | 72 +++++-------------------------
plugins/jenkins/api/blueprint_v200_test.go | 52 ++++-----------------
plugins/jenkins/impl/impl.go | 55 +++++++++++++++--------
plugins/jenkins/tasks/task_data.go | 9 +++-
plugins/jira/impl/impl.go | 4 +-
services/blueprint_makeplan_v200.go | 4 ++
services/blueprint_makeplan_v200_test.go | 2 +-
9 files changed, 81 insertions(+), 129 deletions(-)
diff --git a/plugins/azure/api/blueprint.go b/plugins/azure/api/blueprint.go
index 29d6612bc..0fcbfbccf 100644
--- a/plugins/azure/api/blueprint.go
+++ b/plugins/azure/api/blueprint.go
@@ -37,7 +37,11 @@ func MakePipelinePlan(subtaskMetas []core.SubTaskMeta, connectionId uint64, scop
return nil, errors.Default.Wrap(err, "unable to deserialize pipeline task options")
}
taskOptions["connectionId"] = connectionId
- _, err := tasks.DecodeAndValidateTaskOptions(taskOptions)
+ op, err := tasks.DecodeTaskOptions(taskOptions)
+ if err != nil {
+ return nil, err
+ }
+ _, err = tasks.ValidateTaskOptions(op)
if err != nil {
return nil, err
}
diff --git a/plugins/jenkins/api/blueprint_v100.go b/plugins/jenkins/api/blueprint_v100.go
index 5de27e72c..cb3c2e4e1 100644
--- a/plugins/jenkins/api/blueprint_v100.go
+++ b/plugins/jenkins/api/blueprint_v100.go
@@ -60,7 +60,11 @@ func makePipelinePlanV100(subtaskMetas []core.SubTaskMeta, scope []*core.Bluepri
}
taskOptions["connectionId"] = connection.ID
taskOptions["transformationRules"] = transformationRules
- _, err := tasks.DecodeAndValidateTaskOptions(taskOptions)
+ op, err := tasks.DecodeTaskOptions(taskOptions)
+ if err != nil {
+ return nil, err
+ }
+ _, err = tasks.ValidateTaskOptions(op)
if err != nil {
return nil, err
}
diff --git a/plugins/jenkins/api/blueprint_v200.go b/plugins/jenkins/api/blueprint_v200.go
index b9f2b8894..35c56e101 100644
--- a/plugins/jenkins/api/blueprint_v200.go
+++ b/plugins/jenkins/api/blueprint_v200.go
@@ -18,38 +18,26 @@ limitations under the License.
package api
import (
- goerror "errors"
"fmt"
- "github.com/apache/incubator-devlake/plugins/helper"
- "gorm.io/gorm"
-
"github.com/apache/incubator-devlake/errors"
"github.com/apache/incubator-devlake/models/domainlayer"
"github.com/apache/incubator-devlake/models/domainlayer/devops"
"github.com/apache/incubator-devlake/models/domainlayer/didgen"
"github.com/apache/incubator-devlake/plugins/core"
"github.com/apache/incubator-devlake/plugins/core/dal"
+ "github.com/apache/incubator-devlake/plugins/helper"
"github.com/apache/incubator-devlake/plugins/jenkins/models"
- "github.com/apache/incubator-devlake/plugins/jenkins/tasks"
"github.com/apache/incubator-devlake/utils"
- "github.com/mitchellh/mapstructure"
)
func MakeDataSourcePipelinePlanV200(subtaskMetas []core.SubTaskMeta, connectionId uint64, bpScopes []*core.BlueprintScopeV200) (core.PipelinePlan, []core.Scope, errors.Error) {
- // get the connection info for url
- connection := &models.JenkinsConnection{}
- err := connectionHelper.FirstById(connection, connectionId)
- if err != nil {
- return nil, nil, err
- }
-
plan := make(core.PipelinePlan, len(bpScopes))
- plan, err = makeDataSourcePipelinePlanV200(subtaskMetas, plan, bpScopes, connection)
+ plan, err := makeDataSourcePipelinePlanV200(subtaskMetas, plan, bpScopes, connectionId)
if err != nil {
return nil, nil, err
}
- scopes, err := makeScopesV200(bpScopes, connection)
+ scopes, err := makeScopesV200(bpScopes, connectionId)
if err != nil {
return nil, nil, err
}
@@ -61,47 +49,17 @@ func makeDataSourcePipelinePlanV200(
subtaskMetas []core.SubTaskMeta,
plan core.PipelinePlan,
bpScopes []*core.BlueprintScopeV200,
- connection *models.JenkinsConnection,
+ connectionId uint64,
) (core.PipelinePlan, errors.Error) {
- var err errors.Error
for i, bpScope := range bpScopes {
stage := plan[i]
if stage == nil {
stage = core.PipelineStage{}
}
- jenkinsJob := &models.JenkinsJob{}
- // get repo from db
- err = basicRes.GetDal().First(jenkinsJob,
- dal.Where(`connection_id = ? and full_name = ?`,
- connection.ID, bpScope.Id))
- if err != nil && goerror.Is(err, gorm.ErrRecordNotFound) {
- return nil, errors.Default.Wrap(err, fmt.Sprintf("fail to find jenkinsJob%s", bpScope.Id))
- }
- transformationRule := &models.JenkinsTransformationRule{}
- // get transformation rules from db
- err = basicRes.GetDal().First(transformationRule, dal.Where(`id = ?`,
- jenkinsJob.TransformationRuleId))
- if err != nil && !goerror.Is(err, gorm.ErrRecordNotFound) {
- return nil, err
- }
- // construct task options for jenkins
- var options map[string]interface{}
- err = errors.Convert(mapstructure.Decode(jenkinsJob, &options))
- if err != nil {
- return nil, err
- }
- // make sure task options is valid
- _, err = tasks.DecodeAndValidateTaskOptions(options)
- if err != nil {
- return nil, err
- }
-
- var transformationRuleMap map[string]interface{}
- err = errors.Convert(mapstructure.Decode(transformationRule, &transformationRuleMap))
- if err != nil {
- return nil, err
- }
- options["transformationRules"] = transformationRuleMap
+ // construct task options for Jira
+ options := make(map[string]interface{})
+ options["scopeId"] = bpScope.Id
+ options["connectionId"] = connectionId
subtasks, err := helper.MakePipelinePlanSubtasks(subtaskMetas, bpScope.Entities)
if err != nil {
return nil, err
@@ -117,29 +75,23 @@ func makeDataSourcePipelinePlanV200(
return plan, nil
}
-func makeScopesV200(bpScopes []*core.BlueprintScopeV200, connection *models.JenkinsConnection) ([]core.Scope, errors.Error) {
+func makeScopesV200(bpScopes []*core.BlueprintScopeV200, connectionId uint64) ([]core.Scope, errors.Error) {
scopes := make([]core.Scope, 0)
for _, bpScope := range bpScopes {
jenkinsJob := &models.JenkinsJob{}
// get repo from db
err := basicRes.GetDal().First(jenkinsJob,
dal.Where(`connection_id = ? and full_name = ?`,
- connection.ID, bpScope.Id))
- if err != nil && goerror.Is(err, gorm.ErrRecordNotFound) {
+ connectionId, bpScope.Id))
+ if err != nil {
return nil, errors.Default.Wrap(err, fmt.Sprintf("fail to find jenkinsJob%s", bpScope.Id))
}
- transformationRule := &models.JenkinsTransformationRule{}
- // get transformation rules from db
- err = basicRes.GetDal().First(transformationRule, dal.Where(`id = ?`, jenkinsJob.TransformationRuleId))
- if err != nil && !goerror.Is(err, gorm.ErrRecordNotFound) {
- return nil, err
- }
// add cicd_scope to scopes
if utils.StringsContains(bpScope.Entities, core.DOMAIN_TYPE_CICD) {
scopeCICD := &devops.CicdScope{
DomainEntity: domainlayer.DomainEntity{
- Id: didgen.NewDomainIdGenerator(&models.JenkinsJob{}).Generate(connection.ID, jenkinsJob.FullName),
+ Id: didgen.NewDomainIdGenerator(&models.JenkinsJob{}).Generate(connectionId, jenkinsJob.FullName),
},
Name: jenkinsJob.FullName,
}
diff --git a/plugins/jenkins/api/blueprint_v200_test.go b/plugins/jenkins/api/blueprint_v200_test.go
index 2369e53a0..09b22b507 100644
--- a/plugins/jenkins/api/blueprint_v200_test.go
+++ b/plugins/jenkins/api/blueprint_v200_test.go
@@ -19,11 +19,9 @@ package api
import (
"github.com/apache/incubator-devlake/mocks"
- "github.com/apache/incubator-devlake/models/common"
"github.com/apache/incubator-devlake/models/domainlayer"
"github.com/apache/incubator-devlake/models/domainlayer/devops"
"github.com/apache/incubator-devlake/plugins/core"
- "github.com/apache/incubator-devlake/plugins/helper"
"github.com/apache/incubator-devlake/plugins/jenkins/models"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
@@ -37,28 +35,18 @@ func TestMakeDataSourcePipelinePlanV200(t *testing.T) {
assert.Nil(t, err)
bs := &core.BlueprintScopeV200{
Entities: []string{"CICD"},
- Id: "test",
+ Id: "a/b/ccc",
Name: "",
}
bpScopes := make([]*core.BlueprintScopeV200, 0)
bpScopes = append(bpScopes, bs)
- connection := &models.JenkinsConnection{
- RestConnection: helper.RestConnection{
- BaseConnection: helper.BaseConnection{
- Name: "jenkins",
- Model: common.Model{
- ID: 1,
- },
- },
- },
- }
basicRes = NewMockBasicRes()
plan := make(core.PipelinePlan, len(bpScopes))
- plan, err = makeDataSourcePipelinePlanV200(nil, plan, bpScopes, connection)
+ plan, err = makeDataSourcePipelinePlanV200(nil, plan, bpScopes, 1)
assert.Nil(t, err)
basicRes = NewMockBasicRes()
- scopes, err := makeScopesV200(bpScopes, connection)
+ scopes, err := makeScopesV200(bpScopes, 1)
assert.Nil(t, err)
expectPlan := core.PipelinePlan{
@@ -68,13 +56,8 @@ func TestMakeDataSourcePipelinePlanV200(t *testing.T) {
Subtasks: []string{},
SkipOnFail: false,
Options: map[string]interface{}{
- "connectionId": uint64(1),
- "jobFullName": "a/b/ccc",
- "transformationRuleId": uint64(1),
- "transformationRules": map[string]interface{}{
- "name": "jenkins transformation rule",
- "deploymentPattern": "hey,man,wasup",
- },
+ "connectionId": uint64(1),
+ "scopeId": "a/b/ccc",
},
},
},
@@ -86,10 +69,7 @@ func TestMakeDataSourcePipelinePlanV200(t *testing.T) {
DomainEntity: domainlayer.DomainEntity{
Id: "jenkins:JenkinsJob:1:a/b/ccc",
},
- Name: "a/b/ccc",
- Description: "",
- Url: "",
- CreatedDate: nil,
+ Name: "a/b/ccc",
}
expectScopes = append(expectScopes, scopeCicd)
@@ -99,19 +79,8 @@ func TestMakeDataSourcePipelinePlanV200(t *testing.T) {
// NewMockBasicRes FIXME ...
func NewMockBasicRes() *mocks.BasicRes {
jenkinsJob := &models.JenkinsJob{
- ConnectionId: 1,
- FullName: "a/b/ccc",
- Path: "job/a/job/b/",
- Name: "ccc",
- TransformationRuleId: 1,
- }
-
- transformationRule := &models.JenkinsTransformationRule{
- Model: common.Model{
- ID: 1,
- },
- Name: "jenkins transformation rule",
- DeploymentPattern: "hey,man,wasup",
+ ConnectionId: 1,
+ FullName: "a/b/ccc",
}
mockRes := new(mocks.BasicRes)
mockDal := new(mocks.Dal)
@@ -121,11 +90,6 @@ func NewMockBasicRes() *mocks.BasicRes {
*dst = *jenkinsJob
}).Return(nil).Once()
- mockDal.On("First", mock.Anything, mock.Anything).Run(func(args mock.Arguments) {
- dst := args.Get(0).(*models.JenkinsTransformationRule)
- *dst = *transformationRule
- }).Return(nil).Once()
-
mockRes.On("GetDal").Return(mockDal)
mockRes.On("GetConfig", mock.Anything).Return("")
diff --git a/plugins/jenkins/impl/impl.go b/plugins/jenkins/impl/impl.go
index ed24568f8..35ba87bb6 100644
--- a/plugins/jenkins/impl/impl.go
+++ b/plugins/jenkins/impl/impl.go
@@ -19,12 +19,12 @@ package impl
import (
"fmt"
+ "github.com/apache/incubator-devlake/plugins/core/dal"
"strings"
"time"
"github.com/apache/incubator-devlake/errors"
"github.com/apache/incubator-devlake/plugins/core"
- "github.com/apache/incubator-devlake/plugins/core/dal"
"github.com/apache/incubator-devlake/plugins/helper"
"github.com/apache/incubator-devlake/plugins/jenkins/api"
"github.com/apache/incubator-devlake/plugins/jenkins/models"
@@ -94,16 +94,46 @@ func (plugin Jenkins) SubTaskMetas() []core.SubTaskMeta {
}
}
func (plugin Jenkins) PrepareTaskData(taskCtx core.TaskContext, options map[string]interface{}) (interface{}, errors.Error) {
- op, err := tasks.DecodeAndValidateTaskOptions(options)
- logger := taskCtx.GetLogger()
- logger.Debug("%v", options)
+ // Firstly, let's decode options to JenkinsOptions
+ op, err := tasks.DecodeTaskOptions(options)
if err != nil {
return nil, err
}
- if op.ConnectionId == 0 {
- return nil, errors.BadInput.New("connectionId is invalid")
+ // If this is from BpV200, we should set JobFullName to scopeId
+ if op.JobFullName == "" {
+ op.JobFullName = op.ScopeId
+ }
+ // Validate op and convert JobFullName to JobPath and JobName
+ op, err = tasks.ValidateTaskOptions(op)
+ if err != nil {
+ return nil, err
+ }
+ // get jenkinsJob from db
+ jenkinsJob := &models.JenkinsJob{}
+ err = taskCtx.GetDal().First(jenkinsJob,
+ dal.Where(`connection_id = ? and full_name = ?`,
+ op.ConnectionId, op.ScopeId))
+ if err != nil {
+ return nil, errors.Default.Wrap(err, fmt.Sprintf("fail to find jenkinsJob%s", op.ScopeId))
+ }
+ if op.TransformationRuleId == 0 {
+ op.TransformationRuleId = jenkinsJob.TransformationRuleId
}
+ if !strings.HasSuffix(op.JobPath, "/") {
+ op.JobPath = fmt.Sprintf("%s/", op.JobPath)
+ }
+ // We only set op.JenkinsTransformationRule when it's nil and we have op.TransformationRuleId != 0
+ if op.JenkinsTransformationRule == nil && op.TransformationRuleId != 0 {
+ var transformationRule models.JenkinsTransformationRule
+ err = taskCtx.GetDal().First(&transformationRule, dal.Where("id = ?", op.TransformationRuleId))
+ if err != nil {
+ return nil, errors.BadInput.Wrap(err, "fail to get transformationRule")
+ }
+ op.JenkinsTransformationRule = &transformationRule
+ }
+ logger := taskCtx.GetLogger()
+ logger.Debug("%v", options)
connection := &models.JenkinsConnection{}
connectionHelper := helper.NewConnectionHelper(
taskCtx,
@@ -129,19 +159,6 @@ func (plugin Jenkins) PrepareTaskData(taskCtx core.TaskContext, options map[stri
return nil, errors.BadInput.Wrap(err, "invalid value for `createdDateAfter`")
}
}
-
- if !strings.HasSuffix(op.JobPath, "/") {
- op.JobPath = fmt.Sprintf("%s/", op.JobPath)
- }
-
- if op.JenkinsTransformationRule == nil && op.TransformationRuleId != 0 {
- var transformationRule models.JenkinsTransformationRule
- err = taskCtx.GetDal().First(&transformationRule, dal.Where("id = ?", op.TransformationRuleId))
- if err != nil {
- return nil, errors.BadInput.Wrap(err, "fail to get transformationRule")
- }
- op.JenkinsTransformationRule = &transformationRule
- }
taskData := &tasks.JenkinsTaskData{
Options: op,
ApiClient: apiClient,
diff --git a/plugins/jenkins/tasks/task_data.go b/plugins/jenkins/tasks/task_data.go
index 6f8ddbd74..4b481ecb8 100644
--- a/plugins/jenkins/tasks/task_data.go
+++ b/plugins/jenkins/tasks/task_data.go
@@ -33,6 +33,7 @@ type JenkinsApiParams struct {
type JenkinsOptions struct {
ConnectionId uint64 `json:"connectionId"`
+ ScopeId string
TransformationRuleId uint64 `json:"transformationRuleId"`
JobFullName string `json:"jobFullName"` // "path1/path2/job name"
JobName string `json:"jobName"` // "job name"
@@ -49,12 +50,16 @@ type JenkinsTaskData struct {
CreatedDateAfter *time.Time
}
-func DecodeAndValidateTaskOptions(options map[string]interface{}) (*JenkinsOptions, errors.Error) {
+func DecodeTaskOptions(options map[string]interface{}) (*JenkinsOptions, errors.Error) {
var op JenkinsOptions
err := helper.Decode(options, &op, nil)
if err != nil {
return nil, errors.BadInput.Wrap(err, "could not decode request parameters")
}
+ return &op, nil
+}
+
+func ValidateTaskOptions(op *JenkinsOptions) (*JenkinsOptions, errors.Error) {
if op.ConnectionId == 0 {
return nil, errors.BadInput.New("connectionId is invalid")
}
@@ -71,5 +76,5 @@ func DecodeAndValidateTaskOptions(options map[string]interface{}) (*JenkinsOptio
if op.JenkinsTransformationRule == nil && op.TransformationRuleId == 0 {
op.JenkinsTransformationRule = new(models.JenkinsTransformationRule)
}
- return &op, nil
+ return op, nil
}
diff --git a/plugins/jira/impl/impl.go b/plugins/jira/impl/impl.go
index d18fcc53d..7c5d8e262 100644
--- a/plugins/jira/impl/impl.go
+++ b/plugins/jira/impl/impl.go
@@ -177,7 +177,9 @@ func (plugin Jira) PrepareTaskData(taskCtx core.TaskContext, options map[string]
return nil, errors.Default.Wrap(err, fmt.Sprintf("fail to find board%s", op.ScopeId))
}
op.BoardId = jiraBoard.BoardId
- op.TransformationRuleId = jiraBoard.TransformationRuleId
+ if op.TransformationRuleId == 0 {
+ op.TransformationRuleId = jiraBoard.TransformationRuleId
+ }
}
if op.TransformationRules == nil && op.TransformationRuleId != 0 {
var transformationRule models.JiraTransformationRule
diff --git a/services/blueprint_makeplan_v200.go b/services/blueprint_makeplan_v200.go
index d28b22d78..ac78cb58f 100644
--- a/services/blueprint_makeplan_v200.go
+++ b/services/blueprint_makeplan_v200.go
@@ -106,6 +106,10 @@ func genPlanJsonV200(
return nil, nil, err
}
if pluginBp, ok := plugin.(core.MetricPluginBlueprintV200); ok {
+ // If we enable one metric plugin, even if it has nil option, we still process it
+ if len(metricPluginOptJson) == 0 {
+ metricPluginOptJson = json.RawMessage("{}")
+ }
metricPlans[i], err = pluginBp.MakeMetricPluginPipelinePlanV200(projectName, metricPluginOptJson)
if err != nil {
return nil, nil, err
diff --git a/services/blueprint_makeplan_v200_test.go b/services/blueprint_makeplan_v200_test.go
index 7cee6bf88..21a646066 100644
--- a/services/blueprint_makeplan_v200_test.go
+++ b/services/blueprint_makeplan_v200_test.go
@@ -65,7 +65,7 @@ func TestMakePlanV200(t *testing.T) {
},
}
dora := new(mocks.CompositeMetricPluginBlueprintV200)
- dora.On("MakeMetricPluginPipelinePlanV200", projectName, json.RawMessage(nil)).Return(doraOutputPlan, nil)
+ dora.On("MakeMetricPluginPipelinePlanV200", projectName, json.RawMessage("{}")).Return(doraOutputPlan, nil)
// expectation, establish expectation before any code being launch to avoid unwanted modification
expectedPlan := make(core.PipelinePlan, 0)