You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by li...@apache.org on 2022/12/12 13:09:25 UTC

[incubator-devlake] branch main updated: feat(jenkins): update bp v200 (#3913)

This is an automated email from the ASF dual-hosted git repository.

likyh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git


The following commit(s) were added to refs/heads/main by this push:
     new 74b1714e8 feat(jenkins): update bp v200 (#3913)
74b1714e8 is described below

commit 74b1714e88f4806bcb34fab62924f714709dca8a
Author: Warren Chen <yi...@merico.dev>
AuthorDate: Mon Dec 12 21:09:20 2022 +0800

    feat(jenkins): update bp v200 (#3913)
---
 plugins/azure/api/blueprint.go             |  6 ++-
 plugins/jenkins/api/blueprint_v100.go      |  6 ++-
 plugins/jenkins/api/blueprint_v200.go      | 72 +++++-------------------------
 plugins/jenkins/api/blueprint_v200_test.go | 52 ++++-----------------
 plugins/jenkins/impl/impl.go               | 55 +++++++++++++++--------
 plugins/jenkins/tasks/task_data.go         |  9 +++-
 plugins/jira/impl/impl.go                  |  4 +-
 services/blueprint_makeplan_v200.go        |  4 ++
 services/blueprint_makeplan_v200_test.go   |  2 +-
 9 files changed, 81 insertions(+), 129 deletions(-)

diff --git a/plugins/azure/api/blueprint.go b/plugins/azure/api/blueprint.go
index 29d6612bc..0fcbfbccf 100644
--- a/plugins/azure/api/blueprint.go
+++ b/plugins/azure/api/blueprint.go
@@ -37,7 +37,11 @@ func MakePipelinePlan(subtaskMetas []core.SubTaskMeta, connectionId uint64, scop
 			return nil, errors.Default.Wrap(err, "unable to deserialize pipeline task options")
 		}
 		taskOptions["connectionId"] = connectionId
-		_, err := tasks.DecodeAndValidateTaskOptions(taskOptions)
+		op, err := tasks.DecodeTaskOptions(taskOptions)
+		if err != nil {
+			return nil, err
+		}
+		_, err = tasks.ValidateTaskOptions(op)
 		if err != nil {
 			return nil, err
 		}
diff --git a/plugins/jenkins/api/blueprint_v100.go b/plugins/jenkins/api/blueprint_v100.go
index 5de27e72c..cb3c2e4e1 100644
--- a/plugins/jenkins/api/blueprint_v100.go
+++ b/plugins/jenkins/api/blueprint_v100.go
@@ -60,7 +60,11 @@ func makePipelinePlanV100(subtaskMetas []core.SubTaskMeta, scope []*core.Bluepri
 		}
 		taskOptions["connectionId"] = connection.ID
 		taskOptions["transformationRules"] = transformationRules
-		_, err := tasks.DecodeAndValidateTaskOptions(taskOptions)
+		op, err := tasks.DecodeTaskOptions(taskOptions)
+		if err != nil {
+			return nil, err
+		}
+		_, err = tasks.ValidateTaskOptions(op)
 		if err != nil {
 			return nil, err
 		}
diff --git a/plugins/jenkins/api/blueprint_v200.go b/plugins/jenkins/api/blueprint_v200.go
index b9f2b8894..35c56e101 100644
--- a/plugins/jenkins/api/blueprint_v200.go
+++ b/plugins/jenkins/api/blueprint_v200.go
@@ -18,38 +18,26 @@ limitations under the License.
 package api
 
 import (
-	goerror "errors"
 	"fmt"
 
-	"github.com/apache/incubator-devlake/plugins/helper"
-	"gorm.io/gorm"
-
 	"github.com/apache/incubator-devlake/errors"
 	"github.com/apache/incubator-devlake/models/domainlayer"
 	"github.com/apache/incubator-devlake/models/domainlayer/devops"
 	"github.com/apache/incubator-devlake/models/domainlayer/didgen"
 	"github.com/apache/incubator-devlake/plugins/core"
 	"github.com/apache/incubator-devlake/plugins/core/dal"
+	"github.com/apache/incubator-devlake/plugins/helper"
 	"github.com/apache/incubator-devlake/plugins/jenkins/models"
-	"github.com/apache/incubator-devlake/plugins/jenkins/tasks"
 	"github.com/apache/incubator-devlake/utils"
-	"github.com/mitchellh/mapstructure"
 )
 
 func MakeDataSourcePipelinePlanV200(subtaskMetas []core.SubTaskMeta, connectionId uint64, bpScopes []*core.BlueprintScopeV200) (core.PipelinePlan, []core.Scope, errors.Error) {
-	// get the connection info for url
-	connection := &models.JenkinsConnection{}
-	err := connectionHelper.FirstById(connection, connectionId)
-	if err != nil {
-		return nil, nil, err
-	}
-
 	plan := make(core.PipelinePlan, len(bpScopes))
-	plan, err = makeDataSourcePipelinePlanV200(subtaskMetas, plan, bpScopes, connection)
+	plan, err := makeDataSourcePipelinePlanV200(subtaskMetas, plan, bpScopes, connectionId)
 	if err != nil {
 		return nil, nil, err
 	}
-	scopes, err := makeScopesV200(bpScopes, connection)
+	scopes, err := makeScopesV200(bpScopes, connectionId)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -61,47 +49,17 @@ func makeDataSourcePipelinePlanV200(
 	subtaskMetas []core.SubTaskMeta,
 	plan core.PipelinePlan,
 	bpScopes []*core.BlueprintScopeV200,
-	connection *models.JenkinsConnection,
+	connectionId uint64,
 ) (core.PipelinePlan, errors.Error) {
-	var err errors.Error
 	for i, bpScope := range bpScopes {
 		stage := plan[i]
 		if stage == nil {
 			stage = core.PipelineStage{}
 		}
-		jenkinsJob := &models.JenkinsJob{}
-		// get repo from db
-		err = basicRes.GetDal().First(jenkinsJob,
-			dal.Where(`connection_id = ? and full_name = ?`,
-				connection.ID, bpScope.Id))
-		if err != nil && goerror.Is(err, gorm.ErrRecordNotFound) {
-			return nil, errors.Default.Wrap(err, fmt.Sprintf("fail to find jenkinsJob%s", bpScope.Id))
-		}
-		transformationRule := &models.JenkinsTransformationRule{}
-		// get transformation rules from db
-		err = basicRes.GetDal().First(transformationRule, dal.Where(`id = ?`,
-			jenkinsJob.TransformationRuleId))
-		if err != nil && !goerror.Is(err, gorm.ErrRecordNotFound) {
-			return nil, err
-		}
-		// construct task options for jenkins
-		var options map[string]interface{}
-		err = errors.Convert(mapstructure.Decode(jenkinsJob, &options))
-		if err != nil {
-			return nil, err
-		}
-		// make sure task options is valid
-		_, err = tasks.DecodeAndValidateTaskOptions(options)
-		if err != nil {
-			return nil, err
-		}
-
-		var transformationRuleMap map[string]interface{}
-		err = errors.Convert(mapstructure.Decode(transformationRule, &transformationRuleMap))
-		if err != nil {
-			return nil, err
-		}
-		options["transformationRules"] = transformationRuleMap
+		// construct task options for Jira
+		options := make(map[string]interface{})
+		options["scopeId"] = bpScope.Id
+		options["connectionId"] = connectionId
 		subtasks, err := helper.MakePipelinePlanSubtasks(subtaskMetas, bpScope.Entities)
 		if err != nil {
 			return nil, err
@@ -117,29 +75,23 @@ func makeDataSourcePipelinePlanV200(
 	return plan, nil
 }
 
-func makeScopesV200(bpScopes []*core.BlueprintScopeV200, connection *models.JenkinsConnection) ([]core.Scope, errors.Error) {
+func makeScopesV200(bpScopes []*core.BlueprintScopeV200, connectionId uint64) ([]core.Scope, errors.Error) {
 	scopes := make([]core.Scope, 0)
 	for _, bpScope := range bpScopes {
 		jenkinsJob := &models.JenkinsJob{}
 		// get repo from db
 		err := basicRes.GetDal().First(jenkinsJob,
 			dal.Where(`connection_id = ? and full_name = ?`,
-				connection.ID, bpScope.Id))
-		if err != nil && goerror.Is(err, gorm.ErrRecordNotFound) {
+				connectionId, bpScope.Id))
+		if err != nil {
 			return nil, errors.Default.Wrap(err, fmt.Sprintf("fail to find jenkinsJob%s", bpScope.Id))
 		}
 
-		transformationRule := &models.JenkinsTransformationRule{}
-		// get transformation rules from db
-		err = basicRes.GetDal().First(transformationRule, dal.Where(`id = ?`, jenkinsJob.TransformationRuleId))
-		if err != nil && !goerror.Is(err, gorm.ErrRecordNotFound) {
-			return nil, err
-		}
 		// add cicd_scope to scopes
 		if utils.StringsContains(bpScope.Entities, core.DOMAIN_TYPE_CICD) {
 			scopeCICD := &devops.CicdScope{
 				DomainEntity: domainlayer.DomainEntity{
-					Id: didgen.NewDomainIdGenerator(&models.JenkinsJob{}).Generate(connection.ID, jenkinsJob.FullName),
+					Id: didgen.NewDomainIdGenerator(&models.JenkinsJob{}).Generate(connectionId, jenkinsJob.FullName),
 				},
 				Name: jenkinsJob.FullName,
 			}
diff --git a/plugins/jenkins/api/blueprint_v200_test.go b/plugins/jenkins/api/blueprint_v200_test.go
index 2369e53a0..09b22b507 100644
--- a/plugins/jenkins/api/blueprint_v200_test.go
+++ b/plugins/jenkins/api/blueprint_v200_test.go
@@ -19,11 +19,9 @@ package api
 
 import (
 	"github.com/apache/incubator-devlake/mocks"
-	"github.com/apache/incubator-devlake/models/common"
 	"github.com/apache/incubator-devlake/models/domainlayer"
 	"github.com/apache/incubator-devlake/models/domainlayer/devops"
 	"github.com/apache/incubator-devlake/plugins/core"
-	"github.com/apache/incubator-devlake/plugins/helper"
 	"github.com/apache/incubator-devlake/plugins/jenkins/models"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/mock"
@@ -37,28 +35,18 @@ func TestMakeDataSourcePipelinePlanV200(t *testing.T) {
 	assert.Nil(t, err)
 	bs := &core.BlueprintScopeV200{
 		Entities: []string{"CICD"},
-		Id:       "test",
+		Id:       "a/b/ccc",
 		Name:     "",
 	}
 	bpScopes := make([]*core.BlueprintScopeV200, 0)
 	bpScopes = append(bpScopes, bs)
 
-	connection := &models.JenkinsConnection{
-		RestConnection: helper.RestConnection{
-			BaseConnection: helper.BaseConnection{
-				Name: "jenkins",
-				Model: common.Model{
-					ID: 1,
-				},
-			},
-		},
-	}
 	basicRes = NewMockBasicRes()
 	plan := make(core.PipelinePlan, len(bpScopes))
-	plan, err = makeDataSourcePipelinePlanV200(nil, plan, bpScopes, connection)
+	plan, err = makeDataSourcePipelinePlanV200(nil, plan, bpScopes, 1)
 	assert.Nil(t, err)
 	basicRes = NewMockBasicRes()
-	scopes, err := makeScopesV200(bpScopes, connection)
+	scopes, err := makeScopesV200(bpScopes, 1)
 	assert.Nil(t, err)
 
 	expectPlan := core.PipelinePlan{
@@ -68,13 +56,8 @@ func TestMakeDataSourcePipelinePlanV200(t *testing.T) {
 				Subtasks:   []string{},
 				SkipOnFail: false,
 				Options: map[string]interface{}{
-					"connectionId":         uint64(1),
-					"jobFullName":          "a/b/ccc",
-					"transformationRuleId": uint64(1),
-					"transformationRules": map[string]interface{}{
-						"name":              "jenkins transformation rule",
-						"deploymentPattern": "hey,man,wasup",
-					},
+					"connectionId": uint64(1),
+					"scopeId":      "a/b/ccc",
 				},
 			},
 		},
@@ -86,10 +69,7 @@ func TestMakeDataSourcePipelinePlanV200(t *testing.T) {
 		DomainEntity: domainlayer.DomainEntity{
 			Id: "jenkins:JenkinsJob:1:a/b/ccc",
 		},
-		Name:        "a/b/ccc",
-		Description: "",
-		Url:         "",
-		CreatedDate: nil,
+		Name: "a/b/ccc",
 	}
 
 	expectScopes = append(expectScopes, scopeCicd)
@@ -99,19 +79,8 @@ func TestMakeDataSourcePipelinePlanV200(t *testing.T) {
 // NewMockBasicRes FIXME ...
 func NewMockBasicRes() *mocks.BasicRes {
 	jenkinsJob := &models.JenkinsJob{
-		ConnectionId:         1,
-		FullName:             "a/b/ccc",
-		Path:                 "job/a/job/b/",
-		Name:                 "ccc",
-		TransformationRuleId: 1,
-	}
-
-	transformationRule := &models.JenkinsTransformationRule{
-		Model: common.Model{
-			ID: 1,
-		},
-		Name:              "jenkins transformation rule",
-		DeploymentPattern: "hey,man,wasup",
+		ConnectionId: 1,
+		FullName:     "a/b/ccc",
 	}
 	mockRes := new(mocks.BasicRes)
 	mockDal := new(mocks.Dal)
@@ -121,11 +90,6 @@ func NewMockBasicRes() *mocks.BasicRes {
 		*dst = *jenkinsJob
 	}).Return(nil).Once()
 
-	mockDal.On("First", mock.Anything, mock.Anything).Run(func(args mock.Arguments) {
-		dst := args.Get(0).(*models.JenkinsTransformationRule)
-		*dst = *transformationRule
-	}).Return(nil).Once()
-
 	mockRes.On("GetDal").Return(mockDal)
 	mockRes.On("GetConfig", mock.Anything).Return("")
 
diff --git a/plugins/jenkins/impl/impl.go b/plugins/jenkins/impl/impl.go
index ed24568f8..35ba87bb6 100644
--- a/plugins/jenkins/impl/impl.go
+++ b/plugins/jenkins/impl/impl.go
@@ -19,12 +19,12 @@ package impl
 
 import (
 	"fmt"
+	"github.com/apache/incubator-devlake/plugins/core/dal"
 	"strings"
 	"time"
 
 	"github.com/apache/incubator-devlake/errors"
 	"github.com/apache/incubator-devlake/plugins/core"
-	"github.com/apache/incubator-devlake/plugins/core/dal"
 	"github.com/apache/incubator-devlake/plugins/helper"
 	"github.com/apache/incubator-devlake/plugins/jenkins/api"
 	"github.com/apache/incubator-devlake/plugins/jenkins/models"
@@ -94,16 +94,46 @@ func (plugin Jenkins) SubTaskMetas() []core.SubTaskMeta {
 	}
 }
 func (plugin Jenkins) PrepareTaskData(taskCtx core.TaskContext, options map[string]interface{}) (interface{}, errors.Error) {
-	op, err := tasks.DecodeAndValidateTaskOptions(options)
-	logger := taskCtx.GetLogger()
-	logger.Debug("%v", options)
+	// Firstly, let's decode options to JenkinsOptions
+	op, err := tasks.DecodeTaskOptions(options)
 	if err != nil {
 		return nil, err
 	}
-	if op.ConnectionId == 0 {
-		return nil, errors.BadInput.New("connectionId is invalid")
+	// If this is from BpV200, we should set JobFullName to scopeId
+	if op.JobFullName == "" {
+		op.JobFullName = op.ScopeId
+	}
+	// Validate op and convert JobFullName to JobPath and JobName
+	op, err = tasks.ValidateTaskOptions(op)
+	if err != nil {
+		return nil, err
+	}
+	// get jenkinsJob from db
+	jenkinsJob := &models.JenkinsJob{}
+	err = taskCtx.GetDal().First(jenkinsJob,
+		dal.Where(`connection_id = ? and full_name = ?`,
+			op.ConnectionId, op.ScopeId))
+	if err != nil {
+		return nil, errors.Default.Wrap(err, fmt.Sprintf("fail to find jenkinsJob%s", op.ScopeId))
+	}
+	if op.TransformationRuleId == 0 {
+		op.TransformationRuleId = jenkinsJob.TransformationRuleId
 	}
 
+	if !strings.HasSuffix(op.JobPath, "/") {
+		op.JobPath = fmt.Sprintf("%s/", op.JobPath)
+	}
+	// We only set op.JenkinsTransformationRule when it's nil and we have op.TransformationRuleId != 0
+	if op.JenkinsTransformationRule == nil && op.TransformationRuleId != 0 {
+		var transformationRule models.JenkinsTransformationRule
+		err = taskCtx.GetDal().First(&transformationRule, dal.Where("id = ?", op.TransformationRuleId))
+		if err != nil {
+			return nil, errors.BadInput.Wrap(err, "fail to get transformationRule")
+		}
+		op.JenkinsTransformationRule = &transformationRule
+	}
+	logger := taskCtx.GetLogger()
+	logger.Debug("%v", options)
 	connection := &models.JenkinsConnection{}
 	connectionHelper := helper.NewConnectionHelper(
 		taskCtx,
@@ -129,19 +159,6 @@ func (plugin Jenkins) PrepareTaskData(taskCtx core.TaskContext, options map[stri
 			return nil, errors.BadInput.Wrap(err, "invalid value for `createdDateAfter`")
 		}
 	}
-
-	if !strings.HasSuffix(op.JobPath, "/") {
-		op.JobPath = fmt.Sprintf("%s/", op.JobPath)
-	}
-
-	if op.JenkinsTransformationRule == nil && op.TransformationRuleId != 0 {
-		var transformationRule models.JenkinsTransformationRule
-		err = taskCtx.GetDal().First(&transformationRule, dal.Where("id = ?", op.TransformationRuleId))
-		if err != nil {
-			return nil, errors.BadInput.Wrap(err, "fail to get transformationRule")
-		}
-		op.JenkinsTransformationRule = &transformationRule
-	}
 	taskData := &tasks.JenkinsTaskData{
 		Options:    op,
 		ApiClient:  apiClient,
diff --git a/plugins/jenkins/tasks/task_data.go b/plugins/jenkins/tasks/task_data.go
index 6f8ddbd74..4b481ecb8 100644
--- a/plugins/jenkins/tasks/task_data.go
+++ b/plugins/jenkins/tasks/task_data.go
@@ -33,6 +33,7 @@ type JenkinsApiParams struct {
 
 type JenkinsOptions struct {
 	ConnectionId                      uint64 `json:"connectionId"`
+	ScopeId                           string
 	TransformationRuleId              uint64 `json:"transformationRuleId"`
 	JobFullName                       string `json:"jobFullName"` // "path1/path2/job name"
 	JobName                           string `json:"jobName"`     // "job name"
@@ -49,12 +50,16 @@ type JenkinsTaskData struct {
 	CreatedDateAfter *time.Time
 }
 
-func DecodeAndValidateTaskOptions(options map[string]interface{}) (*JenkinsOptions, errors.Error) {
+func DecodeTaskOptions(options map[string]interface{}) (*JenkinsOptions, errors.Error) {
 	var op JenkinsOptions
 	err := helper.Decode(options, &op, nil)
 	if err != nil {
 		return nil, errors.BadInput.Wrap(err, "could not decode request parameters")
 	}
+	return &op, nil
+}
+
+func ValidateTaskOptions(op *JenkinsOptions) (*JenkinsOptions, errors.Error) {
 	if op.ConnectionId == 0 {
 		return nil, errors.BadInput.New("connectionId is invalid")
 	}
@@ -71,5 +76,5 @@ func DecodeAndValidateTaskOptions(options map[string]interface{}) (*JenkinsOptio
 	if op.JenkinsTransformationRule == nil && op.TransformationRuleId == 0 {
 		op.JenkinsTransformationRule = new(models.JenkinsTransformationRule)
 	}
-	return &op, nil
+	return op, nil
 }
diff --git a/plugins/jira/impl/impl.go b/plugins/jira/impl/impl.go
index d18fcc53d..7c5d8e262 100644
--- a/plugins/jira/impl/impl.go
+++ b/plugins/jira/impl/impl.go
@@ -177,7 +177,9 @@ func (plugin Jira) PrepareTaskData(taskCtx core.TaskContext, options map[string]
 			return nil, errors.Default.Wrap(err, fmt.Sprintf("fail to find board%s", op.ScopeId))
 		}
 		op.BoardId = jiraBoard.BoardId
-		op.TransformationRuleId = jiraBoard.TransformationRuleId
+		if op.TransformationRuleId == 0 {
+			op.TransformationRuleId = jiraBoard.TransformationRuleId
+		}
 	}
 	if op.TransformationRules == nil && op.TransformationRuleId != 0 {
 		var transformationRule models.JiraTransformationRule
diff --git a/services/blueprint_makeplan_v200.go b/services/blueprint_makeplan_v200.go
index d28b22d78..ac78cb58f 100644
--- a/services/blueprint_makeplan_v200.go
+++ b/services/blueprint_makeplan_v200.go
@@ -106,6 +106,10 @@ func genPlanJsonV200(
 			return nil, nil, err
 		}
 		if pluginBp, ok := plugin.(core.MetricPluginBlueprintV200); ok {
+			// If we enable one metric plugin, even if it has nil option, we still process it
+			if len(metricPluginOptJson) == 0 {
+				metricPluginOptJson = json.RawMessage("{}")
+			}
 			metricPlans[i], err = pluginBp.MakeMetricPluginPipelinePlanV200(projectName, metricPluginOptJson)
 			if err != nil {
 				return nil, nil, err
diff --git a/services/blueprint_makeplan_v200_test.go b/services/blueprint_makeplan_v200_test.go
index 7cee6bf88..21a646066 100644
--- a/services/blueprint_makeplan_v200_test.go
+++ b/services/blueprint_makeplan_v200_test.go
@@ -65,7 +65,7 @@ func TestMakePlanV200(t *testing.T) {
 		},
 	}
 	dora := new(mocks.CompositeMetricPluginBlueprintV200)
-	dora.On("MakeMetricPluginPipelinePlanV200", projectName, json.RawMessage(nil)).Return(doraOutputPlan, nil)
+	dora.On("MakeMetricPluginPipelinePlanV200", projectName, json.RawMessage("{}")).Return(doraOutputPlan, nil)
 
 	// expectation, establish expectation before any code being launch to avoid unwanted modification
 	expectedPlan := make(core.PipelinePlan, 0)