You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by ma...@apache.org on 2022/09/20 09:16:03 UTC
[incubator-devlake] branch main updated: feat(github): add dora enrich (#3128)
This is an automated email from the ASF dual-hosted git repository.
mappjzc pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 7fa174c0 feat(github): add dora enrich (#3128)
7fa174c0 is described below
commit 7fa174c0ac1d8de23e60aa64af8371520dcc7087
Author: Warren Chen <yi...@merico.dev>
AuthorDate: Tue Sep 20 17:15:58 2022 +0800
feat(github): add dora enrich (#3128)
* feat(github): add dora enrich
relate to #3127
* feat(github): add unit test for makePipeline
* fix(github): modify unit test
Co-authored-by: mappjzc <zh...@merico.dev>
---
Makefile | 1 +
plugins/github/api/blueprint.go | 131 +++++++++++++++++++++++++----------
plugins/github/api/blueprint_test.go | 96 +++++++++++++++++++++++++
services/blueprint.go | 24 ++++++-
4 files changed, 215 insertions(+), 37 deletions(-)
diff --git a/Makefile b/Makefile
index 6bd35581..b9b70ebf 100644
--- a/Makefile
+++ b/Makefile
@@ -94,6 +94,7 @@ mock:
mockery --dir=./plugins/core --unroll-variadic=false --name='.*'
mockery --dir=./plugins/core/dal --unroll-variadic=false --name='.*'
mockery --dir=./plugins/helper --unroll-variadic=false --name='.*'
+ mockery --dir=./plugins/github/api --unroll-variadic=false --name='.*'
test: unit-test e2e-test
diff --git a/plugins/github/api/blueprint.go b/plugins/github/api/blueprint.go
index a5e77129..55f23592 100644
--- a/plugins/github/api/blueprint.go
+++ b/plugins/github/api/blueprint.go
@@ -37,7 +37,17 @@ import (
)
func MakePipelinePlan(subtaskMetas []core.SubTaskMeta, connectionId uint64, scope []*core.BlueprintScopeV100) (core.PipelinePlan, errors.Error) {
+ repoHelper := RepoHelper{}
+ plan, err := DoMakePipeline(subtaskMetas, connectionId, scope, repoHelper)
+ if err != nil {
+ return nil, err
+ }
+ return plan, nil
+}
+
+func DoMakePipeline(subtaskMetas []core.SubTaskMeta, connectionId uint64, scope []*core.BlueprintScopeV100, repoHelper BpHelper) (core.PipelinePlan, errors.Error) {
var err errors.Error
+ apiRepo, _, _ := &tasks.GithubApiRepo{}, "", ""
plan := make(core.PipelinePlan, len(scope))
for i, scopeElem := range scope {
// handle taskOptions and transformationRules, by dumping them to taskOptions
@@ -94,43 +104,11 @@ func MakePipelinePlan(subtaskMetas []core.SubTaskMeta, connectionId uint64, scop
// collect git data by gitextractor if CODE was requested
if utils.StringsContains(scopeElem.Entities, core.DOMAIN_TYPE_CODE) {
// here is the tricky part, we have to obtain the repo id beforehand
- connection := new(models.GithubConnection)
- err = connectionHelper.FirstById(connection, connectionId)
- if err != nil {
- return nil, err
- }
- token := strings.Split(connection.Token, ",")[0]
- apiClient, err := helper.NewApiClient(
- context.TODO(),
- connection.Endpoint,
- map[string]string{
- "Authorization": fmt.Sprintf("Bearer %s", token),
- },
- 10*time.Second,
- connection.Proxy,
- basicRes,
- )
- if err != nil {
- return nil, err
- }
- res, err := apiClient.Get(fmt.Sprintf("repos/%s/%s", op.Owner, op.Repo), nil, nil)
+ res, token, proxy, err := repoHelper.GetApiRepo(connectionId, op)
+ apiRepo = res.(*tasks.GithubApiRepo)
if err != nil {
return nil, err
}
- defer res.Body.Close()
- if res.StatusCode != http.StatusOK {
- return nil, errors.HttpStatus(res.StatusCode).New(fmt.Sprintf(
- "unexpected status code when requesting repo detail from %s", res.Request.URL.String()))
- }
- body, err := errors.Convert01(io.ReadAll(res.Body))
- if err != nil {
- return nil, err
- }
- apiRepo := new(tasks.GithubApiRepo)
- err = errors.Convert(json.Unmarshal(body, apiRepo))
- if err != nil {
- return nil, errors.Convert(err)
- }
cloneUrl, err := errors.Convert01(url.Parse(apiRepo.CloneUrl))
if err != nil {
return nil, err
@@ -141,11 +119,94 @@ func MakePipelinePlan(subtaskMetas []core.SubTaskMeta, connectionId uint64, scop
Options: map[string]interface{}{
"url": cloneUrl.String(),
"repoId": didgen.NewDomainIdGenerator(&models.GithubRepo{}).Generate(connectionId, apiRepo.GithubId),
- "proxy": connection.Proxy,
+ "proxy": proxy,
},
})
}
plan[i] = stage
+ // dora
+ if doraRules, ok := transformationRules["dora"]; ok && doraRules != nil {
+ j := i + 1
+ // add a new task to next stage
+ if plan[j] != nil {
+ j++
+ }
+ if j == len(plan) {
+ plan = append(plan, nil)
+ }
+ if err != nil {
+ return nil, err
+ }
+ if apiRepo.GithubId == 0 {
+ res, _, _, err := repoHelper.GetApiRepo(connectionId, op)
+ if err != nil {
+ return nil, err
+ }
+ apiRepo = res.(*tasks.GithubApiRepo)
+
+ }
+ doraOption := make(map[string]interface{})
+ doraOption["repoId"] = didgen.NewDomainIdGenerator(&models.GithubRepo{}).Generate(connectionId, apiRepo.GithubId)
+ doraOption["tasks"] = []string{"EnrichTaskEnv"}
+ doraOption["transformation"] = doraRules
+ plan[j] = core.PipelineStage{
+ {
+ Plugin: "dora",
+ Options: doraOption,
+ },
+ }
+ // remove it from github transformationRules
+ delete(transformationRules, "dora")
+ }
}
return plan, nil
}
+
+type BpHelper interface {
+ GetApiRepo(connectionId uint64, op interface{}) (interface{}, string, string, errors.Error)
+}
+
+type RepoHelper struct{}
+
+func (c RepoHelper) GetApiRepo(connectionId uint64, originOp interface{}) (interface{}, string, string, errors.Error) {
+ op := originOp.(*tasks.GithubOptions)
+ // here is the tricky part, we have to obtain the repo id beforehand
+ connection := new(models.GithubConnection)
+ err := connectionHelper.FirstById(connection, connectionId)
+ if err != nil {
+ return nil, "", "", err
+ }
+ token := strings.Split(connection.Token, ",")[0]
+ apiClient, err := helper.NewApiClient(
+ context.TODO(),
+ connection.Endpoint,
+ map[string]string{
+ "Authorization": fmt.Sprintf("Bearer %s", token),
+ },
+ 10*time.Second,
+ connection.Proxy,
+ basicRes,
+ )
+ if err != nil {
+ return nil, "", "", err
+ }
+ res, err := apiClient.Get(fmt.Sprintf("repos/%s/%s", op.Owner, op.Repo), nil, nil)
+ if err != nil {
+ return nil, "", "", err
+ }
+ defer res.Body.Close()
+ if res.StatusCode != http.StatusOK {
+ return nil, "", "", errors.HttpStatus(res.StatusCode).New(fmt.Sprintf(
+ "unexpected status code when requesting repo detail from %s", res.Request.URL.String()))
+ }
+ body, err := errors.Convert01(io.ReadAll(res.Body))
+ if err != nil {
+ return nil, "", "", err
+ }
+ apiRepo := new(tasks.GithubApiRepo)
+ err = errors.Convert(json.Unmarshal(body, apiRepo))
+ if err != nil {
+ return nil, "", "", errors.Convert(err)
+ }
+ return apiRepo, token, connection.Proxy, nil
+}
diff --git a/plugins/github/api/blueprint_test.go b/plugins/github/api/blueprint_test.go
new file mode 100644
index 00000000..4df95d31
--- /dev/null
+++ b/plugins/github/api/blueprint_test.go
@@ -0,0 +1,96 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package api
+
+import (
+ "encoding/json"
+ "fmt"
+ "github.com/apache/incubator-devlake/mocks"
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/github/models"
+ "github.com/apache/incubator-devlake/plugins/github/tasks"
+ "github.com/spf13/cobra"
+ "github.com/stretchr/testify/assert"
+ "testing"
+)
+
+func TestMakePipelinePlan(t *testing.T) {
+ cmd := &cobra.Command{Use: "github"}
+ option := &tasks.GithubOptions{
+ ConnectionId: 1,
+ Tasks: nil,
+ Since: "",
+ Owner: "merico-dev",
+ Repo: "lake",
+ TransformationRules: models.TransformationRules{
+ PrType: "hey,man,wasup",
+ PrComponent: "component/(.*)$",
+ PrBodyClosePattern: "(?mi)(fix|close|resolve|fixes|closes|resolves|fixed|closed|resolved)[\\s]*.*(((and )?(#|https:\\/\\/github.com\\/%s\\/%s\\/issues\\/)\\d+[ ]*)+)",
+ IssueSeverity: "severity/(.*)$",
+ IssuePriority: "^(highest|high|medium|low)$",
+ IssueComponent: "component/(.*)$",
+ IssueTypeBug: "^(bug|failure|error)$",
+ IssueTypeIncident: "",
+ IssueTypeRequirement: "^(feat|feature|proposal|requirement)$",
+ DeployTagPattern: "(?i)deploy",
+ },
+ }
+ mockConnHelper := mocks.NewBpHelper(t)
+ mockConnHelper.On("GetApiRepo", uint64(1), option).Return(&tasks.GithubApiRepo{
+ Name: "test",
+ GithubId: 1,
+ CloneUrl: "CloneUrl",
+ }, "", "", nil)
+ mockMeta := mocks.NewPluginMeta(t)
+ mockMeta.On("RootPkgPath").Return("github.com/apache/incubator-devlake/plugins/github")
+
+ err := core.RegisterPlugin(cmd.Use, mockMeta)
+ if err != nil {
+ panic(err)
+ }
+ bs := &core.BlueprintScopeV100{
+ Entities: []string{"CODE"},
+ Options: json.RawMessage(`{
+ "repo": "lake",
+ "owner": "merico-dev"
+ }`),
+ Transformation: json.RawMessage(`{
+ "prType": "hey,man,wasup",
+ "refdiff": {
+ "tagsPattern": "pattern",
+ "tagsLimit": 10,
+ "tagsOrder": "reverse semver"
+ },
+ "dora": {
+ "environment": "pattern",
+ "environmentRegex": "xxxx"
+ }
+ }`),
+ }
+ scopes := make([]*core.BlueprintScopeV100, 0)
+ scopes = append(scopes, bs)
+ plan, err := DoMakePipeline(nil, 1, scopes, mockConnHelper)
+
+ assert.Nil(t, err)
+ planJson, err1 := json.Marshal(plan)
+ assert.Nil(t, err1)
+ fmt.Println(string(planJson))
+ expectPlan := `[[{"plugin":"github","subtasks":[],"options":{"connectionId":1,"owner":"merico-dev","repo":"lake","transformationRules":{"prType":"hey,man,wasup"}}},{"plugin":"gitextractor","subtasks":null,"options":{"proxy":"","repoId":"github:GithubRepo:1:1","url":"//git:@CloneUrl"}}],[{"plugin":"refdiff","subtasks":null,"options":{"tagsLimit":10,"tagsOrder":"reverse semver","tagsPattern":"pattern"}}],[{"plugin":"dora","subtasks":null,"options":{"repoId":"github:GithubRepo:1:1","tasks" [...]
+ assert.Equal(t, expectPlan, string(planJson))
+
+}
diff --git a/services/blueprint.go b/services/blueprint.go
index f9dccc4c..dfaf3d44 100644
--- a/services/blueprint.go
+++ b/services/blueprint.go
@@ -274,7 +274,8 @@ func GeneratePlanJsonV100(settings *models.BlueprintSettings) (core.PipelinePlan
if err != nil {
return nil, err
}
-
+ hasDoraEnrich := false
+ doraRules := make(map[string]interface{})
plans := make([]core.PipelinePlan, len(connections))
for i, connection := range connections {
if len(connection.Scope) == 0 {
@@ -292,9 +293,28 @@ func GeneratePlanJsonV100(settings *models.BlueprintSettings) (core.PipelinePlan
} else {
return nil, errors.Default.New(fmt.Sprintf("plugin %s does not support blueprint protocol version 1.0.0", connection.Plugin))
}
+ for _, stage := range plans[i] {
+ for _, task := range stage {
+ if task.Plugin == "dora" {
+ hasDoraEnrich = true
+ for k, v := range task.Options {
+ doraRules[k] = v
+ }
+ }
+ }
+ }
}
-
mergedPipelinePlan := MergePipelinePlans(plans...)
+ if hasDoraEnrich {
+ doraRules["tasks"] = []string{"calculateChangeLeadTime", "ConnectIssueDeploy"}
+ plan := core.PipelineStage{
+ &core.PipelineTask{
+ Plugin: "dora",
+ Options: doraRules,
+ },
+ }
+ mergedPipelinePlan = append(mergedPipelinePlan, plan)
+ }
return FormatPipelinePlans(settings.BeforePlan, mergedPipelinePlan, settings.AfterPlan)
}