You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by ab...@apache.org on 2022/09/20 12:47:30 UTC
[incubator-devlake] branch main updated: feat(github): modify unit test
This is an automated email from the ASF dual-hosted git repository.
abeizn pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new d302616e feat(github): modify unit test
d302616e is described below
commit d302616efa9e9123cb38d7fc8a70faa5a59a3654
Author: Yingchu Chen <yi...@merico.dev>
AuthorDate: Tue Sep 20 20:12:35 2022 +0800
feat(github): modify unit test
---
Makefile | 1 -
plugins/github/api/blueprint.go | 241 +++++++++++++++++------------------
plugins/github/api/blueprint_test.go | 72 ++++++-----
plugins/gitlab/api/blueprint.go | 1 +
4 files changed, 158 insertions(+), 157 deletions(-)
diff --git a/Makefile b/Makefile
index b9b70ebf..6bd35581 100644
--- a/Makefile
+++ b/Makefile
@@ -94,7 +94,6 @@ mock:
mockery --dir=./plugins/core --unroll-variadic=false --name='.*'
mockery --dir=./plugins/core/dal --unroll-variadic=false --name='.*'
mockery --dir=./plugins/helper --unroll-variadic=false --name='.*'
- mockery --dir=./plugins/github/api --unroll-variadic=false --name='.*'
test: unit-test e2e-test
diff --git a/plugins/github/api/blueprint.go b/plugins/github/api/blueprint.go
index 55f23592..e8eeffe2 100644
--- a/plugins/github/api/blueprint.go
+++ b/plugins/github/api/blueprint.go
@@ -37,146 +37,147 @@ import (
)
func MakePipelinePlan(subtaskMetas []core.SubTaskMeta, connectionId uint64, scope []*core.BlueprintScopeV100) (core.PipelinePlan, errors.Error) {
- repoHelper := RepoHelper{}
- plan, err := DoMakePipeline(subtaskMetas, connectionId, scope, repoHelper)
- if err != nil {
- return nil, err
- }
- return plan, nil
-}
-
-func DoMakePipeline(subtaskMetas []core.SubTaskMeta, connectionId uint64, scope []*core.BlueprintScopeV100, repoHelper BpHelper) (core.PipelinePlan, errors.Error) {
var err errors.Error
- apiRepo, _, _ := &tasks.GithubApiRepo{}, "", ""
plan := make(core.PipelinePlan, len(scope))
for i, scopeElem := range scope {
- // handle taskOptions and transformationRules, by dumping them to taskOptions
- transformationRules := make(map[string]interface{})
- if len(scopeElem.Transformation) > 0 {
- err = errors.Convert(json.Unmarshal(scopeElem.Transformation, &transformationRules))
- if err != nil {
- return nil, err
- }
- }
- // refdiff
- if refdiffRules, ok := transformationRules["refdiff"]; ok && refdiffRules != nil {
- // add a new task to next stage
- j := i + 1
- if j == len(plan) {
- plan = append(plan, nil)
- }
- plan[j] = core.PipelineStage{
- {
- Plugin: "refdiff",
- Options: refdiffRules.(map[string]interface{}),
- },
- }
- // remove it from github transformationRules
- delete(transformationRules, "refdiff")
- }
- // construct task options for github
- options := make(map[string]interface{})
- err = errors.Convert(json.Unmarshal(scopeElem.Options, &options))
+ plan, err = processScope(subtaskMetas, connectionId, scopeElem, i, plan, nil, nil)
if err != nil {
return nil, err
}
- options["connectionId"] = connectionId
- options["transformationRules"] = transformationRules
- // make sure task options is valid
- op, err := tasks.DecodeAndValidateTaskOptions(options)
+ }
+ return plan, nil
+}
+func processScope(subtaskMetas []core.SubTaskMeta, connectionId uint64, scopeElem *core.BlueprintScopeV100, i int, plan core.PipelinePlan, apiRepo *tasks.GithubApiRepo, connection *models.GithubConnection) (core.PipelinePlan, errors.Error) {
+ var err errors.Error
+ // handle taskOptions and transformationRules, by dumping them to taskOptions
+ transformationRules := make(map[string]interface{})
+ if len(scopeElem.Transformation) > 0 {
+ err = errors.Convert(json.Unmarshal(scopeElem.Transformation, &transformationRules))
if err != nil {
return nil, err
}
- // construct subtasks
- subtasks, err := helper.MakePipelinePlanSubtasks(subtaskMetas, scopeElem.Entities)
- if err != nil {
- return nil, err
+ }
+ // refdiff
+ if refdiffRules, ok := transformationRules["refdiff"]; ok && refdiffRules != nil {
+ // add a new task to next stage
+ j := i + 1
+ if j == len(plan) {
+ plan = append(plan, nil)
}
- stage := plan[i]
- if stage == nil {
- stage = core.PipelineStage{}
+ plan[j] = core.PipelineStage{
+ {
+ Plugin: "refdiff",
+ Options: refdiffRules.(map[string]interface{}),
+ },
}
- stage = append(stage, &core.PipelineTask{
- Plugin: "github",
- Subtasks: subtasks,
- Options: options,
- })
- // collect git data by gitextractor if CODE was requested
- if utils.StringsContains(scopeElem.Entities, core.DOMAIN_TYPE_CODE) {
- // here is the tricky part, we have to obtain the repo id beforehand
- res, token, proxy, err := repoHelper.GetApiRepo(connectionId, op)
- apiRepo = res.(*tasks.GithubApiRepo)
- if err != nil {
- return nil, err
- }
- cloneUrl, err := errors.Convert01(url.Parse(apiRepo.CloneUrl))
+ // remove it from github transformationRules
+ delete(transformationRules, "refdiff")
+ }
+ // construct task options for github
+ options := make(map[string]interface{})
+ err = errors.Convert(json.Unmarshal(scopeElem.Options, &options))
+ if err != nil {
+ return nil, err
+ }
+ options["connectionId"] = connectionId
+ options["transformationRules"] = transformationRules
+ // make sure task options is valid
+ op, err := tasks.DecodeAndValidateTaskOptions(options)
+ if err != nil {
+ return nil, err
+ }
+ // construct subtasks
+ subtasks, err := helper.MakePipelinePlanSubtasks(subtaskMetas, scopeElem.Entities)
+ if err != nil {
+ return nil, err
+ }
+ stage := plan[i]
+ if stage == nil {
+ stage = core.PipelineStage{}
+ }
+ stage = append(stage, &core.PipelineTask{
+ Plugin: "github",
+ Subtasks: subtasks,
+ Options: options,
+ })
+ // collect git data by gitextractor if CODE was requested
+ if utils.StringsContains(scopeElem.Entities, core.DOMAIN_TYPE_CODE) {
+ // here is the tricky part, we have to obtain the repo id beforehand
+ if connection == nil {
+ connection = new(models.GithubConnection)
+ err = connectionHelper.FirstById(connection, connectionId)
if err != nil {
return nil, err
}
- cloneUrl.User = url.UserPassword("git", token)
- stage = append(stage, &core.PipelineTask{
- Plugin: "gitextractor",
- Options: map[string]interface{}{
- "url": cloneUrl.String(),
- "repoId": didgen.NewDomainIdGenerator(&models.GithubRepo{}).Generate(connectionId, apiRepo.GithubId),
- "proxy": proxy,
- },
- })
}
- plan[i] = stage
- // dora
- if doraRules, ok := transformationRules["dora"]; ok && doraRules != nil {
- j := i + 1
- // add a new task to next stage
- if plan[j] != nil {
- j++
- }
- if j == len(plan) {
- plan = append(plan, nil)
- }
+ token := strings.Split(connection.Token, ",")[0]
+ if apiRepo == nil {
+ apiRepo = new(tasks.GithubApiRepo)
+ err = getApiRepo(connection, token, op, apiRepo)
if err != nil {
return nil, err
}
- if apiRepo.GithubId == 0 {
- res, _, _, err := repoHelper.GetApiRepo(connectionId, op)
+ }
+ cloneUrl, err := errors.Convert01(url.Parse(apiRepo.CloneUrl))
+ if err != nil {
+ return nil, err
+ }
+ cloneUrl.User = url.UserPassword("git", token)
+ stage = append(stage, &core.PipelineTask{
+ Plugin: "gitextractor",
+ Options: map[string]interface{}{
+ "url": cloneUrl.String(),
+ "repoId": didgen.NewDomainIdGenerator(&models.GithubRepo{}).Generate(connectionId, apiRepo.GithubId),
+ "proxy": connection.Proxy,
+ },
+ })
+ }
+ // dora
+ if doraRules, ok := transformationRules["dora"]; ok && doraRules != nil {
+ j := i + 1
+ // add a new task to next stage
+ if plan[j] != nil {
+ j++
+ }
+ if j == len(plan) {
+ plan = append(plan, nil)
+ }
+ if err != nil {
+ return nil, err
+ }
+ if apiRepo == nil {
+ if connection == nil {
+ connection = new(models.GithubConnection)
+ err = connectionHelper.FirstById(connection, connectionId)
if err != nil {
return nil, err
}
- apiRepo = res.(*tasks.GithubApiRepo)
-
}
- doraOption := make(map[string]interface{})
- doraOption["repoId"] = didgen.NewDomainIdGenerator(&models.GithubRepo{}).Generate(connectionId, apiRepo.GithubId)
- doraOption["tasks"] = []string{"EnrichTaskEnv"}
- doraOption["transformation"] = doraRules
- plan[j] = core.PipelineStage{
- {
- Plugin: "dora",
- Options: doraOption,
- },
+ token := strings.Split(connection.Token, ",")[0]
+ apiRepo = new(tasks.GithubApiRepo)
+ err = getApiRepo(connection, token, op, apiRepo)
+ if err != nil {
+ return nil, err
}
- // remove it from github transformationRules
- delete(transformationRules, "dora")
}
+ doraOption := make(map[string]interface{})
+ doraOption["repoId"] = didgen.NewDomainIdGenerator(&models.GithubRepo{}).Generate(connectionId, apiRepo.GithubId)
+ doraOption["tasks"] = []string{"EnrichTaskEnv"}
+ doraOption["transformation"] = doraRules
+ plan[j] = core.PipelineStage{
+ {
+ Plugin: "dora",
+ Options: doraOption,
+ },
+ }
+ // remove it from github transformationRules
+ delete(transformationRules, "dora")
}
+ plan[i] = stage
return plan, nil
}
-type BpHelper interface {
- GetApiRepo(connectionId uint64, op interface{}) (interface{}, string, string, errors.Error)
-}
-
-type RepoHelper struct{}
-
-func (c RepoHelper) GetApiRepo(connectionId uint64, originOp interface{}) (interface{}, string, string, errors.Error) {
- op := originOp.(*tasks.GithubOptions)
- // here is the tricky part, we have to obtain the repo id beforehand
- connection := new(models.GithubConnection)
- err := connectionHelper.FirstById(connection, connectionId)
- if err != nil {
- return nil, "", "", err
- }
- token := strings.Split(connection.Token, ",")[0]
+func getApiRepo(connection *models.GithubConnection, token string, op *tasks.GithubOptions, apiRepo *tasks.GithubApiRepo) errors.Error {
apiClient, err := helper.NewApiClient(
context.TODO(),
connection.Endpoint,
@@ -188,25 +189,23 @@ func (c RepoHelper) GetApiRepo(connectionId uint64, originOp interface{}) (inter
basicRes,
)
if err != nil {
- return nil, "", "", err
+ return err
}
res, err := apiClient.Get(fmt.Sprintf("repos/%s/%s", op.Owner, op.Repo), nil, nil)
if err != nil {
- return nil, "", "", err
+ return err
}
- defer res.Body.Close()
+ //defer res.Body.Close()
if res.StatusCode != http.StatusOK {
- return nil, "", "", errors.HttpStatus(res.StatusCode).New(fmt.Sprintf(
- "unexpected status code when requesting repo detail from %s", res.Request.URL.String()))
+ return errors.HttpStatus(res.StatusCode).New(fmt.Sprintf("unexpected status code when requesting repo detail from %s", res.Request.URL.String()))
}
body, err := errors.Convert01(io.ReadAll(res.Body))
if err != nil {
- return nil, "", "", err
+ return err
}
- apiRepo := new(tasks.GithubApiRepo)
err = errors.Convert(json.Unmarshal(body, apiRepo))
if err != nil {
- return nil, "", "", errors.Convert(err)
+ return err
}
- return apiRepo, token, connection.Proxy, nil
+ return nil
}
diff --git a/plugins/github/api/blueprint_test.go b/plugins/github/api/blueprint_test.go
index 24c93c0f..8d5f45aa 100644
--- a/plugins/github/api/blueprint_test.go
+++ b/plugins/github/api/blueprint_test.go
@@ -19,52 +19,49 @@ package api
import (
"encoding/json"
+ "github.com/apache/incubator-devlake/config"
+ "github.com/apache/incubator-devlake/logger"
"github.com/apache/incubator-devlake/mocks"
+ "github.com/apache/incubator-devlake/models/common"
"github.com/apache/incubator-devlake/plugins/core"
"github.com/apache/incubator-devlake/plugins/github/models"
"github.com/apache/incubator-devlake/plugins/github/tasks"
+ "github.com/apache/incubator-devlake/plugins/helper"
+ "github.com/apache/incubator-devlake/runner"
"github.com/stretchr/testify/assert"
"testing"
)
-func TestMakePipelinePlan(t *testing.T) {
- option := &tasks.GithubOptions{
- ConnectionId: 1,
- Tasks: nil,
- Since: "",
- Owner: "merico-dev",
- Repo: "lake",
- TransformationRules: models.TransformationRules{
- PrType: "hey,man,wasup",
- PrComponent: "component/(.*)$",
- PrBodyClosePattern: "(?mi)(fix|close|resolve|fixes|closes|resolves|fixed|closed|resolved)[\\s]*.*(((and )?(#|https:\\/\\/github.com\\/%s\\/%s\\/issues\\/)\\d+[ ]*)+)",
- IssueSeverity: "severity/(.*)$",
- IssuePriority: "^(highest|high|medium|low)$",
- IssueComponent: "component/(.*)$",
- IssueTypeBug: "^(bug|failure|error)$",
- IssueTypeIncident: "",
- IssueTypeRequirement: "^(feat|feature|proposal|requirement)$",
- DeployTagPattern: "(?i)deploy",
+func TestProcessScope(t *testing.T) {
+ cfg := config.GetConfig()
+ log := logger.Global.Nested("github")
+ db, _ := runner.NewGormDb(cfg, log)
+ Init(cfg, log, db)
+ connection := &models.GithubConnection{
+ RestConnection: helper.RestConnection{
+ BaseConnection: helper.BaseConnection{
+ Name: "github-test",
+ Model: common.Model{
+ ID: 1,
+ },
+ },
+ Endpoint: "https://api.github.com/",
+ Proxy: "",
+ RateLimitPerHour: 0,
+ },
+ AccessToken: helper.AccessToken{
+ Token: "123",
},
}
- mockConnHelper := mocks.NewBpHelper(t)
- mockConnHelper.On("GetApiRepo", uint64(1), option).Return(&tasks.GithubApiRepo{
- Name: "test",
- GithubId: 1,
- CloneUrl: "CloneUrl",
- }, "", "", nil)
mockMeta := mocks.NewPluginMeta(t)
mockMeta.On("RootPkgPath").Return("github.com/apache/incubator-devlake/plugins/github")
-
err := core.RegisterPlugin("github", mockMeta)
- if err != nil {
- panic(err)
- }
+ assert.Nil(t, err)
bs := &core.BlueprintScopeV100{
Entities: []string{"CODE"},
Options: json.RawMessage(`{
- "repo": "lake",
- "owner": "merico-dev"
+ "owner": "test",
+ "repo": "testRepo"
}`),
Transformation: json.RawMessage(`{
"prType": "hey,man,wasup",
@@ -79,14 +76,19 @@ func TestMakePipelinePlan(t *testing.T) {
}
}`),
}
+ apiRepo := &tasks.GithubApiRepo{
+ GithubId: 123,
+ CloneUrl: "HttpUrlToRepo",
+ }
scopes := make([]*core.BlueprintScopeV100, 0)
scopes = append(scopes, bs)
- plan, err := DoMakePipeline(nil, 1, scopes, mockConnHelper)
-
- assert.Nil(t, err)
+ plan := make(core.PipelinePlan, len(scopes))
+ for i, scopeElem := range scopes {
+ plan, err = processScope(nil, 1, scopeElem, i, plan, apiRepo, connection)
+ assert.Nil(t, err)
+ }
planJson, err1 := json.Marshal(plan)
assert.Nil(t, err1)
- expectPlan := `[[{"plugin":"github","subtasks":[],"options":{"connectionId":1,"owner":"merico-dev","repo":"lake","transformationRules":{"prType":"hey,man,wasup"}}},{"plugin":"gitextractor","subtasks":null,"options":{"proxy":"","repoId":"github:GithubRepo:1:1","url":"//git:@CloneUrl"}}],[{"plugin":"refdiff","subtasks":null,"options":{"tagsLimit":10,"tagsOrder":"reverse semver","tagsPattern":"pattern"}}],[{"plugin":"dora","subtasks":null,"options":{"repoId":"github:GithubRepo:1:1","tasks" [...]
+ expectPlan := `[[{"plugin":"github","subtasks":[],"options":{"connectionId":1,"owner":"test","repo":"testRepo","transformationRules":{"prType":"hey,man,wasup"}}},{"plugin":"gitextractor","subtasks":null,"options":{"proxy":"","repoId":"github:GithubRepo:1:123","url":"//git:123@HttpUrlToRepo"}}],[{"plugin":"refdiff","subtasks":null,"options":{"tagsLimit":10,"tagsOrder":"reverse semver","tagsPattern":"pattern"}}],[{"plugin":"dora","subtasks":null,"options":{"repoId":"github:GithubRepo:1:12 [...]
assert.Equal(t, expectPlan, string(planJson))
-
}
diff --git a/plugins/gitlab/api/blueprint.go b/plugins/gitlab/api/blueprint.go
index 3c1a1958..9112b403 100644
--- a/plugins/gitlab/api/blueprint.go
+++ b/plugins/gitlab/api/blueprint.go
@@ -113,6 +113,7 @@ func processScope(subtaskMetas []core.SubTaskMeta, connectionId uint64, scopeEle
}
token := strings.Split(connection.Token, ",")[0]
if apiRepo == nil {
+ apiRepo = new(tasks.GitlabApiProject)
err = getApiRepo(connection, token, op, apiRepo)
if err != nil {
return nil, err