You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by ab...@apache.org on 2022/10/13 08:43:49 UTC
[incubator-devlake] branch main updated (dd4df15d -> 3477db54)
This is an automated email from the ASF dual-hosted git repository.
abeizn pushed a change to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
from dd4df15d fix: starrocks plugin sync pg enum data type
new ed07e6b8 fix(github): modify blueprint
new 0ce5b40a fix(github): modify bluprint test
new 3477db54 fix(github): modify blueprint according to review
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
plugins/github/api/blueprint.go | 263 ++++++++++++++++++-----------------
plugins/github/api/blueprint_test.go | 78 +++++++++--
plugins/helper/api_client.go | 9 ++
3 files changed, 206 insertions(+), 144 deletions(-)
[incubator-devlake] 01/03: fix(github): modify blueprint
Posted by ab...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
abeizn pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit ed07e6b88b5d90d2b125671ac83543ce291a5e27
Author: Yingchu Chen <yi...@merico.dev>
AuthorDate: Wed Oct 12 19:38:54 2022 +0800
fix(github): modify blueprint
---
plugins/github/api/blueprint.go | 251 ++++++++++++++++++++--------------------
1 file changed, 124 insertions(+), 127 deletions(-)
diff --git a/plugins/github/api/blueprint.go b/plugins/github/api/blueprint.go
index 48ec1a1f..8acb4d7f 100644
--- a/plugins/github/api/blueprint.go
+++ b/plugins/github/api/blueprint.go
@@ -38,117 +38,87 @@ import (
func MakePipelinePlan(subtaskMetas []core.SubTaskMeta, connectionId uint64, scope []*core.BlueprintScopeV100) (core.PipelinePlan, errors.Error) {
var err errors.Error
- plan := make(core.PipelinePlan, len(scope))
- for i, scopeElem := range scope {
- plan, err = processScope(subtaskMetas, connectionId, scopeElem, i, plan, nil, nil)
- if err != nil {
- return nil, err
- }
- }
- return plan, nil
-}
-func processScope(subtaskMetas []core.SubTaskMeta, connectionId uint64, scopeElem *core.BlueprintScopeV100, i int, plan core.PipelinePlan, apiRepo *tasks.GithubApiRepo, connection *models.GithubConnection) (core.PipelinePlan, errors.Error) {
- var err errors.Error
- // handle taskOptions and transformationRules, by dumping them to taskOptions
- transformationRules := make(map[string]interface{})
- if len(scopeElem.Transformation) > 0 {
- err = errors.Convert(json.Unmarshal(scopeElem.Transformation, &transformationRules))
- if err != nil {
- return nil, err
- }
- }
- // refdiff
- if refdiffRules, ok := transformationRules["refdiff"]; ok && refdiffRules != nil {
- // add a new task to next stage
- j := i + 1
- if j == len(plan) {
- plan = append(plan, nil)
- }
- plan[j] = core.PipelineStage{
- {
- Plugin: "refdiff",
- Options: refdiffRules.(map[string]interface{}),
- },
- }
- // remove it from github transformationRules
- delete(transformationRules, "refdiff")
- }
- // construct task options for github
- options := make(map[string]interface{})
- err = errors.Convert(json.Unmarshal(scopeElem.Options, &options))
+ connection := new(models.GithubConnection)
+ err = connectionHelper.FirstById(connection, connectionId)
if err != nil {
return nil, err
}
- options["connectionId"] = connectionId
- options["transformationRules"] = transformationRules
- // make sure task options is valid
- op, err := tasks.DecodeAndValidateTaskOptions(options)
- if err != nil {
- return nil, err
- }
- // construct subtasks
- subtasks, err := helper.MakePipelinePlanSubtasks(subtaskMetas, scopeElem.Entities)
+ token := strings.Split(connection.Token, ",")[0]
+
+ apiClient, err := helper.NewApiClient(
+ context.TODO(),
+ connection.Endpoint,
+ map[string]string{
+ "Authorization": fmt.Sprintf("Bearer %s", token),
+ },
+ 10*time.Second,
+ connection.Proxy,
+ basicRes,
+ )
+ plan, err := makePipelinePlan(subtaskMetas, connectionId, scope, apiClient, connection)
if err != nil {
return nil, err
}
- stage := plan[i]
- if stage == nil {
- stage = core.PipelineStage{}
- }
- stage = append(stage, &core.PipelineTask{
- Plugin: "github",
- Subtasks: subtasks,
- Options: options,
- })
- // collect git data by gitextractor if CODE was requested
- if utils.StringsContains(scopeElem.Entities, core.DOMAIN_TYPE_CODE) {
- // here is the tricky part, we have to obtain the repo id beforehand
- if connection == nil {
- connection = new(models.GithubConnection)
- err = connectionHelper.FirstById(connection, connectionId)
+ return plan, nil
+}
+func makePipelinePlan(subtaskMetas []core.SubTaskMeta, connectionId uint64, scope []*core.BlueprintScopeV100, apiClient *helper.ApiClient, connection *models.GithubConnection) (core.PipelinePlan, errors.Error) {
+ var err errors.Error
+ plan := make(core.PipelinePlan, len(scope))
+ for i, scopeElem := range scope {
+ // handle taskOptions and transformationRules, by dumping them to taskOptions
+ transformationRules := make(map[string]interface{})
+ if len(scopeElem.Transformation) > 0 {
+ err = errors.Convert(json.Unmarshal(scopeElem.Transformation, &transformationRules))
if err != nil {
return nil, err
}
}
- token := strings.Split(connection.Token, ",")[0]
- if apiRepo == nil {
- apiRepo = new(tasks.GithubApiRepo)
- err = getApiRepo(connection, token, op, apiRepo)
- if err != nil {
- return nil, err
+ // refdiff
+ if refdiffRules, ok := transformationRules["refdiff"]; ok && refdiffRules != nil {
+ // add a new task to next stage
+ j := i + 1
+ if j == len(plan) {
+ plan = append(plan, nil)
+ }
+ plan[j] = core.PipelineStage{
+ {
+ Plugin: "refdiff",
+ Options: refdiffRules.(map[string]interface{}),
+ },
}
+ // remove it from github transformationRules
+ delete(transformationRules, "refdiff")
}
- cloneUrl, err := errors.Convert01(url.Parse(apiRepo.CloneUrl))
+ // construct task options for github
+ options := make(map[string]interface{})
+ err = errors.Convert(json.Unmarshal(scopeElem.Options, &options))
if err != nil {
return nil, err
}
- cloneUrl.User = url.UserPassword("git", token)
- stage = append(stage, &core.PipelineTask{
- Plugin: "gitextractor",
- Options: map[string]interface{}{
- "url": cloneUrl.String(),
- "repoId": didgen.NewDomainIdGenerator(&models.GithubRepo{}).Generate(connectionId, apiRepo.GithubId),
- "proxy": connection.Proxy,
- },
- })
- }
- // dora
- if productionPattern, ok := transformationRules["productionPattern"]; ok && productionPattern != nil {
- j := i + 1
- if j == len(plan) {
- plan = append(plan, nil)
- }
- // add a new task to next stage
- if plan[j] != nil {
- j++
- }
- if j == len(plan) {
- plan = append(plan, nil)
+ options["connectionId"] = connectionId
+ options["transformationRules"] = transformationRules
+ // make sure task options is valid
+ op, err := tasks.DecodeAndValidateTaskOptions(options)
+ if err != nil {
+ return nil, err
}
+ // construct subtasks
+ subtasks, err := helper.MakePipelinePlanSubtasks(subtaskMetas, scopeElem.Entities)
if err != nil {
return nil, err
}
- if apiRepo == nil {
+ stage := plan[i]
+ if stage == nil {
+ stage = core.PipelineStage{}
+ }
+ stage = append(stage, &core.PipelineTask{
+ Plugin: "github",
+ Subtasks: subtasks,
+ Options: options,
+ })
+ // collect git data by gitextractor if CODE was requested
+ if utils.StringsContains(scopeElem.Entities, core.DOMAIN_TYPE_CODE) {
+ // here is the tricky part, we have to obtain the repo id beforehand
if connection == nil {
connection = new(models.GithubConnection)
err = connectionHelper.FirstById(connection, connectionId)
@@ -157,60 +127,87 @@ func processScope(subtaskMetas []core.SubTaskMeta, connectionId uint64, scopeEle
}
}
token := strings.Split(connection.Token, ",")[0]
- apiRepo = new(tasks.GithubApiRepo)
- err = getApiRepo(connection, token, op, apiRepo)
+ apiRepo, err := getApiRepo(op, apiClient)
+ if err != nil {
+ return nil, err
+ }
+
+ cloneUrl, err := errors.Convert01(url.Parse(apiRepo.CloneUrl))
if err != nil {
return nil, err
}
+ cloneUrl.User = url.UserPassword("git", token)
+ stage = append(stage, &core.PipelineTask{
+ Plugin: "gitextractor",
+ Options: map[string]interface{}{
+ "url": cloneUrl.String(),
+ "repoId": didgen.NewDomainIdGenerator(&models.GithubRepo{}).Generate(connectionId, apiRepo.GithubId),
+ "proxy": connection.Proxy,
+ },
+ })
}
- doraOption := make(map[string]interface{})
- doraOption["repoId"] = didgen.NewDomainIdGenerator(&models.GithubRepo{}).Generate(connectionId, apiRepo.GithubId)
- doraRules := make(map[string]interface{})
- doraRules["productionPattern"] = productionPattern
- doraOption["transformationRules"] = doraRules
- plan[j] = core.PipelineStage{
- {
- Plugin: "dora",
- Subtasks: []string{"EnrichTaskEnv"},
- Options: doraOption,
- },
+ // dora
+ if productionPattern, ok := transformationRules["productionPattern"]; ok && productionPattern != nil {
+ j := i + 1
+ if j == len(plan) {
+ plan = append(plan, nil)
+ }
+ // add a new task to next stage
+ if plan[j] != nil {
+ j++
+ }
+ if j == len(plan) {
+ plan = append(plan, nil)
+ }
+ if err != nil {
+ return nil, err
+ }
+
+ apiRepo, err := getApiRepo(op, apiClient)
+ if err != nil {
+ return nil, err
+ }
+
+ doraOption := make(map[string]interface{})
+ doraOption["repoId"] = didgen.NewDomainIdGenerator(&models.GithubRepo{}).Generate(connectionId, apiRepo.GithubId)
+ doraRules := make(map[string]interface{})
+ doraRules["productionPattern"] = productionPattern
+ doraOption["transformationRules"] = doraRules
+ plan[j] = core.PipelineStage{
+ {
+ Plugin: "dora",
+ Subtasks: []string{"EnrichTaskEnv"},
+ Options: doraOption,
+ },
+ }
+ // remove it from github transformationRules
+ delete(transformationRules, "productionPattern")
+ }
+ plan[i] = stage
+ if err != nil {
+ return nil, err
}
- // remove it from github transformationRules
- delete(transformationRules, "productionPattern")
}
- plan[i] = stage
return plan, nil
}
-func getApiRepo(connection *models.GithubConnection, token string, op *tasks.GithubOptions, apiRepo *tasks.GithubApiRepo) errors.Error {
- apiClient, err := helper.NewApiClient(
- context.TODO(),
- connection.Endpoint,
- map[string]string{
- "Authorization": fmt.Sprintf("Bearer %s", token),
- },
- 10*time.Second,
- connection.Proxy,
- basicRes,
- )
- if err != nil {
- return err
- }
+func getApiRepo(op *tasks.GithubOptions, apiClient *helper.ApiClient) (*tasks.GithubApiRepo, errors.Error) {
+ apiRepo := new(tasks.GithubApiRepo)
res, err := apiClient.Get(fmt.Sprintf("repos/%s/%s", op.Owner, op.Repo), nil, nil)
if err != nil {
- return err
+ return nil, err
}
//defer res.Body.Close()
if res.StatusCode != http.StatusOK {
- return errors.HttpStatus(res.StatusCode).New(fmt.Sprintf("unexpected status code when requesting repo detail from %s", res.Request.URL.String()))
+ return nil, errors.HttpStatus(res.StatusCode).New(fmt.Sprintf("unexpected status code when requesting repo detail from %s", res.Request.URL.String()))
}
body, err := errors.Convert01(io.ReadAll(res.Body))
if err != nil {
- return err
+ return nil, err
}
err = errors.Convert(json.Unmarshal(body, apiRepo))
if err != nil {
- return err
+ return nil, err
}
- return nil
+ return apiRepo, nil
}
[incubator-devlake] 03/03: fix(github): modify blueprint according to review
Posted by ab...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
abeizn pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit 3477db5439569f33199ae2b06c7a3fcd5cedffff
Author: Yingchu Chen <yi...@merico.dev>
AuthorDate: Thu Oct 13 12:05:02 2022 +0800
fix(github): modify blueprint according to review
---
plugins/github/api/blueprint.go | 30 ++++++++++++++++--------------
plugins/github/api/blueprint_test.go | 2 --
2 files changed, 16 insertions(+), 16 deletions(-)
diff --git a/plugins/github/api/blueprint.go b/plugins/github/api/blueprint.go
index 40e334fc..e9dc5d1a 100644
--- a/plugins/github/api/blueprint.go
+++ b/plugins/github/api/blueprint.go
@@ -68,12 +68,6 @@ func MakePipelinePlan(subtaskMetas []core.SubTaskMeta, connectionId uint64, scop
func makePipelinePlan(subtaskMetas []core.SubTaskMeta, scope []*core.BlueprintScopeV100, apiClient helper.ApiClientGetter, connection *models.GithubConnection) (core.PipelinePlan, errors.Error) {
var err errors.Error
var repo *tasks.GithubApiRepo
- getApiRepoIfNil := func(op *tasks.GithubOptions) (*tasks.GithubApiRepo, errors.Error) {
- if repo == nil {
- repo, err = getApiRepo(op, apiClient)
- }
- return repo, err
- }
plan := make(core.PipelinePlan, len(scope))
for i, scopeElem := range scope {
// handle taskOptions and transformationRules, by dumping them to taskOptions
@@ -113,6 +107,13 @@ func makePipelinePlan(subtaskMetas []core.SubTaskMeta, scope []*core.BlueprintSc
if err != nil {
return nil, err
}
+ memorizedGetApiRepo := func() (*tasks.GithubApiRepo, errors.Error) {
+ if repo == nil {
+ repo, err = getApiRepo(op, apiClient)
+ }
+ return repo, err
+ }
+
// construct subtasks
subtasks, err := helper.MakePipelinePlanSubtasks(subtaskMetas, scopeElem.Entities)
if err != nil {
@@ -131,7 +132,7 @@ func makePipelinePlan(subtaskMetas []core.SubTaskMeta, scope []*core.BlueprintSc
if utils.StringsContains(scopeElem.Entities, core.DOMAIN_TYPE_CODE) {
// here is the tricky part, we have to obtain the repo id beforehand
token := strings.Split(connection.Token, ",")[0]
- repo, err = getApiRepoIfNil(op)
+ repo, err = memorizedGetApiRepo()
if err != nil {
return nil, err
}
@@ -165,21 +166,21 @@ func makePipelinePlan(subtaskMetas []core.SubTaskMeta, scope []*core.BlueprintSc
if err != nil {
return nil, err
}
- repo, err = getApiRepoIfNil(op)
+ repo, err = memorizedGetApiRepo()
if err != nil {
return nil, err
}
- doraOption := make(map[string]interface{})
- doraOption["repoId"] = didgen.NewDomainIdGenerator(&models.GithubRepo{}).Generate(connection.ID, repo.GithubId)
- doraRules := make(map[string]interface{})
- doraRules["productionPattern"] = productionPattern
- doraOption["transformationRules"] = doraRules
plan[j] = core.PipelineStage{
{
Plugin: "dora",
Subtasks: []string{"EnrichTaskEnv"},
- Options: doraOption,
+ Options: map[string]interface{}{
+ "repoId": didgen.NewDomainIdGenerator(&models.GithubRepo{}).Generate(connection.ID, repo.GithubId),
+ "transformationRules": map[string]interface{}{
+ "productionPattern": productionPattern,
+ },
+ },
},
}
// remove it from github transformationRules
@@ -189,6 +190,7 @@ func makePipelinePlan(subtaskMetas []core.SubTaskMeta, scope []*core.BlueprintSc
if err != nil {
return nil, err
}
+ repo = nil
}
return plan, nil
}
diff --git a/plugins/github/api/blueprint_test.go b/plugins/github/api/blueprint_test.go
index 08dd0390..c663e3ba 100644
--- a/plugins/github/api/blueprint_test.go
+++ b/plugins/github/api/blueprint_test.go
@@ -86,8 +86,6 @@ func TestProcessScope(t *testing.T) {
plan, err := makePipelinePlan(nil, scopes, mockApiCLient, connection)
assert.Nil(t, err)
- //planJson, err1 := json.Marshal(plan)
- //assert.Nil(t, err1)
expectPlan := core.PipelinePlan{
core.PipelineStage{
{
[incubator-devlake] 02/03: fix(github): modify bluprint test
Posted by ab...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
abeizn pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit 0ce5b40aaa922a70cb8c443ee748b5abbb4437f4
Author: Yingchu Chen <yi...@merico.dev>
AuthorDate: Wed Oct 12 22:41:35 2022 +0800
fix(github): modify bluprint test
---
plugins/github/api/blueprint.go | 42 ++++++++++---------
plugins/github/api/blueprint_test.go | 80 ++++++++++++++++++++++++++++++------
plugins/helper/api_client.go | 9 ++++
3 files changed, 98 insertions(+), 33 deletions(-)
diff --git a/plugins/github/api/blueprint.go b/plugins/github/api/blueprint.go
index 8acb4d7f..40e334fc 100644
--- a/plugins/github/api/blueprint.go
+++ b/plugins/github/api/blueprint.go
@@ -55,14 +55,25 @@ func MakePipelinePlan(subtaskMetas []core.SubTaskMeta, connectionId uint64, scop
connection.Proxy,
basicRes,
)
- plan, err := makePipelinePlan(subtaskMetas, connectionId, scope, apiClient, connection)
+ if err != nil {
+ return nil, err
+ }
+ plan, err := makePipelinePlan(subtaskMetas, scope, apiClient, connection)
if err != nil {
return nil, err
}
return plan, nil
}
-func makePipelinePlan(subtaskMetas []core.SubTaskMeta, connectionId uint64, scope []*core.BlueprintScopeV100, apiClient *helper.ApiClient, connection *models.GithubConnection) (core.PipelinePlan, errors.Error) {
+
+func makePipelinePlan(subtaskMetas []core.SubTaskMeta, scope []*core.BlueprintScopeV100, apiClient helper.ApiClientGetter, connection *models.GithubConnection) (core.PipelinePlan, errors.Error) {
var err errors.Error
+ var repo *tasks.GithubApiRepo
+ getApiRepoIfNil := func(op *tasks.GithubOptions) (*tasks.GithubApiRepo, errors.Error) {
+ if repo == nil {
+ repo, err = getApiRepo(op, apiClient)
+ }
+ return repo, err
+ }
plan := make(core.PipelinePlan, len(scope))
for i, scopeElem := range scope {
// handle taskOptions and transformationRules, by dumping them to taskOptions
@@ -95,7 +106,7 @@ func makePipelinePlan(subtaskMetas []core.SubTaskMeta, connectionId uint64, scop
if err != nil {
return nil, err
}
- options["connectionId"] = connectionId
+ options["connectionId"] = connection.ID
options["transformationRules"] = transformationRules
// make sure task options is valid
op, err := tasks.DecodeAndValidateTaskOptions(options)
@@ -119,20 +130,12 @@ func makePipelinePlan(subtaskMetas []core.SubTaskMeta, connectionId uint64, scop
// collect git data by gitextractor if CODE was requested
if utils.StringsContains(scopeElem.Entities, core.DOMAIN_TYPE_CODE) {
// here is the tricky part, we have to obtain the repo id beforehand
- if connection == nil {
- connection = new(models.GithubConnection)
- err = connectionHelper.FirstById(connection, connectionId)
- if err != nil {
- return nil, err
- }
- }
token := strings.Split(connection.Token, ",")[0]
- apiRepo, err := getApiRepo(op, apiClient)
+ repo, err = getApiRepoIfNil(op)
if err != nil {
return nil, err
}
-
- cloneUrl, err := errors.Convert01(url.Parse(apiRepo.CloneUrl))
+ cloneUrl, err := errors.Convert01(url.Parse(repo.CloneUrl))
if err != nil {
return nil, err
}
@@ -141,7 +144,7 @@ func makePipelinePlan(subtaskMetas []core.SubTaskMeta, connectionId uint64, scop
Plugin: "gitextractor",
Options: map[string]interface{}{
"url": cloneUrl.String(),
- "repoId": didgen.NewDomainIdGenerator(&models.GithubRepo{}).Generate(connectionId, apiRepo.GithubId),
+ "repoId": didgen.NewDomainIdGenerator(&models.GithubRepo{}).Generate(connection.ID, repo.GithubId),
"proxy": connection.Proxy,
},
})
@@ -162,14 +165,13 @@ func makePipelinePlan(subtaskMetas []core.SubTaskMeta, connectionId uint64, scop
if err != nil {
return nil, err
}
-
- apiRepo, err := getApiRepo(op, apiClient)
+ repo, err = getApiRepoIfNil(op)
if err != nil {
return nil, err
}
doraOption := make(map[string]interface{})
- doraOption["repoId"] = didgen.NewDomainIdGenerator(&models.GithubRepo{}).Generate(connectionId, apiRepo.GithubId)
+ doraOption["repoId"] = didgen.NewDomainIdGenerator(&models.GithubRepo{}).Generate(connection.ID, repo.GithubId)
doraRules := make(map[string]interface{})
doraRules["productionPattern"] = productionPattern
doraOption["transformationRules"] = doraRules
@@ -191,13 +193,13 @@ func makePipelinePlan(subtaskMetas []core.SubTaskMeta, connectionId uint64, scop
return plan, nil
}
-func getApiRepo(op *tasks.GithubOptions, apiClient *helper.ApiClient) (*tasks.GithubApiRepo, errors.Error) {
- apiRepo := new(tasks.GithubApiRepo)
+func getApiRepo(op *tasks.GithubOptions, apiClient helper.ApiClientGetter) (*tasks.GithubApiRepo, errors.Error) {
+ apiRepo := &tasks.GithubApiRepo{}
res, err := apiClient.Get(fmt.Sprintf("repos/%s/%s", op.Owner, op.Repo), nil, nil)
if err != nil {
return nil, err
}
- //defer res.Body.Close()
+ defer res.Body.Close()
if res.StatusCode != http.StatusOK {
return nil, errors.HttpStatus(res.StatusCode).New(fmt.Sprintf("unexpected status code when requesting repo detail from %s", res.Request.URL.String()))
}
diff --git a/plugins/github/api/blueprint_test.go b/plugins/github/api/blueprint_test.go
index 9a1ad60a..08dd0390 100644
--- a/plugins/github/api/blueprint_test.go
+++ b/plugins/github/api/blueprint_test.go
@@ -18,6 +18,7 @@ limitations under the License.
package api
import (
+ "bytes"
"encoding/json"
"github.com/apache/incubator-devlake/mocks"
"github.com/apache/incubator-devlake/models/common"
@@ -26,6 +27,9 @@ import (
"github.com/apache/incubator-devlake/plugins/github/tasks"
"github.com/apache/incubator-devlake/plugins/helper"
"github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/mock"
+ "io"
+ "net/http"
"testing"
)
@@ -46,9 +50,20 @@ func TestProcessScope(t *testing.T) {
Token: "123",
},
}
+ mockApiCLient := mocks.NewApiClientGetter(t)
+ repo := &tasks.GithubApiRepo{
+ GithubId: 12345,
+ CloneUrl: "*this is cloneUrl*",
+ }
+ js, err := json.Marshal(repo)
+ assert.Nil(t, err)
+ res := &http.Response{}
+ res.Body = io.NopCloser(bytes.NewBuffer(js))
+ res.StatusCode = http.StatusOK
+ mockApiCLient.On("Get", "repos/test/testRepo", mock.Anything, mock.Anything).Return(res, nil)
mockMeta := mocks.NewPluginMeta(t)
mockMeta.On("RootPkgPath").Return("github.com/apache/incubator-devlake/plugins/github")
- err := core.RegisterPlugin("github", mockMeta)
+ err = core.RegisterPlugin("github", mockMeta)
assert.Nil(t, err)
bs := &core.BlueprintScopeV100{
Entities: []string{"CODE"},
@@ -66,19 +81,58 @@ func TestProcessScope(t *testing.T) {
"productionPattern": "xxxx"
}`),
}
- apiRepo := &tasks.GithubApiRepo{
- GithubId: 123,
- CloneUrl: "HttpUrlToRepo",
- }
scopes := make([]*core.BlueprintScopeV100, 0)
scopes = append(scopes, bs)
- plan := make(core.PipelinePlan, len(scopes))
- for i, scopeElem := range scopes {
- plan, err = processScope(nil, 1, scopeElem, i, plan, apiRepo, connection)
- assert.Nil(t, err)
+ plan, err := makePipelinePlan(nil, scopes, mockApiCLient, connection)
+ assert.Nil(t, err)
+
+ //planJson, err1 := json.Marshal(plan)
+ //assert.Nil(t, err1)
+ expectPlan := core.PipelinePlan{
+ core.PipelineStage{
+ {
+ Plugin: "github",
+ Subtasks: []string{},
+ Options: map[string]interface{}{
+ "connectionId": uint64(1),
+ "owner": "test",
+ "repo": "testRepo",
+ "transformationRules": map[string]interface{}{
+ "prType": "hey,man,wasup",
+ },
+ },
+ },
+ {
+ Plugin: "gitextractor",
+ Options: map[string]interface{}{
+ "proxy": "",
+ "repoId": "github:GithubRepo:1:12345",
+ "url": "//git:123@%2Athis%20is%20cloneUrl%2A",
+ },
+ },
+ },
+ core.PipelineStage{
+ {
+ Plugin: "refdiff",
+ Options: map[string]interface{}{
+ "tagsLimit": float64(10),
+ "tagsOrder": "reverse semver",
+ "tagsPattern": "pattern",
+ },
+ },
+ },
+ core.PipelineStage{
+ {
+ Plugin: "dora",
+ Subtasks: []string{"EnrichTaskEnv"},
+ Options: map[string]interface{}{
+ "repoId": "github:GithubRepo:1:12345",
+ "transformationRules": map[string]interface{}{
+ "productionPattern": "xxxx",
+ },
+ },
+ },
+ },
}
- planJson, err1 := json.Marshal(plan)
- assert.Nil(t, err1)
- expectPlan := `[[{"plugin":"github","subtasks":[],"options":{"connectionId":1,"owner":"test","repo":"testRepo","transformationRules":{"prType":"hey,man,wasup"}}},{"plugin":"gitextractor","subtasks":null,"options":{"proxy":"","repoId":"github:GithubRepo:1:123","url":"//git:123@HttpUrlToRepo"}}],[{"plugin":"refdiff","subtasks":null,"options":{"tagsLimit":10,"tagsOrder":"reverse semver","tagsPattern":"pattern"}}],[{"plugin":"dora","subtasks":["EnrichTaskEnv"],"options":{"repoId":"github:Gi [...]
- assert.Equal(t, expectPlan, string(planJson))
+ assert.Equal(t, expectPlan, plan)
}
diff --git a/plugins/helper/api_client.go b/plugins/helper/api_client.go
index 4708b8fc..09afbd55 100644
--- a/plugins/helper/api_client.go
+++ b/plugins/helper/api_client.go
@@ -40,6 +40,15 @@ import (
// ErrIgnoreAndContinue is a error which should be ignored
var ErrIgnoreAndContinue = errors.Default.New("ignore and continue")
+// ApiClientGetter will be used for uint test
+type ApiClientGetter interface {
+ Get(
+ path string,
+ query url.Values,
+ headers http.Header,
+ ) (*http.Response, errors.Error)
+}
+
// ApiClient is designed for simple api requests
type ApiClient struct {
client *http.Client