You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by ab...@apache.org on 2022/09/29 02:36:46 UTC
[incubator-devlake] branch main updated: fix(plugins): add scope for subtasks
This is an automated email from the ASF dual-hosted git repository.
abeizn pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new e13f57f3 fix(plugins): add scope for subtasks
e13f57f3 is described below
commit e13f57f3071af93db2ae0f55c9bc78f359a7fb08
Author: Yingchu Chen <yi...@merico.dev>
AuthorDate: Tue Sep 27 11:56:26 2022 +0800
fix(plugins): add scope for subtasks
---
plugins/github/tasks/account_convertor.go | 2 +-
plugins/github/tasks/cicd_job_collector.go | 2 +-
plugins/github/tasks/cicd_pipeline_convertor.go | 4 ++--
plugins/github/tasks/milestone_collector.go | 5 +++--
plugins/github/tasks/milestone_converter.go | 2 +-
plugins/github/tasks/pr_issue_enricher.go | 2 +-
plugins/gitlab/e2e/pipelines_test.go | 2 +-
plugins/gitlab/impl/impl.go | 2 +-
plugins/gitlab/tasks/account_convertor.go | 2 +-
plugins/gitlab/tasks/job_convertor.go | 3 ++-
...ct_convertor.go => pipeline_commit_convertor.go} | 21 +++++++++++----------
plugins/gitlab/tasks/project_convertor.go | 2 +-
12 files changed, 26 insertions(+), 23 deletions(-)
diff --git a/plugins/github/tasks/account_convertor.go b/plugins/github/tasks/account_convertor.go
index 4b302bfc..e8643226 100644
--- a/plugins/github/tasks/account_convertor.go
+++ b/plugins/github/tasks/account_convertor.go
@@ -52,7 +52,7 @@ func ConvertAccounts(taskCtx core.SubTaskContext) errors.Error {
db := taskCtx.GetDal()
data := taskCtx.GetData().(*GithubTaskData)
- cursor, err := db.Cursor(dal.From(&githubModels.GithubAccount{}))
+ cursor, err := db.Cursor(dal.From(&githubModels.GithubAccount{}), dal.Where("connection_id = ?", data.Options.ConnectionId))
if err != nil {
return err
}
diff --git a/plugins/github/tasks/cicd_job_collector.go b/plugins/github/tasks/cicd_job_collector.go
index 9ec0c87e..c95c0952 100644
--- a/plugins/github/tasks/cicd_job_collector.go
+++ b/plugins/github/tasks/cicd_job_collector.go
@@ -48,7 +48,7 @@ func CollectJobs(taskCtx core.SubTaskContext) errors.Error {
cursor, err := db.Cursor(
dal.Select("id"),
dal.From(models.GithubRun{}.TableName()),
- dal.Where("repo_id = ?", data.Repo.GithubId),
+ dal.Where("repo_id = ? and connection_id = ?", data.Repo.GithubId, data.Options.ConnectionId),
)
if err != nil {
return err
diff --git a/plugins/github/tasks/cicd_pipeline_convertor.go b/plugins/github/tasks/cicd_pipeline_convertor.go
index faf83539..b447ddfc 100644
--- a/plugins/github/tasks/cicd_pipeline_convertor.go
+++ b/plugins/github/tasks/cicd_pipeline_convertor.go
@@ -94,7 +94,7 @@ func ConvertPipelines(taskCtx core.SubTaskContext) errors.Error {
domainPipeline.DurationSec = uint64(line.GithubUpdatedAt.Sub(*line.GithubCreatedAt).Seconds())
}
- domainPipelineProject := &devops.CiCDPipelineCommit{
+ domainPipelineCommit := &devops.CiCDPipelineCommit{
PipelineId: runIdGen.Generate(
data.Options.ConnectionId, line.RepoId, line.ID),
CommitSha: line.HeadSha,
@@ -104,7 +104,7 @@ func ConvertPipelines(taskCtx core.SubTaskContext) errors.Error {
return []interface{}{
domainPipeline,
- domainPipelineProject,
+ domainPipelineCommit,
}, nil
},
})
diff --git a/plugins/github/tasks/milestone_collector.go b/plugins/github/tasks/milestone_collector.go
index aa7f764a..0d5d10b2 100644
--- a/plugins/github/tasks/milestone_collector.go
+++ b/plugins/github/tasks/milestone_collector.go
@@ -45,8 +45,9 @@ func CollectApiMilestones(taskCtx core.SubTaskContext) errors.Error {
RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
Ctx: taskCtx,
Params: GithubApiParams{
- Owner: data.Options.Owner,
- Repo: data.Options.Repo,
+ ConnectionId: data.Options.ConnectionId,
+ Owner: data.Options.Owner,
+ Repo: data.Options.Repo,
},
Table: RAW_MILESTONE_TABLE,
},
diff --git a/plugins/github/tasks/milestone_converter.go b/plugins/github/tasks/milestone_converter.go
index 0ab609cd..4625b86d 100644
--- a/plugins/github/tasks/milestone_converter.go
+++ b/plugins/github/tasks/milestone_converter.go
@@ -53,7 +53,7 @@ func ConvertMilestones(taskCtx core.SubTaskContext) errors.Error {
dal.Select("gi.github_id, gm.*"),
dal.From("_tool_github_issues gi"),
dal.Join("JOIN _tool_github_milestones gm ON gm.milestone_id = gi.milestone_id"),
- dal.Where("gm.repo_id = ?", repoId),
+ dal.Where("gm.repo_id = ? and gm.connection_id = ?", repoId, data.Options.ConnectionId),
}
cursor, err := db.Cursor(clauses...)
if err != nil {
diff --git a/plugins/github/tasks/pr_issue_enricher.go b/plugins/github/tasks/pr_issue_enricher.go
index e38ca1d4..ca5855db 100644
--- a/plugins/github/tasks/pr_issue_enricher.go
+++ b/plugins/github/tasks/pr_issue_enricher.go
@@ -55,7 +55,7 @@ func EnrichPullRequestIssues(taskCtx core.SubTaskContext) (err errors.Error) {
}
}
charPattern := regexp.MustCompile(`[\/a-zA-Z\s,]+`)
- cursor, err := db.Cursor(dal.From(&githubModels.GithubPullRequest{}), dal.Where("repo_id = ?", repoId))
+ cursor, err := db.Cursor(dal.From(&githubModels.GithubPullRequest{}), dal.Where("repo_id = ? and connection_id = ?", repoId, data.Options.ConnectionId))
if err != nil {
return err
}
diff --git a/plugins/gitlab/e2e/pipelines_test.go b/plugins/gitlab/e2e/pipelines_test.go
index a85c342d..cacd5cf7 100644
--- a/plugins/gitlab/e2e/pipelines_test.go
+++ b/plugins/gitlab/e2e/pipelines_test.go
@@ -86,7 +86,7 @@ func TestGitlabPipelineDataFlow(t *testing.T) {
dataflowTester.FlushTabler(&devops.CICDPipeline{})
dataflowTester.FlushTabler(&devops.CiCDPipelineCommit{})
dataflowTester.Subtask(tasks.ConvertPipelineMeta, taskData)
- dataflowTester.Subtask(tasks.ConvertPipelineProjectMeta, taskData)
+ dataflowTester.Subtask(tasks.ConvertPipelineCommitMeta, taskData)
dataflowTester.VerifyTable(
devops.CICDPipeline{},
"./snapshot_tables/cicd_pipelines.csv",
diff --git a/plugins/gitlab/impl/impl.go b/plugins/gitlab/impl/impl.go
index ddb09b33..9127dffb 100644
--- a/plugins/gitlab/impl/impl.go
+++ b/plugins/gitlab/impl/impl.go
@@ -103,7 +103,7 @@ func (plugin Gitlab) SubTaskMetas() []core.SubTaskMeta {
tasks.ConvertMrLabelsMeta,
tasks.ConvertCommitsMeta,
tasks.ConvertPipelineMeta,
- tasks.ConvertPipelineProjectMeta,
+ tasks.ConvertPipelineCommitMeta,
tasks.ConvertJobMeta,
}
}
diff --git a/plugins/gitlab/tasks/account_convertor.go b/plugins/gitlab/tasks/account_convertor.go
index 7467e867..a3d76900 100644
--- a/plugins/gitlab/tasks/account_convertor.go
+++ b/plugins/gitlab/tasks/account_convertor.go
@@ -42,7 +42,7 @@ func ConvertAccounts(taskCtx core.SubTaskContext) errors.Error {
db := taskCtx.GetDal()
data := taskCtx.GetData().(*GitlabTaskData)
- cursor, err := db.Cursor(dal.From(gitlabModels.GitlabAccount{}))
+ cursor, err := db.Cursor(dal.From(gitlabModels.GitlabAccount{}), dal.Where("connection_id = ?", data.Options.ConnectionId))
if err != nil {
return err
}
diff --git a/plugins/gitlab/tasks/job_convertor.go b/plugins/gitlab/tasks/job_convertor.go
index d2dc53f2..09cfb8ca 100644
--- a/plugins/gitlab/tasks/job_convertor.go
+++ b/plugins/gitlab/tasks/job_convertor.go
@@ -52,7 +52,8 @@ func ConvertJobs(taskCtx core.SubTaskContext) (err errors.Error) {
}
}
- cursor, err := db.Cursor(dal.From(gitlabModels.GitlabJob{}))
+ cursor, err := db.Cursor(dal.From(gitlabModels.GitlabJob{}),
+ dal.Where("project_id = ? and connection_id = ?", data.Options.ProjectId, data.Options.ConnectionId))
if err != nil {
return err
}
diff --git a/plugins/gitlab/tasks/pipeline_project_convertor.go b/plugins/gitlab/tasks/pipeline_commit_convertor.go
similarity index 80%
rename from plugins/gitlab/tasks/pipeline_project_convertor.go
rename to plugins/gitlab/tasks/pipeline_commit_convertor.go
index 11da9bcc..923ec826 100644
--- a/plugins/gitlab/tasks/pipeline_project_convertor.go
+++ b/plugins/gitlab/tasks/pipeline_commit_convertor.go
@@ -28,19 +28,20 @@ import (
"reflect"
)
-var ConvertPipelineProjectMeta = core.SubTaskMeta{
- Name: "convertPipelineProjects",
- EntryPoint: ConvertPipelineProjects,
+var ConvertPipelineCommitMeta = core.SubTaskMeta{
+ Name: "convertPipelineCommits",
+ EntryPoint: ConvertPipelineCommits,
EnabledByDefault: true,
Description: "Convert tool layer table gitlab_pipeline_project into domain layer table pipeline",
DomainTypes: []string{core.DOMAIN_TYPE_CROSS},
}
-func ConvertPipelineProjects(taskCtx core.SubTaskContext) errors.Error {
+func ConvertPipelineCommits(taskCtx core.SubTaskContext) errors.Error {
db := taskCtx.GetDal()
data := taskCtx.GetData().(*GitlabTaskData)
- cursor, err := db.Cursor(dal.From(gitlabModels.GitlabPipelineProject{}))
+ cursor, err := db.Cursor(dal.From(gitlabModels.GitlabPipelineProject{}),
+ dal.Where("project_id = ? and connection_id = ?", data.Options.ProjectId, data.Options.ConnectionId))
if err != nil {
return err
}
@@ -60,14 +61,14 @@ func ConvertPipelineProjects(taskCtx core.SubTaskContext) errors.Error {
Table: RAW_PIPELINE_TABLE,
},
Convert: func(inputRow interface{}) ([]interface{}, errors.Error) {
- gitlabPipelineProject := inputRow.(*gitlabModels.GitlabPipelineProject)
+ gitlabPipelineCommit := inputRow.(*gitlabModels.GitlabPipelineProject)
domainPipelineCommit := &devops.CiCDPipelineCommit{
- PipelineId: pipelineIdGen.Generate(data.Options.ConnectionId, gitlabPipelineProject.PipelineId),
- CommitSha: gitlabPipelineProject.Sha,
- Branch: gitlabPipelineProject.Ref,
+ PipelineId: pipelineIdGen.Generate(data.Options.ConnectionId, gitlabPipelineCommit.PipelineId),
+ CommitSha: gitlabPipelineCommit.Sha,
+ Branch: gitlabPipelineCommit.Ref,
RepoId: didgen.NewDomainIdGenerator(&gitlabModels.GitlabProject{}).
- Generate(gitlabPipelineProject.ConnectionId, gitlabPipelineProject.ProjectId),
+ Generate(gitlabPipelineCommit.ConnectionId, gitlabPipelineCommit.ProjectId),
}
return []interface{}{
diff --git a/plugins/gitlab/tasks/project_convertor.go b/plugins/gitlab/tasks/project_convertor.go
index b71c38c4..7c6cff14 100644
--- a/plugins/gitlab/tasks/project_convertor.go
+++ b/plugins/gitlab/tasks/project_convertor.go
@@ -46,7 +46,7 @@ func ConvertApiProjects(taskCtx core.SubTaskContext) errors.Error {
db := taskCtx.GetDal()
clauses := []dal.Clause{
dal.From(&models.GitlabProject{}),
- dal.Where("gitlab_id=?", data.Options.ProjectId),
+ dal.Where("gitlab_id=? and connection_id = ?", data.Options.ProjectId, data.Options.ConnectionId),
}
cursor, err := db.Cursor(clauses...)