You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by ab...@apache.org on 2022/09/05 13:38:53 UTC
[incubator-devlake] branch main updated: feat(gitlab): add pipeline repo
This is an automated email from the ASF dual-hosted git repository.
abeizn pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 43cae213 feat(gitlab): add pipeline repo
43cae213 is described below
commit 43cae2137b11de3840668b4e0902d59f3ef61a4f
Author: Yingchu Chen <yi...@merico.dev>
AuthorDate: Mon Sep 5 20:02:41 2022 +0800
feat(gitlab): add pipeline repo
relate to #2951
---
plugins/gitlab/e2e/pipelines_test.go | 4 +-
.../e2e/snapshot_tables/_tool_gitlab_pipelines.csv | 42 +++++++-------
plugins/gitlab/impl/impl.go | 2 +
plugins/gitlab/models/pipeline.go | 13 +++++
plugins/gitlab/tasks/pipeline_convertor.go | 11 +---
plugins/gitlab/tasks/pipeline_extractor.go | 44 +++++++-------
..._convertor.go => pipeline_project_convertor.go} | 67 ++++++----------------
7 files changed, 78 insertions(+), 105 deletions(-)
diff --git a/plugins/gitlab/e2e/pipelines_test.go b/plugins/gitlab/e2e/pipelines_test.go
index 06da4ace..7c0ccc2a 100644
--- a/plugins/gitlab/e2e/pipelines_test.go
+++ b/plugins/gitlab/e2e/pipelines_test.go
@@ -44,6 +44,7 @@ func TestGitlabPipelineDataFlow(t *testing.T) {
// verify extraction
dataflowTester.FlushTabler(&models.GitlabPipeline{})
+ dataflowTester.FlushTabler(&models.GitlabPipelineProject{})
dataflowTester.Subtask(tasks.ExtractApiPipelinesMeta, taskData)
dataflowTester.VerifyTable(
models.GitlabPipeline{},
@@ -51,11 +52,8 @@ func TestGitlabPipelineDataFlow(t *testing.T) {
[]string{
"connection_id",
"gitlab_id",
- "project_id",
"gitlab_created_at",
"status",
- "ref",
- "sha",
"web_url",
"duration",
"started_at",
diff --git a/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_pipelines.csv b/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_pipelines.csv
index 598620b5..08518bc7 100644
--- a/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_pipelines.csv
+++ b/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_pipelines.csv
@@ -1,21 +1,21 @@
-connection_id,gitlab_id,project_id,gitlab_created_at,status,ref,sha,web_url,duration,started_at,finished_at,coverage,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
-1,457474837,12345678,2022-01-27T10:07:09.429+00:00,failed,renovate/pin-dependencies,b1b82852d48b516a18e56c5bab0ebf54b8f4ccfd,https://gitlab.com/merico-dev/ee/charts/-/pipelines/457474837,0,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,152,
-1,457474996,12345678,2022-01-27T10:07:18.884+00:00,failed,renovate/jest-monorepo,739ab912e6e1ed27cecd8a2d00bfd6fa52afd90a,https://gitlab.com/merico-dev/ee/charts/-/pipelines/457474996,0,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,153,
-1,457475160,12345678,2022-01-27T10:07:26.435+00:00,failed,renovate/lodash-monorepo,44d127e0ab7dbc4bc259b55929c9d00b62fc3bf4,https://gitlab.com/merico-dev/ee/charts/-/pipelines/457475160,0,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,154,
-1,457475337,12345678,2022-01-27T10:07:36.502+00:00,failed,renovate/shx-0.x,10a6464b6bd2cf4b59b8ac37ce1466e013f5a20d,https://gitlab.com/merico-dev/ee/charts/-/pipelines/457475337,1,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,155,
-1,485811050,12345678,2022-03-07T06:26:42.109+00:00,failed,EE-7121,c791ea6949d6b4aadf79b15ba666cb690c6527ac,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485811050,0,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,156,
-1,485811059,12345678,2022-03-07T06:26:43.784+00:00,failed,refs/merge-requests/74/head,c791ea6949d6b4aadf79b15ba666cb690c6527ac,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485811059,0,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,157,
-1,485813816,12345678,2022-03-07T06:33:56.824+00:00,failed,EE-7121,ecc7c0b2874c812ed882c9effbbda26e0abc7110,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485813816,0,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,158,
-1,485813830,12345678,2022-03-07T06:33:58.889+00:00,failed,refs/merge-requests/74/head,ecc7c0b2874c812ed882c9effbbda26e0abc7110,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485813830,0,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,159,
-1,485814501,12345678,2022-03-07T06:35:28.111+00:00,failed,EE-7121,6a3346f8434cc65fbe3f7a80a0edec5b4014a733,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485814501,0,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,160,
-1,485814516,12345678,2022-03-07T06:35:31.255+00:00,failed,refs/merge-requests/74/head,6a3346f8434cc65fbe3f7a80a0edec5b4014a733,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485814516,0,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,161,
-1,485814871,12345678,2022-03-07T06:36:50.020+00:00,failed,refs/merge-requests/74/head,f731732471961fed061a01fa99631f69e861c4a2,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485814871,42,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,162,
-1,485817670,12345678,2022-03-07T06:45:09.471+00:00,failed,refs/merge-requests/74/head,5b95c5aebce1eae6a1b95ecf6fbc870851455375,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485817670,1956,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,163,
-1,485837602,12345678,2022-03-07T07:20:45.859+00:00,failed,refs/merge-requests/74/head,a79e2aaeee9916338a11b665cb4201f24627f3f1,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485837602,434,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,164,
-1,485842553,12345678,2022-03-07T07:30:47.018+00:00,failed,refs/merge-requests/74/head,9e0934cedcd83abab8d34c5bb0f597c7837a55f0,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485842553,287,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,165,
-1,485845850,12345678,2022-03-07T07:38:58.611+00:00,failed,refs/merge-requests/74/head,b3b581940f6d4d3226db31e48eab2b3b73f4c6db,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485845850,419,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,166,
-1,485852752,12345678,2022-03-07T07:46:09.385+00:00,failed,refs/merge-requests/74/head,82df93eeb72c71271dad5a85359f39661de899fe,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485852752,319,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,167,
-1,485865876,12345678,2022-03-07T08:04:56.406+00:00,failed,refs/merge-requests/74/head,db45efd231dbaca1337bd29bc1f65f404754e6bd,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485865876,480,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,168,
-1,485877118,12345678,2022-03-07T08:22:48.943+00:00,failed,refs/merge-requests/74/head,09f81b1b2d083411c0bfecd32d7728479b594503,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485877118,289,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,169,
-1,485905167,12345678,2022-03-07T09:02:09.994+00:00,failed,refs/merge-requests/74/head,cce432655861efafcd05edc8de09e305c772d2b8,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485905167,687,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,170,
-1,485932863,12345678,2022-03-07T09:34:57.476+00:00,success,refs/merge-requests/74/head,12fc3a42080bb98ca520817bd4fe0ca33c0bb279,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485932863,398,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,171,
+connection_id,gitlab_id,gitlab_created_at,status,web_url,duration,started_at,finished_at,coverage,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
+1,457474837,2022-01-27T10:07:09.429+00:00,failed,https://gitlab.com/merico-dev/ee/charts/-/pipelines/457474837,0,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,152,
+1,457474996,2022-01-27T10:07:18.884+00:00,failed,https://gitlab.com/merico-dev/ee/charts/-/pipelines/457474996,0,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,153,
+1,457475160,2022-01-27T10:07:26.435+00:00,failed,https://gitlab.com/merico-dev/ee/charts/-/pipelines/457475160,0,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,154,
+1,457475337,2022-01-27T10:07:36.502+00:00,failed,https://gitlab.com/merico-dev/ee/charts/-/pipelines/457475337,1,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,155,
+1,485811050,2022-03-07T06:26:42.109+00:00,failed,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485811050,0,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,156,
+1,485811059,2022-03-07T06:26:43.784+00:00,failed,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485811059,0,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,157,
+1,485813816,2022-03-07T06:33:56.824+00:00,failed,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485813816,0,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,158,
+1,485813830,2022-03-07T06:33:58.889+00:00,failed,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485813830,0,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,159,
+1,485814501,2022-03-07T06:35:28.111+00:00,failed,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485814501,0,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,160,
+1,485814516,2022-03-07T06:35:31.255+00:00,failed,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485814516,0,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,161,
+1,485814871,2022-03-07T06:36:50.020+00:00,failed,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485814871,42,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,162,
+1,485817670,2022-03-07T06:45:09.471+00:00,failed,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485817670,1956,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,163,
+1,485837602,2022-03-07T07:20:45.859+00:00,failed,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485837602,434,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,164,
+1,485842553,2022-03-07T07:30:47.018+00:00,failed,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485842553,287,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,165,
+1,485845850,2022-03-07T07:38:58.611+00:00,failed,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485845850,419,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,166,
+1,485852752,2022-03-07T07:46:09.385+00:00,failed,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485852752,319,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,167,
+1,485865876,2022-03-07T08:04:56.406+00:00,failed,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485865876,480,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,168,
+1,485877118,2022-03-07T08:22:48.943+00:00,failed,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485877118,289,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,169,
+1,485905167,2022-03-07T09:02:09.994+00:00,failed,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485905167,687,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,170,
+1,485932863,2022-03-07T09:34:57.476+00:00,success,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485932863,398,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,171,
diff --git a/plugins/gitlab/impl/impl.go b/plugins/gitlab/impl/impl.go
index 99086a22..82e1f722 100644
--- a/plugins/gitlab/impl/impl.go
+++ b/plugins/gitlab/impl/impl.go
@@ -62,6 +62,7 @@ func (plugin Gitlab) GetTablesInfo() []core.Tabler {
&models.GitlabMrLabel{},
&models.GitlabMrNote{},
&models.GitlabPipeline{},
+ &models.GitlabPipelineProject{},
&models.GitlabProject{},
&models.GitlabProjectCommit{},
&models.GitlabReviewer{},
@@ -102,6 +103,7 @@ func (plugin Gitlab) SubTaskMetas() []core.SubTaskMeta {
tasks.ConvertMrLabelsMeta,
tasks.ConvertCommitsMeta,
tasks.ConvertPipelineMeta,
+ tasks.ConvertPipelineProjectMeta,
tasks.ConvertJobMeta,
}
}
diff --git a/plugins/gitlab/models/pipeline.go b/plugins/gitlab/models/pipeline.go
index 8e0de87f..6a7cb623 100644
--- a/plugins/gitlab/models/pipeline.go
+++ b/plugins/gitlab/models/pipeline.go
@@ -46,3 +46,16 @@ type GitlabPipeline struct {
func (GitlabPipeline) TableName() string {
return "_tool_gitlab_pipelines"
}
+
+type GitlabPipelineProject struct {
+ ConnectionId uint64 `gorm:"primaryKey"`
+ PipelineId int `gorm:"primaryKey"`
+ ProjectId int `gorm:"primaryKey;type:varchar(255)"`
+ Ref string `gorm:"type:varchar(255)"`
+ Sha string `gorm:"type:varchar(255)"`
+ common.NoPKModel
+}
+
+func (GitlabPipelineProject) TableName() string {
+ return "_tool_gitlab_pipeline_projects"
+}
diff --git a/plugins/gitlab/tasks/pipeline_convertor.go b/plugins/gitlab/tasks/pipeline_convertor.go
index f3e5a787..c5579f71 100644
--- a/plugins/gitlab/tasks/pipeline_convertor.go
+++ b/plugins/gitlab/tasks/pipeline_convertor.go
@@ -60,7 +60,7 @@ func ConvertPipelines(taskCtx core.SubTaskContext) error {
ConnectionId: data.Options.ConnectionId,
ProjectId: data.Options.ProjectId,
},
- Table: RAW_USER_TABLE,
+ Table: RAW_PIPELINE_TABLE,
},
Convert: func(inputRow interface{}) ([]interface{}, error) {
gitlabPipeline := inputRow.(*gitlabModels.GitlabPipeline)
@@ -74,11 +74,7 @@ func ConvertPipelines(taskCtx core.SubTaskContext) error {
DomainEntity: domainlayer.DomainEntity{
Id: pipelineIdGen.Generate(data.Options.ConnectionId, gitlabPipeline.GitlabId),
},
-
- Name: fmt.Sprintf("%d", gitlabPipeline.GitlabId),
- CommitSha: gitlabPipeline.Sha,
- Branch: gitlabPipeline.Ref,
- Repo: fmt.Sprintf("%d", gitlabPipeline.ProjectId),
+ Name: fmt.Sprintf("%d", gitlabPipeline.GitlabId),
Result: devops.GetResult(&devops.ResultRule{
Failed: []string{"failed"},
Abort: []string{"canceled", "skipped"},
@@ -88,8 +84,7 @@ func ConvertPipelines(taskCtx core.SubTaskContext) error {
InProgress: []string{"created", "waiting_for_resource", "preparing", "pending", "running", "manual", "scheduled"},
Default: devops.DONE,
}, gitlabPipeline.Status),
- Type: "CI/CD",
-
+ Type: "CI/CD",
CreatedDate: createdAt,
FinishedDate: gitlabPipeline.GitlabUpdatedAt,
}
diff --git a/plugins/gitlab/tasks/pipeline_extractor.go b/plugins/gitlab/tasks/pipeline_extractor.go
index a3a23543..3efc44a4 100644
--- a/plugins/gitlab/tasks/pipeline_extractor.go
+++ b/plugins/gitlab/tasks/pipeline_extractor.go
@@ -75,16 +75,31 @@ func ExtractApiPipelines(taskCtx core.SubTaskContext) error {
duration := int(gitlabApiPipeline.UpdatedAt.ToTime().Sub(gitlabApiPipeline.CreatedAt.ToTime()).Seconds())
gitlabApiPipeline.Duration = duration
- gitlabPipeline, err := convertPipeline(gitlabApiPipeline, data.Options.ProjectId)
+ gitlabPipeline := &models.GitlabPipeline{
+ GitlabId: gitlabApiPipeline.Id,
+ WebUrl: gitlabApiPipeline.WebUrl,
+ Status: gitlabApiPipeline.Status,
+ GitlabCreatedAt: helper.Iso8601TimeToTime(gitlabApiPipeline.CreatedAt),
+ GitlabUpdatedAt: helper.Iso8601TimeToTime(gitlabApiPipeline.UpdatedAt),
+ StartedAt: helper.Iso8601TimeToTime(gitlabApiPipeline.StartedAt),
+ FinishedAt: helper.Iso8601TimeToTime(gitlabApiPipeline.FinishedAt),
+ Duration: gitlabApiPipeline.Duration,
+ ConnectionId: data.Options.ConnectionId,
+ }
if err != nil {
return nil, err
}
- // use data.Options.ProjectId to set the value of ProjectId for it
- gitlabPipeline.ProjectId = data.Options.ProjectId
- gitlabPipeline.ConnectionId = data.Options.ConnectionId
- results := make([]interface{}, 0, 1)
- results = append(results, gitlabPipeline)
+ pipelineProject := &models.GitlabPipelineProject{
+ ConnectionId: data.Options.ConnectionId,
+ PipelineId: gitlabPipeline.GitlabId,
+ ProjectId: data.Options.ProjectId,
+ Ref: gitlabApiPipeline.Ref,
+ Sha: gitlabApiPipeline.Sha,
+ }
+
+ results := make([]interface{}, 0, 2)
+ results = append(results, gitlabPipeline, pipelineProject)
return results, nil
},
@@ -96,20 +111,3 @@ func ExtractApiPipelines(taskCtx core.SubTaskContext) error {
return extractor.Execute()
}
-
-func convertPipeline(pipeline *ApiPipeline, projectId int) (*models.GitlabPipeline, error) {
- gitlabPipeline := &models.GitlabPipeline{
- GitlabId: pipeline.Id,
- ProjectId: projectId,
- Ref: pipeline.Ref,
- Sha: pipeline.Sha,
- WebUrl: pipeline.WebUrl,
- Status: pipeline.Status,
- GitlabCreatedAt: helper.Iso8601TimeToTime(pipeline.CreatedAt),
- GitlabUpdatedAt: helper.Iso8601TimeToTime(pipeline.UpdatedAt),
- StartedAt: helper.Iso8601TimeToTime(pipeline.StartedAt),
- FinishedAt: helper.Iso8601TimeToTime(pipeline.FinishedAt),
- Duration: pipeline.Duration,
- }
- return gitlabPipeline, nil
-}
diff --git a/plugins/gitlab/tasks/pipeline_convertor.go b/plugins/gitlab/tasks/pipeline_project_convertor.go
similarity index 53%
copy from plugins/gitlab/tasks/pipeline_convertor.go
copy to plugins/gitlab/tasks/pipeline_project_convertor.go
index f3e5a787..005d71cf 100644
--- a/plugins/gitlab/tasks/pipeline_convertor.go
+++ b/plugins/gitlab/tasks/pipeline_project_convertor.go
@@ -18,10 +18,6 @@ limitations under the License.
package tasks
import (
- "fmt"
- "reflect"
- "time"
-
"github.com/apache/incubator-devlake/models/domainlayer"
"github.com/apache/incubator-devlake/models/domainlayer/devops"
"github.com/apache/incubator-devlake/models/domainlayer/didgen"
@@ -29,21 +25,22 @@ import (
"github.com/apache/incubator-devlake/plugins/core/dal"
gitlabModels "github.com/apache/incubator-devlake/plugins/gitlab/models"
"github.com/apache/incubator-devlake/plugins/helper"
+ "reflect"
)
-var ConvertPipelineMeta = core.SubTaskMeta{
- Name: "convertPipelines",
- EntryPoint: ConvertPipelines,
+var ConvertPipelineProjectMeta = core.SubTaskMeta{
+ Name: "convertPipelineProjects",
+ EntryPoint: ConvertPipelineProjects,
EnabledByDefault: true,
- Description: "Convert tool layer table gitlab_pipeline into domain layer table pipeline",
+ Description: "Convert tool layer table gitlab_pipeline_project into domain layer table pipeline",
DomainTypes: []string{core.DOMAIN_TYPE_CROSS},
}
-func ConvertPipelines(taskCtx core.SubTaskContext) error {
+func ConvertPipelineProjects(taskCtx core.SubTaskContext) error {
db := taskCtx.GetDal()
data := taskCtx.GetData().(*GitlabTaskData)
- cursor, err := db.Cursor(dal.From(gitlabModels.GitlabPipeline{}))
+ cursor, err := db.Cursor(dal.From(gitlabModels.GitlabPipelineProject{}))
if err != nil {
return err
}
@@ -52,7 +49,7 @@ func ConvertPipelines(taskCtx core.SubTaskContext) error {
pipelineIdGen := didgen.NewDomainIdGenerator(&gitlabModels.GitlabPipeline{})
converter, err := helper.NewDataConverter(helper.DataConverterArgs{
- InputRowType: reflect.TypeOf(gitlabModels.GitlabPipeline{}),
+ InputRowType: reflect.TypeOf(gitlabModels.GitlabPipelineProject{}),
Input: cursor,
RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
Ctx: taskCtx,
@@ -60,53 +57,23 @@ func ConvertPipelines(taskCtx core.SubTaskContext) error {
ConnectionId: data.Options.ConnectionId,
ProjectId: data.Options.ProjectId,
},
- Table: RAW_USER_TABLE,
+ Table: RAW_PIPELINE_TABLE,
},
Convert: func(inputRow interface{}) ([]interface{}, error) {
- gitlabPipeline := inputRow.(*gitlabModels.GitlabPipeline)
-
- createdAt := time.Now()
- if gitlabPipeline.GitlabCreatedAt != nil {
- createdAt = *gitlabPipeline.GitlabCreatedAt
- }
+ gitlabPipelineProject := inputRow.(*gitlabModels.GitlabPipelineProject)
- domainPipeline := &devops.CICDPipeline{
+ domainPipelineRepo := &devops.CiCDPipelineRepo{
DomainEntity: domainlayer.DomainEntity{
- Id: pipelineIdGen.Generate(data.Options.ConnectionId, gitlabPipeline.GitlabId),
+ Id: pipelineIdGen.Generate(data.Options.ConnectionId, gitlabPipelineProject.PipelineId),
},
-
- Name: fmt.Sprintf("%d", gitlabPipeline.GitlabId),
- CommitSha: gitlabPipeline.Sha,
- Branch: gitlabPipeline.Ref,
- Repo: fmt.Sprintf("%d", gitlabPipeline.ProjectId),
- Result: devops.GetResult(&devops.ResultRule{
- Failed: []string{"failed"},
- Abort: []string{"canceled", "skipped"},
- Default: devops.SUCCESS,
- }, gitlabPipeline.Status),
- Status: devops.GetStatus(&devops.StatusRule{
- InProgress: []string{"created", "waiting_for_resource", "preparing", "pending", "running", "manual", "scheduled"},
- Default: devops.DONE,
- }, gitlabPipeline.Status),
- Type: "CI/CD",
-
- CreatedDate: createdAt,
- FinishedDate: gitlabPipeline.GitlabUpdatedAt,
+ CommitSha: gitlabPipelineProject.Sha,
+ Branch: gitlabPipelineProject.Ref,
+ Repo: didgen.NewDomainIdGenerator(&gitlabModels.GitlabProject{}).
+ Generate(gitlabPipelineProject.ConnectionId, gitlabPipelineProject.PipelineId),
}
- // rebuild the FinishedDate and DurationSec by Status
- finishedAt := time.Now()
- if domainPipeline.Status != devops.DONE {
- domainPipeline.FinishedDate = nil
- } else if gitlabPipeline.GitlabUpdatedAt != nil {
- finishedAt = *gitlabPipeline.GitlabUpdatedAt
- }
- durationTime := finishedAt.Sub(createdAt)
-
- domainPipeline.DurationSec = uint64(durationTime.Seconds())
-
return []interface{}{
- domainPipeline,
+ domainPipelineRepo,
}, nil
},
})