You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by wa...@apache.org on 2022/11/03 07:19:21 UTC
[incubator-devlake] branch main updated: fix(gitlab): modify pipeline (#3659)
This is an automated email from the ASF dual-hosted git repository.
warren pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 7a64e710 fix(gitlab): modify pipeline (#3659)
7a64e710 is described below
commit 7a64e710d968909a9aff4dae85aec3d367ba3a87
Author: Warren Chen <yi...@merico.dev>
AuthorDate: Thu Nov 3 15:19:18 2022 +0800
fix(gitlab): modify pipeline (#3659)
closes #3658
---
.../e2e/raw_tables/_raw_gitlab_api_pipeline.csv | 4 +--
.../e2e/snapshot_tables/_tool_gitlab_pipelines.csv | 4 +--
.../_tool_gitlab_project_commits.csv | 38 ----------------------
.../gitlab/e2e/snapshot_tables/cicd_pipelines.csv | 4 +--
plugins/gitlab/tasks/pipeline_convertor.go | 10 +++---
plugins/gitlab/tasks/pipeline_extractor.go | 5 +--
6 files changed, 13 insertions(+), 52 deletions(-)
diff --git a/plugins/gitlab/e2e/raw_tables/_raw_gitlab_api_pipeline.csv b/plugins/gitlab/e2e/raw_tables/_raw_gitlab_api_pipeline.csv
index da645137..87381da9 100644
--- a/plugins/gitlab/e2e/raw_tables/_raw_gitlab_api_pipeline.csv
+++ b/plugins/gitlab/e2e/raw_tables/_raw_gitlab_api_pipeline.csv
@@ -1,8 +1,8 @@
id,params,data,url,input,created_at
-152,"{""ConnectionId"":1,""ProjectId"":12345678}","{""id"":457474837,""iid"":1,""project_id"":12345678,""sha"":""b1b82852d48b516a18e56c5bab0ebf54b8f4ccfd"",""ref"":""renovate/pin-dependencies"",""status"":""failed"",""source"":""external"",""created_at"":""2022-01-27T10:07:09.429Z"",""updated_at"":""2022-01-27T10:07:09.553Z"",""web_url"":""https://gitlab.com/merico-dev/ee/charts/-/pipelines/457474837""}",https://gitlab.com/api/v4/projects/12345678/pipelines?page=1&per_page=100&sort=asc&w [...]
+152,"{""ConnectionId"":1,""ProjectId"":12345678}","{""id"":457474837,""iid"":1,""project_id"":12345678,""sha"":""b1b82852d48b516a18e56c5bab0ebf54b8f4ccfd"",""ref"":""renovate/pin-dependencies"",""status"":""running"",""source"":""external"",""created_at"":""2022-01-27T10:07:09.429Z"",""updated_at"":null,""web_url"":""https://gitlab.com/merico-dev/ee/charts/-/pipelines/457474837""}",https://gitlab.com/api/v4/projects/12345678/pipelines?page=1&per_page=100&sort=asc&with_stats=true,null,202 [...]
153,"{""ConnectionId"":1,""ProjectId"":12345678}","{""id"":457474996,""iid"":2,""project_id"":12345678,""sha"":""739ab912e6e1ed27cecd8a2d00bfd6fa52afd90a"",""ref"":""renovate/jest-monorepo"",""status"":""failed"",""source"":""external"",""created_at"":""2022-01-27T10:07:18.884Z"",""updated_at"":""2022-01-27T10:07:19.043Z"",""web_url"":""https://gitlab.com/merico-dev/ee/charts/-/pipelines/457474996""}",https://gitlab.com/api/v4/projects/12345678/pipelines?page=1&per_page=100&sort=asc&with [...]
154,"{""ConnectionId"":1,""ProjectId"":12345678}","{""id"":457475160,""iid"":3,""project_id"":12345678,""sha"":""44d127e0ab7dbc4bc259b55929c9d00b62fc3bf4"",""ref"":""renovate/lodash-monorepo"",""status"":""failed"",""source"":""external"",""created_at"":""2022-01-27T10:07:26.435Z"",""updated_at"":""2022-01-27T10:07:26.638Z"",""web_url"":""https://gitlab.com/merico-dev/ee/charts/-/pipelines/457475160""}",https://gitlab.com/api/v4/projects/12345678/pipelines?page=1&per_page=100&sort=asc&wi [...]
-155,"{""ConnectionId"":1,""ProjectId"":12345678}","{""id"":457475337,""iid"":4,""project_id"":12345678,""sha"":""10a6464b6bd2cf4b59b8ac37ce1466e013f5a20d"",""ref"":""renovate/shx-0.x"",""status"":""failed"",""source"":""external"",""created_at"":""2022-01-27T10:07:36.502Z"",""updated_at"":""2022-01-27T10:07:37.545Z"",""web_url"":""https://gitlab.com/merico-dev/ee/charts/-/pipelines/457475337""}",https://gitlab.com/api/v4/projects/12345678/pipelines?page=1&per_page=100&sort=asc&with_stats [...]
+155,"{""ConnectionId"":1,""ProjectId"":12345678}","{""id"":457475337,""iid"":4,""project_id"":12345678,""sha"":""10a6464b6bd2cf4b59b8ac37ce1466e013f5a20d"",""ref"":""renovate/shx-0.x"",""status"":""running"",""source"":""external"",""created_at"":""2022-01-27T10:07:36.502Z"",""updated_at"":null,""web_url"":""https://gitlab.com/merico-dev/ee/charts/-/pipelines/457475337""}",https://gitlab.com/api/v4/projects/12345678/pipelines?page=1&per_page=100&sort=asc&with_stats=true,null,2022-06-16 1 [...]
156,"{""ConnectionId"":1,""ProjectId"":12345678}","{""id"":485811050,""iid"":5,""project_id"":12345678,""sha"":""c791ea6949d6b4aadf79b15ba666cb690c6527ac"",""ref"":""EE-7121"",""status"":""failed"",""source"":""push"",""created_at"":""2022-03-07T06:26:42.109Z"",""updated_at"":""2022-03-07T06:26:42.109Z"",""web_url"":""https://gitlab.com/merico-dev/ee/charts/-/pipelines/485811050""}",https://gitlab.com/api/v4/projects/12345678/pipelines?page=1&per_page=100&sort=asc&with_stats=true,null,20 [...]
157,"{""ConnectionId"":1,""ProjectId"":12345678}","{""id"":485811059,""iid"":6,""project_id"":12345678,""sha"":""c791ea6949d6b4aadf79b15ba666cb690c6527ac"",""ref"":""refs/merge-requests/74/head"",""status"":""failed"",""source"":""merge_request_event"",""created_at"":""2022-03-07T06:26:43.784Z"",""updated_at"":""2022-03-07T06:26:43.784Z"",""web_url"":""https://gitlab.com/merico-dev/ee/charts/-/pipelines/485811059""}",https://gitlab.com/api/v4/projects/12345678/pipelines?page=1&per_page=1 [...]
158,"{""ConnectionId"":1,""ProjectId"":12345678}","{""id"":485813816,""iid"":7,""project_id"":12345678,""sha"":""ecc7c0b2874c812ed882c9effbbda26e0abc7110"",""ref"":""EE-7121"",""status"":""failed"",""source"":""push"",""created_at"":""2022-03-07T06:33:56.824Z"",""updated_at"":""2022-03-07T06:33:56.824Z"",""web_url"":""https://gitlab.com/merico-dev/ee/charts/-/pipelines/485813816""}",https://gitlab.com/api/v4/projects/12345678/pipelines?page=1&per_page=100&sort=asc&with_stats=true,null,20 [...]
diff --git a/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_pipelines.csv b/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_pipelines.csv
index 57ac49b7..42560897 100644
--- a/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_pipelines.csv
+++ b/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_pipelines.csv
@@ -1,8 +1,8 @@
connection_id,gitlab_id,gitlab_created_at,project_id,status,web_url,duration,started_at,finished_at,coverage,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
-1,457474837,2022-01-27T10:07:09.429+00:00,12345678,failed,https://gitlab.com/merico-dev/ee/charts/-/pipelines/457474837,0,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,152,
+1,457474837,2022-01-27T10:07:09.429+00:00,12345678,running,https://gitlab.com/merico-dev/ee/charts/-/pipelines/457474837,0,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,152,
1,457474996,2022-01-27T10:07:18.884+00:00,12345678,failed,https://gitlab.com/merico-dev/ee/charts/-/pipelines/457474996,0,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,153,
1,457475160,2022-01-27T10:07:26.435+00:00,12345678,failed,https://gitlab.com/merico-dev/ee/charts/-/pipelines/457475160,0,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,154,
-1,457475337,2022-01-27T10:07:36.502+00:00,12345678,failed,https://gitlab.com/merico-dev/ee/charts/-/pipelines/457475337,1,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,155,
+1,457475337,2022-01-27T10:07:36.502+00:00,12345678,running,https://gitlab.com/merico-dev/ee/charts/-/pipelines/457475337,0,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,155,
1,485811050,2022-03-07T06:26:42.109+00:00,12345678,failed,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485811050,0,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,156,
1,485811059,2022-03-07T06:26:43.784+00:00,12345678,failed,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485811059,0,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,157,
1,485813816,2022-03-07T06:33:56.824+00:00,12345678,failed,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485813816,0,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,158,
diff --git a/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_project_commits.csv b/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_project_commits.csv
deleted file mode 100644
index 719c4f82..00000000
--- a/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_project_commits.csv
+++ /dev/null
@@ -1,38 +0,0 @@
-connection_id,gitlab_project_id,commit_sha,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
-1,12345678,05976c113b02fb71d5c43682bd2f9d96a3454efa,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,34,
-1,12345678,083864201849f6707a34d24e61c43638696f95a6,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,6,
-1,12345678,08973065d33c6fd2fb1f592b7a1e0fbe2ee72edd,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,11,
-1,12345678,0a4574d52b5964c37bfcb2952f6c6a8a9c6d5bcf,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,7,
-1,12345678,0dbb8bfdbbf7373656fa3eb036bdcbb25ae162da,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,26,
-1,12345678,1cba71b355d2bd4e1350602cfb9b98cce5a5cb4d,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,8,
-1,12345678,1f16955d45cec904c8968bb27689d91a71e9a2f2,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,12,
-1,12345678,211795deaaaaddb23b797261cdcd148e6083d5b2,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,21,
-1,12345678,25f2b34099a3e1867fcf73a54e50f78f3c8d8f75,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,9,
-1,12345678,382084b42697577d3a6adf71ce73d4b5ddd22977,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,2,
-1,12345678,3f04e0a61d0c4d2dd736a6bcaa3a06826269a533,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,3,
-1,12345678,45bed79fcdc6ba3444245f55660f839db50d79c8,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,33,
-1,12345678,48295660f8bd0c0a024849d4407830face57d4bc,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,37,
-1,12345678,4f0d2f7b1a47d21b01a68e59c12d6f8885ab230e,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,24,
-1,12345678,553f7613b178301da4075202afe91ebda5d97ab5,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,16,
-1,12345678,5a141ad0c28960b02c40e6c476d8df3cc76f58c4,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,31,
-1,12345678,5b6a2be437e0c66ba6276030a61a10583c3f83fa,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,22,
-1,12345678,5d6c369200c633b06d85c9375f61fef297761ef1,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,19,
-1,12345678,645a724b558a7fa3885fa233d06f68f833d893fc,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,27,
-1,12345678,64abdcdd9ee9c821f56faf93c5bb0ebc57f575e4,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,28,
-1,12345678,6ebf519f25328e49a004f5a574c8b69ba7420be1,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,15,
-1,12345678,81b12c995b2fc12e20fd05113cf36dbd57b6e7b0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,20,
-1,12345678,846deac12a1263e4631ced45abbc5e157b23c1be,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,30,
-1,12345678,8a40765e605966842b05b5d8cdf2dd6890556d09,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,14,
-1,12345678,8a96911c5350f7575215f02f0242f7e27ad3d1f4,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,10,
-1,12345678,91e5666bdfe90e5ada3e65aaa9240b2b0b1a072f,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,17,
-1,12345678,93d13cf14a470995ec35186292bf165e1152dc2a,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,32,
-1,12345678,9a80506e158547d87e7ad2902c0196deff5b5c07,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,35,
-1,12345678,9e5ec04b0a8273b73936ea150275919b0602c260,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,13,
-1,12345678,9ef8183140bae8c39a440d9c7eb47f3bc7dcab75,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,18,
-1,12345678,a09f2d4ae88e78d7424fba1ae6d0c036b677271d,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,36,
-1,12345678,abbe0ab2c7bb1dc2cfaa3ef3062f378fb908ba71,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,5,
-1,12345678,ad25fcda15e310a7eb34b3e38ff9b962b54f0e03,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,1,
-1,12345678,bb6dfec8dcdacb5b6b37dbd8eb1aa6f9e8c06138,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,23,
-1,12345678,c4d406c077af582837d3611a46eded70151f93fb,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,29,
-1,12345678,e01d4f03811cd0da9949848731236e0aa261cf54,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,4,
-1,12345678,e24fecacb1a2c85f904bef57ffd5598ca2df023c,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,25,
diff --git a/plugins/gitlab/e2e/snapshot_tables/cicd_pipelines.csv b/plugins/gitlab/e2e/snapshot_tables/cicd_pipelines.csv
index 3494cfc4..f6cb8d00 100644
--- a/plugins/gitlab/e2e/snapshot_tables/cicd_pipelines.csv
+++ b/plugins/gitlab/e2e/snapshot_tables/cicd_pipelines.csv
@@ -1,8 +1,8 @@
id,name,result,status,type,duration_sec,created_date,finished_date,environment,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
-gitlab:GitlabPipeline:1:457474837,gitlab:GitlabProject:1:12345678,FAILURE,DONE,,0,2022-01-27T10:07:09.429+00:00,2022-01-27T10:07:09.553+00:00,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,152,
+gitlab:GitlabPipeline:1:457474837,gitlab:GitlabProject:1:12345678,SUCCESS,IN_PROGRESS,,0,2022-01-27T10:07:09.429+00:00,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,152,
gitlab:GitlabPipeline:1:457474996,gitlab:GitlabProject:1:12345678,FAILURE,DONE,,0,2022-01-27T10:07:18.884+00:00,2022-01-27T10:07:19.043+00:00,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,153,
gitlab:GitlabPipeline:1:457475160,gitlab:GitlabProject:1:12345678,FAILURE,DONE,,0,2022-01-27T10:07:26.435+00:00,2022-01-27T10:07:26.638+00:00,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,154,
-gitlab:GitlabPipeline:1:457475337,gitlab:GitlabProject:1:12345678,FAILURE,DONE,,1,2022-01-27T10:07:36.502+00:00,2022-01-27T10:07:37.545+00:00,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,155,
+gitlab:GitlabPipeline:1:457475337,gitlab:GitlabProject:1:12345678,SUCCESS,IN_PROGRESS,,0,2022-01-27T10:07:36.502+00:00,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,155,
gitlab:GitlabPipeline:1:485811050,gitlab:GitlabProject:1:12345678,FAILURE,DONE,,0,2022-03-07T06:26:42.109+00:00,2022-03-07T06:26:42.109+00:00,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,156,
gitlab:GitlabPipeline:1:485811059,gitlab:GitlabProject:1:12345678,FAILURE,DONE,,0,2022-03-07T06:26:43.784+00:00,2022-03-07T06:26:43.784+00:00,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,157,
gitlab:GitlabPipeline:1:485813816,gitlab:GitlabProject:1:12345678,FAILURE,DONE,,0,2022-03-07T06:33:56.824+00:00,2022-03-07T06:33:56.824+00:00,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,158,
diff --git a/plugins/gitlab/tasks/pipeline_convertor.go b/plugins/gitlab/tasks/pipeline_convertor.go
index 366f5a02..7b49852b 100644
--- a/plugins/gitlab/tasks/pipeline_convertor.go
+++ b/plugins/gitlab/tasks/pipeline_convertor.go
@@ -92,15 +92,13 @@ func ConvertPipelines(taskCtx core.SubTaskContext) errors.Error {
}
// rebuild the FinishedDate and DurationSec by Status
- finishedAt := time.Now()
if domainPipeline.Status != devops.DONE {
domainPipeline.FinishedDate = nil
- } else if gitlabPipeline.GitlabUpdatedAt != nil {
- finishedAt = *gitlabPipeline.GitlabUpdatedAt
+ domainPipeline.DurationSec = 0
+ } else if domainPipeline.FinishedDate != nil {
+ durationTime := domainPipeline.FinishedDate.Sub(createdAt)
+ domainPipeline.DurationSec = uint64(durationTime.Seconds())
}
- durationTime := finishedAt.Sub(createdAt)
-
- domainPipeline.DurationSec = uint64(durationTime.Seconds())
return []interface{}{
domainPipeline,
diff --git a/plugins/gitlab/tasks/pipeline_extractor.go b/plugins/gitlab/tasks/pipeline_extractor.go
index 9f4529b8..325183c3 100644
--- a/plugins/gitlab/tasks/pipeline_extractor.go
+++ b/plugins/gitlab/tasks/pipeline_extractor.go
@@ -74,8 +74,9 @@ func ExtractApiPipelines(taskCtx core.SubTaskContext) errors.Error {
return nil, err
}
- duration := int(gitlabApiPipeline.UpdatedAt.ToTime().Sub(gitlabApiPipeline.CreatedAt.ToTime()).Seconds())
- gitlabApiPipeline.Duration = duration
+ if gitlabApiPipeline.UpdatedAt != nil && gitlabApiPipeline.CreatedAt != nil {
+ gitlabApiPipeline.Duration = int(gitlabApiPipeline.UpdatedAt.ToTime().Sub(gitlabApiPipeline.CreatedAt.ToTime()).Seconds())
+ }
gitlabPipeline := &models.GitlabPipeline{
GitlabId: gitlabApiPipeline.Id,
ProjectId: data.Options.ProjectId,