You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by wa...@apache.org on 2022/08/03 07:38:51 UTC
[incubator-devlake] 01/02: feat: gitlab-ci
This is an automated email from the ASF dual-hosted git repository.
warren pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit c45c2b4e2f15b54604646c0728b04f333bac3521
Author: Nddtfjiang <jz...@qq.com>
AuthorDate: Fri Jul 29 14:48:08 2022 +0000
feat: gitlab-ci
Add CollectApiJobs
Add ExtractApiJobs
Alert _tool_gitlab_pipelines
Nddtfjiang <zh...@merico.dev>
---
plugins/gitee/api/connection.go | 3 -
plugins/gitlab/e2e/pipelines_test.go | 4 +-
.../e2e/snapshot_tables/_tool_gitlab_pipelines.csv | 42 ++++----
plugins/gitlab/impl/impl.go | 2 +
plugins/gitlab/models/{pipeline.go => job.go} | 35 +++---
.../migrationscripts/20220729_pipeline_and_job.go | 119 +++++++++++++++++++++
plugins/gitlab/models/migrationscripts/register.go | 1 +
plugins/gitlab/models/pipeline.go | 28 ++---
plugins/gitlab/tasks/job_collector.go | 54 ++++++++++
plugins/gitlab/tasks/job_extractor.go | 104 ++++++++++++++++++
plugins/gitlab/tasks/pipeline_extractor.go | 73 +++++++------
11 files changed, 378 insertions(+), 87 deletions(-)
diff --git a/plugins/gitee/api/connection.go b/plugins/gitee/api/connection.go
index 5fd6e5e7..a0052d38 100644
--- a/plugins/gitee/api/connection.go
+++ b/plugins/gitee/api/connection.go
@@ -133,8 +133,5 @@ GET /plugins/gitee/connections/:connectionId
func GetConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) {
connection := &models.GiteeConnection{}
err := connectionHelper.First(connection, input.Params)
- if err != nil {
- return nil, err
- }
return &core.ApiResourceOutput{Body: connection}, err
}
diff --git a/plugins/gitlab/e2e/pipelines_test.go b/plugins/gitlab/e2e/pipelines_test.go
index 06da4ace..ec571393 100644
--- a/plugins/gitlab/e2e/pipelines_test.go
+++ b/plugins/gitlab/e2e/pipelines_test.go
@@ -58,8 +58,8 @@ func TestGitlabPipelineDataFlow(t *testing.T) {
"sha",
"web_url",
"duration",
- "started_at",
- "finished_at",
+ "gitlab_started_at",
+ "gitlab_finished_at",
"coverage",
"_raw_data_params",
"_raw_data_table",
diff --git a/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_pipelines.csv b/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_pipelines.csv
index 73b4672c..46a0333f 100644
--- a/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_pipelines.csv
+++ b/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_pipelines.csv
@@ -1,21 +1,21 @@
-connection_id,gitlab_id,project_id,gitlab_created_at,status,ref,sha,web_url,duration,started_at,finished_at,coverage,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
-1,457474837,12345678,2022-01-27T10:07:09.429+00:00,failed,renovate/pin-dependencies,b1b82852d48b516a18e56c5bab0ebf54b8f4ccfd,https://gitlab.com/merico-dev/ee/charts/-/pipelines/457474837,0,2022-01-27T10:07:09.429+00:00,2022-01-27T10:07:09.553+00:00,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,152,
-1,457474996,12345678,2022-01-27T10:07:18.884+00:00,failed,renovate/jest-monorepo,739ab912e6e1ed27cecd8a2d00bfd6fa52afd90a,https://gitlab.com/merico-dev/ee/charts/-/pipelines/457474996,0,2022-01-27T10:07:18.884+00:00,2022-01-27T10:07:19.043+00:00,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,153,
-1,457475160,12345678,2022-01-27T10:07:26.435+00:00,failed,renovate/lodash-monorepo,44d127e0ab7dbc4bc259b55929c9d00b62fc3bf4,https://gitlab.com/merico-dev/ee/charts/-/pipelines/457475160,0,2022-01-27T10:07:26.435+00:00,2022-01-27T10:07:26.638+00:00,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,154,
-1,457475337,12345678,2022-01-27T10:07:36.502+00:00,failed,renovate/shx-0.x,10a6464b6bd2cf4b59b8ac37ce1466e013f5a20d,https://gitlab.com/merico-dev/ee/charts/-/pipelines/457475337,1,2022-01-27T10:07:36.502+00:00,2022-01-27T10:07:37.545+00:00,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,155,
-1,485811050,12345678,2022-03-07T06:26:42.109+00:00,failed,EE-7121,c791ea6949d6b4aadf79b15ba666cb690c6527ac,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485811050,0,2022-03-07T06:26:42.109+00:00,2022-03-07T06:26:42.109+00:00,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,156,
-1,485811059,12345678,2022-03-07T06:26:43.784+00:00,failed,refs/merge-requests/74/head,c791ea6949d6b4aadf79b15ba666cb690c6527ac,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485811059,0,2022-03-07T06:26:43.784+00:00,2022-03-07T06:26:43.784+00:00,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,157,
-1,485813816,12345678,2022-03-07T06:33:56.824+00:00,failed,EE-7121,ecc7c0b2874c812ed882c9effbbda26e0abc7110,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485813816,0,2022-03-07T06:33:56.824+00:00,2022-03-07T06:33:56.824+00:00,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,158,
-1,485813830,12345678,2022-03-07T06:33:58.889+00:00,failed,refs/merge-requests/74/head,ecc7c0b2874c812ed882c9effbbda26e0abc7110,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485813830,0,2022-03-07T06:33:58.889+00:00,2022-03-07T06:33:58.889+00:00,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,159,
-1,485814501,12345678,2022-03-07T06:35:28.111+00:00,failed,EE-7121,6a3346f8434cc65fbe3f7a80a0edec5b4014a733,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485814501,0,2022-03-07T06:35:28.111+00:00,2022-03-07T06:35:28.111+00:00,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,160,
-1,485814516,12345678,2022-03-07T06:35:31.255+00:00,failed,refs/merge-requests/74/head,6a3346f8434cc65fbe3f7a80a0edec5b4014a733,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485814516,0,2022-03-07T06:35:31.255+00:00,2022-03-07T06:35:31.255+00:00,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,161,
-1,485814871,12345678,2022-03-07T06:36:50.020+00:00,failed,refs/merge-requests/74/head,f731732471961fed061a01fa99631f69e861c4a2,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485814871,42,2022-03-07T06:36:50.020+00:00,2022-03-07T06:37:32.103+00:00,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,162,
-1,485817670,12345678,2022-03-07T06:45:09.471+00:00,failed,refs/merge-requests/74/head,5b95c5aebce1eae6a1b95ecf6fbc870851455375,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485817670,1956,2022-03-07T06:45:09.471+00:00,2022-03-07T07:17:46.305+00:00,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,163,
-1,485837602,12345678,2022-03-07T07:20:45.859+00:00,failed,refs/merge-requests/74/head,a79e2aaeee9916338a11b665cb4201f24627f3f1,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485837602,434,2022-03-07T07:20:45.859+00:00,2022-03-07T07:28:00.277+00:00,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,164,
-1,485842553,12345678,2022-03-07T07:30:47.018+00:00,failed,refs/merge-requests/74/head,9e0934cedcd83abab8d34c5bb0f597c7837a55f0,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485842553,287,2022-03-07T07:30:47.018+00:00,2022-03-07T07:35:34.998+00:00,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,165,
-1,485845850,12345678,2022-03-07T07:38:58.611+00:00,failed,refs/merge-requests/74/head,b3b581940f6d4d3226db31e48eab2b3b73f4c6db,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485845850,419,2022-03-07T07:38:58.611+00:00,2022-03-07T07:45:58.412+00:00,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,166,
-1,485852752,12345678,2022-03-07T07:46:09.385+00:00,failed,refs/merge-requests/74/head,82df93eeb72c71271dad5a85359f39661de899fe,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485852752,319,2022-03-07T07:46:09.385+00:00,2022-03-07T07:51:28.709+00:00,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,167,
-1,485865876,12345678,2022-03-07T08:04:56.406+00:00,failed,refs/merge-requests/74/head,db45efd231dbaca1337bd29bc1f65f404754e6bd,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485865876,480,2022-03-07T08:04:56.406+00:00,2022-03-07T08:12:56.453+00:00,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,168,
-1,485877118,12345678,2022-03-07T08:22:48.943+00:00,failed,refs/merge-requests/74/head,09f81b1b2d083411c0bfecd32d7728479b594503,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485877118,289,2022-03-07T08:22:48.943+00:00,2022-03-07T08:27:38.364+00:00,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,169,
-1,485905167,12345678,2022-03-07T09:02:09.994+00:00,failed,refs/merge-requests/74/head,cce432655861efafcd05edc8de09e305c772d2b8,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485905167,687,2022-03-07T09:02:09.994+00:00,2022-03-07T09:13:37.013+00:00,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,170,
-1,485932863,12345678,2022-03-07T09:34:57.476+00:00,success,refs/merge-requests/74/head,12fc3a42080bb98ca520817bd4fe0ca33c0bb279,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485932863,398,2022-03-07T09:34:57.476+00:00,2022-03-07T09:41:36.267+00:00,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,171,
+connection_id,gitlab_id,project_id,gitlab_created_at,status,ref,sha,web_url,duration,gitlab_started_at,gitlab_finished_at,coverage,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
+1,457474837,12345678,2022-01-27T10:07:09.429+00:00,failed,renovate/pin-dependencies,b1b82852d48b516a18e56c5bab0ebf54b8f4ccfd,https://gitlab.com/merico-dev/ee/charts/-/pipelines/457474837,0,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,152,
+1,457474996,12345678,2022-01-27T10:07:18.884+00:00,failed,renovate/jest-monorepo,739ab912e6e1ed27cecd8a2d00bfd6fa52afd90a,https://gitlab.com/merico-dev/ee/charts/-/pipelines/457474996,0,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,153,
+1,457475160,12345678,2022-01-27T10:07:26.435+00:00,failed,renovate/lodash-monorepo,44d127e0ab7dbc4bc259b55929c9d00b62fc3bf4,https://gitlab.com/merico-dev/ee/charts/-/pipelines/457475160,0,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,154,
+1,457475337,12345678,2022-01-27T10:07:36.502+00:00,failed,renovate/shx-0.x,10a6464b6bd2cf4b59b8ac37ce1466e013f5a20d,https://gitlab.com/merico-dev/ee/charts/-/pipelines/457475337,1,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,155,
+1,485811050,12345678,2022-03-07T06:26:42.109+00:00,failed,EE-7121,c791ea6949d6b4aadf79b15ba666cb690c6527ac,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485811050,0,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,156,
+1,485811059,12345678,2022-03-07T06:26:43.784+00:00,failed,refs/merge-requests/74/head,c791ea6949d6b4aadf79b15ba666cb690c6527ac,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485811059,0,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,157,
+1,485813816,12345678,2022-03-07T06:33:56.824+00:00,failed,EE-7121,ecc7c0b2874c812ed882c9effbbda26e0abc7110,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485813816,0,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,158,
+1,485813830,12345678,2022-03-07T06:33:58.889+00:00,failed,refs/merge-requests/74/head,ecc7c0b2874c812ed882c9effbbda26e0abc7110,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485813830,0,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,159,
+1,485814501,12345678,2022-03-07T06:35:28.111+00:00,failed,EE-7121,6a3346f8434cc65fbe3f7a80a0edec5b4014a733,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485814501,0,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,160,
+1,485814516,12345678,2022-03-07T06:35:31.255+00:00,failed,refs/merge-requests/74/head,6a3346f8434cc65fbe3f7a80a0edec5b4014a733,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485814516,0,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,161,
+1,485814871,12345678,2022-03-07T06:36:50.020+00:00,failed,refs/merge-requests/74/head,f731732471961fed061a01fa99631f69e861c4a2,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485814871,42,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,162,
+1,485817670,12345678,2022-03-07T06:45:09.471+00:00,failed,refs/merge-requests/74/head,5b95c5aebce1eae6a1b95ecf6fbc870851455375,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485817670,1956,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,163,
+1,485837602,12345678,2022-03-07T07:20:45.859+00:00,failed,refs/merge-requests/74/head,a79e2aaeee9916338a11b665cb4201f24627f3f1,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485837602,434,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,164,
+1,485842553,12345678,2022-03-07T07:30:47.018+00:00,failed,refs/merge-requests/74/head,9e0934cedcd83abab8d34c5bb0f597c7837a55f0,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485842553,287,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,165,
+1,485845850,12345678,2022-03-07T07:38:58.611+00:00,failed,refs/merge-requests/74/head,b3b581940f6d4d3226db31e48eab2b3b73f4c6db,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485845850,419,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,166,
+1,485852752,12345678,2022-03-07T07:46:09.385+00:00,failed,refs/merge-requests/74/head,82df93eeb72c71271dad5a85359f39661de899fe,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485852752,319,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,167,
+1,485865876,12345678,2022-03-07T08:04:56.406+00:00,failed,refs/merge-requests/74/head,db45efd231dbaca1337bd29bc1f65f404754e6bd,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485865876,480,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,168,
+1,485877118,12345678,2022-03-07T08:22:48.943+00:00,failed,refs/merge-requests/74/head,09f81b1b2d083411c0bfecd32d7728479b594503,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485877118,289,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,169,
+1,485905167,12345678,2022-03-07T09:02:09.994+00:00,failed,refs/merge-requests/74/head,cce432655861efafcd05edc8de09e305c772d2b8,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485905167,687,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,170,
+1,485932863,12345678,2022-03-07T09:34:57.476+00:00,success,refs/merge-requests/74/head,12fc3a42080bb98ca520817bd4fe0ca33c0bb279,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485932863,398,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,171,
diff --git a/plugins/gitlab/impl/impl.go b/plugins/gitlab/impl/impl.go
index 4c62da6f..8b2401a1 100644
--- a/plugins/gitlab/impl/impl.go
+++ b/plugins/gitlab/impl/impl.go
@@ -64,6 +64,8 @@ func (plugin Gitlab) SubTaskMetas() []core.SubTaskMeta {
tasks.ExtractApiMrCommitsMeta,
tasks.CollectApiPipelinesMeta,
tasks.ExtractApiPipelinesMeta,
+ tasks.CollectApiJobsMeta,
+ tasks.ExtractApiJobsMeta,
tasks.EnrichMergeRequestsMeta,
tasks.CollectAccountsMeta,
tasks.ExtractAccountsMeta,
diff --git a/plugins/gitlab/models/pipeline.go b/plugins/gitlab/models/job.go
similarity index 59%
copy from plugins/gitlab/models/pipeline.go
copy to plugins/gitlab/models/job.go
index 2c16a74c..9e437a9a 100644
--- a/plugins/gitlab/models/pipeline.go
+++ b/plugins/gitlab/models/job.go
@@ -18,27 +18,32 @@ limitations under the License.
package models
import (
- "github.com/apache/incubator-devlake/models/common"
"time"
+
+ "github.com/apache/incubator-devlake/models/common"
)
-type GitlabPipeline struct {
+type GitlabJob struct {
ConnectionId uint64 `gorm:"primaryKey"`
- GitlabId int `gorm:"primaryKey"`
- ProjectId int `gorm:"index"`
- GitlabCreatedAt time.Time
- Status string `gorm:"type:varchar(100)"`
- Ref string `gorm:"type:varchar(255)"`
- Sha string `gorm:"type:varchar(255)"`
- WebUrl string `gorm:"type:varchar(255)"`
- Duration int
- StartedAt *time.Time
- FinishedAt *time.Time
- Coverage string
+ GitlabId int `gorm:"primaryKey"`
+ ProjectId int `gorm:"index"`
+ Status string `gorm:"type:varchar(255)"`
+ Stage string `gorm:"type:varchar(255)"`
+ Name string `gorm:"type:varchar(255)"`
+ Ref string `gorm:"type:varchar(255)"`
+ Tag bool `gorm:"type:boolean"`
+ AllowFailure bool `json:"allow_failure"`
+ Duration float64 `gorm:"type:text"`
+ WebUrl string `gorm:"type:varchar(255)"`
+
+ GitlabCreatedAt *time.Time
+ GitlabStartedAt *time.Time
+ GitlabFinishedAt *time.Time
+
common.NoPKModel
}
-func (GitlabPipeline) TableName() string {
- return "_tool_gitlab_pipelines"
+func (GitlabJob) TableName() string {
+ return "_tool_gitlab_jobs"
}
diff --git a/plugins/gitlab/models/migrationscripts/20220729_pipeline_and_job.go b/plugins/gitlab/models/migrationscripts/20220729_pipeline_and_job.go
new file mode 100644
index 00000000..7a5a384f
--- /dev/null
+++ b/plugins/gitlab/models/migrationscripts/20220729_pipeline_and_job.go
@@ -0,0 +1,119 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+ "context"
+ "time"
+
+ "github.com/apache/incubator-devlake/models/common"
+ "github.com/apache/incubator-devlake/plugins/gitlab/models/migrationscripts/archived"
+ "gorm.io/gorm"
+)
+
+type alertPipeline struct{}
+
+type GitlabPipeline20220729 struct {
+ ConnectionId uint64 `gorm:"primaryKey"`
+
+ GitlabId int `gorm:"primaryKey"`
+ ProjectId int `gorm:"index"`
+ Status string `gorm:"type:varchar(100)"`
+ Ref string `gorm:"type:varchar(255)"`
+ Sha string `gorm:"type:varchar(255)"`
+ WebUrl string `gorm:"type:varchar(255)"`
+ Duration int
+
+ GitlabCreatedAt *time.Time
+ GitlabUpdatedAt *time.Time
+ GitlabStartedAt *time.Time
+ GitlabFinishedAt *time.Time
+ Coverage string
+
+ common.NoPKModel
+}
+
+func (GitlabPipeline20220729) TableName() string {
+ return "_tool_gitlab_pipelines"
+}
+
+type GitlabJob20220729 struct {
+ ConnectionId uint64 `gorm:"primaryKey"`
+
+ GitlabId int `gorm:"primaryKey"`
+ ProjectId int `gorm:"index"`
+ Status string `gorm:"type:varchar(255)"`
+ Stage string `gorm:"type:varchar(255)"`
+ Name string `gorm:"type:varchar(255)"`
+ Ref string `gorm:"type:varchar(255)"`
+ Tag bool `gorm:"type:boolean"`
+ AllowFailure bool `json:"allow_failure"`
+ Duration float64 `gorm:"type:text"`
+ WebUrl string `gorm:"type:varchar(255)"`
+
+ GitlabCreatedAt *time.Time
+ GitlabStartedAt *time.Time
+ GitlabFinishedAt *time.Time
+
+ common.NoPKModel
+}
+
+func (GitlabJob20220729) TableName() string {
+ return "_tool_gitlab_jobs"
+}
+
+func (*alertPipeline) Up(ctx context.Context, db *gorm.DB) error {
+ err := db.Migrator().DropColumn(&archived.GitlabPipeline{}, "started_at")
+ if err != nil {
+ return err
+ }
+ err = db.Migrator().DropColumn(&archived.GitlabPipeline{}, "finished_at")
+ if err != nil {
+ return err
+ }
+
+ err = db.Migrator().AddColumn(&GitlabPipeline20220729{}, "gitlab_updated_at")
+ if err != nil {
+ return err
+ }
+
+ err = db.Migrator().AddColumn(&GitlabPipeline20220729{}, "gitlab_started_at")
+ if err != nil {
+ return err
+ }
+
+ err = db.Migrator().AddColumn(&GitlabPipeline20220729{}, "gitlab_finished_at")
+ if err != nil {
+ return err
+ }
+
+ err = db.Migrator().AutoMigrate(&GitlabJob20220729{})
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func (*alertPipeline) Version() uint64 {
+ return 20220729231236
+}
+
+func (*alertPipeline) Name() string {
+ return "pipeline and job"
+}
diff --git a/plugins/gitlab/models/migrationscripts/register.go b/plugins/gitlab/models/migrationscripts/register.go
index c1365f7d..b47b0d6f 100644
--- a/plugins/gitlab/models/migrationscripts/register.go
+++ b/plugins/gitlab/models/migrationscripts/register.go
@@ -25,5 +25,6 @@ import (
func All() []migration.Script {
return []migration.Script{
new(addInitTables),
+ new(alertPipeline),
}
}
diff --git a/plugins/gitlab/models/pipeline.go b/plugins/gitlab/models/pipeline.go
index 2c16a74c..6f9d3532 100644
--- a/plugins/gitlab/models/pipeline.go
+++ b/plugins/gitlab/models/pipeline.go
@@ -18,24 +18,28 @@ limitations under the License.
package models
import (
- "github.com/apache/incubator-devlake/models/common"
"time"
+
+ "github.com/apache/incubator-devlake/models/common"
)
type GitlabPipeline struct {
ConnectionId uint64 `gorm:"primaryKey"`
- GitlabId int `gorm:"primaryKey"`
- ProjectId int `gorm:"index"`
- GitlabCreatedAt time.Time
- Status string `gorm:"type:varchar(100)"`
- Ref string `gorm:"type:varchar(255)"`
- Sha string `gorm:"type:varchar(255)"`
- WebUrl string `gorm:"type:varchar(255)"`
- Duration int
- StartedAt *time.Time
- FinishedAt *time.Time
- Coverage string
+ GitlabId int `gorm:"primaryKey"`
+ ProjectId int `gorm:"index"`
+ Status string `gorm:"type:varchar(100)"`
+ Ref string `gorm:"type:varchar(255)"`
+ Sha string `gorm:"type:varchar(255)"`
+ WebUrl string `gorm:"type:varchar(255)"`
+ Duration int
+
+ GitlabCreatedAt *time.Time
+ GitlabUpdatedAt *time.Time
+ GitlabStartedAt *time.Time
+ GitlabFinishedAt *time.Time
+ Coverage string
+
common.NoPKModel
}
diff --git a/plugins/gitlab/tasks/job_collector.go b/plugins/gitlab/tasks/job_collector.go
new file mode 100644
index 00000000..a8a26c42
--- /dev/null
+++ b/plugins/gitlab/tasks/job_collector.go
@@ -0,0 +1,54 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/helper"
+)
+
+const RAW_JOB_TABLE = "gitlab_api_job"
+
+var CollectApiJobsMeta = core.SubTaskMeta{
+ Name: "collectApiJobs",
+ EntryPoint: CollectApiJobs,
+ EnabledByDefault: true,
+ Description: "Collect job data from gitlab api",
+ DomainTypes: []string{core.DOMAIN_TYPE_CICD},
+}
+
+func CollectApiJobs(taskCtx core.SubTaskContext) error {
+ rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_JOB_TABLE)
+
+ collector, err := helper.NewApiCollector(helper.ApiCollectorArgs{
+ RawDataSubTaskArgs: *rawDataSubTaskArgs,
+ ApiClient: data.ApiClient,
+ PageSize: 100,
+ Incremental: false,
+ UrlTemplate: "projects/{{ .Params.ProjectId }}/jobs",
+ Query: GetQuery,
+ ResponseParser: GetRawMessageFromResponse,
+ AfterResponse: ignoreHTTPStatus403, // ignore 403 for CI/CD disable
+ })
+
+ if err != nil {
+ return err
+ }
+
+ return collector.Execute()
+}
diff --git a/plugins/gitlab/tasks/job_extractor.go b/plugins/gitlab/tasks/job_extractor.go
new file mode 100644
index 00000000..d7d99d04
--- /dev/null
+++ b/plugins/gitlab/tasks/job_extractor.go
@@ -0,0 +1,104 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "encoding/json"
+
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/gitlab/models"
+ "github.com/apache/incubator-devlake/plugins/helper"
+)
+
+type ApiJob struct {
+ Id int `json:"id"`
+ Status string
+ Stage string
+ Name string
+ Ref string
+ Tag bool
+ AllowFailure bool `json:"allow_failure"`
+ Duration float64
+ WebUrl string `json:"web_url"`
+
+ CreatedAt *helper.Iso8601Time `json:"created_at"`
+ StartedAt *helper.Iso8601Time `json:"started_at"`
+ FinishedAt *helper.Iso8601Time `json:"finished_at"`
+}
+
+var ExtractApiJobsMeta = core.SubTaskMeta{
+ Name: "extractApiJobs",
+ EntryPoint: ExtractApiJobs,
+ EnabledByDefault: true,
+ Description: "Extract raw pipelines data into tool layer table GitlabPipeline",
+ DomainTypes: []string{core.DOMAIN_TYPE_CICD},
+}
+
+func ExtractApiJobs(taskCtx core.SubTaskContext) error {
+ rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_JOB_TABLE)
+
+ extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{
+ RawDataSubTaskArgs: *rawDataSubTaskArgs,
+ Extract: func(row *helper.RawData) ([]interface{}, error) {
+ // create gitlab commit
+ gitlabApiJob := &ApiJob{}
+ err := json.Unmarshal(row.Data, gitlabApiJob)
+ if err != nil {
+ return nil, err
+ }
+
+ gitlabPipeline, err := convertJob(gitlabApiJob, data.Options.ProjectId)
+ if err != nil {
+ return nil, err
+ }
+
+ // use data.Options.ProjectId to set the value of ProjectId for it
+ gitlabPipeline.ProjectId = data.Options.ProjectId
+ gitlabPipeline.ConnectionId = data.Options.ConnectionId
+ results := make([]interface{}, 0, 1)
+ results = append(results, gitlabPipeline)
+
+ return results, nil
+ },
+ })
+
+ if err != nil {
+ return err
+ }
+
+ return extractor.Execute()
+}
+
+func convertJob(job *ApiJob, projectId int) (*models.GitlabJob, error) {
+ return &models.GitlabJob{
+ GitlabId: job.Id,
+ ProjectId: projectId,
+ Status: job.Status,
+ Stage: job.Stage,
+ Name: job.Name,
+ Ref: job.Ref,
+ Tag: job.Tag,
+ AllowFailure: job.AllowFailure,
+ Duration: job.Duration,
+ WebUrl: job.WebUrl,
+
+ GitlabCreatedAt: helper.Iso8601TimeToTime(job.CreatedAt),
+ GitlabStartedAt: helper.Iso8601TimeToTime(job.StartedAt),
+ GitlabFinishedAt: helper.Iso8601TimeToTime(job.FinishedAt),
+ }, nil
+}
diff --git a/plugins/gitlab/tasks/pipeline_extractor.go b/plugins/gitlab/tasks/pipeline_extractor.go
index 74a57cd0..25885b5f 100644
--- a/plugins/gitlab/tasks/pipeline_extractor.go
+++ b/plugins/gitlab/tasks/pipeline_extractor.go
@@ -25,29 +25,32 @@ import (
"github.com/apache/incubator-devlake/plugins/helper"
)
-type ApiPipeline struct {
- GitlabId int `json:"id"`
- ProjectId int `json:"project_id"`
- GitlabCreatedAt *helper.Iso8601Time `json:"created_at"`
- UpdatedAt *helper.Iso8601Time `json:"updated_at"`
- Ref string
- Sha string
- Duration int
- WebUrl string `json:"web_url"`
- Status string
+type ApiDetailedStatus struct {
+ Icon string
+ Text string
+ Label string
+ Group string
+ Tooltip string
+ HasDetails bool `json:"has_details"`
+ DetailsPath string `json:"details_path"`
+ Favicon string
}
-type ApiSinglePipelineResponse struct {
- GitlabId int `json:"id"`
- ProjectId int `json:"project_id"`
- GitlabCreatedAt *helper.Iso8601Time `json:"created_at"`
- Ref string
- Sha string
- WebUrl string `json:"web_url"`
- Duration int
- UpdatedAt *helper.Iso8601Time `json:"updated_at"`
- Coverage string
- Status string
+type ApiPipeline struct {
+ Id int `json:"id"`
+ Ref string
+ Sha string
+ Status string
+ Tag bool
+ Duration int
+ WebUrl string `json:"web_url"`
+
+ CreatedAt *helper.Iso8601Time `json:"created_at"`
+ UpdatedAt *helper.Iso8601Time `json:"updated_at"`
+ StartedAt *helper.Iso8601Time `json:"started_at"`
+ FinishedAt *helper.Iso8601Time `json:"finished_at"`
+
+ ApiDetailedStatus
}
var ExtractApiPipelinesMeta = core.SubTaskMeta{
@@ -70,9 +73,10 @@ func ExtractApiPipelines(taskCtx core.SubTaskContext) error {
if err != nil {
return nil, err
}
- duration := int(gitlabApiPipeline.UpdatedAt.ToTime().Sub(gitlabApiPipeline.GitlabCreatedAt.ToTime()).Seconds())
+
+ duration := int(gitlabApiPipeline.UpdatedAt.ToTime().Sub(gitlabApiPipeline.CreatedAt.ToTime()).Seconds())
gitlabApiPipeline.Duration = duration
- gitlabPipeline, err := convertPipeline(gitlabApiPipeline)
+ gitlabPipeline, err := convertPipeline(gitlabApiPipeline, data.Options.ProjectId)
if err != nil {
return nil, err
}
@@ -94,18 +98,19 @@ func ExtractApiPipelines(taskCtx core.SubTaskContext) error {
return extractor.Execute()
}
-func convertPipeline(pipeline *ApiPipeline) (*models.GitlabPipeline, error) {
+func convertPipeline(pipeline *ApiPipeline, projectId int) (*models.GitlabPipeline, error) {
gitlabPipeline := &models.GitlabPipeline{
- GitlabId: pipeline.GitlabId,
- ProjectId: pipeline.ProjectId,
- GitlabCreatedAt: pipeline.GitlabCreatedAt.ToTime(),
- Ref: pipeline.Ref,
- Sha: pipeline.Sha,
- WebUrl: pipeline.WebUrl,
- Status: pipeline.Status,
- StartedAt: helper.Iso8601TimeToTime(pipeline.GitlabCreatedAt),
- FinishedAt: helper.Iso8601TimeToTime(pipeline.UpdatedAt),
- Duration: pipeline.Duration,
+ GitlabId: pipeline.Id,
+ ProjectId: projectId,
+ Ref: pipeline.Ref,
+ Sha: pipeline.Sha,
+ WebUrl: pipeline.WebUrl,
+ Status: pipeline.Status,
+ GitlabCreatedAt: helper.Iso8601TimeToTime(pipeline.CreatedAt),
+ GitlabUpdatedAt: helper.Iso8601TimeToTime(pipeline.UpdatedAt),
+ GitlabStartedAt: helper.Iso8601TimeToTime(pipeline.StartedAt),
+ GitlabFinishedAt: helper.Iso8601TimeToTime(pipeline.FinishedAt),
+ Duration: pipeline.Duration,
}
return gitlabPipeline, nil
}