You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by ab...@apache.org on 2023/08/18 08:35:09 UTC
[incubator-devlake] branch release-v0.18 updated: cherry-pick #5908 fix: converting pipelines to deployments should ignore skipped items to v0.18 (beta6) (#5913)
This is an automated email from the ASF dual-hosted git repository.
abeizn pushed a commit to branch release-v0.18
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/release-v0.18 by this push:
new 2865173a7 cherry-pick #5908 fix: converting pipelines to deployments should ignore skipped items to v0.18 (beta6) (#5913)
2865173a7 is described below
commit 2865173a764b9f9627657b6930f80042023bfbc9
Author: Klesh Wong <zh...@merico.dev>
AuthorDate: Fri Aug 18 16:35:04 2023 +0800
cherry-pick #5908 fix: converting pipelines to deployments should ignore skipped items to v0.18 (beta6) (#5913)
* fix: converting pipelines to deployments should ignore skipped items
Closes #5885
* fix: linting and test cases
---
.../models/domainlayer/devops/cicd_pipeline.go | 43 +++++++++++++---------
.../plugins/bamboo/tasks/deploy_build_convertor.go | 4 +-
.../plugins/bamboo/tasks/job_build_convertor.go | 4 +-
.../plugins/bamboo/tasks/plan_build_convertor.go | 4 +-
.../bitbucket/tasks/deployment_convertor.go | 4 +-
.../plugins/bitbucket/tasks/pipeline_convertor.go | 9 +++--
.../bitbucket/tasks/pipeline_steps_convertor.go | 11 +++---
.../cicd_deployment_commits.csv | 2 +-
.../deployment_generator/cicd_pipeline_commits.csv | 2 +
.../e2e/deployment_generator/cicd_pipelines.csv | 4 +-
.../dora/e2e/deployment_generator/cicd_tasks.csv | 1 +
.../dora/tasks/deployment_commits_generator.go | 18 +++++----
.../dora/tasks/incident_deploy_connector.go | 2 +-
.../dora/tasks/prev_deployment_commit_enricher.go | 2 +-
backend/plugins/github/tasks/cicd_job_convertor.go | 28 ++++++--------
backend/plugins/github/tasks/cicd_run_convertor.go | 25 +++++--------
backend/plugins/gitlab/tasks/job_convertor.go | 7 ++--
backend/plugins/gitlab/tasks/pipeline_convertor.go | 9 +++--
backend/plugins/jenkins/e2e/builds_test.go | 6 +--
backend/plugins/jenkins/impl/impl.go | 2 +-
.../plugins/jenkins/tasks/build_cicd_convertor.go | 33 ++++++++---------
backend/plugins/jenkins/tasks/stage_convertor.go | 10 ++---
backend/plugins/webhook/api/deployments.go | 4 +-
23 files changed, 121 insertions(+), 113 deletions(-)
diff --git a/backend/core/models/domainlayer/devops/cicd_pipeline.go b/backend/core/models/domainlayer/devops/cicd_pipeline.go
index 167d62c67..40612f2e8 100644
--- a/backend/core/models/domainlayer/devops/cicd_pipeline.go
+++ b/backend/core/models/domainlayer/devops/cicd_pipeline.go
@@ -42,16 +42,17 @@ func (CICDPipeline) TableName() string {
// this is for the field `result` in table.cicd_pipelines and table.cicd_tasks
const (
- SUCCESS = "SUCCESS"
- FAILURE = "FAILURE"
- ABORT = "ABORT"
- MANUAL = "MANUAL"
+ RESULT_SUCCESS = "SUCCESS"
+ RESULT_FAILURE = "FAILURE"
+ RESULT_ABORT = "ABORT"
+ RESULT_MANUAL = "MANUAL"
+ RESULT_SKIPPED = "SKIPPED"
)
// this is for the field `status` in table.cicd_pipelines and table.cicd_tasks
const (
- IN_PROGRESS = "IN_PROGRESS"
- DONE = "DONE"
+ STATUS_IN_PROGRESS = "IN_PROGRESS"
+ STATUS_DONE = "DONE"
)
type ResultRule struct {
@@ -59,12 +60,13 @@ type ResultRule struct {
Failed []string
Abort []string
Manual []string
+ Skipped []string
Default string
}
-type StatusRule struct {
- InProgress []string
- Done []string
- Manual []string
+type StatusRule[T comparable] struct {
+ InProgress []T
+ Done []T
+ Manual []T
Default string
}
@@ -72,42 +74,47 @@ type StatusRule struct {
func GetResult(rule *ResultRule, input interface{}) string {
for _, suc := range rule.Success {
if suc == input {
- return SUCCESS
+ return RESULT_SUCCESS
}
}
for _, fail := range rule.Failed {
if fail == input {
- return FAILURE
+ return RESULT_FAILURE
}
}
for _, abort := range rule.Abort {
if abort == input {
- return ABORT
+ return RESULT_ABORT
}
}
for _, manual := range rule.Manual {
if manual == input {
- return MANUAL
+ return RESULT_MANUAL
+ }
+ }
+ for _, skipped := range rule.Skipped {
+ if skipped == input {
+ return RESULT_SKIPPED
}
}
return rule.Default
}
// GetStatus compare the input with rule for return the enmu value of status
-func GetStatus(rule *StatusRule, input interface{}) string {
+func GetStatus[T comparable](rule *StatusRule[T], input T) string {
for _, inp := range rule.InProgress {
if inp == input {
- return IN_PROGRESS
+ return STATUS_IN_PROGRESS
}
}
for _, done := range rule.Done {
if done == input {
- return DONE
+ return STATUS_DONE
}
}
for _, manual := range rule.Manual {
if manual == input {
- return MANUAL
+ return RESULT_MANUAL
}
}
return rule.Default
diff --git a/backend/plugins/bamboo/tasks/deploy_build_convertor.go b/backend/plugins/bamboo/tasks/deploy_build_convertor.go
index a0ada3719..c6b08128f 100644
--- a/backend/plugins/bamboo/tasks/deploy_build_convertor.go
+++ b/backend/plugins/bamboo/tasks/deploy_build_convertor.go
@@ -81,9 +81,9 @@ func ConvertDeployBuilds(taskCtx plugin.SubTaskContext) errors.Error {
Default: "",
}, deployBuild.DeploymentState),
- Status: devops.GetStatus(&devops.StatusRule{
+ Status: devops.GetStatus(&devops.StatusRule[string]{
Done: []string{"Finished", "FINISHED"},
- Default: devops.IN_PROGRESS,
+ Default: devops.STATUS_IN_PROGRESS,
}, deployBuild.LifeCycleState),
//DurationSec: uint64(deployBuild),
diff --git a/backend/plugins/bamboo/tasks/job_build_convertor.go b/backend/plugins/bamboo/tasks/job_build_convertor.go
index 8ef924b6d..292c4dc8f 100644
--- a/backend/plugins/bamboo/tasks/job_build_convertor.go
+++ b/backend/plugins/bamboo/tasks/job_build_convertor.go
@@ -73,9 +73,9 @@ func ConvertJobBuilds(taskCtx plugin.SubTaskContext) errors.Error {
Default: "",
}, line.BuildState),
- Status: devops.GetStatus(&devops.StatusRule{
+ Status: devops.GetStatus(&devops.StatusRule[string]{
Done: []string{"Finished"},
- Default: devops.IN_PROGRESS,
+ Default: devops.STATUS_IN_PROGRESS,
}, line.LifeCycleState),
}
diff --git a/backend/plugins/bamboo/tasks/plan_build_convertor.go b/backend/plugins/bamboo/tasks/plan_build_convertor.go
index 25818b7e8..27d4512ea 100644
--- a/backend/plugins/bamboo/tasks/plan_build_convertor.go
+++ b/backend/plugins/bamboo/tasks/plan_build_convertor.go
@@ -79,9 +79,9 @@ func ConvertPlanBuilds(taskCtx plugin.SubTaskContext) errors.Error {
Default: "",
}, line.BuildState),
- Status: devops.GetStatus(&devops.StatusRule{
+ Status: devops.GetStatus(&devops.StatusRule[string]{
Done: []string{"Finished"},
- Default: devops.IN_PROGRESS,
+ Default: devops.STATUS_IN_PROGRESS,
}, line.LifeCycleState),
}
diff --git a/backend/plugins/bitbucket/tasks/deployment_convertor.go b/backend/plugins/bitbucket/tasks/deployment_convertor.go
index 3de4696eb..a94bb1f1b 100644
--- a/backend/plugins/bitbucket/tasks/deployment_convertor.go
+++ b/backend/plugins/bitbucket/tasks/deployment_convertor.go
@@ -92,9 +92,9 @@ func ConvertDeployments(taskCtx plugin.SubTaskContext) errors.Error {
Success: []string{"COMPLETED"},
Default: "",
}, bitbucketDeployment.Status),
- Status: devops.GetStatus(&devops.StatusRule{
+ Status: devops.GetStatus(&devops.StatusRule[string]{
Done: []string{"COMPLETED", "UNDEPLOYED"},
- Default: devops.IN_PROGRESS,
+ Default: devops.STATUS_IN_PROGRESS,
}, bitbucketDeployment.Status),
Environment: bitbucketDeployment.Environment,
CreatedDate: *bitbucketDeployment.CreatedOn,
diff --git a/backend/plugins/bitbucket/tasks/pipeline_convertor.go b/backend/plugins/bitbucket/tasks/pipeline_convertor.go
index 3f38cd291..c5eb48157 100644
--- a/backend/plugins/bitbucket/tasks/pipeline_convertor.go
+++ b/backend/plugins/bitbucket/tasks/pipeline_convertor.go
@@ -88,14 +88,15 @@ func ConvertPipelines(taskCtx plugin.SubTaskContext) errors.Error {
Generate(data.Options.ConnectionId, bitbucketPipeline.RefName),
Result: devops.GetResult(&devops.ResultRule{
Failed: []string{models.FAILED, models.ERROR, models.UNDEPLOYED},
- Abort: []string{models.STOPPED, models.SKIPPED},
+ Abort: []string{models.STOPPED},
Success: []string{models.SUCCESSFUL, models.COMPLETED},
Manual: []string{models.PAUSED, models.HALTED},
- Default: devops.SUCCESS,
+ Skipped: []string{models.SKIPPED},
+ Default: devops.RESULT_SUCCESS,
}, bitbucketPipeline.Result),
- Status: devops.GetStatus(&devops.StatusRule{
+ Status: devops.GetStatus(&devops.StatusRule[string]{
InProgress: []string{models.IN_PROGRESS, models.PENDING, models.BUILDING},
- Default: devops.DONE,
+ Default: devops.STATUS_DONE,
}, bitbucketPipeline.Status),
Type: bitbucketPipeline.Type,
Environment: bitbucketPipeline.Environment,
diff --git a/backend/plugins/bitbucket/tasks/pipeline_steps_convertor.go b/backend/plugins/bitbucket/tasks/pipeline_steps_convertor.go
index c36e25f15..b8d1ab76d 100644
--- a/backend/plugins/bitbucket/tasks/pipeline_steps_convertor.go
+++ b/backend/plugins/bitbucket/tasks/pipeline_steps_convertor.go
@@ -70,14 +70,15 @@ func ConvertPipelineSteps(taskCtx plugin.SubTaskContext) errors.Error {
PipelineId: pipelineIdGen.Generate(data.Options.ConnectionId, bitbucketPipelineStep.PipelineId),
Result: devops.GetResult(&devops.ResultRule{
Failed: []string{models.FAILED, models.ERROR, models.UNDEPLOYED},
- Abort: []string{models.STOPPED, models.SKIPPED},
+ Abort: []string{models.STOPPED},
Success: []string{models.SUCCESSFUL, models.COMPLETED},
Manual: []string{models.PAUSED, models.HALTED},
- Default: devops.SUCCESS,
+ Skipped: []string{models.SKIPPED},
+ Default: devops.RESULT_SUCCESS,
}, bitbucketPipelineStep.Result),
- Status: devops.GetStatus(&devops.StatusRule{
+ Status: devops.GetStatus(&devops.StatusRule[string]{
InProgress: []string{models.IN_PROGRESS, models.PENDING, models.BUILDING},
- Default: devops.DONE,
+ Default: devops.STATUS_DONE,
}, bitbucketPipelineStep.State),
CicdScopeId: repoIdGen.Generate(data.Options.ConnectionId, data.Options.FullName),
}
@@ -87,7 +88,7 @@ func ConvertPipelineSteps(taskCtx plugin.SubTaskContext) errors.Error {
}
domainTask.StartedDate = *bitbucketPipelineStep.StartedOn
// rebuild the FinishedDate
- if domainTask.Status == devops.DONE {
+ if domainTask.Status == devops.STATUS_DONE {
domainTask.FinishedDate = bitbucketPipelineStep.CompletedOn
domainTask.DurationSec = uint64(bitbucketPipelineStep.DurationInSeconds)
}
diff --git a/backend/plugins/dora/e2e/deployment_generator/cicd_deployment_commits.csv b/backend/plugins/dora/e2e/deployment_generator/cicd_deployment_commits.csv
index 781b4d596..980576383 100644
--- a/backend/plugins/dora/e2e/deployment_generator/cicd_deployment_commits.csv
+++ b/backend/plugins/dora/e2e/deployment_generator/cicd_deployment_commits.csv
@@ -1,5 +1,5 @@
id,commit_sha,cicd_scope_id,cicd_deployment_id,result,repo_url,environment,started_date,finished_date
-gitlab:GitlabPipeline:1:457475337:https://gitlab.com/gitlab-data/snowflake_spend,10a6464b6bd2cf4b59b8ac37ce1466e013f5a20d,cicd1,gitlab:GitlabPipeline:1:457475337,,https://gitlab.com/gitlab-data/snowflake_spend,PRODUCTION,,
+gitlab:GitlabPipeline:1:457475337:https://gitlab.com/gitlab-data/snowflake_spend,10a6464b6bd2cf4b59b8ac37ce1466e013f5a20d,cicd1,gitlab:GitlabPipeline:1:457475337,ABORTED,https://gitlab.com/gitlab-data/snowflake_spend,PRODUCTION,,
gitlab:GitlabPipeline:1:485811050:https://gitlab.com/gitlab-data/snowflake_spend,c791ea6949d6b4aadf79b15ba666cb690c6527ac,cicd1,gitlab:GitlabPipeline:1:485811050,FAILURE,https://gitlab.com/gitlab-data/snowflake_spend,PRODUCTION,2022-03-07T06:26:42.109+00:00,2022-03-07T06:26:42.109+00:00
gitlab:GitlabPipeline:1:485813816:https://gitlab.com/gitlab-data/snowflake_spend,ecc7c0b2874c812ed882c9effbbda26e0abc7110,cicd1,gitlab:GitlabPipeline:1:485813816,FAILURE,https://gitlab.com/gitlab-data/snowflake_spend,PRODUCTION,2022-03-07T06:33:56.824+00:00,2022-03-07T06:33:56.824+00:00
gitlab:GitlabPipeline:1:485814501:https://gitlab.com/gitlab-data/snowflake_spend,6a3346f8434cc65fbe3f7a80a0edec5b4014a733,cicd1,gitlab:GitlabPipeline:1:485814501,FAILURE,https://gitlab.com/gitlab-data/snowflake_spend,PRODUCTION,2022-03-07T06:35:28.111+00:00,2022-03-07T06:35:28.111+00:00
diff --git a/backend/plugins/dora/e2e/deployment_generator/cicd_pipeline_commits.csv b/backend/plugins/dora/e2e/deployment_generator/cicd_pipeline_commits.csv
index e34ee8b6d..5adda0c47 100644
--- a/backend/plugins/dora/e2e/deployment_generator/cicd_pipeline_commits.csv
+++ b/backend/plugins/dora/e2e/deployment_generator/cicd_pipeline_commits.csv
@@ -12,3 +12,5 @@ gitlab:GitlabPipeline:1:485811059,c791ea6949d6b4aadf79b15ba666cb690c6527ac,refs/
gitlab:GitlabPipeline:1:485811050,c791ea6949d6b4aadf79b15ba666cb690c6527ac,EE-7121,cicd1,https://gitlab.com/gitlab-data/snowflake_spend
gitlab:GitlabPipeline:1:457475160,44d127e0ab7dbc4bc259b55929c9d00b62fc3bf4,renovate/lodash-monorepo,cicd1,https://gitlab.com/gitlab-data/snowflake_spend
gitlab:GitlabPipeline:1:457475337,10a6464b6bd2cf4b59b8ac37ce1466e013f5a20d,renovate/shx-0.x,cicd1,https://gitlab.com/gitlab-data/snowflake_spend
+gitlab:GitlabPipeline:1:485932864,22a6464b6bd2cf4b59b8ac37ce1466e013f5a20d,renovate/shx-0.x,cicd1,https://gitlab.com/gitlab-data/snowflake_spend
+gitlab:GitlabPipeline:1:485932865,33a6464b6bd2cf4b59b8ac37ce1466e013f5a20d,renovate/shx-0.x,cicd1,https://gitlab.com/gitlab-data/snowflake_spend
diff --git a/backend/plugins/dora/e2e/deployment_generator/cicd_pipelines.csv b/backend/plugins/dora/e2e/deployment_generator/cicd_pipelines.csv
index 32302c103..aa57b288d 100644
--- a/backend/plugins/dora/e2e/deployment_generator/cicd_pipelines.csv
+++ b/backend/plugins/dora/e2e/deployment_generator/cicd_pipelines.csv
@@ -1,6 +1,6 @@
id,name,result,status,type,duration_sec,environment,created_date,finished_date,cicd_scope_id
gitlab:GitlabPipeline:1:457475160,cicd1,FAILURE,DONE,,0,,2022-01-27T10:07:26.435+00:00,2022-01-27T10:07:26.638+00:00,cicd1
-gitlab:GitlabPipeline:1:457475337,cicd1,,IN_PROGRESS,,0,,2022-01-27T10:07:36.502+00:00,,cicd1
+gitlab:GitlabPipeline:1:457475337,cicd1,ABORTED,IN_PROGRESS,,0,,2022-01-27T10:07:36.502+00:00,,cicd1
gitlab:GitlabPipeline:1:485811050,cicd1,FAILURE,DONE,DEPLOYMENT,0,PRODUCTION,2022-03-07T06:26:42.109+00:00,2022-03-07T06:26:42.109+00:00,cicd1
gitlab:GitlabPipeline:1:485811059,cicd1,FAILURE,DONE,,0,,2022-03-07T06:26:43.784+00:00,2022-03-07T06:26:43.784+00:00,cicd1
gitlab:GitlabPipeline:1:485813816,cicd1,FAILURE,DONE,DEPLOYMENT,0,PRODUCTION,2022-03-07T06:33:56.824+00:00,2022-03-07T06:33:56.824+00:00,cicd1
@@ -12,3 +12,5 @@ gitlab:GitlabPipeline:1:485817670,cicd1,FAILURE,DONE,,1956,,2022-03-07T06:45:09.
gitlab:GitlabPipeline:1:485845850,cicd1,FAILURE,DONE,,419,,2022-03-07T07:38:58.611+00:00,2022-03-07T07:45:58.412+00:00,cicd1
gitlab:GitlabPipeline:1:485877118,cicd1,FAILURE,DONE,,289,,2022-03-07T08:22:48.943+00:00,2022-03-07T08:27:38.364+00:00,cicd1
gitlab:GitlabPipeline:1:485932863,cicd1,SUCCESS,DONE,DEPLOYMENT,398,,2022-03-07T09:34:57.476+00:00,2022-03-07T09:41:36.267+00:00,cicd1
+gitlab:GitlabPipeline:1:485932864,cicd1,SKIPPED,DONE,DEPLOYMENT,398,,2022-03-07T09:34:57.476+00:00,2022-03-07T09:41:36.267+00:00,cicd1
+gitlab:GitlabPipeline:1:485932865,cicd1,SUCCESS,DONE,,398,,2022-03-07T09:34:57.476+00:00,2022-03-07T09:41:36.267+00:00,cicd1
\ No newline at end of file
diff --git a/backend/plugins/dora/e2e/deployment_generator/cicd_tasks.csv b/backend/plugins/dora/e2e/deployment_generator/cicd_tasks.csv
index 7c35ccc10..16b855aac 100644
--- a/backend/plugins/dora/e2e/deployment_generator/cicd_tasks.csv
+++ b/backend/plugins/dora/e2e/deployment_generator/cicd_tasks.csv
@@ -10,3 +10,4 @@ gitlab:GitlabJob:1:107,format,gitlab:GitlabPipeline:1:457475337,SUCCESS,DONE,,,2
gitlab:GitlabJob:1:108,format,gitlab:GitlabPipeline:1:457475337,SUCCESS,DONE,,,2,2022-07-25T15:35:11.707+00:00,2022-07-25T15:35:14.224+00:00,gitlab:GitlabProject:1:44
gitlab:GitlabJob:1:109,compile,gitlab:GitlabPipeline:1:457475337,SUCCESS,DONE,DEPLOYMENT,PRODUCTION,3,2022-07-25T15:35:14.724+00:00,2022-07-25T15:35:17.828+00:00,gitlab:GitlabProject:1:44
gitlab:GitlabJob:1:110,format,gitlab:GitlabPipeline:1:457475337,SUCCESS,DONE,,,2,2022-07-25T15:36:18.097+00:00,2022-07-25T15:36:20.954+00:00,gitlab:GitlabProject:1:44
+gitlab:GitlabJob:1:111,format,gitlab:GitlabPipeline:1:485932865,SKIPPED,DONE,DEPLOYMENT,PRODUCTION,2,2022-07-25T15:36:18.097+00:00,2022-07-25T15:36:20.954+00:00,gitlab:GitlabProject:1:44
diff --git a/backend/plugins/dora/tasks/deployment_commits_generator.go b/backend/plugins/dora/tasks/deployment_commits_generator.go
index 744cde5e1..b1307a1e9 100644
--- a/backend/plugins/dora/tasks/deployment_commits_generator.go
+++ b/backend/plugins/dora/tasks/deployment_commits_generator.go
@@ -69,16 +69,16 @@ func GenerateDeploymentCommits(taskCtx plugin.SubTaskContext) errors.Error {
p.finished_date,
p.environment,
p.cicd_scope_id,
- EXISTS(SELECT 1 FROM cicd_tasks t WHERE t.pipeline_id = p.id AND t.environment = ?)
+ EXISTS(SELECT 1 FROM cicd_tasks t WHERE t.pipeline_id = p.id AND t.environment = ? AND t.result <> ?)
as has_testing_tasks,
- EXISTS(SELECT 1 FROM cicd_tasks t WHERE t.pipeline_id = p.id AND t.environment = ?)
+ EXISTS(SELECT 1 FROM cicd_tasks t WHERE t.pipeline_id = p.id AND t.environment = ? AND t.result <> ?)
as has_staging_tasks,
- EXISTS( SELECT 1 FROM cicd_tasks t WHERE t.pipeline_id = p.id AND t.environment = ?)
+ EXISTS( SELECT 1 FROM cicd_tasks t WHERE t.pipeline_id = p.id AND t.environment = ? AND t.result <> ?)
as has_production_tasks
`,
- devops.TESTING,
- devops.STAGING,
- devops.PRODUCTION,
+ devops.TESTING, devops.RESULT_SKIPPED,
+ devops.STAGING, devops.RESULT_SKIPPED,
+ devops.PRODUCTION, devops.RESULT_SKIPPED,
),
dal.From("cicd_pipeline_commits pc"),
dal.Join("LEFT JOIN cicd_pipelines p ON (p.id = pc.pipeline_id)"),
@@ -87,13 +87,15 @@ func GenerateDeploymentCommits(taskCtx plugin.SubTaskContext) errors.Error {
`
pm.project_name = ? AND (
p.type = ? OR EXISTS(
- SELECT 1 FROM cicd_tasks t WHERE t.pipeline_id = p.id AND t.type = ?
+ SELECT 1 FROM cicd_tasks t WHERE t.pipeline_id = p.id AND t.type = ? AND t.result <> ?
)
- )
+ ) AND p.result <> ?
`,
data.Options.ProjectName,
devops.DEPLOYMENT,
devops.DEPLOYMENT,
+ devops.RESULT_SKIPPED,
+ devops.RESULT_SKIPPED,
),
)
if err != nil {
diff --git a/backend/plugins/dora/tasks/incident_deploy_connector.go b/backend/plugins/dora/tasks/incident_deploy_connector.go
index 839b75486..cb013503a 100644
--- a/backend/plugins/dora/tasks/incident_deploy_connector.go
+++ b/backend/plugins/dora/tasks/incident_deploy_connector.go
@@ -93,7 +93,7 @@ func ConnectIncidentToDeployment(taskCtx plugin.SubTaskContext) errors.Error {
and cicd_deployment_commits.environment = ?
and pm.table = ?
and pm.project_name = ?`,
- issue.CreatedDate, devops.SUCCESS, devops.PRODUCTION, "cicd_scopes", data.Options.ProjectName,
+ issue.CreatedDate, devops.RESULT_SUCCESS, devops.PRODUCTION, "cicd_scopes", data.Options.ProjectName,
),
dal.Orderby("finished_date DESC"),
dal.Limit(1),
diff --git a/backend/plugins/dora/tasks/prev_deployment_commit_enricher.go b/backend/plugins/dora/tasks/prev_deployment_commit_enricher.go
index cfeec0902..a60985247 100644
--- a/backend/plugins/dora/tasks/prev_deployment_commit_enricher.go
+++ b/backend/plugins/dora/tasks/prev_deployment_commit_enricher.go
@@ -58,7 +58,7 @@ func EnrichPrevSuccessDeploymentCommit(taskCtx plugin.SubTaskContext) errors.Err
AND dc.repo_url IS NOT NULL AND dc.repo_url != ''
AND pm.project_name = ? AND dc.result = ?
`,
- data.Options.ProjectName, devops.SUCCESS,
+ data.Options.ProjectName, devops.RESULT_SUCCESS,
),
dal.Orderby(`dc.cicd_scope_id, dc.repo_url, dc.environment, dc.finished_date`),
)
diff --git a/backend/plugins/github/tasks/cicd_job_convertor.go b/backend/plugins/github/tasks/cicd_job_convertor.go
index b59ae50da..d5896ba8c 100644
--- a/backend/plugins/github/tasks/cicd_job_convertor.go
+++ b/backend/plugins/github/tasks/cicd_job_convertor.go
@@ -18,6 +18,8 @@ limitations under the License.
package tasks
import (
+ "reflect"
+
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models/domainlayer"
@@ -26,8 +28,6 @@ import (
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/github/models"
- "reflect"
- "strings"
)
var ConvertJobsMeta = plugin.SubTaskMeta{
@@ -85,22 +85,18 @@ func ConvertJobs(taskCtx plugin.SubTaskContext) (err errors.Error) {
CicdScopeId: repoIdGen.Generate(data.Options.ConnectionId, line.RepoId),
Type: line.Type,
Environment: line.Environment,
+ Result: devops.GetResult(&devops.ResultRule{
+ Failed: []string{"failure", "FAILURE"},
+ Success: []string{"success", "SUCCESS"},
+ Skipped: []string{"skipped", "SKIPPED"},
+ }, line.Conclusion),
+ Status: devops.GetStatus(&devops.StatusRule[string]{
+ Done: []string{"completed", "COMPLETED"},
+ Default: devops.STATUS_IN_PROGRESS,
+ }, line.Status),
}
- if strings.Contains(line.Conclusion, "SUCCESS") {
- domainJob.Result = devops.SUCCESS
- } else if strings.Contains(line.Conclusion, "FAILURE") {
- domainJob.Result = devops.FAILURE
- } else if strings.Contains(line.Conclusion, "ABORT") {
- domainJob.Result = devops.ABORT
- } else {
- domainJob.Result = ""
- }
-
- if line.Status != "COMPLETED" {
- domainJob.Status = devops.IN_PROGRESS
- } else {
- domainJob.Status = devops.DONE
+ if domainJob.Status == devops.STATUS_DONE {
domainJob.DurationSec = uint64(line.CompletedAt.Sub(*line.StartedAt).Seconds())
}
diff --git a/backend/plugins/github/tasks/cicd_run_convertor.go b/backend/plugins/github/tasks/cicd_run_convertor.go
index 09bccc84f..d6b386f18 100644
--- a/backend/plugins/github/tasks/cicd_run_convertor.go
+++ b/backend/plugins/github/tasks/cicd_run_convertor.go
@@ -19,7 +19,6 @@ package tasks
import (
"reflect"
- "strings"
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
@@ -85,21 +84,17 @@ func ConvertRuns(taskCtx plugin.SubTaskContext) errors.Error {
CicdScopeId: repoIdGen.Generate(data.Options.ConnectionId, line.RepoId),
Type: line.Type,
Environment: line.Environment,
+ Result: devops.GetResult(&devops.ResultRule{
+ Failed: []string{"failure", "FAILURE"},
+ Success: []string{"success", "SUCCESS"},
+ Skipped: []string{"skipped", "SKIPPED"},
+ }, line.Conclusion),
+ Status: devops.GetStatus(&devops.StatusRule[string]{
+ Done: []string{"completed", "COMPLETED"},
+ Default: devops.STATUS_IN_PROGRESS,
+ }, line.Status),
}
- if strings.Contains(line.Conclusion, "success") {
- domainPipeline.Result = devops.SUCCESS
- } else if strings.Contains(line.Conclusion, "failure") {
- domainPipeline.Result = devops.FAILURE
- } else if strings.Contains(line.Conclusion, "abort") {
- domainPipeline.Result = devops.ABORT
- } else {
- domainPipeline.Result = ""
- }
-
- if line.Status != "completed" {
- domainPipeline.Status = devops.IN_PROGRESS
- } else {
- domainPipeline.Status = devops.DONE
+ if domainPipeline.Status == devops.STATUS_DONE {
domainPipeline.DurationSec = uint64(line.GithubUpdatedAt.Sub(*line.GithubCreatedAt).Seconds())
}
diff --git a/backend/plugins/gitlab/tasks/job_convertor.go b/backend/plugins/gitlab/tasks/job_convertor.go
index 6f9605452..b8bbacb09 100644
--- a/backend/plugins/gitlab/tasks/job_convertor.go
+++ b/backend/plugins/gitlab/tasks/job_convertor.go
@@ -86,15 +86,16 @@ func ConvertJobs(taskCtx plugin.SubTaskContext) (err errors.Error) {
PipelineId: pipelineIdGen.Generate(data.Options.ConnectionId, gitlabJob.PipelineId),
Result: devops.GetResult(&devops.ResultRule{
Failed: []string{"failed"},
- Abort: []string{"canceled", "skipped"},
+ Abort: []string{"canceled"},
Manual: []string{"manual"},
Success: []string{"success"},
+ Skipped: []string{"skipped"},
Default: "",
}, gitlabJob.Status),
- Status: devops.GetStatus(&devops.StatusRule{
+ Status: devops.GetStatus(&devops.StatusRule[string]{
InProgress: []string{"created", "waiting_for_resource", "preparing", "pending", "running", "scheduled"},
Manual: []string{"manual"},
- Default: devops.DONE,
+ Default: devops.STATUS_DONE,
}, gitlabJob.Status),
DurationSec: uint64(gitlabJob.Duration),
diff --git a/backend/plugins/gitlab/tasks/pipeline_convertor.go b/backend/plugins/gitlab/tasks/pipeline_convertor.go
index 320a508ed..a2bfddc2b 100644
--- a/backend/plugins/gitlab/tasks/pipeline_convertor.go
+++ b/backend/plugins/gitlab/tasks/pipeline_convertor.go
@@ -86,13 +86,14 @@ func ConvertPipelines(taskCtx plugin.SubTaskContext) errors.Error {
Generate(data.Options.ConnectionId, data.Options.ProjectId),
Result: devops.GetResult(&devops.ResultRule{
Failed: []string{"failed"},
- Abort: []string{"canceled", "skipped"},
+ Abort: []string{"canceled"},
Success: []string{"success"},
+ Skipped: []string{"skipped"},
Default: "",
}, gitlabPipeline.Status),
- Status: devops.GetStatus(&devops.StatusRule{
+ Status: devops.GetStatus(&devops.StatusRule[string]{
InProgress: []string{"created", "waiting_for_resource", "preparing", "pending", "running", "manual", "scheduled"},
- Default: devops.DONE,
+ Default: devops.STATUS_DONE,
}, gitlabPipeline.Status),
CreatedDate: createdAt,
FinishedDate: gitlabPipeline.GitlabUpdatedAt,
@@ -102,7 +103,7 @@ func ConvertPipelines(taskCtx plugin.SubTaskContext) errors.Error {
}
// rebuild the FinishedDate and DurationSec by Status
- if domainPipeline.Status != devops.DONE {
+ if domainPipeline.Status != devops.STATUS_DONE {
domainPipeline.FinishedDate = nil
domainPipeline.DurationSec = 0
} else if domainPipeline.FinishedDate != nil {
diff --git a/backend/plugins/jenkins/e2e/builds_test.go b/backend/plugins/jenkins/e2e/builds_test.go
index 948a4ab90..f6b8b581a 100644
--- a/backend/plugins/jenkins/e2e/builds_test.go
+++ b/backend/plugins/jenkins/e2e/builds_test.go
@@ -102,10 +102,10 @@ func TestJenkinsBuildsDataFlow(t *testing.T) {
)
dataflowTester.Subtask(tasks.ConvertBuildReposMeta, taskData)
- dataflowTester.Subtask(tasks.ConvertBuildsToCICDMeta, taskData)
+ dataflowTester.Subtask(tasks.ConvertBuildsToCicdTasksMeta, taskData)
// verify env when prod env is omitted
- dataflowTester.Subtask(tasks.ConvertBuildsToCICDMeta, taskData)
+ dataflowTester.Subtask(tasks.ConvertBuildsToCicdTasksMeta, taskData)
dataflowTester.VerifyTable(
devops.CICDTask{},
"./snapshot_tables/cicd_tasks_no_prod_regex.csv",
@@ -117,7 +117,7 @@ func TestJenkinsBuildsDataFlow(t *testing.T) {
// continue
_ = regexEnricher.TryAdd(devops.PRODUCTION, `test-sub-sub-dir\/devlake.*`)
dataflowTester.FlushTabler(&devops.CICDTask{})
- dataflowTester.Subtask(tasks.ConvertBuildsToCICDMeta, taskData)
+ dataflowTester.Subtask(tasks.ConvertBuildsToCicdTasksMeta, taskData)
dataflowTester.VerifyTable(
devops.CICDTask{},
"./snapshot_tables/cicd_tasks.csv",
diff --git a/backend/plugins/jenkins/impl/impl.go b/backend/plugins/jenkins/impl/impl.go
index 6bb0ed362..bc44bec07 100644
--- a/backend/plugins/jenkins/impl/impl.go
+++ b/backend/plugins/jenkins/impl/impl.go
@@ -95,7 +95,7 @@ func (p Jenkins) SubTaskMetas() []plugin.SubTaskMeta {
tasks.CollectApiStagesMeta,
tasks.ExtractApiStagesMeta,
tasks.EnrichApiBuildWithStagesMeta,
- tasks.ConvertBuildsToCICDMeta,
+ tasks.ConvertBuildsToCicdTasksMeta,
tasks.ConvertStagesMeta,
tasks.ConvertBuildReposMeta,
}
diff --git a/backend/plugins/jenkins/tasks/build_cicd_convertor.go b/backend/plugins/jenkins/tasks/build_cicd_convertor.go
index f95380979..e3c4a13c1 100644
--- a/backend/plugins/jenkins/tasks/build_cicd_convertor.go
+++ b/backend/plugins/jenkins/tasks/build_cicd_convertor.go
@@ -31,15 +31,15 @@ import (
"github.com/apache/incubator-devlake/plugins/jenkins/models"
)
-var ConvertBuildsToCICDMeta = plugin.SubTaskMeta{
+var ConvertBuildsToCicdTasksMeta = plugin.SubTaskMeta{
Name: "convertBuildsToCICD",
- EntryPoint: ConvertBuildsToCICD,
+ EntryPoint: ConvertBuildsToCicdTasks,
EnabledByDefault: true,
Description: "convert builds to cicd",
DomainTypes: []string{plugin.DOMAIN_TYPE_CICD},
}
-func ConvertBuildsToCICD(taskCtx plugin.SubTaskContext) (err errors.Error) {
+func ConvertBuildsToCicdTasks(taskCtx plugin.SubTaskContext) (err errors.Error) {
db := taskCtx.GetDal()
data := taskCtx.GetData().(*JenkinsTaskData)
clauses := []dal.Clause{
@@ -71,23 +71,22 @@ func ConvertBuildsToCICD(taskCtx plugin.SubTaskContext) (err errors.Error) {
Convert: func(inputRow interface{}) ([]interface{}, errors.Error) {
jenkinsBuild := inputRow.(*models.JenkinsBuild)
durationSec := int64(jenkinsBuild.Duration / 1000)
- jenkinsPipelineResult := ""
- jenkinsPipelineStatus := ""
+ jenkinsPipelineStatus := devops.GetStatus(&devops.StatusRule[bool]{
+ InProgress: []bool{true},
+ Done: []bool{false},
+ }, jenkinsBuild.Building)
+ var jenkinsPipelineResult string
+ if !jenkinsBuild.Building {
+ jenkinsPipelineResult = devops.GetResult(&devops.ResultRule{
+ Success: []string{"SUCCESS"},
+ Failed: []string{"FAILURE"},
+ Abort: []string{"ABORTED"},
+ }, jenkinsBuild.Result)
+ }
var jenkinsPipelineFinishedDate *time.Time
results := make([]interface{}, 0)
- if jenkinsBuild.Result == "SUCCESS" {
- jenkinsPipelineResult = devops.SUCCESS
- } else if jenkinsBuild.Result == "FAILURE" {
- jenkinsPipelineResult = devops.FAILURE
- } else {
- jenkinsPipelineResult = devops.ABORT
- }
- if jenkinsBuild.Building {
- jenkinsPipelineStatus = devops.IN_PROGRESS
- jenkinsPipelineResult = ""
- } else {
- jenkinsPipelineStatus = devops.DONE
+ if jenkinsPipelineStatus == devops.STATUS_DONE {
finishTime := jenkinsBuild.StartTime.Add(time.Duration(durationSec * int64(time.Second)))
jenkinsPipelineFinishedDate = &finishTime
}
diff --git a/backend/plugins/jenkins/tasks/stage_convertor.go b/backend/plugins/jenkins/tasks/stage_convertor.go
index 73fca37bc..91eca0a93 100644
--- a/backend/plugins/jenkins/tasks/stage_convertor.go
+++ b/backend/plugins/jenkins/tasks/stage_convertor.go
@@ -101,18 +101,18 @@ func ConvertStages(taskCtx plugin.SubTaskContext) (err errors.Error) {
}
durationSec := int64(body.DurationMillis / 1000)
jenkinsTaskResult := ""
- jenkinsTaskStatus := devops.DONE
+ jenkinsTaskStatus := devops.STATUS_DONE
var jenkinsTaskFinishedDate *time.Time
results := make([]interface{}, 0)
if body.Status == "SUCCESS" {
- jenkinsTaskResult = devops.SUCCESS
+ jenkinsTaskResult = devops.RESULT_SUCCESS
} else if body.Status == "FAILED" {
- jenkinsTaskResult = devops.FAILURE
+ jenkinsTaskResult = devops.RESULT_FAILURE
} else if body.Status == "ABORTED" {
- jenkinsTaskResult = devops.ABORT
+ jenkinsTaskResult = devops.RESULT_ABORT
} else {
jenkinsTaskResult = ""
- jenkinsTaskStatus = devops.IN_PROGRESS
+ jenkinsTaskStatus = devops.STATUS_IN_PROGRESS
}
startedDate := time.Unix(body.StartTimeMillis/1000, 0)
diff --git a/backend/plugins/webhook/api/deployments.go b/backend/plugins/webhook/api/deployments.go
index 16aabcf9b..33bc63285 100644
--- a/backend/plugins/webhook/api/deployments.go
+++ b/backend/plugins/webhook/api/deployments.go
@@ -100,7 +100,7 @@ func PostDeploymentCicdTask(input *plugin.ApiResourceInput) (*plugin.ApiResource
request.FinishedDate = &now
}
if request.Result == "" {
- request.Result = devops.SUCCESS
+ request.Result = devops.RESULT_SUCCESS
}
duration := uint64(request.FinishedDate.Sub(*request.StartedDate).Seconds())
@@ -113,7 +113,7 @@ func PostDeploymentCicdTask(input *plugin.ApiResourceInput) (*plugin.ApiResource
CicdScopeId: scopeId,
Name: fmt.Sprintf(`deployment for %s`, request.CommitSha),
Result: request.Result,
- Status: devops.DONE,
+ Status: devops.STATUS_DONE,
Environment: request.Environment,
CreatedDate: *request.CreatedDate,
StartedDate: request.StartedDate,