You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by li...@apache.org on 2023/05/15 08:09:18 UTC
[incubator-devlake] branch release-v0.17 updated: cherry #5180 env left empty when regex is omitted to v0.17 (#5183)
This is an automated email from the ASF dual-hosted git repository.
likyh pushed a commit to branch release-v0.17
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/release-v0.17 by this push:
new b54fa2aa8 cherry #5180 env left empty when regex is omitted to v0.17 (#5183)
b54fa2aa8 is described below
commit b54fa2aa831c83a500d100948cec67e4734c7bf4
Author: Klesh Wong <zh...@merico.dev>
AuthorDate: Mon May 15 16:09:13 2023 +0800
cherry #5180 env left empty when regex is omitted to v0.17 (#5183)
* fix: bitbucket pipeline/step should be marked as PRODUCTION when corresponding regex is omitted
* fix: github runs/jobs should be marked as PRODUCTION when corresponding regex is omitted
* fix: bamboo builds should be marked as PRODUCTION when corresponding regex is omitted
* fix: gitlab pipelines/jobs should be marked as PRODUCTION when corresponding regex is omitted
* fix: jenkins builds/stages should be marked as PRODUCTION when corresponding regex is omitted
---
.../helpers/pluginhelper/api/enrich_with_regex.go | 10 +++-
backend/plugins/bamboo/e2e/job_build_test.go | 14 ++++-
.../_tool_bamboo_job_builds_no_prod_env.csv | 10 ++++
.../plugins/bamboo/tasks/job_build_extractor.go | 2 +-
.../plugins/bitbucket/e2e/pipeline_steps_test.go | 6 +-
.../_tool_bitbucket_pipeline_steps.csv | 52 ++++++++---------
.../plugins/bitbucket/tasks/pipeline_extractor.go | 2 +-
.../bitbucket/tasks/pipeline_steps_extractor.go | 2 +-
backend/plugins/github/e2e/cicd_job_test.go | 16 +++--
backend/plugins/github/e2e/cicd_run_test.go | 18 ++++--
.../_tool_github_jobs_no_prod_env.csv | 21 +++++++
.../_tool_github_runs_no_prod_regex.csv | 21 +++++++
backend/plugins/github/tasks/cicd_job_extractor.go | 5 +-
backend/plugins/github/tasks/cicd_run_extractor.go | 3 +-
.../plugins/github_graphql/tasks/job_collector.go | 2 +-
backend/plugins/gitlab/e2e/job_test.go | 21 +++++--
backend/plugins/gitlab/e2e/pipelines_test.go | 14 ++++-
.../_tool_gitlab_pipelines_no_prod_regex.csv | 21 +++++++
.../snapshot_tables/cicd_tasks_no_prod_regex.csv | 68 ++++++++++++++++++++++
backend/plugins/gitlab/tasks/job_convertor.go | 12 ++--
backend/plugins/gitlab/tasks/pipeline_extractor.go | 2 +-
backend/plugins/jenkins/e2e/builds_test.go | 17 +++++-
.../cicd_tasks_after_stages_no_prod_regex.csv | 15 +++++
.../snapshot_tables/cicd_tasks_no_prod_regex.csv | 8 +++
backend/plugins/jenkins/e2e/stages_test.go | 13 ++++-
.../plugins/jenkins/tasks/build_cicd_convertor.go | 4 +-
backend/plugins/jenkins/tasks/stage_convertor.go | 2 +-
27 files changed, 314 insertions(+), 67 deletions(-)
diff --git a/backend/helpers/pluginhelper/api/enrich_with_regex.go b/backend/helpers/pluginhelper/api/enrich_with_regex.go
index a4d31697e..92e0c0b28 100644
--- a/backend/helpers/pluginhelper/api/enrich_with_regex.go
+++ b/backend/helpers/pluginhelper/api/enrich_with_regex.go
@@ -84,7 +84,7 @@ func (r *RegexEnricher) TryAdd(name, pattern string) errors.Error {
return nil
}
-// ReturnNameIfMatched will return name if any of the targets matches the regex with the given name
+// ReturnNameIfMatched will return name if any of the targets matches the regex associated with the given name
func (r *RegexEnricher) ReturnNameIfMatched(name string, targets ...string) string {
if regex, ok := r.regexpMap[name]; !ok {
return ""
@@ -97,3 +97,11 @@ func (r *RegexEnricher) ReturnNameIfMatched(name string, targets ...string) stri
}
return ""
}
+
+// ReturnNameIfMatchedOrOmitted returns the given name if regex of the given name is omitted or fallback to ReturnNameIfMatched
+func (r *RegexEnricher) ReturnNameIfOmittedOrMatched(name string, targets ...string) string {
+ if _, ok := r.regexpMap[name]; !ok {
+ return name
+ }
+ return r.ReturnNameIfMatched(name, targets...)
+}
diff --git a/backend/plugins/bamboo/e2e/job_build_test.go b/backend/plugins/bamboo/e2e/job_build_test.go
index 79b80fe4b..82677f2f5 100644
--- a/backend/plugins/bamboo/e2e/job_build_test.go
+++ b/backend/plugins/bamboo/e2e/job_build_test.go
@@ -45,12 +45,24 @@ func TestBambooJobBuildDataFlow(t *testing.T) {
RegexEnricher: helper.NewRegexEnricher(),
}
taskData.RegexEnricher.TryAdd(devops.DEPLOYMENT, taskData.Options.DeploymentPattern)
- taskData.RegexEnricher.TryAdd(devops.PRODUCTION, taskData.Options.ProductionPattern)
// import raw data table
// SELECT * FROM _raw_bamboo_api_job_build INTO OUTFILE "/tmp/_raw_bamboo_api_job_build.csv" FIELDS TERMINATED BY ',' OPTIONALLY ENCLOSED BY '"' LINES TERMINATED BY '\r\n';
dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_bamboo_api_job_build.csv", "_raw_bamboo_api_job_build")
+ // verify env when production regex is not set
+ dataflowTester.FlushTabler(&models.BambooJobBuild{})
+ dataflowTester.Subtask(tasks.ExtractJobBuildMeta, taskData)
+ dataflowTester.Subtask(tasks.ExtractJobBuildMeta, taskData)
+ dataflowTester.VerifyTable(
+ models.BambooJobBuild{},
+ "./snapshot_tables/_tool_bamboo_job_builds_no_prod_env.csv",
+ e2ehelper.ColumnWithRawData(
+ "environment",
+ ),
+ )
+
// verify extraction
+ taskData.RegexEnricher.TryAdd(devops.PRODUCTION, taskData.Options.ProductionPattern)
dataflowTester.FlushTabler(&models.BambooJobBuild{})
dataflowTester.Subtask(tasks.ExtractJobBuildMeta, taskData)
dataflowTester.VerifyTable(
diff --git a/backend/plugins/bamboo/e2e/snapshot_tables/_tool_bamboo_job_builds_no_prod_env.csv b/backend/plugins/bamboo/e2e/snapshot_tables/_tool_bamboo_job_builds_no_prod_env.csv
new file mode 100644
index 000000000..bb0a5c9c8
--- /dev/null
+++ b/backend/plugins/bamboo/e2e/snapshot_tables/_tool_bamboo_job_builds_no_prod_env.csv
@@ -0,0 +1,10 @@
+connection_id,job_build_key,environment,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
+3,TEST1-TEST1-JOB1-22,PRODUCTION,"{""connectionId"":3,""ProjectKey"":""TEST1""}",_raw_bamboo_api_job_build,199,
+3,TEST1-TEST1-JOB1-23,PRODUCTION,"{""connectionId"":3,""ProjectKey"":""TEST1""}",_raw_bamboo_api_job_build,198,
+3,TEST1-TEST2-JOB1-2,PRODUCTION,"{""connectionId"":3,""ProjectKey"":""TEST1""}",_raw_bamboo_api_job_build,123,
+3,TEST1-TEST2-JOB1-3,PRODUCTION,"{""connectionId"":3,""ProjectKey"":""TEST1""}",_raw_bamboo_api_job_build,122,
+3,TEST1-TEST3-JOB1-2,PRODUCTION,"{""connectionId"":3,""ProjectKey"":""TEST1""}",_raw_bamboo_api_job_build,154,
+3,TEST1-TEST3-JOB1-3,PRODUCTION,"{""connectionId"":3,""ProjectKey"":""TEST1""}",_raw_bamboo_api_job_build,153,
+3,TEST1-TEST4-JOB1-1,PRODUCTION,"{""connectionId"":3,""ProjectKey"":""TEST1""}",_raw_bamboo_api_job_build,139,
+3,TEST1-TEST4-JOB1-2,PRODUCTION,"{""connectionId"":3,""ProjectKey"":""TEST1""}",_raw_bamboo_api_job_build,138,
+3,TEST1-TEST4-JOB1-3,PRODUCTION,"{""connectionId"":3,""ProjectKey"":""TEST1""}",_raw_bamboo_api_job_build,137,
diff --git a/backend/plugins/bamboo/tasks/job_build_extractor.go b/backend/plugins/bamboo/tasks/job_build_extractor.go
index 893fc46f7..6d6608297 100644
--- a/backend/plugins/bamboo/tasks/job_build_extractor.go
+++ b/backend/plugins/bamboo/tasks/job_build_extractor.go
@@ -55,7 +55,7 @@ func ExtractJobBuild(taskCtx plugin.SubTaskContext) errors.Error {
body.PlanName = plan.PlanName
body.PlanBuildKey = fmt.Sprintf("%s-%v", plan.PlanKey, body.Number)
body.Type = data.RegexEnricher.ReturnNameIfMatched(devops.DEPLOYMENT, body.JobName)
- body.Environment = data.RegexEnricher.ReturnNameIfMatched(devops.PRODUCTION, body.JobName)
+ body.Environment = data.RegexEnricher.ReturnNameIfOmittedOrMatched(devops.PRODUCTION, body.JobName)
results := make([]interface{}, 0)
results = append(results, body)
for _, v := range res.VcsRevisions.VcsRevision {
diff --git a/backend/plugins/bitbucket/e2e/pipeline_steps_test.go b/backend/plugins/bitbucket/e2e/pipeline_steps_test.go
index 96703f780..0f9bdf73b 100644
--- a/backend/plugins/bitbucket/e2e/pipeline_steps_test.go
+++ b/backend/plugins/bitbucket/e2e/pipeline_steps_test.go
@@ -34,8 +34,8 @@ func TestBitbucketPipelineStepsDataFlow(t *testing.T) {
dataflowTester := e2ehelper.NewDataFlowTester(t, "bitbucket", bitbucket)
regexEnricher := helper.NewRegexEnricher()
- _ = regexEnricher.TryAdd(devops.DEPLOYMENT, "main")
- _ = regexEnricher.TryAdd(devops.PRODUCTION, "pipeline")
+ _ = regexEnricher.TryAdd(devops.DEPLOYMENT, "staging")
+ // _ = regexEnricher.TryAdd(devops.PRODUCTION, "pipeline") // when production regex is omitted, all steps will be treated as production
taskData := &tasks.BitbucketTaskData{
Options: &tasks.BitbucketOptions{
ConnectionId: 1,
@@ -67,6 +67,8 @@ func TestBitbucketPipelineStepsDataFlow(t *testing.T) {
"run_number",
//"trigger",
//"result",
+ "type",
+ "environment",
),
)
diff --git a/backend/plugins/bitbucket/e2e/snapshot_tables/_tool_bitbucket_pipeline_steps.csv b/backend/plugins/bitbucket/e2e/snapshot_tables/_tool_bitbucket_pipeline_steps.csv
index d8f157a66..b5aa2f3e3 100644
--- a/backend/plugins/bitbucket/e2e/snapshot_tables/_tool_bitbucket_pipeline_steps.csv
+++ b/backend/plugins/bitbucket/e2e/snapshot_tables/_tool_bitbucket_pipeline_steps.csv
@@ -1,26 +1,26 @@
-connection_id,bitbucket_id,pipeline_id,repo_id,name,state,max_time,started_on,completed_on,duration_in_seconds,build_seconds_used,run_number,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
-1,{009577b4-99a8-4498-99bf-a00ef7254ca3},{5decbbb2-f4e6-4f3a-aab0-f74c40497c51},likyh/likyhphp,staging,COMPLETED,120,2023-02-20T09:38:47.203+00:00,2023-02-20T09:38:52.307+00:00,5,5,1,"{""ConnectionId"":1,""FullName"":""likyh/likyhphp""}",_raw_bitbucket_api_pipeline_steps,285,
-1,{0706054d-bde3-4d27-af26-8ff9bc675e6c},{5decbbb2-f4e6-4f3a-aab0-f74c40497c51},likyh/likyhphp,production,COMPLETED,120,2023-02-20T09:39:04.211+00:00,2023-02-20T09:39:09.267+00:00,5,5,1,"{""ConnectionId"":1,""FullName"":""likyh/likyhphp""}",_raw_bitbucket_api_pipeline_steps,286,
-1,{089a4513-1ea5-4890-8c08-f647ad5ff836},{8b417a17-72e6-4e16-ae4a-97f6e279f68e},likyh/likyhphp,staging,PENDING,120,,,0,0,1,"{""ConnectionId"":1,""FullName"":""likyh/likyhphp""}",_raw_bitbucket_api_pipeline_steps,405,
-1,{0fa0137f-e029-459e-8760-289ed9e6b57d},{5decbbb2-f4e6-4f3a-aab0-f74c40497c51},likyh/likyhphp,Test,COMPLETED,120,2023-02-20T09:38:30.581+00:00,2023-02-20T09:38:35.012+00:00,4,4,1,"{""ConnectionId"":1,""FullName"":""likyh/likyhphp""}",_raw_bitbucket_api_pipeline_steps,284,
-1,{1218e6e3-f34e-4ebc-a767-2a69285681ca},{2d91d69f-d84b-423d-ae4d-c1d3f2c27166},likyh/likyhphp,staging,COMPLETED,120,2023-02-20T09:23:50.415+00:00,2023-02-20T09:23:55.180+00:00,5,4,1,"{""ConnectionId"":1,""FullName"":""likyh/likyhphp""}",_raw_bitbucket_api_pipeline_steps,144,
-1,{148bfd03-8b7f-4483-982c-18b020124350},{b5d2ab02-2013-415e-b146-61ff3bf19502},likyh/likyhphp,production,COMPLETED,120,2023-02-20T09:23:40.970+00:00,2023-02-20T09:23:40.970+00:00,0,0,1,"{""ConnectionId"":1,""FullName"":""likyh/likyhphp""}",_raw_bitbucket_api_pipeline_steps,533,
-1,{14c24a3e-ffff-4d15-956d-83bba2ea8191},{5fb099c8-ebdd-4d3e-97d8-a12ebfca5446},likyh/likyhphp,staging,PENDING,120,,,0,0,1,"{""ConnectionId"":1,""FullName"":""likyh/likyhphp""}",_raw_bitbucket_api_pipeline_steps,291,
-1,{263500ac-4242-4996-95bd-739b5cafd3ef},{1e20072e-b22a-443f-ad92-3d9435393d5b},likyh/likyhphp,Test,COMPLETED,120,2023-02-20T09:12:34.572+00:00,2023-02-20T09:12:39.614+00:00,5,5,1,"{""ConnectionId"":1,""FullName"":""likyh/likyhphp""}",_raw_bitbucket_api_pipeline_steps,91,
-1,{2915edc8-9a5a-4a76-831b-f7ce30938d4e},{b5d2ab02-2013-415e-b146-61ff3bf19502},likyh/likyhphp,staging,COMPLETED,120,2023-02-20T09:23:40.928+00:00,2023-02-20T09:23:40.928+00:00,0,0,1,"{""ConnectionId"":1,""FullName"":""likyh/likyhphp""}",_raw_bitbucket_api_pipeline_steps,532,
-1,{61e34fbf-87a2-4271-827e-b336ff4c4396},{d7bf8394-6e18-42c4-b3a3-70514fc4079f},likyh/likyhphp,staging,COMPLETED,120,2023-02-20T09:15:45.450+00:00,2023-02-20T09:15:50.609+00:00,5,5,1,"{""ConnectionId"":1,""FullName"":""likyh/likyhphp""}",_raw_bitbucket_api_pipeline_steps,632,
-1,{6d73cef4-ade5-413a-b3ff-92472198790f},{5fb099c8-ebdd-4d3e-97d8-a12ebfca5446},likyh/likyhphp,Test,PENDING,120,,,0,0,1,"{""ConnectionId"":1,""FullName"":""likyh/likyhphp""}",_raw_bitbucket_api_pipeline_steps,290,
-1,{74ceee44-533a-489b-8d9b-3c9c0ef57ff3},{2d91d69f-d84b-423d-ae4d-c1d3f2c27166},likyh/likyhphp,production,COMPLETED,120,2023-02-20T09:24:05.997+00:00,2023-02-20T09:24:12.269+00:00,6,6,1,"{""ConnectionId"":1,""FullName"":""likyh/likyhphp""}",_raw_bitbucket_api_pipeline_steps,145,
-1,{9b079475-6fed-40bb-b591-711351c12f71},{2d91d69f-d84b-423d-ae4d-c1d3f2c27166},likyh/likyhphp,Test,COMPLETED,120,2023-02-20T09:23:31.978+00:00,2023-02-20T09:23:36.923+00:00,5,4,1,"{""ConnectionId"":1,""FullName"":""likyh/likyhphp""}",_raw_bitbucket_api_pipeline_steps,143,
-1,{a72dea81-cd04-45db-8af9-958ed814d050},{1e20072e-b22a-443f-ad92-3d9435393d5b},likyh/likyhphp,Test,COMPLETED,120,2023-02-20T09:12:04.411+00:00,2023-02-20T09:12:10.347+00:00,6,5,1,"{""ConnectionId"":1,""FullName"":""likyh/likyhphp""}",_raw_bitbucket_api_pipeline_steps,90,
-1,{ac0a2d0b-d67d-47c6-8e23-ee33ccdfe2d8},{b5d2ab02-2013-415e-b146-61ff3bf19502},likyh/likyhphp,Test,COMPLETED,120,2023-02-20T09:23:40.750+00:00,2023-02-20T09:23:40.750+00:00,0,0,1,"{""ConnectionId"":1,""FullName"":""likyh/likyhphp""}",_raw_bitbucket_api_pipeline_steps,531,
-1,{c2da4b2d-f971-4c5f-90b7-1c55ea80121c},{d7bf8394-6e18-42c4-b3a3-70514fc4079f},likyh/likyhphp,production,COMPLETED,120,2023-02-20T09:16:01.886+00:00,2023-02-20T09:16:09.885+00:00,8,7,1,"{""ConnectionId"":1,""FullName"":""likyh/likyhphp""}",_raw_bitbucket_api_pipeline_steps,633,
-1,{c596152f-6c3e-45db-a841-f8d312366cb1},{02c4dfe8-70a6-442c-8bd8-19061979c2cf},likyh/likyhphp,Test,PENDING,120,,,0,0,1,"{""ConnectionId"":1,""FullName"":""likyh/likyhphp""}",_raw_bitbucket_api_pipeline_steps,11,
-1,{caf86c6c-c60a-4e62-b838-6480ad110182},{1e20072e-b22a-443f-ad92-3d9435393d5b},likyh/likyhphp,Test,COMPLETED,120,2023-02-20T09:11:43.393+00:00,2023-02-20T09:11:53.145+00:00,10,9,1,"{""ConnectionId"":1,""FullName"":""likyh/likyhphp""}",_raw_bitbucket_api_pipeline_steps,89,
-1,{d464ab91-8b50-4ad5-8c22-d2a6d1ca5d22},{01fad871-cd21-4a80-bd59-74148dd5be8e},likyh/likyhphp,Test,COMPLETED,120,2023-02-20T08:58:31.520+00:00,2023-02-20T08:58:45.945+00:00,14,14,1,"{""ConnectionId"":1,""FullName"":""likyh/likyhphp""}",_raw_bitbucket_api_pipeline_steps,1,
-1,{d986f8fb-c8a3-4d2b-95cd-75f294c06262},{02c4dfe8-70a6-442c-8bd8-19061979c2cf},likyh/likyhphp,staging,PENDING,120,,,0,0,1,"{""ConnectionId"":1,""FullName"":""likyh/likyhphp""}",_raw_bitbucket_api_pipeline_steps,12,
-1,{dae66dc7-4d3e-43c9-ae04-8a8bcc1747c6},{02c4dfe8-70a6-442c-8bd8-19061979c2cf},likyh/likyhphp,production,PENDING,120,,,0,0,1,"{""ConnectionId"":1,""FullName"":""likyh/likyhphp""}",_raw_bitbucket_api_pipeline_steps,13,
-1,{e4753fd7-469e-4d15-a689-0fd882fee2d4},{d7bf8394-6e18-42c4-b3a3-70514fc4079f},likyh/likyhphp,Test,COMPLETED,120,2023-02-20T09:17:33.116+00:00,2023-02-20T09:17:37.835+00:00,5,4,2,"{""ConnectionId"":1,""FullName"":""likyh/likyhphp""}",_raw_bitbucket_api_pipeline_steps,631,
-1,{e74ae72c-1c09-4cb1-8c19-16e070204f07},{8b417a17-72e6-4e16-ae4a-97f6e279f68e},likyh/likyhphp,Test,PENDING,120,,,0,0,1,"{""ConnectionId"":1,""FullName"":""likyh/likyhphp""}",_raw_bitbucket_api_pipeline_steps,404,
-1,{ea7acd3f-d51c-41a1-9f53-458178d227f0},{5fb099c8-ebdd-4d3e-97d8-a12ebfca5446},likyh/likyhphp,production,PENDING,120,,,0,0,1,"{""ConnectionId"":1,""FullName"":""likyh/likyhphp""}",_raw_bitbucket_api_pipeline_steps,292,
-1,{eafe4c3f-6cfc-42c8-81c2-575dc6dee0f7},{8b417a17-72e6-4e16-ae4a-97f6e279f68e},likyh/likyhphp,production,PENDING,120,,,0,0,1,"{""ConnectionId"":1,""FullName"":""likyh/likyhphp""}",_raw_bitbucket_api_pipeline_steps,406,
+connection_id,bitbucket_id,pipeline_id,repo_id,name,state,max_time,started_on,completed_on,duration_in_seconds,build_seconds_used,run_number,type,environment,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
+1,{009577b4-99a8-4498-99bf-a00ef7254ca3},{5decbbb2-f4e6-4f3a-aab0-f74c40497c51},likyh/likyhphp,staging,COMPLETED,120,2023-02-20T09:38:47.203+00:00,2023-02-20T09:38:52.307+00:00,5,5,1,DEPLOYMENT,PRODUCTION,"{""ConnectionId"":1,""FullName"":""likyh/likyhphp""}",_raw_bitbucket_api_pipeline_steps,285,
+1,{0706054d-bde3-4d27-af26-8ff9bc675e6c},{5decbbb2-f4e6-4f3a-aab0-f74c40497c51},likyh/likyhphp,production,COMPLETED,120,2023-02-20T09:39:04.211+00:00,2023-02-20T09:39:09.267+00:00,5,5,1,,PRODUCTION,"{""ConnectionId"":1,""FullName"":""likyh/likyhphp""}",_raw_bitbucket_api_pipeline_steps,286,
+1,{089a4513-1ea5-4890-8c08-f647ad5ff836},{8b417a17-72e6-4e16-ae4a-97f6e279f68e},likyh/likyhphp,staging,PENDING,120,,,0,0,1,DEPLOYMENT,PRODUCTION,"{""ConnectionId"":1,""FullName"":""likyh/likyhphp""}",_raw_bitbucket_api_pipeline_steps,405,
+1,{0fa0137f-e029-459e-8760-289ed9e6b57d},{5decbbb2-f4e6-4f3a-aab0-f74c40497c51},likyh/likyhphp,Test,COMPLETED,120,2023-02-20T09:38:30.581+00:00,2023-02-20T09:38:35.012+00:00,4,4,1,,PRODUCTION,"{""ConnectionId"":1,""FullName"":""likyh/likyhphp""}",_raw_bitbucket_api_pipeline_steps,284,
+1,{1218e6e3-f34e-4ebc-a767-2a69285681ca},{2d91d69f-d84b-423d-ae4d-c1d3f2c27166},likyh/likyhphp,staging,COMPLETED,120,2023-02-20T09:23:50.415+00:00,2023-02-20T09:23:55.180+00:00,5,4,1,DEPLOYMENT,PRODUCTION,"{""ConnectionId"":1,""FullName"":""likyh/likyhphp""}",_raw_bitbucket_api_pipeline_steps,144,
+1,{148bfd03-8b7f-4483-982c-18b020124350},{b5d2ab02-2013-415e-b146-61ff3bf19502},likyh/likyhphp,production,COMPLETED,120,2023-02-20T09:23:40.970+00:00,2023-02-20T09:23:40.970+00:00,0,0,1,,PRODUCTION,"{""ConnectionId"":1,""FullName"":""likyh/likyhphp""}",_raw_bitbucket_api_pipeline_steps,533,
+1,{14c24a3e-ffff-4d15-956d-83bba2ea8191},{5fb099c8-ebdd-4d3e-97d8-a12ebfca5446},likyh/likyhphp,staging,PENDING,120,,,0,0,1,DEPLOYMENT,PRODUCTION,"{""ConnectionId"":1,""FullName"":""likyh/likyhphp""}",_raw_bitbucket_api_pipeline_steps,291,
+1,{263500ac-4242-4996-95bd-739b5cafd3ef},{1e20072e-b22a-443f-ad92-3d9435393d5b},likyh/likyhphp,Test,COMPLETED,120,2023-02-20T09:12:34.572+00:00,2023-02-20T09:12:39.614+00:00,5,5,1,,PRODUCTION,"{""ConnectionId"":1,""FullName"":""likyh/likyhphp""}",_raw_bitbucket_api_pipeline_steps,91,
+1,{2915edc8-9a5a-4a76-831b-f7ce30938d4e},{b5d2ab02-2013-415e-b146-61ff3bf19502},likyh/likyhphp,staging,COMPLETED,120,2023-02-20T09:23:40.928+00:00,2023-02-20T09:23:40.928+00:00,0,0,1,DEPLOYMENT,PRODUCTION,"{""ConnectionId"":1,""FullName"":""likyh/likyhphp""}",_raw_bitbucket_api_pipeline_steps,532,
+1,{61e34fbf-87a2-4271-827e-b336ff4c4396},{d7bf8394-6e18-42c4-b3a3-70514fc4079f},likyh/likyhphp,staging,COMPLETED,120,2023-02-20T09:15:45.450+00:00,2023-02-20T09:15:50.609+00:00,5,5,1,DEPLOYMENT,PRODUCTION,"{""ConnectionId"":1,""FullName"":""likyh/likyhphp""}",_raw_bitbucket_api_pipeline_steps,632,
+1,{6d73cef4-ade5-413a-b3ff-92472198790f},{5fb099c8-ebdd-4d3e-97d8-a12ebfca5446},likyh/likyhphp,Test,PENDING,120,,,0,0,1,,PRODUCTION,"{""ConnectionId"":1,""FullName"":""likyh/likyhphp""}",_raw_bitbucket_api_pipeline_steps,290,
+1,{74ceee44-533a-489b-8d9b-3c9c0ef57ff3},{2d91d69f-d84b-423d-ae4d-c1d3f2c27166},likyh/likyhphp,production,COMPLETED,120,2023-02-20T09:24:05.997+00:00,2023-02-20T09:24:12.269+00:00,6,6,1,,PRODUCTION,"{""ConnectionId"":1,""FullName"":""likyh/likyhphp""}",_raw_bitbucket_api_pipeline_steps,145,
+1,{9b079475-6fed-40bb-b591-711351c12f71},{2d91d69f-d84b-423d-ae4d-c1d3f2c27166},likyh/likyhphp,Test,COMPLETED,120,2023-02-20T09:23:31.978+00:00,2023-02-20T09:23:36.923+00:00,5,4,1,,PRODUCTION,"{""ConnectionId"":1,""FullName"":""likyh/likyhphp""}",_raw_bitbucket_api_pipeline_steps,143,
+1,{a72dea81-cd04-45db-8af9-958ed814d050},{1e20072e-b22a-443f-ad92-3d9435393d5b},likyh/likyhphp,Test,COMPLETED,120,2023-02-20T09:12:04.411+00:00,2023-02-20T09:12:10.347+00:00,6,5,1,,PRODUCTION,"{""ConnectionId"":1,""FullName"":""likyh/likyhphp""}",_raw_bitbucket_api_pipeline_steps,90,
+1,{ac0a2d0b-d67d-47c6-8e23-ee33ccdfe2d8},{b5d2ab02-2013-415e-b146-61ff3bf19502},likyh/likyhphp,Test,COMPLETED,120,2023-02-20T09:23:40.750+00:00,2023-02-20T09:23:40.750+00:00,0,0,1,,PRODUCTION,"{""ConnectionId"":1,""FullName"":""likyh/likyhphp""}",_raw_bitbucket_api_pipeline_steps,531,
+1,{c2da4b2d-f971-4c5f-90b7-1c55ea80121c},{d7bf8394-6e18-42c4-b3a3-70514fc4079f},likyh/likyhphp,production,COMPLETED,120,2023-02-20T09:16:01.886+00:00,2023-02-20T09:16:09.885+00:00,8,7,1,,PRODUCTION,"{""ConnectionId"":1,""FullName"":""likyh/likyhphp""}",_raw_bitbucket_api_pipeline_steps,633,
+1,{c596152f-6c3e-45db-a841-f8d312366cb1},{02c4dfe8-70a6-442c-8bd8-19061979c2cf},likyh/likyhphp,Test,PENDING,120,,,0,0,1,,PRODUCTION,"{""ConnectionId"":1,""FullName"":""likyh/likyhphp""}",_raw_bitbucket_api_pipeline_steps,11,
+1,{caf86c6c-c60a-4e62-b838-6480ad110182},{1e20072e-b22a-443f-ad92-3d9435393d5b},likyh/likyhphp,Test,COMPLETED,120,2023-02-20T09:11:43.393+00:00,2023-02-20T09:11:53.145+00:00,10,9,1,,PRODUCTION,"{""ConnectionId"":1,""FullName"":""likyh/likyhphp""}",_raw_bitbucket_api_pipeline_steps,89,
+1,{d464ab91-8b50-4ad5-8c22-d2a6d1ca5d22},{01fad871-cd21-4a80-bd59-74148dd5be8e},likyh/likyhphp,Test,COMPLETED,120,2023-02-20T08:58:31.520+00:00,2023-02-20T08:58:45.945+00:00,14,14,1,,PRODUCTION,"{""ConnectionId"":1,""FullName"":""likyh/likyhphp""}",_raw_bitbucket_api_pipeline_steps,1,
+1,{d986f8fb-c8a3-4d2b-95cd-75f294c06262},{02c4dfe8-70a6-442c-8bd8-19061979c2cf},likyh/likyhphp,staging,PENDING,120,,,0,0,1,DEPLOYMENT,PRODUCTION,"{""ConnectionId"":1,""FullName"":""likyh/likyhphp""}",_raw_bitbucket_api_pipeline_steps,12,
+1,{dae66dc7-4d3e-43c9-ae04-8a8bcc1747c6},{02c4dfe8-70a6-442c-8bd8-19061979c2cf},likyh/likyhphp,production,PENDING,120,,,0,0,1,,PRODUCTION,"{""ConnectionId"":1,""FullName"":""likyh/likyhphp""}",_raw_bitbucket_api_pipeline_steps,13,
+1,{e4753fd7-469e-4d15-a689-0fd882fee2d4},{d7bf8394-6e18-42c4-b3a3-70514fc4079f},likyh/likyhphp,Test,COMPLETED,120,2023-02-20T09:17:33.116+00:00,2023-02-20T09:17:37.835+00:00,5,4,2,,PRODUCTION,"{""ConnectionId"":1,""FullName"":""likyh/likyhphp""}",_raw_bitbucket_api_pipeline_steps,631,
+1,{e74ae72c-1c09-4cb1-8c19-16e070204f07},{8b417a17-72e6-4e16-ae4a-97f6e279f68e},likyh/likyhphp,Test,PENDING,120,,,0,0,1,,PRODUCTION,"{""ConnectionId"":1,""FullName"":""likyh/likyhphp""}",_raw_bitbucket_api_pipeline_steps,404,
+1,{ea7acd3f-d51c-41a1-9f53-458178d227f0},{5fb099c8-ebdd-4d3e-97d8-a12ebfca5446},likyh/likyhphp,production,PENDING,120,,,0,0,1,,PRODUCTION,"{""ConnectionId"":1,""FullName"":""likyh/likyhphp""}",_raw_bitbucket_api_pipeline_steps,292,
+1,{eafe4c3f-6cfc-42c8-81c2-575dc6dee0f7},{8b417a17-72e6-4e16-ae4a-97f6e279f68e},likyh/likyhphp,production,PENDING,120,,,0,0,1,,PRODUCTION,"{""ConnectionId"":1,""FullName"":""likyh/likyhphp""}",_raw_bitbucket_api_pipeline_steps,406,
diff --git a/backend/plugins/bitbucket/tasks/pipeline_extractor.go b/backend/plugins/bitbucket/tasks/pipeline_extractor.go
index 1cd1eb8c3..2a72e3fed 100644
--- a/backend/plugins/bitbucket/tasks/pipeline_extractor.go
+++ b/backend/plugins/bitbucket/tasks/pipeline_extractor.go
@@ -101,7 +101,7 @@ func ExtractApiPipelines(taskCtx plugin.SubTaskContext) errors.Error {
BitbucketCreatedOn: api.Iso8601TimeToTime(bitbucketApiPipeline.CreatedOn),
BitbucketCompleteOn: api.Iso8601TimeToTime(bitbucketApiPipeline.CompletedOn),
Type: data.RegexEnricher.ReturnNameIfMatched(devops.DEPLOYMENT, bitbucketApiPipeline.Target.RefName),
- Environment: data.RegexEnricher.ReturnNameIfMatched(devops.PRODUCTION, bitbucketApiPipeline.Target.RefName),
+ Environment: data.RegexEnricher.ReturnNameIfOmittedOrMatched(devops.PRODUCTION, bitbucketApiPipeline.Target.RefName),
}
if err != nil {
return nil, err
diff --git a/backend/plugins/bitbucket/tasks/pipeline_steps_extractor.go b/backend/plugins/bitbucket/tasks/pipeline_steps_extractor.go
index 6ea4234eb..5ad13535f 100644
--- a/backend/plugins/bitbucket/tasks/pipeline_steps_extractor.go
+++ b/backend/plugins/bitbucket/tasks/pipeline_steps_extractor.go
@@ -94,7 +94,7 @@ func ExtractPipelineSteps(taskCtx plugin.SubTaskContext) errors.Error {
BuildSecondsUsed: apiPipelineStep.BuildSecondsUsed,
RunNumber: apiPipelineStep.RunNumber,
Type: data.RegexEnricher.ReturnNameIfMatched(devops.DEPLOYMENT, apiPipelineStep.Name),
- Environment: data.RegexEnricher.ReturnNameIfMatched(devops.PRODUCTION, apiPipelineStep.Name),
+ Environment: data.RegexEnricher.ReturnNameIfOmittedOrMatched(devops.PRODUCTION, apiPipelineStep.Name),
}
return []interface{}{
bitbucketStep,
diff --git a/backend/plugins/github/e2e/cicd_job_test.go b/backend/plugins/github/e2e/cicd_job_test.go
index 86ef28c0c..9e828961b 100644
--- a/backend/plugins/github/e2e/cicd_job_test.go
+++ b/backend/plugins/github/e2e/cicd_job_test.go
@@ -18,9 +18,10 @@ limitations under the License.
package e2e
import (
+ "testing"
+
"github.com/apache/incubator-devlake/core/models/common"
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
- "testing"
"github.com/apache/incubator-devlake/core/models/domainlayer/devops"
"github.com/apache/incubator-devlake/helpers/e2ehelper"
@@ -34,7 +35,6 @@ func TestGithubCICDJobDataFlow(t *testing.T) {
dataflowTester := e2ehelper.NewDataFlowTester(t, "github", github)
regexEnricher := helper.NewRegexEnricher()
_ = regexEnricher.TryAdd(devops.DEPLOYMENT, "deploywindows.*")
- _ = regexEnricher.TryAdd(devops.PRODUCTION, "deploywindows.*")
taskData := &tasks.GithubTaskData{
Options: &tasks.GithubOptions{
ConnectionId: 1,
@@ -48,16 +48,24 @@ func TestGithubCICDJobDataFlow(t *testing.T) {
// SELECT * FROM _raw_github_api_jobs INTO OUTFILE "/tmp/_raw_github_api_jobs.csv" FIELDS TERMINATED BY ',' OPTIONALLY ENCLOSED BY '"' LINES TERMINATED BY '\r\n';
dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_github_api_jobs.csv", "_raw_github_api_jobs")
- // verify extraction
+ // verify when production regex is omitted
dataflowTester.FlushTabler(&models.GithubJob{})
- dataflowTester.FlushTabler(&devops.CICDTask{})
+ dataflowTester.Subtask(tasks.ExtractJobsMeta, taskData)
+ dataflowTester.VerifyTableWithOptions(&models.GithubJob{}, e2ehelper.TableOptions{
+ CSVRelPath: "./snapshot_tables/_tool_github_jobs_no_prod_env.csv",
+ IgnoreTypes: []interface{}{common.NoPKModel{}},
+ })
+ _ = regexEnricher.TryAdd(devops.PRODUCTION, "deploywindows.*")
+ // verify extraction
+ dataflowTester.FlushTabler(&models.GithubJob{})
dataflowTester.Subtask(tasks.ExtractJobsMeta, taskData)
dataflowTester.VerifyTableWithOptions(&models.GithubJob{}, e2ehelper.TableOptions{
CSVRelPath: "./snapshot_tables/_tool_github_jobs.csv",
IgnoreTypes: []interface{}{common.NoPKModel{}},
})
+ dataflowTester.FlushTabler(&devops.CICDTask{})
dataflowTester.Subtask(tasks.ConvertJobsMeta, taskData)
dataflowTester.VerifyTableWithOptions(&devops.CICDTask{}, e2ehelper.TableOptions{
CSVRelPath: "./snapshot_tables/cicd_tasks.csv",
diff --git a/backend/plugins/github/e2e/cicd_run_test.go b/backend/plugins/github/e2e/cicd_run_test.go
index e69a692d7..0d7d01df6 100644
--- a/backend/plugins/github/e2e/cicd_run_test.go
+++ b/backend/plugins/github/e2e/cicd_run_test.go
@@ -18,9 +18,10 @@ limitations under the License.
package e2e
import (
+ "testing"
+
"github.com/apache/incubator-devlake/core/models/common"
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
- "testing"
"github.com/apache/incubator-devlake/core/models/domainlayer/devops"
"github.com/apache/incubator-devlake/helpers/e2ehelper"
@@ -34,7 +35,6 @@ func TestGithubCICDRunDataFlow(t *testing.T) {
dataflowTester := e2ehelper.NewDataFlowTester(t, "github", github)
regexEnricher := helper.NewRegexEnricher()
_ = regexEnricher.TryAdd(devops.DEPLOYMENT, "CodeQL.*")
- _ = regexEnricher.TryAdd(devops.PRODUCTION, "CodeQL.*")
taskData := &tasks.GithubTaskData{
Options: &tasks.GithubOptions{
ConnectionId: 1,
@@ -49,17 +49,25 @@ func TestGithubCICDRunDataFlow(t *testing.T) {
dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_github_api_runs.csv", "_raw_github_api_runs")
dataflowTester.ImportCsvIntoTabler("./raw_tables/_tool_github_repos.csv", &models.GithubRepo{})
- // verify extraction
+ // verify when production regex is omitted
dataflowTester.FlushTabler(&models.GithubRun{})
- dataflowTester.FlushTabler(&devops.CICDPipeline{})
- dataflowTester.FlushTabler(&devops.CiCDPipelineCommit{})
+ dataflowTester.Subtask(tasks.ExtractRunsMeta, taskData)
+ dataflowTester.VerifyTableWithOptions(&models.GithubRun{}, e2ehelper.TableOptions{
+ CSVRelPath: "./snapshot_tables/_tool_github_runs_no_prod_regex.csv",
+ IgnoreTypes: []interface{}{common.NoPKModel{}},
+ })
+ // verify extraction
+ dataflowTester.FlushTabler(&models.GithubRun{})
+ _ = regexEnricher.TryAdd(devops.PRODUCTION, "CodeQL.*")
dataflowTester.Subtask(tasks.ExtractRunsMeta, taskData)
dataflowTester.VerifyTableWithOptions(&models.GithubRun{}, e2ehelper.TableOptions{
CSVRelPath: "./snapshot_tables/_tool_github_runs.csv",
IgnoreTypes: []interface{}{common.NoPKModel{}},
})
+ dataflowTester.FlushTabler(&devops.CICDPipeline{})
+ dataflowTester.FlushTabler(&devops.CiCDPipelineCommit{})
dataflowTester.Subtask(tasks.ConvertRunsMeta, taskData)
dataflowTester.VerifyTableWithOptions(&devops.CICDPipeline{}, e2ehelper.TableOptions{
CSVRelPath: "./snapshot_tables/cicd_pipelines.csv",
diff --git a/backend/plugins/github/e2e/snapshot_tables/_tool_github_jobs_no_prod_env.csv b/backend/plugins/github/e2e/snapshot_tables/_tool_github_jobs_no_prod_env.csv
new file mode 100644
index 000000000..43fe67345
--- /dev/null
+++ b/backend/plugins/github/e2e/snapshot_tables/_tool_github_jobs_no_prod_env.csv
@@ -0,0 +1,21 @@
+connection_id,repo_id,id,run_id,run_url,node_id,head_sha,url,html_url,status,conclusion,started_at,completed_at,name,steps,check_run_url,labels,runner_id,runner_name,runner_group_id,type,environment
+1,134018330,1924918168,577324558,https://api.github.com/repos/panjf2000/ants/actions/runs/577324558,MDg6Q2hlY2tSdW4xOTI0OTE4MTY4,cb4adab28f63313592a9a395656b8413184ea336,https://api.github.com/repos/panjf2000/ants/actions/jobs/1924918168,https://github.com/panjf2000/ants/runs/1924918168?check_suite_focus=true,COMPLETED,SUCCESS,2021-02-18T06:59:13.000+00:00,2021-02-18T06:59:33.000+00:00,Golangci-Lint,"[{""name"": ""Set up job"", ""number"": 1, ""status"": ""completed"", ""conclusion"": "" [...]
+1,134018330,1924918171,577324554,https://api.github.com/repos/panjf2000/ants/actions/runs/577324554,MDg6Q2hlY2tSdW4xOTI0OTE4MTcx,cb4adab28f63313592a9a395656b8413184ea336,https://api.github.com/repos/panjf2000/ants/actions/jobs/1924918171,https://github.com/panjf2000/ants/runs/1924918171?check_suite_focus=true,COMPLETED,CANCELLED,2021-02-18T06:59:13.000+00:00,2021-02-18T07:01:18.000+00:00,deployubuntu,"[{""name"": ""Set up job"", ""number"": 1, ""status"": ""completed"", ""conclusion"": " [...]
+1,134018330,1924918191,577324554,https://api.github.com/repos/panjf2000/ants/actions/runs/577324554,MDg6Q2hlY2tSdW4xOTI0OTE4MTkx,cb4adab28f63313592a9a395656b8413184ea336,https://api.github.com/repos/panjf2000/ants/actions/jobs/1924918191,https://github.com/panjf2000/ants/runs/1924918191?check_suite_focus=true,COMPLETED,CANCELLED,2021-02-18T06:59:21.000+00:00,2021-02-18T07:01:18.000+00:00,deploymacos,"[{""name"": ""Set up job"", ""number"": 1, ""status"": ""completed"", ""conclusion"": "" [...]
+1,134018330,1924918205,577324554,https://api.github.com/repos/panjf2000/ants/actions/runs/577324554,MDg6Q2hlY2tSdW4xOTI0OTE4MjA1,cb4adab28f63313592a9a395656b8413184ea336,https://api.github.com/repos/panjf2000/ants/actions/jobs/1924918205,https://github.com/panjf2000/ants/runs/1924918205?check_suite_focus=true,COMPLETED,CANCELLED,2021-02-18T06:59:15.000+00:00,2021-02-18T07:01:09.000+00:00,deploywindows,"[{""name"": ""Set up job"", ""number"": 1, ""status"": ""completed"", ""conclusion"": [...]
+1,134018330,1924918228,577324554,https://api.github.com/repos/panjf2000/ants/actions/runs/577324554,MDg6Q2hlY2tSdW4xOTI0OTE4MjI4,cb4adab28f63313592a9a395656b8413184ea336,https://api.github.com/repos/panjf2000/ants/actions/jobs/1924918228,https://github.com/panjf2000/ants/runs/1924918228?check_suite_focus=true,COMPLETED,CANCELLED,2021-02-18T06:59:13.000+00:00,2021-02-18T07:01:18.000+00:00,deployubuntu,"[{""name"": ""Set up job"", ""number"": 1, ""status"": ""completed"", ""conclusion"": " [...]
+1,134018330,1924918243,577324554,https://api.github.com/repos/panjf2000/ants/actions/runs/577324554,MDg6Q2hlY2tSdW4xOTI0OTE4MjQz,cb4adab28f63313592a9a395656b8413184ea336,https://api.github.com/repos/panjf2000/ants/actions/jobs/1924918243,https://github.com/panjf2000/ants/runs/1924918243?check_suite_focus=true,COMPLETED,CANCELLED,2021-02-18T06:59:19.000+00:00,2021-02-18T07:01:18.000+00:00,deploymacos,"[{""name"": ""Set up job"", ""number"": 1, ""status"": ""completed"", ""conclusion"": "" [...]
+1,134018330,1924918261,577324554,https://api.github.com/repos/panjf2000/ants/actions/runs/577324554,MDg6Q2hlY2tSdW4xOTI0OTE4MjYx,cb4adab28f63313592a9a395656b8413184ea336,https://api.github.com/repos/panjf2000/ants/actions/jobs/1924918261,https://github.com/panjf2000/ants/runs/1924918261?check_suite_focus=true,COMPLETED,CANCELLED,2021-02-18T06:59:15.000+00:00,2021-02-18T07:01:09.000+00:00,deploywindows,"[{""name"": ""Set up job"", ""number"": 1, ""status"": ""completed"", ""conclusion"": [...]
+1,134018330,1924918319,577324571,https://api.github.com/repos/panjf2000/ants/actions/runs/577324571,MDg6Q2hlY2tSdW4xOTI0OTE4MzE5,cb4adab28f63313592a9a395656b8413184ea336,https://api.github.com/repos/panjf2000/ants/actions/jobs/1924918319,https://github.com/panjf2000/ants/runs/1924918319?check_suite_focus=true,COMPLETED,SUCCESS,2021-02-18T06:59:16.000+00:00,2021-02-18T07:00:17.000+00:00,Analyze,"[{""name"": ""Set up job"", ""number"": 1, ""status"": ""completed"", ""conclusion"": ""succes [...]
+1,134018330,1924932184,577330055,https://api.github.com/repos/panjf2000/ants/actions/runs/577330055,MDg6Q2hlY2tSdW4xOTI0OTMyMTg0,fd8d670fd09489e6ea7693c0a382ba85d2694f16,https://api.github.com/repos/panjf2000/ants/actions/jobs/1924932184,https://github.com/panjf2000/ants/runs/1924932184?check_suite_focus=true,COMPLETED,SUCCESS,2021-02-18T07:02:02.000+00:00,2021-02-18T07:02:56.000+00:00,Analyze,"[{""name"": ""Set up job"", ""number"": 1, ""status"": ""completed"", ""conclusion"": ""succes [...]
+1,134018330,1924932219,577330056,https://api.github.com/repos/panjf2000/ants/actions/runs/577330056,MDg6Q2hlY2tSdW4xOTI0OTMyMjE5,fd8d670fd09489e6ea7693c0a382ba85d2694f16,https://api.github.com/repos/panjf2000/ants/actions/jobs/1924932219,https://github.com/panjf2000/ants/runs/1924932219?check_suite_focus=true,COMPLETED,SUCCESS,2021-02-18T07:02:03.000+00:00,2021-02-18T07:05:03.000+00:00,deployubuntu,"[{""name"": ""Set up job"", ""number"": 1, ""status"": ""completed"", ""conclusion"": ""s [...]
+1,134018330,1924932237,577330056,https://api.github.com/repos/panjf2000/ants/actions/runs/577330056,MDg6Q2hlY2tSdW4xOTI0OTMyMjM3,fd8d670fd09489e6ea7693c0a382ba85d2694f16,https://api.github.com/repos/panjf2000/ants/actions/jobs/1924932237,https://github.com/panjf2000/ants/runs/1924932237?check_suite_focus=true,IN_PROGRESS,,2021-02-18T07:02:06.000+00:00,2021-02-18T07:04:44.000+00:00,deploymacos,"[{""name"": ""Set up job"", ""number"": 1, ""status"": ""completed"", ""conclusion"": ""success [...]
+1,134018330,1924932251,577330056,https://api.github.com/repos/panjf2000/ants/actions/runs/577330056,MDg6Q2hlY2tSdW4xOTI0OTMyMjUx,fd8d670fd09489e6ea7693c0a382ba85d2694f16,https://api.github.com/repos/panjf2000/ants/actions/jobs/1924932251,https://github.com/panjf2000/ants/runs/1924932251?check_suite_focus=true,IN_PROGRESS,,2021-02-18T07:02:03.000+00:00,2021-02-18T07:05:57.000+00:00,deploywindows,"[{""name"": ""Set up job"", ""number"": 1, ""status"": ""completed"", ""conclusion"": ""succe [...]
+1,134018330,1924932263,577330057,https://api.github.com/repos/panjf2000/ants/actions/runs/577330057,MDg6Q2hlY2tSdW4xOTI0OTMyMjYz,fd8d670fd09489e6ea7693c0a382ba85d2694f16,https://api.github.com/repos/panjf2000/ants/actions/jobs/1924932263,https://github.com/panjf2000/ants/runs/1924932263?check_suite_focus=true,COMPLETED,FAILURE,2021-02-18T07:02:05.000+00:00,2021-02-18T07:02:19.000+00:00,Golangci-Lint,"[{""name"": ""Set up job"", ""number"": 1, ""status"": ""completed"", ""conclusion"": "" [...]
+1,134018330,1924932266,577330056,https://api.github.com/repos/panjf2000/ants/actions/runs/577330056,MDg6Q2hlY2tSdW4xOTI0OTMyMjY2,fd8d670fd09489e6ea7693c0a382ba85d2694f16,https://api.github.com/repos/panjf2000/ants/actions/jobs/1924932266,https://github.com/panjf2000/ants/runs/1924932266?check_suite_focus=true,COMPLETED,SUCCESS,2021-02-18T07:02:03.000+00:00,2021-02-18T07:04:44.000+00:00,deployubuntu,"[{""name"": ""Set up job"", ""number"": 1, ""status"": ""completed"", ""conclusion"": ""s [...]
+1,134018330,1924932293,577330056,https://api.github.com/repos/panjf2000/ants/actions/runs/577330056,MDg6Q2hlY2tSdW4xOTI0OTMyMjkz,fd8d670fd09489e6ea7693c0a382ba85d2694f16,https://api.github.com/repos/panjf2000/ants/actions/jobs/1924932293,https://github.com/panjf2000/ants/runs/1924932293?check_suite_focus=true,COMPLETED,SUCCESS,2021-02-18T07:02:06.000+00:00,2021-02-18T07:04:44.000+00:00,deploymacos,"[{""name"": ""Set up job"", ""number"": 1, ""status"": ""completed"", ""conclusion"": ""su [...]
+1,134018330,1924932319,577330056,https://api.github.com/repos/panjf2000/ants/actions/runs/577330056,MDg6Q2hlY2tSdW4xOTI0OTMyMzE5,fd8d670fd09489e6ea7693c0a382ba85d2694f16,https://api.github.com/repos/panjf2000/ants/actions/jobs/1924932319,https://github.com/panjf2000/ants/runs/1924932319?check_suite_focus=true,COMPLETED,SUCCESS,2021-02-18T07:02:03.000+00:00,2021-02-18T07:05:53.000+00:00,deploywindows,"[{""name"": ""Set up job"", ""number"": 1, ""status"": ""completed"", ""conclusion"": "" [...]
+1,134018330,1940449839,583528173,https://api.github.com/repos/panjf2000/ants/actions/runs/583528173,MDg6Q2hlY2tSdW4xOTQwNDQ5ODM5,fd8d670fd09489e6ea7693c0a382ba85d2694f16,https://api.github.com/repos/panjf2000/ants/actions/jobs/1940449839,https://github.com/panjf2000/ants/runs/1940449839?check_suite_focus=true,COMPLETED,SUCCESS,2021-02-20T05:10:17.000+00:00,2021-02-20T05:11:12.000+00:00,Analyze,"[{""name"": ""Set up job"", ""number"": 1, ""status"": ""completed"", ""conclusion"": ""succes [...]
+1,134018330,1992620044,604839350,https://api.github.com/repos/panjf2000/ants/actions/runs/604839350,MDg6Q2hlY2tSdW4xOTkyNjIwMDQ0,fd8d670fd09489e6ea7693c0a382ba85d2694f16,https://api.github.com/repos/panjf2000/ants/actions/jobs/1992620044,https://github.com/panjf2000/ants/runs/1992620044?check_suite_focus=true,COMPLETED,FAILURE,2021-02-27T05:10:19.000+00:00,2021-02-27T05:11:20.000+00:00,Analyze,"[{""name"": ""Set up job"", ""number"": 1, ""status"": ""completed"", ""conclusion"": ""succes [...]
+1,134018330,2011825638,613518923,https://api.github.com/repos/panjf2000/ants/actions/runs/613518923,MDg6Q2hlY2tSdW4yMDExODI1NjM4,5431f73492ade2e5b947a98f6032595c32cf730e,https://api.github.com/repos/panjf2000/ants/actions/jobs/2011825638,https://github.com/panjf2000/ants/runs/2011825638?check_suite_focus=true,COMPLETED,SUCCESS,2021-03-02T09:24:49.000+00:00,2021-03-02T09:25:11.000+00:00,Golangci-Lint,"[{""name"": ""Set up job"", ""number"": 1, ""status"": ""completed"", ""conclusion"": "" [...]
+1,134018330,2139659897,664533609,https://api.github.com/repos/panjf2000/ants/actions/runs/664533609,MDg6Q2hlY2tSdW4yMTM5NjU5ODk3,e45d13c6303d4ec82d16cd4111a49a7de0ad0712,https://api.github.com/repos/panjf2000/ants/actions/jobs/2139659897,https://github.com/panjf2000/ants/runs/2139659897?check_suite_focus=true,COMPLETED,SUCCESS,2021-03-18T12:39:24.000+00:00,2021-03-18T12:40:35.000+00:00,Analyze,"[{""name"": ""Set up job"", ""number"": 1, ""status"": ""completed"", ""conclusion"": ""succes [...]
diff --git a/backend/plugins/github/e2e/snapshot_tables/_tool_github_runs_no_prod_regex.csv b/backend/plugins/github/e2e/snapshot_tables/_tool_github_runs_no_prod_regex.csv
new file mode 100644
index 000000000..1b86c263f
--- /dev/null
+++ b/backend/plugins/github/e2e/snapshot_tables/_tool_github_runs_no_prod_regex.csv
@@ -0,0 +1,21 @@
+connection_id,repo_id,id,name,node_id,head_branch,head_sha,path,run_number,event,status,conclusion,workflow_id,check_suite_id,check_suite_node_id,url,html_url,github_created_at,github_updated_at,run_attempt,run_started_at,jobs_url,logs_url,check_suite_url,artifacts_url,cancel_url,rerun_url,workflow_url,type,environment
+1,134018330,2559400712,CodeQL,WFR_kwLOB_z1Gs6YjVsI,Fix_rm_redundancy_code,5dd23ddff8621e6ae36eb24b20d4c4a06dd73dc9,.github/workflows/codeql.yml,141,pull_request,completed,success,5904664,7087122717,CS_kwDOB_z1Gs8AAAABpmzpHQ,https://api.github.com/repos/panjf2000/ants/actions/runs/2559400712,https://github.com/panjf2000/ants/actions/runs/2559400712,2022-06-25T04:17:45.000+00:00,2022-06-26T12:36:58.000+00:00,2,2022-06-26T12:35:50.000+00:00,https://api.github.com/repos/panjf2000/ants/action [...]
+1,134018330,2559400713,Lint,WFR_kwLOB_z1Gs6YjVsJ,Fix_rm_redundancy_code,5dd23ddff8621e6ae36eb24b20d4c4a06dd73dc9,.github/workflows/lint.yml,71,pull_request,completed,success,5904665,7087122718,CS_kwDOB_z1Gs8AAAABpmzpHg,https://api.github.com/repos/panjf2000/ants/actions/runs/2559400713,https://github.com/panjf2000/ants/actions/runs/2559400713,2022-06-25T04:17:45.000+00:00,2022-06-26T12:36:22.000+00:00,2,2022-06-26T12:35:50.000+00:00,https://api.github.com/repos/panjf2000/ants/actions/run [...]
+1,134018330,2559400714,Tests,WFR_kwLOB_z1Gs6YjVsK,Fix_rm_redundancy_code,5dd23ddff8621e6ae36eb24b20d4c4a06dd73dc9,.github/workflows/ci.yml,71,pull_request,completed,success,5904663,7087122719,CS_kwDOB_z1Gs8AAAABpmzpHw,https://api.github.com/repos/panjf2000/ants/actions/runs/2559400714,https://github.com/panjf2000/ants/actions/runs/2559400714,2022-06-25T04:17:45.000+00:00,2022-06-26T12:41:24.000+00:00,2,2022-06-26T12:35:50.000+00:00,https://api.github.com/repos/panjf2000/ants/actions/runs [...]
+1,134018330,2559507315,CodeQL,WFR_kwLOB_z1Gs6Yjvtz,master,f85611741eb1f5451697ac589008d28f240887fc,.github/workflows/codeql.yml,142,schedule,in_progress,,5904664,7087322798,CS_kwDOB_z1Gs8AAAABpm_2rg,https://api.github.com/repos/panjf2000/ants/actions/runs/2559507315,https://github.com/panjf2000/ants/actions/runs/2559507315,2022-06-25T05:02:56.000+00:00,2022-06-25T05:03:53.000+00:00,1,2022-06-25T05:02:56.000+00:00,https://api.github.com/repos/panjf2000/ants/actions/runs/2559507315/jobs,ht [...]
+1,134018330,2566218975,Tests,WFR_kwLOB_z1Gs6Y9WTf,master,32664cb1408f8d9ffa7236335025a4cd94a306ce,.github/workflows/ci.yml,72,push,in_progress,,5904663,7099938409,CS_kwDOB_z1Gs8AAAABpzB2aQ,https://api.github.com/repos/panjf2000/ants/actions/runs/2566218975,https://github.com/panjf2000/ants/actions/runs/2566218975,2022-06-27T01:29:54.000+00:00,2022-06-27T01:37:33.000+00:00,1,2022-06-27T01:29:54.000+00:00,https://api.github.com/repos/panjf2000/ants/actions/runs/2566218975/jobs,https://api. [...]
+1,134018330,2566218976,CodeQL,WFR_kwLOB_z1Gs6Y9WTg,master,32664cb1408f8d9ffa7236335025a4cd94a306ce,.github/workflows/codeql.yml,143,push,completed,success,5904664,7099938410,CS_kwDOB_z1Gs8AAAABpzB2ag,https://api.github.com/repos/panjf2000/ants/actions/runs/2566218976,https://github.com/panjf2000/ants/actions/runs/2566218976,2022-06-27T01:29:54.000+00:00,2022-06-27T01:30:55.000+00:00,1,2022-06-27T01:29:54.000+00:00,https://api.github.com/repos/panjf2000/ants/actions/runs/2566218976/jobs,h [...]
+1,134018330,2566218977,Lint,WFR_kwLOB_z1Gs6Y9WTh,master,32664cb1408f8d9ffa7236335025a4cd94a306ce,.github/workflows/lint.yml,72,push,completed,failure,5904665,7099938411,CS_kwDOB_z1Gs8AAAABpzB2aw,https://api.github.com/repos/panjf2000/ants/actions/runs/2566218977,https://github.com/panjf2000/ants/actions/runs/2566218977,2022-06-27T01:29:54.000+00:00,2022-06-27T01:30:28.000+00:00,1,2022-06-27T01:29:54.000+00:00,https://api.github.com/repos/panjf2000/ants/actions/runs/2566218977/jobs,https: [...]
+1,134018330,2589885628,Tests,WFR_kwLOB_z1Gs6aXoS8,master,fa938334e73faf88a15b59622ab1da61a643c5da,.github/workflows/ci.yml,75,pull_request,completed,success,5904663,7161479138,CS_kwDOB_z1Gs8AAAABqtt_4g,https://api.github.com/repos/panjf2000/ants/actions/runs/2589885628,https://github.com/panjf2000/ants/actions/runs/2589885628,2022-06-30T12:23:37.000+00:00,2022-07-01T13:40:47.000+00:00,2,2022-07-01T13:34:14.000+00:00,https://api.github.com/repos/panjf2000/ants/actions/runs/2589885628/jobs [...]
+1,134018330,2589885635,CodeQL,WFR_kwLOB_z1Gs6aXoTD,master,fa938334e73faf88a15b59622ab1da61a643c5da,.github/workflows/codeql.yml,146,pull_request,completed,failure,5904664,7161479152,CS_kwDOB_z1Gs8AAAABqtt_8A,https://api.github.com/repos/panjf2000/ants/actions/runs/2589885635,https://github.com/panjf2000/ants/actions/runs/2589885635,2022-06-30T12:23:37.000+00:00,2022-07-01T13:35:19.000+00:00,2,2022-07-01T13:34:14.000+00:00,https://api.github.com/repos/panjf2000/ants/actions/runs/258988563 [...]
+1,134018330,2589885639,Lint,WFR_kwLOB_z1Gs6aXoTH,master,fa938334e73faf88a15b59622ab1da61a643c5da,.github/workflows/lint.yml,75,pull_request,completed,success,5904665,7161479158,CS_kwDOB_z1Gs8AAAABqtt_9g,https://api.github.com/repos/panjf2000/ants/actions/runs/2589885639,https://github.com/panjf2000/ants/actions/runs/2589885639,2022-06-30T12:23:37.000+00:00,2022-07-01T13:34:43.000+00:00,2,2022-07-01T13:34:14.000+00:00,https://api.github.com/repos/panjf2000/ants/actions/runs/2589885639/job [...]
+1,134018330,2600408985,CodeQL,WFR_kwLOB_z1Gs6a_xeZ,master,32664cb1408f8d9ffa7236335025a4cd94a306ce,.github/workflows/codeql.yml,147,schedule,completed,success,5904664,7187902086,CS_kwDOB_z1Gs8AAAABrG6uhg,https://api.github.com/repos/panjf2000/ants/actions/runs/2600408985,https://github.com/panjf2000/ants/actions/runs/2600408985,2022-07-02T05:05:26.000+00:00,2022-07-02T05:06:23.000+00:00,1,2022-07-02T05:05:26.000+00:00,https://api.github.com/repos/panjf2000/ants/actions/runs/2600408985/jo [...]
+1,134018330,2639945362,CodeQL,WFR_kwLOB_z1Gs6dWl6S,master,32664cb1408f8d9ffa7236335025a4cd94a306ce,.github/workflows/codeql.yml,148,schedule,completed,success,5904664,7284226378,CS_kwDOB_z1Gs8AAAABsix5Sg,https://api.github.com/repos/panjf2000/ants/actions/runs/2639945362,https://github.com/panjf2000/ants/actions/runs/2639945362,2022-07-09T05:02:44.000+00:00,2022-07-09T05:03:48.000+00:00,1,2022-07-09T05:02:44.000+00:00,https://api.github.com/repos/panjf2000/ants/actions/runs/2639945362/jo [...]
+1,134018330,2680721264,CodeQL,WFR_kwLOB_z1Gs6fyI9w,master,32664cb1408f8d9ffa7236335025a4cd94a306ce,.github/workflows/codeql.yml,149,schedule,completed,success,5904664,7383284464,CS_kwDOB_z1Gs8AAAABuBP68A,https://api.github.com/repos/panjf2000/ants/actions/runs/2680721264,https://github.com/panjf2000/ants/actions/runs/2680721264,2022-07-16T05:03:38.000+00:00,2022-07-16T05:04:51.000+00:00,1,2022-07-16T05:03:38.000+00:00,https://api.github.com/repos/panjf2000/ants/actions/runs/2680721264/jo [...]
+1,134018330,2722539966,CodeQL,WFR_kwLOB_z1Gs6iRqm-,master,32664cb1408f8d9ffa7236335025a4cd94a306ce,.github/workflows/codeql.yml,150,schedule,completed,success,5904664,7487244521,CS_kwDOB_z1Gs8AAAABvkZI6Q,https://api.github.com/repos/panjf2000/ants/actions/runs/2722539966,https://github.com/panjf2000/ants/actions/runs/2722539966,2022-07-23T05:04:59.000+00:00,2022-07-23T05:05:58.000+00:00,1,2022-07-23T05:04:59.000+00:00,https://api.github.com/repos/panjf2000/ants/actions/runs/2722539966/jo [...]
+1,134018330,2764660507,CodeQL,WFR_kwLOB_z1Gs6kyV8b,master,32664cb1408f8d9ffa7236335025a4cd94a306ce,.github/workflows/codeql.yml,151,schedule,completed,success,5904664,7589122087,CS_kwDOB_z1Gs8AAAABxFjQJw,https://api.github.com/repos/panjf2000/ants/actions/runs/2764660507,https://github.com/panjf2000/ants/actions/runs/2764660507,2022-07-30T05:06:06.000+00:00,2022-07-30T05:07:04.000+00:00,1,2022-07-30T05:06:06.000+00:00,https://api.github.com/repos/panjf2000/ants/actions/runs/2764660507/jo [...]
+1,134018330,2807709308,CodeQL,WFR_kwLOB_z1Gs6nWj58,master,32664cb1408f8d9ffa7236335025a4cd94a306ce,.github/workflows/codeql.yml,154,schedule,completed,success,5904664,7693176674,CS_kwDOB_z1Gs8AAAAByoyPYg,https://api.github.com/repos/panjf2000/ants/actions/runs/2807709308,https://github.com/panjf2000/ants/actions/runs/2807709308,2022-08-06T05:02:43.000+00:00,2022-08-06T05:03:58.000+00:00,1,2022-08-06T05:02:43.000+00:00,https://api.github.com/repos/panjf2000/ants/actions/runs/2807709308/jo [...]
+1,134018330,2850801364,CodeQL,WFR_kwLOB_z1Gs6p68bU,master,32664cb1408f8d9ffa7236335025a4cd94a306ce,.github/workflows/codeql.yml,155,schedule,completed,success,5904664,7797647541,CS_kwDOB_z1Gs8AAAAB0MaotQ,https://api.github.com/repos/panjf2000/ants/actions/runs/2850801364,https://github.com/panjf2000/ants/actions/runs/2850801364,2022-08-13T05:02:51.000+00:00,2022-08-13T05:03:45.000+00:00,1,2022-08-13T05:02:51.000+00:00,https://api.github.com/repos/panjf2000/ants/actions/runs/2850801364/jo [...]
+1,134018330,2893573709,CodeQL,WFR_kwLOB_z1Gs6seG5N,master,32664cb1408f8d9ffa7236335025a4cd94a306ce,.github/workflows/codeql.yml,156,schedule,completed,success,5904664,7899725937,CS_kwDOB_z1Gs8AAAAB1txAcQ,https://api.github.com/repos/panjf2000/ants/actions/runs/2893573709,https://github.com/panjf2000/ants/actions/runs/2893573709,2022-08-20T05:04:53.000+00:00,2022-08-20T05:06:10.000+00:00,1,2022-08-20T05:04:53.000+00:00,https://api.github.com/repos/panjf2000/ants/actions/runs/2893573709/jo [...]
+1,134018330,2938072864,CodeQL,WFR_kwLOB_z1Gs6vH28g,master,06e6934c35c336b1a2bd3005fb21dc3914a45747,.github/workflows/codeql.yml,157,schedule,completed,success,5904664,8009261503,CS_kwDOB_z1Gs8AAAAB3WOhvw,https://api.github.com/repos/panjf2000/ants/actions/runs/2938072864,https://github.com/panjf2000/ants/actions/runs/2938072864,2022-08-27T05:13:50.000+00:00,2022-08-27T05:15:06.000+00:00,1,2022-08-27T05:13:50.000+00:00,https://api.github.com/repos/panjf2000/ants/actions/runs/2938072864/jo [...]
+1,134018330,2983238245,CodeQL,WFR_kwLOB_z1Gs6x0Jpl,master,06e6934c35c336b1a2bd3005fb21dc3914a45747,.github/workflows/codeql.yml,158,schedule,completed,success,5904664,8117851893,CS_kwDOB_z1Gs8AAAAB49yW9Q,https://api.github.com/repos/panjf2000/ants/actions/runs/2983238245,https://github.com/panjf2000/ants/actions/runs/2983238245,2022-09-03T05:15:09.000+00:00,2022-09-03T05:16:16.000+00:00,1,2022-09-03T05:15:09.000+00:00,https://api.github.com/repos/panjf2000/ants/actions/runs/2983238245/jo [...]
diff --git a/backend/plugins/github/tasks/cicd_job_extractor.go b/backend/plugins/github/tasks/cicd_job_extractor.go
index 41ab90ca0..edb96cccf 100644
--- a/backend/plugins/github/tasks/cicd_job_extractor.go
+++ b/backend/plugins/github/tasks/cicd_job_extractor.go
@@ -19,12 +19,13 @@ package tasks
import (
"encoding/json"
+ "strings"
+
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models/domainlayer/devops"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/github/models"
- "strings"
)
var ExtractJobsMeta = plugin.SubTaskMeta{
@@ -78,7 +79,7 @@ func ExtractJobs(taskCtx plugin.SubTaskContext) errors.Error {
RunnerName: githubJob.RunnerName,
RunnerGroupID: githubJob.RunnerGroupID,
Type: data.RegexEnricher.ReturnNameIfMatched(devops.DEPLOYMENT, githubJob.Name),
- Environment: data.RegexEnricher.ReturnNameIfMatched(devops.PRODUCTION, githubJob.Name),
+ Environment: data.RegexEnricher.ReturnNameIfOmittedOrMatched(devops.PRODUCTION, githubJob.Name),
}
results = append(results, githubJobResult)
return results, nil
diff --git a/backend/plugins/github/tasks/cicd_run_extractor.go b/backend/plugins/github/tasks/cicd_run_extractor.go
index 2283e9b2f..de3326f09 100644
--- a/backend/plugins/github/tasks/cicd_run_extractor.go
+++ b/backend/plugins/github/tasks/cicd_run_extractor.go
@@ -19,6 +19,7 @@ package tasks
import (
"encoding/json"
+
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models/domainlayer/devops"
"github.com/apache/incubator-devlake/core/plugin"
@@ -85,7 +86,7 @@ func ExtractRuns(taskCtx plugin.SubTaskContext) errors.Error {
RerunURL: githubRun.RerunURL,
WorkflowURL: githubRun.WorkflowURL,
Type: data.RegexEnricher.ReturnNameIfMatched(devops.DEPLOYMENT, githubRun.Name),
- Environment: data.RegexEnricher.ReturnNameIfMatched(devops.PRODUCTION, githubRun.Name),
+ Environment: data.RegexEnricher.ReturnNameIfOmittedOrMatched(devops.PRODUCTION, githubRun.Name),
}
results = append(results, githubRunResult)
return results, nil
diff --git a/backend/plugins/github_graphql/tasks/job_collector.go b/backend/plugins/github_graphql/tasks/job_collector.go
index 2ec5c3242..805a86c5e 100644
--- a/backend/plugins/github_graphql/tasks/job_collector.go
+++ b/backend/plugins/github_graphql/tasks/job_collector.go
@@ -192,7 +192,7 @@ func CollectGraphqlJobs(taskCtx plugin.SubTaskContext) errors.Error {
Name: checkRun.Name,
Steps: paramsBytes,
Type: data.RegexEnricher.ReturnNameIfMatched(devops.DEPLOYMENT, checkRun.Name),
- Environment: data.RegexEnricher.ReturnNameIfMatched(devops.PRODUCTION, checkRun.Name),
+ Environment: data.RegexEnricher.ReturnNameIfOmittedOrMatched(devops.PRODUCTION, checkRun.Name),
// these columns can not fill by graphql
//HeadSha: ``, // use _tool_github_runs
//RunURL: ``,
diff --git a/backend/plugins/gitlab/e2e/job_test.go b/backend/plugins/gitlab/e2e/job_test.go
index 6088b725e..1705f59a8 100644
--- a/backend/plugins/gitlab/e2e/job_test.go
+++ b/backend/plugins/gitlab/e2e/job_test.go
@@ -18,29 +18,29 @@ limitations under the License.
package e2e
import (
+ "testing"
+
"github.com/apache/incubator-devlake/core/models/common"
"github.com/apache/incubator-devlake/core/models/domainlayer/devops"
"github.com/apache/incubator-devlake/helpers/e2ehelper"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/gitlab/impl"
"github.com/apache/incubator-devlake/plugins/gitlab/models"
"github.com/apache/incubator-devlake/plugins/gitlab/tasks"
- "testing"
)
func TestGitlabJobDataFlow(t *testing.T) {
var gitlab impl.Gitlab
dataflowTester := e2ehelper.NewDataFlowTester(t, "gitlab", gitlab)
-
+ regexEnricher := api.NewRegexEnricher()
+ _ = regexEnricher.TryAdd(devops.DEPLOYMENT, "(?i)compile")
taskData := &tasks.GitlabTaskData{
Options: &tasks.GitlabOptions{
ConnectionId: 1,
ProjectId: 44,
- GitlabTransformationRule: &models.GitlabTransformationRule{
- DeploymentPattern: "(?i)compile",
- ProductionPattern: "(?i)compile",
- },
},
+ RegexEnricher: regexEnricher,
}
// import raw data table
// SELECT * FROM _raw_gitlab_api_job INTO OUTFILE "/tmp/_raw_gitlab_api_job.csv" FIELDS TERMINATED BY ',' OPTIONALLY ENCLOSED BY '"' LINES TERMINATED BY '\r\n';
@@ -71,7 +71,16 @@ func TestGitlabJobDataFlow(t *testing.T) {
),
)
+ // verifi when production regex is omitted
+ dataflowTester.FlushTabler(&devops.CICDTask{})
+ dataflowTester.Subtask(tasks.ConvertJobMeta, taskData)
+ dataflowTester.VerifyTableWithOptions(&devops.CICDTask{}, e2ehelper.TableOptions{
+ CSVRelPath: "./snapshot_tables/cicd_tasks_no_prod_regex.csv",
+ IgnoreTypes: []interface{}{common.NoPKModel{}},
+ })
+
// verify conversion
+ _ = regexEnricher.TryAdd(devops.PRODUCTION, "(?i)compile")
dataflowTester.FlushTabler(&devops.CICDTask{})
dataflowTester.Subtask(tasks.ConvertJobMeta, taskData)
dataflowTester.VerifyTableWithOptions(&devops.CICDTask{}, e2ehelper.TableOptions{
diff --git a/backend/plugins/gitlab/e2e/pipelines_test.go b/backend/plugins/gitlab/e2e/pipelines_test.go
index 0f4bb38d6..9c0bb8b70 100644
--- a/backend/plugins/gitlab/e2e/pipelines_test.go
+++ b/backend/plugins/gitlab/e2e/pipelines_test.go
@@ -36,7 +36,6 @@ func TestGitlabPipelineDataFlow(t *testing.T) {
regexEnricher := api.NewRegexEnricher()
_ = regexEnricher.TryAdd(devops.DEPLOYMENT, "EE-7121")
- _ = regexEnricher.TryAdd(devops.PRODUCTION, "EE-7121")
taskData := &tasks.GitlabTaskData{
Options: &tasks.GitlabOptions{
@@ -51,7 +50,20 @@ func TestGitlabPipelineDataFlow(t *testing.T) {
dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_gitlab_api_pipeline.csv", "_raw_gitlab_api_pipeline")
dataflowTester.ImportCsvIntoTabler("./raw_tables/_tool_gitlab_projects.csv", &models.GitlabProject{})
+ // verify env when production is omitted
+ dataflowTester.FlushTabler(&models.GitlabPipeline{})
+ dataflowTester.FlushTabler(&models.GitlabPipelineProject{})
+ dataflowTester.Subtask(tasks.ExtractApiPipelinesMeta, taskData)
+ dataflowTester.VerifyTable(
+ models.GitlabPipeline{},
+ "./snapshot_tables/_tool_gitlab_pipelines_no_prod_regex.csv",
+ e2ehelper.ColumnWithRawData(
+ "environment",
+ ),
+ )
+
// verify extraction
+ _ = regexEnricher.TryAdd(devops.PRODUCTION, "EE-7121")
dataflowTester.FlushTabler(&models.GitlabPipeline{})
dataflowTester.FlushTabler(&models.GitlabPipelineProject{})
dataflowTester.Subtask(tasks.ExtractApiPipelinesMeta, taskData)
diff --git a/backend/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_pipelines_no_prod_regex.csv b/backend/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_pipelines_no_prod_regex.csv
new file mode 100644
index 000000000..deb620e2e
--- /dev/null
+++ b/backend/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_pipelines_no_prod_regex.csv
@@ -0,0 +1,21 @@
+connection_id,gitlab_id,environment,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
+1,457474837,PRODUCTION,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,152,
+1,457474996,PRODUCTION,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,153,
+1,457475160,PRODUCTION,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,154,
+1,457475337,PRODUCTION,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,155,
+1,485811050,PRODUCTION,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,156,
+1,485811059,PRODUCTION,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,157,
+1,485813816,PRODUCTION,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,158,
+1,485813830,PRODUCTION,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,159,
+1,485814501,PRODUCTION,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,160,
+1,485814516,PRODUCTION,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,161,
+1,485814871,PRODUCTION,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,162,
+1,485817670,PRODUCTION,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,163,
+1,485837602,PRODUCTION,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,164,
+1,485842553,PRODUCTION,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,165,
+1,485845850,PRODUCTION,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,166,
+1,485852752,PRODUCTION,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,167,
+1,485865876,PRODUCTION,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,168,
+1,485877118,PRODUCTION,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,169,
+1,485905167,PRODUCTION,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,170,
+1,485932863,PRODUCTION,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,171,
diff --git a/backend/plugins/gitlab/e2e/snapshot_tables/cicd_tasks_no_prod_regex.csv b/backend/plugins/gitlab/e2e/snapshot_tables/cicd_tasks_no_prod_regex.csv
new file mode 100644
index 000000000..abc6d6aa9
--- /dev/null
+++ b/backend/plugins/gitlab/e2e/snapshot_tables/cicd_tasks_no_prod_regex.csv
@@ -0,0 +1,68 @@
+id,name,pipeline_id,result,status,type,environment,duration_sec,started_date,finished_date,cicd_scope_id
+gitlab:GitlabJob:1:100,compile,gitlab:GitlabPipeline:1:24,SUCCESS,DONE,DEPLOYMENT,PRODUCTION,2,2022-07-25T15:06:57.051+00:00,2022-07-25T15:06:59.885+00:00,gitlab:GitlabProject:1:44
+gitlab:GitlabJob:1:101,format,gitlab:GitlabPipeline:1:25,SUCCESS,DONE,,PRODUCTION,3,2022-07-25T15:13:37.206+00:00,2022-07-25T15:13:40.246+00:00,gitlab:GitlabProject:1:44
+gitlab:GitlabJob:1:102,format,gitlab:GitlabPipeline:1:26,SUCCESS,DONE,,PRODUCTION,2,2022-07-25T15:30:22.560+00:00,2022-07-25T15:30:25.315+00:00,gitlab:GitlabProject:1:44
+gitlab:GitlabJob:1:103,format,gitlab:GitlabPipeline:1:27,SUCCESS,DONE,,PRODUCTION,2,2022-07-25T15:30:55.671+00:00,2022-07-25T15:30:58.650+00:00,gitlab:GitlabProject:1:44
+gitlab:GitlabJob:1:104,format,gitlab:GitlabPipeline:1:28,SUCCESS,DONE,,PRODUCTION,2,2022-07-25T15:32:04.954+00:00,2022-07-25T15:32:07.726+00:00,gitlab:GitlabProject:1:44
+gitlab:GitlabJob:1:105,compile,gitlab:GitlabPipeline:1:28,FAILURE,DONE,DEPLOYMENT,PRODUCTION,3,2022-07-25T15:32:07.953+00:00,2022-07-25T15:32:11.077+00:00,gitlab:GitlabProject:1:44
+gitlab:GitlabJob:1:106,format,gitlab:GitlabPipeline:1:29,SUCCESS,DONE,,PRODUCTION,2,2022-07-25T15:33:26.382+00:00,2022-07-25T15:33:29.356+00:00,gitlab:GitlabProject:1:44
+gitlab:GitlabJob:1:107,format,gitlab:GitlabPipeline:1:30,SUCCESS,DONE,,PRODUCTION,2,2022-07-25T15:34:23.665+00:00,2022-07-25T15:34:26.392+00:00,gitlab:GitlabProject:1:44
+gitlab:GitlabJob:1:108,format,gitlab:GitlabPipeline:1:31,SUCCESS,DONE,,PRODUCTION,2,2022-07-25T15:35:11.707+00:00,2022-07-25T15:35:14.224+00:00,gitlab:GitlabProject:1:44
+gitlab:GitlabJob:1:109,compile,gitlab:GitlabPipeline:1:31,SUCCESS,DONE,DEPLOYMENT,PRODUCTION,3,2022-07-25T15:35:14.724+00:00,2022-07-25T15:35:17.828+00:00,gitlab:GitlabProject:1:44
+gitlab:GitlabJob:1:110,format,gitlab:GitlabPipeline:1:32,SUCCESS,DONE,,PRODUCTION,2,2022-07-25T15:36:18.097+00:00,2022-07-25T15:36:20.954+00:00,gitlab:GitlabProject:1:44
+gitlab:GitlabJob:1:111,format,gitlab:GitlabPipeline:1:33,SUCCESS,DONE,,PRODUCTION,3,2022-07-25T15:38:03.463+00:00,2022-07-25T15:38:06.467+00:00,gitlab:GitlabProject:1:44
+gitlab:GitlabJob:1:112,format,gitlab:GitlabPipeline:1:34,SUCCESS,DONE,,PRODUCTION,3,2022-07-25T21:19:14.509+00:00,2022-07-25T21:19:17.811+00:00,gitlab:GitlabProject:1:44
+gitlab:GitlabJob:1:113,format,gitlab:GitlabPipeline:1:35,SUCCESS,DONE,,PRODUCTION,5,2022-07-26T09:37:05.694+00:00,2022-07-26T09:37:10.873+00:00,gitlab:GitlabProject:1:44
+gitlab:GitlabJob:1:114,format,gitlab:GitlabPipeline:1:36,SUCCESS,DONE,,PRODUCTION,2,2022-07-26T09:37:38.057+00:00,2022-07-26T09:37:40.975+00:00,gitlab:GitlabProject:1:44
+gitlab:GitlabJob:1:115,format,gitlab:GitlabPipeline:1:37,SUCCESS,DONE,,PRODUCTION,3,2022-07-26T09:38:29.318+00:00,2022-07-26T09:38:32.970+00:00,gitlab:GitlabProject:1:44
+gitlab:GitlabJob:1:116,format,gitlab:GitlabPipeline:1:38,SUCCESS,DONE,,PRODUCTION,3,2022-07-26T21:19:13.888+00:00,2022-07-26T21:19:17.021+00:00,gitlab:GitlabProject:1:44
+gitlab:GitlabJob:1:117,format,gitlab:GitlabPipeline:1:39,SUCCESS,DONE,,PRODUCTION,3,2022-07-27T08:19:24.376+00:00,2022-07-27T08:19:28.159+00:00,gitlab:GitlabProject:1:44
+gitlab:GitlabJob:1:118,format,gitlab:GitlabPipeline:1:40,SUCCESS,DONE,,PRODUCTION,4,2022-07-27T21:19:32.288+00:00,2022-07-27T21:19:36.850+00:00,gitlab:GitlabProject:1:44
+gitlab:GitlabJob:1:119,format,gitlab:GitlabPipeline:1:41,FAILURE,DONE,,PRODUCTION,0,2022-07-28T21:19:24.257+00:00,2022-07-28T23:00:17.842+00:00,gitlab:GitlabProject:1:44
+gitlab:GitlabJob:1:120,format,gitlab:GitlabPipeline:1:41,SUCCESS,DONE,,PRODUCTION,56,2022-07-29T02:10:58.370+00:00,2022-07-29T02:11:55.170+00:00,gitlab:GitlabProject:1:44
+gitlab:GitlabJob:1:121,format,gitlab:GitlabPipeline:1:42,FAILURE,DONE,,PRODUCTION,0,2022-07-29T21:19:02.884+00:00,2022-07-29T23:00:24.840+00:00,gitlab:GitlabProject:1:44
+gitlab:GitlabJob:1:122,format,gitlab:GitlabPipeline:1:43,FAILURE,DONE,,PRODUCTION,0,2022-07-30T21:19:26.310+00:00,2022-07-30T23:00:25.126+00:00,gitlab:GitlabProject:1:44
+gitlab:GitlabJob:1:123,format,gitlab:GitlabPipeline:1:44,FAILURE,DONE,,PRODUCTION,0,2022-07-31T21:19:05.348+00:00,2022-07-31T23:00:29.135+00:00,gitlab:GitlabProject:1:44
+gitlab:GitlabJob:1:124,format,gitlab:GitlabPipeline:1:45,FAILURE,DONE,,PRODUCTION,0,2022-08-01T21:19:02.489+00:00,2022-08-01T23:00:22.874+00:00,gitlab:GitlabProject:1:44
+gitlab:GitlabJob:1:125,format,gitlab:GitlabPipeline:1:46,FAILURE,DONE,,PRODUCTION,0,2022-08-02T21:19:25.568+00:00,2022-08-02T23:00:23.221+00:00,gitlab:GitlabProject:1:44
+gitlab:GitlabJob:1:126,format,gitlab:GitlabPipeline:1:47,FAILURE,DONE,,PRODUCTION,0,2022-08-03T08:19:06.570+00:00,2022-08-03T10:00:05.573+00:00,gitlab:GitlabProject:1:44
+gitlab:GitlabJob:1:127,format,gitlab:GitlabPipeline:1:48,FAILURE,DONE,,PRODUCTION,0,2022-08-03T21:19:21.010+00:00,2022-08-03T23:00:06.114+00:00,gitlab:GitlabProject:1:44
+gitlab:GitlabJob:1:128,format,gitlab:GitlabPipeline:1:49,FAILURE,DONE,,PRODUCTION,0,2022-08-04T21:19:12.398+00:00,2022-08-04T23:00:25.717+00:00,gitlab:GitlabProject:1:44
+gitlab:GitlabJob:1:129,format,gitlab:GitlabPipeline:1:50,FAILURE,DONE,,PRODUCTION,0,2022-08-05T21:19:09.648+00:00,2022-08-05T23:00:18.441+00:00,gitlab:GitlabProject:1:44
+gitlab:GitlabJob:1:130,format,gitlab:GitlabPipeline:1:51,FAILURE,DONE,,PRODUCTION,0,2022-08-06T21:19:29.253+00:00,2022-08-06T23:00:04.246+00:00,gitlab:GitlabProject:1:44
+gitlab:GitlabJob:1:131,format,gitlab:GitlabPipeline:1:52,FAILURE,DONE,,PRODUCTION,0,2022-08-07T21:19:33.476+00:00,2022-08-07T23:00:01.350+00:00,gitlab:GitlabProject:1:44
+gitlab:GitlabJob:1:132,format,gitlab:GitlabPipeline:1:53,FAILURE,DONE,,PRODUCTION,0,2022-08-08T21:19:02.531+00:00,2022-08-08T23:00:30.138+00:00,gitlab:GitlabProject:1:44
+gitlab:GitlabJob:1:133,format,gitlab:GitlabPipeline:1:54,FAILURE,DONE,,PRODUCTION,0,2022-08-09T21:19:34.379+00:00,2022-08-09T23:00:15.331+00:00,gitlab:GitlabProject:1:44
+gitlab:GitlabJob:1:134,format,gitlab:GitlabPipeline:1:55,FAILURE,DONE,,PRODUCTION,0,2022-08-10T08:19:08.693+00:00,2022-08-10T10:00:10.203+00:00,gitlab:GitlabProject:1:44
+gitlab:GitlabJob:1:135,format,gitlab:GitlabPipeline:1:56,FAILURE,DONE,,PRODUCTION,0,2022-08-10T21:19:05.714+00:00,2022-08-10T23:00:41.546+00:00,gitlab:GitlabProject:1:44
+gitlab:GitlabJob:1:136,format,gitlab:GitlabPipeline:1:57,FAILURE,DONE,,PRODUCTION,0,2022-08-11T21:19:25.605+00:00,2022-08-11T23:00:08.674+00:00,gitlab:GitlabProject:1:44
+gitlab:GitlabJob:1:137,format,gitlab:GitlabPipeline:1:58,FAILURE,DONE,,PRODUCTION,0,2022-08-12T21:19:08.350+00:00,2022-08-12T23:00:03.492+00:00,gitlab:GitlabProject:1:44
+gitlab:GitlabJob:1:138,format,gitlab:GitlabPipeline:1:59,FAILURE,DONE,,PRODUCTION,0,2022-08-13T21:19:06.775+00:00,2022-08-13T23:00:06.728+00:00,gitlab:GitlabProject:1:44
+gitlab:GitlabJob:1:139,format,gitlab:GitlabPipeline:1:60,FAILURE,DONE,,PRODUCTION,0,2022-08-14T21:19:07.007+00:00,2022-08-14T23:00:22.581+00:00,gitlab:GitlabProject:1:44
+gitlab:GitlabJob:1:140,format,gitlab:GitlabPipeline:1:61,FAILURE,DONE,,PRODUCTION,0,2022-08-15T21:19:09.087+00:00,2022-08-15T23:00:31.590+00:00,gitlab:GitlabProject:1:44
+gitlab:GitlabJob:1:141,format,gitlab:GitlabPipeline:1:62,FAILURE,DONE,,PRODUCTION,0,2022-08-16T21:19:12.248+00:00,2022-08-16T23:00:16.800+00:00,gitlab:GitlabProject:1:44
+gitlab:GitlabJob:1:142,format,gitlab:GitlabPipeline:1:63,FAILURE,DONE,,PRODUCTION,0,2022-08-17T08:20:06.419+00:00,2022-08-17T10:00:36.594+00:00,gitlab:GitlabProject:1:44
+gitlab:GitlabJob:1:143,format,gitlab:GitlabPipeline:1:64,FAILURE,DONE,,PRODUCTION,0,2022-08-17T21:19:11.908+00:00,2022-08-17T23:00:23.915+00:00,gitlab:GitlabProject:1:44
+gitlab:GitlabJob:1:144,format,gitlab:GitlabPipeline:1:65,FAILURE,DONE,,PRODUCTION,0,2022-08-18T21:19:14.072+00:00,2022-08-18T23:00:26.546+00:00,gitlab:GitlabProject:1:44
+gitlab:GitlabJob:1:145,format,gitlab:GitlabPipeline:1:66,FAILURE,DONE,,PRODUCTION,0,2022-08-19T21:19:03.364+00:00,2022-08-19T23:00:19.772+00:00,gitlab:GitlabProject:1:44
+gitlab:GitlabJob:1:146,format,gitlab:GitlabPipeline:1:67,FAILURE,DONE,,PRODUCTION,0,2022-08-20T21:19:37.743+00:00,2022-08-20T23:00:09.418+00:00,gitlab:GitlabProject:1:44
+gitlab:GitlabJob:1:147,format,gitlab:GitlabPipeline:1:68,FAILURE,DONE,,PRODUCTION,0,2022-08-21T21:19:02.164+00:00,2022-08-21T23:00:18.538+00:00,gitlab:GitlabProject:1:44
+gitlab:GitlabJob:1:148,format,gitlab:GitlabPipeline:1:69,FAILURE,DONE,,PRODUCTION,0,2022-08-22T21:19:16.175+00:00,2022-08-22T23:00:08.653+00:00,gitlab:GitlabProject:1:44
+gitlab:GitlabJob:1:149,format,gitlab:GitlabPipeline:1:70,FAILURE,DONE,,PRODUCTION,0,2022-08-23T21:19:13.313+00:00,2022-08-23T23:00:20.712+00:00,gitlab:GitlabProject:1:44
+gitlab:GitlabJob:1:150,format,gitlab:GitlabPipeline:1:71,FAILURE,DONE,,PRODUCTION,0,2022-08-24T08:19:19.653+00:00,2022-08-24T10:00:04.660+00:00,gitlab:GitlabProject:1:44
+gitlab:GitlabJob:1:151,format,gitlab:GitlabPipeline:1:72,FAILURE,DONE,,PRODUCTION,0,2022-08-24T21:19:29.226+00:00,2022-08-24T23:00:14.036+00:00,gitlab:GitlabProject:1:44
+gitlab:GitlabJob:1:152,format,gitlab:GitlabPipeline:1:73,FAILURE,DONE,,PRODUCTION,0,2022-08-25T21:19:10.938+00:00,2022-08-25T23:00:08.594+00:00,gitlab:GitlabProject:1:44
+gitlab:GitlabJob:1:86,format,gitlab:GitlabPipeline:1:16,FAILURE,DONE,,PRODUCTION,0,2022-07-25T13:40:42.020+00:00,2022-07-25T13:40:42.892+00:00,gitlab:GitlabProject:1:44
+gitlab:GitlabJob:1:87,format,gitlab:GitlabPipeline:1:16,FAILURE,DONE,,PRODUCTION,0,2022-07-25T13:41:11.601+00:00,2022-07-25T13:41:11.932+00:00,gitlab:GitlabProject:1:44
+gitlab:GitlabJob:1:88,format,gitlab:GitlabPipeline:1:17,FAILURE,DONE,,PRODUCTION,0,2022-07-25T13:42:59.674+00:00,2022-07-25T13:42:59.998+00:00,gitlab:GitlabProject:1:44
+gitlab:GitlabJob:1:89,format,gitlab:GitlabPipeline:1:17,ABORT,DONE,,PRODUCTION,0,2022-07-25T13:46:15.482+00:00,2022-07-25T13:49:42.952+00:00,gitlab:GitlabProject:1:44
+gitlab:GitlabJob:1:90,format,gitlab:GitlabPipeline:1:18,ABORT,DONE,,PRODUCTION,0,2022-07-25T13:50:40.680+00:00,2022-07-25T14:19:03.023+00:00,gitlab:GitlabProject:1:44
+gitlab:GitlabJob:1:91,format,gitlab:GitlabPipeline:1:18,FAILURE,DONE,,PRODUCTION,2,2022-07-25T14:26:02.616+00:00,2022-07-25T14:26:05.480+00:00,gitlab:GitlabProject:1:44
+gitlab:GitlabJob:1:92,format,gitlab:GitlabPipeline:1:18,FAILURE,DONE,,PRODUCTION,1,2022-07-25T14:47:12.876+00:00,2022-07-25T14:47:14.295+00:00,gitlab:GitlabProject:1:44
+gitlab:GitlabJob:1:93,format,gitlab:GitlabPipeline:1:19,FAILURE,DONE,,PRODUCTION,1,2022-07-25T14:53:56.227+00:00,2022-07-25T14:53:57.910+00:00,gitlab:GitlabProject:1:44
+gitlab:GitlabJob:1:94,format,gitlab:GitlabPipeline:1:20,FAILURE,DONE,,PRODUCTION,1,2022-07-25T14:55:26.493+00:00,2022-07-25T14:55:28.331+00:00,gitlab:GitlabProject:1:44
+gitlab:GitlabJob:1:95,format,gitlab:GitlabPipeline:1:21,FAILURE,DONE,,PRODUCTION,1,2022-07-25T14:56:59.811+00:00,2022-07-25T14:57:01.498+00:00,gitlab:GitlabProject:1:44
+gitlab:GitlabJob:1:96,format,gitlab:GitlabPipeline:1:21,FAILURE,DONE,,PRODUCTION,5,2022-07-25T14:59:29.276+00:00,2022-07-25T14:59:34.282+00:00,gitlab:GitlabProject:1:44
+gitlab:GitlabJob:1:97,format,gitlab:GitlabPipeline:1:22,SUCCESS,DONE,,PRODUCTION,3,2022-07-25T15:00:43.749+00:00,2022-07-25T15:00:46.895+00:00,gitlab:GitlabProject:1:44
+gitlab:GitlabJob:1:98,format,gitlab:GitlabPipeline:1:23,SUCCESS,DONE,,PRODUCTION,2,2022-07-25T15:03:23.471+00:00,2022-07-25T15:03:26.432+00:00,gitlab:GitlabProject:1:44
+gitlab:GitlabJob:1:99,format,gitlab:GitlabPipeline:1:24,SUCCESS,DONE,,PRODUCTION,2,2022-07-25T15:06:54.037+00:00,2022-07-25T15:06:56.819+00:00,gitlab:GitlabProject:1:44
diff --git a/backend/plugins/gitlab/tasks/job_convertor.go b/backend/plugins/gitlab/tasks/job_convertor.go
index 10275ead8..32d876017 100644
--- a/backend/plugins/gitlab/tasks/job_convertor.go
+++ b/backend/plugins/gitlab/tasks/job_convertor.go
@@ -18,6 +18,8 @@ limitations under the License.
package tasks
import (
+ "reflect"
+
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models/domainlayer"
@@ -26,7 +28,6 @@ import (
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
gitlabModels "github.com/apache/incubator-devlake/plugins/gitlab/models"
- "reflect"
)
var ConvertJobMeta = plugin.SubTaskMeta{
@@ -40,10 +41,7 @@ var ConvertJobMeta = plugin.SubTaskMeta{
func ConvertJobs(taskCtx plugin.SubTaskContext) (err errors.Error) {
db := taskCtx.GetDal()
data := taskCtx.GetData().(*GitlabTaskData)
- deploymentPattern := data.Options.DeploymentPattern
- productionPattern := data.Options.ProductionPattern
- regexEnricher := api.NewRegexEnricher()
- err = regexEnricher.AddRegexp(deploymentPattern, productionPattern)
+ regexEnricher := data.RegexEnricher
cursor, err := db.Cursor(dal.From(gitlabModels.GitlabJob{}),
dal.Where("project_id = ? and connection_id = ?", data.Options.ProjectId, data.Options.ConnectionId))
@@ -99,8 +97,8 @@ func ConvertJobs(taskCtx plugin.SubTaskContext) (err errors.Error) {
FinishedDate: gitlabJob.FinishedAt,
CicdScopeId: projectIdGen.Generate(data.Options.ConnectionId, gitlabJob.ProjectId),
}
- domainJob.Type = regexEnricher.GetEnrichResult(deploymentPattern, gitlabJob.Name, devops.DEPLOYMENT)
- domainJob.Environment = regexEnricher.GetEnrichResult(productionPattern, gitlabJob.Name, devops.PRODUCTION)
+ domainJob.Type = regexEnricher.ReturnNameIfMatched(devops.DEPLOYMENT, gitlabJob.Name)
+ domainJob.Environment = regexEnricher.ReturnNameIfOmittedOrMatched(devops.PRODUCTION, gitlabJob.Name)
return []interface{}{
domainJob,
diff --git a/backend/plugins/gitlab/tasks/pipeline_extractor.go b/backend/plugins/gitlab/tasks/pipeline_extractor.go
index 39c840119..0c275a7d5 100644
--- a/backend/plugins/gitlab/tasks/pipeline_extractor.go
+++ b/backend/plugins/gitlab/tasks/pipeline_extractor.go
@@ -95,7 +95,7 @@ func ExtractApiPipelines(taskCtx plugin.SubTaskContext) errors.Error {
ConnectionId: data.Options.ConnectionId,
Type: data.RegexEnricher.ReturnNameIfMatched(devops.DEPLOYMENT, gitlabApiPipeline.Ref),
- Environment: data.RegexEnricher.ReturnNameIfMatched(devops.PRODUCTION, gitlabApiPipeline.Ref),
+ Environment: data.RegexEnricher.ReturnNameIfOmittedOrMatched(devops.PRODUCTION, gitlabApiPipeline.Ref),
IsDetailRequired: false,
}
diff --git a/backend/plugins/jenkins/e2e/builds_test.go b/backend/plugins/jenkins/e2e/builds_test.go
index aba601aba..948a4ab90 100644
--- a/backend/plugins/jenkins/e2e/builds_test.go
+++ b/backend/plugins/jenkins/e2e/builds_test.go
@@ -35,7 +35,6 @@ func TestJenkinsBuildsDataFlow(t *testing.T) {
regexEnricher := api.NewRegexEnricher()
_ = regexEnricher.TryAdd(devops.DEPLOYMENT, `test-sub-sub-dir\/devlake.*`)
- _ = regexEnricher.TryAdd(devops.PRODUCTION, `test-sub-sub-dir\/devlake.*`)
taskData := &tasks.JenkinsTaskData{
Options: &tasks.JenkinsOptions{
ConnectionId: 1,
@@ -102,9 +101,23 @@ func TestJenkinsBuildsDataFlow(t *testing.T) {
},
)
- dataflowTester.Subtask(tasks.ConvertBuildsToCICDMeta, taskData)
dataflowTester.Subtask(tasks.ConvertBuildReposMeta, taskData)
+ dataflowTester.Subtask(tasks.ConvertBuildsToCICDMeta, taskData)
+
+ // verify env when prod env is omitted
+ dataflowTester.Subtask(tasks.ConvertBuildsToCICDMeta, taskData)
+ dataflowTester.VerifyTable(
+ devops.CICDTask{},
+ "./snapshot_tables/cicd_tasks_no_prod_regex.csv",
+ e2ehelper.ColumnWithRawData(
+ "environment",
+ ),
+ )
+ // continue
+ _ = regexEnricher.TryAdd(devops.PRODUCTION, `test-sub-sub-dir\/devlake.*`)
+ dataflowTester.FlushTabler(&devops.CICDTask{})
+ dataflowTester.Subtask(tasks.ConvertBuildsToCICDMeta, taskData)
dataflowTester.VerifyTable(
devops.CICDTask{},
"./snapshot_tables/cicd_tasks.csv",
diff --git a/backend/plugins/jenkins/e2e/snapshot_tables/cicd_tasks_after_stages_no_prod_regex.csv b/backend/plugins/jenkins/e2e/snapshot_tables/cicd_tasks_after_stages_no_prod_regex.csv
new file mode 100644
index 000000000..8c5c203fa
--- /dev/null
+++ b/backend/plugins/jenkins/e2e/snapshot_tables/cicd_tasks_after_stages_no_prod_regex.csv
@@ -0,0 +1,15 @@
+id,environment,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
+jenkins:JenkinsStage:1:Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake#10:8,PRODUCTION,"{""ConnectionId"":1,""FullName"":""Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake""}",_raw_jenkins_api_stages,13577,
+jenkins:JenkinsStage:1:Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake#11:9,PRODUCTION,"{""ConnectionId"":1,""FullName"":""Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake""}",_raw_jenkins_api_stages,13578,
+jenkins:JenkinsStage:1:Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake#12:1,PRODUCTION,"{""ConnectionId"":1,""FullName"":""Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake""}",_raw_jenkins_api_stages,13579,
+jenkins:JenkinsStage:1:Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake#13:2,PRODUCTION,"{""ConnectionId"":1,""FullName"":""Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake""}",_raw_jenkins_api_stages,13580,
+jenkins:JenkinsStage:1:Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake#14:3,PRODUCTION,"{""ConnectionId"":1,""FullName"":""Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake""}",_raw_jenkins_api_stages,13581,
+jenkins:JenkinsStage:1:Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake#1:6,PRODUCTION,"{""ConnectionId"":1,""FullName"":""Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake""}",_raw_jenkins_api_stages,1,
+jenkins:JenkinsStage:1:Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake#2:7,PRODUCTION,"{""ConnectionId"":1,""FullName"":""Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake""}",_raw_jenkins_api_stages,2,
+jenkins:JenkinsStage:1:Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake#3:8,PRODUCTION,"{""ConnectionId"":1,""FullName"":""Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake""}",_raw_jenkins_api_stages,3,
+jenkins:JenkinsStage:1:Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake#4:9,PRODUCTION,"{""ConnectionId"":1,""FullName"":""Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake""}",_raw_jenkins_api_stages,4,
+jenkins:JenkinsStage:1:Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake#5:10,PRODUCTION,"{""ConnectionId"":1,""FullName"":""Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake""}",_raw_jenkins_api_stages,5,
+jenkins:JenkinsStage:1:Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake#6:11,PRODUCTION,"{""ConnectionId"":1,""FullName"":""Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake""}",_raw_jenkins_api_stages,6,
+jenkins:JenkinsStage:1:Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake#7:12,PRODUCTION,"{""ConnectionId"":1,""FullName"":""Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake""}",_raw_jenkins_api_stages,13574,
+jenkins:JenkinsStage:1:Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake#8:6,PRODUCTION,"{""ConnectionId"":1,""FullName"":""Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake""}",_raw_jenkins_api_stages,13575,
+jenkins:JenkinsStage:1:Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake#9:7,PRODUCTION,"{""ConnectionId"":1,""FullName"":""Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake""}",_raw_jenkins_api_stages,13576,
diff --git a/backend/plugins/jenkins/e2e/snapshot_tables/cicd_tasks_no_prod_regex.csv b/backend/plugins/jenkins/e2e/snapshot_tables/cicd_tasks_no_prod_regex.csv
new file mode 100644
index 000000000..c6af70c36
--- /dev/null
+++ b/backend/plugins/jenkins/e2e/snapshot_tables/cicd_tasks_no_prod_regex.csv
@@ -0,0 +1,8 @@
+id,environment,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
+jenkins:JenkinsBuild:1:Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake#17,PRODUCTION,"{""ConnectionId"":1,""FullName"":""Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake""}",_raw_jenkins_api_builds,124,
+jenkins:JenkinsBuild:1:Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake#21,PRODUCTION,"{""ConnectionId"":1,""FullName"":""Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake""}",_raw_jenkins_api_builds,94,
+jenkins:JenkinsBuild:1:Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake#23,PRODUCTION,"{""ConnectionId"":1,""FullName"":""Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake""}",_raw_jenkins_api_builds,96,
+jenkins:JenkinsBuild:1:Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake#27,PRODUCTION,"{""ConnectionId"":1,""FullName"":""Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake""}",_raw_jenkins_api_builds,123,
+jenkins:JenkinsBuild:1:Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake#31,PRODUCTION,"{""ConnectionId"":1,""FullName"":""Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake""}",_raw_jenkins_api_builds,93,
+jenkins:JenkinsBuild:1:Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake#37,PRODUCTION,"{""ConnectionId"":1,""FullName"":""Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake""}",_raw_jenkins_api_builds,122,
+jenkins:JenkinsBuild:1:Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake#47,PRODUCTION,"{""ConnectionId"":1,""FullName"":""Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake""}",_raw_jenkins_api_builds,121,
diff --git a/backend/plugins/jenkins/e2e/stages_test.go b/backend/plugins/jenkins/e2e/stages_test.go
index 8f76c028d..cbc33bc9a 100644
--- a/backend/plugins/jenkins/e2e/stages_test.go
+++ b/backend/plugins/jenkins/e2e/stages_test.go
@@ -34,7 +34,6 @@ func TestJenkinsStagesDataFlow(t *testing.T) {
regexEnricher := api.NewRegexEnricher()
_ = regexEnricher.TryAdd(devops.DEPLOYMENT, `gitlabAutoSync`)
- _ = regexEnricher.TryAdd(devops.PRODUCTION, `gitlabAutoSync`)
taskData := &tasks.JenkinsTaskData{
Options: &tasks.JenkinsOptions{
ConnectionId: 1,
@@ -75,6 +74,18 @@ func TestJenkinsStagesDataFlow(t *testing.T) {
),
)
+ // verify env when production regex is omitted
+ dataflowTester.FlushTabler(&devops.CICDTask{})
+ dataflowTester.Subtask(tasks.ConvertStagesMeta, taskData)
+ dataflowTester.VerifyTable(
+ devops.CICDTask{},
+ "./snapshot_tables/cicd_tasks_after_stages_no_prod_regex.csv",
+ e2ehelper.ColumnWithRawData(
+ "environment",
+ ),
+ )
+
+ _ = regexEnricher.TryAdd(devops.PRODUCTION, `gitlabAutoSync`)
dataflowTester.FlushTabler(&devops.CICDTask{})
dataflowTester.Subtask(tasks.ConvertStagesMeta, taskData)
dataflowTester.VerifyTable(
diff --git a/backend/plugins/jenkins/tasks/build_cicd_convertor.go b/backend/plugins/jenkins/tasks/build_cicd_convertor.go
index 13a3d2059..f95380979 100644
--- a/backend/plugins/jenkins/tasks/build_cicd_convertor.go
+++ b/backend/plugins/jenkins/tasks/build_cicd_convertor.go
@@ -103,7 +103,7 @@ func ConvertBuildsToCICD(taskCtx plugin.SubTaskContext) (err errors.Error) {
CreatedDate: jenkinsBuild.StartTime,
CicdScopeId: jobIdGen.Generate(jenkinsBuild.ConnectionId, data.Options.JobFullName),
Type: data.RegexEnricher.ReturnNameIfMatched(devops.DEPLOYMENT, jenkinsBuild.FullName),
- Environment: data.RegexEnricher.ReturnNameIfMatched(devops.PRODUCTION, jenkinsBuild.FullName),
+ Environment: data.RegexEnricher.ReturnNameIfOmittedOrMatched(devops.PRODUCTION, jenkinsBuild.FullName),
}
jenkinsPipeline.RawDataOrigin = jenkinsBuild.RawDataOrigin
results = append(results, jenkinsPipeline)
@@ -121,7 +121,7 @@ func ConvertBuildsToCICD(taskCtx plugin.SubTaskContext) (err errors.Error) {
FinishedDate: jenkinsPipelineFinishedDate,
CicdScopeId: jobIdGen.Generate(jenkinsBuild.ConnectionId, data.Options.JobFullName),
Type: data.RegexEnricher.ReturnNameIfMatched(devops.DEPLOYMENT, jenkinsBuild.FullName),
- Environment: data.RegexEnricher.ReturnNameIfMatched(devops.PRODUCTION, jenkinsBuild.FullName),
+ Environment: data.RegexEnricher.ReturnNameIfOmittedOrMatched(devops.PRODUCTION, jenkinsBuild.FullName),
PipelineId: buildIdGen.Generate(jenkinsBuild.ConnectionId, jenkinsBuild.FullName),
}
results = append(results, jenkinsTask)
diff --git a/backend/plugins/jenkins/tasks/stage_convertor.go b/backend/plugins/jenkins/tasks/stage_convertor.go
index 1a921b5b8..73fca37bc 100644
--- a/backend/plugins/jenkins/tasks/stage_convertor.go
+++ b/backend/plugins/jenkins/tasks/stage_convertor.go
@@ -131,7 +131,7 @@ func ConvertStages(taskCtx plugin.SubTaskContext) (err errors.Error) {
FinishedDate: jenkinsTaskFinishedDate,
CicdScopeId: jobIdGen.Generate(body.ConnectionId, data.Options.JobFullName),
Type: data.RegexEnricher.ReturnNameIfMatched(devops.DEPLOYMENT, body.Name),
- Environment: data.RegexEnricher.ReturnNameIfMatched(devops.PRODUCTION, body.Name),
+ Environment: data.RegexEnricher.ReturnNameIfOmittedOrMatched(devops.PRODUCTION, body.Name),
}
results = append(results, jenkinsTask)
return results, nil