You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by zh...@apache.org on 2023/01/05 10:34:25 UTC
[incubator-devlake] branch main updated: fix(gitlab): update job convertor (#4113)
This is an automated email from the ASF dual-hosted git repository.
zhangliang2022 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new ccc2e5444 fix(gitlab): update job convertor (#4113)
ccc2e5444 is described below
commit ccc2e5444e198e1c786324c45a35520068e8b273
Author: Warren Chen <yi...@merico.dev>
AuthorDate: Thu Jan 5 18:34:19 2023 +0800
fix(gitlab): update job convertor (#4113)
---
plugins/gitlab/e2e/job_test.go | 9 ++++++---
plugins/gitlab/e2e/snapshot_tables/cicd_tasks.csv | 6 +++---
plugins/gitlab/tasks/job_convertor.go | 22 ++++++----------------
3 files changed, 15 insertions(+), 22 deletions(-)
diff --git a/plugins/gitlab/e2e/job_test.go b/plugins/gitlab/e2e/job_test.go
index 78f7ca701..c27a26dbb 100644
--- a/plugins/gitlab/e2e/job_test.go
+++ b/plugins/gitlab/e2e/job_test.go
@@ -36,9 +36,12 @@ func TestGitlabJobDataFlow(t *testing.T) {
taskData := &tasks.GitlabTaskData{
Options: &tasks.GitlabOptions{
- ConnectionId: 1,
- ProjectId: 44,
- GitlabTransformationRule: new(models.GitlabTransformationRule),
+ ConnectionId: 1,
+ ProjectId: 44,
+ GitlabTransformationRule: &models.GitlabTransformationRule{
+ DeploymentPattern: "(?i)compile",
+ ProductionPattern: "(?i)compile",
+ },
},
}
// import raw data table
diff --git a/plugins/gitlab/e2e/snapshot_tables/cicd_tasks.csv b/plugins/gitlab/e2e/snapshot_tables/cicd_tasks.csv
index f12e185d7..2e30b786b 100644
--- a/plugins/gitlab/e2e/snapshot_tables/cicd_tasks.csv
+++ b/plugins/gitlab/e2e/snapshot_tables/cicd_tasks.csv
@@ -1,14 +1,14 @@
id,name,pipeline_id,result,status,type,environment,duration_sec,started_date,finished_date,cicd_scope_id
-gitlab:GitlabJob:1:100,compile,gitlab:GitlabPipeline:1:24,SUCCESS,DONE,,,2,2022-07-25T15:06:57.051+00:00,2022-07-25T15:06:59.885+00:00,gitlab:GitlabProject:1:44
+gitlab:GitlabJob:1:100,compile,gitlab:GitlabPipeline:1:24,SUCCESS,DONE,DEPLOYMENT,PRODUCTION,2,2022-07-25T15:06:57.051+00:00,2022-07-25T15:06:59.885+00:00,gitlab:GitlabProject:1:44
gitlab:GitlabJob:1:101,format,gitlab:GitlabPipeline:1:25,SUCCESS,DONE,,,3,2022-07-25T15:13:37.206+00:00,2022-07-25T15:13:40.246+00:00,gitlab:GitlabProject:1:44
gitlab:GitlabJob:1:102,format,gitlab:GitlabPipeline:1:26,SUCCESS,DONE,,,2,2022-07-25T15:30:22.560+00:00,2022-07-25T15:30:25.315+00:00,gitlab:GitlabProject:1:44
gitlab:GitlabJob:1:103,format,gitlab:GitlabPipeline:1:27,SUCCESS,DONE,,,2,2022-07-25T15:30:55.671+00:00,2022-07-25T15:30:58.650+00:00,gitlab:GitlabProject:1:44
gitlab:GitlabJob:1:104,format,gitlab:GitlabPipeline:1:28,SUCCESS,DONE,,,2,2022-07-25T15:32:04.954+00:00,2022-07-25T15:32:07.726+00:00,gitlab:GitlabProject:1:44
-gitlab:GitlabJob:1:105,compile,gitlab:GitlabPipeline:1:28,FAILURE,DONE,,,3,2022-07-25T15:32:07.953+00:00,2022-07-25T15:32:11.077+00:00,gitlab:GitlabProject:1:44
+gitlab:GitlabJob:1:105,compile,gitlab:GitlabPipeline:1:28,FAILURE,DONE,DEPLOYMENT,PRODUCTION,3,2022-07-25T15:32:07.953+00:00,2022-07-25T15:32:11.077+00:00,gitlab:GitlabProject:1:44
gitlab:GitlabJob:1:106,format,gitlab:GitlabPipeline:1:29,SUCCESS,DONE,,,2,2022-07-25T15:33:26.382+00:00,2022-07-25T15:33:29.356+00:00,gitlab:GitlabProject:1:44
gitlab:GitlabJob:1:107,format,gitlab:GitlabPipeline:1:30,SUCCESS,DONE,,,2,2022-07-25T15:34:23.665+00:00,2022-07-25T15:34:26.392+00:00,gitlab:GitlabProject:1:44
gitlab:GitlabJob:1:108,format,gitlab:GitlabPipeline:1:31,SUCCESS,DONE,,,2,2022-07-25T15:35:11.707+00:00,2022-07-25T15:35:14.224+00:00,gitlab:GitlabProject:1:44
-gitlab:GitlabJob:1:109,compile,gitlab:GitlabPipeline:1:31,SUCCESS,DONE,,,3,2022-07-25T15:35:14.724+00:00,2022-07-25T15:35:17.828+00:00,gitlab:GitlabProject:1:44
+gitlab:GitlabJob:1:109,compile,gitlab:GitlabPipeline:1:31,SUCCESS,DONE,DEPLOYMENT,PRODUCTION,3,2022-07-25T15:35:14.724+00:00,2022-07-25T15:35:17.828+00:00,gitlab:GitlabProject:1:44
gitlab:GitlabJob:1:110,format,gitlab:GitlabPipeline:1:32,SUCCESS,DONE,,,2,2022-07-25T15:36:18.097+00:00,2022-07-25T15:36:20.954+00:00,gitlab:GitlabProject:1:44
gitlab:GitlabJob:1:111,format,gitlab:GitlabPipeline:1:33,SUCCESS,DONE,,,3,2022-07-25T15:38:03.463+00:00,2022-07-25T15:38:06.467+00:00,gitlab:GitlabProject:1:44
gitlab:GitlabJob:1:112,format,gitlab:GitlabPipeline:1:34,SUCCESS,DONE,,,3,2022-07-25T21:19:14.509+00:00,2022-07-25T21:19:17.811+00:00,gitlab:GitlabProject:1:44
diff --git a/plugins/gitlab/tasks/job_convertor.go b/plugins/gitlab/tasks/job_convertor.go
index 1c95f0a46..32bbe4e08 100644
--- a/plugins/gitlab/tasks/job_convertor.go
+++ b/plugins/gitlab/tasks/job_convertor.go
@@ -18,9 +18,6 @@ limitations under the License.
package tasks
import (
- "reflect"
- "regexp"
-
"github.com/apache/incubator-devlake/errors"
"github.com/apache/incubator-devlake/models/domainlayer"
"github.com/apache/incubator-devlake/models/domainlayer/devops"
@@ -29,6 +26,7 @@ import (
"github.com/apache/incubator-devlake/plugins/core/dal"
gitlabModels "github.com/apache/incubator-devlake/plugins/gitlab/models"
"github.com/apache/incubator-devlake/plugins/helper"
+ "reflect"
)
var ConvertJobMeta = core.SubTaskMeta{
@@ -42,15 +40,10 @@ var ConvertJobMeta = core.SubTaskMeta{
func ConvertJobs(taskCtx core.SubTaskContext) (err errors.Error) {
db := taskCtx.GetDal()
data := taskCtx.GetData().(*GitlabTaskData)
-
- var deployTagRegexp *regexp.Regexp
deploymentPattern := data.Options.DeploymentPattern
- if len(deploymentPattern) > 0 {
- deployTagRegexp, err = errors.Convert01(regexp.Compile(deploymentPattern))
- if err != nil {
- return errors.Default.Wrap(err, "regexp compile deploymentPattern failed")
- }
- }
+ productionPattern := data.Options.ProductionPattern
+ regexEnricher := helper.NewRegexEnricher()
+ err = regexEnricher.AddRegexp(deploymentPattern, productionPattern)
cursor, err := db.Cursor(dal.From(gitlabModels.GitlabJob{}),
dal.Where("project_id = ? and connection_id = ?", data.Options.ProjectId, data.Options.ConnectionId))
@@ -106,11 +99,8 @@ func ConvertJobs(taskCtx core.SubTaskContext) (err errors.Error) {
FinishedDate: gitlabJob.FinishedAt,
CicdScopeId: projectIdGen.Generate(data.Options.ConnectionId, gitlabJob.ProjectId),
}
- if deployTagRegexp != nil {
- if deployFlag := deployTagRegexp.FindString(gitlabJob.Name); deployFlag != "" {
- domainJob.Type = devops.DEPLOYMENT
- }
- }
+ domainJob.Type = regexEnricher.GetEnrichResult(deploymentPattern, gitlabJob.Name, devops.DEPLOYMENT)
+ domainJob.Environment = regexEnricher.GetEnrichResult(productionPattern, gitlabJob.Name, devops.PRODUCTION)
return []interface{}{
domainJob,