You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by ab...@apache.org on 2022/09/27 09:09:05 UTC
[incubator-devlake] branch main updated: fix(jenkins): use prefix index of dataSource
This is an automated email from the ASF dual-hosted git repository.
abeizn pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new f7017d98 fix(jenkins): use prefix index of dataSource
f7017d98 is described below
commit f7017d98ca5db93c93457098e9f834b6acb6eb25
Author: Yingchu Chen <yi...@merico.dev>
AuthorDate: Tue Sep 27 15:52:06 2022 +0800
fix(jenkins): use prefix index of dataSource
---
plugins/dora/dora.go | 4 +++-
plugins/dora/e2e/env_enricher_test.go | 21 ++++++++++++++++++++-
.../e2e/raw_tables/lake_cicd_tasks_for_prefix.csv | 2 ++
.../e2e/snapshot_tables/lake_cicd_tasks_prefix.csv | 2 ++
plugins/dora/tasks/cicd_task_env_enricher.go | 10 +++++-----
plugins/dora/tasks/task_data.go | 8 ++++----
plugins/jenkins/api/blueprint.go | 2 +-
7 files changed, 37 insertions(+), 12 deletions(-)
diff --git a/plugins/dora/dora.go b/plugins/dora/dora.go
index 6c98f3c2..3d4e4003 100644
--- a/plugins/dora/dora.go
+++ b/plugins/dora/dora.go
@@ -31,14 +31,16 @@ func main() {
cmd := &cobra.Command{Use: "dora"}
repoId := cmd.Flags().StringP("repoId", "r", "", "repo id")
- _ = cmd.MarkFlagRequired("repoId")
+ //_ = cmd.MarkFlagRequired("repoId")
productionPattern := cmd.Flags().String("productionPattern", "(?i)production", "environment production name")
stagingPattern := cmd.Flags().String("stagingPattern", "(?i)stag", "environment stag name")
testingPattern := cmd.Flags().String("testingPattern", "(?i)test", "environment test name")
+ prefix := cmd.Flags().StringP("prefix", "p", "", "id prefix")
cmd.Run = func(cmd *cobra.Command, args []string) {
runner.DirectRun(cmd, args, PluginEntry, map[string]interface{}{
"repoId": *repoId,
+ "prefix": *prefix,
"productionPattern": *productionPattern,
"stagingPattern": *stagingPattern,
"testingPattern": *testingPattern,
diff --git a/plugins/dora/e2e/env_enricher_test.go b/plugins/dora/e2e/env_enricher_test.go
index 48bc68be..cc108162 100644
--- a/plugins/dora/e2e/env_enricher_test.go
+++ b/plugins/dora/e2e/env_enricher_test.go
@@ -48,10 +48,29 @@ func TestEnrichEnvDataFlow(t *testing.T) {
dataflowTester.ImportCsvIntoTabler("./raw_tables/lake_cicd_pipeline_commits.csv", &devops.CiCDPipelineCommit{})
dataflowTester.ImportCsvIntoTabler("./raw_tables/lake_cicd_tasks.csv", &devops.CICDTask{})
- // verify converter
+ // verify enrich with repoId
dataflowTester.Subtask(tasks.EnrichTaskEnvMeta, taskData)
dataflowTester.VerifyTableWithOptions(&devops.CICDTask{}, e2ehelper.TableOptions{
CSVRelPath: "./snapshot_tables/lake_cicd_tasks.csv",
IgnoreTypes: []interface{}{common.NoPKModel{}},
})
+
+ // verify enrich with prefix
+ dataflowTester.FlushTabler(&devops.CICDTask{})
+ dataflowTester.ImportCsvIntoTabler("./raw_tables/lake_cicd_tasks_for_prefix.csv", &devops.CICDTask{})
+ taskDataPrefix := &tasks.DoraTaskData{
+ Options: &tasks.DoraOptions{
+ TransformationRules: tasks.TransformationRules{
+ Prefix: "jenkins",
+ ProductionPattern: "(?i)deploy",
+ StagingPattern: "(?i)stag",
+ TestingPattern: "(?i)test",
+ },
+ },
+ }
+ dataflowTester.Subtask(tasks.EnrichTaskEnvMeta, taskDataPrefix)
+ dataflowTester.VerifyTableWithOptions(&devops.CICDTask{}, e2ehelper.TableOptions{
+ CSVRelPath: "./snapshot_tables/lake_cicd_tasks_prefix.csv",
+ IgnoreTypes: []interface{}{common.NoPKModel{}},
+ })
}
diff --git a/plugins/dora/e2e/raw_tables/lake_cicd_tasks_for_prefix.csv b/plugins/dora/e2e/raw_tables/lake_cicd_tasks_for_prefix.csv
new file mode 100644
index 00000000..bd241115
--- /dev/null
+++ b/plugins/dora/e2e/raw_tables/lake_cicd_tasks_for_prefix.csv
@@ -0,0 +1,2 @@
+id,created_at,updated_at,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark,name,pipeline_id,result,status,type,duration_sec,started_date,finished_date,environment
+jenkins:GithubJob:1:8269872730,2022-09-15 03:20:03.867,2022-09-15 03:20:03.867,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_jobs,21,"",deploy,jenkins:GithubRun:1:3022348786,SUCCESS,DONE,"",1006,2022-09-09 12:21:30,2022-09-09 12:38:16,""
diff --git a/plugins/dora/e2e/snapshot_tables/lake_cicd_tasks_prefix.csv b/plugins/dora/e2e/snapshot_tables/lake_cicd_tasks_prefix.csv
new file mode 100644
index 00000000..482b9f9b
--- /dev/null
+++ b/plugins/dora/e2e/snapshot_tables/lake_cicd_tasks_prefix.csv
@@ -0,0 +1,2 @@
+id,name,pipeline_id,result,status,type,environment,duration_sec,started_date,finished_date
+jenkins:GithubJob:1:8269872730,deploy,jenkins:GithubRun:1:3022348786,SUCCESS,DONE,,PRODUCTION,1006,2022-09-09T12:21:30.000+00:00,2022-09-09T12:38:16.000+00:00
diff --git a/plugins/dora/tasks/cicd_task_env_enricher.go b/plugins/dora/tasks/cicd_task_env_enricher.go
index c976b85b..6d2ee16d 100644
--- a/plugins/dora/tasks/cicd_task_env_enricher.go
+++ b/plugins/dora/tasks/cicd_task_env_enricher.go
@@ -19,6 +19,7 @@ package tasks
import (
"database/sql"
+ "fmt"
"reflect"
"regexp"
@@ -46,7 +47,7 @@ func EnrichTasksEnv(taskCtx core.SubTaskContext) (err errors.Error) {
// TODO: STAGE 2
// stagingNamePattern := data.Options.StagingPattern
// testingNamePattern := data.Options.TestingPattern
- dataSource := data.Options.DataSoure
+ prefix := data.Options.Prefix
productionNameRegexp, errRegexp := regexp.Compile(productionNamePattern)
if errRegexp != nil {
@@ -63,17 +64,16 @@ func EnrichTasksEnv(taskCtx core.SubTaskContext) (err errors.Error) {
// }
var cursor *sql.Rows
- if len(dataSource) == 0 {
+ if len(prefix) == 0 {
cursor, err = db.Cursor(
dal.From(&devops.CICDTask{}),
dal.Join("left join cicd_pipeline_commits cpr on cpr.repo_id = ? and cicd_tasks.pipeline_id = cpr.pipeline_id ", repoId),
dal.Where("status=? ", devops.DONE))
} else {
-
+ likeString := fmt.Sprintf(`%s:%s`, prefix, "%")
cursor, err = db.Cursor(
dal.From(&devops.CICDTask{}),
- dal.Join("left join cicd_pipeline_commits cpr on cpr.repo_id = ? and cicd_tasks.pipeline_id = cpr.pipeline_id ", repoId),
- dal.Where("status=? and SUBSTRING_INDEX(id, ':', 1) in ? ", devops.DONE, dataSource))
+ dal.Where("status=? and id like ? ", devops.DONE, likeString))
}
if err != nil {
return err
diff --git a/plugins/dora/tasks/task_data.go b/plugins/dora/tasks/task_data.go
index 45355986..978943a6 100644
--- a/plugins/dora/tasks/task_data.go
+++ b/plugins/dora/tasks/task_data.go
@@ -26,10 +26,10 @@ type DoraApiParams struct {
}
type TransformationRules struct {
- ProductionPattern string `mapstructure:"productionPattern" json:"productionPattern"`
- StagingPattern string `mapstructure:"stagingPattern" json:"stagingPattern"`
- TestingPattern string `mapstructure:"testingPattern" json:"testingPattern"`
- DataSoure []string `mapstructure:"dataSoure" json:"dataSoure"`
+ ProductionPattern string `mapstructure:"productionPattern" json:"productionPattern"`
+ StagingPattern string `mapstructure:"stagingPattern" json:"stagingPattern"`
+ TestingPattern string `mapstructure:"testingPattern" json:"testingPattern"`
+ Prefix string `mapstructure:"prefix" json:"prefix"`
}
type DoraOptions struct {
diff --git a/plugins/jenkins/api/blueprint.go b/plugins/jenkins/api/blueprint.go
index 1508b119..37fae67c 100644
--- a/plugins/jenkins/api/blueprint.go
+++ b/plugins/jenkins/api/blueprint.go
@@ -76,7 +76,7 @@ func MakePipelinePlan(subtaskMetas []core.SubTaskMeta, connectionId uint64, scop
}
doraOption := make(map[string]interface{})
doraOption["tasks"] = []string{"EnrichTaskEnv"}
- doraOption["dataSource"] = []string{"jenkins"}
+ doraOption["prefix"] = "jenkins"
doraRules := make(map[string]interface{})
doraRules["productionPattern"] = productionPattern
doraOption["transformationRules"] = doraRules