You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by ab...@apache.org on 2022/08/05 12:33:24 UTC
[incubator-devlake] 02/02: feat(jenkins): stage convertor
This is an automated email from the ASF dual-hosted git repository.
abeizn pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit ba5c53ba6351b255a08c96b033b70ab6e221f536
Author: Yingchu Chen <yi...@merico.dev>
AuthorDate: Fri Aug 5 10:00:29 2022 +0800
feat(jenkins): stage convertor
---
docker-compose.yml | 6 +-
plugins/jenkins/impl/impl.go | 1 +
.../migrationscripts/20220714_add_init_tables.go | 53 ++++----
...d_convertor_cicd.go => build_cicd_convertor.go} | 30 +++--
plugins/jenkins/tasks/build_enricher.go | 21 +++-
plugins/jenkins/tasks/stage_collector.go | 2 +-
plugins/jenkins/tasks/stage_convertor.go | 139 +++++++++++++++++++++
7 files changed, 206 insertions(+), 46 deletions(-)
diff --git a/docker-compose.yml b/docker-compose.yml
index f15d5f67..aee2117d 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -42,7 +42,7 @@ services:
# POSTGRES_PASSWORD: merico
grafana:
- image: mericodev/devlake-dashboard:v0.12.1-beta
+ image: mericodev/devlake-dashboard:latest
build:
context: grafana/
ports:
@@ -60,7 +60,7 @@ services:
- mysql
devlake:
- image: mericodev/devlake:v0.12.1-beta
+ image: mericodev/devlake:latest
build:
context: "."
args:
@@ -75,7 +75,7 @@ services:
- mysql
config-ui:
- image: mericodev/devlake-config-ui:v0.12.1-beta
+ image: mericodev/devlake-config-ui:latest
build:
context: "config-ui"
ports:
diff --git a/plugins/jenkins/impl/impl.go b/plugins/jenkins/impl/impl.go
index 30036228..299a43a1 100644
--- a/plugins/jenkins/impl/impl.go
+++ b/plugins/jenkins/impl/impl.go
@@ -59,6 +59,7 @@ func (plugin Jenkins) SubTaskMetas() []core.SubTaskMeta {
tasks.ExtractApiStagesMeta,
tasks.EnrichApiBuildsMeta,
tasks.ConvertBuildsToCICDMeta,
+ tasks.ConvertStagesMeta,
tasks.ConvertJobsMeta,
tasks.ConvertBuildsMeta,
}
diff --git a/plugins/jenkins/models/migrationscripts/20220714_add_init_tables.go b/plugins/jenkins/models/migrationscripts/20220714_add_init_tables.go
index 79055179..5830eec2 100644
--- a/plugins/jenkins/models/migrationscripts/20220714_add_init_tables.go
+++ b/plugins/jenkins/models/migrationscripts/20220714_add_init_tables.go
@@ -19,6 +19,9 @@ package migrationscripts
import (
"context"
+ "github.com/apache/incubator-devlake/config"
+ "github.com/apache/incubator-devlake/plugins/core"
+ "gorm.io/gorm/clause"
"github.com/apache/incubator-devlake/plugins/jenkins/models/migrationscripts/archived"
"gorm.io/gorm"
@@ -40,37 +43,37 @@ func (*addInitTables) Up(ctx context.Context, db *gorm.DB) error {
err = db.Migrator().AutoMigrate(
&archived.JenkinsJob{},
&archived.JenkinsBuild{},
- //&archived.JenkinsConnection{},
+ &archived.JenkinsConnection{},
)
if err != nil {
return err
}
- //v := config.GetConfig()
+ v := config.GetConfig()
- //encKey := v.GetString("ENCODE_KEY")
- //endPoint := v.GetString("JENKINS_ENDPOINT")
- //useName := v.GetString("JENKINS_USERNAME")
- //passWord := v.GetString("JENKINS_PASSWORD")
- //if encKey == "" || endPoint == "" || useName == "" || passWord == "" {
- // return nil
- //}
- //conn := &archived.JenkinsConnection{}
- //conn.Name = "init jenkins connection"
- //conn.ID = 1
- //conn.Endpoint = endPoint
- //conn.Proxy = v.GetString("JENKINS_PROXY")
- //conn.RateLimitPerHour = v.GetInt("JENKINS_API_REQUESTS_PER_HOUR")
- //conn.Username = useName
- //conn.Password, err = core.Encrypt(encKey, passWord)
- //if err != nil {
- // return err
- //}
- //err = db.Clauses(clause.OnConflict{DoNothing: true}).Create(conn).Error
- //
- //if err != nil {
- // return err
- //}
+ encKey := v.GetString("ENCODE_KEY")
+ endPoint := v.GetString("JENKINS_ENDPOINT")
+ useName := v.GetString("JENKINS_USERNAME")
+ passWord := v.GetString("JENKINS_PASSWORD")
+ if encKey == "" || endPoint == "" || useName == "" || passWord == "" {
+ return nil
+ }
+ conn := &archived.JenkinsConnection{}
+ conn.Name = "init jenkins connection"
+ conn.ID = 1
+ conn.Endpoint = endPoint
+ conn.Proxy = v.GetString("JENKINS_PROXY")
+ conn.RateLimitPerHour = v.GetInt("JENKINS_API_REQUESTS_PER_HOUR")
+ conn.Username = useName
+ conn.Password, err = core.Encrypt(encKey, passWord)
+ if err != nil {
+ return err
+ }
+ err = db.Clauses(clause.OnConflict{DoNothing: true}).Create(conn).Error
+
+ if err != nil {
+ return err
+ }
return nil
}
diff --git a/plugins/jenkins/tasks/build_convertor_cicd.go b/plugins/jenkins/tasks/build_cicd_convertor.go
similarity index 89%
rename from plugins/jenkins/tasks/build_convertor_cicd.go
rename to plugins/jenkins/tasks/build_cicd_convertor.go
index a227fdc9..3fe3d645 100644
--- a/plugins/jenkins/tasks/build_convertor_cicd.go
+++ b/plugins/jenkins/tasks/build_cicd_convertor.go
@@ -46,7 +46,7 @@ type JenkinsBuildWithRepo struct {
Building bool
Branch string `gorm:"type:varchar(255)"`
RepoUrl string `gorm:"type:varchar(255)"`
- StageCount int
+ HasStages bool
common.NoPKModel
}
@@ -64,8 +64,9 @@ func ConvertBuildsToCICD(taskCtx core.SubTaskContext) error {
clauses := []dal.Clause{
dal.Select(`tjb.connection_id, tjb.duration, tjb.display_name, tjb.estimated_duration, tjb.number,
+ tjb._raw_data_remark, tjb._raw_data_id, tjb._raw_data_table, tjb._raw_data_params,
tjb.result, tjb.timestamp, tjb.start_time, tjbr.commit_sha, tjb.type, tjb.class,
- tjb.triggered_by, tjb.building, tjbr.branch, tjbr.repo_url`),
+ tjb.triggered_by, tjb.building, tjbr.branch, tjbr.repo_url, tjb.has_stages`),
dal.From("_tool_jenkins_builds tjb"),
dal.Join("left join _tool_jenkins_build_repos tjbr on tjbr.build_name = tjb.display_name"),
dal.Where("tjb.connection_id = ?", data.Options.ConnectionId),
@@ -76,9 +77,6 @@ func ConvertBuildsToCICD(taskCtx core.SubTaskContext) error {
}
defer cursor.Close()
- //jobIdGen := didgen.NewDomainIdGenerator(&models.JenkinsJob{})
- //buildIdGen := didgen.NewDomainIdGenerator(&models.JenkinsBuild{})
-
converter, err := helper.NewDataConverter(helper.DataConverterArgs{
InputRowType: reflect.TypeOf(JenkinsBuildWithRepo{}),
Input: cursor,
@@ -124,25 +122,26 @@ func ConvertBuildsToCICD(taskCtx core.SubTaskContext) error {
Result: jenkinsPipelineResult,
Status: jenkinsPipelineStatus,
FinishedDate: jenkinsPipelineFinishedDate,
- //Type: "",
- DurationSec: uint64(durationSec),
- CreatedDate: jenkinsBuildWithRepo.StartTime,
+ Type: "CI/CD",
+ DurationSec: uint64(durationSec),
+ CreatedDate: jenkinsBuildWithRepo.StartTime,
}
if jenkinsPipelineFinishedDate != nil {
}
+ jenkinsPipeline.RawDataOrigin = jenkinsBuildWithRepo.RawDataOrigin
results = append(results, jenkinsPipeline)
}
- if jenkinsBuildWithRepo.StageCount == 0 {
+ if !jenkinsBuildWithRepo.HasStages {
jenkinsTask := &devops.CICDTask{
DomainEntity: domainlayer.DomainEntity{
Id: fmt.Sprintf("%s:%s:%d:%s", "jenkins", "JenkinsTask", jenkinsBuildWithRepo.ConnectionId,
jenkinsBuildWithRepo.DisplayName),
},
- Name: jenkinsBuildWithRepo.DisplayName,
- Result: jenkinsPipelineResult,
- Status: jenkinsPipelineStatus,
- //Type: "",
+ Name: jenkinsBuildWithRepo.DisplayName,
+ Result: jenkinsPipelineResult,
+ Status: jenkinsPipelineStatus,
+ Type: "CI/CD",
DurationSec: uint64(durationSec),
StatedDate: jenkinsBuildWithRepo.StartTime,
FinishedDate: jenkinsPipelineFinishedDate,
@@ -150,9 +149,7 @@ func ConvertBuildsToCICD(taskCtx core.SubTaskContext) error {
if jenkinsBuildWithRepo.TriggeredBy != "" {
tmp := make([]*JenkinsBuildWithRepo, 0)
clauses := []dal.Clause{
- dal.Select(`tjb.connection_id, tjb.duration, tjb.display_name, tjb.estimated_duration, tjb.number,
- tjb.result, tjb.timestamp, tjbr.commit_sha, tjb.type, tjb.class,
- tjb.triggered_by, tjb.building, tjbr.branch, tjbr.repo_url`),
+ dal.Select(`tjb.display_name, tjb.result, tjb.timestamp, tjbr.commit_sha`),
dal.From("_tool_jenkins_builds tjb"),
dal.Join("left join _tool_jenkins_build_repos tjbr on tjbr.build_name = tjb.display_name"),
dal.Where("tjb.connection_id = ? and tjb.display_name = ?", data.Options.ConnectionId, jenkinsBuildWithRepo.TriggeredBy),
@@ -169,6 +166,7 @@ func ConvertBuildsToCICD(taskCtx core.SubTaskContext) error {
jenkinsTask.PipelineId = fmt.Sprintf("%s:%s:%d:%s:%s", "jenkins", "JenkinsPipeline", jenkinsBuildWithRepo.ConnectionId,
jenkinsBuildWithRepo.CommitSha, jenkinsBuildWithRepo.DisplayName)
}
+ jenkinsTask.RawDataOrigin = jenkinsBuildWithRepo.RawDataOrigin
results = append(results, jenkinsTask)
}
diff --git a/plugins/jenkins/tasks/build_enricher.go b/plugins/jenkins/tasks/build_enricher.go
index ceed2726..76efb5fc 100644
--- a/plugins/jenkins/tasks/build_enricher.go
+++ b/plugins/jenkins/tasks/build_enricher.go
@@ -21,6 +21,8 @@ import (
"github.com/apache/incubator-devlake/plugins/core"
"github.com/apache/incubator-devlake/plugins/core/dal"
"github.com/apache/incubator-devlake/plugins/jenkins/models"
+ "strconv"
+ "strings"
)
// this struct should be moved to `gitub_api_common.go`
@@ -43,6 +45,9 @@ func EnrichApiBuilds(taskCtx core.SubTaskContext) error {
dal.Groupby("build_name"),
}
cursor, err := db.Cursor(clauses...)
+ defer cursor.Close()
+ taskCtx.SetProgress(0, -1)
+
for cursor.Next() {
var buildName string
err = cursor.Scan(&buildName)
@@ -53,11 +58,25 @@ func EnrichApiBuilds(taskCtx core.SubTaskContext) error {
continue
}
build := &models.JenkinsBuild{}
+ build.ConnectionId = data.Options.ConnectionId
+ str := strings.Split(buildName, "#")
+ build.JobName = strings.TrimSpace(str[0])
+ number, err := strconv.Atoi(strings.TrimSpace(str[1]))
+ if err != nil {
+ return err
+ }
+ build.Number = int64(number)
+ err = db.First(build)
+ if err != nil {
+ return err
+ }
build.HasStages = true
- err = db.Update(&models.JenkinsBuild{}, dal.Where("connection_id = ? and build_name = ?", data.Options.ConnectionId, buildName))
+
+ err = db.Update(build)
if err != nil {
return err
}
+ taskCtx.IncProgress(1)
}
return nil
}
diff --git a/plugins/jenkins/tasks/stage_collector.go b/plugins/jenkins/tasks/stage_collector.go
index 19f6a83f..a3d05928 100644
--- a/plugins/jenkins/tasks/stage_collector.go
+++ b/plugins/jenkins/tasks/stage_collector.go
@@ -50,7 +50,7 @@ func CollectApiStages(taskCtx core.SubTaskContext) error {
clauses := []dal.Clause{
dal.Select("tjb.job_name,tjb.number, tjb.display_name"),
dal.From("_tool_jenkins_builds tjb"),
- dal.Where(`tjb.connection_id = ? and tjb.type = ?`,
+ dal.Where(`tjb.connection_id = ? and tjb.class = ?`,
data.Options.ConnectionId, "WorkflowRun"),
}
diff --git a/plugins/jenkins/tasks/stage_convertor.go b/plugins/jenkins/tasks/stage_convertor.go
new file mode 100644
index 00000000..e3e1bca8
--- /dev/null
+++ b/plugins/jenkins/tasks/stage_convertor.go
@@ -0,0 +1,139 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "fmt"
+ "github.com/apache/incubator-devlake/models/common"
+ "github.com/apache/incubator-devlake/models/domainlayer"
+ "github.com/apache/incubator-devlake/models/domainlayer/devops"
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/core/dal"
+ "github.com/apache/incubator-devlake/plugins/helper"
+ "reflect"
+ "time"
+)
+
+type JenkinsBuildWithRepoStage struct {
+ // collected fields
+ ConnectionId uint64 `gorm:"primaryKey"`
+ ID string `json:"id" gorm:"primaryKey;type:varchar(255)"`
+ Name string `json:"name" gorm:"type:varchar(255)"`
+ ExecNode string `json:"execNode" gorm:"type:varchar(255)"`
+ CommitSha string `gorm:"type:varchar(255)"`
+ Result string // Result
+ Status string `json:"status" gorm:"type:varchar(255)"`
+ StartTimeMillis int64 `json:"startTimeMillis"`
+ DurationMillis int `json:"durationMillis"`
+ PauseDurationMillis int `json:"pauseDurationMillis"`
+ Type string `gorm:"index;type:varchar(255)"`
+ BuildName string `gorm:"primaryKey;type:varchar(255)"`
+ Branch string `gorm:"type:varchar(255)"`
+ RepoUrl string `gorm:"type:varchar(255)"`
+ common.NoPKModel
+}
+
+var ConvertStagesMeta = core.SubTaskMeta{
+ Name: "convertStages",
+ EntryPoint: ConvertStages,
+ EnabledByDefault: true,
+ Description: "convert jenkins_stages",
+ DomainTypes: []string{core.DOMAIN_TYPE_CICD},
+}
+
+func ConvertStages(taskCtx core.SubTaskContext) error {
+ db := taskCtx.GetDal()
+ data := taskCtx.GetData().(*JenkinsTaskData)
+ clauses := []dal.Clause{
+ dal.Select(`tjb.connection_id, tjs.build_name, tjs.name, tjs._raw_data_remark,
+ tjs._raw_data_id, tjs._raw_data_table, tjs._raw_data_params,
+ tjs.status, tjs.start_time_millis, tjs.duration_millis,
+ tjs.pause_duration_millis, tjs.type,
+ tjbr.commit_sha, tjb.class,
+ tjb.triggered_by, tjb.building, tjbr.branch, tjbr.repo_url`),
+ dal.From("_tool_jenkins_builds tjb"),
+ dal.Join("left join _tool_jenkins_build_repos tjbr on tjbr.build_name = tjb.display_name"),
+ dal.Join("left join _tool_jenkins_stages tjs on tjs.build_name = tjb.display_name"),
+ dal.Where("tjb.connection_id = ? ", data.Options.ConnectionId),
+ }
+
+ cursor, err := db.Cursor(clauses...)
+ if err != nil {
+ return err
+ }
+ defer cursor.Close()
+
+ convertor, err := helper.NewDataConverter(helper.DataConverterArgs{
+ InputRowType: reflect.TypeOf(JenkinsBuildWithRepoStage{}),
+ Input: cursor,
+ RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
+ Params: JenkinsApiParams{
+ ConnectionId: data.Options.ConnectionId,
+ },
+ Ctx: taskCtx,
+ Table: RAW_STAGE_TABLE,
+ },
+ Convert: func(inputRow interface{}) ([]interface{}, error) {
+ body := inputRow.(*JenkinsBuildWithRepoStage)
+ if body.Name == "" {
+ return nil, err
+ }
+ durationSec := int64(body.DurationMillis / 1000)
+ jenkinsTaskResult := ""
+ jenkinsTaskStatus := "DONE"
+ var jenkinsTaskFinishedDate *time.Time
+ results := make([]interface{}, 0)
+ if body.Result == "SUCCESS" {
+ jenkinsTaskResult = devops.SUCCESS
+ } else if body.Result == "FAILED" {
+ jenkinsTaskResult = devops.FAILURE
+ } else {
+ jenkinsTaskResult = devops.ABORT
+ }
+ startedDate := time.Unix(body.StartTimeMillis/1000, 0)
+ finishedDate := startedDate.Add(time.Duration(durationSec * int64(time.Second)))
+ jenkinsTaskFinishedDate = &finishedDate
+
+ jenkinsTask := &devops.CICDTask{
+ DomainEntity: domainlayer.DomainEntity{
+ Id: fmt.Sprintf("%s:%s:%d:%s:%s", "jenkins", "JenkinsTask", body.ConnectionId,
+ body.BuildName, body.Name),
+ },
+ Name: body.Name,
+ PipelineId: fmt.Sprintf("%s:%s:%d:%s:%s", "jenkins", "JenkinsPipeline", body.ConnectionId,
+ body.CommitSha, body.BuildName),
+ Result: jenkinsTaskResult,
+ Status: jenkinsTaskStatus,
+ Type: "CI/CD",
+ DurationSec: uint64(body.DurationMillis / 1000),
+ StatedDate: time.Unix(durationSec, 0),
+ FinishedDate: jenkinsTaskFinishedDate,
+ }
+ jenkinsTask.RawDataOrigin = body.RawDataOrigin
+
+ results = append(results, jenkinsTask)
+ return results, nil
+ },
+ })
+
+ if err != nil {
+ return err
+ }
+
+ return convertor.Execute()
+}