You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by mi...@apache.org on 2024/09/20 04:22:41 UTC
(incubator-devlake) 04/39: feat: jenkins dora config (#8033)
This is an automated email from the ASF dual-hosted git repository.
mintsweet pushed a commit to branch feat-dora-config
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit 2d9e8f696cdf29c2bf9d2b19885937e175ca9d98
Author: abeizn <zi...@merico.dev>
AuthorDate: Wed Sep 11 16:50:40 2024 +0800
feat: jenkins dora config (#8033)
---
backend/plugins/github/api/connection_api.go | 4 +-
backend/plugins/jenkins/api/connection_api.go | 94 ++++++++++++++++++++++
backend/plugins/jenkins/impl/impl.go | 3 +
backend/plugins/jenkins/models/build.go | 20 ++---
...{register.go => 20240911_add_url_for_builds.go} | 40 +++++----
.../jenkins/models/migrationscripts/register.go | 1 +
backend/plugins/jenkins/models/response.go | 1 +
backend/plugins/jenkins/tasks/build_collector.go | 2 +-
backend/plugins/jenkins/tasks/build_extractor.go | 1 +
9 files changed, 137 insertions(+), 29 deletions(-)
diff --git a/backend/plugins/github/api/connection_api.go b/backend/plugins/github/api/connection_api.go
index 02c0265de..de0991ef6 100644
--- a/backend/plugins/github/api/connection_api.go
+++ b/backend/plugins/github/api/connection_api.go
@@ -410,7 +410,7 @@ func TestExistingConnection(input *plugin.ApiResourceInput) (*plugin.ApiResource
// @Tags plugins/github
// @Param id path int true "id"
// @Param connectionId path int true "connectionId"
-// @Success 200 {object} models.GithubScopeConfigDeployment
+// @Success 200 {array} string "List of Environment Names"
// @Failure 400 {object} shared.ApiBody "Bad Request"
// @Failure 500 {object} shared.ApiBody "Internal Error"
// @Router /plugins/github/connections/{connectionId}/deployments [GET]
@@ -437,7 +437,7 @@ func GetConnectionDeployments(input *plugin.ApiResourceInput) (*plugin.ApiResour
// @Tags plugins/github
// @Param id path int true "id"
// @Param connectionId path int true "connectionId"
-// @Success 200 {object} models.GithubScopeConfigDeployment
+// @Success 200 {object} map[string]interface{}
// @Failure 400 {object} shared.ApiBody "Bad Request"
// @Failure 500 {object} shared.ApiBody "Internal Error"
// @Router /plugins/github/connections/{connectionId}/transform-to-deployments [POST]
diff --git a/backend/plugins/jenkins/api/connection_api.go b/backend/plugins/jenkins/api/connection_api.go
index b213bb068..4b5dd5ceb 100644
--- a/backend/plugins/jenkins/api/connection_api.go
+++ b/backend/plugins/jenkins/api/connection_api.go
@@ -19,6 +19,7 @@ package api
import (
"context"
+ "fmt"
"net/http"
"strings"
@@ -176,3 +177,96 @@ func ListConnections(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput,
func GetConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
return dsHelper.ConnApi.GetDetail(input)
}
+
+// GetConnectionTransformToDeployments return one connection deployments
+// @Summary return one connection deployments
+// @Description return one connection deployments
+// @Tags plugins/jenkins
+// @Param id path int true "id"
+// @Param connectionId path int true "connectionId"
+// @Success 200 {object} map[string]interface{}
+// @Failure 400 {object} shared.ApiBody "Bad Request"
+// @Failure 500 {object} shared.ApiBody "Internal Error"
+// @Router /plugins/jenkins/connections/{connectionId}/transform-to-deployments [POST]
+func GetConnectionTransformToDeployments(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
+ db := basicRes.GetDal()
+ connectionId := input.Params["connectionId"]
+ deploymentPattern := input.Body["deploymentPattern"]
+ productionPattern := input.Body["productionPattern"]
+ page, err := api.ParsePageParam(input.Body, "page", 1)
+ if err != nil {
+ return nil, errors.Default.New("invalid page value")
+ }
+ pageSize, err := api.ParsePageParam(input.Body, "pageSize", 10)
+ if err != nil {
+ return nil, errors.Default.New("invalid pageSize value")
+ }
+
+ cursor, err := db.RawCursor(`
+ SELECT DISTINCT number, job_name, full_name, url, start_time
+ FROM(
+ SELECT number, job_name, full_name, url, start_time
+ FROM _tool_jenkins_builds
+ WHERE connection_id = ? AND full_name REGEXP ?
+ AND full_name REGEXP ?
+ UNION
+ SELECT number, job_name, full_name, url, start_time
+ FROM _tool_jenkins_stages s
+ LEFT JOIN _tool_jenkins_builds b ON b.full_name = s.build_name
+ WHERE s.connection_id = ? AND s.name REGEXP ?
+ AND s.name REGEXP ?
+ ) AS t
+ ORDER BY start_time DESC
+ `, connectionId, deploymentPattern, productionPattern, connectionId, deploymentPattern, productionPattern)
+ if err != nil {
+ return nil, errors.Default.Wrap(err, "error on get")
+ }
+ defer cursor.Close()
+
+ type selectFileds struct {
+ Number int
+ JobName string
+ FullName string
+ URL string
+ }
+ type transformedFields struct {
+ Name string
+ URL string
+ }
+ var allRuns []transformedFields
+ for cursor.Next() {
+ sf := &selectFileds{}
+ err = db.Fetch(cursor, sf)
+ if err != nil {
+ return nil, errors.Default.Wrap(err, "error on fetch")
+ }
+ // Directly transform and append to allRuns
+ transformed := transformedFields{
+ Name: fmt.Sprintf("#%d - %s", sf.Number, sf.JobName),
+ URL: sf.URL,
+ }
+ allRuns = append(allRuns, transformed)
+ }
+ // Calculate total count
+ totalCount := len(allRuns)
+
+ // Paginate in memory
+ start := (page - 1) * pageSize
+ end := start + pageSize
+ if start > totalCount {
+ start = totalCount
+ }
+ if end > totalCount {
+ end = totalCount
+ }
+ pagedRuns := allRuns[start:end]
+
+ // Return result containing paged runs and total count
+ result := map[string]interface{}{
+ "total": totalCount,
+ "data": pagedRuns,
+ }
+ return &plugin.ApiResourceOutput{
+ Body: result,
+ }, nil
+}
diff --git a/backend/plugins/jenkins/impl/impl.go b/backend/plugins/jenkins/impl/impl.go
index 7411f7543..7be75a8e1 100644
--- a/backend/plugins/jenkins/impl/impl.go
+++ b/backend/plugins/jenkins/impl/impl.go
@@ -216,6 +216,9 @@ func (p Jenkins) ApiResources() map[string]map[string]plugin.ApiResourceHandler
"POST": api.CreateScopeConfig,
"GET": api.GetScopeConfigList,
},
+ "connections/:connectionId/transform-to-deployments": {
+ "POST": api.GetConnectionTransformToDeployments,
+ },
"connections/:connectionId/scope-configs/:scopeConfigId": {
"PATCH": api.UpdateScopeConfig,
"GET": api.GetScopeConfig,
diff --git a/backend/plugins/jenkins/models/build.go b/backend/plugins/jenkins/models/build.go
index 203d81d00..804c5521c 100644
--- a/backend/plugins/jenkins/models/build.go
+++ b/backend/plugins/jenkins/models/build.go
@@ -18,22 +18,24 @@ limitations under the License.
package models
import (
- "github.com/apache/incubator-devlake/core/models/common"
"time"
+
+ "github.com/apache/incubator-devlake/core/models/common"
)
// JenkinsBuild db entity for jenkins build
type JenkinsBuild struct {
common.NoPKModel
// collected fields
- ConnectionId uint64 `gorm:"primaryKey"`
- JobName string `gorm:"index;type:varchar(255)"`
- JobPath string `gorm:"index;type:varchar(255)"`
- Duration float64 // build time
- FullName string `gorm:"primaryKey;type:varchar(255)"` // "path/job name#7"
- EstimatedDuration float64 // EstimatedDuration
- Number int64 `gorm:"index"`
- Result string // Result
+ ConnectionId uint64 `gorm:"primaryKey"`
+ JobName string `gorm:"index;type:varchar(255)"`
+ JobPath string `gorm:"index;type:varchar(255)"`
+ Duration float64 // build time
+ FullName string `gorm:"primaryKey;type:varchar(255)"` // "path/job name#7"
+ EstimatedDuration float64 // EstimatedDuration
+ Number int64 `gorm:"index"`
+ Result string // Result
+ Url string
Timestamp int64 // start time
StartTime time.Time // convered by timestamp
Type string `gorm:"index;type:varchar(255)"`
diff --git a/backend/plugins/jenkins/models/migrationscripts/register.go b/backend/plugins/jenkins/models/migrationscripts/20240911_add_url_for_builds.go
similarity index 59%
copy from backend/plugins/jenkins/models/migrationscripts/register.go
copy to backend/plugins/jenkins/models/migrationscripts/20240911_add_url_for_builds.go
index 1dc1eb84b..d3959b55a 100644
--- a/backend/plugins/jenkins/models/migrationscripts/register.go
+++ b/backend/plugins/jenkins/models/migrationscripts/20240911_add_url_for_builds.go
@@ -18,23 +18,29 @@ limitations under the License.
package migrationscripts
import (
- "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/core/context"
+ "github.com/apache/incubator-devlake/core/errors"
)
-// All return all the migration scripts
-func All() []plugin.MigrationScript {
- return []plugin.MigrationScript{
- new(addInitTables),
- new(modifyAllEntities),
- new(modifyJenkinsBuild),
- new(addJobFields),
- new(addJobPathForBuilds),
- new(changeIndexOfJobPath),
- new(addTransformationRule20221128),
- new(addFullNameForBuilds),
- new(addConnectionIdToTransformationRule),
- new(renameTr2ScopeConfig),
- new(addRawParamTableForScope),
- new(addNumberToJenkinsBuildCommit),
- }
+type addUrlForBuilds struct{}
+
+type jenkinsBuild20240911 struct {
+ Url string
+}
+
+func (jenkinsBuild20240911) TableName() string {
+ return "_tool_jenkins_builds"
+}
+
+func (script *addUrlForBuilds) Up(basicRes context.BasicRes) errors.Error {
+ db := basicRes.GetDal()
+ return db.AutoMigrate(&jenkinsBuild20240911{})
+}
+
+func (*addUrlForBuilds) Version() uint64 {
+ return 20240911231237
+}
+
+func (*addUrlForBuilds) Name() string {
+ return "add url for builds"
}
diff --git a/backend/plugins/jenkins/models/migrationscripts/register.go b/backend/plugins/jenkins/models/migrationscripts/register.go
index 1dc1eb84b..ff2c9ac7a 100644
--- a/backend/plugins/jenkins/models/migrationscripts/register.go
+++ b/backend/plugins/jenkins/models/migrationscripts/register.go
@@ -36,5 +36,6 @@ func All() []plugin.MigrationScript {
new(renameTr2ScopeConfig),
new(addRawParamTableForScope),
new(addNumberToJenkinsBuildCommit),
+ new(addUrlForBuilds),
}
}
diff --git a/backend/plugins/jenkins/models/response.go b/backend/plugins/jenkins/models/response.go
index 643bec691..962a875af 100644
--- a/backend/plugins/jenkins/models/response.go
+++ b/backend/plugins/jenkins/models/response.go
@@ -102,6 +102,7 @@ type ApiBuildResponse struct {
Class string `json:"_class"`
Number int64 `json:"number"`
Result string `json:"result"`
+ Url string `json:"url"`
Building bool `json:"building"`
Actions []Action `json:"actions"`
Duration float64 `json:"duration"`
diff --git a/backend/plugins/jenkins/tasks/build_collector.go b/backend/plugins/jenkins/tasks/build_collector.go
index 0d792c614..c0b47f95f 100644
--- a/backend/plugins/jenkins/tasks/build_collector.go
+++ b/backend/plugins/jenkins/tasks/build_collector.go
@@ -90,7 +90,7 @@ func collectSingleJobApiBuilds(taskCtx plugin.SubTaskContext) errors.Error {
Query: func(reqData *helper.RequestData, createdAfter *time.Time) (url.Values, errors.Error) {
query := url.Values{}
treeValue := fmt.Sprintf(
- "allBuilds[timestamp,number,duration,building,estimatedDuration,fullDisplayName,result,actions[lastBuiltRevision[SHA1,branch[name]],remoteUrls,mercurialRevisionNumber,causes[*]],changeSet[kind,revisions[revision]]]{%d,%d}",
+ "allBuilds[timestamp,number,duration,building,estimatedDuration,fullDisplayName,result,url,actions[lastBuiltRevision[SHA1,branch[name]],remoteUrls,mercurialRevisionNumber,causes[*]],changeSet[kind,revisions[revision]]]{%d,%d}",
reqData.Pager.Skip, reqData.Pager.Skip+reqData.Pager.Size)
query.Set("tree", treeValue)
return query, nil
diff --git a/backend/plugins/jenkins/tasks/build_extractor.go b/backend/plugins/jenkins/tasks/build_extractor.go
index d039034ab..842c46d2c 100644
--- a/backend/plugins/jenkins/tasks/build_extractor.go
+++ b/backend/plugins/jenkins/tasks/build_extractor.go
@@ -81,6 +81,7 @@ func ExtractApiBuilds(taskCtx plugin.SubTaskContext) errors.Error {
EstimatedDuration: body.EstimatedDuration,
Number: body.Number,
Result: body.Result,
+ Url: body.Url,
Timestamp: body.Timestamp,
Class: class,
Building: body.Building,