You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by ab...@apache.org on 2024/04/15 08:04:42 UTC
(incubator-devlake) branch feat#7280 updated: feat: add deployment metadata to the table of the 'DORA details - deployment frequency' dashboard
This is an automated email from the ASF dual-hosted git repository.
abeizn pushed a commit to branch feat#7280
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/feat#7280 by this push:
new cb840c58a feat: add deployment metadata to the table of the 'DORA details - deployment frequency' dashboard
cb840c58a is described below
commit cb840c58abb1d9a73d718fa1eae8a83bfbe47263
Author: abeizn <zi...@merico.dev>
AuthorDate: Mon Apr 15 16:04:28 2024 +0800
feat: add deployment metadata to the table of the 'DORA details - deployment frequency' dashboard
---
.../models/domainlayer/devops/cicd_deployment.go | 2 +
.../domainlayer/devops/cicd_deployment_commit.go | 4 ++
.../models/domainlayer/devops/cicd_pipeline.go | 5 +-
.../domainlayer/devops/cicd_pipeline_commmit.go | 14 ++--
...0410_add_display_title_and_url_to_deployment.go | 77 ++++++++++++++++++++++
backend/core/models/migrationscripts/register.go | 1 +
.../20240411_add_link_href_to_plan_build.go | 55 ++++++++++++++++
.../bamboo/models/migrationscripts/register.go | 1 +
backend/plugins/bamboo/models/plan_build.go | 8 ++-
backend/plugins/bamboo/models/plan_build_commit.go | 2 +
.../deploy_build_to_deployment_commit_convertor.go | 10 +--
.../tasks/deploy_build_to_deployment_convertor.go | 1 +
.../plugins/bamboo/tasks/plan_build_convertor.go | 13 +++-
.../plugins/bamboo/tasks/plan_build_extractor.go | 10 +++
.../plugins/bamboo/tasks/plan_commit_convertor.go | 6 +-
.../bitbucket/tasks/deployment_convertor.go | 12 ++--
.../plugins/bitbucket/tasks/pipeline_convertor.go | 18 +++--
.../plugins/bitbucket/tasks/pipeline_extractor.go | 3 -
.../plugins/circleci/tasks/workflow_converter.go | 17 +++--
.../dora/tasks/deployment_commits_generator.go | 2 +
backend/plugins/dora/tasks/deployment_generator.go | 2 +
backend/plugins/github/models/deployment.go | 8 ++-
.../20240410_add_workflow_display_title.go | 55 ++++++++++++++++
.../github/models/migrationscripts/register.go | 1 +
backend/plugins/github/models/run.go | 4 +-
backend/plugins/github/tasks/cicd_run_convertor.go | 12 ++--
.../plugins/github/tasks/deployment_convertor.go | 10 +--
.../github_graphql/tasks/deployment_collector.go | 14 ++++
.../github_graphql/tasks/deployment_extractor.go | 3 +
.../20240411_add_web_url_to_pipeline.go} | 41 +++++++-----
.../gitlab/models/migrationscripts/register.go | 1 +
backend/plugins/gitlab/models/pipeline.go | 1 +
.../plugins/gitlab/tasks/deployment_convertor.go | 2 +
.../gitlab/tasks/pipeline_commit_convertor.go | 2 +
.../gitlab/tasks/pipeline_detail_convertor.go | 2 +
.../gitlab/tasks/pipeline_detail_extractor.go | 3 -
backend/plugins/gitlab/tasks/pipeline_extractor.go | 1 +
.../plugins/azuredevops/azuredevops/migrations.py | 6 ++
.../plugins/azuredevops/azuredevops/models.py | 2 +
.../azuredevops/azuredevops/streams/builds.py | 4 ++
.../pydevlake/pydevlake/domain_layer/devops.py | 5 ++
41 files changed, 369 insertions(+), 71 deletions(-)
diff --git a/backend/core/models/domainlayer/devops/cicd_deployment.go b/backend/core/models/domainlayer/devops/cicd_deployment.go
index 199e95fa2..b30710772 100644
--- a/backend/core/models/domainlayer/devops/cicd_deployment.go
+++ b/backend/core/models/domainlayer/devops/cicd_deployment.go
@@ -25,6 +25,8 @@ type CICDDeployment struct {
domainlayer.DomainEntity
CicdScopeId string `gorm:"index;type:varchar(255)"`
Name string `gorm:"type:varchar(255)"`
+ DisplayTitle string
+ Url string
Result string `gorm:"type:varchar(100)"`
Status string `gorm:"type:varchar(100)"`
OriginalStatus string `gorm:"type:varchar(100)"`
diff --git a/backend/core/models/domainlayer/devops/cicd_deployment_commit.go b/backend/core/models/domainlayer/devops/cicd_deployment_commit.go
index aa8b9db8d..7b24cee6b 100644
--- a/backend/core/models/domainlayer/devops/cicd_deployment_commit.go
+++ b/backend/core/models/domainlayer/devops/cicd_deployment_commit.go
@@ -26,6 +26,8 @@ type CicdDeploymentCommit struct {
CicdScopeId string `gorm:"index;type:varchar(255)"`
CicdDeploymentId string `gorm:"type:varchar(255)"` // if it is converted from a cicd_pipeline_commit
Name string `gorm:"type:varchar(255)"`
+ DisplayTitle string
+ Url string
Result string `gorm:"type:varchar(100)"`
Status string `gorm:"type:varchar(100)"`
OriginalStatus string `gorm:"type:varchar(100)"`
@@ -70,5 +72,7 @@ func (cicdDeploymentCommit CicdDeploymentCommit) ToDeployment() *CICDDeployment
},
QueuedDurationSec: cicdDeploymentCommit.QueuedDurationSec,
DurationSec: cicdDeploymentCommit.DurationSec,
+ DisplayTitle: cicdDeploymentCommit.DisplayTitle,
+ Url: cicdDeploymentCommit.Url,
}
}
diff --git a/backend/core/models/domainlayer/devops/cicd_pipeline.go b/backend/core/models/domainlayer/devops/cicd_pipeline.go
index d8748e39a..e05d573b6 100644
--- a/backend/core/models/domainlayer/devops/cicd_pipeline.go
+++ b/backend/core/models/domainlayer/devops/cicd_pipeline.go
@@ -18,15 +18,18 @@ limitations under the License.
package devops
import (
- "github.com/spf13/cast"
"strings"
+ "github.com/spf13/cast"
+
"github.com/apache/incubator-devlake/core/models/domainlayer"
)
type CICDPipeline struct {
domainlayer.DomainEntity
Name string `gorm:"type:varchar(255)"`
+ DisplayTitle string
+ Url string
Result string `gorm:"type:varchar(100)"`
Status string `gorm:"type:varchar(100)"`
OriginalStatus string `gorm:"type:varchar(100)"`
diff --git a/backend/core/models/domainlayer/devops/cicd_pipeline_commmit.go b/backend/core/models/domainlayer/devops/cicd_pipeline_commmit.go
index 10e0dac57..bbf8f18fa 100644
--- a/backend/core/models/domainlayer/devops/cicd_pipeline_commmit.go
+++ b/backend/core/models/domainlayer/devops/cicd_pipeline_commmit.go
@@ -23,12 +23,14 @@ import (
type CiCDPipelineCommit struct {
common.NoPKModel
- PipelineId string `gorm:"primaryKey;type:varchar(255)"`
- CommitSha string `gorm:"primaryKey;type:varchar(255)"`
- CommitMsg string
- Branch string `gorm:"type:varchar(255)"`
- RepoId string `gorm:"index;type:varchar(255)"`
- RepoUrl string
+ PipelineId string `gorm:"primaryKey;type:varchar(255)"`
+ CommitSha string `gorm:"primaryKey;type:varchar(255)"`
+ CommitMsg string
+ DisplayTitle string
+ Url string
+ Branch string `gorm:"type:varchar(255)"`
+ RepoId string `gorm:"index;type:varchar(255)"`
+ RepoUrl string
}
func (CiCDPipelineCommit) TableName() string {
diff --git a/backend/core/models/migrationscripts/20240410_add_display_title_and_url_to_deployment.go b/backend/core/models/migrationscripts/20240410_add_display_title_and_url_to_deployment.go
new file mode 100644
index 000000000..66876cb26
--- /dev/null
+++ b/backend/core/models/migrationscripts/20240410_add_display_title_and_url_to_deployment.go
@@ -0,0 +1,77 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+ "github.com/apache/incubator-devlake/core/context"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/helpers/migrationhelper"
+)
+
+var _ plugin.MigrationScript = (*addDisplayTitleAndUrl)(nil)
+
+type cicdDeployment20240410 struct {
+ DisplayTitle string
+ Url string
+}
+
+func (cicdDeployment20240410) TableName() string {
+ return "cicd_deployments"
+}
+
+type cicdDeploymentCommit20240410 struct {
+ DisplayTitle string
+ Url string
+}
+
+func (cicdDeploymentCommit20240410) TableName() string {
+ return "cicd_deployment_commits"
+}
+
+type cicdPipeline20240410 struct {
+ DisplayTitle string
+ Url string
+}
+
+func (cicdPipeline20240410) TableName() string {
+ return "cicd_pipelines"
+}
+
+type cicdPipelineCommit20240410 struct {
+ DisplayTitle string
+ Url string
+}
+
+func (cicdPipelineCommit20240410) TableName() string {
+ return "cicd_pipeline_commits"
+}
+
+type addDisplayTitleAndUrl struct{}
+
+func (*addDisplayTitleAndUrl) Up(basicRes context.BasicRes) errors.Error {
+ return migrationhelper.AutoMigrateTables(basicRes, &cicdDeployment20240410{}, &cicdDeploymentCommit20240410{}, &cicdPipeline20240410{}, &cicdPipelineCommit20240410{})
+}
+
+func (*addDisplayTitleAndUrl) Version() uint64 {
+ return 20240410111248
+}
+
+func (*addDisplayTitleAndUrl) Name() string {
+ return "add display title and url to deployments and pipelines"
+}
diff --git a/backend/core/models/migrationscripts/register.go b/backend/core/models/migrationscripts/register.go
index f4641d03b..3c3e2e45f 100644
--- a/backend/core/models/migrationscripts/register.go
+++ b/backend/core/models/migrationscripts/register.go
@@ -111,5 +111,6 @@ func All() []plugin.MigrationScript {
new(addSubtabknameToDeployment),
new(addStore),
new(addSubtaskField),
+ new(addDisplayTitleAndUrl),
}
}
diff --git a/backend/plugins/bamboo/models/migrationscripts/20240411_add_link_href_to_plan_build.go b/backend/plugins/bamboo/models/migrationscripts/20240411_add_link_href_to_plan_build.go
new file mode 100644
index 000000000..cb212b8ba
--- /dev/null
+++ b/backend/plugins/bamboo/models/migrationscripts/20240411_add_link_href_to_plan_build.go
@@ -0,0 +1,55 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+ "github.com/apache/incubator-devlake/core/context"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/helpers/migrationhelper"
+)
+
+type addLinkHrefToBambooPlanBuild struct{}
+
+type bambooPlanBuild20240411 struct {
+ LinkHref string
+}
+
+func (bambooPlanBuild20240411) TableName() string {
+ return "_tool_bamboo_plan_builds"
+}
+
+type BambooPlanBuildVcsRevision20240411 struct {
+ DisplayTitle string `json:"displayTitle"`
+ Url string `json:"url"`
+}
+
+func (BambooPlanBuildVcsRevision20240411) TableName() string {
+ return "_tool_bamboo_plan_build_commits"
+}
+
+func (u *addLinkHrefToBambooPlanBuild) Up(baseRes context.BasicRes) errors.Error {
+ return migrationhelper.AutoMigrateTables(baseRes, &bambooPlanBuild20240411{}, &BambooPlanBuildVcsRevision20240411{})
+}
+
+func (*addLinkHrefToBambooPlanBuild) Version() uint64 {
+ return 20240411150361
+}
+
+func (*addLinkHrefToBambooPlanBuild) Name() string {
+ return "add display_title and url to bamboo plan build and bamboo plan build commit"
+}
diff --git a/backend/plugins/bamboo/models/migrationscripts/register.go b/backend/plugins/bamboo/models/migrationscripts/register.go
index c257892f9..d7ef35c7e 100644
--- a/backend/plugins/bamboo/models/migrationscripts/register.go
+++ b/backend/plugins/bamboo/models/migrationscripts/register.go
@@ -37,5 +37,6 @@ func All() []plugin.MigrationScript {
new(renameMultiBambooRawTables20230920),
new(addMissingPrimaryKeyForBambooPlanBuildVcsRevision),
new(addQueuedFieldsInJobBuild20231128),
+ new(addLinkHrefToBambooPlanBuild),
}
}
diff --git a/backend/plugins/bamboo/models/plan_build.go b/backend/plugins/bamboo/models/plan_build.go
index 1b74e6eca..4b0351ecc 100644
--- a/backend/plugins/bamboo/models/plan_build.go
+++ b/backend/plugins/bamboo/models/plan_build.go
@@ -28,6 +28,7 @@ type BambooPlanBuild struct {
ConnectionId uint64 `gorm:"primaryKey"`
PlanBuildKey string `gorm:"primaryKey"`
Expand string `json:"expand"`
+ LinkHref string `json:"linkHref"`
Number int `json:"number"`
BuildNumber int `json:"buildNumber"`
PlanName string `json:"planName"`
@@ -79,6 +80,7 @@ func (apiRes *ApiBambooPlanBuild) Convert() *BambooPlanBuild {
return &BambooPlanBuild{
PlanBuildKey: apiRes.Key,
Expand: apiRes.Expand,
+ LinkHref: apiRes.Link.Href,
Number: apiRes.Number,
BuildNumber: apiRes.BuildNumber,
PlanName: apiRes.PlanName,
@@ -115,7 +117,11 @@ func (apiRes *ApiBambooPlanBuild) Convert() *BambooPlanBuild {
}
type ApiBambooPlanBuild struct {
- Expand string `json:"expand"`
+ Expand string `json:"expand"`
+ Link struct {
+ Href string `json:"href"`
+ Rel string `json:"rel"`
+ } `json:"link"`
PlanName string `json:"planName"`
ProjectName string `json:"projectName"`
BuildResultKey string `json:"buildResultKey"`
diff --git a/backend/plugins/bamboo/models/plan_build_commit.go b/backend/plugins/bamboo/models/plan_build_commit.go
index a9998c1ab..f1c99ab5a 100644
--- a/backend/plugins/bamboo/models/plan_build_commit.go
+++ b/backend/plugins/bamboo/models/plan_build_commit.go
@@ -28,6 +28,8 @@ type BambooPlanBuildVcsRevision struct {
RepositoryId int
RepositoryName string `json:"repositoryName"`
VcsRevisionKey string `gorm:"primaryKey"`
+ DisplayTitle string `json:"displayTitle"`
+ Url string `json:"url"`
common.NoPKModel
}
diff --git a/backend/plugins/bamboo/tasks/deploy_build_to_deployment_commit_convertor.go b/backend/plugins/bamboo/tasks/deploy_build_to_deployment_commit_convertor.go
index ab2d8199b..e29155b50 100644
--- a/backend/plugins/bamboo/tasks/deploy_build_to_deployment_commit_convertor.go
+++ b/backend/plugins/bamboo/tasks/deploy_build_to_deployment_commit_convertor.go
@@ -19,11 +19,12 @@ package tasks
import (
"fmt"
- "github.com/apache/incubator-devlake/core/models/common"
"reflect"
"strconv"
"time"
+ "github.com/apache/incubator-devlake/core/models/common"
+
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models/domainlayer"
@@ -136,9 +137,10 @@ func ConvertDeployBuildsToDeploymentCommits(taskCtx plugin.SubTaskContext) error
StartedDate: input.ExecutedDate,
FinishedDate: input.FinishedDate,
},
- CommitSha: input.VcsRevisionKey,
- RefName: input.PlanBranchName,
- RepoId: strconv.Itoa(input.RepositoryId),
+ CommitSha: input.VcsRevisionKey,
+ RefName: input.PlanBranchName,
+ RepoId: strconv.Itoa(input.RepositoryId),
+ DisplayTitle: input.GenerateCICDDeploymentCommitName(),
}
if data.RegexEnricher.ReturnNameIfMatched(devops.ENV_NAME_PATTERN, input.Environment) != "" {
deploymentCommit.Environment = devops.PRODUCTION
diff --git a/backend/plugins/bamboo/tasks/deploy_build_to_deployment_convertor.go b/backend/plugins/bamboo/tasks/deploy_build_to_deployment_convertor.go
index 0a2c014bc..badf42a53 100644
--- a/backend/plugins/bamboo/tasks/deploy_build_to_deployment_convertor.go
+++ b/backend/plugins/bamboo/tasks/deploy_build_to_deployment_convertor.go
@@ -103,6 +103,7 @@ func ConvertDeployBuildsToDeployments(taskCtx plugin.SubTaskContext) errors.Erro
StartedDate: input.ExecutedDate,
FinishedDate: input.FinishedDate,
},
+ DisplayTitle: name,
}
if data.RegexEnricher.ReturnNameIfMatched(devops.ENV_NAME_PATTERN, input.Environment) != "" {
deployment.Environment = devops.PRODUCTION
diff --git a/backend/plugins/bamboo/tasks/plan_build_convertor.go b/backend/plugins/bamboo/tasks/plan_build_convertor.go
index 14d27ac20..c86e8ae53 100644
--- a/backend/plugins/bamboo/tasks/plan_build_convertor.go
+++ b/backend/plugins/bamboo/tasks/plan_build_convertor.go
@@ -18,6 +18,9 @@ limitations under the License.
package tasks
import (
+ "reflect"
+ "time"
+
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models/domainlayer"
@@ -26,8 +29,6 @@ import (
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/bamboo/models"
- "reflect"
- "time"
)
var ConvertPlanBuildsMeta = plugin.SubTaskMeta{
@@ -40,6 +41,7 @@ var ConvertPlanBuildsMeta = plugin.SubTaskMeta{
func ConvertPlanBuilds(taskCtx plugin.SubTaskContext) errors.Error {
db := taskCtx.GetDal()
+ logger := taskCtx.GetLogger()
rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_PLAN_BUILD_TABLE)
cursor, err := db.Cursor(
dal.From(&models.BambooPlanBuild{}),
@@ -87,6 +89,13 @@ func ConvertPlanBuilds(taskCtx plugin.SubTaskContext) errors.Error {
Default: devops.STATUS_OTHER,
}, line.LifeCycleState),
OriginalStatus: line.LifeCycleState,
+ DisplayTitle: line.GenerateCICDPipeLineName(),
+ }
+ homepage, err := getBambooHomePage(line.LinkHref)
+ if err != nil {
+ logger.Warn(err, "get bamboo home")
+ } else {
+ domainPlanBuild.Url = homepage + "/browse/" + line.PlanKey
}
domainPlanBuild.Type = line.Type
diff --git a/backend/plugins/bamboo/tasks/plan_build_extractor.go b/backend/plugins/bamboo/tasks/plan_build_extractor.go
index 273e61764..7f5c6eb5b 100644
--- a/backend/plugins/bamboo/tasks/plan_build_extractor.go
+++ b/backend/plugins/bamboo/tasks/plan_build_extractor.go
@@ -31,6 +31,7 @@ var _ plugin.SubTaskEntryPoint = ExtractPlanBuild
func ExtractPlanBuild(taskCtx plugin.SubTaskContext) errors.Error {
rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_PLAN_BUILD_TABLE)
+ logger := taskCtx.GetLogger()
extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{
RawDataSubTaskArgs: *rawDataSubTaskArgs,
@@ -47,6 +48,13 @@ func ExtractPlanBuild(taskCtx plugin.SubTaskContext) errors.Error {
body.Type = data.RegexEnricher.ReturnNameIfMatched(devops.DEPLOYMENT, body.PlanName)
body.Environment = data.RegexEnricher.ReturnNameIfMatched(devops.PRODUCTION, body.PlanName)
+ var url string
+ homepage, errGetHomePage := getBambooHomePage(body.LinkHref)
+ if errGetHomePage != nil {
+ logger.Warn(errGetHomePage, "get bamboo home")
+ } else {
+ url = homepage + "/browse/" + body.PlanKey
+ }
results := make([]interface{}, 0)
results = append(results, body)
// As job build can get more accuracy repo info,
@@ -59,6 +67,8 @@ func ExtractPlanBuild(taskCtx plugin.SubTaskContext) errors.Error {
RepositoryId: v.RepositoryId,
RepositoryName: v.RepositoryName,
VcsRevisionKey: v.VcsRevisionKey,
+ DisplayTitle: body.GenerateCICDPipeLineName(),
+ Url: url,
})
}
diff --git a/backend/plugins/bamboo/tasks/plan_commit_convertor.go b/backend/plugins/bamboo/tasks/plan_commit_convertor.go
index ec9384b49..13104d89c 100644
--- a/backend/plugins/bamboo/tasks/plan_commit_convertor.go
+++ b/backend/plugins/bamboo/tasks/plan_commit_convertor.go
@@ -60,8 +60,10 @@ func ConvertPlanVcs(taskCtx plugin.SubTaskContext) errors.Error {
Convert: func(inputRow interface{}) ([]interface{}, errors.Error) {
line := inputRow.(*models.BambooPlanBuildVcsRevision)
domainPlanVcs := &devops.CiCDPipelineCommit{
- PipelineId: planBuildIdGen.Generate(data.Options.ConnectionId, line.PlanBuildKey),
- CommitSha: line.VcsRevisionKey,
+ PipelineId: planBuildIdGen.Generate(data.Options.ConnectionId, line.PlanBuildKey),
+ CommitSha: line.VcsRevisionKey,
+ DisplayTitle: line.DisplayTitle,
+ Url: line.Url,
}
domainPlanVcs.RepoId = repoMap[line.RepositoryId]
fakeRepoUrl, err := generateFakeRepoUrl(data.ApiClient.GetEndpoint(), line.RepositoryId)
diff --git a/backend/plugins/bitbucket/tasks/deployment_convertor.go b/backend/plugins/bitbucket/tasks/deployment_convertor.go
index ca6ea9a4a..8b2fbbcc3 100644
--- a/backend/plugins/bitbucket/tasks/deployment_convertor.go
+++ b/backend/plugins/bitbucket/tasks/deployment_convertor.go
@@ -111,11 +111,13 @@ func ConvertDeployments(taskCtx plugin.SubTaskContext) errors.Error {
StartedDate: bitbucketDeployment.StartedOn,
FinishedDate: bitbucketDeployment.CompletedOn,
},
- DurationSec: duration,
- CommitSha: bitbucketDeployment.CommitSha,
- RefName: bitbucketDeployment.RefName,
- RepoId: repoId,
- RepoUrl: repo.HTMLUrl,
+ DurationSec: duration,
+ CommitSha: bitbucketDeployment.CommitSha,
+ RefName: bitbucketDeployment.RefName,
+ RepoId: repoId,
+ RepoUrl: repo.HTMLUrl,
+ DisplayTitle: bitbucketDeployment.Name,
+ Url: bitbucketDeployment.WebUrl,
}
if domainDeployCommit.Environment == devops.TEST {
// Theoretically, environment cannot be "Test" according to
diff --git a/backend/plugins/bitbucket/tasks/pipeline_convertor.go b/backend/plugins/bitbucket/tasks/pipeline_convertor.go
index 9a8d80bdf..580dd5b90 100644
--- a/backend/plugins/bitbucket/tasks/pipeline_convertor.go
+++ b/backend/plugins/bitbucket/tasks/pipeline_convertor.go
@@ -76,11 +76,13 @@ func ConvertPipelines(taskCtx plugin.SubTaskContext) errors.Error {
domainEntityId := pipelineIdGen.Generate(data.Options.ConnectionId, bitbucketPipeline.BitbucketId)
results := make([]interface{}, 0, 2)
domainPipelineCommit := &devops.CiCDPipelineCommit{
- PipelineId: domainEntityId,
- RepoId: repoId,
- CommitSha: bitbucketPipeline.CommitSha,
- Branch: bitbucketPipeline.RefName,
- RepoUrl: repo.HTMLUrl,
+ PipelineId: domainEntityId,
+ RepoId: repoId,
+ CommitSha: bitbucketPipeline.CommitSha,
+ Branch: bitbucketPipeline.RefName,
+ RepoUrl: repo.HTMLUrl,
+ DisplayTitle: fmt.Sprintf("%s/%d", domainEntityId, bitbucketPipeline.BuildNumber),
+ Url: bitbucketPipeline.WebUrl,
}
domainPipeline := &devops.CICDPipeline{
DomainEntity: domainlayer.DomainEntity{
@@ -106,8 +108,10 @@ func ConvertPipelines(taskCtx plugin.SubTaskContext) errors.Error {
StartedDate: bitbucketPipeline.BitbucketCreatedOn,
FinishedDate: bitbucketPipeline.BitbucketCompleteOn,
},
- DurationSec: float64(bitbucketPipeline.DurationInSeconds),
- CicdScopeId: repoId,
+ DurationSec: float64(bitbucketPipeline.DurationInSeconds),
+ CicdScopeId: repoId,
+ DisplayTitle: fmt.Sprintf("%s/%d", domainEntityId, bitbucketPipeline.BuildNumber),
+ Url: bitbucketPipeline.WebUrl,
}
results = append(results, domainPipelineCommit, domainPipeline)
return results, nil
diff --git a/backend/plugins/bitbucket/tasks/pipeline_extractor.go b/backend/plugins/bitbucket/tasks/pipeline_extractor.go
index dd7e68880..ec924db8d 100644
--- a/backend/plugins/bitbucket/tasks/pipeline_extractor.go
+++ b/backend/plugins/bitbucket/tasks/pipeline_extractor.go
@@ -106,9 +106,6 @@ func ExtractApiPipelines(taskCtx plugin.SubTaskContext) errors.Error {
Type: data.RegexEnricher.ReturnNameIfMatched(devops.DEPLOYMENT, bitbucketApiPipeline.Target.RefName),
Environment: data.RegexEnricher.ReturnNameIfOmittedOrMatched(devops.PRODUCTION, bitbucketApiPipeline.Target.RefName),
}
- if err != nil {
- return nil, err
- }
if bitbucketApiPipeline.State.Result != nil {
bitbucketPipeline.Result = bitbucketApiPipeline.State.Result.Name
} else if bitbucketApiPipeline.State.Stage != nil {
diff --git a/backend/plugins/circleci/tasks/workflow_converter.go b/backend/plugins/circleci/tasks/workflow_converter.go
index 9e923c9d3..91806fad5 100644
--- a/backend/plugins/circleci/tasks/workflow_converter.go
+++ b/backend/plugins/circleci/tasks/workflow_converter.go
@@ -18,6 +18,7 @@ limitations under the License.
package tasks
import (
+ "fmt"
"reflect"
"github.com/apache/incubator-devlake/core/dal"
@@ -80,8 +81,9 @@ func ConvertWorkflows(taskCtx plugin.SubTaskContext) errors.Error {
Failure: []string{"failed", "failing", "error"}, // not_run,canceled
Default: devops.RESULT_DEFAULT,
}, userTool.Status),
- Type: data.RegexEnricher.ReturnNameIfMatched(devops.DEPLOYMENT, userTool.Name),
- Environment: data.RegexEnricher.ReturnNameIfOmittedOrMatched(devops.PRODUCTION, userTool.Name),
+ Type: data.RegexEnricher.ReturnNameIfMatched(devops.DEPLOYMENT, userTool.Name),
+ Environment: data.RegexEnricher.ReturnNameIfOmittedOrMatched(devops.PRODUCTION, userTool.Name),
+ DisplayTitle: fmt.Sprintf("%s#%d", userTool.Name, userTool.PipelineNumber),
}
result := make([]interface{}, 0, 2)
result = append(result, pipeline)
@@ -91,11 +93,12 @@ func ConvertWorkflows(taskCtx plugin.SubTaskContext) errors.Error {
if p, err := findPipelineById(db, userTool.PipelineId); err == nil {
if p.Vcs.Revision != "" {
result = append(result, &devops.CiCDPipelineCommit{
- PipelineId: pipeline.Id,
- CommitSha: p.Vcs.Revision,
- Branch: p.Vcs.Branch,
- RepoId: p.Vcs.OriginRepositoryUrl,
- RepoUrl: p.Vcs.OriginRepositoryUrl,
+ PipelineId: pipeline.Id,
+ CommitSha: p.Vcs.Revision,
+ Branch: p.Vcs.Branch,
+ RepoId: p.Vcs.OriginRepositoryUrl,
+ RepoUrl: p.Vcs.OriginRepositoryUrl,
+ DisplayTitle: pipeline.DisplayTitle,
})
}
}
diff --git a/backend/plugins/dora/tasks/deployment_commits_generator.go b/backend/plugins/dora/tasks/deployment_commits_generator.go
index e29846997..826e17728 100644
--- a/backend/plugins/dora/tasks/deployment_commits_generator.go
+++ b/backend/plugins/dora/tasks/deployment_commits_generator.go
@@ -158,6 +158,8 @@ func GenerateDeploymentCommits(taskCtx plugin.SubTaskContext) errors.Error {
CicdScopeId: pipelineCommit.CicdScopeId,
CicdDeploymentId: pipelineCommit.PipelineId,
Name: pipelineCommit.PipelineName,
+ DisplayTitle: pipelineCommit.DisplayTitle,
+ Url: pipelineCommit.Url,
Result: pipelineCommit.Result,
OriginalStatus: pipelineCommit.OriginalStatus,
OriginalResult: pipelineCommit.OriginalResult,
diff --git a/backend/plugins/dora/tasks/deployment_generator.go b/backend/plugins/dora/tasks/deployment_generator.go
index 08102fb42..0defd4205 100644
--- a/backend/plugins/dora/tasks/deployment_generator.go
+++ b/backend/plugins/dora/tasks/deployment_generator.go
@@ -132,6 +132,8 @@ func GenerateDeployment(taskCtx plugin.SubTaskContext) errors.Error {
},
CicdScopeId: pipelineExInfo.CicdScopeId,
Name: pipelineExInfo.Name,
+ DisplayTitle: pipelineExInfo.DisplayTitle,
+ Url: pipelineExInfo.Url,
Result: pipelineExInfo.Result,
Status: pipelineExInfo.Status,
OriginalStatus: pipelineExInfo.OriginalStatus,
diff --git a/backend/plugins/github/models/deployment.go b/backend/plugins/github/models/deployment.go
index f6d87684a..6a818be45 100644
--- a/backend/plugins/github/models/deployment.go
+++ b/backend/plugins/github/models/deployment.go
@@ -25,9 +25,11 @@ import (
type GithubDeployment struct {
common.NoPKModel `json:"-" mapstructure:"-"`
- ConnectionId uint64 `json:"connection_id" gorm:"primaryKey"`
- GithubId int `json:"github_id"`
- Id string `json:"id" gorm:"type:varchar(255);primaryKey"`
+ ConnectionId uint64 `json:"connection_id" gorm:"primaryKey"`
+ GithubId int `json:"github_id"`
+ Id string `json:"id" gorm:"type:varchar(255);primaryKey"`
+ DisplayTitle string
+ Url string
DatabaseId uint `json:"database_id"`
CommitOid string `json:"commit_oid" gorm:"type:varchar(255)"`
Description string `json:"description" gorm:"type:varchar(255)"`
diff --git a/backend/plugins/github/models/migrationscripts/20240410_add_workflow_display_title.go b/backend/plugins/github/models/migrationscripts/20240410_add_workflow_display_title.go
new file mode 100644
index 000000000..4cd789f4e
--- /dev/null
+++ b/backend/plugins/github/models/migrationscripts/20240410_add_workflow_display_title.go
@@ -0,0 +1,55 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+ "github.com/apache/incubator-devlake/core/context"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/helpers/migrationhelper"
+)
+
+type addWorkflowDisplayTitle struct{}
+
+type run20240410 struct {
+ DisplayTitle string
+}
+
+func (run20240410) TableName() string {
+ return "_tool_github_runs"
+}
+
+type deployment20240410 struct {
+ DisplayTitle string
+ Url string
+}
+
+func (deployment20240410) TableName() string {
+ return "_tool_github_deployments"
+}
+
+func (u *addWorkflowDisplayTitle) Up(baseRes context.BasicRes) errors.Error {
+ return migrationhelper.AutoMigrateTables(baseRes, &run20240410{}, &deployment20240410{})
+}
+
+func (*addWorkflowDisplayTitle) Version() uint64 {
+ return 20240410150358
+}
+
+func (*addWorkflowDisplayTitle) Name() string {
+ return "add display title and url to github runs and deployments"
+}
diff --git a/backend/plugins/github/models/migrationscripts/register.go b/backend/plugins/github/models/migrationscripts/register.go
index a179a1bfd..1f09898df 100644
--- a/backend/plugins/github/models/migrationscripts/register.go
+++ b/backend/plugins/github/models/migrationscripts/register.go
@@ -47,5 +47,6 @@ func All() []plugin.MigrationScript {
new(modifyGithubMilestone),
new(addEnvNamePattern),
new(modifyIssueTypeLength),
+ new(addWorkflowDisplayTitle),
}
}
diff --git a/backend/plugins/github/models/run.go b/backend/plugins/github/models/run.go
index 7bf55e59e..34d211647 100644
--- a/backend/plugins/github/models/run.go
+++ b/backend/plugins/github/models/run.go
@@ -18,8 +18,9 @@ limitations under the License.
package models
import (
- "github.com/apache/incubator-devlake/core/models/common"
"time"
+
+ "github.com/apache/incubator-devlake/core/models/common"
)
type GithubRun struct {
@@ -39,6 +40,7 @@ type GithubRun struct {
WorkflowID int `json:"workflow_id"`
CheckSuiteID int64 `json:"check_suite_id"`
CheckSuiteNodeID string `json:"check_suite_node_id" gorm:"type:varchar(255)"`
+ DisplayTitle string `json:"display_title" gorm:"type:text"`
URL string `json:"url" gorm:"type:varchar(255)"`
HTMLURL string `json:"html_url" gorm:"type:varchar(255)"`
GithubCreatedAt *time.Time `json:"created_at"`
diff --git a/backend/plugins/github/tasks/cicd_run_convertor.go b/backend/plugins/github/tasks/cicd_run_convertor.go
index 505d85fce..d4917862d 100644
--- a/backend/plugins/github/tasks/cicd_run_convertor.go
+++ b/backend/plugins/github/tasks/cicd_run_convertor.go
@@ -117,6 +117,8 @@ func ConvertRuns(taskCtx plugin.SubTaskContext) errors.Error {
Default: devops.STATUS_OTHER,
}, line.Status),
OriginalStatus: line.Status,
+ DisplayTitle: line.DisplayTitle,
+ Url: line.URL,
}
if line.GithubUpdatedAt != nil && line.RunStartedAt != nil {
domainPipeline.DurationSec = float64(line.GithubUpdatedAt.Sub(*line.RunStartedAt).Milliseconds() / 1e3)
@@ -125,10 +127,12 @@ func ConvertRuns(taskCtx plugin.SubTaskContext) errors.Error {
domainPipelineCommit := &devops.CiCDPipelineCommit{
PipelineId: runIdGen.Generate(
data.Options.ConnectionId, line.RepoId, line.ID),
- CommitSha: line.HeadSha,
- Branch: line.HeadBranch,
- RepoId: repoIdGen.Generate(data.Options.ConnectionId, repoId),
- RepoUrl: repo.HTMLUrl,
+ CommitSha: line.HeadSha,
+ Branch: line.HeadBranch,
+ RepoId: repoIdGen.Generate(data.Options.ConnectionId, repoId),
+ RepoUrl: repo.HTMLUrl,
+ DisplayTitle: line.DisplayTitle,
+ Url: line.URL,
}
return []interface{}{
diff --git a/backend/plugins/github/tasks/deployment_convertor.go b/backend/plugins/github/tasks/deployment_convertor.go
index 0d9f89dc6..47cdd2319 100644
--- a/backend/plugins/github/tasks/deployment_convertor.go
+++ b/backend/plugins/github/tasks/deployment_convertor.go
@@ -95,10 +95,12 @@ func ConvertDeployment(taskCtx plugin.SubTaskContext) errors.Error {
StartedDate: &githubDeployment.CreatedDate,
FinishedDate: &githubDeployment.UpdatedDate,
},
- CommitSha: githubDeployment.CommitOid,
- RefName: githubDeployment.RefName,
- RepoId: deploymentScopeIdGen.Generate(githubDeployment.ConnectionId, githubDeployment.GithubId),
- RepoUrl: githubDeployment.RepositoryUrl,
+ CommitSha: githubDeployment.CommitOid,
+ RefName: githubDeployment.RefName,
+ RepoId: deploymentScopeIdGen.Generate(githubDeployment.ConnectionId, githubDeployment.GithubId),
+ RepoUrl: githubDeployment.RepositoryUrl,
+ DisplayTitle: githubDeployment.DisplayTitle,
+ Url: githubDeployment.Url,
}
durationSec := float64(githubDeployment.UpdatedDate.Sub(githubDeployment.CreatedDate).Milliseconds() / 1e3)
diff --git a/backend/plugins/github_graphql/tasks/deployment_collector.go b/backend/plugins/github_graphql/tasks/deployment_collector.go
index 437088629..b81e3d107 100644
--- a/backend/plugins/github_graphql/tasks/deployment_collector.go
+++ b/backend/plugins/github_graphql/tasks/deployment_collector.go
@@ -81,6 +81,20 @@ type GraphqlQueryDeploymentDeployment struct {
} `graphql:"repository"`
CreatedAt time.Time
UpdatedAt time.Time
+ Commit struct {
+ Oid string `graphql:"oid"`
+ Message string `graphql:"message"`
+ Author struct {
+ Name string `graphql:"name"`
+ Email string `graphql:"email"`
+ } `graphql:"author"`
+ CommittedDate time.Time `graphql:"committedDate"`
+ } `graphql:"commit"`
+ // Commits struct {
+ // PageInfo *api.GraphqlQueryPageInfo
+ // Nodes []GraphqlQueryCommit `graphql:"nodes"`
+ // TotalCount graphql.Int
+ // } `graphql:"commits(first: 100)"`
}
// CollectDeployments will request github api via graphql and store the result into raw layer.
diff --git a/backend/plugins/github_graphql/tasks/deployment_extractor.go b/backend/plugins/github_graphql/tasks/deployment_extractor.go
index b2150effd..1eb6c252b 100644
--- a/backend/plugins/github_graphql/tasks/deployment_extractor.go
+++ b/backend/plugins/github_graphql/tasks/deployment_extractor.go
@@ -19,6 +19,7 @@ package tasks
import (
"encoding/json"
+ "strings"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models/common"
@@ -81,6 +82,8 @@ func convertGithubDeployment(deployment GraphqlQueryDeploymentDeployment, connec
GithubId: githubId,
NoPKModel: common.NewNoPKModel(),
Id: deployment.Id,
+ DisplayTitle: strings.Split(deployment.Commit.Message, "\n")[0],
+ Url: deployment.Repository.Url + "/deployments/" + deployment.Environment,
DatabaseId: deployment.DatabaseId,
Payload: deployment.Payload,
Description: deployment.Description,
diff --git a/backend/plugins/bamboo/models/migrationscripts/register.go b/backend/plugins/gitlab/models/migrationscripts/20240411_add_web_url_to_pipeline.go
similarity index 50%
copy from backend/plugins/bamboo/models/migrationscripts/register.go
copy to backend/plugins/gitlab/models/migrationscripts/20240411_add_web_url_to_pipeline.go
index c257892f9..afd1434f6 100644
--- a/backend/plugins/bamboo/models/migrationscripts/register.go
+++ b/backend/plugins/gitlab/models/migrationscripts/20240411_add_web_url_to_pipeline.go
@@ -18,24 +18,29 @@ limitations under the License.
package migrationscripts
import (
- "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/core/context"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/helpers/migrationhelper"
)
-// All return all the migration scripts
-func All() []plugin.MigrationScript {
- return []plugin.MigrationScript{
- new(addInitTables),
- new(addConnectionIdToTransformationRule),
- new(addTypeAndEnvironment),
- new(renameTr2ScopeConfig),
- new(addRawParamTableForScope),
- new(addScopeConfigId),
- new(addEnvNamePattern),
- new(addPlanResultKey),
- new(renameToolBambooDeployBuild20230919),
- new(renameToolBambooDeployEnvironments20230919),
- new(renameMultiBambooRawTables20230920),
- new(addMissingPrimaryKeyForBambooPlanBuildVcsRevision),
- new(addQueuedFieldsInJobBuild20231128),
- }
+type addWebUrlToGitlabPipelineProject struct{}
+
+type gitlabPipelineProject20240411 struct {
+ WebUrl string `gorm:"type:varchar(255)"`
+}
+
+func (gitlabPipelineProject20240411) TableName() string {
+ return "_tool_gitlab_pipeline_projects"
+}
+
+func (u *addWebUrlToGitlabPipelineProject) Up(baseRes context.BasicRes) errors.Error {
+ return migrationhelper.AutoMigrateTables(baseRes, &gitlabPipelineProject20240411{})
+}
+
+func (*addWebUrlToGitlabPipelineProject) Version() uint64 {
+ return 20240411150359
+}
+
+func (*addWebUrlToGitlabPipelineProject) Name() string {
+ return "add web url to gitlab pipeline projects"
}
diff --git a/backend/plugins/gitlab/models/migrationscripts/register.go b/backend/plugins/gitlab/models/migrationscripts/register.go
index c6d97dd20..270bb296f 100644
--- a/backend/plugins/gitlab/models/migrationscripts/register.go
+++ b/backend/plugins/gitlab/models/migrationscripts/register.go
@@ -47,5 +47,6 @@ func All() []plugin.MigrationScript {
new(modifyDeploymentMessageType),
new(addTimeToGitlabPipelineProject),
new(modifyDeploymentCommitTitle),
+ new(addWebUrlToGitlabPipelineProject),
}
}
diff --git a/backend/plugins/gitlab/models/pipeline.go b/backend/plugins/gitlab/models/pipeline.go
index a11e00024..00dce35f6 100644
--- a/backend/plugins/gitlab/models/pipeline.go
+++ b/backend/plugins/gitlab/models/pipeline.go
@@ -58,6 +58,7 @@ type GitlabPipelineProject struct {
PipelineId int `gorm:"primaryKey"`
ProjectId int `gorm:"primaryKey"`
Ref string `gorm:"type:varchar(255)"`
+ WebUrl string `gorm:"type:varchar(255)"`
Sha string `gorm:"type:varchar(255)"`
GitlabCreatedAt *time.Time
GitlabUpdatedAt *time.Time
diff --git a/backend/plugins/gitlab/tasks/deployment_convertor.go b/backend/plugins/gitlab/tasks/deployment_convertor.go
index 4db399b95..d7ff0d774 100644
--- a/backend/plugins/gitlab/tasks/deployment_convertor.go
+++ b/backend/plugins/gitlab/tasks/deployment_convertor.go
@@ -125,6 +125,8 @@ func ConvertDeployment(taskCtx plugin.SubTaskContext) errors.Error {
RefName: gitlabDeployment.Ref,
RepoId: projectIdGen.Generate(data.Options.ConnectionId, data.Options.ProjectId),
RepoUrl: repo.WebUrl,
+ DisplayTitle: gitlabDeployment.DeployableCommitTitle,
+ Url: repo.WebUrl + "/environments",
}
if data.RegexEnricher != nil {
if data.RegexEnricher.ReturnNameIfMatched(devops.ENV_NAME_PATTERN, gitlabDeployment.Environment) != "" {
diff --git a/backend/plugins/gitlab/tasks/pipeline_commit_convertor.go b/backend/plugins/gitlab/tasks/pipeline_commit_convertor.go
index a15928c4d..4984270d9 100644
--- a/backend/plugins/gitlab/tasks/pipeline_commit_convertor.go
+++ b/backend/plugins/gitlab/tasks/pipeline_commit_convertor.go
@@ -82,6 +82,8 @@ func ConvertPipelineCommits(taskCtx plugin.SubTaskContext) errors.Error {
RepoId: didgen.NewDomainIdGenerator(&models.GitlabProject{}).
Generate(gitlabPipelineCommit.ConnectionId, gitlabPipelineCommit.ProjectId),
RepoUrl: repo.WebUrl,
+ // DisplayTitle: gitlabPipelineCommit.Ref,
+ Url: gitlabPipelineCommit.WebUrl,
}
return []interface{}{
diff --git a/backend/plugins/gitlab/tasks/pipeline_detail_convertor.go b/backend/plugins/gitlab/tasks/pipeline_detail_convertor.go
index 3818a9a69..1b652274f 100644
--- a/backend/plugins/gitlab/tasks/pipeline_detail_convertor.go
+++ b/backend/plugins/gitlab/tasks/pipeline_detail_convertor.go
@@ -103,6 +103,8 @@ func ConvertDetailPipelines(taskCtx plugin.SubTaskContext) errors.Error {
Environment: gitlabPipeline.Environment,
Type: gitlabPipeline.Type,
DurationSec: float64(gitlabPipeline.Duration),
+ // DisplayTitle: gitlabPipeline.Ref,
+ Url: gitlabPipeline.WebUrl,
}
return []interface{}{
domainPipeline,
diff --git a/backend/plugins/gitlab/tasks/pipeline_detail_extractor.go b/backend/plugins/gitlab/tasks/pipeline_detail_extractor.go
index 83dab29c7..a5e597e23 100644
--- a/backend/plugins/gitlab/tasks/pipeline_detail_extractor.go
+++ b/backend/plugins/gitlab/tasks/pipeline_detail_extractor.go
@@ -71,9 +71,6 @@ func ExtractApiPipelineDetails(taskCtx plugin.SubTaskContext) errors.Error {
Type: data.RegexEnricher.ReturnNameIfMatched(devops.DEPLOYMENT, gitlabApiPipeline.Ref),
Environment: data.RegexEnricher.ReturnNameIfMatched(devops.PRODUCTION, gitlabApiPipeline.Ref),
}
- if err != nil {
- return nil, err
- }
results := make([]interface{}, 0, 1)
results = append(results, gitlabPipeline)
diff --git a/backend/plugins/gitlab/tasks/pipeline_extractor.go b/backend/plugins/gitlab/tasks/pipeline_extractor.go
index 738109ac4..9cce27c80 100644
--- a/backend/plugins/gitlab/tasks/pipeline_extractor.go
+++ b/backend/plugins/gitlab/tasks/pipeline_extractor.go
@@ -87,6 +87,7 @@ func ExtractApiPipelines(taskCtx plugin.SubTaskContext) errors.Error {
PipelineId: gitlabApiPipeline.Id,
ProjectId: data.Options.ProjectId,
Ref: gitlabApiPipeline.Ref,
+ WebUrl: gitlabApiPipeline.WebUrl,
Sha: gitlabApiPipeline.Sha,
GitlabCreatedAt: common.Iso8601TimeToTime(gitlabApiPipeline.CreatedAt),
GitlabUpdatedAt: common.Iso8601TimeToTime(gitlabApiPipeline.UpdatedAt),
diff --git a/backend/python/plugins/azuredevops/azuredevops/migrations.py b/backend/python/plugins/azuredevops/azuredevops/migrations.py
index b6924e037..47e2e81c9 100644
--- a/backend/python/plugins/azuredevops/azuredevops/migrations.py
+++ b/backend/python/plugins/azuredevops/azuredevops/migrations.py
@@ -199,3 +199,9 @@ def add_updated_date_field_in_tool_azuredevops_gitrepositories(b: MigrationScrip
table = "_tool_azuredevops_gitrepositories"
b.execute(f'ALTER TABLE {table} add COLUMN updated_date datetime(3) DEFAULT NULL', Dialect.MYSQL)
b.execute(f'ALTER TABLE {table} add COLUMN updated_date TIMESTAMPTZ DEFAULT NULL', Dialect.POSTGRESQL)
+
+@migration(20240415163000, name="add queue_time field in _tool_azuredevops_builds")
+def add_queue_time_field_in_tool_azuredevops_builds(b: MigrationScriptBuilder):
+ table = '_tool_azuredevops_builds'
+ b.add_column(table, 'display_title', 'LONGTEXT')
+ b.add_column(table, 'url', 'LONGTEXT')
\ No newline at end of file
diff --git a/backend/python/plugins/azuredevops/azuredevops/models.py b/backend/python/plugins/azuredevops/azuredevops/models.py
index 8863f1b9f..28f7692d8 100644
--- a/backend/python/plugins/azuredevops/azuredevops/models.py
+++ b/backend/python/plugins/azuredevops/azuredevops/models.py
@@ -115,6 +115,8 @@ class Build(ToolModel, table=True):
result: Optional[BuildResult]
source_branch: str
source_version: str
+ display_title: str = Field(source='/triggerInfo/ci.message')
+ url: str = Field(source='/_links/web/href')
class Job(ToolModel, table=True):
diff --git a/backend/python/plugins/azuredevops/azuredevops/streams/builds.py b/backend/python/plugins/azuredevops/azuredevops/streams/builds.py
index af2fdf252..bbf9e7420 100644
--- a/backend/python/plugins/azuredevops/azuredevops/streams/builds.py
+++ b/backend/python/plugins/azuredevops/azuredevops/streams/builds.py
@@ -96,6 +96,8 @@ class Builds(Stream):
environment=environment,
type=type,
cicd_scope_id=ctx.scope.domain_id(),
+ display_title=b.display_title,
+ url=b.url,
)
if b.source_version is not None:
@@ -105,4 +107,6 @@ class Builds(Stream):
branch=b.source_branch,
repo_id=ctx.scope.domain_id(),
repo_url=ctx.scope.url,
+ display_title=b.display_title,
+ url=b.url,
)
diff --git a/backend/python/pydevlake/pydevlake/domain_layer/devops.py b/backend/python/pydevlake/pydevlake/domain_layer/devops.py
index 3b6608fb3..c8ef108d4 100644
--- a/backend/python/pydevlake/pydevlake/domain_layer/devops.py
+++ b/backend/python/pydevlake/pydevlake/domain_layer/devops.py
@@ -69,6 +69,9 @@ class CICDPipeline(DomainModel, table=True):
type: Optional[CICDType]
environment: Optional[CICDEnvironment]
+ display_title: Optional[str]
+ url: Optional[str]
+
class CiCDPipelineCommit(NoPKModel, table=True):
__tablename__ = 'cicd_pipeline_commits'
@@ -77,6 +80,8 @@ class CiCDPipelineCommit(NoPKModel, table=True):
branch: str
repo_id: str
repo_url: str
+ display_title: Optional[str]
+ url: Optional[str]
class CicdScope(DomainScope):