You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by ly...@apache.org on 2024/07/29 06:20:32 UTC
(incubator-devlake) branch release-v1.0 updated: cherry pick #7652 #7783 to v1.0 (#7790)
This is an automated email from the ASF dual-hosted git repository.
lynwee pushed a commit to branch release-v1.0
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/release-v1.0 by this push:
new e196ad3af cherry pick #7652 #7783 to v1.0 (#7790)
e196ad3af is described below
commit e196ad3af241ac3b22c84b4dc78c7d5e67ec2799
Author: Lynwee <15...@qq.com>
AuthorDate: Mon Jul 29 14:20:28 2024 +0800
cherry pick #7652 #7783 to v1.0 (#7790)
* feat(dora): add incident related tables
* fix(ci): fix lint/e2e/test errors
* fix(ci): fix unit test errors
* feat(webhook): issues with type = "INCIDENT" will be stored/updated in table `incidents`
* fix(incident): fix method `IsIncident`
* feat(dora): add new option `DisableIssueToIncidentGenerator`
* fix(dora): fix test
---
.../core/models/domainlayer/code/pull_request.go | 53 +++++++
...=> project_incident_deployment_relationship.go} | 6 +-
.../models/domainlayer/domaininfo/domaininfo.go | 4 +-
.../domainlayer/ticket/{issue.go => incident.go} | 70 ++--------
.../incident_assginee.go} | 20 +--
backend/core/models/domainlayer/ticket/issue.go | 51 ++++++-
...40621_init_incidents_and_incident_assignees.go} | 35 ++---
.../20240621_rename_project_issue_metrics.go} | 31 ++---
.../archived/incident.go} | 71 ++--------
.../archived/incident_assignee.go} | 18 ++-
backend/core/models/migrationscripts/register.go | 2 +
.../dora/e2e/connect_incident_to_deploy_test.go | 6 +-
... project_incident_deployment_relationships.csv} | 0
backend/plugins/dora/impl/impl.go | 2 +
backend/plugins/dora/impl/impl_test.go | 2 +
.../dora/tasks/incident_deploy_connector.go | 8 +-
.../dora/tasks/incident_from_issue_generator.go | 154 +++++++++++++++++++++
backend/plugins/dora/tasks/task_data.go | 3 +-
backend/plugins/starrocks/utils/utils.go | 2 +-
backend/plugins/webhook/api/issues.go | 67 +++++++--
.../pydevlake/domain_layer/crossdomain.py | 4 +-
backend/server/services/project.go | 6 +-
grafana/dashboards/DORA.json | 10 +-
grafana/dashboards/DORAByTeam.json | 10 +-
grafana/dashboards/DORADebug.json | 12 +-
.../dashboards/DORADetails-ChangeFailureRate.json | 6 +-
.../DORADetails-FailedDeploymentRecoveryTime.json | 6 +-
27 files changed, 425 insertions(+), 234 deletions(-)
diff --git a/backend/core/models/domainlayer/code/pull_request.go b/backend/core/models/domainlayer/code/pull_request.go
index 348129e76..997ab2bbb 100644
--- a/backend/core/models/domainlayer/code/pull_request.go
+++ b/backend/core/models/domainlayer/code/pull_request.go
@@ -18,6 +18,8 @@ limitations under the License.
package code
import (
+ "fmt"
+ "github.com/apache/incubator-devlake/core/models/domainlayer/ticket"
"time"
"github.com/apache/incubator-devlake/core/models/domainlayer"
@@ -63,3 +65,54 @@ type PullRequest struct {
func (PullRequest) TableName() string {
return "pull_requests"
}
+
+func (pr PullRequest) ConvertStatusToIncidentStatus() string {
+ switch pr.Status {
+ case OPEN:
+ return ticket.TODO
+ case CLOSED:
+ return ticket.OTHER
+ case MERGED:
+ return ticket.DONE
+ default:
+ return ticket.OTHER
+ }
+}
+
+func (pr PullRequest) ToIncident() (*ticket.Incident, error) {
+ incident := &ticket.Incident{
+ DomainEntity: pr.DomainEntity,
+ Url: pr.Url,
+ IncidentKey: fmt.Sprintf("%d", pr.PullRequestKey),
+ Title: pr.Title,
+ Description: pr.Description,
+ Status: pr.ConvertStatusToIncidentStatus(),
+ OriginalStatus: pr.OriginalStatus,
+ ResolutionDate: pr.MergedDate,
+ CreatedDate: &pr.CreatedDate,
+ OriginalEstimateMinutes: nil,
+ TimeSpentMinutes: nil,
+ TimeRemainingMinutes: nil,
+ CreatorId: pr.AuthorId,
+ CreatorName: pr.AuthorName,
+ ParentIncidentId: pr.ParentPrId,
+ Priority: "",
+ Severity: "",
+ Urgency: "",
+ Component: pr.Component,
+ OriginalProject: "",
+ }
+
+ if pr.MergedDate != nil {
+ incident.UpdatedDate = pr.MergedDate
+ }
+ if incident.UpdatedDate == nil {
+ incident.UpdatedDate = pr.ClosedDate
+ }
+
+ if pr.MergedDate != nil {
+ temp := uint(pr.MergedDate.Sub(pr.CreatedDate).Minutes())
+ incident.LeadTimeMinutes = &temp
+ }
+ return incident, nil
+}
diff --git a/backend/core/models/domainlayer/crossdomain/project_issue_metric.go b/backend/core/models/domainlayer/crossdomain/project_incident_deployment_relationship.go
similarity index 85%
copy from backend/core/models/domainlayer/crossdomain/project_issue_metric.go
copy to backend/core/models/domainlayer/crossdomain/project_incident_deployment_relationship.go
index 5c2a3cf51..755741c1b 100644
--- a/backend/core/models/domainlayer/crossdomain/project_issue_metric.go
+++ b/backend/core/models/domainlayer/crossdomain/project_incident_deployment_relationship.go
@@ -21,12 +21,12 @@ import (
"github.com/apache/incubator-devlake/core/models/domainlayer"
)
-type ProjectIssueMetric struct {
+type ProjectIncidentDeploymentRelationship struct {
domainlayer.DomainEntity
ProjectName string `gorm:"primaryKey;type:varchar(100)"`
DeploymentId string
}
-func (ProjectIssueMetric) TableName() string {
- return "project_issue_metrics"
+func (ProjectIncidentDeploymentRelationship) TableName() string {
+ return "project_incident_deployment_relationships"
}
diff --git a/backend/core/models/domainlayer/domaininfo/domaininfo.go b/backend/core/models/domainlayer/domaininfo/domaininfo.go
index 422f537ab..6837126f8 100644
--- a/backend/core/models/domainlayer/domaininfo/domaininfo.go
+++ b/backend/core/models/domainlayer/domaininfo/domaininfo.go
@@ -60,7 +60,7 @@ func GetDomainTablesInfo() []dal.Tabler {
&crossdomain.IssueCommit{},
&crossdomain.IssueRepoCommit{},
&crossdomain.ProjectMapping{},
- &crossdomain.ProjectIssueMetric{},
+ &crossdomain.ProjectIncidentDeploymentRelationship{},
&crossdomain.ProjectPrMetric{},
&crossdomain.PullRequestIssue{},
&crossdomain.RefsIssuesDiffs{},
@@ -91,5 +91,7 @@ func GetDomainTablesInfo() []dal.Tabler {
&ticket.IssueAssignee{},
&ticket.IssueRelationship{},
&ticket.IssueCustomArrayField{},
+ &ticket.Incident{},
+ &ticket.IncidentAssignee{},
}
}
diff --git a/backend/core/models/domainlayer/ticket/issue.go b/backend/core/models/domainlayer/ticket/incident.go
similarity index 55%
copy from backend/core/models/domainlayer/ticket/issue.go
copy to backend/core/models/domainlayer/ticket/incident.go
index 7f29404d8..b565cd3b9 100644
--- a/backend/core/models/domainlayer/ticket/issue.go
+++ b/backend/core/models/domainlayer/ticket/incident.go
@@ -18,24 +18,21 @@ limitations under the License.
package ticket
import (
- "time"
-
"github.com/apache/incubator-devlake/core/models/domainlayer"
+ "time"
)
-type Issue struct {
+// Incident comes from two tables: issues and pull_requests, it won't be produced from tool layer tables so far.
+// 1.From issues, all fields are the same with issues' record, include id field.
+// 2.From pull_requests, id field is pull_requests.id, other fields should be transformed or calculated.
+type Incident struct {
domainlayer.DomainEntity
Url string `gorm:"type:varchar(255)"`
- IconURL string `gorm:"type:varchar(255);column:icon_url"`
- IssueKey string `gorm:"type:varchar(255)"`
+ IncidentKey string `gorm:"type:varchar(255)"` // issue_key/pull_request_key
Title string
Description string
- EpicKey string `gorm:"type:varchar(255)"`
- Type string `gorm:"type:varchar(100)"`
- OriginalType string `gorm:"type:varchar(500)"`
Status string `gorm:"type:varchar(100)"`
OriginalStatus string `gorm:"type:varchar(100)"`
- StoryPoint *float64
ResolutionDate *time.Time
CreatedDate *time.Time
UpdatedDate *time.Time
@@ -45,9 +42,7 @@ type Issue struct {
TimeRemainingMinutes *int64
CreatorId string `gorm:"type:varchar(255)"`
CreatorName string `gorm:"type:varchar(255)"`
- AssigneeId string `gorm:"type:varchar(255)"`
- AssigneeName string `gorm:"type:varchar(255)"`
- ParentIssueId string `gorm:"type:varchar(255)"`
+ ParentIncidentId string `gorm:"type:varchar(255)"`
Priority string `gorm:"type:varchar(255)"`
Severity string `gorm:"type:varchar(255)"`
Urgency string `gorm:"type:varchar(255)"`
@@ -55,53 +50,6 @@ type Issue struct {
OriginalProject string `gorm:"type:varchar(255)"`
}
-func (Issue) TableName() string {
- return "issues"
-}
-
-const (
- BUG = "BUG"
- REQUIREMENT = "REQUIREMENT"
- INCIDENT = "INCIDENT"
- TASK = "TASK"
- SUBTASK = "SUBTASK"
-
- // status
- TODO = "TODO"
- DONE = "DONE"
- IN_PROGRESS = "IN_PROGRESS"
- OTHER = "OTHER"
-)
-
-type StatusRule struct {
- InProgress []string
- Todo []string
- Done []string
- Other []string
- Default string
-}
-
-// GetStatus compare the input with rule for return the enmu value of status
-func GetStatus(rule *StatusRule, input interface{}) string {
- for _, inp := range rule.InProgress {
- if inp == input {
- return IN_PROGRESS
- }
- }
- for _, Todo := range rule.Todo {
- if Todo == input {
- return TODO
- }
- }
- for _, done := range rule.Done {
- if done == input {
- return DONE
- }
- }
- for _, Other := range rule.Other {
- if Other == input {
- return OTHER
- }
- }
- return rule.Default
+func (Incident) TableName() string {
+ return "incidents"
}
diff --git a/backend/core/models/domainlayer/crossdomain/project_issue_metric.go b/backend/core/models/domainlayer/ticket/incident_assginee.go
similarity index 67%
copy from backend/core/models/domainlayer/crossdomain/project_issue_metric.go
copy to backend/core/models/domainlayer/ticket/incident_assginee.go
index 5c2a3cf51..d4e16c993 100644
--- a/backend/core/models/domainlayer/crossdomain/project_issue_metric.go
+++ b/backend/core/models/domainlayer/ticket/incident_assginee.go
@@ -15,18 +15,18 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
-package crossdomain
+package ticket
-import (
- "github.com/apache/incubator-devlake/core/models/domainlayer"
-)
+import "github.com/apache/incubator-devlake/core/models/common"
-type ProjectIssueMetric struct {
- domainlayer.DomainEntity
- ProjectName string `gorm:"primaryKey;type:varchar(100)"`
- DeploymentId string
+type IncidentAssignee struct {
+ IncidentId string `gorm:"primaryKey;type:varchar(255)"`
+ AssigneeId string `gorm:"primaryKey;type:varchar(255)"`
+ AssigneeName string `gorm:"type:varchar(255)"`
+
+ common.NoPKModel
}
-func (ProjectIssueMetric) TableName() string {
- return "project_issue_metrics"
+func (IncidentAssignee) TableName() string {
+ return "incident_assignees"
}
diff --git a/backend/core/models/domainlayer/ticket/issue.go b/backend/core/models/domainlayer/ticket/issue.go
index 7f29404d8..816571a4a 100644
--- a/backend/core/models/domainlayer/ticket/issue.go
+++ b/backend/core/models/domainlayer/ticket/issue.go
@@ -18,9 +18,9 @@ limitations under the License.
package ticket
import (
- "time"
-
+ "errors"
"github.com/apache/incubator-devlake/core/models/domainlayer"
+ "time"
)
type Issue struct {
@@ -105,3 +105,50 @@ func GetStatus(rule *StatusRule, input interface{}) string {
}
return rule.Default
}
+
+func (issue Issue) IsIncident() bool {
+ return issue.Type == INCIDENT
+}
+
+func (issue Issue) ToIncidentAssignee() (*IncidentAssignee, error) {
+ if !issue.IsIncident() {
+ return nil, errors.New("issue type is not INCIDENT, cannot generate incident_assignee")
+ }
+ return &IncidentAssignee{
+ IncidentId: issue.Id,
+ AssigneeId: issue.AssigneeId,
+ AssigneeName: issue.AssigneeName,
+ NoPKModel: issue.DomainEntity.NoPKModel,
+ }, nil
+}
+
+func (issue Issue) ToIncident() (*Incident, error) {
+ if !issue.IsIncident() {
+ return nil, errors.New("issue type is not INCIDENT, cannot generate incident")
+ }
+ incident := &Incident{
+ DomainEntity: issue.DomainEntity,
+ Url: issue.Url,
+ IncidentKey: issue.IssueKey,
+ Title: issue.Title,
+ Description: issue.Description,
+ Status: issue.Status,
+ OriginalStatus: issue.OriginalStatus,
+ ResolutionDate: issue.ResolutionDate,
+ CreatedDate: issue.CreatedDate,
+ UpdatedDate: issue.UpdatedDate,
+ LeadTimeMinutes: issue.LeadTimeMinutes,
+ OriginalEstimateMinutes: issue.OriginalEstimateMinutes,
+ TimeSpentMinutes: issue.TimeSpentMinutes,
+ TimeRemainingMinutes: issue.TimeRemainingMinutes,
+ CreatorId: issue.CreatorId,
+ CreatorName: issue.CreatorName,
+ ParentIncidentId: issue.ParentIssueId,
+ Priority: issue.Priority,
+ Severity: issue.Severity,
+ Urgency: issue.Urgency,
+ Component: issue.Component,
+ OriginalProject: issue.OriginalProject,
+ }
+ return incident, nil
+}
diff --git a/backend/plugins/dora/tasks/task_data.go b/backend/core/models/migrationscripts/20240621_init_incidents_and_incident_assignees.go
similarity index 56%
copy from backend/plugins/dora/tasks/task_data.go
copy to backend/core/models/migrationscripts/20240621_init_incidents_and_incident_assignees.go
index 0af005446..d98275162 100644
--- a/backend/plugins/dora/tasks/task_data.go
+++ b/backend/core/models/migrationscripts/20240621_init_incidents_and_incident_assignees.go
@@ -15,34 +15,29 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
-package tasks
+package migrationscripts
import (
+ "github.com/apache/incubator-devlake/core/context"
"github.com/apache/incubator-devlake/core/errors"
- helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "github.com/apache/incubator-devlake/core/models/migrationscripts/archived"
+ "github.com/apache/incubator-devlake/helpers/migrationhelper"
)
-type DoraApiParams struct {
- ProjectName string
-}
+type initIncidentRelatedTables struct{}
-type DoraOptions struct {
- Tasks []string `json:"tasks,omitempty"`
- Since string
- ProjectName string `json:"projectName"`
- ScopeId *string `json:"scopeId,omitempty"`
+func (u *initIncidentRelatedTables) Up(basicRes context.BasicRes) errors.Error {
+ return migrationhelper.AutoMigrateTables(
+ basicRes,
+ &archived.IncidentAssignee{},
+ &archived.Incident{},
+ )
}
-type DoraTaskData struct {
- Options *DoraOptions
+func (*initIncidentRelatedTables) Version() uint64 {
+ return 20240621152500
}
-func DecodeAndValidateTaskOptions(options map[string]interface{}) (*DoraOptions, errors.Error) {
- var op DoraOptions
- err := helper.Decode(options, &op, nil)
- if err != nil {
- return nil, errors.Default.Wrap(err, "error decoding DORA task options")
- }
-
- return &op, nil
+func (*initIncidentRelatedTables) Name() string {
+ return "init tables: incidents and incident_assignees"
}
diff --git a/backend/plugins/dora/tasks/task_data.go b/backend/core/models/migrationscripts/20240621_rename_project_issue_metrics.go
similarity index 56%
copy from backend/plugins/dora/tasks/task_data.go
copy to backend/core/models/migrationscripts/20240621_rename_project_issue_metrics.go
index 0af005446..76fe3db48 100644
--- a/backend/plugins/dora/tasks/task_data.go
+++ b/backend/core/models/migrationscripts/20240621_rename_project_issue_metrics.go
@@ -15,34 +15,25 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
-package tasks
+package migrationscripts
import (
+ "github.com/apache/incubator-devlake/core/context"
"github.com/apache/incubator-devlake/core/errors"
- helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "github.com/apache/incubator-devlake/core/models/migrationscripts/archived"
)
-type DoraApiParams struct {
- ProjectName string
-}
+type renameProjectIssueMetrics struct{}
-type DoraOptions struct {
- Tasks []string `json:"tasks,omitempty"`
- Since string
- ProjectName string `json:"projectName"`
- ScopeId *string `json:"scopeId,omitempty"`
+func (u *renameProjectIssueMetrics) Up(basicRes context.BasicRes) errors.Error {
+ db := basicRes.GetDal()
+ return db.RenameTable(archived.ProjectIssueMetric{}.TableName(), "project_incident_deployment_relationships")
}
-type DoraTaskData struct {
- Options *DoraOptions
+func (*renameProjectIssueMetrics) Version() uint64 {
+ return 20240621162000
}
-func DecodeAndValidateTaskOptions(options map[string]interface{}) (*DoraOptions, errors.Error) {
- var op DoraOptions
- err := helper.Decode(options, &op, nil)
- if err != nil {
- return nil, errors.Default.Wrap(err, "error decoding DORA task options")
- }
-
- return &op, nil
+func (*renameProjectIssueMetrics) Name() string {
+ return "rename project_issue_metrics to project_incident_deployment_relationships"
}
diff --git a/backend/core/models/domainlayer/ticket/issue.go b/backend/core/models/migrationscripts/archived/incident.go
similarity index 50%
copy from backend/core/models/domainlayer/ticket/issue.go
copy to backend/core/models/migrationscripts/archived/incident.go
index 7f29404d8..8c00af341 100644
--- a/backend/core/models/domainlayer/ticket/issue.go
+++ b/backend/core/models/migrationscripts/archived/incident.go
@@ -15,27 +15,20 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
-package ticket
+package archived
import (
"time"
-
- "github.com/apache/incubator-devlake/core/models/domainlayer"
)
-type Issue struct {
- domainlayer.DomainEntity
+type Incident struct {
+ DomainEntity
Url string `gorm:"type:varchar(255)"`
- IconURL string `gorm:"type:varchar(255);column:icon_url"`
- IssueKey string `gorm:"type:varchar(255)"`
+ IncidentKey string `gorm:"type:varchar(255)"` // issue_key/pull_request_key
Title string
Description string
- EpicKey string `gorm:"type:varchar(255)"`
- Type string `gorm:"type:varchar(100)"`
- OriginalType string `gorm:"type:varchar(500)"`
Status string `gorm:"type:varchar(100)"`
OriginalStatus string `gorm:"type:varchar(100)"`
- StoryPoint *float64
ResolutionDate *time.Time
CreatedDate *time.Time
UpdatedDate *time.Time
@@ -45,63 +38,15 @@ type Issue struct {
TimeRemainingMinutes *int64
CreatorId string `gorm:"type:varchar(255)"`
CreatorName string `gorm:"type:varchar(255)"`
- AssigneeId string `gorm:"type:varchar(255)"`
- AssigneeName string `gorm:"type:varchar(255)"`
- ParentIssueId string `gorm:"type:varchar(255)"`
+ ParentIncidentId string `gorm:"type:varchar(255)"`
Priority string `gorm:"type:varchar(255)"`
Severity string `gorm:"type:varchar(255)"`
Urgency string `gorm:"type:varchar(255)"`
Component string `gorm:"type:varchar(255)"`
- OriginalProject string `gorm:"type:varchar(255)"`
-}
-
-func (Issue) TableName() string {
- return "issues"
-}
-
-const (
- BUG = "BUG"
- REQUIREMENT = "REQUIREMENT"
- INCIDENT = "INCIDENT"
- TASK = "TASK"
- SUBTASK = "SUBTASK"
-
- // status
- TODO = "TODO"
- DONE = "DONE"
- IN_PROGRESS = "IN_PROGRESS"
- OTHER = "OTHER"
-)
-type StatusRule struct {
- InProgress []string
- Todo []string
- Done []string
- Other []string
- Default string
+ OriginalProject string `gorm:"type:varchar(255)"`
}
-// GetStatus compare the input with rule for return the enmu value of status
-func GetStatus(rule *StatusRule, input interface{}) string {
- for _, inp := range rule.InProgress {
- if inp == input {
- return IN_PROGRESS
- }
- }
- for _, Todo := range rule.Todo {
- if Todo == input {
- return TODO
- }
- }
- for _, done := range rule.Done {
- if done == input {
- return DONE
- }
- }
- for _, Other := range rule.Other {
- if Other == input {
- return OTHER
- }
- }
- return rule.Default
+func (Incident) TableName() string {
+ return "incidents"
}
diff --git a/backend/core/models/domainlayer/crossdomain/project_issue_metric.go b/backend/core/models/migrationscripts/archived/incident_assignee.go
similarity index 71%
rename from backend/core/models/domainlayer/crossdomain/project_issue_metric.go
rename to backend/core/models/migrationscripts/archived/incident_assignee.go
index 5c2a3cf51..4207e2fd3 100644
--- a/backend/core/models/domainlayer/crossdomain/project_issue_metric.go
+++ b/backend/core/models/migrationscripts/archived/incident_assignee.go
@@ -15,18 +15,16 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
-package crossdomain
+package archived
-import (
- "github.com/apache/incubator-devlake/core/models/domainlayer"
-)
+type IncidentAssignee struct {
+ IncidentId string `gorm:"primaryKey;type:varchar(255)"`
+ AssigneeId string `gorm:"primaryKey;type:varchar(255)"`
+ AssigneeName string `gorm:"type:varchar(255)"`
-type ProjectIssueMetric struct {
- domainlayer.DomainEntity
- ProjectName string `gorm:"primaryKey;type:varchar(100)"`
- DeploymentId string
+ NoPKModel
}
-func (ProjectIssueMetric) TableName() string {
- return "project_issue_metrics"
+func (IncidentAssignee) TableName() string {
+ return "incident_assignees"
}
diff --git a/backend/core/models/migrationscripts/register.go b/backend/core/models/migrationscripts/register.go
index b4139b2ee..5a5d3c403 100644
--- a/backend/core/models/migrationscripts/register.go
+++ b/backend/core/models/migrationscripts/register.go
@@ -126,5 +126,7 @@ func All() []plugin.MigrationScript {
new(addPullRequestIdIndexToPullRequestCommits),
new(addPullRequestIdIndexToPullRequestComments),
new(addIsDraftToPr),
+ new(initIncidentRelatedTables),
+ new(renameProjectIssueMetrics),
}
}
diff --git a/backend/plugins/dora/e2e/connect_incident_to_deploy_test.go b/backend/plugins/dora/e2e/connect_incident_to_deploy_test.go
index ee340d078..ed625063d 100644
--- a/backend/plugins/dora/e2e/connect_incident_to_deploy_test.go
+++ b/backend/plugins/dora/e2e/connect_incident_to_deploy_test.go
@@ -45,10 +45,10 @@ func TestConnectIncidentToDeploymentDataFlow(t *testing.T) {
dataflowTester.ImportCsvIntoTabler("./raw_tables/issues.csv", &ticket.Issue{})
// verify converter
- dataflowTester.FlushTabler(&crossdomain.ProjectIssueMetric{})
+ dataflowTester.FlushTabler(&crossdomain.ProjectIncidentDeploymentRelationship{})
dataflowTester.Subtask(tasks.ConnectIncidentToDeploymentMeta, taskData)
- dataflowTester.VerifyTableWithOptions(&crossdomain.ProjectIssueMetric{}, e2ehelper.TableOptions{
- CSVRelPath: "./snapshot_tables/project_issue_metrics.csv",
+ dataflowTester.VerifyTableWithOptions(&crossdomain.ProjectIncidentDeploymentRelationship{}, e2ehelper.TableOptions{
+ CSVRelPath: "./snapshot_tables/project_incident_deployment_relationships.csv",
IgnoreTypes: []interface{}{common.NoPKModel{}},
})
}
diff --git a/backend/plugins/dora/e2e/snapshot_tables/project_issue_metrics.csv b/backend/plugins/dora/e2e/snapshot_tables/project_incident_deployment_relationships.csv
similarity index 100%
rename from backend/plugins/dora/e2e/snapshot_tables/project_issue_metrics.csv
rename to backend/plugins/dora/e2e/snapshot_tables/project_incident_deployment_relationships.csv
diff --git a/backend/plugins/dora/impl/impl.go b/backend/plugins/dora/impl/impl.go
index f96a04b5b..a0764a962 100644
--- a/backend/plugins/dora/impl/impl.go
+++ b/backend/plugins/dora/impl/impl.go
@@ -95,6 +95,7 @@ func (p Dora) SubTaskMetas() []plugin.SubTaskMeta {
tasks.EnrichTaskEnvMeta,
tasks.CalculateChangeLeadTimeMeta,
tasks.ConnectIncidentToDeploymentMeta,
+ tasks.IssuesToIncidentsMeta,
}
}
@@ -160,6 +161,7 @@ func (p Dora) MakeMetricPluginPipelinePlanV200(projectName string, options json.
Subtasks: []string{
"calculateChangeLeadTime",
"ConnectIncidentToDeployment",
+ tasks.IssuesToIncidentsMeta.Name,
},
},
},
diff --git a/backend/plugins/dora/impl/impl_test.go b/backend/plugins/dora/impl/impl_test.go
index 461897dc2..54cc7d5c4 100644
--- a/backend/plugins/dora/impl/impl_test.go
+++ b/backend/plugins/dora/impl/impl_test.go
@@ -19,6 +19,7 @@ package impl
import (
"encoding/json"
+ "github.com/apache/incubator-devlake/plugins/dora/tasks"
"testing"
coreModels "github.com/apache/incubator-devlake/core/models"
@@ -61,6 +62,7 @@ func TestMakeMetricPluginPipelinePlanV200(t *testing.T) {
Subtasks: []string{
"calculateChangeLeadTime",
"ConnectIncidentToDeployment",
+ tasks.IssuesToIncidentsMeta.Name,
},
Options: map[string]interface{}{"projectName": projectName},
},
diff --git a/backend/plugins/dora/tasks/incident_deploy_connector.go b/backend/plugins/dora/tasks/incident_deploy_connector.go
index 6d9a1365d..9dc70f812 100644
--- a/backend/plugins/dora/tasks/incident_deploy_connector.go
+++ b/backend/plugins/dora/tasks/incident_deploy_connector.go
@@ -44,13 +44,15 @@ type simpleCicdDeploymentCommit struct {
FinishedDate *time.Time
}
+// ConnectIncidentToDeployment will generate data to crossdomain.ProjectIncidentDeploymentRelationship.
+// FIXME: it should be generated from Incident.
func ConnectIncidentToDeployment(taskCtx plugin.SubTaskContext) errors.Error {
db := taskCtx.GetDal()
data := taskCtx.GetData().(*DoraTaskData)
// Clear previous results from the project
- err := db.Exec("DELETE FROM project_issue_metrics WHERE project_name = ?", data.Options.ProjectName)
+ err := db.Exec("DELETE FROM project_incident_deployment_relationships WHERE project_name = ?", data.Options.ProjectName)
if err != nil {
- return errors.Default.Wrap(err, "error deleting previous project_issue_metrics")
+ return errors.Default.Wrap(err, "error deleting previous project_incident_deployment_relationships")
}
// select all issues belongs to the board
clauses := []dal.Clause{
@@ -80,7 +82,7 @@ func ConnectIncidentToDeployment(taskCtx plugin.SubTaskContext) errors.Error {
Input: cursor,
Convert: func(inputRow interface{}) ([]interface{}, errors.Error) {
issue := inputRow.(*ticket.Issue)
- projectIssueMetric := &crossdomain.ProjectIssueMetric{
+ projectIssueMetric := &crossdomain.ProjectIncidentDeploymentRelationship{
DomainEntity: domainlayer.DomainEntity{
Id: issue.Id,
},
diff --git a/backend/plugins/dora/tasks/incident_from_issue_generator.go b/backend/plugins/dora/tasks/incident_from_issue_generator.go
new file mode 100644
index 000000000..c84c890b4
--- /dev/null
+++ b/backend/plugins/dora/tasks/incident_from_issue_generator.go
@@ -0,0 +1,154 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ goerrors "errors"
+ "github.com/apache/incubator-devlake/core/dal"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/log"
+ "github.com/apache/incubator-devlake/core/models/common"
+ "github.com/apache/incubator-devlake/core/models/domainlayer/ticket"
+ "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "reflect"
+)
+
+var IssuesToIncidentsMeta = plugin.SubTaskMeta{
+ Name: "ConvertIssuesToIncidents",
+ EntryPoint: ConvertIssuesToIncidents,
+ EnabledByDefault: true,
+ Description: "Connect issue to incident",
+ DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET},
+}
+
+func ConvertIssuesToIncidents(taskCtx plugin.SubTaskContext) errors.Error {
+ db := taskCtx.GetDal()
+ data := taskCtx.GetData().(*DoraTaskData)
+ if data.DisableIssueToIncidentGenerator {
+ return nil
+ }
+ // clear previous incidents and incident_assignees from the project
+ deleteIncidentsSql := `
+ DELETE
+ FROM incidents
+ WHERE id IN
+ (SELECT i.id
+ FROM issues i
+ LEFT JOIN board_issues bi ON bi.issue_id = i.id
+ LEFT JOIN project_mapping pm ON pm.row_id = bi.board_id
+ WHERE i.type = "INCIDENT"
+ AND pm.project_name = ?
+ AND pm.table = "boards")
+ `
+ if err := db.Exec(deleteIncidentsSql, data.Options.ProjectName); err != nil {
+ return errors.Default.Wrap(err, "error deleting previous incidents")
+
+ }
+ deleteIncidentAssigneesSql := `
+ DELETE
+ FROM incident_assignees
+ WHERE incident_id IN
+ (SELECT i.id
+ FROM issues i
+ LEFT JOIN board_issues bi ON bi.issue_id = i.id
+ LEFT JOIN project_mapping pm ON pm.row_id = bi.board_id
+ WHERE i.type = "INCIDENT"
+ AND pm.project_name = ?
+ AND pm.table = "boards")
+ `
+ if err := db.Exec(deleteIncidentAssigneesSql, data.Options.ProjectName); err != nil {
+ return errors.Default.Wrap(err, "error deleting previous incident_assignees")
+ }
+
+ // select all issues belongs to the board
+ clauses := []dal.Clause{
+ dal.Select("i.*"),
+ dal.From(`issues i`),
+ dal.Join(`left join board_issues bi on bi.issue_id = i.id`),
+ dal.Join(`left join project_mapping pm on pm.row_id = bi.board_id`),
+ dal.Where(
+ "i.type = ? and pm.project_name = ? and pm.table = ?",
+ "INCIDENT", data.Options.ProjectName, "boards",
+ ),
+ }
+ cursor, err := db.Cursor(clauses...)
+ if err != nil {
+ return err
+ }
+ defer cursor.Close()
+
+ enricher, err := api.NewDataConverter(api.DataConverterArgs{
+ RawDataSubTaskArgs: api.RawDataSubTaskArgs{
+ Ctx: taskCtx,
+ Params: DoraApiParams{
+ ProjectName: data.Options.ProjectName,
+ },
+ Table: ticket.Issue{}.TableName(),
+ },
+ InputRowType: reflect.TypeOf(ticket.Issue{}),
+ Input: cursor,
+ Convert: func(inputRow interface{}) ([]interface{}, errors.Error) {
+ issue := inputRow.(*ticket.Issue)
+ incident, err := issue.ToIncident()
+ if err != nil {
+ return nil, errors.Convert(err)
+ }
+ incidentAssignees, err := generateIncidentAssigneeFromIssue(db, taskCtx.GetLogger(), issue)
+ if err != nil {
+ return nil, errors.Convert(err)
+ }
+ ret := []interface{}{incident}
+ for _, assignee := range incidentAssignees {
+ ret = append(ret, assignee)
+ }
+ return ret, nil
+ },
+ })
+ if err != nil {
+ return err
+ }
+ return enricher.Execute()
+}
+
+func generateIncidentAssigneeFromIssue(db dal.Dal, logger log.Logger, issue *ticket.Issue) ([]*ticket.IncidentAssignee, error) {
+ if issue == nil {
+ return nil, goerrors.New("issue is nil")
+ }
+ var issueAssignees []*ticket.IssueAssignee
+ if err := db.All(&issueAssignees, dal.Where("issue_id = ?", issue.Id)); err != nil {
+ logger.Error(err, "Failed to fetch issue assignees")
+ return nil, err
+ }
+ incidentAssignee, err := issue.ToIncidentAssignee()
+ if err != nil {
+ return nil, err
+ }
+ incidentAssignees := []*ticket.IncidentAssignee{
+ incidentAssignee,
+ }
+ for _, issueAssignee := range issueAssignees {
+ incidentAssignees = append(incidentAssignees, &ticket.IncidentAssignee{
+ IncidentId: issueAssignee.IssueId,
+ AssigneeId: issueAssignee.AssigneeId,
+ AssigneeName: issueAssignee.AssigneeName,
+ NoPKModel: common.NewNoPKModel(),
+ })
+ }
+ return incidentAssignees, nil
+}
diff --git a/backend/plugins/dora/tasks/task_data.go b/backend/plugins/dora/tasks/task_data.go
index 0af005446..3beaf339f 100644
--- a/backend/plugins/dora/tasks/task_data.go
+++ b/backend/plugins/dora/tasks/task_data.go
@@ -34,7 +34,8 @@ type DoraOptions struct {
}
type DoraTaskData struct {
- Options *DoraOptions
+ Options *DoraOptions
+ DisableIssueToIncidentGenerator bool
}
func DecodeAndValidateTaskOptions(options map[string]interface{}) (*DoraOptions, errors.Error) {
diff --git a/backend/plugins/starrocks/utils/utils.go b/backend/plugins/starrocks/utils/utils.go
index 010e7bded..36ba3813e 100644
--- a/backend/plugins/starrocks/utils/utils.go
+++ b/backend/plugins/starrocks/utils/utils.go
@@ -59,7 +59,7 @@ func GetTablesByDomainLayer(domainLayer string) []string {
"board_repos",
"issue_commits",
"issue_repo_commits",
- "project_issue_metrics",
+ "project_incident_deployment_relationships",
"project_mapping",
"project_pr_metrics",
"pull_request_issues",
diff --git a/backend/plugins/webhook/api/issues.go b/backend/plugins/webhook/api/issues.go
index 96c695a28..a73b60e67 100644
--- a/backend/plugins/webhook/api/issues.go
+++ b/backend/plugins/webhook/api/issues.go
@@ -19,6 +19,8 @@ package api
import (
"fmt"
+ "github.com/apache/incubator-devlake/core/log"
+ "github.com/apache/incubator-devlake/helpers/dbhelper"
"net/http"
"time"
@@ -62,6 +64,24 @@ type WebhookIssueRequest struct {
//DeploymentId string
}
+func saveIncidentRelatedRecordsFromIssue(db dal.Transaction, logger log.Logger, issue *ticket.Issue) error {
+ incident, err := issue.ToIncident()
+ if err != nil {
+ return err
+ }
+ if err := db.CreateOrUpdate(incident); err != nil {
+ return err
+ }
+ assignee, err := issue.ToIncidentAssignee()
+ if err != nil {
+ return err
+ }
+ if err := db.CreateOrUpdate(assignee); err != nil {
+ return err
+ }
+ return nil
+}
+
// PostIssue
// @Summary receive a record as defined and save it
// @Description receive a record as follow and save it, example: {"url":"","issue_key":"DLK-1234","title":"a feature from DLK","description":"","epic_key":"","type":"BUG","status":"TODO","original_status":"created","story_point":0,"resolution_date":null,"created_date":"2020-01-01T12:00:00+00:00","updated_date":null,"lead_time_minutes":0,"parent_issue_key":"DLK-1200","priority":"","original_estimate_minutes":0,"time_spent_minutes":0,"time_remaining_minutes":0,"creator_id":"user1131","crea [...]
@@ -89,7 +109,9 @@ func PostIssue(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, error
if err != nil {
return &plugin.ApiResourceOutput{Body: err.Error(), Status: http.StatusBadRequest}, nil
}
- db := basicRes.GetDal()
+ txHelper := dbhelper.NewTxHelper(basicRes, &err)
+ defer txHelper.End()
+ tx := txHelper.Begin()
domainIssue := &ticket.Issue{
DomainEntity: domainlayer.DomainEntity{
Id: fmt.Sprintf("%s:%d:%s", "webhook", connection.ID, request.IssueKey),
@@ -134,7 +156,7 @@ func PostIssue(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, error
}
// check if board exists
- count, err := db.Count(dal.From(&ticket.Board{}), dal.Where("id = ?", domainBoardId))
+ count, err := tx.Count(dal.From(&ticket.Board{}), dal.Where("id = ?", domainBoardId))
if err != nil {
return nil, err
}
@@ -146,22 +168,28 @@ func PostIssue(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, error
Id: domainBoardId,
},
}
- err = db.Create(domainBoard)
+ err = tx.Create(domainBoard)
if err != nil {
return nil, err
}
}
// save
- err = db.CreateOrUpdate(domainIssue)
+ err = tx.CreateOrUpdate(domainIssue)
if err != nil {
return nil, err
}
- err = db.CreateOrUpdate(boardIssue)
+ err = tx.CreateOrUpdate(boardIssue)
if err != nil {
return nil, err
}
+ if domainIssue.IsIncident() {
+ if err := saveIncidentRelatedRecordsFromIssue(tx, logger, domainIssue); err != nil {
+ logger.Error(err, "failed to save incident related records")
+ return nil, errors.Convert(err)
+ }
+ }
return &plugin.ApiResourceOutput{Body: nil, Status: http.StatusOK}, nil
}
@@ -181,19 +209,40 @@ func CloseIssue(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, erro
return nil, err
}
- db := basicRes.GetDal()
+ txHelper := dbhelper.NewTxHelper(basicRes, &err)
+ defer txHelper.End()
+ tx := txHelper.Begin()
+
+ issueId := fmt.Sprintf("%s:%d:%s", "webhook", connection.ID, input.Params[`issueKey`])
domainIssue := &ticket.Issue{}
- err = db.First(domainIssue, dal.Where("id = ?", fmt.Sprintf("%s:%d:%s", "webhook", connection.ID, input.Params[`issueKey`])))
+ err = tx.First(domainIssue, dal.Where("id = ?", issueId))
if err != nil {
return nil, errors.NotFound.Wrap(err, `issue not found`)
}
domainIssue.Status = ticket.DONE
domainIssue.OriginalStatus = ``
-
// save
- err = db.Update(domainIssue)
+ err = tx.Update(domainIssue)
if err != nil {
return nil, err
}
+
+ if domainIssue.IsIncident() {
+ domainIncident := &ticket.Incident{}
+ incidentId := issueId
+ err = tx.First(domainIncident, dal.Where("id = ?", incidentId))
+ if err == nil {
+ domainIncident.Status = ticket.DONE
+ domainIncident.OriginalStatus = ``
+ // save
+ err = tx.Update(domainIncident)
+ if err != nil {
+ return nil, err
+ }
+ } else {
+ logger.Warn(err, "failed to find incident")
+ }
+ }
+
return &plugin.ApiResourceOutput{Body: nil, Status: http.StatusOK}, nil
}
diff --git a/backend/python/pydevlake/pydevlake/domain_layer/crossdomain.py b/backend/python/pydevlake/pydevlake/domain_layer/crossdomain.py
index 3d7d83e7a..b82ee1f02 100644
--- a/backend/python/pydevlake/pydevlake/domain_layer/crossdomain.py
+++ b/backend/python/pydevlake/pydevlake/domain_layer/crossdomain.py
@@ -50,8 +50,8 @@ class IssueRepoCommit(NoPKModel, table=True):
commit_sha: str = Field(primary_key=True)
-class ProjectIssueMetric(NoPKModel, table=True):
- __tablename__ = "project_issue_metrics"
+class ProjectIncidentDeploymentRelationship(NoPKModel, table=True):
+ __tablename__ = "project_incident_deployment_relationships"
project_name: str = Field(primary_key=True)
deployment_id: str
diff --git a/backend/server/services/project.go b/backend/server/services/project.go
index 58ed4b173..2d70fd4d7 100644
--- a/backend/server/services/project.go
+++ b/backend/server/services/project.go
@@ -239,9 +239,9 @@ func PatchProject(name string, body map[string]interface{}) (*models.ApiOutputPr
return nil, err
}
- // ProjectIssueMetric
+ // ProjectIncidentDeploymentRelationship
err = tx.UpdateColumn(
- &crossdomain.ProjectIssueMetric{},
+ &crossdomain.ProjectIncidentDeploymentRelationship{},
"project_name", project.Name,
dal.Where("project_name = ?", name),
)
@@ -350,7 +350,7 @@ func DeleteProject(name string) errors.Error {
if err != nil {
return errors.Default.Wrap(err, "error deleting project PR metric")
}
- err = tx.Delete(&crossdomain.ProjectIssueMetric{}, dal.Where("project_name = ?", name))
+ err = tx.Delete(&crossdomain.ProjectIncidentDeploymentRelationship{}, dal.Where("project_name = ?", name))
if err != nil {
return errors.Default.Wrap(err, "error deleting project Issue metric")
}
diff --git a/grafana/dashboards/DORA.json b/grafana/dashboards/DORA.json
index 0bf338685..5d4191978 100644
--- a/grafana/dashboards/DORA.json
+++ b/grafana/dashboards/DORA.json
@@ -235,7 +235,7 @@
"metricColumn": "none",
"queryType": "randomWalk",
"rawQuery": true,
- "rawSql": "-- Metric 1: Deployment Frequency\nwith last_few_calendar_months as(\n-- construct the last few calendar months within the selected time period in the top-right corner\n\tSELECT CAST(($__timeTo()-INTERVAL (H+T+U) DAY) AS date) day\n\tFROM ( SELECT 0 H\n\t\t\tUNION ALL SELECT 100 UNION ALL SELECT 200 UNION ALL SELECT 300\n\t\t) H CROSS JOIN ( SELECT 0 T\n\t\t\tUNION ALL SELECT 10 UNION ALL SELECT 20 UNION ALL SELECT 30\n\t\t\tUNION ALL SELECT 40 UNION ALL SELECT [...]
+ "rawSql": "-- Metric 1: Deployment Frequency\nwith last_few_calendar_months as(\n-- construct the last few calendar months within the selected time period in the top-right corner\n\tSELECT CAST(($__timeTo()-INTERVAL (H+T+U) DAY) AS date) day\n\tFROM ( SELECT 0 H\n\t\t\tUNION ALL SELECT 100 UNION ALL SELECT 200 UNION ALL SELECT 300\n\t\t) H CROSS JOIN ( SELECT 0 T\n\t\t\tUNION ALL SELECT 10 UNION ALL SELECT 20 UNION ALL SELECT 30\n\t\t\tUNION ALL SELECT 40 UNION ALL SELECT [...]
"refId": "A",
"select": [
[
@@ -669,7 +669,7 @@
"metricColumn": "none",
"queryType": "randomWalk",
"rawQuery": true,
- "rawSql": "-- Metric 3: change failure rate\nwith _deployments as (\n-- When deploying multiple commits in one pipeline, GitLab and BitBucket may generate more than one deployment. However, DevLake consider these deployments as ONE production deployment and use the last one's finished_date as the finished date.\n\tSELECT\n\t\tcdc.cicd_deployment_id as deployment_id,\n\t\tmax(cdc.finished_date) as deployment_finished_date\n\tFROM \n\t\tcicd_deployment_commits cdc\n\t\tJOIN proje [...]
+ "rawSql": "-- Metric 3: change failure rate\nwith _deployments as (\n-- When deploying multiple commits in one pipeline, GitLab and BitBucket may generate more than one deployment. However, DevLake consider these deployments as ONE production deployment and use the last one's finished_date as the finished date.\n\tSELECT\n\t\tcdc.cicd_deployment_id as deployment_id,\n\t\tmax(cdc.finished_date) as deployment_finished_date\n\tFROM \n\t\tcicd_deployment_commits cdc\n\t\tJOIN proje [...]
"refId": "A",
"select": [
[
@@ -821,7 +821,7 @@
"metricColumn": "none",
"queryType": "randomWalk",
"rawQuery": true,
- "rawSql": "-- ***** 2023 report ***** --\n-- Metric 4: Failed deployment recovery time\nwith _deployments as (\n SELECT\n cdc.cicd_deployment_id as deployment_id,\n max(cdc.finished_date) as deployment_finished_date\n FROM \n cicd_deployment_commits cdc\n JOIN project_mapping pm on cdc.cicd_scope_id = pm.row_id and pm.`table` = 'cicd_scopes'\n WHERE\n pm.project_name in ($project)\n and cdc.result = 'SUCCESS'\n and cdc [...]
+ "rawSql": "-- ***** 2023 report ***** --\n-- Metric 4: Failed deployment recovery time\nwith _deployments as (\n SELECT\n cdc.cicd_deployment_id as deployment_id,\n max(cdc.finished_date) as deployment_finished_date\n FROM \n cicd_deployment_commits cdc\n JOIN project_mapping pm on cdc.cicd_scope_id = pm.row_id and pm.`table` = 'cicd_scopes'\n WHERE\n pm.project_name in ($project)\n and cdc.result = 'SUCCESS'\n and cdc [...]
"refId": "A",
"select": [
[
@@ -1237,7 +1237,7 @@
"hide": false,
"metricColumn": "none",
"rawQuery": true,
- "rawSql": "-- Metric 3: change failure rate per month\nwith _deployments as (\n-- When deploying multiple commits in one pipeline, GitLab and BitBucket may generate more than one deployment. However, DevLake consider these deployments as ONE production deployment and use the last one's finished_date as the finished date.\n\tSELECT\n\t\tcdc.cicd_deployment_id as deployment_id,\n\t\tmax(cdc.finished_date) as deployment_finished_date\n\tFROM \n\t\tcicd_deployment_commits cdc\n\t\t [...]
+ "rawSql": "-- Metric 3: change failure rate per month\nwith _deployments as (\n-- When deploying multiple commits in one pipeline, GitLab and BitBucket may generate more than one deployment. However, DevLake consider these deployments as ONE production deployment and use the last one's finished_date as the finished date.\n\tSELECT\n\t\tcdc.cicd_deployment_id as deployment_id,\n\t\tmax(cdc.finished_date) as deployment_finished_date\n\tFROM \n\t\tcicd_deployment_commits cdc\n\t\t [...]
"refId": "A",
"select": [
[
@@ -1393,7 +1393,7 @@
"hide": false,
"metricColumn": "none",
"rawQuery": true,
- "rawSql": "-- ***** 2023 report ***** --\n-- Metric 4: Failed deployment recovery time\nwith _deployments as (\n SELECT\n cdc.cicd_deployment_id as deployment_id,\n max(cdc.finished_date) as deployment_finished_date\n FROM \n cicd_deployment_commits cdc\n JOIN project_mapping pm on cdc.cicd_scope_id = pm.row_id and pm.`table` = 'cicd_scopes'\n WHERE\n pm.project_name in ($project)\n and cdc.result = 'SUCCESS'\n and cdc [...]
+ "rawSql": "-- ***** 2023 report ***** --\n-- Metric 4: Failed deployment recovery time\nwith _deployments as (\n SELECT\n cdc.cicd_deployment_id as deployment_id,\n max(cdc.finished_date) as deployment_finished_date\n FROM \n cicd_deployment_commits cdc\n JOIN project_mapping pm on cdc.cicd_scope_id = pm.row_id and pm.`table` = 'cicd_scopes'\n WHERE\n pm.project_name in ($project)\n and cdc.result = 'SUCCESS'\n and cdc [...]
"refId": "A",
"select": [
[
diff --git a/grafana/dashboards/DORAByTeam.json b/grafana/dashboards/DORAByTeam.json
index 5dcda9980..c1901c7bd 100644
--- a/grafana/dashboards/DORAByTeam.json
+++ b/grafana/dashboards/DORAByTeam.json
@@ -208,7 +208,7 @@
"format": "table",
"hide": false,
"rawQuery": true,
- "rawSql": "-- Metric 1: Deployment Frequency\nwith last_few_calendar_months as(\n-- construct the last few calendar months within the selected time period in the top-right corner\n\tSELECT CAST(($__timeTo()-INTERVAL (H+T+U) DAY) AS date) day\n\tFROM ( SELECT 0 H\n\t\t\tUNION ALL SELECT 100 UNION ALL SELECT 200 UNION ALL SELECT 300\n\t\t) H CROSS JOIN ( SELECT 0 T\n\t\t\tUNION ALL SELECT 10 UNION ALL SELECT 20 UNION ALL SELECT 30\n\t\t\tUNION ALL SELECT 40 UNION ALL SELECT [...]
+ "rawSql": "-- Metric 1: Deployment Frequency\nwith last_few_calendar_months as(\n-- construct the last few calendar months within the selected time period in the top-right corner\n\tSELECT CAST(($__timeTo()-INTERVAL (H+T+U) DAY) AS date) day\n\tFROM ( SELECT 0 H\n\t\t\tUNION ALL SELECT 100 UNION ALL SELECT 200 UNION ALL SELECT 300\n\t\t) H CROSS JOIN ( SELECT 0 T\n\t\t\tUNION ALL SELECT 10 UNION ALL SELECT 20 UNION ALL SELECT 30\n\t\t\tUNION ALL SELECT 40 UNION ALL SELECT [...]
"refId": "A",
"sql": {
"columns": [
@@ -578,7 +578,7 @@
"format": "table",
"hide": false,
"rawQuery": true,
- "rawSql": "-- Metric 4: change failure rate\nwith _deployments as (\n-- When deploying multiple commits in one pipeline, GitLab and BitBucket may generate more than one deployment. However, DevLake consider these deployments as ONE production deployment and use the last one's finished_date as the finished date.\n\tSELECT\n\t\tcdc.cicd_deployment_id as deployment_id,\n\t\tmax(cdc.finished_date) as deployment_finished_date\n\tFROM \n\t\tcicd_deployment_commits cdc\n\t JOIN com [...]
+ "rawSql": "-- Metric 4: change failure rate\nwith _deployments as (\n-- When deploying multiple commits in one pipeline, GitLab and BitBucket may generate more than one deployment. However, DevLake consider these deployments as ONE production deployment and use the last one's finished_date as the finished date.\n\tSELECT\n\t\tcdc.cicd_deployment_id as deployment_id,\n\t\tmax(cdc.finished_date) as deployment_finished_date\n\tFROM \n\t\tcicd_deployment_commits cdc\n\t JOIN com [...]
"refId": "A",
"sql": {
"columns": [
@@ -699,7 +699,7 @@
"format": "table",
"hide": false,
"rawQuery": true,
- "rawSql": "-- ***** 2023 report ***** --\n-- Metric 4: Failed deployment recovery time\nwith _deployments as (\n SELECT\n cdc.cicd_deployment_id as deployment_id,\n max(cdc.finished_date) as deployment_finished_date\n FROM \n cicd_deployment_commits cdc\n\t\tJOIN commits c on cdc.commit_sha = c.sha\n join user_accounts ua on c.author_id = ua.account_id\n join users u on ua.user_id = u.id\n join team_users tu on u.id = tu.user_id [...]
+ "rawSql": "-- ***** 2023 report ***** --\n-- Metric 4: Failed deployment recovery time\nwith _deployments as (\n SELECT\n cdc.cicd_deployment_id as deployment_id,\n max(cdc.finished_date) as deployment_finished_date\n FROM \n cicd_deployment_commits cdc\n\t\tJOIN commits c on cdc.commit_sha = c.sha\n join user_accounts ua on c.author_id = ua.account_id\n join users u on ua.user_id = u.id\n join team_users tu on u.id = tu.user_id [...]
"refId": "A",
"sql": {
"columns": [
@@ -1032,7 +1032,7 @@
"format": "table",
"hide": false,
"rawQuery": true,
- "rawSql": "-- Metric 4: change failure rate per month\nwith _deployments as (\n-- When deploying multiple commits in one pipeline, GitLab and BitBucket may generate more than one deployment. However, DevLake consider these deployments as ONE production deployment and use the last one's finished_date as the finished date.\n\tSELECT\n\t\tcdc.cicd_deployment_id as deployment_id,\n\t\tmax(cdc.finished_date) as deployment_finished_date\n\tFROM \n\t\tcicd_deployment_commits cdc\n\t\t [...]
+ "rawSql": "-- Metric 4: change failure rate per month\nwith _deployments as (\n-- When deploying multiple commits in one pipeline, GitLab and BitBucket may generate more than one deployment. However, DevLake consider these deployments as ONE production deployment and use the last one's finished_date as the finished date.\n\tSELECT\n\t\tcdc.cicd_deployment_id as deployment_id,\n\t\tmax(cdc.finished_date) as deployment_finished_date\n\tFROM \n\t\tcicd_deployment_commits cdc\n\t\t [...]
"refId": "A",
"sql": {
"columns": [
@@ -1141,7 +1141,7 @@
"format": "table",
"hide": false,
"rawQuery": true,
- "rawSql": "-- ***** 2023 report ***** --\n-- Metric 4: Failed deployment recovery time\nwith _deployments as (\n SELECT\n cdc.cicd_deployment_id as deployment_id,\n max(cdc.finished_date) as deployment_finished_date\n FROM \n cicd_deployment_commits cdc\n\t\tJOIN commits c on cdc.commit_sha = c.sha\n\tjoin user_accounts ua on c.author_id = ua.account_id\n join users u on ua.user_id = u.id\n join team_users tu on u.id = tu.user_id\n join tea [...]
+ "rawSql": "-- ***** 2023 report ***** --\n-- Metric 4: Failed deployment recovery time\nwith _deployments as (\n SELECT\n cdc.cicd_deployment_id as deployment_id,\n max(cdc.finished_date) as deployment_finished_date\n FROM \n cicd_deployment_commits cdc\n\t\tJOIN commits c on cdc.commit_sha = c.sha\n\tjoin user_accounts ua on c.author_id = ua.account_id\n join users u on ua.user_id = u.id\n join team_users tu on u.id = tu.user_id\n join tea [...]
"refId": "A",
"sql": {
"columns": [
diff --git a/grafana/dashboards/DORADebug.json b/grafana/dashboards/DORADebug.json
index ddc6f4caf..2b3408b01 100644
--- a/grafana/dashboards/DORADebug.json
+++ b/grafana/dashboards/DORADebug.json
@@ -2847,7 +2847,7 @@
]
}
],
- "title": "Step 1. All types of issues in table.issues (rows with 2 green columns will be used to construct project_issue_metrics)",
+ "title": "Step 1. All types of issues in table.issues (rows with 2 green columns will be used to construct project_incident_deployment_relationships)",
"type": "table"
},
{
@@ -2948,7 +2948,7 @@
"metricColumn": "none",
"queryType": "randomWalk",
"rawQuery": true,
- "rawSql": "-- ***** 2023 report ***** --\n-- Metric 4: Failed deployment recovery time\nwith _deployments as (\n SELECT\n cdc.cicd_deployment_id as deployment_id,\n max(cdc.finished_date) as deployment_finished_date\n FROM \n cicd_deployment_commits cdc\n JOIN project_mapping pm on cdc.cicd_scope_id = pm.row_id and pm.`table` = 'cicd_scopes'\n WHERE\n pm.project_name in ($project)\n and cdc.result = 'SUCCESS'\n and cdc [...]
+ "rawSql": "-- ***** 2023 report ***** --\n-- Metric 4: Failed deployment recovery time\nwith _deployments as (\n SELECT\n cdc.cicd_deployment_id as deployment_id,\n max(cdc.finished_date) as deployment_finished_date\n FROM \n cicd_deployment_commits cdc\n JOIN project_mapping pm on cdc.cicd_scope_id = pm.row_id and pm.`table` = 'cicd_scopes'\n WHERE\n pm.project_name in ($project)\n and cdc.result = 'SUCCESS'\n and cdc [...]
"refId": "A",
"select": [
[
@@ -3231,7 +3231,7 @@
"hide": false,
"metricColumn": "none",
"rawQuery": true,
- "rawSql": "-- ***** 2023 report ***** --\n-- Metric 4: Failed deployment recovery time\nwith _deployments as (\n SELECT\n cdc.cicd_deployment_id as deployment_id,\n max(cdc.finished_date) as deployment_finished_date\n FROM \n cicd_deployment_commits cdc\n JOIN project_mapping pm on cdc.cicd_scope_id = pm.row_id and pm.`table` = 'cicd_scopes'\n WHERE\n pm.project_name in ($project)\n and cdc.result = 'SUCCESS'\n and cdc [...]
+ "rawSql": "-- ***** 2023 report ***** --\n-- Metric 4: Failed deployment recovery time\nwith _deployments as (\n SELECT\n cdc.cicd_deployment_id as deployment_id,\n max(cdc.finished_date) as deployment_finished_date\n FROM \n cicd_deployment_commits cdc\n JOIN project_mapping pm on cdc.cicd_scope_id = pm.row_id and pm.`table` = 'cicd_scopes'\n WHERE\n pm.project_name in ($project)\n and cdc.result = 'SUCCESS'\n and cdc [...]
"refId": "A",
"select": [
[
@@ -3369,7 +3369,7 @@
"metricColumn": "none",
"queryType": "randomWalk",
"rawQuery": true,
- "rawSql": "-- Metric 3: change failure rate\nwith _deployments as (\n-- When deploying multiple commits in one pipeline, GitLab and BitBucket may generate more than one deployment. However, DevLake consider these deployments as ONE production deployment and use the last one's finished_date as the finished date.\n\tSELECT\n\t\tcdc.cicd_deployment_id as deployment_id,\n\t\tmax(cdc.finished_date) as deployment_finished_date\n\tFROM \n\t\tcicd_deployment_commits cdc\n\t\tJOIN proje [...]
+ "rawSql": "-- Metric 3: change failure rate\nwith _deployments as (\n-- When deploying multiple commits in one pipeline, GitLab and BitBucket may generate more than one deployment. However, DevLake consider these deployments as ONE production deployment and use the last one's finished_date as the finished date.\n\tSELECT\n\t\tcdc.cicd_deployment_id as deployment_id,\n\t\tmax(cdc.finished_date) as deployment_finished_date\n\tFROM \n\t\tcicd_deployment_commits cdc\n\t\tJOIN proje [...]
"refId": "A",
"select": [
[
@@ -3589,7 +3589,7 @@
"metricColumn": "none",
"queryType": "randomWalk",
"rawQuery": true,
- "rawSql": "select \n -- in CFR we use deployment_commit_id as the deployment_id in a specific repo\n cdc.cicd_deployment_id as deployment_id,\n cdc.finished_date,\n pim.id as incident_id,\n if (pim.id is not null, 'TRUE', 'FALSE') as has_failure\nfrom \n cicd_deployment_commits cdc\n left join project_issue_metrics pim on cdc.cicd_deployment_id = pim.deployment_id\n left join project_mapping pm on cdc.cicd_scope_id = pm.row_id and pm.`table` = 'cicd_scopes'\nwhere \n p [...]
+ "rawSql": "select \n -- in CFR we use deployment_commit_id as the deployment_id in a specific repo\n cdc.cicd_deployment_id as deployment_id,\n cdc.finished_date,\n pim.id as incident_id,\n if (pim.id is not null, 'TRUE', 'FALSE') as has_failure\nfrom \n cicd_deployment_commits cdc\n left join project_incident_deployment_relationships pim on cdc.cicd_deployment_id = pim.deployment_id\n left join project_mapping pm on cdc.cicd_scope_id = pm.row_id and pm.`table` = 'cicd_ [...]
"refId": "A",
"select": [
[
@@ -3736,7 +3736,7 @@
"hide": false,
"metricColumn": "none",
"rawQuery": true,
- "rawSql": "-- Metric 3: change failure rate per month\nwith _deployments as (\n-- When deploying multiple commits in one pipeline, GitLab and BitBucket may generate more than one deployment. However, DevLake consider these deployments as ONE production deployment and use the last one's finished_date as the finished date.\n\tSELECT\n\t\tcdc.cicd_deployment_id as deployment_id,\n\t\tmax(cdc.finished_date) as deployment_finished_date\n\tFROM \n\t\tcicd_deployment_commits cdc\n\t\t [...]
+ "rawSql": "-- Metric 3: change failure rate per month\nwith _deployments as (\n-- When deploying multiple commits in one pipeline, GitLab and BitBucket may generate more than one deployment. However, DevLake consider these deployments as ONE production deployment and use the last one's finished_date as the finished date.\n\tSELECT\n\t\tcdc.cicd_deployment_id as deployment_id,\n\t\tmax(cdc.finished_date) as deployment_finished_date\n\tFROM \n\t\tcicd_deployment_commits cdc\n\t\t [...]
"refId": "A",
"select": [
[
diff --git a/grafana/dashboards/DORADetails-ChangeFailureRate.json b/grafana/dashboards/DORADetails-ChangeFailureRate.json
index 3322f683a..e9d165ce7 100644
--- a/grafana/dashboards/DORADetails-ChangeFailureRate.json
+++ b/grafana/dashboards/DORADetails-ChangeFailureRate.json
@@ -164,7 +164,7 @@
"metricColumn": "none",
"queryType": "randomWalk",
"rawQuery": true,
- "rawSql": "-- Metric 3: change failure rate\nwith _deployments as (\n-- When deploying multiple commits in one pipeline, GitLab and BitBucket may generate more than one deployment. However, DevLake consider these deployments as ONE production deployment and use the last one's finished_date as the finished date.\n\tSELECT\n\t\tcdc.cicd_deployment_id as deployment_id,\n\t\tmax(cdc.finished_date) as deployment_finished_date\n\tFROM \n\t\tcicd_deployment_commits cdc\n\t\tJOIN proje [...]
+ "rawSql": "-- Metric 3: change failure rate\nwith _deployments as (\n-- When deploying multiple commits in one pipeline, GitLab and BitBucket may generate more than one deployment. However, DevLake consider these deployments as ONE production deployment and use the last one's finished_date as the finished date.\n\tSELECT\n\t\tcdc.cicd_deployment_id as deployment_id,\n\t\tmax(cdc.finished_date) as deployment_finished_date\n\tFROM \n\t\tcicd_deployment_commits cdc\n\t\tJOIN proje [...]
"refId": "A",
"select": [
[
@@ -365,7 +365,7 @@
"metricColumn": "none",
"queryType": "randomWalk",
"rawQuery": true,
- "rawSql": "-- Metric 3: change failure rate\nwith _deployments as (\n-- When deploying multiple commits in one pipeline, GitLab and BitBucket may generate more than one deployment. However, DevLake consider these deployments as ONE production deployment and use the last one's finished_date as the finished date.\n\tSELECT\n\t\tcdc.cicd_deployment_id as deployment_id,\n\t\tmax(cdc.finished_date) as deployment_finished_date\n\tFROM \n\t\tcicd_deployment_commits cdc\n\t\tJOIN proje [...]
+ "rawSql": "-- Metric 3: change failure rate\nwith _deployments as (\n-- When deploying multiple commits in one pipeline, GitLab and BitBucket may generate more than one deployment. However, DevLake consider these deployments as ONE production deployment and use the last one's finished_date as the finished date.\n\tSELECT\n\t\tcdc.cicd_deployment_id as deployment_id,\n\t\tmax(cdc.finished_date) as deployment_finished_date\n\tFROM \n\t\tcicd_deployment_commits cdc\n\t\tJOIN proje [...]
"refId": "A",
"select": [
[
@@ -502,7 +502,7 @@
"metricColumn": "none",
"queryType": "randomWalk",
"rawQuery": true,
- "rawSql": "-- Metric 3: change failure rate\nwith _deployments as (\n-- When deploying multiple commits in one pipeline, GitLab and BitBucket may generate more than one deployment. However, DevLake consider these deployments as ONE production deployment and use the last one's finished_date as the finished date.\n\tSELECT\n\t\tcdc.cicd_deployment_id as deployment_id,\n\t\tmax(cdc.finished_date) as deployment_finished_date\n\tFROM \n\t\tcicd_deployment_commits cdc\n\t\tJOIN proje [...]
+ "rawSql": "-- Metric 3: change failure rate\nwith _deployments as (\n-- When deploying multiple commits in one pipeline, GitLab and BitBucket may generate more than one deployment. However, DevLake consider these deployments as ONE production deployment and use the last one's finished_date as the finished date.\n\tSELECT\n\t\tcdc.cicd_deployment_id as deployment_id,\n\t\tmax(cdc.finished_date) as deployment_finished_date\n\tFROM \n\t\tcicd_deployment_commits cdc\n\t\tJOIN proje [...]
"refId": "A",
"select": [
[
diff --git a/grafana/dashboards/DORADetails-FailedDeploymentRecoveryTime.json b/grafana/dashboards/DORADetails-FailedDeploymentRecoveryTime.json
index 056e40c3d..edc06606d 100644
--- a/grafana/dashboards/DORADetails-FailedDeploymentRecoveryTime.json
+++ b/grafana/dashboards/DORADetails-FailedDeploymentRecoveryTime.json
@@ -121,7 +121,7 @@
"format": "table",
"hide": false,
"rawQuery": true,
- "rawSql": "-- ***** 2023 report ***** --\n-- Metric 4: Failed deployment recovery time\nwith _deployments as (\n SELECT\n cdc.cicd_deployment_id as deployment_id,\n max(cdc.finished_date) as deployment_finished_date\n FROM \n cicd_deployment_commits cdc\n JOIN project_mapping pm on cdc.cicd_scope_id = pm.row_id and pm.`table` = 'cicd_scopes'\n WHERE\n pm.project_name in ($project)\n and cdc.result = 'SUCCESS'\n and cdc [...]
+ "rawSql": "-- ***** 2023 report ***** --\n-- Metric 4: Failed deployment recovery time\nwith _deployments as (\n SELECT\n cdc.cicd_deployment_id as deployment_id,\n max(cdc.finished_date) as deployment_finished_date\n FROM \n cicd_deployment_commits cdc\n JOIN project_mapping pm on cdc.cicd_scope_id = pm.row_id and pm.`table` = 'cicd_scopes'\n WHERE\n pm.project_name in ($project)\n and cdc.result = 'SUCCESS'\n and cdc [...]
"refId": "D",
"sql": {
"columns": [
@@ -315,7 +315,7 @@
"metricColumn": "none",
"queryType": "randomWalk",
"rawQuery": true,
- "rawSql": "-- ***** 2023 report ***** --\n-- Metric 4: Failed deployment recovery time\nwith _deployments as (\n SELECT\n cdc.cicd_deployment_id as deployment_id,\n max(cdc.finished_date) as deployment_finished_date\n FROM \n cicd_deployment_commits cdc\n JOIN project_mapping pm on cdc.cicd_scope_id = pm.row_id and pm.`table` = 'cicd_scopes'\n WHERE\n pm.project_name in ($project)\n and cdc.result = 'SUCCESS'\n and cdc [...]
+ "rawSql": "-- ***** 2023 report ***** --\n-- Metric 4: Failed deployment recovery time\nwith _deployments as (\n SELECT\n cdc.cicd_deployment_id as deployment_id,\n max(cdc.finished_date) as deployment_finished_date\n FROM \n cicd_deployment_commits cdc\n JOIN project_mapping pm on cdc.cicd_scope_id = pm.row_id and pm.`table` = 'cicd_scopes'\n WHERE\n pm.project_name in ($project)\n and cdc.result = 'SUCCESS'\n and cdc [...]
"refId": "A",
"select": [
[
@@ -409,7 +409,7 @@
"format": "table",
"hide": false,
"rawQuery": true,
- "rawSql": "-- ***** 2023 report ***** --\n-- Metric 4: Failed deployment recovery time\nwith _deployments as (\n SELECT\n cdc.cicd_deployment_id as deployment_id,\n max(cdc.finished_date) as deployment_finished_date\n FROM \n cicd_deployment_commits cdc\n JOIN project_mapping pm on cdc.cicd_scope_id = pm.row_id and pm.`table` = 'cicd_scopes'\n WHERE\n pm.project_name in ($project)\n and cdc.result = 'SUCCESS'\n and cdc [...]
+ "rawSql": "-- ***** 2023 report ***** --\n-- Metric 4: Failed deployment recovery time\nwith _deployments as (\n SELECT\n cdc.cicd_deployment_id as deployment_id,\n max(cdc.finished_date) as deployment_finished_date\n FROM \n cicd_deployment_commits cdc\n JOIN project_mapping pm on cdc.cicd_scope_id = pm.row_id and pm.`table` = 'cicd_scopes'\n WHERE\n pm.project_name in ($project)\n and cdc.result = 'SUCCESS'\n and cdc [...]
"refId": "D",
"sql": {
"columns": [