You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by ab...@apache.org on 2024/04/28 10:17:00 UTC
(incubator-devlake) tag revert-dora-empty-v20240229-6 created (now 4823fc326)
This is an automated email from the ASF dual-hosted git repository.
abeizn pushed a change to tag revert-dora-empty-v20240229-6
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
at 4823fc326 (commit)
This tag includes the following new commits:
new f83a09f62 Revert "fix: empty dora data when run the tasks (#7070) (#7072)"
new 4823fc326 Revert "fix: cicd deployment dora delete when data nil (#7033) (#7044)"
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(incubator-devlake) 01/02: Revert "fix: empty dora data when run the tasks (#7070) (#7072)"
Posted by ab...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
abeizn pushed a commit to tag revert-dora-empty-v20240229-6
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit f83a09f620e4323218a1f058adc29319a6e99b06
Author: abeizn <zi...@merico.dev>
AuthorDate: Thu Mar 7 11:48:54 2024 +0800
Revert "fix: empty dora data when run the tasks (#7070) (#7072)"
This reverts commit f82045dfc2119110719954987368cbd7372feb60.
---
.../dora/tasks/change_lead_time_calculator.go | 19 +++++++++---
.../dora/tasks/deployment_commits_generator.go | 34 ++++++++------------
backend/plugins/dora/tasks/deployment_generator.go | 36 +++++++++-------------
.../dora/tasks/incident_deploy_connector.go | 19 +++++++++---
4 files changed, 55 insertions(+), 53 deletions(-)
diff --git a/backend/plugins/dora/tasks/change_lead_time_calculator.go b/backend/plugins/dora/tasks/change_lead_time_calculator.go
index 8e29f1678..dc5134cbe 100644
--- a/backend/plugins/dora/tasks/change_lead_time_calculator.go
+++ b/backend/plugins/dora/tasks/change_lead_time_calculator.go
@@ -18,6 +18,7 @@ limitations under the License.
package tasks
import (
+ "fmt"
"math"
"reflect"
"time"
@@ -46,11 +47,6 @@ func CalculateChangeLeadTime(taskCtx plugin.SubTaskContext) errors.Error {
db := taskCtx.GetDal()
logger := taskCtx.GetLogger()
data := taskCtx.GetData().(*DoraTaskData)
- // Clear previous results from the project
- err := db.Exec("DELETE FROM project_pr_metrics WHERE project_name = ? ", data.Options.ProjectName)
- if err != nil {
- return errors.Default.Wrap(err, "error deleting previous project_pr_metrics")
- }
// Get pull requests by repo project_name
var clauses = []dal.Clause{
@@ -64,6 +60,19 @@ func CalculateChangeLeadTime(taskCtx plugin.SubTaskContext) errors.Error {
return err
}
defer cursor.Close()
+ count, err := db.Count(clauses...)
+ if err != nil {
+ return errors.Default.Wrap(err, "error getting count of clauses")
+ }
+ if count == 0 {
+ // Clear previous results from the project
+ deleteSql := fmt.Sprintf("DELETE FROM project_pr_metrics WHERE project_name = '%s'", data.Options.ProjectName)
+ err := db.Exec(deleteSql)
+ if err != nil {
+ return errors.Default.Wrap(err, "error deleting previous project_pr_metrics")
+ }
+ return nil
+ }
converter, err := api.NewDataConverter(api.DataConverterArgs{
RawDataSubTaskArgs: api.RawDataSubTaskArgs{
diff --git a/backend/plugins/dora/tasks/deployment_commits_generator.go b/backend/plugins/dora/tasks/deployment_commits_generator.go
index 64133ce06..d960f5785 100644
--- a/backend/plugins/dora/tasks/deployment_commits_generator.go
+++ b/backend/plugins/dora/tasks/deployment_commits_generator.go
@@ -101,38 +101,30 @@ func GenerateDeploymentCommits(taskCtx plugin.SubTaskContext) errors.Error {
}
if data.Options.ScopeId != nil {
clauses = append(clauses, dal.Where(`p.cicd_scope_id = ?`, data.Options.ScopeId))
- // Clear previous results from the project
- deleteSql := `DELETE FROM cicd_deployment_commits WHERE cicd_scope_id = ? ;`
- err := db.Exec(deleteSql, data.Options.ScopeId)
- if err != nil {
- return errors.Default.Wrap(err, "error deleting previous cicd_deployment_commits")
- }
} else {
clauses = append(clauses,
dal.Join("LEFT JOIN project_mapping pm ON (pm.table = 'cicd_scopes' AND pm.row_id = p.cicd_scope_id)"),
dal.Where(`pm.project_name = ?`, data.Options.ProjectName),
)
- // Clear previous results from the project
- deleteSql := `DELETE FROM cicd_deployment_commits
- WHERE cicd_scope_id IN (
- SELECT cicd_scope_id
- FROM (
- SELECT cdc.cicd_scope_id
- FROM cicd_deployment_commits cdc
- LEFT JOIN project_mapping pm ON (pm.table = 'cicd_scopes' AND pm.row_id = cdc.cicd_scope_id)
- WHERE pm.project_name = ?
- ) AS subquery
- );`
- err := db.Exec(deleteSql, data.Options.ProjectName)
- if err != nil {
- return errors.Default.Wrap(err, "error deleting previous cicd_deployment_commits")
- }
}
cursor, err := db.Cursor(clauses...)
if err != nil {
return err
}
defer cursor.Close()
+ count, err := db.Count(clauses...)
+ if err != nil {
+ return errors.Default.Wrap(err, "error getting count of clauses")
+ }
+ if count == 0 {
+ // Clear previous results from the project
+ deleteSql := fmt.Sprintf("DELETE cdc FROM cicd_deployment_commits cdc LEFT JOIN project_mapping pm ON (pm.table = 'cicd_scopes' AND pm.row_id = cdc.cicd_scope_id) WHERE pm.project_name = '%s'", data.Options.ProjectName)
+ err := db.Exec(deleteSql)
+ if err != nil {
+ return errors.Default.Wrap(err, "error deleting previous cicd_deployment_commits")
+ }
+ return nil
+ }
enricher, err := api.NewDataConverter(api.DataConverterArgs{
RawDataSubTaskArgs: api.RawDataSubTaskArgs{
diff --git a/backend/plugins/dora/tasks/deployment_generator.go b/backend/plugins/dora/tasks/deployment_generator.go
index 237b7216c..f7caada5e 100644
--- a/backend/plugins/dora/tasks/deployment_generator.go
+++ b/backend/plugins/dora/tasks/deployment_generator.go
@@ -18,6 +18,7 @@ limitations under the License.
package tasks
import (
+ "fmt"
"reflect"
"time"
@@ -79,39 +80,30 @@ func GenerateDeployment(taskCtx plugin.SubTaskContext) errors.Error {
clauses = append(clauses,
dal.Where("p.cicd_scope_id = ?", data.Options.ScopeId),
)
- // Clear previous results from the cicd_scope_id
- deleteSql := `DELETE FROM cicd_deployments WHERE cicd_scope_id = ?;`
- err := db.Exec(deleteSql, data.Options.ScopeId)
- if err != nil {
- return errors.Default.Wrap(err, "error deleting previous deployments")
- }
} else {
clauses = append(clauses,
dal.Join("LEFT JOIN project_mapping pm ON (pm.table = 'cicd_scopes' AND pm.row_id = p.cicd_scope_id)"),
dal.Where("pm.project_name = ?", data.Options.ProjectName),
)
- // Clear previous results from the project
- deleteSql := `DELETE FROM cicd_deployments
- WHERE cicd_scope_id IN (
- SELECT cicd_scope_id
- FROM (
- SELECT cd.cicd_scope_id
- FROM cicd_deployments cd
- LEFT JOIN project_mapping pm ON (pm.table = 'cicd_scopes' AND pm.row_id = cd.cicd_scope_id)
- WHERE pm.project_name = ?
- ) AS subquery
- );`
- err := db.Exec(deleteSql, data.Options.ProjectName)
- if err != nil {
- return errors.Default.Wrap(err, "error deleting previous deployments")
- }
}
-
cursor, err := db.Cursor(clauses...)
if err != nil {
return err
}
defer cursor.Close()
+ count, err := db.Count(clauses...)
+ if err != nil {
+ return errors.Default.Wrap(err, "error getting count of clauses")
+ }
+ if count == 0 {
+ // Clear previous results from the project
+ deleteSql := fmt.Sprintf("DELETE cd FROM cicd_deployments cd LEFT JOIN project_mapping pm ON (pm.table = 'cicd_scopes' AND pm.row_id = cd.cicd_scope_id) WHERE pm.project_name = '%s'", data.Options.ProjectName)
+ err := db.Exec(deleteSql)
+ if err != nil {
+ return errors.Default.Wrap(err, "error deleting previous deployments")
+ }
+ return nil
+ }
enricher, err := api.NewDataConverter(api.DataConverterArgs{
RawDataSubTaskArgs: api.RawDataSubTaskArgs{
diff --git a/backend/plugins/dora/tasks/incident_deploy_connector.go b/backend/plugins/dora/tasks/incident_deploy_connector.go
index 6d9a1365d..f2903d03c 100644
--- a/backend/plugins/dora/tasks/incident_deploy_connector.go
+++ b/backend/plugins/dora/tasks/incident_deploy_connector.go
@@ -18,6 +18,7 @@ limitations under the License.
package tasks
import (
+ "fmt"
"reflect"
"time"
@@ -47,11 +48,6 @@ type simpleCicdDeploymentCommit struct {
func ConnectIncidentToDeployment(taskCtx plugin.SubTaskContext) errors.Error {
db := taskCtx.GetDal()
data := taskCtx.GetData().(*DoraTaskData)
- // Clear previous results from the project
- err := db.Exec("DELETE FROM project_issue_metrics WHERE project_name = ?", data.Options.ProjectName)
- if err != nil {
- return errors.Default.Wrap(err, "error deleting previous project_issue_metrics")
- }
// select all issues belongs to the board
clauses := []dal.Clause{
dal.From(`issues i`),
@@ -67,6 +63,19 @@ func ConnectIncidentToDeployment(taskCtx plugin.SubTaskContext) errors.Error {
return err
}
defer cursor.Close()
+ count, err := db.Count(clauses...)
+ if err != nil {
+ return errors.Default.Wrap(err, "error getting count of clauses")
+ }
+ if count == 0 {
+ // Clear previous results from the project
+ deleteSql := fmt.Sprintf("DELETE FROM project_issue_metrics WHERE project_name = '%s'", data.Options.ProjectName)
+ err := db.Exec(deleteSql)
+ if err != nil {
+ return errors.Default.Wrap(err, "error deleting previous project_issue_metrics")
+ }
+ return nil
+ }
enricher, err := api.NewDataConverter(api.DataConverterArgs{
RawDataSubTaskArgs: api.RawDataSubTaskArgs{
(incubator-devlake) 02/02: Revert "fix: cicd deployment dora delete when data nil (#7033) (#7044)"
Posted by ab...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
abeizn pushed a commit to tag revert-dora-empty-v20240229-6
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit 4823fc326524180323490b379e5d3d8fabd7c660
Author: abeizn <zi...@merico.dev>
AuthorDate: Thu Mar 7 11:48:59 2024 +0800
Revert "fix: cicd deployment dora delete when data nil (#7033) (#7044)"
This reverts commit ec3eaf0f1c4f7147d9e88b8f501206f45d5c851d.
---
.../plugins/dora/tasks/change_lead_time_calculator.go | 19 ++-----------------
.../dora/tasks/deployment_commits_generator.go | 13 -------------
backend/plugins/dora/tasks/deployment_generator.go | 14 --------------
.../plugins/dora/tasks/incident_deploy_connector.go | 14 --------------
backend/plugins/github_graphql/tasks/pr_extractor.go | 2 ++
5 files changed, 4 insertions(+), 58 deletions(-)
diff --git a/backend/plugins/dora/tasks/change_lead_time_calculator.go b/backend/plugins/dora/tasks/change_lead_time_calculator.go
index dc5134cbe..f36303b8c 100644
--- a/backend/plugins/dora/tasks/change_lead_time_calculator.go
+++ b/backend/plugins/dora/tasks/change_lead_time_calculator.go
@@ -18,7 +18,6 @@ limitations under the License.
package tasks
import (
- "fmt"
"math"
"reflect"
"time"
@@ -49,30 +48,16 @@ func CalculateChangeLeadTime(taskCtx plugin.SubTaskContext) errors.Error {
data := taskCtx.GetData().(*DoraTaskData)
// Get pull requests by repo project_name
- var clauses = []dal.Clause{
+ cursor, err := db.Cursor(
dal.Select("pr.id, pr.pull_request_key, pr.author_id, pr.merge_commit_sha, pr.created_date, pr.merged_date"),
dal.From("pull_requests pr"),
dal.Join(`LEFT JOIN project_mapping pm ON (pm.row_id = pr.base_repo_id)`),
dal.Where("pr.merged_date IS NOT NULL AND pm.project_name = ? AND pm.table = 'repos'", data.Options.ProjectName),
- }
- cursor, err := db.Cursor(clauses...)
+ )
if err != nil {
return err
}
defer cursor.Close()
- count, err := db.Count(clauses...)
- if err != nil {
- return errors.Default.Wrap(err, "error getting count of clauses")
- }
- if count == 0 {
- // Clear previous results from the project
- deleteSql := fmt.Sprintf("DELETE FROM project_pr_metrics WHERE project_name = '%s'", data.Options.ProjectName)
- err := db.Exec(deleteSql)
- if err != nil {
- return errors.Default.Wrap(err, "error deleting previous project_pr_metrics")
- }
- return nil
- }
converter, err := api.NewDataConverter(api.DataConverterArgs{
RawDataSubTaskArgs: api.RawDataSubTaskArgs{
diff --git a/backend/plugins/dora/tasks/deployment_commits_generator.go b/backend/plugins/dora/tasks/deployment_commits_generator.go
index d960f5785..85da0e5fe 100644
--- a/backend/plugins/dora/tasks/deployment_commits_generator.go
+++ b/backend/plugins/dora/tasks/deployment_commits_generator.go
@@ -112,19 +112,6 @@ func GenerateDeploymentCommits(taskCtx plugin.SubTaskContext) errors.Error {
return err
}
defer cursor.Close()
- count, err := db.Count(clauses...)
- if err != nil {
- return errors.Default.Wrap(err, "error getting count of clauses")
- }
- if count == 0 {
- // Clear previous results from the project
- deleteSql := fmt.Sprintf("DELETE cdc FROM cicd_deployment_commits cdc LEFT JOIN project_mapping pm ON (pm.table = 'cicd_scopes' AND pm.row_id = cdc.cicd_scope_id) WHERE pm.project_name = '%s'", data.Options.ProjectName)
- err := db.Exec(deleteSql)
- if err != nil {
- return errors.Default.Wrap(err, "error deleting previous cicd_deployment_commits")
- }
- return nil
- }
enricher, err := api.NewDataConverter(api.DataConverterArgs{
RawDataSubTaskArgs: api.RawDataSubTaskArgs{
diff --git a/backend/plugins/dora/tasks/deployment_generator.go b/backend/plugins/dora/tasks/deployment_generator.go
index f7caada5e..3ae9da7e6 100644
--- a/backend/plugins/dora/tasks/deployment_generator.go
+++ b/backend/plugins/dora/tasks/deployment_generator.go
@@ -18,7 +18,6 @@ limitations under the License.
package tasks
import (
- "fmt"
"reflect"
"time"
@@ -91,19 +90,6 @@ func GenerateDeployment(taskCtx plugin.SubTaskContext) errors.Error {
return err
}
defer cursor.Close()
- count, err := db.Count(clauses...)
- if err != nil {
- return errors.Default.Wrap(err, "error getting count of clauses")
- }
- if count == 0 {
- // Clear previous results from the project
- deleteSql := fmt.Sprintf("DELETE cd FROM cicd_deployments cd LEFT JOIN project_mapping pm ON (pm.table = 'cicd_scopes' AND pm.row_id = cd.cicd_scope_id) WHERE pm.project_name = '%s'", data.Options.ProjectName)
- err := db.Exec(deleteSql)
- if err != nil {
- return errors.Default.Wrap(err, "error deleting previous deployments")
- }
- return nil
- }
enricher, err := api.NewDataConverter(api.DataConverterArgs{
RawDataSubTaskArgs: api.RawDataSubTaskArgs{
diff --git a/backend/plugins/dora/tasks/incident_deploy_connector.go b/backend/plugins/dora/tasks/incident_deploy_connector.go
index f2903d03c..cb013503a 100644
--- a/backend/plugins/dora/tasks/incident_deploy_connector.go
+++ b/backend/plugins/dora/tasks/incident_deploy_connector.go
@@ -18,7 +18,6 @@ limitations under the License.
package tasks
import (
- "fmt"
"reflect"
"time"
@@ -63,19 +62,6 @@ func ConnectIncidentToDeployment(taskCtx plugin.SubTaskContext) errors.Error {
return err
}
defer cursor.Close()
- count, err := db.Count(clauses...)
- if err != nil {
- return errors.Default.Wrap(err, "error getting count of clauses")
- }
- if count == 0 {
- // Clear previous results from the project
- deleteSql := fmt.Sprintf("DELETE FROM project_issue_metrics WHERE project_name = '%s'", data.Options.ProjectName)
- err := db.Exec(deleteSql)
- if err != nil {
- return errors.Default.Wrap(err, "error deleting previous project_issue_metrics")
- }
- return nil
- }
enricher, err := api.NewDataConverter(api.DataConverterArgs{
RawDataSubTaskArgs: api.RawDataSubTaskArgs{
diff --git a/backend/plugins/github_graphql/tasks/pr_extractor.go b/backend/plugins/github_graphql/tasks/pr_extractor.go
index fd90d1f99..9abcc4a06 100644
--- a/backend/plugins/github_graphql/tasks/pr_extractor.go
+++ b/backend/plugins/github_graphql/tasks/pr_extractor.go
@@ -19,6 +19,7 @@ package tasks
import (
"encoding/json"
+ "fmt"
"regexp"
"github.com/apache/incubator-devlake/core/errors"
@@ -52,6 +53,7 @@ func ExtractPrs(taskCtx plugin.SubTaskContext) errors.Error {
}
}
if config != nil && len(config.PrComponent) > 0 {
+ fmt.Println("config.PrComponent1", config.PrComponent)
labelComponentRegex, err = errors.Convert01(regexp.Compile(config.PrComponent))
if err != nil {
return errors.Default.Wrap(err, "regexp Compile prComponent failed")