You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by wa...@apache.org on 2022/06/10 03:43:07 UTC
[incubator-devlake] 06/06: refactor: caculate all pairs on prepare
This is an automated email from the ASF dual-hosted git repository.
warren pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit d7565fc150f2c22b69c54d23f7d36be2f85ab3f4
Author: Nddtfjiang <jz...@qq.com>
AuthorDate: Thu Jun 9 12:31:59 2022 +0000
refactor: caculate all pairs on prepare
Calculate AllPairs on PrepareTaskData
Changed db to dal on CaculateTagPattern and CalculateCommitPairs.
Nddtfjiang <zh...@merico.dev>
---
plugins/refdiff/refdiff.go | 8 ++-
.../refdiff/tasks/ref_commit_diff_calculator.go | 50 +--------------
plugins/refdiff/tasks/ref_issue_diff_calculator.go | 12 ++--
plugins/refdiff/tasks/refdiff_task_data.go | 73 +++++++++++++++++++---
4 files changed, 75 insertions(+), 68 deletions(-)
diff --git a/plugins/refdiff/refdiff.go b/plugins/refdiff/refdiff.go
index cec4370c..52dce6c8 100644
--- a/plugins/refdiff/refdiff.go
+++ b/plugins/refdiff/refdiff.go
@@ -62,12 +62,16 @@ func (plugin RefDiff) PrepareTaskData(taskCtx core.TaskContext, options map[stri
return nil, err
}
- db := taskCtx.GetDb()
+ db := taskCtx.GetDal()
tagsPattern := op.TagsPattern
tagsLimit := op.TagsLimit
tagsOrder := op.TagsOrder
- op.TagsRefs, err = tasks.CaculateTagPattern(db, tagsPattern, tagsLimit, tagsOrder)
+ rs, err := tasks.CaculateTagPattern(db, tagsPattern, tagsLimit, tagsOrder)
+ if err != nil {
+ return nil, err
+ }
+ op.AllPairs, err = tasks.CalculateCommitPairs(db, op.RepoId, op.Pairs, rs)
if err != nil {
return nil, err
}
diff --git a/plugins/refdiff/tasks/ref_commit_diff_calculator.go b/plugins/refdiff/tasks/ref_commit_diff_calculator.go
index 7e2b3504..af71c905 100644
--- a/plugins/refdiff/tasks/ref_commit_diff_calculator.go
+++ b/plugins/refdiff/tasks/ref_commit_diff_calculator.go
@@ -27,51 +27,6 @@ import (
"gorm.io/gorm/clause"
)
-// Calculate the commits pairs both from Options.Pairs and TagPattern
-func CalculateCommitsPairs(taskCtx core.SubTaskContext) (RefCommitPairs, error) {
- data := taskCtx.GetData().(*RefdiffTaskData)
- repoId := data.Options.RepoId
- pairs := data.Options.Pairs
- rs := data.Options.TagsRefs
- db := taskCtx.GetDb()
-
- commitPairs := make(RefCommitPairs, 0, len(rs)+len(pairs))
- for i := 1; i < len(rs); i++ {
- commitPairs = append(commitPairs, [4]string{rs[i-1].CommitSha, rs[i].CommitSha, rs[i-1].Name, rs[i].Name})
- }
-
- // caculate pairs part
- // convert ref pairs into commit pairs
- ref2sha := func(refName string) (string, error) {
- ref := &code.Ref{}
- if refName == "" {
- return "", fmt.Errorf("ref name is empty")
- }
- ref.Id = fmt.Sprintf("%s:%s", repoId, refName)
- err := db.First(ref).Error
- if err != nil {
- return "", fmt.Errorf("faild to load Ref info for repoId:%s, refName:%s", repoId, refName)
- }
- return ref.CommitSha, nil
- }
-
- for i, refPair := range pairs {
- // get new ref's commit sha
- newCommit, err := ref2sha(refPair.NewRef)
- if err != nil {
- return RefCommitPairs{}, fmt.Errorf("failed to load commit sha for NewRef on pair #%d: %w", i, err)
- }
- // get old ref's commit sha
- oldCommit, err := ref2sha(refPair.OldRef)
- if err != nil {
- return RefCommitPairs{}, fmt.Errorf("failed to load commit sha for OleRef on pair #%d: %w", i, err)
- }
- commitPairs = append(commitPairs, RefCommitPair{newCommit, oldCommit, refPair.NewRef, refPair.OldRef})
- }
-
- return commitPairs, nil
-}
-
func CalculateCommitsDiff(taskCtx core.SubTaskContext) error {
data := taskCtx.GetData().(*RefdiffTaskData)
repoId := data.Options.RepoId
@@ -80,10 +35,7 @@ func CalculateCommitsDiff(taskCtx core.SubTaskContext) error {
logger := taskCtx.GetLogger()
insertCountLimitOfRefsCommitsDiff := int(65535 / reflect.ValueOf(code.RefsCommitsDiff{}).NumField())
- commitPairs, err := CalculateCommitsPairs(taskCtx)
- if err != nil {
- return err
- }
+ commitPairs := data.Options.AllPairs
commitNodeGraph := utils.NewCommitNodeGraph()
diff --git a/plugins/refdiff/tasks/ref_issue_diff_calculator.go b/plugins/refdiff/tasks/ref_issue_diff_calculator.go
index 3f6809f5..43e9c853 100644
--- a/plugins/refdiff/tasks/ref_issue_diff_calculator.go
+++ b/plugins/refdiff/tasks/ref_issue_diff_calculator.go
@@ -26,20 +26,16 @@ import (
"github.com/apache/incubator-devlake/plugins/helper"
)
-// Calculate the pair list both from Options.Pairs and TagPattern
+// CaculatePairList Calculate the pair list both from Options.Pairs and TagPattern
func CaculatePairList(taskCtx core.SubTaskContext) (RefPairLists, error) {
data := taskCtx.GetData().(*RefdiffTaskData)
repoId := data.Options.RepoId
- pairs := data.Options.Pairs
- rs := data.Options.TagsRefs
+ pairs := data.Options.AllPairs
- pairList := make(RefPairLists, 0, len(rs)+len(pairs))
- for i := 1; i < len(rs); i++ {
- pairList = append(pairList, RefPairList{fmt.Sprintf("%s:%s", repoId, rs[i-1].Id), fmt.Sprintf("%s:%s", repoId, rs[i].Id)})
- }
+ pairList := make(RefPairLists, 0, len(pairs))
for _, pair := range pairs {
- pairList = append(pairList, RefPairList{fmt.Sprintf("%s:%s", repoId, pair.NewRef), fmt.Sprintf("%s:%s", repoId, pair.OldRef)})
+ pairList = append(pairList, RefPairList{fmt.Sprintf("%s:%s", repoId, pair[2]), fmt.Sprintf("%s:%s", repoId, pair[3])})
}
return pairList, nil
diff --git a/plugins/refdiff/tasks/refdiff_task_data.go b/plugins/refdiff/tasks/refdiff_task_data.go
index 37781308..8e0c9fec 100644
--- a/plugins/refdiff/tasks/refdiff_task_data.go
+++ b/plugins/refdiff/tasks/refdiff_task_data.go
@@ -25,7 +25,7 @@ import (
"time"
"github.com/apache/incubator-devlake/models/domainlayer/code"
- "gorm.io/gorm"
+ "github.com/apache/incubator-devlake/plugins/core/dal"
)
type RefdiffOptions struct {
@@ -33,10 +33,11 @@ type RefdiffOptions struct {
Tasks []string `json:"tasks,omitempty"`
Pairs []RefPair
- TagsPattern string // The Pattern to match from all tags
- TagsLimit int // How many tags be matched should be used.
- TagsOrder string // The Rule to Order the tag list
- TagsRefs []code.Ref // Caculate out by TagsPattern TagsLimit TagsOrder from db
+ TagsPattern string // The Pattern to match from all tags
+ TagsLimit int // How many tags be matched should be used.
+ TagsOrder string // The Rule to Order the tag list
+
+ AllPairs RefCommitPairs // Pairs and TagsPattern Pairs
}
type RefdiffTaskData struct {
@@ -133,15 +134,20 @@ func (rs RefsReverseSemver) Swap(i, j int) {
rs[i], rs[j] = rs[j], rs[i]
}
-// Calculate the TagPattern order by tagsOrder and return the Refs
-func CaculateTagPattern(db *gorm.DB, tagsPattern string, tagsLimit int, tagsOrder string) (Refs, error) {
+// CaculateTagPattern Calculate the TagPattern order by tagsOrder and return the Refs
+func CaculateTagPattern(db dal.Dal, tagsPattern string, tagsLimit int, tagsOrder string) (Refs, error) {
rs := Refs{}
// caculate Pattern part
if tagsPattern == "" || tagsLimit <= 1 {
return rs, nil
}
- rows, err := db.Model(&code.Ref{}).Order("created_date desc").Rows()
+ rows, err := db.Cursor(
+ dal.From("refs"),
+ dal.Where(""),
+ dal.Orderby("created_date desc"),
+ )
+
if err != nil {
return rs, err
}
@@ -152,7 +158,7 @@ func CaculateTagPattern(db *gorm.DB, tagsPattern string, tagsLimit int, tagsOrde
}
for rows.Next() {
var ref code.Ref
- err = db.ScanRows(rows, &ref)
+ err = db.Fetch(rows, &ref)
if err != nil {
return rs, err
}
@@ -180,3 +186,52 @@ func CaculateTagPattern(db *gorm.DB, tagsPattern string, tagsLimit int, tagsOrde
return rs, nil
}
+
+// CalculateCommitPairs Calculate the commits pairs both from Options.Pairs and TagPattern
+func CalculateCommitPairs(db dal.Dal, repoId string, pairs []RefPair, rs Refs) (RefCommitPairs, error) {
+ commitPairs := make(RefCommitPairs, 0, len(rs)+len(pairs))
+ for i := 1; i < len(rs); i++ {
+ commitPairs = append(commitPairs, [4]string{rs[i-1].CommitSha, rs[i].CommitSha, rs[i-1].Name, rs[i].Name})
+ }
+
+ // caculate pairs part
+ // convert ref pairs into commit pairs
+ ref2sha := func(refName string) (string, error) {
+ ref := &code.Ref{}
+ if refName == "" {
+ return "", fmt.Errorf("ref name is empty")
+ }
+ ref.Id = fmt.Sprintf("%s:%s", repoId, refName)
+ err := db.First(ref)
+ if err != nil {
+ return "", fmt.Errorf("faild to load Ref info for repoId:%s, refName:%s", repoId, refName)
+ }
+ return ref.CommitSha, nil
+ }
+
+ for i, refPair := range pairs {
+ // get new ref's commit sha
+ newCommit, err := ref2sha(refPair.NewRef)
+ if err != nil {
+ return RefCommitPairs{}, fmt.Errorf("failed to load commit sha for NewRef on pair #%d: %w", i, err)
+ }
+ // get old ref's commit sha
+ oldCommit, err := ref2sha(refPair.OldRef)
+ if err != nil {
+ return RefCommitPairs{}, fmt.Errorf("failed to load commit sha for OleRef on pair #%d: %w", i, err)
+ }
+
+ have := false
+ for _, cp := range commitPairs {
+ if cp[0] == newCommit && cp[1] == oldCommit {
+ have = true
+ break
+ }
+ }
+ if !have {
+ commitPairs = append(commitPairs, RefCommitPair{newCommit, oldCommit, refPair.NewRef, refPair.OldRef})
+ }
+ }
+
+ return commitPairs, nil
+}