You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by ab...@apache.org on 2022/11/24 12:40:22 UTC
[incubator-devlake] branch main updated: feat: add new table ref_commits for the refdiff plugin (#3801)
This is an automated email from the ASF dual-hosted git repository.
abeizn pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 286f39448 feat: add new table ref_commits for the refdiff plugin (#3801)
286f39448 is described below
commit 286f39448b3a97f4e98e0d36576d3979ed48a866
Author: abeizn <zi...@merico.dev>
AuthorDate: Thu Nov 24 20:40:17 2022 +0800
feat: add new table ref_commits for the refdiff plugin (#3801)
* feat: add new table ref_commits for the refdiff plugin
---
models/domainlayer/code/commits_diff.go | 15 ++++++-
models/domainlayer/domaininfo/domaininfo.go | 3 +-
.../20221109_modfiy_commits_diffs.go | 18 +++++++-
.../e2e/project_deployment_commit_diff_test.go | 4 +-
plugins/refdiff/tasks/commit_diff_calculator.go | 49 +++++++++++++++++-----
.../project_deployment_commit_diff_calculator.go | 6 +--
6 files changed, 76 insertions(+), 19 deletions(-)
diff --git a/models/domainlayer/code/commits_diff.go b/models/domainlayer/code/commits_diff.go
index 8fc83632f..a6b546b6b 100644
--- a/models/domainlayer/code/commits_diff.go
+++ b/models/domainlayer/code/commits_diff.go
@@ -28,11 +28,22 @@ func (CommitsDiff) TableName() string {
return "commits_diffs"
}
-type FinishedCommitsDiffs struct {
+type RefCommit struct {
+ NewRefId string `gorm:"primaryKey;type:varchar(255)"`
+ OldRefId string `gorm:"primaryKey;type:varchar(255)"`
+ NewCommitSha string `gorm:"type:varchar(40)"`
+ OldCommitSha string `gorm:"type:varchar(40)"`
+}
+
+func (RefCommit) TableName() string {
+ return "ref_commits"
+}
+
+type FinishedCommitsDiff struct {
NewCommitSha string `gorm:"primaryKey;type:varchar(40)"`
OldCommitSha string `gorm:"primaryKey;type:varchar(40)"`
}
-func (FinishedCommitsDiffs) TableName() string {
+func (FinishedCommitsDiff) TableName() string {
return "finished_commits_diffs"
}
diff --git a/models/domainlayer/domaininfo/domaininfo.go b/models/domainlayer/domaininfo/domaininfo.go
index f4a9d529c..3450923dd 100644
--- a/models/domainlayer/domaininfo/domaininfo.go
+++ b/models/domainlayer/domaininfo/domaininfo.go
@@ -42,7 +42,8 @@ func GetDomainTablesInfo() []Tabler {
&code.PullRequestLabel{},
&code.Ref{},
&code.CommitsDiff{},
- &code.FinishedCommitsDiffs{},
+ &code.RefCommit{},
+ &code.FinishedCommitsDiff{},
&code.RefsPrCherrypick{},
&code.Repo{},
&code.RepoCommit{},
diff --git a/models/migrationscripts/20221109_modfiy_commits_diffs.go b/models/migrationscripts/20221109_modfiy_commits_diffs.go
index 1c35662aa..9860db737 100644
--- a/models/migrationscripts/20221109_modfiy_commits_diffs.go
+++ b/models/migrationscripts/20221109_modfiy_commits_diffs.go
@@ -52,9 +52,9 @@ func (RefsCommitsDiff20221109) TableName() string {
}
type CommitsDiff20221109 struct {
- CommitSha string `gorm:"primaryKey;type:varchar(40)"`
NewCommitSha string `gorm:"primaryKey;type:varchar(40)"`
OldCommitSha string `gorm:"primaryKey;type:varchar(40)"`
+ CommitSha string `gorm:"primaryKey;type:varchar(40)"`
SortingIndex int
}
@@ -62,6 +62,17 @@ func (CommitsDiff20221109) TableName() string {
return "commits_diffs"
}
+type RefCommits20221109 struct {
+ NewRefId string `gorm:"primaryKey;type:varchar(255)"`
+ OldRefId string `gorm:"primaryKey;type:varchar(255)"`
+ NewCommitSha string `gorm:"type:varchar(40)"`
+ OldCommitSha string `gorm:"type:varchar(40)"`
+}
+
+func (RefCommits20221109) TableName() string {
+ return "ref_commits"
+}
+
func (script *modifyCommitsDiffs) Up(basicRes core.BasicRes) errors.Error {
db := basicRes.GetDal()
// create table
@@ -70,6 +81,11 @@ func (script *modifyCommitsDiffs) Up(basicRes core.BasicRes) errors.Error {
return err
}
+ err = db.AutoMigrate(&RefCommits20221109{})
+ if err != nil {
+ return err
+ }
+
err = db.AutoMigrate(&FinishedCommitsDiffs20221109{})
if err != nil {
return err
diff --git a/plugins/refdiff/e2e/project_deployment_commit_diff_test.go b/plugins/refdiff/e2e/project_deployment_commit_diff_test.go
index 3de043e0a..5b5028f58 100644
--- a/plugins/refdiff/e2e/project_deployment_commit_diff_test.go
+++ b/plugins/refdiff/e2e/project_deployment_commit_diff_test.go
@@ -50,14 +50,14 @@ func TestRepoDataFlow(t *testing.T) {
// verify extraction
dataflowTester.FlushTabler(&code.CommitsDiff{})
- dataflowTester.FlushTabler(&code.FinishedCommitsDiffs{})
+ dataflowTester.FlushTabler(&code.FinishedCommitsDiff{})
dataflowTester.Subtask(tasks.CalculateProjectDeploymentCommitsDiffMeta, taskData)
dataflowTester.VerifyTableWithOptions(&code.CommitsDiff{}, e2ehelper.TableOptions{
CSVRelPath: "./snapshot_tables/commits_diffs.csv",
})
- dataflowTester.VerifyTableWithOptions(&code.FinishedCommitsDiffs{}, e2ehelper.TableOptions{
+ dataflowTester.VerifyTableWithOptions(&code.FinishedCommitsDiff{}, e2ehelper.TableOptions{
CSVRelPath: "./snapshot_tables/finished_commits_diffs.csv",
})
}
diff --git a/plugins/refdiff/tasks/commit_diff_calculator.go b/plugins/refdiff/tasks/commit_diff_calculator.go
index a6ba0b13d..98795c679 100644
--- a/plugins/refdiff/tasks/commit_diff_calculator.go
+++ b/plugins/refdiff/tasks/commit_diff_calculator.go
@@ -18,6 +18,7 @@ limitations under the License.
package tasks
import (
+ "fmt"
"reflect"
"github.com/apache/incubator-devlake/errors"
@@ -40,25 +41,43 @@ func CalculateCommitsDiff(taskCtx core.SubTaskContext) errors.Error {
}
// get all data from finish_commits_diffs
- commitPairs := data.Options.AllPairs
- var ExistFinishedCommitDiff []code.FinishedCommitsDiffs
- err := db.All(&ExistFinishedCommitDiff,
+ commitPairsSrc := data.Options.AllPairs
+ var commitPairs RefCommitPairs
+ var existFinishedCommitDiff []code.FinishedCommitsDiff
+ err := db.All(&existFinishedCommitDiff,
dal.Select("*"),
dal.From("finished_commits_diffs"),
)
if err != nil {
return err
}
- for i := 0; i < len(commitPairs); i++ {
- pair := commitPairs[i]
- for _, item := range ExistFinishedCommitDiff {
+
+ refCommit := &code.RefCommit{}
+ for _, pair := range commitPairsSrc {
+ newRefId := fmt.Sprintf("%s:%s", repoId, pair[2])
+ oldRefId := fmt.Sprintf("%s:%s", repoId, pair[3])
+ isAppend := true
+ isRefAppend := true
+ for _, item := range existFinishedCommitDiff {
if pair[0] == item.NewCommitSha && pair[1] == item.OldCommitSha {
- commitPairs = append(commitPairs[:i], commitPairs[i+1:]...)
- i--
+ isAppend = false
+ if pair[2] == newRefId && pair[3] == oldRefId {
+ isRefAppend = false
+ }
break
}
}
+ if isAppend {
+ commitPairs = append(commitPairs, pair)
+ }
+ if isRefAppend {
+ refCommit.NewCommitSha = pair[0]
+ refCommit.OldCommitSha = pair[1]
+ refCommit.NewRefId = newRefId
+ refCommit.OldRefId = oldRefId
+ }
}
+
if len(commitPairs) == 0 {
logger.Info("commit pair has been produced.")
return nil
@@ -98,7 +117,7 @@ func CalculateCommitsDiff(taskCtx core.SubTaskContext) errors.Error {
// calculate diffs for commits pairs and store them into database
commitsDiff := &code.CommitsDiff{}
- finishedCommitDiff := &code.FinishedCommitsDiffs{}
+ finishedCommitDiff := &code.FinishedCommitsDiff{}
lenCommitPairs := len(commitPairs)
taskCtx.SetProgress(0, lenCommitPairs)
@@ -111,6 +130,7 @@ func CalculateCommitsDiff(taskCtx core.SubTaskContext) errors.Error {
// ref might advance, keep commit sha for debugging
commitsDiff.NewCommitSha = pair[0]
commitsDiff.OldCommitSha = pair[1]
+
finishedCommitDiff.NewCommitSha = pair[0]
finishedCommitDiff.OldCommitSha = pair[1]
@@ -126,7 +146,8 @@ func CalculateCommitsDiff(taskCtx core.SubTaskContext) errors.Error {
lostSha, oldCount, newCount := commitNodeGraph.CalculateLostSha(pair[1], pair[0])
commitsDiffs := []code.CommitsDiff{}
- finishedCommitDiffs := []code.FinishedCommitsDiffs{}
+ refCommits := []code.RefCommit{}
+ finishedCommitDiffs := []code.FinishedCommitsDiff{}
commitsDiff.SortingIndex = 1
for _, sha := range lostSha {
@@ -154,6 +175,14 @@ func CalculateCommitsDiff(taskCtx core.SubTaskContext) errors.Error {
}
}
+ refCommits = append(refCommits, *refCommit)
+ if len(refCommits) > 0 {
+ err = db.CreateIfNotExist(refCommits)
+ if err != nil {
+ return err
+ }
+ }
+
finishedCommitDiffs = append(finishedCommitDiffs, *finishedCommitDiff)
if len(finishedCommitDiffs) > 0 {
err = db.CreateIfNotExist(finishedCommitDiffs)
diff --git a/plugins/refdiff/tasks/project_deployment_commit_diff_calculator.go b/plugins/refdiff/tasks/project_deployment_commit_diff_calculator.go
index 0f07c31fa..990c7dfd1 100644
--- a/plugins/refdiff/tasks/project_deployment_commit_diff_calculator.go
+++ b/plugins/refdiff/tasks/project_deployment_commit_diff_calculator.go
@@ -27,7 +27,7 @@ import (
"github.com/apache/incubator-devlake/plugins/refdiff/utils"
)
-func CommitDiffConvertor(pipelineCommitShaList []string, existFinishedCommitDiff []code.FinishedCommitsDiffs) (commitPairs []code.CommitsDiff, finishedCommitDiffs []code.FinishedCommitsDiffs) {
+func CommitDiffConvertor(pipelineCommitShaList []string, existFinishedCommitDiff []code.FinishedCommitsDiff) (commitPairs []code.CommitsDiff, finishedCommitDiffs []code.FinishedCommitsDiff) {
for i := 0; i < len(pipelineCommitShaList)-1; i++ {
for _, item := range existFinishedCommitDiff {
if pipelineCommitShaList[i+1] == item.NewCommitSha && pipelineCommitShaList[i] == item.OldCommitSha {
@@ -36,7 +36,7 @@ func CommitDiffConvertor(pipelineCommitShaList []string, existFinishedCommitDiff
}
}
commitPairs = append(commitPairs, code.CommitsDiff{NewCommitSha: pipelineCommitShaList[i+1], OldCommitSha: pipelineCommitShaList[i]})
- finishedCommitDiffs = append(finishedCommitDiffs, code.FinishedCommitsDiffs{NewCommitSha: pipelineCommitShaList[i+1], OldCommitSha: pipelineCommitShaList[i]})
+ finishedCommitDiffs = append(finishedCommitDiffs, code.FinishedCommitsDiff{NewCommitSha: pipelineCommitShaList[i+1], OldCommitSha: pipelineCommitShaList[i]})
}
return commitPairs, finishedCommitDiffs
}
@@ -62,7 +62,7 @@ func CalculateProjectDeploymentCommitsDiff(taskCtx core.SubTaskContext) errors.E
}
defer cursorScope.Close()
- var existFinishedCommitDiff []code.FinishedCommitsDiffs
+ var existFinishedCommitDiff []code.FinishedCommitsDiff
err = db.All(&existFinishedCommitDiff,
dal.Select("*"),
dal.From("finished_commits_diffs"),