You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by kl...@apache.org on 2022/06/24 11:59:37 UTC
[incubator-devlake] branch main updated: [Refactor][Gitee] use GetDal instead of deprecated GetDb (#2340)
This is an automated email from the ASF dual-hosted git repository.
klesh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 73026bec [Refactor][Gitee] use GetDal instead of deprecated GetDb (#2340)
73026bec is described below
commit 73026becbe4ee37181a0233adfc6728e22ad5ce3
Author: Mr.An <42...@users.noreply.github.com>
AuthorDate: Fri Jun 24 19:59:32 2022 +0800
[Refactor][Gitee] use GetDal instead of deprecated GetDb (#2340)
* update commit collector.go
* fix#bug g.pkTypes list is not fully traversed
* update migrationscripts
* update models
* Refactor commit db to dal
* Refactor commit_stat db to dal
* Refactor issue db to dal
* Refactor issue_comment db to dal
* Refactor issue_label db to dal
* Refactor pr db to dal
* Refactor pr commit db to dal
* Refactor pr commit db to dal
* Refactor pr review db to dal
* Refactor pr issue and pr label db to dal
* Refactor repo db to dal
* Refactor db to dal
* Generate domain repoId, authorId, committerId through connectionId
* Refactor db to dal
* fix golint error
* fix golint error info
---
plugins/gitee/models/commit_stats.go | 1 +
plugins/gitee/models/connection.go | 3 +-
plugins/gitee/models/issue.go | 1 +
plugins/gitee/models/issue_comment.go | 5 +--
plugins/gitee/models/issue_label.go | 5 +--
.../migrationscripts/archived/commit_stat.go | 1 +
.../models/migrationscripts/archived/connection.go | 1 +
.../models/migrationscripts/archived/issue.go | 1 +
.../migrationscripts/archived/issue_comment.go | 5 +--
.../migrationscripts/archived/issue_label.go | 5 +--
.../migrationscripts/archived/pull_request.go | 1 +
.../archived/pull_request_comment.go | 5 +--
.../archived/pull_request_commit.go | 1 +
.../archived/pull_request_issue.go | 5 +--
.../archived/pull_request_label.go | 5 +--
.../gitee/models/migrationscripts/archived/repo.go | 3 +-
.../migrationscripts/archived/repo_commit.go | 5 +--
.../models/migrationscripts/archived/reviewer.go | 1 +
.../gitee/models/migrationscripts/archived/user.go | 3 +-
.../gitee/models/migrationscripts/init_schema.go | 2 +-
plugins/gitee/models/pull_request.go | 1 +
plugins/gitee/models/pull_request_comment.go | 5 +--
plugins/gitee/models/pull_request_commit.go | 1 +
plugins/gitee/models/pull_request_issue.go | 5 +--
plugins/gitee/models/pull_request_label.go | 5 +--
plugins/gitee/models/repo.go | 3 +-
plugins/gitee/models/repo_commit.go | 5 +--
plugins/gitee/models/reviewer.go | 1 +
plugins/gitee/models/user.go | 3 +-
plugins/gitee/tasks/commit_collector.go | 33 ++++++++++++++++--
plugins/gitee/tasks/commit_convertor.go | 29 ++++++++--------
plugins/gitee/tasks/commit_extractor.go | 5 +--
plugins/gitee/tasks/commit_stats_collector.go | 29 ++++++++++------
plugins/gitee/tasks/commit_stats_extractor.go | 9 +++--
plugins/gitee/tasks/issue_collector.go | 13 ++++---
plugins/gitee/tasks/issue_comment_collector.go | 27 ++++++++++-----
plugins/gitee/tasks/issue_comment_convertor.go | 20 ++++++-----
plugins/gitee/tasks/issue_comment_extractor.go | 8 +++--
plugins/gitee/tasks/issue_convertor.go | 17 +++++----
plugins/gitee/tasks/issue_extractor.go | 40 +++++++---------------
plugins/gitee/tasks/issue_label_convertor.go | 18 ++++++----
plugins/gitee/tasks/pr_collector.go | 15 +++++---
plugins/gitee/tasks/pr_comment_convertor.go | 20 ++++++-----
plugins/gitee/tasks/pr_commit_collector.go | 14 +++++---
plugins/gitee/tasks/pr_commit_convertor.go | 17 +++++----
plugins/gitee/tasks/pr_commit_extractor.go | 3 +-
plugins/gitee/tasks/pr_convertor.go | 15 +++++---
plugins/gitee/tasks/pr_extractor.go | 20 +++++------
plugins/gitee/tasks/pr_issue_convertor.go | 20 ++++++-----
plugins/gitee/tasks/pr_issue_enricher.go | 18 ++++++----
plugins/gitee/tasks/pr_label_convertor.go | 18 ++++++----
plugins/gitee/tasks/pr_review_collector.go | 15 +++++---
plugins/gitee/tasks/pr_review_extractor.go | 3 +-
plugins/gitee/tasks/repo_collector.go | 2 +-
plugins/gitee/tasks/repo_convertor.go | 15 ++++----
plugins/gitee/tasks/repo_extractor.go | 19 +++++-----
plugins/gitee/tasks/shared.go | 10 +++---
plugins/gitee/tasks/task_data.go | 14 ++++----
plugins/gitee/tasks/user_convertor.go | 11 +++---
59 files changed, 358 insertions(+), 227 deletions(-)
diff --git a/plugins/gitee/models/commit_stats.go b/plugins/gitee/models/commit_stats.go
index d4992490..829eaefe 100644
--- a/plugins/gitee/models/commit_stats.go
+++ b/plugins/gitee/models/commit_stats.go
@@ -24,6 +24,7 @@ import (
)
type GiteeCommitStat struct {
+ ConnectionId uint64 `gorm:"primaryKey"`
Sha string `gorm:"primaryKey;type:varchar(40)"`
Additions int `gorm:"comment:Added lines of code"`
Deletions int `gorm:"comment:Deleted lines of code"`
diff --git a/plugins/gitee/models/connection.go b/plugins/gitee/models/connection.go
index a5360449..7129654d 100644
--- a/plugins/gitee/models/connection.go
+++ b/plugins/gitee/models/connection.go
@@ -41,9 +41,10 @@ type TestConnectionRequest struct {
helper.AccessToken `mapstructure:",squash"`
}
-type Config struct {
+type TransformationRules struct {
PrType string `mapstructure:"prType" env:"GITEE_PR_TYPE" json:"prType"`
PrComponent string `mapstructure:"prComponent" env:"GITEE_PR_COMPONENT" json:"prComponent"`
+ PrBodyClosePattern string `mapstructure:"prBodyClosePattern" json:"prBodyClosePattern"`
IssueSeverity string `mapstructure:"issueSeverity" env:"GITEE_ISSUE_SEVERITY" json:"issueSeverity"`
IssuePriority string `mapstructure:"issuePriority" env:"GITEE_ISSUE_PRIORITY" json:"issuePriority"`
IssueComponent string `mapstructure:"issueComponent" env:"GITEE_ISSUE_COMPONENT" json:"issueComponent"`
diff --git a/plugins/gitee/models/issue.go b/plugins/gitee/models/issue.go
index aa2ea170..b4103ef0 100644
--- a/plugins/gitee/models/issue.go
+++ b/plugins/gitee/models/issue.go
@@ -24,6 +24,7 @@ import (
)
type GiteeIssue struct {
+ ConnectionId uint64 `gorm:"primaryKey"`
GiteeId int `gorm:"primaryKey"`
RepoId int `gorm:"index"`
Number string `gorm:"index;comment:Used in API requests ex. api/repo/1/issue/<THIS_NUMBER>"`
diff --git a/plugins/gitee/models/issue_comment.go b/plugins/gitee/models/issue_comment.go
index 9d170040..861b660e 100644
--- a/plugins/gitee/models/issue_comment.go
+++ b/plugins/gitee/models/issue_comment.go
@@ -24,8 +24,9 @@ import (
)
type GiteeIssueComment struct {
- GiteeId int `gorm:"primaryKey"`
- IssueId int `gorm:"index;comment:References the Issue"`
+ ConnectionId uint64 `gorm:"primaryKey"`
+ GiteeId int `gorm:"primaryKey"`
+ IssueId int `gorm:"index;comment:References the Issue"`
Body string
AuthorUsername string `gorm:"type:varchar(255)"`
AuthorUserId int
diff --git a/plugins/gitee/models/issue_label.go b/plugins/gitee/models/issue_label.go
index 77f910f9..408b0f39 100644
--- a/plugins/gitee/models/issue_label.go
+++ b/plugins/gitee/models/issue_label.go
@@ -25,8 +25,9 @@ import (
// Pull Requests are considered Issues in Gitee.
type GiteeIssueLabel struct {
- IssueId int `gorm:"primaryKey;autoIncrement:false"`
- LabelName string `gorm:"primaryKey;type:varchar(255)"`
+ ConnectionId uint64 `gorm:"primaryKey"`
+ IssueId int `gorm:"primaryKey;autoIncrement:false"`
+ LabelName string `gorm:"primaryKey;type:varchar(255)"`
common.NoPKModel
}
diff --git a/plugins/gitee/models/migrationscripts/archived/commit_stat.go b/plugins/gitee/models/migrationscripts/archived/commit_stat.go
index 77a9e0a4..5a70ddd5 100644
--- a/plugins/gitee/models/migrationscripts/archived/commit_stat.go
+++ b/plugins/gitee/models/migrationscripts/archived/commit_stat.go
@@ -24,6 +24,7 @@ import (
)
type GiteeCommitStat struct {
+ ConnectionId uint64 `gorm:"primaryKey"`
Sha string `gorm:"primaryKey;type:varchar(40)"`
Additions int `gorm:"comment:Added lines of code"`
Deletions int `gorm:"comment:Deleted lines of code"`
diff --git a/plugins/gitee/models/migrationscripts/archived/connection.go b/plugins/gitee/models/migrationscripts/archived/connection.go
index c1fc5daf..96188971 100644
--- a/plugins/gitee/models/migrationscripts/archived/connection.go
+++ b/plugins/gitee/models/migrationscripts/archived/connection.go
@@ -58,6 +58,7 @@ type ApiUserResponse struct {
type Config struct {
PrType string `mapstructure:"prType" env:"GITEE_PR_TYPE" json:"prType"`
PrComponent string `mapstructure:"prComponent" env:"GITEE_PR_COMPONENT" json:"prComponent"`
+ PrBodyClosePattern string `mapstructure:"prBodyClosePattern" json:"prBodyClosePattern"`
IssueSeverity string `mapstructure:"issueSeverity" env:"GITEE_ISSUE_SEVERITY" json:"issueSeverity"`
IssuePriority string `mapstructure:"issuePriority" env:"GITEE_ISSUE_PRIORITY" json:"issuePriority"`
IssueComponent string `mapstructure:"issueComponent" env:"GITEE_ISSUE_COMPONENT" json:"issueComponent"`
diff --git a/plugins/gitee/models/migrationscripts/archived/issue.go b/plugins/gitee/models/migrationscripts/archived/issue.go
index 5d9e6240..4dd0b381 100644
--- a/plugins/gitee/models/migrationscripts/archived/issue.go
+++ b/plugins/gitee/models/migrationscripts/archived/issue.go
@@ -24,6 +24,7 @@ import (
)
type GiteeIssue struct {
+ ConnectionId uint64 `gorm:"primaryKey"`
GiteeId int `gorm:"primaryKey"`
RepoId int `gorm:"index"`
Number string `gorm:"index;comment:Used in API requests ex. api/repo/1/issue/<THIS_NUMBER>"`
diff --git a/plugins/gitee/models/migrationscripts/archived/issue_comment.go b/plugins/gitee/models/migrationscripts/archived/issue_comment.go
index ea40c2d9..58fa0fbf 100644
--- a/plugins/gitee/models/migrationscripts/archived/issue_comment.go
+++ b/plugins/gitee/models/migrationscripts/archived/issue_comment.go
@@ -24,8 +24,9 @@ import (
)
type GiteeIssueComment struct {
- GiteeId int `gorm:"primaryKey"`
- IssueId int `gorm:"index;comment:References the Issue"`
+ ConnectionId uint64 `gorm:"primaryKey"`
+ GiteeId int `gorm:"primaryKey"`
+ IssueId int `gorm:"index;comment:References the Issue"`
Body string
AuthorUsername string `gorm:"type:varchar(255)"`
AuthorUserId int
diff --git a/plugins/gitee/models/migrationscripts/archived/issue_label.go b/plugins/gitee/models/migrationscripts/archived/issue_label.go
index 9db73278..add179b5 100644
--- a/plugins/gitee/models/migrationscripts/archived/issue_label.go
+++ b/plugins/gitee/models/migrationscripts/archived/issue_label.go
@@ -23,8 +23,9 @@ import "github.com/apache/incubator-devlake/models/migrationscripts/archived"
// Pull Requests are considered Issues in Gitee.
type GiteeIssueLabel struct {
- IssueId int `gorm:"primaryKey;autoIncrement:false"`
- LabelName string `gorm:"primaryKey;type:varchar(255)"`
+ ConnectionId uint64 `gorm:"primaryKey"`
+ IssueId int `gorm:"primaryKey;autoIncrement:false"`
+ LabelName string `gorm:"primaryKey;type:varchar(255)"`
archived.NoPKModel
}
diff --git a/plugins/gitee/models/migrationscripts/archived/pull_request.go b/plugins/gitee/models/migrationscripts/archived/pull_request.go
index 39ced959..90e22e26 100644
--- a/plugins/gitee/models/migrationscripts/archived/pull_request.go
+++ b/plugins/gitee/models/migrationscripts/archived/pull_request.go
@@ -24,6 +24,7 @@ import (
)
type GiteePullRequest struct {
+ ConnectionId uint64 `gorm:"primaryKey"`
GiteeId int `gorm:"primaryKey"`
RepoId int `gorm:"index"`
Number int `gorm:"index"` // This number is used in GET requests to the API associated to reviewers / comments / etc.
diff --git a/plugins/gitee/models/migrationscripts/archived/pull_request_comment.go b/plugins/gitee/models/migrationscripts/archived/pull_request_comment.go
index 79bf54be..16864b7a 100644
--- a/plugins/gitee/models/migrationscripts/archived/pull_request_comment.go
+++ b/plugins/gitee/models/migrationscripts/archived/pull_request_comment.go
@@ -24,8 +24,9 @@ import (
)
type GiteePullRequestComment struct {
- GiteeId int `gorm:"primaryKey"`
- PullRequestId int `gorm:"index"`
+ ConnectionId uint64 `gorm:"primaryKey"`
+ GiteeId int `gorm:"primaryKey"`
+ PullRequestId int `gorm:"index"`
Body string
AuthorUsername string `gorm:"type:varchar(255)"`
AuthorUserId int
diff --git a/plugins/gitee/models/migrationscripts/archived/pull_request_commit.go b/plugins/gitee/models/migrationscripts/archived/pull_request_commit.go
index 54b3617f..5249387b 100644
--- a/plugins/gitee/models/migrationscripts/archived/pull_request_commit.go
+++ b/plugins/gitee/models/migrationscripts/archived/pull_request_commit.go
@@ -20,6 +20,7 @@ package archived
import "github.com/apache/incubator-devlake/models/migrationscripts/archived"
type GiteePullRequestCommit struct {
+ ConnectionId uint64 `gorm:"primaryKey"`
CommitSha string `gorm:"primaryKey;type:varchar(40)"`
PullRequestId int `gorm:"primaryKey;autoIncrement:false"`
archived.NoPKModel
diff --git a/plugins/gitee/models/migrationscripts/archived/pull_request_issue.go b/plugins/gitee/models/migrationscripts/archived/pull_request_issue.go
index 20e3d13e..416efb84 100644
--- a/plugins/gitee/models/migrationscripts/archived/pull_request_issue.go
+++ b/plugins/gitee/models/migrationscripts/archived/pull_request_issue.go
@@ -20,8 +20,9 @@ package archived
import "github.com/apache/incubator-devlake/models/migrationscripts/archived"
type GiteePullRequestIssue struct {
- PullRequestId int `gorm:"primaryKey"`
- IssueId int `gorm:"primaryKey"`
+ ConnectionId uint64 `gorm:"primaryKey"`
+ PullRequestId int `gorm:"primaryKey"`
+ IssueId int `gorm:"primaryKey"`
PullRequestNumber int
IssueNumber int
archived.NoPKModel
diff --git a/plugins/gitee/models/migrationscripts/archived/pull_request_label.go b/plugins/gitee/models/migrationscripts/archived/pull_request_label.go
index 69f25c7e..84a52f23 100644
--- a/plugins/gitee/models/migrationscripts/archived/pull_request_label.go
+++ b/plugins/gitee/models/migrationscripts/archived/pull_request_label.go
@@ -20,8 +20,9 @@ package archived
import "github.com/apache/incubator-devlake/models/migrationscripts/archived"
type GiteePullRequestLabel struct {
- PullId int `gorm:"primaryKey;autoIncrement:false"`
- LabelName string `gorm:"primaryKey;type:varchar(255)"`
+ ConnectionId uint64 `gorm:"primaryKey"`
+ PullId int `gorm:"primaryKey;autoIncrement:false"`
+ LabelName string `gorm:"primaryKey;type:varchar(255)"`
archived.NoPKModel
}
diff --git a/plugins/gitee/models/migrationscripts/archived/repo.go b/plugins/gitee/models/migrationscripts/archived/repo.go
index 989aeae2..06dbec68 100644
--- a/plugins/gitee/models/migrationscripts/archived/repo.go
+++ b/plugins/gitee/models/migrationscripts/archived/repo.go
@@ -24,6 +24,7 @@ import (
)
type GiteeRepo struct {
+ ConnectionId uint64 `gorm:"primaryKey"`
GiteeId int `gorm:"primaryKey"`
Name string `gorm:"type:varchar(255)"`
HTMLUrl string `gorm:"type:varchar(255)"`
@@ -39,5 +40,5 @@ type GiteeRepo struct {
}
func (GiteeRepo) TableName() string {
- return "_tool_gitee_repo"
+ return "_tool_gitee_repos"
}
diff --git a/plugins/gitee/models/migrationscripts/archived/repo_commit.go b/plugins/gitee/models/migrationscripts/archived/repo_commit.go
index fa70dfb2..b35636b8 100644
--- a/plugins/gitee/models/migrationscripts/archived/repo_commit.go
+++ b/plugins/gitee/models/migrationscripts/archived/repo_commit.go
@@ -22,8 +22,9 @@ import (
)
type GiteeRepoCommit struct {
- RepoId int `gorm:"primaryKey"`
- CommitSha string `gorm:"primaryKey;type:varchar(40)"`
+ ConnectionId uint64 `gorm:"primaryKey"`
+ RepoId int `gorm:"primaryKey"`
+ CommitSha string `gorm:"primaryKey;type:varchar(40)"`
archived.NoPKModel
}
diff --git a/plugins/gitee/models/migrationscripts/archived/reviewer.go b/plugins/gitee/models/migrationscripts/archived/reviewer.go
index 69aeee6b..d7549879 100644
--- a/plugins/gitee/models/migrationscripts/archived/reviewer.go
+++ b/plugins/gitee/models/migrationscripts/archived/reviewer.go
@@ -20,6 +20,7 @@ package archived
import "github.com/apache/incubator-devlake/models/migrationscripts/archived"
type GiteeReviewer struct {
+ ConnectionId uint64 `gorm:"primaryKey"`
GiteeId int `gorm:"primaryKey"`
Login string `gorm:"type:varchar(255)"`
PullRequestId int `gorm:"primaryKey"`
diff --git a/plugins/gitee/models/migrationscripts/archived/user.go b/plugins/gitee/models/migrationscripts/archived/user.go
index 729b484d..8a78d62c 100644
--- a/plugins/gitee/models/migrationscripts/archived/user.go
+++ b/plugins/gitee/models/migrationscripts/archived/user.go
@@ -22,7 +22,8 @@ import (
)
type GiteeUser struct {
- Id int `json:"id" gorm:"primaryKey"`
+ ConnectionId uint64 `gorm:"primaryKey"`
+ Id int `json:"id" gorm:"primaryKey;autoIncrement:false"`
Login string `json:"login" gorm:"type:varchar(255)"`
Name string `json:"name" gorm:"type:varchar(255)"`
AvatarUrl string `json:"avatar_url" gorm:"type:varchar(255)"`
diff --git a/plugins/gitee/models/migrationscripts/init_schema.go b/plugins/gitee/models/migrationscripts/init_schema.go
index 305d4702..7666edae 100644
--- a/plugins/gitee/models/migrationscripts/init_schema.go
+++ b/plugins/gitee/models/migrationscripts/init_schema.go
@@ -118,7 +118,7 @@ func (*InitSchemas) Up(ctx context.Context, db *gorm.DB) error {
}
func (*InitSchemas) Version() uint64 {
- return 20220617231243
+ return 20220617231268
}
func (*InitSchemas) Name() string {
diff --git a/plugins/gitee/models/pull_request.go b/plugins/gitee/models/pull_request.go
index 7fb1caaf..a6cd2c6b 100644
--- a/plugins/gitee/models/pull_request.go
+++ b/plugins/gitee/models/pull_request.go
@@ -24,6 +24,7 @@ import (
)
type GiteePullRequest struct {
+ ConnectionId uint64 `gorm:"primaryKey"`
GiteeId int `gorm:"primaryKey"`
RepoId int `gorm:"index"`
Number int `gorm:"index"` // This number is used in GET requests to the API associated to reviewers / comments / etc.
diff --git a/plugins/gitee/models/pull_request_comment.go b/plugins/gitee/models/pull_request_comment.go
index 6e0b804a..27adb7c5 100644
--- a/plugins/gitee/models/pull_request_comment.go
+++ b/plugins/gitee/models/pull_request_comment.go
@@ -24,8 +24,9 @@ import (
)
type GiteePullRequestComment struct {
- GiteeId int `gorm:"primaryKey"`
- PullRequestId int `gorm:"index"`
+ ConnectionId uint64 `gorm:"primaryKey"`
+ GiteeId int `gorm:"primaryKey"`
+ PullRequestId int `gorm:"index"`
Body string
AuthorUsername string `gorm:"type:varchar(255)"`
AuthorUserId int
diff --git a/plugins/gitee/models/pull_request_commit.go b/plugins/gitee/models/pull_request_commit.go
index 6705b665..d5a0ea74 100644
--- a/plugins/gitee/models/pull_request_commit.go
+++ b/plugins/gitee/models/pull_request_commit.go
@@ -22,6 +22,7 @@ import (
)
type GiteePullRequestCommit struct {
+ ConnectionId uint64 `gorm:"primaryKey"`
CommitSha string `gorm:"primaryKey;type:varchar(40)"`
PullRequestId int `gorm:"primaryKey;autoIncrement:false"`
common.NoPKModel
diff --git a/plugins/gitee/models/pull_request_issue.go b/plugins/gitee/models/pull_request_issue.go
index 8527ff4d..fb0a817b 100644
--- a/plugins/gitee/models/pull_request_issue.go
+++ b/plugins/gitee/models/pull_request_issue.go
@@ -20,8 +20,9 @@ package models
import "github.com/apache/incubator-devlake/models/common"
type GiteePullRequestIssue struct {
- PullRequestId int `gorm:"primaryKey"`
- IssueId int `gorm:"primaryKey"`
+ ConnectionId uint64 `gorm:"primaryKey"`
+ PullRequestId int `gorm:"primaryKey"`
+ IssueId int `gorm:"primaryKey"`
PullRequestNumber int
IssueNumber string
common.NoPKModel
diff --git a/plugins/gitee/models/pull_request_label.go b/plugins/gitee/models/pull_request_label.go
index 619eeb0d..280e5542 100644
--- a/plugins/gitee/models/pull_request_label.go
+++ b/plugins/gitee/models/pull_request_label.go
@@ -22,8 +22,9 @@ import (
)
type GiteePullRequestLabel struct {
- PullId int `gorm:"primaryKey;autoIncrement:false"`
- LabelName string `gorm:"primaryKey;type:varchar(255)"`
+ ConnectionId uint64 `gorm:"primaryKey"`
+ PullId int `gorm:"primaryKey;autoIncrement:false"`
+ LabelName string `gorm:"primaryKey;type:varchar(255)"`
common.NoPKModel
}
diff --git a/plugins/gitee/models/repo.go b/plugins/gitee/models/repo.go
index 84a4b493..d538aa5d 100644
--- a/plugins/gitee/models/repo.go
+++ b/plugins/gitee/models/repo.go
@@ -24,6 +24,7 @@ import (
)
type GiteeRepo struct {
+ ConnectionId uint64 `gorm:"primaryKey"`
GiteeId int `gorm:"primaryKey"`
Name string `gorm:"type:varchar(255)"`
HTMLUrl string `gorm:"type:varchar(255)"`
@@ -39,5 +40,5 @@ type GiteeRepo struct {
}
func (GiteeRepo) TableName() string {
- return "_tool_gitee_repo"
+ return "_tool_gitee_repos"
}
diff --git a/plugins/gitee/models/repo_commit.go b/plugins/gitee/models/repo_commit.go
index 7e66077e..f3c6ffbe 100644
--- a/plugins/gitee/models/repo_commit.go
+++ b/plugins/gitee/models/repo_commit.go
@@ -22,8 +22,9 @@ import (
)
type GiteeRepoCommit struct {
- RepoId int `gorm:"primaryKey"`
- CommitSha string `gorm:"primaryKey;type:varchar(40)"`
+ ConnectionId uint64 `gorm:"primaryKey"`
+ RepoId int `gorm:"primaryKey"`
+ CommitSha string `gorm:"primaryKey;type:varchar(40)"`
common.NoPKModel
}
diff --git a/plugins/gitee/models/reviewer.go b/plugins/gitee/models/reviewer.go
index fc2c6a1d..251493ad 100644
--- a/plugins/gitee/models/reviewer.go
+++ b/plugins/gitee/models/reviewer.go
@@ -22,6 +22,7 @@ import (
)
type GiteeReviewer struct {
+ ConnectionId uint64 `gorm:"primaryKey"`
GiteeId int `gorm:"primaryKey"`
Login string `gorm:"type:varchar(255)"`
PullRequestId int `gorm:"primaryKey"`
diff --git a/plugins/gitee/models/user.go b/plugins/gitee/models/user.go
index acf6309e..935b863f 100644
--- a/plugins/gitee/models/user.go
+++ b/plugins/gitee/models/user.go
@@ -20,7 +20,8 @@ package models
import "github.com/apache/incubator-devlake/models/common"
type GiteeUser struct {
- Id int `json:"id" gorm:"primaryKey"`
+ ConnectionId uint64 `gorm:"primaryKey"`
+ Id int `json:"id" gorm:"primaryKey;autoIncrement:false"`
Login string `json:"login" gorm:"type:varchar(255)"`
Name string `json:"name" gorm:"type:varchar(255)"`
AvatarUrl string `json:"avatar_url" gorm:"type:varchar(255)"`
diff --git a/plugins/gitee/tasks/commit_collector.go b/plugins/gitee/tasks/commit_collector.go
index 0d37a632..cded9309 100644
--- a/plugins/gitee/tasks/commit_collector.go
+++ b/plugins/gitee/tasks/commit_collector.go
@@ -18,9 +18,13 @@ limitations under the License.
package tasks
import (
+ "fmt"
"net/url"
"strconv"
+ "github.com/apache/incubator-devlake/plugins/core/dal"
+ "github.com/apache/incubator-devlake/plugins/gitee/models"
+
"github.com/apache/incubator-devlake/plugins/core"
"github.com/apache/incubator-devlake/plugins/helper"
)
@@ -35,18 +39,41 @@ var CollectCommitsMeta = core.SubTaskMeta{
}
func CollectApiCommits(taskCtx core.SubTaskContext) error {
+ db := taskCtx.GetDal()
rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_COMMIT_TABLE)
+ since := data.Since
+ incremental := false
+ if since == nil {
+ latestUpdated := &models.GiteeCommit{}
+ err := db.All(
+ &latestUpdated,
+ dal.Join("left join _tool_gitee_repo_commits on _tool_gitee_commits.sha = _tool_gitee_repo_commits.commit_sha"),
+ dal.Join("left join _tool_gitee_repos on _tool_gitee_repo_commits.repo_id = _tool_gitee_repos.gitee_id"),
+ dal.Where("_tool_gitee_repo_commits.repo_id = ? AND _tool_gitee_repo_commits.connection_id = ?", data.Repo.GiteeId, data.Repo.ConnectionId),
+ dal.Orderby("committed_date DESC"),
+ dal.Limit(1),
+ )
+
+ if err != nil {
+ return fmt.Errorf("failed to get latest gitee commit record: %w", err)
+ }
+ if latestUpdated.Sha != "" {
+ since = &latestUpdated.CommittedDate
+ incremental = true
+ }
+ }
collector, err := helper.NewApiCollector(helper.ApiCollectorArgs{
RawDataSubTaskArgs: *rawDataSubTaskArgs,
ApiClient: data.ApiClient,
PageSize: 100,
- Incremental: false,
+ Incremental: incremental,
UrlTemplate: "repos/{{ .Params.Owner }}/{{ .Params.Repo }}/commits",
Query: func(reqData *helper.RequestData) (url.Values, error) {
query := url.Values{}
- query.Set("with_stats", "true")
- query.Set("sort", "asc")
+ if since != nil {
+ query.Set("since", since.String())
+ }
query.Set("page", strconv.Itoa(reqData.Pager.Page))
query.Set("per_page", strconv.Itoa(reqData.Pager.Size))
return query, nil
diff --git a/plugins/gitee/tasks/commit_convertor.go b/plugins/gitee/tasks/commit_convertor.go
index d9405edc..1565d534 100644
--- a/plugins/gitee/tasks/commit_convertor.go
+++ b/plugins/gitee/tasks/commit_convertor.go
@@ -20,11 +20,12 @@ package tasks
import (
"reflect"
+ "github.com/apache/incubator-devlake/plugins/core/dal"
+
"github.com/apache/incubator-devlake/models/domainlayer/code"
"github.com/apache/incubator-devlake/models/domainlayer/didgen"
"github.com/apache/incubator-devlake/plugins/core"
"github.com/apache/incubator-devlake/plugins/gitee/models"
- giteeModels "github.com/apache/incubator-devlake/plugins/gitee/models"
"github.com/apache/incubator-devlake/plugins/helper"
)
@@ -38,26 +39,26 @@ var ConvertCommitsMeta = core.SubTaskMeta{
func ConvertCommits(taskCtx core.SubTaskContext) error {
rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_COMMIT_TABLE)
- db := taskCtx.GetDb()
+ db := taskCtx.GetDal()
repoId := data.Repo.GiteeId
// select all commits belongs to the project
- cursor, err := db.Table("_tool_gitee_commits gc").
- Joins(`left join _tool_gitee_repo_commits gpc on (
- gpc.commit_sha = gc.sha
- )`).
- Select("gc.*").
- Where("gpc.repo_id = ?", repoId).
- Rows()
+ cursor, err := db.Cursor(
+ dal.Select("gc.*"),
+ dal.From("_tool_gitee_commits gc"),
+ dal.Join(`left join _tool_gitee_repo_commits grc on (
+ grc.commit_sha = gc.sha
+ )`),
+ dal.Where("grc.repo_id = ? AND grc.connection_id = ?", repoId, data.Options.ConnectionId),
+ )
if err != nil {
return err
}
defer cursor.Close()
- // TODO: adopt batch indate operation
userDidGen := didgen.NewDomainIdGenerator(&models.GiteeUser{})
- repoDidGen := didgen.NewDomainIdGenerator(&giteeModels.GiteeRepo{})
- domainRepoId := repoDidGen.Generate(repoId)
+ repoDidGen := didgen.NewDomainIdGenerator(&models.GiteeRepo{})
+ domainRepoId := repoDidGen.Generate(data.Options.ConnectionId, repoId)
converter, err := helper.NewDataConverter(helper.DataConverterArgs{
RawDataSubTaskArgs: *rawDataSubTaskArgs,
@@ -73,14 +74,14 @@ func ConvertCommits(taskCtx core.SubTaskContext) error {
commit.Message = giteeCommit.Message
commit.Additions = giteeCommit.Additions
commit.Deletions = giteeCommit.Deletions
- commit.AuthorId = userDidGen.Generate(giteeCommit.AuthorId)
+ commit.AuthorId = userDidGen.Generate(data.Options.ConnectionId, giteeCommit.AuthorId)
commit.AuthorName = giteeCommit.AuthorName
commit.AuthorEmail = giteeCommit.AuthorEmail
commit.AuthoredDate = giteeCommit.AuthoredDate
commit.CommitterName = giteeCommit.CommitterName
commit.CommitterEmail = giteeCommit.CommitterEmail
commit.CommittedDate = giteeCommit.CommittedDate
- commit.CommitterId = userDidGen.Generate(giteeCommit.CommitterId)
+ commit.CommitterId = userDidGen.Generate(data.Options.ConnectionId, giteeCommit.CommitterId)
// convert repo / commits relationship
repoCommit := &code.RepoCommit{
diff --git a/plugins/gitee/tasks/commit_extractor.go b/plugins/gitee/tasks/commit_extractor.go
index 1ebeb81e..7635afa4 100644
--- a/plugins/gitee/tasks/commit_extractor.go
+++ b/plugins/gitee/tasks/commit_extractor.go
@@ -93,8 +93,9 @@ func ExtractApiCommits(taskCtx core.SubTaskContext) error {
}
giteeRepoCommit := &models.GiteeRepoCommit{
- RepoId: data.Repo.GiteeId,
- CommitSha: commit.Sha,
+ ConnectionId: data.Options.ConnectionId,
+ RepoId: data.Repo.GiteeId,
+ CommitSha: commit.Sha,
}
results = append(results, giteeCommit)
results = append(results, giteeRepoCommit)
diff --git a/plugins/gitee/tasks/commit_stats_collector.go b/plugins/gitee/tasks/commit_stats_collector.go
index 83902bf4..29e4b61b 100644
--- a/plugins/gitee/tasks/commit_stats_collector.go
+++ b/plugins/gitee/tasks/commit_stats_collector.go
@@ -25,6 +25,8 @@ import (
"net/url"
"reflect"
+ "github.com/apache/incubator-devlake/plugins/core/dal"
+
"github.com/apache/incubator-devlake/plugins/helper"
"github.com/apache/incubator-devlake/plugins/core"
@@ -41,26 +43,33 @@ var CollectApiCommitStatsMeta = core.SubTaskMeta{
}
func CollectApiCommitStats(taskCtx core.SubTaskContext) error {
- db := taskCtx.GetDb()
+ db := taskCtx.GetDal()
rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_COMMIT_STATS_TABLE)
var latestUpdated models.GiteeCommitStat
- err := db.Model(&latestUpdated).Joins("left join _tool_gitee_repo_commits on _tool_gitee_commit_stats.sha = _tool_gitee_repo_commits.commit_sha").
- Where("_tool_gitee_repo_commits.repo_id = ?", data.Repo.GiteeId).
- Order("committed_date DESC").Limit(1).Find(&latestUpdated).Error
+
+ err := db.First(
+ &latestUpdated,
+ dal.Join("left join _tool_gitee_repo_commits on _tool_gitee_commit_stats.sha = _tool_gitee_repo_commits.commit_sha"),
+ dal.Where("_tool_gitee_repo_commits.repo_id = ? and _tool_gitee_repo_commits.connection_id = ?", data.Repo.GiteeId, data.Repo.ConnectionId),
+ dal.Orderby("committed_date DESC"),
+ dal.Limit(1),
+ )
+
if err != nil {
return fmt.Errorf("failed to get latest gitee commit record: %w", err)
}
- cursor, err := db.Model(&models.GiteeCommit{}).
- Joins("left join _tool_gitee_repo_commits on _tool_gitee_commits.sha = _tool_gitee_repo_commits.commit_sha").
- Where("_tool_gitee_repo_commits.repo_id = ? and _tool_gitee_commits.committed_date >= ?",
- data.Repo.GiteeId, latestUpdated.CommittedDate.String()).
- Rows()
+ cursor, err := db.Cursor(
+ dal.Join("left join _tool_gitee_repo_commits on _tool_gitee_commits.sha = _tool_gitee_repo_commits.commit_sha"),
+ dal.From(models.GiteeCommit{}.TableName()),
+ dal.Where("_tool_gitee_repo_commits.repo_id = ? and _tool_gitee_repo_commits.connection_id = ? and _tool_gitee_commits.committed_date >= ?",
+ data.Repo.GiteeId, data.Repo.ConnectionId, latestUpdated.CommittedDate.String()),
+ )
if err != nil {
return err
}
- iterator, err := helper.NewCursorIterator(db, cursor, reflect.TypeOf(models.GiteeCommit{}))
+ iterator, err := helper.NewDalCursorIterator(db, cursor, reflect.TypeOf(models.GiteeCommit{}))
if err != nil {
return err
}
diff --git a/plugins/gitee/tasks/commit_stats_extractor.go b/plugins/gitee/tasks/commit_stats_extractor.go
index 680b01a6..9c5f2b55 100644
--- a/plugins/gitee/tasks/commit_stats_extractor.go
+++ b/plugins/gitee/tasks/commit_stats_extractor.go
@@ -20,6 +20,8 @@ package tasks
import (
"encoding/json"
+ "github.com/apache/incubator-devlake/plugins/core/dal"
+
"github.com/apache/incubator-devlake/plugins/core"
"github.com/apache/incubator-devlake/plugins/gitee/models"
"github.com/apache/incubator-devlake/plugins/helper"
@@ -50,7 +52,7 @@ type ApiSingleCommitResponse struct {
}
func ExtractApiCommitStats(taskCtx core.SubTaskContext) error {
- rawDataSubTaskArgs, _ := CreateRawDataSubTaskArgs(taskCtx, RAW_COMMIT_STATS_TABLE)
+ rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_COMMIT_STATS_TABLE)
extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{
RawDataSubTaskArgs: *rawDataSubTaskArgs,
@@ -64,9 +66,9 @@ func ExtractApiCommitStats(taskCtx core.SubTaskContext) error {
return nil, nil
}
- db := taskCtx.GetDb()
+ db := taskCtx.GetDal()
commit := &models.GiteeCommit{}
- err = db.Model(commit).Where("sha = ?", body.Sha).Limit(1).Find(commit).Error
+ err = db.First(commit, dal.Where("sha = ?", body.Sha), dal.Limit(1))
if err != nil {
return nil, err
}
@@ -75,6 +77,7 @@ func ExtractApiCommitStats(taskCtx core.SubTaskContext) error {
commit.Deletions = body.Stats.Deletions
commitStat := &models.GiteeCommitStat{
+ ConnectionId: data.Options.ConnectionId,
Additions: body.Stats.Additions,
Deletions: body.Stats.Deletions,
CommittedDate: body.Commit.Committer.Date.ToTime(),
diff --git a/plugins/gitee/tasks/issue_collector.go b/plugins/gitee/tasks/issue_collector.go
index 5b325084..83c821f0 100644
--- a/plugins/gitee/tasks/issue_collector.go
+++ b/plugins/gitee/tasks/issue_collector.go
@@ -23,6 +23,8 @@ import (
"net/http"
"net/url"
+ "github.com/apache/incubator-devlake/plugins/core/dal"
+
"github.com/apache/incubator-devlake/plugins/helper"
"github.com/apache/incubator-devlake/plugins/core"
@@ -39,7 +41,7 @@ var CollectApiIssuesMeta = core.SubTaskMeta{
}
func CollectApiIssues(taskCtx core.SubTaskContext) error {
- db := taskCtx.GetDb()
+ db := taskCtx.GetDal()
rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_ISSUE_TABLE)
since := data.Since
@@ -47,9 +49,12 @@ func CollectApiIssues(taskCtx core.SubTaskContext) error {
// user didn't specify a time range to sync, try load from database
if since == nil {
var latestUpdated models.GiteeIssue
- err := db.Model(&latestUpdated).
- Where("repo_id = ?", data.Repo.GiteeId).
- Order("gitee_updated_at DESC").Limit(1).Find(&latestUpdated).Error
+ err := db.All(
+ &latestUpdated,
+ dal.Where("repo_id = ? and connection_id = ?", data.Repo.GiteeId, data.Repo.ConnectionId),
+ dal.Orderby("gitee_updated_at DESC"),
+ dal.Limit(1),
+ )
if err != nil {
return fmt.Errorf("failed to get latest gitee issue record: %w", err)
}
diff --git a/plugins/gitee/tasks/issue_comment_collector.go b/plugins/gitee/tasks/issue_comment_collector.go
index df0d6364..38a1c22d 100644
--- a/plugins/gitee/tasks/issue_comment_collector.go
+++ b/plugins/gitee/tasks/issue_comment_collector.go
@@ -21,6 +21,7 @@ import (
"fmt"
"net/url"
+ . "github.com/apache/incubator-devlake/plugins/core/dal"
"github.com/apache/incubator-devlake/plugins/helper"
"github.com/apache/incubator-devlake/plugins/core"
@@ -37,7 +38,7 @@ var CollectApiIssueCommentsMeta = core.SubTaskMeta{
}
func CollectApiIssueComments(taskCtx core.SubTaskContext) error {
- db := taskCtx.GetDb()
+ db := taskCtx.GetDal()
rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_COMMENTS_TABLE)
since := data.Since
@@ -46,18 +47,26 @@ func CollectApiIssueComments(taskCtx core.SubTaskContext) error {
// actually, for gitee pull, since doesn't make any sense, gitee pull api doesn't support it
if since == nil {
var latestUpdatedIssueComment models.GiteeIssueComment
- err := db.Model(&latestUpdatedIssueComment).
- Joins("left join _tool_gitee_issues on _tool_gitee_issues.gitee_id = _tool_gitee_issue_comments.issue_id").
- Where("_tool_gitee_issues.repo_id = ?", data.Repo.GiteeId).
- Order("gitee_updated_at DESC").Limit(1).Find(&latestUpdatedIssueComment).Error
+ err := db.All(
+ &latestUpdatedIssueComment,
+ Join("left join _tool_gitee_issues on _tool_gitee_issues.gitee_id = _tool_gitee_issue_comments.issue_id"),
+ Where(
+ "_tool_gitee_issues.repo_id = ? AND _tool_gitee_issues.connection_id = ?", data.Repo.GiteeId, data.Repo.ConnectionId,
+ ),
+ Orderby("gitee_updated_at DESC"),
+ Limit(1),
+ )
if err != nil {
return fmt.Errorf("failed to get latest gitee issue record: %w", err)
}
var latestUpdatedPrComt models.GiteePullRequestComment
- err = db.Model(&latestUpdatedPrComt).
- Joins("left join _tool_gitee_pull_requests on _tool_gitee_pull_requests.gitee_id = _tool_gitee_pull_request_comments.pull_request_id").
- Where("_tool_gitee_pull_requests.repo_id = ?", data.Repo.GiteeId).
- Order("gitee_updated_at DESC").Limit(1).Find(&latestUpdatedPrComt).Error
+ err = db.All(
+ &latestUpdatedPrComt,
+ Join("left join _tool_gitee_pull_requests on _tool_gitee_pull_requests.gitee_id = _tool_gitee_pull_request_comments.pull_request_id"),
+ Where("_tool_gitee_pull_requests.repo_id = ? AND _tool_gitee_pull_requests.connection_id = ?", data.Repo.GiteeId, data.Repo.ConnectionId),
+ Orderby("gitee_updated_at DESC"),
+ Limit(1),
+ )
if err != nil {
return fmt.Errorf("failed to get latest gitee issue record: %w", err)
}
diff --git a/plugins/gitee/tasks/issue_comment_convertor.go b/plugins/gitee/tasks/issue_comment_convertor.go
index 279d6846..a2453fc8 100644
--- a/plugins/gitee/tasks/issue_comment_convertor.go
+++ b/plugins/gitee/tasks/issue_comment_convertor.go
@@ -20,6 +20,8 @@ package tasks
import (
"reflect"
+ "github.com/apache/incubator-devlake/plugins/core/dal"
+
"github.com/apache/incubator-devlake/models/domainlayer"
"github.com/apache/incubator-devlake/models/domainlayer/didgen"
"github.com/apache/incubator-devlake/models/domainlayer/ticket"
@@ -36,14 +38,16 @@ var ConvertIssueCommentsMeta = core.SubTaskMeta{
}
func ConvertIssueComments(taskCtx core.SubTaskContext) error {
- db := taskCtx.GetDb()
+ db := taskCtx.GetDal()
rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_COMMENTS_TABLE)
repoId := data.Repo.GiteeId
- cursor, err := db.Model(&models.GiteeIssueComment{}).
- Joins("left join _tool_gitee_issues "+
- "on _tool_gitee_issues.gitee_id = _tool_gitee_issue_comments.issue_id").
- Where("repo_id = ?", repoId).Rows()
+ cursor, err := db.Cursor(
+ dal.From(&models.GiteeIssueComment{}),
+ dal.Join("left join _tool_gitee_issues "+
+ "on _tool_gitee_issues.gitee_id = _tool_gitee_issue_comments.issue_id"),
+ dal.Where("repo_id = ? and _tool_gitee_issues.connection_id = ?", repoId, data.Options.ConnectionId),
+ )
if err != nil {
return err
}
@@ -60,11 +64,11 @@ func ConvertIssueComments(taskCtx core.SubTaskContext) error {
giteeIssueComment := inputRow.(*models.GiteeIssueComment)
domainIssueComment := &ticket.IssueComment{
DomainEntity: domainlayer.DomainEntity{
- Id: issueIdGen.Generate(giteeIssueComment.GiteeId),
+ Id: issueIdGen.Generate(data.Options.ConnectionId, giteeIssueComment.GiteeId),
},
- IssueId: issueIdGen.Generate(giteeIssueComment.IssueId),
+ IssueId: issueIdGen.Generate(data.Options.ConnectionId, giteeIssueComment.IssueId),
Body: giteeIssueComment.Body,
- UserId: userIdGen.Generate(giteeIssueComment.AuthorUserId),
+ UserId: userIdGen.Generate(data.Options.ConnectionId, giteeIssueComment.AuthorUserId),
CreatedDate: giteeIssueComment.GiteeCreatedAt,
}
return []interface{}{
diff --git a/plugins/gitee/tasks/issue_comment_extractor.go b/plugins/gitee/tasks/issue_comment_extractor.go
index 2fcfb76c..3caa6a46 100644
--- a/plugins/gitee/tasks/issue_comment_extractor.go
+++ b/plugins/gitee/tasks/issue_comment_extractor.go
@@ -20,6 +20,8 @@ package tasks
import (
"encoding/json"
+ "github.com/apache/incubator-devlake/plugins/core/dal"
+
"github.com/apache/incubator-devlake/plugins/core"
"github.com/apache/incubator-devlake/plugins/gitee/models"
"github.com/apache/incubator-devlake/plugins/helper"
@@ -84,18 +86,19 @@ func ExtractApiIssueComments(taskCtx core.SubTaskContext) error {
return nil, err
}
issue := &models.GiteeIssue{}
- err = taskCtx.GetDb().Where("number = ? and repo_id = ?", issueINumber, data.Repo.GiteeId).Limit(1).Find(issue).Error
+ err = taskCtx.GetDal().All(issue, dal.Where("connection_id = ? and number = ? and repo_id = ?", data.Options.ConnectionId, issueINumber, data.Repo.GiteeId))
if err != nil {
return nil, err
}
//if we can not find issues with issue number above, move the comments to gitee_pull_request_comments
if issue.GiteeId == 0 {
pr := &models.GiteePullRequest{}
- err = taskCtx.GetDb().Where("number = ? and repo_id = ?", issueINumber, data.Repo.GiteeId).Limit(1).Find(pr).Error
+ err = taskCtx.GetDal().All(issue, dal.Where("connection_id = ? and number = ? and repo_id = ?", data.Options.ConnectionId, issueINumber, data.Repo.GiteeId))
if err != nil {
return nil, err
}
giteePrComment := &models.GiteePullRequestComment{
+ ConnectionId: data.Options.ConnectionId,
GiteeId: apiComment.GiteeId,
PullRequestId: pr.GiteeId,
Body: apiComment.Body,
@@ -107,6 +110,7 @@ func ExtractApiIssueComments(taskCtx core.SubTaskContext) error {
results = append(results, giteePrComment)
} else {
giteeIssueComment := &models.GiteeIssueComment{
+ ConnectionId: data.Options.ConnectionId,
GiteeId: apiComment.GiteeId,
IssueId: issue.GiteeId,
Body: apiComment.Body,
diff --git a/plugins/gitee/tasks/issue_convertor.go b/plugins/gitee/tasks/issue_convertor.go
index 0fe2d3a1..5ebd2467 100644
--- a/plugins/gitee/tasks/issue_convertor.go
+++ b/plugins/gitee/tasks/issue_convertor.go
@@ -20,6 +20,8 @@ package tasks
import (
"reflect"
+ "github.com/apache/incubator-devlake/plugins/core/dal"
+
"github.com/apache/incubator-devlake/plugins/core"
"github.com/apache/incubator-devlake/plugins/helper"
@@ -37,12 +39,15 @@ var ConvertIssuesMeta = core.SubTaskMeta{
}
func ConvertIssues(taskCtx core.SubTaskContext) error {
- db := taskCtx.GetDb()
+ db := taskCtx.GetDal()
rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_ISSUE_TABLE)
repoId := data.Repo.GiteeId
issue := &giteeModels.GiteeIssue{}
- cursor, err := db.Model(issue).Where("repo_id = ?", repoId).Rows()
+ cursor, err := db.Cursor(
+ dal.From(issue),
+ dal.Where("repo_id = ? and connection_id=?", repoId, data.Options.ConnectionId),
+ )
if err != nil {
return err
@@ -60,15 +65,15 @@ func ConvertIssues(taskCtx core.SubTaskContext) error {
Convert: func(inputRow interface{}) ([]interface{}, error) {
issue := inputRow.(*giteeModels.GiteeIssue)
domainIssue := &ticket.Issue{
- DomainEntity: domainlayer.DomainEntity{Id: issueIdGen.Generate(issue.GiteeId)},
+ DomainEntity: domainlayer.DomainEntity{Id: issueIdGen.Generate(data.Options.ConnectionId, issue.GiteeId)},
IssueKey: issue.Number,
Title: issue.Title,
Description: issue.Body,
Priority: issue.Priority,
Type: issue.Type,
- AssigneeId: userIdGen.Generate(issue.AssigneeId),
+ AssigneeId: userIdGen.Generate(data.Options.ConnectionId, issue.AssigneeId),
AssigneeName: issue.AssigneeName,
- CreatorId: userIdGen.Generate(issue.AuthorId),
+ CreatorId: userIdGen.Generate(data.Options.ConnectionId, issue.AuthorId),
CreatorName: issue.AuthorName,
LeadTimeMinutes: issue.LeadTimeMinutes,
Url: issue.Url,
@@ -84,7 +89,7 @@ func ConvertIssues(taskCtx core.SubTaskContext) error {
domainIssue.Status = ticket.TODO
}
boardIssue := &ticket.BoardIssue{
- BoardId: boardIdGen.Generate(repoId),
+ BoardId: boardIdGen.Generate(data.Options.ConnectionId, repoId),
IssueId: domainIssue.Id,
}
return []interface{}{
diff --git a/plugins/gitee/tasks/issue_extractor.go b/plugins/gitee/tasks/issue_extractor.go
index 69cb7a57..c37476ec 100644
--- a/plugins/gitee/tasks/issue_extractor.go
+++ b/plugins/gitee/tasks/issue_extractor.go
@@ -96,7 +96,7 @@ type IssuesResponse struct {
func ExtractApiIssues(taskCtx core.SubTaskContext) error {
rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_ISSUE_TABLE)
- config := data.Options.Config
+ config := data.Options.TransformationRules
var issueSeverityRegex *regexp.Regexp
var issueComponentRegex *regexp.Regexp
var issuePriorityRegex *regexp.Regexp
@@ -104,44 +104,26 @@ func ExtractApiIssues(taskCtx core.SubTaskContext) error {
var issueTypeRequirementRegex *regexp.Regexp
var issueTypeIncidentRegex *regexp.Regexp
var issueSeverity = config.IssueSeverity
- if issueSeverity == "" {
- issueSeverity = taskCtx.GetConfig("GITEE_ISSUE_SEVERITY")
- }
- var issueComponent = config.IssueComponent
- if issueComponent == "" {
- issueComponent = taskCtx.GetConfig("GITEE_ISSUE_COMPONENT")
- }
- var issuePriority = config.IssuePriority
- if issuePriority == "" {
- issuePriority = taskCtx.GetConfig("GITEE_ISSUE_PRIORITY")
- }
- var issueTypeBug = config.IssueTypeBug
- if issueTypeBug == "" {
- issueTypeBug = taskCtx.GetConfig("GITEE_ISSUE_TYPE_BUG")
- }
- var issueTypeRequirement = config.IssueTypeRequirement
- if issueTypeRequirement == "" {
- issueTypeRequirement = taskCtx.GetConfig("GITEE_ISSUE_TYPE_REQUIREMENT")
- }
- var issueTypeIncident = config.IssueTypeIncident
- if issueTypeIncident == "" {
- issueTypeIncident = taskCtx.GetConfig("GITEE_ISSUE_TYPE_INCIDENT")
- }
if len(issueSeverity) > 0 {
issueSeverityRegex = regexp.MustCompile(issueSeverity)
}
+ var issueComponent = config.IssueComponent
if len(issueComponent) > 0 {
issueComponentRegex = regexp.MustCompile(issueComponent)
}
+ var issuePriority = config.IssuePriority
if len(issuePriority) > 0 {
issuePriorityRegex = regexp.MustCompile(issuePriority)
}
+ var issueTypeBug = config.IssueTypeBug
if len(issueTypeBug) > 0 {
issueTypeBugRegex = regexp.MustCompile(issueTypeBug)
}
+ var issueTypeRequirement = config.IssueTypeRequirement
if len(issueTypeRequirement) > 0 {
issueTypeRequirementRegex = regexp.MustCompile(issueTypeRequirement)
}
+ var issueTypeIncident = config.IssueTypeIncident
if len(issueTypeIncident) > 0 {
issueTypeIncidentRegex = regexp.MustCompile(issueTypeIncident)
}
@@ -163,14 +145,15 @@ func ExtractApiIssues(taskCtx core.SubTaskContext) error {
return nil, nil
}
results := make([]interface{}, 0, 2)
- giteeIssue, err := convertGiteeIssue(body, data.Repo.GiteeId)
+ giteeIssue, err := convertGiteeIssue(body, data.Options.ConnectionId, data.Repo.GiteeId)
if err != nil {
return nil, err
}
for _, label := range body.Labels {
results = append(results, &models.GiteeIssueLabel{
- IssueId: giteeIssue.GiteeId,
- LabelName: label.Name,
+ ConnectionId: data.Options.ConnectionId,
+ IssueId: giteeIssue.GiteeId,
+ LabelName: label.Name,
})
if issueSeverityRegex != nil {
groups := issueSeverityRegex.FindStringSubmatch(label.Name)
@@ -223,8 +206,9 @@ func ExtractApiIssues(taskCtx core.SubTaskContext) error {
return extractor.Execute()
}
-func convertGiteeIssue(issue *IssuesResponse, repositoryId int) (*models.GiteeIssue, error) {
+func convertGiteeIssue(issue *IssuesResponse, connectionId uint64, repositoryId int) (*models.GiteeIssue, error) {
giteeIssue := &models.GiteeIssue{
+ ConnectionId: connectionId,
GiteeId: issue.GiteeId,
RepoId: repositoryId,
Number: issue.Number,
diff --git a/plugins/gitee/tasks/issue_label_convertor.go b/plugins/gitee/tasks/issue_label_convertor.go
index d017cffa..95ba664d 100644
--- a/plugins/gitee/tasks/issue_label_convertor.go
+++ b/plugins/gitee/tasks/issue_label_convertor.go
@@ -20,6 +20,8 @@ package tasks
import (
"reflect"
+ "github.com/apache/incubator-devlake/plugins/core/dal"
+
"github.com/apache/incubator-devlake/models/domainlayer/didgen"
"github.com/apache/incubator-devlake/models/domainlayer/ticket"
"github.com/apache/incubator-devlake/plugins/core"
@@ -35,15 +37,17 @@ var ConvertIssueLabelsMeta = core.SubTaskMeta{
}
func ConvertIssueLabels(taskCtx core.SubTaskContext) error {
- db := taskCtx.GetDb()
+ db := taskCtx.GetDal()
rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_ISSUE_TABLE)
repoId := data.Repo.GiteeId
- cursor, err := db.Model(&models.GiteeIssueLabel{}).
- Joins(`left join _tool_gitee_issues on _tool_gitee_issues.gitee_id = _tool_gitee_issue_labels.issue_id`).
- Where("_tool_gitee_issues.repo_id = ?", repoId).
- Order("issue_id ASC").
- Rows()
+ cursor, err := db.Cursor(
+ dal.From(&models.GiteeIssueLabel{}),
+ dal.Join(`left join _tool_gitee_issues on _tool_gitee_issues.gitee_id = _tool_gitee_issue_labels.issue_id`),
+ dal.Where("_tool_gitee_issues.repo_id = ? and _tool_gitee_issues.connection_id = ?", repoId, data.Options.ConnectionId),
+ dal.Orderby("issue_id ASC"),
+ )
+
if err != nil {
return err
}
@@ -57,7 +61,7 @@ func ConvertIssueLabels(taskCtx core.SubTaskContext) error {
Convert: func(inputRow interface{}) ([]interface{}, error) {
issueLabel := inputRow.(*models.GiteeIssueLabel)
domainIssueLabel := &ticket.IssueLabel{
- IssueId: issueIdGen.Generate(issueLabel.IssueId),
+ IssueId: issueIdGen.Generate(data.Options.ConnectionId, issueLabel.IssueId),
LabelName: issueLabel.LabelName,
}
return []interface{}{
diff --git a/plugins/gitee/tasks/pr_collector.go b/plugins/gitee/tasks/pr_collector.go
index c20a308d..f22eb1f8 100644
--- a/plugins/gitee/tasks/pr_collector.go
+++ b/plugins/gitee/tasks/pr_collector.go
@@ -23,6 +23,8 @@ import (
"net/http"
"net/url"
+ "github.com/apache/incubator-devlake/plugins/core/dal"
+
"github.com/apache/incubator-devlake/plugins/gitee/models"
"github.com/apache/incubator-devlake/plugins/helper"
@@ -40,15 +42,20 @@ var CollectApiPullRequestsMeta = core.SubTaskMeta{
}
func CollectApiPullRequests(taskCtx core.SubTaskContext) error {
- db := taskCtx.GetDb()
+ db := taskCtx.GetDal()
rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_PULL_REQUEST_TABLE)
since := data.Since
incremental := false
if since == nil {
var latestUpdated models.GiteePullRequest
- err := db.Model(&latestUpdated).
- Where("repo_id = ?", data.Repo.GiteeId).
- Order("gitee_updated_at DESC").Limit(1).Find(&latestUpdated).Error
+
+ err := db.All(
+ &latestUpdated,
+ dal.Where("repo_id = ? and connection_id=?", data.Repo.GiteeId, data.Options.ConnectionId),
+ dal.Orderby("gitee_updated_at DESC"),
+ dal.Limit(1),
+ )
+
if err != nil {
return fmt.Errorf("failed to get latest gitee issue record: %w", err)
}
diff --git a/plugins/gitee/tasks/pr_comment_convertor.go b/plugins/gitee/tasks/pr_comment_convertor.go
index 03aff955..8a8c1312 100644
--- a/plugins/gitee/tasks/pr_comment_convertor.go
+++ b/plugins/gitee/tasks/pr_comment_convertor.go
@@ -20,6 +20,8 @@ package tasks
import (
"reflect"
+ "github.com/apache/incubator-devlake/plugins/core/dal"
+
"github.com/apache/incubator-devlake/models/domainlayer"
"github.com/apache/incubator-devlake/models/domainlayer/code"
"github.com/apache/incubator-devlake/models/domainlayer/didgen"
@@ -36,14 +38,16 @@ var ConvertPullRequestCommentsMeta = core.SubTaskMeta{
}
func ConvertPullRequestComments(taskCtx core.SubTaskContext) error {
- db := taskCtx.GetDb()
+ db := taskCtx.GetDal()
rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_COMMENTS_TABLE)
repoId := data.Repo.GiteeId
- cursor, err := db.Model(&models.GiteePullRequestComment{}).
- Joins("left join _tool_gitee_pull_requests "+
- "on _tool_gitee_pull_requests.gitee_id = _tool_gitee_pull_request_comments.pull_request_id").
- Where("repo_id = ?", repoId).Rows()
+ cursor, err := db.Cursor(
+ dal.From(&models.GiteePullRequestComment{}),
+ dal.Join("left join _tool_gitee_pull_requests "+
+ "on _tool_gitee_pull_requests.gitee_id = _tool_gitee_pull_request_comments.pull_request_id"),
+ dal.Where("repo_id = ? and _tool_gitee_pull_requests.connection_id = ?", repoId, data.Options.ConnectionId),
+ )
if err != nil {
return err
}
@@ -60,11 +64,11 @@ func ConvertPullRequestComments(taskCtx core.SubTaskContext) error {
giteePullRequestComment := inputRow.(*models.GiteePullRequestComment)
domainPrComment := &code.PullRequestComment{
DomainEntity: domainlayer.DomainEntity{
- Id: prIdGen.Generate(giteePullRequestComment.GiteeId),
+ Id: prIdGen.Generate(data.Options.ConnectionId, giteePullRequestComment.GiteeId),
},
- PullRequestId: prIdGen.Generate(giteePullRequestComment.PullRequestId),
+ PullRequestId: prIdGen.Generate(data.Options.ConnectionId, giteePullRequestComment.PullRequestId),
Body: giteePullRequestComment.Body,
- UserId: userIdGen.Generate(giteePullRequestComment.AuthorUserId),
+ UserId: userIdGen.Generate(data.Options.ConnectionId, giteePullRequestComment.AuthorUserId),
CreatedDate: giteePullRequestComment.GiteeCreatedAt,
CommitSha: "",
Position: 0,
diff --git a/plugins/gitee/tasks/pr_commit_collector.go b/plugins/gitee/tasks/pr_commit_collector.go
index 5cdd8e3e..b82d5949 100644
--- a/plugins/gitee/tasks/pr_commit_collector.go
+++ b/plugins/gitee/tasks/pr_commit_collector.go
@@ -24,6 +24,8 @@ import (
"net/url"
"reflect"
+ "github.com/apache/incubator-devlake/plugins/core/dal"
+
"github.com/apache/incubator-devlake/plugins/helper"
"github.com/apache/incubator-devlake/plugins/core"
@@ -45,18 +47,20 @@ type SimplePr struct {
}
func CollectApiPullRequestCommits(taskCtx core.SubTaskContext) error {
- db := taskCtx.GetDb()
+ db := taskCtx.GetDal()
rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_PULL_REQUEST_COMMIT_TABLE)
incremental := false
- cursor, err := db.Model(&models.GiteePullRequest{}).Select("number, gitee_id").
- Where("repo_id = ?", data.Repo.GiteeId).
- Rows()
+ cursor, err := db.Cursor(
+ dal.Select("number, gitee_id"),
+ dal.From(models.GiteePullRequest{}.TableName()),
+ dal.Where("repo_id = ? and connection_id=?", data.Repo.GiteeId, data.Options.ConnectionId),
+ )
if err != nil {
return err
}
- iterator, err := helper.NewCursorIterator(db, cursor, reflect.TypeOf(SimplePr{}))
+ iterator, err := helper.NewDalCursorIterator(db, cursor, reflect.TypeOf(SimplePr{}))
if err != nil {
return err
}
diff --git a/plugins/gitee/tasks/pr_commit_convertor.go b/plugins/gitee/tasks/pr_commit_convertor.go
index e0370135..2a38b97b 100644
--- a/plugins/gitee/tasks/pr_commit_convertor.go
+++ b/plugins/gitee/tasks/pr_commit_convertor.go
@@ -20,6 +20,8 @@ package tasks
import (
"reflect"
+ "github.com/apache/incubator-devlake/plugins/core/dal"
+
"github.com/apache/incubator-devlake/models/domainlayer/code"
"github.com/apache/incubator-devlake/models/domainlayer/didgen"
"github.com/apache/incubator-devlake/plugins/core"
@@ -35,16 +37,19 @@ var ConvertPullRequestCommitsMeta = core.SubTaskMeta{
}
func ConvertPullRequestCommits(taskCtx core.SubTaskContext) (err error) {
- db := taskCtx.GetDb()
+ db := taskCtx.GetDal()
rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_PULL_REQUEST_COMMIT_TABLE)
repoId := data.Repo.GiteeId
pullIdGen := didgen.NewDomainIdGenerator(&models.GiteePullRequest{})
- cursor, err := db.Model(&models.GiteePullRequestCommit{}).
- Joins(`left join _tool_gitee_pull_requests on _tool_gitee_pull_requests.gitee_id = _tool_gitee_pull_request_commits.pull_request_id`).
- Where("_tool_gitee_pull_requests.repo_id = ?", repoId).
- Order("pull_request_id ASC").Rows()
+ cursor, err := db.Cursor(
+ dal.From(&models.GiteePullRequestCommit{}),
+ dal.Join(`left join _tool_gitee_pull_requests on _tool_gitee_pull_requests.gitee_id = _tool_gitee_pull_request_commits.pull_request_id`),
+ dal.Where("_tool_gitee_pull_requests.repo_id = ? and _tool_gitee_pull_requests.connection_id = ?", repoId, data.Options.ConnectionId),
+ dal.Orderby("pull_request_id ASC"),
+ )
+
if err != nil {
return err
}
@@ -58,7 +63,7 @@ func ConvertPullRequestCommits(taskCtx core.SubTaskContext) (err error) {
giteePullRequestCommit := inputRow.(*models.GiteePullRequestCommit)
domainPrCommit := &code.PullRequestCommit{
CommitSha: giteePullRequestCommit.CommitSha,
- PullRequestId: pullIdGen.Generate(giteePullRequestCommit.PullRequestId),
+ PullRequestId: pullIdGen.Generate(data.Options.ConnectionId, giteePullRequestCommit.PullRequestId),
}
return []interface{}{
domainPrCommit,
diff --git a/plugins/gitee/tasks/pr_commit_extractor.go b/plugins/gitee/tasks/pr_commit_extractor.go
index 30befb6e..0e7b8b68 100644
--- a/plugins/gitee/tasks/pr_commit_extractor.go
+++ b/plugins/gitee/tasks/pr_commit_extractor.go
@@ -65,7 +65,7 @@ type PullRequestCommit struct {
}
func ExtractApiPullRequestCommits(taskCtx core.SubTaskContext) error {
- rawDataSubTaskArgs, _ := CreateRawDataSubTaskArgs(taskCtx, RAW_PULL_REQUEST_COMMIT_TABLE)
+ rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_PULL_REQUEST_COMMIT_TABLE)
extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{
RawDataSubTaskArgs: *rawDataSubTaskArgs,
Extract: func(row *helper.RawData) ([]interface{}, error) {
@@ -92,6 +92,7 @@ func ExtractApiPullRequestCommits(taskCtx core.SubTaskContext) error {
results = append(results, giteeCommit)
giteePullRequestCommit := &models.GiteePullRequestCommit{
+ ConnectionId: data.Options.ConnectionId,
CommitSha: apiPullRequestCommit.Sha,
PullRequestId: pull.GiteeId,
}
diff --git a/plugins/gitee/tasks/pr_convertor.go b/plugins/gitee/tasks/pr_convertor.go
index 28d4d4de..4b245f66 100644
--- a/plugins/gitee/tasks/pr_convertor.go
+++ b/plugins/gitee/tasks/pr_convertor.go
@@ -20,6 +20,8 @@ package tasks
import (
"reflect"
+ "github.com/apache/incubator-devlake/plugins/core/dal"
+
"github.com/apache/incubator-devlake/models/domainlayer"
"github.com/apache/incubator-devlake/models/domainlayer/code"
"github.com/apache/incubator-devlake/models/domainlayer/didgen"
@@ -36,11 +38,14 @@ var ConvertPullRequestsMeta = core.SubTaskMeta{
}
func ConvertPullRequests(taskCtx core.SubTaskContext) error {
- db := taskCtx.GetDb()
+ db := taskCtx.GetDal()
rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_PULL_REQUEST_TABLE)
repoId := data.Repo.GiteeId
- cursor, err := db.Model(&models.GiteePullRequest{}).Where("repo_id = ?", repoId).Rows()
+ cursor, err := db.Cursor(
+ dal.From(&models.GiteePullRequest{}),
+ dal.Where("repo_id = ? and connection_id = ?", repoId, data.Options.ConnectionId),
+ )
if err != nil {
return err
}
@@ -58,13 +63,13 @@ func ConvertPullRequests(taskCtx core.SubTaskContext) error {
pr := inputRow.(*models.GiteePullRequest)
domainPr := &code.PullRequest{
DomainEntity: domainlayer.DomainEntity{
- Id: prIdGen.Generate(pr.GiteeId),
+ Id: prIdGen.Generate(data.Options.ConnectionId, pr.GiteeId),
},
- BaseRepoId: repoIdGen.Generate(pr.RepoId),
+ BaseRepoId: repoIdGen.Generate(data.Options.ConnectionId, pr.RepoId),
Status: pr.State,
Title: pr.Title,
Url: pr.Url,
- AuthorId: userIdGen.Generate(pr.AuthorId),
+ AuthorId: userIdGen.Generate(data.Options.ConnectionId, pr.AuthorId),
AuthorName: pr.AuthorName,
Description: pr.Body,
CreatedDate: pr.GiteeCreatedAt,
diff --git a/plugins/gitee/tasks/pr_extractor.go b/plugins/gitee/tasks/pr_extractor.go
index 3a8f0540..ffd2bd08 100644
--- a/plugins/gitee/tasks/pr_extractor.go
+++ b/plugins/gitee/tasks/pr_extractor.go
@@ -77,20 +77,14 @@ type GiteeApiPullResponse struct {
func ExtractApiPullRequests(taskCtx core.SubTaskContext) error {
rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_PULL_REQUEST_TABLE)
- config := data.Options.Config
+ config := data.Options.TransformationRules
var labelTypeRegex *regexp.Regexp
var labelComponentRegex *regexp.Regexp
var prType = config.PrType
- if prType == "" {
- prType = taskCtx.GetConfig("GITEE_PR_TYPE")
- }
- var prComponent = config.PrComponent
- if prComponent == "" {
- prComponent = taskCtx.GetConfig("GITEE_PR_COMPONENT")
- }
if len(prType) > 0 {
labelTypeRegex = regexp.MustCompile(prType)
}
+ var prComponent = config.PrComponent
if len(prComponent) > 0 {
labelComponentRegex = regexp.MustCompile(prComponent)
}
@@ -109,14 +103,15 @@ func ExtractApiPullRequests(taskCtx core.SubTaskContext) error {
if pullResponse.GiteeId == 0 {
return nil, nil
}
- giteePr, err := convertGiteePullRequest(pullResponse, data.Repo.GiteeId)
+ giteePr, err := convertGiteePullRequest(pullResponse, data.Options.ConnectionId, data.Repo.GiteeId)
if err != nil {
return nil, err
}
for _, label := range pullResponse.Labels {
results = append(results, &models.GiteePullRequestLabel{
- PullId: giteePr.GiteeId,
- LabelName: label.Name,
+ ConnectionId: data.Options.ConnectionId,
+ PullId: giteePr.GiteeId,
+ LabelName: label.Name,
})
// if pr.Type has not been set and prType is set in .env, process the below
if labelTypeRegex != nil {
@@ -146,8 +141,9 @@ func ExtractApiPullRequests(taskCtx core.SubTaskContext) error {
return extractor.Execute()
}
-func convertGiteePullRequest(pull *GiteeApiPullResponse, repoId int) (*models.GiteePullRequest, error) {
+func convertGiteePullRequest(pull *GiteeApiPullResponse, connId uint64, repoId int) (*models.GiteePullRequest, error) {
giteePull := &models.GiteePullRequest{
+ ConnectionId: connId,
GiteeId: pull.GiteeId,
RepoId: repoId,
Number: pull.Number,
diff --git a/plugins/gitee/tasks/pr_issue_convertor.go b/plugins/gitee/tasks/pr_issue_convertor.go
index 07a24300..eab4bae8 100644
--- a/plugins/gitee/tasks/pr_issue_convertor.go
+++ b/plugins/gitee/tasks/pr_issue_convertor.go
@@ -21,6 +21,8 @@ import (
"reflect"
"strconv"
+ "github.com/apache/incubator-devlake/plugins/core/dal"
+
"github.com/apache/incubator-devlake/models/domainlayer/crossdomain"
"github.com/apache/incubator-devlake/models/domainlayer/didgen"
"github.com/apache/incubator-devlake/plugins/core"
@@ -36,15 +38,17 @@ var ConvertPullRequestIssuesMeta = core.SubTaskMeta{
}
func ConvertPullRequestIssues(taskCtx core.SubTaskContext) error {
- db := taskCtx.GetDb()
+ db := taskCtx.GetDal()
rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_PULL_REQUEST_TABLE)
repoId := data.Repo.GiteeId
- cursor, err := db.Model(&models.GiteePullRequestIssue{}).
- Joins(`left join _tool_gitee_pull_requests on _tool_gitee_pull_requests.gitee_id = _tool_gitee_pull_request_issues.pull_request_id`).
- Where("_tool_gitee_pull_requests.repo_id = ?", repoId).
- Order("pull_request_id ASC").
- Rows()
+ cursor, err := db.Cursor(
+ dal.From(&models.GiteePullRequestIssue{}),
+ dal.Join(`left join _tool_gitee_pull_requests on _tool_gitee_pull_requests.gitee_id = _tool_gitee_pull_request_issues.pull_request_id`),
+ dal.Where("_tool_gitee_pull_requests.repo_id = ? and _tool_gitee_pull_requests.connection_id = ?", repoId, data.Options.ConnectionId),
+ dal.Orderby("pull_request_id ASC"),
+ )
+
if err != nil {
return err
}
@@ -60,8 +64,8 @@ func ConvertPullRequestIssues(taskCtx core.SubTaskContext) error {
giteePrIssue := inputRow.(*models.GiteePullRequestIssue)
issueNum, _ := strconv.Atoi(giteePrIssue.IssueNumber)
pullRequestIssue := &crossdomain.PullRequestIssue{
- PullRequestId: prIdGen.Generate(giteePrIssue.PullRequestId),
- IssueId: issueIdGen.Generate(giteePrIssue.IssueId),
+ PullRequestId: prIdGen.Generate(data.Options.ConnectionId, giteePrIssue.PullRequestId),
+ IssueId: issueIdGen.Generate(data.Options.ConnectionId, giteePrIssue.IssueId),
IssueNumber: issueNum,
PullRequestNumber: giteePrIssue.PullRequestNumber,
}
diff --git a/plugins/gitee/tasks/pr_issue_enricher.go b/plugins/gitee/tasks/pr_issue_enricher.go
index 3fa136cf..dff9ab5d 100644
--- a/plugins/gitee/tasks/pr_issue_enricher.go
+++ b/plugins/gitee/tasks/pr_issue_enricher.go
@@ -23,6 +23,8 @@ import (
"strconv"
"strings"
+ "github.com/apache/incubator-devlake/plugins/core/dal"
+
"github.com/apache/incubator-devlake/plugins/core"
"github.com/apache/incubator-devlake/plugins/gitee/models"
"github.com/apache/incubator-devlake/plugins/helper"
@@ -36,12 +38,12 @@ var EnrichPullRequestIssuesMeta = core.SubTaskMeta{
}
func EnrichPullRequestIssues(taskCtx core.SubTaskContext) (err error) {
- db := taskCtx.GetDb()
+ db := taskCtx.GetDal()
rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_PULL_REQUEST_TABLE)
repoId := data.Repo.GiteeId
var prBodyCloseRegex *regexp.Regexp
- prBodyClosePattern := taskCtx.GetConfig("GITEE_PR_BODY_CLOSE_PATTERN")
+ prBodyClosePattern := data.Options.PrBodyClosePattern
//the pattern before the issue number, sometimes, the issue number is #1098, sometimes it is https://xxx/#1098
prBodyClosePattern = strings.Replace(prBodyClosePattern, "%s", data.Options.Owner, 1)
prBodyClosePattern = strings.Replace(prBodyClosePattern, "%s", data.Options.Repo, 1)
@@ -49,9 +51,7 @@ func EnrichPullRequestIssues(taskCtx core.SubTaskContext) (err error) {
prBodyCloseRegex = regexp.MustCompile(prBodyClosePattern)
}
charPattern := regexp.MustCompile(`[a-zA-Z\s,]+`)
- cursor, err := db.Model(&models.GiteePullRequest{}).
- Where("repo_id = ?", repoId).
- Rows()
+ cursor, err := db.Cursor(dal.From(&models.GiteePullRequest{}), dal.Where("repo_id = ?", repoId))
if err != nil {
return err
}
@@ -89,8 +89,11 @@ func EnrichPullRequestIssues(taskCtx core.SubTaskContext) (err error) {
if numFormatErr != nil {
continue
}
- err = db.Where("number = ? and repo_id = ?", issueNumber, repoId).
- Limit(1).Find(issue).Error
+ err = db.All(
+ issue,
+ dal.Where("number = ? and repo_id = ? and connection_id = ?", issueNumber, repoId, data.Options.ConnectionId),
+ dal.Limit(1),
+ )
if err != nil {
return nil, err
}
@@ -98,6 +101,7 @@ func EnrichPullRequestIssues(taskCtx core.SubTaskContext) (err error) {
continue
}
giteePullRequstIssue := &models.GiteePullRequestIssue{
+ ConnectionId: data.Options.ConnectionId,
PullRequestId: giteePullRequst.GiteeId,
IssueId: issue.GiteeId,
PullRequestNumber: giteePullRequst.Number,
diff --git a/plugins/gitee/tasks/pr_label_convertor.go b/plugins/gitee/tasks/pr_label_convertor.go
index 14b87161..c4fbdb39 100644
--- a/plugins/gitee/tasks/pr_label_convertor.go
+++ b/plugins/gitee/tasks/pr_label_convertor.go
@@ -20,6 +20,8 @@ package tasks
import (
"reflect"
+ "github.com/apache/incubator-devlake/plugins/core/dal"
+
"github.com/apache/incubator-devlake/models/domainlayer/code"
"github.com/apache/incubator-devlake/models/domainlayer/didgen"
"github.com/apache/incubator-devlake/plugins/core"
@@ -35,15 +37,17 @@ var ConvertPullRequestLabelsMeta = core.SubTaskMeta{
}
func ConvertPullRequestLabels(taskCtx core.SubTaskContext) error {
- db := taskCtx.GetDb()
+ db := taskCtx.GetDal()
rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_PULL_REQUEST_TABLE)
repoId := data.Repo.GiteeId
- cursor, err := db.Model(&models.GiteePullRequestLabel{}).
- Joins(`left join _tool_gitee_pull_requests on _tool_gitee_pull_requests.gitee_id = _tool_gitee_pull_request_labels.pull_id`).
- Where("_tool_gitee_pull_requests.repo_id = ?", repoId).
- Order("pull_id ASC").
- Rows()
+ cursor, err := db.Cursor(
+ dal.From(&models.GiteePullRequestLabel{}),
+ dal.Join(`left join _tool_gitee_pull_requests on _tool_gitee_pull_requests.gitee_id = _tool_gitee_pull_request_labels.pull_id`),
+ dal.Where("_tool_gitee_pull_requests.repo_id = ? and _tool_gitee_pull_requests.connection_id = ?", repoId, data.Options.ConnectionId),
+ dal.Orderby("pull_id ASC"),
+ )
+
if err != nil {
return err
}
@@ -57,7 +61,7 @@ func ConvertPullRequestLabels(taskCtx core.SubTaskContext) error {
Convert: func(inputRow interface{}) ([]interface{}, error) {
prLabel := inputRow.(*models.GiteePullRequestLabel)
domainPrLabel := &code.PullRequestLabel{
- PullRequestId: prIdGen.Generate(prLabel.PullId),
+ PullRequestId: prIdGen.Generate(data.Options.ConnectionId, prLabel.PullId),
LabelName: prLabel.LabelName,
}
return []interface{}{
diff --git a/plugins/gitee/tasks/pr_review_collector.go b/plugins/gitee/tasks/pr_review_collector.go
index bfc21cec..88be6d02 100644
--- a/plugins/gitee/tasks/pr_review_collector.go
+++ b/plugins/gitee/tasks/pr_review_collector.go
@@ -24,6 +24,8 @@ import (
"net/url"
"reflect"
+ "github.com/apache/incubator-devlake/plugins/core/dal"
+
"github.com/apache/incubator-devlake/plugins/helper"
"github.com/apache/incubator-devlake/plugins/core"
@@ -41,18 +43,21 @@ var CollectApiPullRequestReviewsMeta = core.SubTaskMeta{
}
func CollectApiPullRequestReviews(taskCtx core.SubTaskContext) error {
- db := taskCtx.GetDb()
+ db := taskCtx.GetDal()
rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_PULL_REQUEST_REVIEW_TABLE)
incremental := false
- cursor, err := db.Model(&models.GiteePullRequest{}).Select("number, gitee_id").
- Where("repo_id = ?", data.Repo.GiteeId).
- Rows()
+ cursor, err := db.Cursor(
+ dal.Select("number, gitee_id"),
+ dal.From(models.GiteePullRequest{}.TableName()),
+ dal.Where("repo_id = ? and connection_id=?", data.Repo.GiteeId, data.Options.ConnectionId),
+ )
+
if err != nil {
return err
}
- iterator, err := helper.NewCursorIterator(db, cursor, reflect.TypeOf(SimplePr{}))
+ iterator, err := helper.NewDalCursorIterator(db, cursor, reflect.TypeOf(SimplePr{}))
if err != nil {
return err
}
diff --git a/plugins/gitee/tasks/pr_review_extractor.go b/plugins/gitee/tasks/pr_review_extractor.go
index 5fa79c38..8b863a20 100644
--- a/plugins/gitee/tasks/pr_review_extractor.go
+++ b/plugins/gitee/tasks/pr_review_extractor.go
@@ -46,7 +46,7 @@ type PullRequestReview struct {
}
func ExtractApiPullRequestReviews(taskCtx core.SubTaskContext) error {
- rawDataSubTaskArgs, _ := CreateRawDataSubTaskArgs(taskCtx, RAW_PULL_REQUEST_REVIEW_TABLE)
+ rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_PULL_REQUEST_REVIEW_TABLE)
extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{
RawDataSubTaskArgs: *rawDataSubTaskArgs,
Extract: func(row *helper.RawData) ([]interface{}, error) {
@@ -66,6 +66,7 @@ func ExtractApiPullRequestReviews(taskCtx core.SubTaskContext) error {
results := make([]interface{}, 0, 1)
giteeReviewer := &models.GiteeReviewer{
+ ConnectionId: data.Options.ConnectionId,
GiteeId: apiPullRequestReview.User.Id,
Login: apiPullRequestReview.User.Login,
PullRequestId: pull.GiteeId,
diff --git a/plugins/gitee/tasks/repo_collector.go b/plugins/gitee/tasks/repo_collector.go
index ad2670c1..4fcc4af0 100644
--- a/plugins/gitee/tasks/repo_collector.go
+++ b/plugins/gitee/tasks/repo_collector.go
@@ -29,7 +29,7 @@ import (
"github.com/apache/incubator-devlake/plugins/core"
)
-const RAW_REPOSITORIES_TABLE = "gitee_api_repo"
+const RAW_REPOSITORIES_TABLE = "gitee_api_repos"
var CollectApiRepoMeta = core.SubTaskMeta{
Name: "collectApiRepo",
diff --git a/plugins/gitee/tasks/repo_convertor.go b/plugins/gitee/tasks/repo_convertor.go
index 21cc8b82..a47ec9fe 100644
--- a/plugins/gitee/tasks/repo_convertor.go
+++ b/plugins/gitee/tasks/repo_convertor.go
@@ -21,6 +21,8 @@ import (
"fmt"
"reflect"
+ "github.com/apache/incubator-devlake/plugins/core/dal"
+
"github.com/apache/incubator-devlake/models/domainlayer"
"github.com/apache/incubator-devlake/models/domainlayer/code"
"github.com/apache/incubator-devlake/models/domainlayer/didgen"
@@ -39,12 +41,13 @@ var ConvertRepoMeta = core.SubTaskMeta{
func ConvertRepo(taskCtx core.SubTaskContext) error {
rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_REPOSITORIES_TABLE)
- db := taskCtx.GetDb()
+ db := taskCtx.GetDal()
repoId := data.Repo.GiteeId
- cursor, err := db.Model(&models.GiteeRepo{}).
- Where("gitee_id = ?", repoId).
- Rows()
+ cursor, err := db.Cursor(
+ dal.From(&models.GiteeRepo{}),
+ dal.Where("gitee_id = ?", repoId),
+ )
if err != nil {
return err
}
@@ -60,7 +63,7 @@ func ConvertRepo(taskCtx core.SubTaskContext) error {
repository := inputRow.(*models.GiteeRepo)
domainRepository := &code.Repo{
DomainEntity: domainlayer.DomainEntity{
- Id: repoIdGen.Generate(repository.GiteeId),
+ Id: repoIdGen.Generate(data.Options.ConnectionId, repository.GiteeId),
},
Name: fmt.Sprintf("%s/%s", repository.OwnerLogin, repository.Name),
Url: repository.HTMLUrl,
@@ -73,7 +76,7 @@ func ConvertRepo(taskCtx core.SubTaskContext) error {
domainBoard := &ticket.Board{
DomainEntity: domainlayer.DomainEntity{
- Id: repoIdGen.Generate(repository.GiteeId),
+ Id: repoIdGen.Generate(data.Options.ConnectionId, repository.GiteeId),
},
Name: fmt.Sprintf("%s/%s", repository.OwnerLogin, repository.Name),
Url: fmt.Sprintf("%s/%s", repository.HTMLUrl, "issues"),
diff --git a/plugins/gitee/tasks/repo_extractor.go b/plugins/gitee/tasks/repo_extractor.go
index 02ce4a7c..68f95ecc 100644
--- a/plugins/gitee/tasks/repo_extractor.go
+++ b/plugins/gitee/tasks/repo_extractor.go
@@ -60,15 +60,16 @@ func ExtractApiRepositories(taskCtx core.SubTaskContext) error {
}
results := make([]interface{}, 0, 1)
giteeRepository := &models.GiteeRepo{
- GiteeId: repo.GiteeId,
- Name: repo.Name,
- HTMLUrl: repo.HTMLUrl,
- Description: repo.Description,
- OwnerId: repo.Owner.Id,
- OwnerLogin: repo.Owner.Login,
- Language: repo.Language,
- CreatedDate: repo.CreatedAt.ToTime(),
- UpdatedDate: helper.Iso8601TimeToTime(repo.UpdatedAt),
+ ConnectionId: data.Options.ConnectionId,
+ GiteeId: repo.GiteeId,
+ Name: repo.Name,
+ HTMLUrl: repo.HTMLUrl,
+ Description: repo.Description,
+ OwnerId: repo.Owner.Id,
+ OwnerLogin: repo.Owner.Login,
+ Language: repo.Language,
+ CreatedDate: repo.CreatedAt.ToTime(),
+ UpdatedDate: helper.Iso8601TimeToTime(repo.UpdatedAt),
}
data.Repo = giteeRepository
diff --git a/plugins/gitee/tasks/shared.go b/plugins/gitee/tasks/shared.go
index 76b2de18..2e704ccc 100644
--- a/plugins/gitee/tasks/shared.go
+++ b/plugins/gitee/tasks/shared.go
@@ -46,8 +46,9 @@ type RateLimitInfo struct {
}
type GiteeApiParams struct {
- Repo string
- Owner string
+ ConnectionId uint64
+ Repo string
+ Owner string
}
type GiteeInput struct {
@@ -93,8 +94,9 @@ func CreateRawDataSubTaskArgs(taskCtx core.SubTaskContext, Table string) (*helpe
RawDataSubTaskArgs := &helper.RawDataSubTaskArgs{
Ctx: taskCtx,
Params: GiteeApiParams{
- Repo: data.Options.Repo,
- Owner: data.Options.Owner,
+ ConnectionId: data.Options.ConnectionId,
+ Repo: data.Options.Repo,
+ Owner: data.Options.Owner,
},
Table: Table,
}
diff --git a/plugins/gitee/tasks/task_data.go b/plugins/gitee/tasks/task_data.go
index c990378b..b8faca57 100644
--- a/plugins/gitee/tasks/task_data.go
+++ b/plugins/gitee/tasks/task_data.go
@@ -25,13 +25,13 @@ import (
)
type GiteeOptions struct {
- ConnectionId uint64 `json:"connectionId"`
- Tasks []string `json:"tasks,omitempty"`
- Since string
- Owner string
- Repo string
- Token string
- models.Config `mapstructure:",squash"`
+ ConnectionId uint64 `json:"connectionId"`
+ Tasks []string `json:"tasks,omitempty"`
+ Since string
+ Owner string
+ Repo string
+ Token string
+ models.TransformationRules `mapstructure:"transformationRules" json:"transformationRules"`
}
type GiteeTaskData struct {
diff --git a/plugins/gitee/tasks/user_convertor.go b/plugins/gitee/tasks/user_convertor.go
index 4c149f4e..dcbb732b 100644
--- a/plugins/gitee/tasks/user_convertor.go
+++ b/plugins/gitee/tasks/user_convertor.go
@@ -20,6 +20,8 @@ package tasks
import (
"reflect"
+ "github.com/apache/incubator-devlake/plugins/core/dal"
+
"github.com/apache/incubator-devlake/models/domainlayer"
"github.com/apache/incubator-devlake/models/domainlayer/didgen"
"github.com/apache/incubator-devlake/models/domainlayer/user"
@@ -36,11 +38,10 @@ var ConvertUsersMeta = core.SubTaskMeta{
}
func ConvertUsers(taskCtx core.SubTaskContext) error {
- db := taskCtx.GetDb()
- rawDataSubTaskArgs, _ := CreateRawDataSubTaskArgs(taskCtx, RAW_COMMIT_TABLE)
+ db := taskCtx.GetDal()
+ rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_COMMIT_TABLE)
- cursor, err := db.Model(&models.GiteeUser{}).
- Rows()
+ cursor, err := db.Cursor(dal.From(&models.GiteeUser{}))
if err != nil {
return err
}
@@ -55,7 +56,7 @@ func ConvertUsers(taskCtx core.SubTaskContext) error {
Convert: func(inputRow interface{}) ([]interface{}, error) {
GiteeUser := inputRow.(*models.GiteeUser)
domainUser := &user.User{
- DomainEntity: domainlayer.DomainEntity{Id: userIdGen.Generate(GiteeUser.Id)},
+ DomainEntity: domainlayer.DomainEntity{Id: userIdGen.Generate(data.Options.ConnectionId, GiteeUser.Id)},
Name: GiteeUser.Login,
AvatarUrl: GiteeUser.AvatarUrl,
}