You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by wa...@apache.org on 2022/06/18 09:24:18 UTC
[incubator-devlake] branch main updated (2cbc097e -> 5f0804ba)
This is an automated email from the ASF dual-hosted git repository.
warren pushed a change to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
from 2cbc097e feat: multi-data connections support for Jenkins and Gitlab (#2239)
new 9f239e8a init commite gitee plugin
new 03f5a14d update gitee plugin
new a86c5596 add asf license and env.example
new d8394e32 update impl.go
new f1431a70 update commit and pr review code
new 495420cf update api connection.go
new 8b4c27db multi-data connections support for gitee
new 0313c121 add default gitee endpoint
new 5d57ed67 remove gitee unused configuration information
new 5f0804ba update gitee main
The 10 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.env.example | 8 +
config/config.go | 1 +
plugins/gitee/README.md | 117 ++++++++++++++
plugins/{gitlab => gitee}/api/connection.go | 72 ++++-----
plugins/{github => gitee}/api/init.go | 0
plugins/{github/github.go => gitee/gitee.go} | 16 +-
plugins/{github => gitee}/impl/impl.go | 88 ++++++-----
plugins/{gitlab => gitee}/models/commit.go | 12 +-
plugins/{github => gitee}/models/commit_stats.go | 16 +-
plugins/{gitlab => gitee}/models/connection.go | 41 +++--
plugins/{github => gitee}/models/issue.go | 18 +--
.../project.go => gitee/models/issue_comment.go} | 18 ++-
.../story_label.go => gitee/models/issue_label.go} | 11 +-
.../models/migrationscripts/archived}/commit.go | 14 +-
.../migrationscripts/archived/commit_stat.go | 13 +-
.../models/migrationscripts/archived/connection.go | 25 ++-
.../models/migrationscripts/archived/issue.go | 15 +-
.../migrationscripts/archived/issue_comment.go} | 18 ++-
.../migrationscripts/archived/issue_label.go | 13 +-
.../migrationscripts/archived/pull_request.go | 25 ++-
.../archived/pull_request_comment.go} | 18 ++-
.../archived/pull_request_commit.go | 6 +-
.../archived/pull_request_issue.go | 6 +-
.../archived/pull_request_label.go} | 10 +-
.../models/migrationscripts/archived/repo.go} | 22 ++-
.../migrationscripts/archived/repo_commit.go | 10 +-
.../models/migrationscripts/archived/reviewer.go | 8 +-
.../gitee/models/migrationscripts/archived/user.go | 47 ++++++
.../gitee/models/migrationscripts/init_schema.go | 126 +++++++++++++++
.../models/pr.go => gitee/models/pull_request.go} | 28 ++--
.../models/pull_request_comment.go} | 21 +--
.../models/pull_request_commit.go} | 6 +-
.../models/pull_request_issue.go} | 8 +-
.../models/pull_request_label.go} | 8 +-
.../{ae/models/project.go => gitee/models/repo.go} | 22 ++-
plugins/{github => gitee}/models/repo_commit.go | 10 +-
plugins/{github => gitee}/models/reviewer.go | 8 +-
plugins/gitee/models/user.go | 45 ++++++
plugins/{gitlab => gitee}/tasks/api_client.go | 24 +--
.../tasks/commit_collector.go} | 32 ++--
.../{gitlab => gitee}/tasks/commit_convertor.go | 66 ++++----
plugins/gitee/tasks/commit_extractor.go | 126 +++++++++++++++
.../tasks/commit_stats_collector.go | 51 +++---
.../tasks/commit_stats_extractor.go | 28 +---
.../tasks/issue_collector.go} | 56 +++----
plugins/gitee/tasks/issue_comment_collector.go | 109 +++++++++++++
.../tasks/issue_comment_convertor.go | 46 +++---
.../tasks/issue_comment_extractor.go} | 93 ++++++-----
plugins/{github => gitee}/tasks/issue_convertor.go | 52 +++----
plugins/{github => gitee}/tasks/issue_extractor.go | 171 ++++++++++++---------
.../tasks/issue_label_convertor.go | 42 ++---
plugins/{github => gitee}/tasks/pr_collector.go | 53 ++-----
.../tasks/pr_comment_convertor.go | 46 +++---
.../tasks/pr_commit_collector.go} | 54 +++----
.../{github => gitee}/tasks/pr_commit_convertor.go | 35 ++---
.../{github => gitee}/tasks/pr_commit_extractor.go | 56 ++++---
plugins/{github => gitee}/tasks/pr_convertor.go | 44 ++----
plugins/gitee/tasks/pr_extractor.go | 171 +++++++++++++++++++++
.../{github => gitee}/tasks/pr_issue_convertor.go | 43 +++---
.../{github => gitee}/tasks/pr_issue_enricher.go | 48 +++---
.../{github => gitee}/tasks/pr_label_convertor.go | 42 ++---
.../{github => gitee}/tasks/pr_review_collector.go | 46 ++----
.../{github => gitee}/tasks/pr_review_extractor.go | 42 ++---
plugins/{github => gitee}/tasks/repo_collector.go | 22 +--
plugins/{github => gitee}/tasks/repo_convertor.go | 40 ++---
plugins/gitee/tasks/repo_extractor.go | 90 +++++++++++
.../utils/utils.go => gitee/tasks/shared.go} | 74 +++++++--
plugins/{github => gitee}/tasks/task_data.go | 11 +-
plugins/{github => gitee}/tasks/user_convertor.go | 31 ++--
69 files changed, 1778 insertions(+), 1016 deletions(-)
create mode 100644 plugins/gitee/README.md
copy plugins/{gitlab => gitee}/api/connection.go (67%)
copy plugins/{github => gitee}/api/init.go (100%)
copy plugins/{github/github.go => gitee/gitee.go} (86%)
copy plugins/{github => gitee}/impl/impl.go (64%)
copy plugins/{gitlab => gitee}/models/commit.go (87%)
copy plugins/{github => gitee}/models/commit_stats.go (75%)
copy plugins/{gitlab => gitee}/models/connection.go (64%)
copy plugins/{github => gitee}/models/issue.go (83%)
copy plugins/{ae/models/project.go => gitee/models/issue_comment.go} (69%)
copy plugins/{tapd/models/story_label.go => gitee/models/issue_label.go} (76%)
copy plugins/{gitlab/models => gitee/models/migrationscripts/archived}/commit.go (86%)
copy plugins/{github => gitee}/models/migrationscripts/archived/commit_stat.go (75%)
copy plugins/{gitlab => gitee}/models/migrationscripts/archived/connection.go (75%)
copy plugins/{github => gitee}/models/migrationscripts/archived/issue.go (83%)
copy plugins/{ae/models/migrationscripts/archived/project.go => gitee/models/migrationscripts/archived/issue_comment.go} (70%)
copy plugins/{github => gitee}/models/migrationscripts/archived/issue_label.go (74%)
copy plugins/{github => gitee}/models/migrationscripts/archived/pull_request.go (72%)
copy plugins/{ae/models/migrationscripts/archived/project.go => gitee/models/migrationscripts/archived/pull_request_comment.go} (70%)
copy plugins/{github => gitee}/models/migrationscripts/archived/pull_request_commit.go (88%)
copy plugins/{github => gitee}/models/migrationscripts/archived/pull_request_issue.go (88%)
copy plugins/{github/models/migrationscripts/archived/repo_commit.go => gitee/models/migrationscripts/archived/pull_request_label.go} (78%)
copy plugins/{github/models/migrationscripts/archived/issue_event.go => gitee/models/migrationscripts/archived/repo.go} (59%)
copy plugins/{github => gitee}/models/migrationscripts/archived/repo_commit.go (82%)
copy plugins/{github => gitee}/models/migrationscripts/archived/reviewer.go (86%)
create mode 100644 plugins/gitee/models/migrationscripts/archived/user.go
create mode 100644 plugins/gitee/models/migrationscripts/init_schema.go
copy plugins/{github/models/pr.go => gitee/models/pull_request.go} (72%)
copy plugins/{github/models/commit_stats.go => gitee/models/pull_request_comment.go} (70%)
copy plugins/{github/models/pr_commit.go => gitee/models/pull_request_commit.go} (88%)
copy plugins/{github/models/pr_issue.go => gitee/models/pull_request_issue.go} (86%)
copy plugins/{tapd/models/story_label.go => gitee/models/pull_request_label.go} (83%)
copy plugins/{ae/models/project.go => gitee/models/repo.go} (58%)
copy plugins/{github => gitee}/models/repo_commit.go (83%)
copy plugins/{github => gitee}/models/reviewer.go (86%)
create mode 100644 plugins/gitee/models/user.go
copy plugins/{gitlab => gitee}/tasks/api_client.go (75%)
copy plugins/{gitlab/tasks/mr_collector.go => gitee/tasks/commit_collector.go} (62%)
copy plugins/{gitlab => gitee}/tasks/commit_convertor.go (54%)
create mode 100644 plugins/gitee/tasks/commit_extractor.go
copy plugins/{github => gitee}/tasks/commit_stats_collector.go (66%)
copy plugins/{github => gitee}/tasks/commit_stats_extractor.go (78%)
copy plugins/{github/tasks/event_collector.go => gitee/tasks/issue_collector.go} (57%)
create mode 100644 plugins/gitee/tasks/issue_comment_collector.go
copy plugins/{github => gitee}/tasks/issue_comment_convertor.go (60%)
copy plugins/{github/tasks/comment_extractor.go => gitee/tasks/issue_comment_extractor.go} (52%)
copy plugins/{github => gitee}/tasks/issue_convertor.go (64%)
copy plugins/{github => gitee}/tasks/issue_extractor.go (52%)
copy plugins/{gitlab => gitee}/tasks/issue_label_convertor.go (61%)
copy plugins/{github => gitee}/tasks/pr_collector.go (62%)
copy plugins/{github => gitee}/tasks/pr_comment_convertor.go (59%)
copy plugins/{github/tasks/pr_review_collector.go => gitee/tasks/pr_commit_collector.go} (60%)
copy plugins/{github => gitee}/tasks/pr_commit_convertor.go (60%)
copy plugins/{github => gitee}/tasks/pr_commit_extractor.go (73%)
copy plugins/{github => gitee}/tasks/pr_convertor.go (65%)
create mode 100644 plugins/gitee/tasks/pr_extractor.go
copy plugins/{github => gitee}/tasks/pr_issue_convertor.go (57%)
copy plugins/{github => gitee}/tasks/pr_issue_enricher.go (72%)
copy plugins/{github => gitee}/tasks/pr_label_convertor.go (56%)
copy plugins/{github => gitee}/tasks/pr_review_collector.go (65%)
copy plugins/{github => gitee}/tasks/pr_review_extractor.go (67%)
copy plugins/{github => gitee}/tasks/repo_collector.go (77%)
copy plugins/{github => gitee}/tasks/repo_convertor.go (70%)
create mode 100644 plugins/gitee/tasks/repo_extractor.go
copy plugins/{github/utils/utils.go => gitee/tasks/shared.go} (72%)
copy plugins/{github => gitee}/tasks/task_data.go (85%)
copy plugins/{github => gitee}/tasks/user_convertor.go (69%)
[incubator-devlake] 04/10: update impl.go
Posted by wa...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
warren pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit d8394e32eb9789b2a1370eeba6a0d10568f5cad2
Author: Mr.An <42...@users.noreply.github.com>
AuthorDate: Wed Jun 15 12:32:03 2022 +0800
update impl.go
---
plugins/gitee/impl/impl.go | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/plugins/gitee/impl/impl.go b/plugins/gitee/impl/impl.go
index c1e4d799..9cde8eb3 100644
--- a/plugins/gitee/impl/impl.go
+++ b/plugins/gitee/impl/impl.go
@@ -58,8 +58,8 @@ func (plugin Gitee) SubTaskMetas() []core.SubTaskMeta {
tasks.ExtractApiIssueCommentsMeta,
tasks.CollectApiPullRequestCommitsMeta,
tasks.ExtractApiPullRequestCommitsMeta,
- //tasks.CollectApiPullRequestReviewsMeta,
- //tasks.ExtractApiPullRequestReviewsMeta,
+ tasks.CollectApiPullRequestReviewsMeta,
+ tasks.ExtractApiPullRequestReviewsMeta,
tasks.CollectApiCommitStatsMeta,
tasks.ExtractApiCommitStatsMeta,
tasks.EnrichPullRequestIssuesMeta,
[incubator-devlake] 07/10: multi-data connections support for gitee
Posted by wa...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
warren pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit 8b4c27db61d90c2d5e5791b5078c80a0f2f476c8
Author: Mr.An <42...@users.noreply.github.com>
AuthorDate: Sat Jun 18 16:58:39 2022 +0800
multi-data connections support for gitee
---
plugins/gitee/README.md | 8 +--
plugins/gitee/gitee.go | 44 ++++++++----
plugins/gitee/impl/impl.go | 79 ++++++++++++++++++++--
plugins/gitee/models/connection.go | 4 ++
.../{ => migrationscripts/archived}/connection.go | 40 ++++++++---
.../gitee/models/migrationscripts/init_schema.go | 76 ++++++++++++++++++++-
plugins/gitee/tasks/api_client.go | 30 ++++----
plugins/gitee/tasks/commit_collector.go | 1 -
plugins/gitee/tasks/commit_stats_collector.go | 1 -
plugins/gitee/tasks/issue_collector.go | 1 -
plugins/gitee/tasks/issue_comment_collector.go | 1 -
plugins/gitee/tasks/issue_comment_extractor.go | 16 ++++-
plugins/gitee/tasks/pr_collector.go | 1 -
plugins/gitee/tasks/pr_commit_collector.go | 1 -
plugins/gitee/tasks/pr_review_collector.go | 1 -
plugins/gitee/tasks/repo_collector.go | 1 -
plugins/gitee/tasks/shared.go | 12 ----
plugins/gitee/tasks/task_data.go | 13 ++--
18 files changed, 247 insertions(+), 83 deletions(-)
diff --git a/plugins/gitee/README.md b/plugins/gitee/README.md
index ce1bc6b6..cfb2b925 100644
--- a/plugins/gitee/README.md
+++ b/plugins/gitee/README.md
@@ -57,8 +57,7 @@ In order to collect data, you have to compose a JSON looks like following one, a
"plugin": "gitee",
"options": {
"repo": "lake",
- "owner": "merico-dev",
- "token": "xxxx"
+ "owner": "merico-dev"
}
}
]
@@ -73,8 +72,7 @@ and if you want to perform certain subtasks.
"subtasks": ["collectXXX", "extractXXX", "convertXXX"],
"options": {
"repo": "lake",
- "owner": "merico-dev",
- "token": "xxxx"
+ "owner": "merico-dev"
}
}
]
@@ -94,7 +92,6 @@ curl --location --request POST 'localhost:8080/pipelines' \
"options": {
"repo": "lake",
"owner": "merico-dev"
- "token": "xxxx"
}
}]]
}
@@ -113,7 +110,6 @@ curl --location --request POST 'localhost:8080/pipelines' \
"options": {
"repo": "lake",
"owner": "merico-dev"
- "token": "xxxx"
}
}]]
}
diff --git a/plugins/gitee/gitee.go b/plugins/gitee/gitee.go
index c6f19cf9..7c239996 100644
--- a/plugins/gitee/gitee.go
+++ b/plugins/gitee/gitee.go
@@ -26,19 +26,39 @@ import (
var PluginEntry impl.Gitee //nolint
func main() {
- giteeCmd := &cobra.Command{Use: "gitee"}
- owner := giteeCmd.Flags().StringP("owner", "o", "", "gitee owner")
- repo := giteeCmd.Flags().StringP("repo", "r", "", "gitee repo")
- token := giteeCmd.Flags().StringP("auth", "a", "", "access token")
- _ = giteeCmd.MarkFlagRequired("owner")
- _ = giteeCmd.MarkFlagRequired("repo")
-
- giteeCmd.Run = func(cmd *cobra.Command, args []string) {
+ cmd := &cobra.Command{Use: "github"}
+ connectionId := cmd.Flags().Uint64P("connectionId", "c", 0, "gitee connection id")
+ owner := cmd.Flags().StringP("owner", "o", "", "gitee owner")
+ repo := cmd.Flags().StringP("repo", "r", "", "gitee repo")
+ _ = cmd.MarkFlagRequired("connectionId")
+ _ = cmd.MarkFlagRequired("owner")
+ _ = cmd.MarkFlagRequired("repo")
+
+ prType := cmd.Flags().String("prType", "type/(.*)$", "pr type")
+ prComponent := cmd.Flags().String("prComponent", "component/(.*)$", "pr component")
+ prBodyClosePattern := cmd.Flags().String("prBodyClosePattern", "(?mi)(fix|close|resolve|fixes|closes|resolves|fixed|closed|resolved)[\\s]*.*(((and )?(#|https:\\/\\/github.com\\/%s\\/%s\\/issues\\/)\\d+[ ]*)+)", "pr body close pattern")
+ issueSeverity := cmd.Flags().String("issueSeverity", "severity/(.*)$", "issue severity")
+ issuePriority := cmd.Flags().String("issuePriority", "^(highest|high|medium|low)$", "issue priority")
+ issueComponent := cmd.Flags().String("issueComponent", "component/(.*)$", "issue component")
+ issueTypeBug := cmd.Flags().String("issueTypeBug", "^(bug|failure|error)$", "issue type bug")
+ issueTypeIncident := cmd.Flags().String("issueTypeIncident", "", "issue type incident")
+ issueTypeRequirement := cmd.Flags().String("issueTypeRequirement", "^(feat|feature|proposal|requirement)$", "issue type requirement")
+
+ cmd.Run = func(cmd *cobra.Command, args []string) {
runner.DirectRun(cmd, args, PluginEntry, map[string]interface{}{
- "owner": *owner,
- "repo": *repo,
- "token": *token,
+ "connectionId": *connectionId,
+ "owner": *owner,
+ "repo": *repo,
+ "prType": *prType,
+ "prComponent": *prComponent,
+ "prBodyClosePattern": *prBodyClosePattern,
+ "issueSeverity": *issueSeverity,
+ "issuePriority": *issuePriority,
+ "issueComponent": *issueComponent,
+ "issueTypeBug": *issueTypeBug,
+ "issueTypeIncident": *issueTypeIncident,
+ "issueTypeRequirement": *issueTypeRequirement,
})
}
- runner.RunCmd(giteeCmd)
+ runner.RunCmd(cmd)
}
diff --git a/plugins/gitee/impl/impl.go b/plugins/gitee/impl/impl.go
index 9cde8eb3..c7589f66 100644
--- a/plugins/gitee/impl/impl.go
+++ b/plugins/gitee/impl/impl.go
@@ -18,11 +18,15 @@ limitations under the License.
package impl
import (
+ "fmt"
+
"github.com/apache/incubator-devlake/migration"
"github.com/apache/incubator-devlake/plugins/core"
"github.com/apache/incubator-devlake/plugins/gitee/api"
+ "github.com/apache/incubator-devlake/plugins/gitee/models"
"github.com/apache/incubator-devlake/plugins/gitee/models/migrationscripts"
"github.com/apache/incubator-devlake/plugins/gitee/tasks"
+ "github.com/apache/incubator-devlake/plugins/helper"
"github.com/mitchellh/mapstructure"
"github.com/spf13/viper"
"gorm.io/gorm"
@@ -37,6 +41,7 @@ var _ core.Migratable = (*Gitee)(nil)
type Gitee string
func (plugin Gitee) Init(config *viper.Viper, logger core.Logger, db *gorm.DB) error {
+ api.Init(config, logger, db)
return nil
}
@@ -86,7 +91,67 @@ func (plugin Gitee) PrepareTaskData(taskCtx core.TaskContext, options map[string
return nil, err
}
- apiClient, err := tasks.NewGiteeApiClient(taskCtx)
+ if op.Owner == "" {
+ return nil, fmt.Errorf("owner is required for Gitee execution")
+ }
+
+ if op.Repo == "" {
+ return nil, fmt.Errorf("repo is required for Gitee execution")
+ }
+
+ if op.PrType == "" {
+ op.PrType = "type/(.*)$"
+ }
+
+ if op.PrComponent == "" {
+ op.PrComponent = "component/(.*)$"
+ }
+
+ if op.IssueSeverity == "" {
+ op.IssueSeverity = "severity/(.*)$"
+ }
+
+ if op.IssuePriority == "" {
+ op.IssuePriority = "^(highest|high|medium|low)$"
+ }
+
+ if op.IssueComponent == "" {
+ op.IssueComponent = "component/(.*)$"
+ }
+
+ if op.IssueTypeBug == "" {
+ op.IssueTypeBug = "^(bug|failure|error)$"
+ }
+
+ if op.IssueTypeIncident == "" {
+ op.IssueTypeIncident = ""
+ }
+
+ if op.IssueTypeRequirement == "" {
+ op.IssueTypeRequirement = "^(feat|feature|proposal|requirement)$"
+ }
+
+ if op.ConnectionId == 0 {
+ return nil, fmt.Errorf("connectionId is invalid")
+ }
+
+ connection := &models.GiteeConnection{}
+ connectionHelper := helper.NewConnectionHelper(
+ taskCtx,
+ nil,
+ )
+
+ if err != nil {
+ return nil, err
+ }
+
+ err = connectionHelper.FirstById(connection, op.ConnectionId)
+
+ if err != nil {
+ return nil, err
+ }
+ apiClient, err := tasks.NewGiteeApiClient(taskCtx, connection)
+
if err != nil {
return nil, err
}
@@ -102,7 +167,9 @@ func (plugin Gitee) RootPkgPath() string {
}
func (plugin Gitee) MigrationScripts() []migration.Script {
- return []migration.Script{new(migrationscripts.InitSchemas), new(migrationscripts.InitSchemas)}
+ return []migration.Script{
+ new(migrationscripts.InitSchemas),
+ }
}
func (plugin Gitee) ApiResources() map[string]map[string]core.ApiResourceHandler {
@@ -111,11 +178,13 @@ func (plugin Gitee) ApiResources() map[string]map[string]core.ApiResourceHandler
"POST": api.TestConnection,
},
"connections": {
- "GET": api.ListConnections,
+ "POST": api.PostConnections,
+ "GET": api.ListConnections,
},
"connections/:connectionId": {
- "GET": api.GetConnection,
- "PATCH": api.PatchConnection,
+ "GET": api.GetConnection,
+ "PATCH": api.PatchConnection,
+ "DELETE": api.DeleteConnection,
},
}
}
diff --git a/plugins/gitee/models/connection.go b/plugins/gitee/models/connection.go
index f161df73..a5360449 100644
--- a/plugins/gitee/models/connection.go
+++ b/plugins/gitee/models/connection.go
@@ -51,3 +51,7 @@ type Config struct {
IssueTypeIncident string `mapstructure:"issueTypeIncident" env:"GITEE_ISSUE_TYPE_INCIDENT" json:"issueTypeIncident"`
IssueTypeRequirement string `mapstructure:"issueTypeRequirement" env:"GITEE_ISSUE_TYPE_REQUIREMENT" json:"issueTypeRequirement"`
}
+
+func (GiteeConnection) TableName() string {
+ return "_tool_gitee_connections"
+}
diff --git a/plugins/gitee/models/connection.go b/plugins/gitee/models/migrationscripts/archived/connection.go
similarity index 66%
copy from plugins/gitee/models/connection.go
copy to plugins/gitee/models/migrationscripts/archived/connection.go
index f161df73..c1fc5daf 100644
--- a/plugins/gitee/models/connection.go
+++ b/plugins/gitee/models/migrationscripts/archived/connection.go
@@ -15,32 +15,46 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
-package models
+package archived
-import "github.com/apache/incubator-devlake/plugins/helper"
+import (
+ "github.com/apache/incubator-devlake/models/migrationscripts/archived"
+)
type GiteeConnection struct {
- helper.RestConnection `mapstructure:",squash"`
- helper.AccessToken `mapstructure:",squash"`
+ RestConnection `mapstructure:",squash"`
+ AccessToken `mapstructure:",squash"`
}
-type GiteeResponse struct {
+type RestConnection struct {
+ BaseConnection `mapstructure:",squash"`
+ Endpoint string `mapstructure:"endpoint" validate:"required" json:"endpoint"`
+ Proxy string `mapstructure:"proxy" json:"proxy"`
+ RateLimit int `comment:"api request rate limt per hour" json:"rateLimit"`
+}
+
+type BaseConnection struct {
+ Name string `gorm:"type:varchar(100);uniqueIndex" json:"name" validate:"required"`
+ archived.Model
+}
+
+type AccessToken struct {
+ Token string `mapstructure:"token" validate:"required" json:"token" encrypt:"yes"`
+}
+
+// This object conforms to what the frontend currently expects.
+type GitlabResponse struct {
Name string `json:"name"`
ID int `json:"id"`
GiteeConnection
}
+// Using User because it requires authentication.
type ApiUserResponse struct {
Id int
Name string `json:"name"`
}
-type TestConnectionRequest struct {
- Endpoint string `json:"endpoint" validate:"required"`
- Proxy string `json:"proxy"`
- helper.AccessToken `mapstructure:",squash"`
-}
-
type Config struct {
PrType string `mapstructure:"prType" env:"GITEE_PR_TYPE" json:"prType"`
PrComponent string `mapstructure:"prComponent" env:"GITEE_PR_COMPONENT" json:"prComponent"`
@@ -51,3 +65,7 @@ type Config struct {
IssueTypeIncident string `mapstructure:"issueTypeIncident" env:"GITEE_ISSUE_TYPE_INCIDENT" json:"issueTypeIncident"`
IssueTypeRequirement string `mapstructure:"issueTypeRequirement" env:"GITEE_ISSUE_TYPE_REQUIREMENT" json:"issueTypeRequirement"`
}
+
+func (GiteeConnection) TableName() string {
+ return "_tool_gitee_connections"
+}
diff --git a/plugins/gitee/models/migrationscripts/init_schema.go b/plugins/gitee/models/migrationscripts/init_schema.go
index 44e3e33a..305d4702 100644
--- a/plugins/gitee/models/migrationscripts/init_schema.go
+++ b/plugins/gitee/models/migrationscripts/init_schema.go
@@ -19,6 +19,12 @@ package migrationscripts
import (
"context"
+ "fmt"
+
+ "github.com/apache/incubator-devlake/config"
+ "gorm.io/gorm/clause"
+
+ "github.com/apache/incubator-devlake/plugins/core"
"github.com/apache/incubator-devlake/plugins/gitee/models/migrationscripts/archived"
"gorm.io/gorm"
@@ -27,7 +33,25 @@ import (
type InitSchemas struct{}
func (*InitSchemas) Up(ctx context.Context, db *gorm.DB) error {
- return db.Migrator().AutoMigrate(
+ rawTableList := []string{
+ "_raw_gitee_api_commit",
+ "_raw_gitee_api_issues",
+ "_raw_gitee_api_pull_requests",
+ "_raw_gitee_api_pull_request_commits",
+ "_raw_gitee_api_pull_request_reviews",
+ "_raw_gitee_api_repo",
+ "_raw_gitee_api_comments",
+ "_raw_gitee_api_commits",
+ "_raw_gitee_issue_comments",
+ }
+ for _, v := range rawTableList {
+ err := db.Exec(fmt.Sprintf("DROP TABLE IF EXISTS %s CASCADE", v)).Error
+ if err != nil {
+ return err
+ }
+ }
+
+ err := db.Migrator().DropTable(
&archived.GiteeRepo{},
&archived.GiteeCommit{},
&archived.GiteeRepoCommit{},
@@ -42,11 +66,59 @@ func (*InitSchemas) Up(ctx context.Context, db *gorm.DB) error {
&archived.GiteePullRequestCommit{},
&archived.GiteePullRequestIssue{},
&archived.GiteeReviewer{},
+ &archived.GiteeConnection{},
)
+
+ if err != nil {
+ return err
+ }
+
+ err = db.Migrator().AutoMigrate(
+ &archived.GiteeRepo{},
+ &archived.GiteeCommit{},
+ &archived.GiteeRepoCommit{},
+ &archived.GiteePullRequest{},
+ &archived.GiteePullRequestLabel{},
+ &archived.GiteeUser{},
+ &archived.GiteePullRequestComment{},
+ &archived.GiteeIssue{},
+ &archived.GiteeIssueComment{},
+ &archived.GiteeCommitStat{},
+ &archived.GiteeIssueLabel{},
+ &archived.GiteePullRequestCommit{},
+ &archived.GiteePullRequestIssue{},
+ &archived.GiteeReviewer{},
+ &archived.GiteeConnection{},
+ )
+
+ if err != nil {
+ return err
+ }
+
+ conn := &archived.GiteeConnection{}
+ v := config.GetConfig()
+ encKey := v.GetString(core.EncodeKeyEnvStr)
+
+ conn.Name = "init gitee connection"
+ conn.ID = 1
+ conn.Endpoint = v.GetString("GITEE_ENDPOINT")
+ conn.Token, err = core.Encrypt(encKey, v.GetString("GITEE_AUTH"))
+ if err != nil {
+ return err
+ }
+ conn.Proxy = v.GetString("GITEE_PROXY")
+ conn.RateLimit = v.GetInt("GITEE_API_REQUESTS_PER_HOUR")
+
+ err = db.Clauses(clause.OnConflict{DoNothing: true}).Create(conn).Error
+
+ if err != nil {
+ return err
+ }
+ return nil
}
func (*InitSchemas) Version() uint64 {
- return 20220617201204
+ return 20220617231243
}
func (*InitSchemas) Name() string {
diff --git a/plugins/gitee/tasks/api_client.go b/plugins/gitee/tasks/api_client.go
index 94c63256..0f567186 100644
--- a/plugins/gitee/tasks/api_client.go
+++ b/plugins/gitee/tasks/api_client.go
@@ -23,32 +23,26 @@ import (
"strconv"
"time"
+ "github.com/apache/incubator-devlake/plugins/gitee/models"
+
"github.com/apache/incubator-devlake/plugins/core"
"github.com/apache/incubator-devlake/plugins/helper"
- "github.com/apache/incubator-devlake/utils"
)
-func NewGiteeApiClient(taskCtx core.TaskContext) (*helper.ApiAsyncClient, error) {
- endpoint := taskCtx.GetConfig("GITEE_ENDPOINT")
- if endpoint == "" {
- return nil, fmt.Errorf("endpint is required")
- }
- userRateLimit, err := utils.StrToIntOr(taskCtx.GetConfig("GITEE_API_REQUESTS_PER_HOUR"), 0)
+func NewGiteeApiClient(taskCtx core.TaskContext, connection *models.GiteeConnection) (*helper.ApiAsyncClient, error) {
+
+ apiClient, err := helper.NewApiClient(connection.Endpoint, nil, 0, connection.Proxy, taskCtx.GetContext())
if err != nil {
return nil, err
}
- auth := taskCtx.GetConfig("GITEE_AUTH")
- if auth == "" {
- return nil, fmt.Errorf("GITEE_AUTH is required")
- }
- proxy := taskCtx.GetConfig("GITEE_PROXY")
- headers := map[string]string{}
+ apiClient.SetBeforeFunction(func(req *http.Request) error {
+ query := req.URL.Query()
+ query.Set("access_token", connection.Token)
+ req.URL.RawQuery = query.Encode()
+ return nil
+ })
- apiClient, err := helper.NewApiClient(endpoint, headers, 0, proxy, taskCtx.GetContext())
- if err != nil {
- return nil, err
- }
apiClient.SetAfterFunction(func(res *http.Response) error {
if res.StatusCode == http.StatusUnauthorized {
return fmt.Errorf("authentication failed, please check your Basic Auth Token")
@@ -57,7 +51,7 @@ func NewGiteeApiClient(taskCtx core.TaskContext) (*helper.ApiAsyncClient, error)
})
rateLimiter := &helper.ApiRateLimitCalculator{
- UserRateLimitPerHour: userRateLimit,
+ UserRateLimitPerHour: connection.RateLimit,
DynamicRateLimit: func(res *http.Response) (int, time.Duration, error) {
rateLimitHeader := res.Header.Get("RateLimit-Limit")
if rateLimitHeader == "" {
diff --git a/plugins/gitee/tasks/commit_collector.go b/plugins/gitee/tasks/commit_collector.go
index dc834e53..0d37a632 100644
--- a/plugins/gitee/tasks/commit_collector.go
+++ b/plugins/gitee/tasks/commit_collector.go
@@ -45,7 +45,6 @@ func CollectApiCommits(taskCtx core.SubTaskContext) error {
UrlTemplate: "repos/{{ .Params.Owner }}/{{ .Params.Repo }}/commits",
Query: func(reqData *helper.RequestData) (url.Values, error) {
query := url.Values{}
- query.Set("access_token", data.Options.Token)
query.Set("with_stats", "true")
query.Set("sort", "asc")
query.Set("page", strconv.Itoa(reqData.Pager.Page))
diff --git a/plugins/gitee/tasks/commit_stats_collector.go b/plugins/gitee/tasks/commit_stats_collector.go
index cdc23c1d..83902bf4 100644
--- a/plugins/gitee/tasks/commit_stats_collector.go
+++ b/plugins/gitee/tasks/commit_stats_collector.go
@@ -85,7 +85,6 @@ func CollectApiCommitStats(taskCtx core.SubTaskContext) error {
*/
Query: func(reqData *helper.RequestData) (url.Values, error) {
query := url.Values{}
- query.Set("access_token", data.Options.Token)
query.Set("state", "all")
query.Set("direction", "asc")
query.Set("page", fmt.Sprintf("%v", reqData.Pager.Page))
diff --git a/plugins/gitee/tasks/issue_collector.go b/plugins/gitee/tasks/issue_collector.go
index 4b882233..5b325084 100644
--- a/plugins/gitee/tasks/issue_collector.go
+++ b/plugins/gitee/tasks/issue_collector.go
@@ -67,7 +67,6 @@ func CollectApiIssues(taskCtx core.SubTaskContext) error {
UrlTemplate: "repos/{{ .Params.Owner }}/{{ .Params.Repo }}/issues",
Query: func(reqData *helper.RequestData) (url.Values, error) {
query := url.Values{}
- query.Set("access_token", data.Options.Token)
query.Set("state", "all")
if since != nil {
query.Set("since", since.String())
diff --git a/plugins/gitee/tasks/issue_comment_collector.go b/plugins/gitee/tasks/issue_comment_collector.go
index a74b08fb..df0d6364 100644
--- a/plugins/gitee/tasks/issue_comment_collector.go
+++ b/plugins/gitee/tasks/issue_comment_collector.go
@@ -87,7 +87,6 @@ func CollectApiIssueComments(taskCtx core.SubTaskContext) error {
UrlTemplate: "repos/{{ .Params.Owner }}/{{ .Params.Repo }}/issues/comments",
Query: func(reqData *helper.RequestData) (url.Values, error) {
query := url.Values{}
- query.Set("access_token", data.Options.Token)
query.Set("state", "all")
if since != nil {
query.Set("since", since.String())
diff --git a/plugins/gitee/tasks/issue_comment_extractor.go b/plugins/gitee/tasks/issue_comment_extractor.go
index 8e321b05..2fcfb76c 100644
--- a/plugins/gitee/tasks/issue_comment_extractor.go
+++ b/plugins/gitee/tasks/issue_comment_extractor.go
@@ -36,11 +36,21 @@ var ExtractApiIssueCommentsMeta = core.SubTaskMeta{
type IssueComment struct {
GiteeId int `json:"id"`
Body string
- User struct {
+
+ User struct {
Login string
Id int
}
- IssueUrl string `json:"issue_url"`
+
+ Target struct {
+ Issue struct {
+ Id int `json:"id"`
+ Title string `json:"title"`
+ Number string `json:"number"`
+ }
+ PullRequest string `json:"pull_request"`
+ }
+
GiteeCreatedAt helper.Iso8601Time `json:"created_at"`
GiteeUpdatedAt helper.Iso8601Time `json:"updated_at"`
}
@@ -69,7 +79,7 @@ func ExtractApiIssueComments(taskCtx core.SubTaskContext) error {
return nil, nil
}
//If this is a pr, ignore
- issueINumber, err := GetIssueIdByIssueUrl(apiComment.IssueUrl)
+ issueINumber := apiComment.Target.Issue.Number
if err != nil {
return nil, err
}
diff --git a/plugins/gitee/tasks/pr_collector.go b/plugins/gitee/tasks/pr_collector.go
index 31f80ddc..c20a308d 100644
--- a/plugins/gitee/tasks/pr_collector.go
+++ b/plugins/gitee/tasks/pr_collector.go
@@ -68,7 +68,6 @@ func CollectApiPullRequests(taskCtx core.SubTaskContext) error {
Query: func(reqData *helper.RequestData) (url.Values, error) {
query := url.Values{}
- query.Set("access_token", data.Options.Token)
query.Set("state", "all")
if since != nil {
query.Set("since", since.String())
diff --git a/plugins/gitee/tasks/pr_commit_collector.go b/plugins/gitee/tasks/pr_commit_collector.go
index 8a5a1388..5cdd8e3e 100644
--- a/plugins/gitee/tasks/pr_commit_collector.go
+++ b/plugins/gitee/tasks/pr_commit_collector.go
@@ -71,7 +71,6 @@ func CollectApiPullRequestCommits(taskCtx core.SubTaskContext) error {
Query: func(reqData *helper.RequestData) (url.Values, error) {
query := url.Values{}
- query.Set("access_token", data.Options.Token)
query.Set("state", "all")
query.Set("page", fmt.Sprintf("%v", reqData.Pager.Page))
query.Set("direction", "asc")
diff --git a/plugins/gitee/tasks/pr_review_collector.go b/plugins/gitee/tasks/pr_review_collector.go
index 468236f6..bfc21cec 100644
--- a/plugins/gitee/tasks/pr_review_collector.go
+++ b/plugins/gitee/tasks/pr_review_collector.go
@@ -67,7 +67,6 @@ func CollectApiPullRequestReviews(taskCtx core.SubTaskContext) error {
Query: func(reqData *helper.RequestData) (url.Values, error) {
query := url.Values{}
- query.Set("access_token", data.Options.Token)
query.Set("page", fmt.Sprintf("%v", reqData.Pager.Page))
query.Set("per_page", fmt.Sprintf("%v", reqData.Pager.Size))
query.Set("sort", "asc")
diff --git a/plugins/gitee/tasks/repo_collector.go b/plugins/gitee/tasks/repo_collector.go
index 15111300..ad2670c1 100644
--- a/plugins/gitee/tasks/repo_collector.go
+++ b/plugins/gitee/tasks/repo_collector.go
@@ -47,7 +47,6 @@ func CollectApiRepositories(taskCtx core.SubTaskContext) error {
UrlTemplate: "repos/{{ .Params.Owner }}/{{ .Params.Repo }}",
Query: func(reqData *helper.RequestData) (url.Values, error) {
query := url.Values{}
- query.Set("access_token", data.Options.Token)
query.Set("state", "all")
query.Set("page", fmt.Sprintf("%v", reqData.Pager.Page))
query.Set("direction", "asc")
diff --git a/plugins/gitee/tasks/shared.go b/plugins/gitee/tasks/shared.go
index 48238dcb..76b2de18 100644
--- a/plugins/gitee/tasks/shared.go
+++ b/plugins/gitee/tasks/shared.go
@@ -48,7 +48,6 @@ type RateLimitInfo struct {
type GiteeApiParams struct {
Repo string
Owner string
- Token string
}
type GiteeInput struct {
@@ -96,7 +95,6 @@ func CreateRawDataSubTaskArgs(taskCtx core.SubTaskContext, Table string) (*helpe
Params: GiteeApiParams{
Repo: data.Options.Repo,
Owner: data.Options.Owner,
- Token: data.Options.Token,
},
Table: Table,
}
@@ -194,13 +192,3 @@ func GetPagingFromLinkHeader(link string) (PagingInfo, error) {
return result, errors.New("the link string provided is invalid. There is likely no next page of data to fetch")
}
}
-
-func GetIssueIdByIssueUrl(s string) (int, error) {
- regex := regexp.MustCompile(`.*/issues/(\d+)`)
- groups := regex.FindStringSubmatch(s)
- if len(groups) > 0 {
- return strconv.Atoi(groups[1])
- } else {
- return 0, errors.New("invalid issue url")
- }
-}
diff --git a/plugins/gitee/tasks/task_data.go b/plugins/gitee/tasks/task_data.go
index 76956641..c990378b 100644
--- a/plugins/gitee/tasks/task_data.go
+++ b/plugins/gitee/tasks/task_data.go
@@ -25,12 +25,13 @@ import (
)
type GiteeOptions struct {
- Tasks []string `json:"tasks,omitempty"`
- Since string
- Owner string
- Repo string
- Token string
- models.Config
+ ConnectionId uint64 `json:"connectionId"`
+ Tasks []string `json:"tasks,omitempty"`
+ Since string
+ Owner string
+ Repo string
+ Token string
+ models.Config `mapstructure:",squash"`
}
type GiteeTaskData struct {
[incubator-devlake] 06/10: update api connection.go
Posted by wa...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
warren pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit 495420cfb4a5713b9a271e038be764959167cd26
Author: Mr.An <42...@users.noreply.github.com>
AuthorDate: Sat Jun 18 02:26:19 2022 +0800
update api connection.go
---
plugins/gitee/api/connection.go | 80 +++++++++++++++++---------------------
plugins/gitee/api/init.go | 42 ++++++++++++++++++++
plugins/gitee/models/connection.go | 25 ++++--------
3 files changed, 84 insertions(+), 63 deletions(-)
diff --git a/plugins/gitee/api/connection.go b/plugins/gitee/api/connection.go
index 7c912eb9..b074dc7b 100644
--- a/plugins/gitee/api/connection.go
+++ b/plugins/gitee/api/connection.go
@@ -23,29 +23,23 @@ import (
"net/url"
"time"
- "github.com/apache/incubator-devlake/config"
- "github.com/apache/incubator-devlake/plugins/core"
"github.com/apache/incubator-devlake/plugins/gitee/models"
- "github.com/apache/incubator-devlake/plugins/helper"
- "github.com/go-playground/validator/v10"
+ "github.com/apache/incubator-devlake/plugins/helper"
"github.com/mitchellh/mapstructure"
-)
-var vld = validator.New()
+ "github.com/apache/incubator-devlake/plugins/core"
+)
/*
POST /plugins/gitee/test
*/
func TestConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) {
- // decode
- var err error
var connection models.TestConnectionRequest
- err = mapstructure.Decode(input.Body, &connection)
+ err := mapstructure.Decode(input.Body, &connection)
if err != nil {
return nil, err
}
- // validate
err = vld.Struct(connection)
if err != nil {
return nil, err
@@ -61,8 +55,9 @@ func TestConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput, erro
if err != nil {
return nil, err
}
- query := make(url.Values)
- query["access_token"] = []string{connection.Auth}
+
+ query := url.Values{}
+ query.Set("access_token", connection.Token)
res, err := apiClient.Get("user", query, nil)
if err != nil {
@@ -81,68 +76,63 @@ func TestConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput, erro
}
/*
-PATCH /plugins/gitee/connections/:connectionId
+POST /plugins/gitee/connections
*/
-func PatchConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) {
- v := config.GetConfig()
+func PostConnections(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) {
connection := &models.GiteeConnection{}
- err := helper.EncodeStruct(v, connection, "env")
+ err := connectionHelper.Create(connection, input)
if err != nil {
return nil, err
}
- // update from request and save to .env
- err = helper.DecodeStruct(v, connection, input.Body, "env")
+ return &core.ApiResourceOutput{Body: connection, Status: http.StatusOK}, nil
+}
+
+/*
+PATCH /plugins/gitee/connections/:connectionId
+*/
+func PatchConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) {
+ connection := &models.GiteeConnection{}
+ err := connectionHelper.Patch(connection, input)
if err != nil {
return nil, err
}
- err = config.WriteConfig(v)
+ return &core.ApiResourceOutput{Body: connection, Status: http.StatusOK}, nil
+}
+
+/*
+DELETE /plugins/gitee/connections/:connectionId
+*/
+func DeleteConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) {
+ connection := &models.GiteeConnection{}
+ err := connectionHelper.First(connection, input.Params)
if err != nil {
return nil, err
}
- response := models.GiteeResponse{
- GiteeConnection: *connection,
- Name: "Gitee",
- ID: 1,
- }
- return &core.ApiResourceOutput{Body: response, Status: http.StatusOK}, nil
+ err = connectionHelper.Delete(connection)
+ return &core.ApiResourceOutput{Body: connection}, err
}
/*
GET /plugins/gitee/connections
*/
func ListConnections(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) {
- // RETURN ONLY 1 SOURCE (FROM ENV) until multi-connection is developed.
- v := config.GetConfig()
- connection := &models.GiteeConnection{}
-
- err := helper.EncodeStruct(v, connection, "env")
+ var connections []models.GiteeConnection
+ err := connectionHelper.List(&connections)
if err != nil {
return nil, err
}
- response := models.GiteeResponse{
- GiteeConnection: *connection,
- Name: "Gitee",
- ID: 1,
- }
- return &core.ApiResourceOutput{Body: []models.GiteeResponse{response}}, nil
+ return &core.ApiResourceOutput{Body: connections}, nil
}
/*
GET /plugins/gitee/connections/:connectionId
*/
func GetConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) {
- // RETURN ONLY 1 SOURCE FROM ENV (Ignore ID until multi-connection is developed.)
- v := config.GetConfig()
connection := &models.GiteeConnection{}
- err := helper.EncodeStruct(v, connection, "env")
+ err := connectionHelper.First(connection, input.Params)
if err != nil {
return nil, err
}
- response := &models.GiteeResponse{
- GiteeConnection: *connection,
- Name: "Gitee",
- ID: 1,
- }
- return &core.ApiResourceOutput{Body: response}, nil
+ return &core.ApiResourceOutput{Body: connection}, err
}
diff --git a/plugins/gitee/api/init.go b/plugins/gitee/api/init.go
new file mode 100644
index 00000000..aef88dcb
--- /dev/null
+++ b/plugins/gitee/api/init.go
@@ -0,0 +1,42 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package api
+
+import (
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/github/models"
+ "github.com/apache/incubator-devlake/plugins/helper"
+ "github.com/go-playground/validator/v10"
+ "github.com/spf13/viper"
+ "gorm.io/gorm"
+)
+
+type ApiUserPublicEmailResponse []models.PublicEmail
+
+var vld *validator.Validate
+var connectionHelper *helper.ConnectionApiHelper
+var basicRes core.BasicRes
+
+func Init(config *viper.Viper, logger core.Logger, database *gorm.DB) {
+ basicRes = helper.NewDefaultBasicRes(config, logger, database)
+ vld = validator.New()
+ connectionHelper = helper.NewConnectionHelper(
+ basicRes,
+ vld,
+ )
+}
diff --git a/plugins/gitee/models/connection.go b/plugins/gitee/models/connection.go
index 24ad1b6d..f161df73 100644
--- a/plugins/gitee/models/connection.go
+++ b/plugins/gitee/models/connection.go
@@ -17,30 +17,28 @@ limitations under the License.
package models
-// This object conforms to what the frontend currently sends.
+import "github.com/apache/incubator-devlake/plugins/helper"
+
type GiteeConnection struct {
- Endpoint string `mapstructure:"endpoint" validate:"required" env:"GITEE_ENDPOINT" json:"endpoint"`
- Auth string `mapstructure:"auth" validate:"required" env:"GITEE_AUTH" json:"auth"`
- Proxy string `mapstructure:"proxy" env:"GITEE_PROXY" json:"proxy"`
+ helper.RestConnection `mapstructure:",squash"`
+ helper.AccessToken `mapstructure:",squash"`
}
-// This object conforms to what the frontend currently expects.
type GiteeResponse struct {
Name string `json:"name"`
ID int `json:"id"`
GiteeConnection
}
-// Using User because it requires authentication.
type ApiUserResponse struct {
Id int
Name string `json:"name"`
}
type TestConnectionRequest struct {
- Endpoint string `json:"endpoint" validate:"required"`
- Auth string `json:"auth" validate:"required"`
- Proxy string `json:"proxy"`
+ Endpoint string `json:"endpoint" validate:"required"`
+ Proxy string `json:"proxy"`
+ helper.AccessToken `mapstructure:",squash"`
}
type Config struct {
@@ -53,12 +51,3 @@ type Config struct {
IssueTypeIncident string `mapstructure:"issueTypeIncident" env:"GITEE_ISSUE_TYPE_INCIDENT" json:"issueTypeIncident"`
IssueTypeRequirement string `mapstructure:"issueTypeRequirement" env:"GITEE_ISSUE_TYPE_REQUIREMENT" json:"issueTypeRequirement"`
}
-
-// Using Public Email because it requires authentication, and it is public information anyway.
-// We're not using email information for anything here.
-type PublicEmail struct {
- Email string
- Primary bool
- Verified bool
- Visibility string
-}
[incubator-devlake] 02/10: update gitee plugin
Posted by wa...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
warren pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit 03f5a14d731188f4131534eb17c474694c583d65
Author: Mr.An <42...@users.noreply.github.com>
AuthorDate: Wed Jun 15 12:13:02 2022 +0800
update gitee plugin
---
plugins/gitee/README.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/plugins/gitee/README.md b/plugins/gitee/README.md
index 98a51514..ce1bc6b6 100644
--- a/plugins/gitee/README.md
+++ b/plugins/gitee/README.md
@@ -45,7 +45,7 @@ Click **Save Settings** to update additional settings.
### Regular Expression Configuration
Define regex pattern in .env
-- gitee_PR_BODY_CLOSE_PATTERN: Define key word to associate issue in pr body, please check the example in .env.example
+- GITEE_PR_BODY_CLOSE_PATTERN: Define key word to associate issue in pr body, please check the example in .env.example
## Sample Request
In order to collect data, you have to compose a JSON looks like following one, and send it by selecting `Advanced Mode` on `Create Pipeline Run` page:
[incubator-devlake] 05/10: update commit and pr review code
Posted by wa...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
warren pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit f1431a707a78e2e9c9ae007d75d0763edcfc4b1c
Author: Mr.An <42...@users.noreply.github.com>
AuthorDate: Fri Jun 17 11:55:39 2022 +0800
update commit and pr review code
---
plugins/gitee/models/migrationscripts/init_schema.go | 2 +-
plugins/gitee/tasks/commit_extractor.go | 2 --
plugins/gitee/tasks/pr_review_collector.go | 5 +++--
plugins/gitee/tasks/pr_review_extractor.go | 8 ++++----
4 files changed, 8 insertions(+), 9 deletions(-)
diff --git a/plugins/gitee/models/migrationscripts/init_schema.go b/plugins/gitee/models/migrationscripts/init_schema.go
index 5eceb089..44e3e33a 100644
--- a/plugins/gitee/models/migrationscripts/init_schema.go
+++ b/plugins/gitee/models/migrationscripts/init_schema.go
@@ -46,7 +46,7 @@ func (*InitSchemas) Up(ctx context.Context, db *gorm.DB) error {
}
func (*InitSchemas) Version() uint64 {
- return 20220407201202
+ return 20220617201204
}
func (*InitSchemas) Name() string {
diff --git a/plugins/gitee/tasks/commit_extractor.go b/plugins/gitee/tasks/commit_extractor.go
index 3f280af5..1ebeb81e 100644
--- a/plugins/gitee/tasks/commit_extractor.go
+++ b/plugins/gitee/tasks/commit_extractor.go
@@ -48,7 +48,6 @@ type GiteeCommit struct {
type GiteeApiCommitResponse struct {
Author *models.GiteeUser `json:"author"`
- AuthorId int
CommentsUrl string `json:"comments_url"`
Commit GiteeCommit `json:"commit"`
Committer *models.GiteeUser `json:"committer"`
@@ -114,7 +113,6 @@ func ExtractApiCommits(taskCtx core.SubTaskContext) error {
func ConvertCommit(commit *GiteeApiCommitResponse) (*models.GiteeCommit, error) {
giteeCommit := &models.GiteeCommit{
Sha: commit.Sha,
- AuthorId: commit.Author.Id,
Message: commit.Commit.Message,
AuthorName: commit.Commit.Author.Name,
AuthorEmail: commit.Commit.Author.Email,
diff --git a/plugins/gitee/tasks/pr_review_collector.go b/plugins/gitee/tasks/pr_review_collector.go
index 0a64752e..468236f6 100644
--- a/plugins/gitee/tasks/pr_review_collector.go
+++ b/plugins/gitee/tasks/pr_review_collector.go
@@ -30,6 +30,7 @@ import (
"github.com/apache/incubator-devlake/plugins/gitee/models"
)
+//gitee
const RAW_PULL_REQUEST_REVIEW_TABLE = "gitee_api_pull_request_reviews"
var CollectApiPullRequestReviewsMeta = core.SubTaskMeta{
@@ -62,14 +63,14 @@ func CollectApiPullRequestReviews(taskCtx core.SubTaskContext) error {
Incremental: incremental,
Input: iterator,
- UrlTemplate: "repos/{{ .Params.Owner }}/{{ .Params.Repo }}/pulls/3/review",
+ UrlTemplate: "repos/{{ .Params.Owner }}/{{ .Params.Repo }}/pulls/{{ .Input.Number }}/operate_logs",
Query: func(reqData *helper.RequestData) (url.Values, error) {
query := url.Values{}
query.Set("access_token", data.Options.Token)
query.Set("page", fmt.Sprintf("%v", reqData.Pager.Page))
query.Set("per_page", fmt.Sprintf("%v", reqData.Pager.Size))
-
+ query.Set("sort", "asc")
return query, nil
},
ResponseParser: func(res *http.Response) ([]json.RawMessage, error) {
diff --git a/plugins/gitee/tasks/pr_review_extractor.go b/plugins/gitee/tasks/pr_review_extractor.go
index fd4c0499..5fa79c38 100644
--- a/plugins/gitee/tasks/pr_review_extractor.go
+++ b/plugins/gitee/tasks/pr_review_extractor.go
@@ -38,10 +38,11 @@ type PullRequestReview struct {
User struct {
Id int
Login string
+ Name string
}
- Body string
- State string
- SubmittedAt helper.Iso8601Time `json:"submitted_at"`
+ Content string
+ ActionType string `json:"action_type"`
+ CreatedAt helper.Iso8601Time `json:"created_at"`
}
func ExtractApiPullRequestReviews(taskCtx core.SubTaskContext) error {
@@ -62,7 +63,6 @@ func ExtractApiPullRequestReviews(taskCtx core.SubTaskContext) error {
if err != nil {
return nil, err
}
- // need to extract 2 kinds of entities here
results := make([]interface{}, 0, 1)
giteeReviewer := &models.GiteeReviewer{
[incubator-devlake] 08/10: add default gitee endpoint
Posted by wa...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
warren pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit 0313c1212427ab15de991c31ea0310211a327cf4
Author: Mr.An <42...@users.noreply.github.com>
AuthorDate: Sat Jun 18 17:01:42 2022 +0800
add default gitee endpoint
---
config/config.go | 1 +
1 file changed, 1 insertion(+)
diff --git a/config/config.go b/config/config.go
index aeab9def..02fad7cf 100644
--- a/config/config.go
+++ b/config/config.go
@@ -71,6 +71,7 @@ func setDefaultValue(v *viper.Viper) {
v.SetDefault("TEMPORAL_TASK_QUEUE", "DEVLAKE_TASK_QUEUE")
v.SetDefault("GITLAB_ENDPOINT", "https://gitlab.com/api/v4/")
v.SetDefault("GITHUB_ENDPOINT", "https://api.github.com/")
+ v.SetDefault("GITEE_ENDPOINT", "https://gitee.com/api/v5/")
}
// replaceNewEnvItemInOldContent replace old config to new config in env file content
[incubator-devlake] 10/10: update gitee main
Posted by wa...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
warren pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit 5f0804ba062c1fcb99828934ba7b6bb407e3b77a
Author: Mr.An <42...@users.noreply.github.com>
AuthorDate: Sat Jun 18 17:09:18 2022 +0800
update gitee main
---
plugins/gitee/gitee.go | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/plugins/gitee/gitee.go b/plugins/gitee/gitee.go
index 7c239996..7ad70737 100644
--- a/plugins/gitee/gitee.go
+++ b/plugins/gitee/gitee.go
@@ -26,7 +26,7 @@ import (
var PluginEntry impl.Gitee //nolint
func main() {
- cmd := &cobra.Command{Use: "github"}
+ cmd := &cobra.Command{Use: "gitee"}
connectionId := cmd.Flags().Uint64P("connectionId", "c", 0, "gitee connection id")
owner := cmd.Flags().StringP("owner", "o", "", "gitee owner")
repo := cmd.Flags().StringP("repo", "r", "", "gitee repo")
[incubator-devlake] 01/10: init commite gitee plugin
Posted by wa...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
warren pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit 9f239e8a8fbdbcd9e4600678533d4aa6773e1307
Author: Mr.An <42...@users.noreply.github.com>
AuthorDate: Wed Jun 15 12:08:55 2022 +0800
init commite gitee plugin
---
plugins/gitee/README.md | 121 ++++++++++
plugins/gitee/api/connection.go | 131 +++++++++++
plugins/gitee/gitee.go | 27 +++
plugins/gitee/impl/impl.go | 104 +++++++++
plugins/gitee/models/commit.go | 47 ++++
plugins/gitee/models/commit_stats.go | 36 +++
plugins/gitee/models/connection.go | 64 ++++++
plugins/gitee/models/issue.go | 52 +++++
plugins/gitee/models/issue_comment.go | 39 ++++
plugins/gitee/models/issue_label.go | 35 +++
.../models/migrationscripts/archived/commit.go | 47 ++++
.../migrationscripts/archived/commit_stat.go | 36 +++
.../models/migrationscripts/archived/issue.go | 52 +++++
.../migrationscripts/archived/issue_comment.go | 39 ++++
.../migrationscripts/archived/issue_label.go | 33 +++
.../migrationscripts/archived/pull_request.go | 59 +++++
.../archived/pull_request_comment.go | 39 ++++
.../archived/pull_request_commit.go | 30 +++
.../archived/pull_request_issue.go | 32 +++
.../archived/pull_request_label.go | 30 +++
.../gitee/models/migrationscripts/archived/repo.go | 43 ++++
.../migrationscripts/archived/repo_commit.go | 32 +++
.../models/migrationscripts/archived/reviewer.go | 32 +++
.../gitee/models/migrationscripts/archived/user.go | 47 ++++
.../gitee/models/migrationscripts/init_schema.go | 54 +++++
plugins/gitee/models/pull_request.go | 59 +++++
plugins/gitee/models/pull_request_comment.go | 39 ++++
plugins/gitee/models/pull_request_commit.go | 32 +++
plugins/gitee/models/pull_request_issue.go | 32 +++
plugins/gitee/models/pull_request_label.go | 32 +++
plugins/gitee/models/repo.go | 43 ++++
plugins/gitee/models/repo_commit.go | 32 +++
plugins/gitee/models/reviewer.go | 34 +++
plugins/gitee/models/user.go | 45 ++++
plugins/gitee/tasks/api_client.go | 84 +++++++
plugins/gitee/tasks/commit_collector.go | 64 ++++++
plugins/gitee/tasks/commit_convertor.go | 102 +++++++++
plugins/gitee/tasks/commit_extractor.go | 111 +++++++++
plugins/gitee/tasks/commit_stats_collector.go | 112 +++++++++
plugins/gitee/tasks/commit_stats_extractor.go | 98 ++++++++
plugins/gitee/tasks/issue_collector.go | 97 ++++++++
plugins/gitee/tasks/issue_comment_collector.go | 110 +++++++++
plugins/gitee/tasks/issue_comment_convertor.go | 80 +++++++
plugins/gitee/tasks/issue_comment_extractor.go | 119 ++++++++++
plugins/gitee/tasks/issue_convertor.go | 101 ++++++++
plugins/gitee/tasks/issue_extractor.go | 253 +++++++++++++++++++++
plugins/gitee/tasks/issue_label_convertor.go | 73 ++++++
plugins/gitee/tasks/pr_collector.go | 99 ++++++++
plugins/gitee/tasks/pr_comment_convertor.go | 82 +++++++
plugins/gitee/tasks/pr_commit_collector.go | 96 ++++++++
plugins/gitee/tasks/pr_commit_convertor.go | 73 ++++++
plugins/gitee/tasks/pr_commit_extractor.go | 127 +++++++++++
plugins/gitee/tasks/pr_convertor.go | 92 ++++++++
plugins/gitee/tasks/pr_extractor.go | 171 ++++++++++++++
plugins/gitee/tasks/pr_issue_convertor.go | 78 +++++++
plugins/gitee/tasks/pr_issue_enricher.go | 116 ++++++++++
plugins/gitee/tasks/pr_label_convertor.go | 73 ++++++
plugins/gitee/tasks/pr_review_collector.go | 89 ++++++++
plugins/gitee/tasks/pr_review_extractor.go | 84 +++++++
plugins/gitee/tasks/repo_collector.go | 72 ++++++
plugins/gitee/tasks/repo_convertor.go | 95 ++++++++
plugins/gitee/tasks/repo_extractor.go | 90 ++++++++
plugins/gitee/tasks/shared.go | 206 +++++++++++++++++
plugins/gitee/tasks/task_data.go | 41 ++++
plugins/gitee/tasks/user_convertor.go | 72 ++++++
65 files changed, 4769 insertions(+)
diff --git a/plugins/gitee/README.md b/plugins/gitee/README.md
new file mode 100644
index 00000000..98a51514
--- /dev/null
+++ b/plugins/gitee/README.md
@@ -0,0 +1,121 @@
+# Gitee Pond
+
+<div align="center">
+
+| [English](README.md) | [中文](README-zh-CN.md) |
+| --- | --- |
+
+</div>
+
+<br>
+
+## Summary
+
+## Configuration
+
+### Provider (Datasource) Connection
+The connection aspect of the configuration screen requires the following key fields to connect to the **Gitee API**. As gitee is a _single-source data provider_ at the moment, the connection name is read-only as there is only one instance to manage. As we continue our development roadmap we may enable _multi-source_ connections for gitee in the future.
+
+- **Connection Name** [`READONLY`]
+ - ⚠️ Defaults to "**Gitee**" and may not be changed.
+- **Endpoint URL** (REST URL, starts with `https://` or `http://`)
+ - This should be a valid REST API Endpoint eg. `https://gitee.com/api/v5/`
+ - ⚠️ URL should end with`/`
+- **Auth Token(s)** (Personal Access Token)
+ - For help on **Creating a personal access token**
+ - Provide at least one token for Authentication with the . This field accepts a comma-separated list of values for multiple tokens. The data collection will take longer for gitee since they have a **rate limit of 2k requests per hour**. You can accelerate the process by configuring _multiple_ personal access tokens.
+
+"For API requests using `Basic Authentication` or `OAuth`
+
+
+If you have a need for more api rate limits, you can set many tokens in the config file and we will use all of your tokens.
+
+For an overview of the **gitee REST API**, please see official [gitee Docs on REST](https://gitee.com/api/v5/swagger)
+
+Click **Save Connection** to update connection settings.
+
+
+### Provider (Datasource) Settings
+Manage additional settings and options for the gitee Datasource Provider. Currently there is only one **optional** setting, *Proxy URL*. If you are behind a corporate firewall or VPN you may need to utilize a proxy server.
+
+**gitee Proxy URL [ `Optional`]**
+Enter a valid proxy server address on your Network, e.g. `http://your-proxy-server.com:1080`
+
+Click **Save Settings** to update additional settings.
+
+### Regular Expression Configuration
+Define regex pattern in .env
+- gitee_PR_BODY_CLOSE_PATTERN: Define key word to associate issue in pr body, please check the example in .env.example
+
+## Sample Request
+In order to collect data, you have to compose a JSON looks like following one, and send it by selecting `Advanced Mode` on `Create Pipeline Run` page:
+1. Configure-UI Mode
+```json
+[
+ [
+ {
+ "plugin": "gitee",
+ "options": {
+ "repo": "lake",
+ "owner": "merico-dev",
+ "token": "xxxx"
+ }
+ }
+ ]
+]
+```
+and if you want to perform certain subtasks.
+```json
+[
+ [
+ {
+ "plugin": "gitee",
+ "subtasks": ["collectXXX", "extractXXX", "convertXXX"],
+ "options": {
+ "repo": "lake",
+ "owner": "merico-dev",
+ "token": "xxxx"
+ }
+ }
+ ]
+]
+```
+
+2. Curl Mode:
+ You can also trigger data collection by making a POST request to `/pipelines`.
+```
+curl --location --request POST 'localhost:8080/pipelines' \
+--header 'Content-Type: application/json' \
+--data-raw '
+{
+ "name": "gitee 20211126",
+ "tasks": [[{
+ "plugin": "gitee",
+ "options": {
+ "repo": "lake",
+ "owner": "merico-dev"
+ "token": "xxxx"
+ }
+ }]]
+}
+'
+```
+and if you want to perform certain subtasks.
+```
+curl --location --request POST 'localhost:8080/pipelines' \
+--header 'Content-Type: application/json' \
+--data-raw '
+{
+ "name": "gitee 20211126",
+ "tasks": [[{
+ "plugin": "gitee",
+ "subtasks": ["collectXXX", "extractXXX", "convertXXX"],
+ "options": {
+ "repo": "lake",
+ "owner": "merico-dev"
+ "token": "xxxx"
+ }
+ }]]
+}
+'
+```
diff --git a/plugins/gitee/api/connection.go b/plugins/gitee/api/connection.go
new file mode 100644
index 00000000..98884b8a
--- /dev/null
+++ b/plugins/gitee/api/connection.go
@@ -0,0 +1,131 @@
+package api
+
+import (
+ "fmt"
+ "net/http"
+ "net/url"
+ "time"
+
+ "github.com/apache/incubator-devlake/config"
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/gitee/models"
+ "github.com/apache/incubator-devlake/plugins/helper"
+
+ "github.com/go-playground/validator/v10"
+ "github.com/mitchellh/mapstructure"
+)
+
+var vld = validator.New()
+
+/*
+POST /plugins/gitee/test
+*/
+func TestConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) {
+ // decode
+ var err error
+ var connection models.TestConnectionRequest
+ err = mapstructure.Decode(input.Body, &connection)
+ if err != nil {
+ return nil, err
+ }
+ // validate
+ err = vld.Struct(connection)
+ if err != nil {
+ return nil, err
+ }
+ // test connection
+ apiClient, err := helper.NewApiClient(
+ connection.Endpoint,
+ nil,
+ 3*time.Second,
+ connection.Proxy,
+ nil,
+ )
+ if err != nil {
+ return nil, err
+ }
+ query := make(url.Values)
+ query["access_token"] = []string{connection.Auth}
+
+ res, err := apiClient.Get("user", query, nil)
+ if err != nil {
+ return nil, err
+ }
+ resBody := &models.ApiUserResponse{}
+ err = helper.UnmarshalResponse(res, resBody)
+ if err != nil {
+ return nil, err
+ }
+
+ if res.StatusCode != http.StatusOK {
+ return nil, fmt.Errorf("unexpected status code: %d", res.StatusCode)
+ }
+ return nil, nil
+}
+
+/*
+PATCH /plugins/gitee/connections/:connectionId
+*/
+func PatchConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) {
+ v := config.GetConfig()
+ connection := &models.GiteeConnection{}
+ err := helper.EncodeStruct(v, connection, "env")
+ if err != nil {
+ return nil, err
+ }
+ // update from request and save to .env
+ err = helper.DecodeStruct(v, connection, input.Body, "env")
+ if err != nil {
+ return nil, err
+ }
+ err = config.WriteConfig(v)
+ if err != nil {
+ return nil, err
+ }
+ response := models.GiteeResponse{
+ GiteeConnection: *connection,
+ Name: "Gitee",
+ ID: 1,
+ }
+ return &core.ApiResourceOutput{Body: response, Status: http.StatusOK}, nil
+}
+
+/*
+GET /plugins/gitee/connections
+*/
+func ListConnections(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) {
+ // RETURN ONLY 1 SOURCE (FROM ENV) until multi-connection is developed.
+ v := config.GetConfig()
+ connection := &models.GiteeConnection{}
+
+ err := helper.EncodeStruct(v, connection, "env")
+ if err != nil {
+ return nil, err
+ }
+ response := models.GiteeResponse{
+ GiteeConnection: *connection,
+ Name: "Gitee",
+ ID: 1,
+ }
+
+ return &core.ApiResourceOutput{Body: []models.GiteeResponse{response}}, nil
+}
+
+/*
+GET /plugins/gitee/connections/:connectionId
+*/
+func GetConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) {
+ // RETURN ONLY 1 SOURCE FROM ENV (Ignore ID until multi-connection is developed.)
+ v := config.GetConfig()
+ connection := &models.GiteeConnection{}
+ err := helper.EncodeStruct(v, connection, "env")
+ if err != nil {
+ return nil, err
+ }
+ response := &models.GiteeResponse{
+ GiteeConnection: *connection,
+ Name: "Gitee",
+ ID: 1,
+ }
+ return &core.ApiResourceOutput{Body: response}, nil
+}
diff --git a/plugins/gitee/gitee.go b/plugins/gitee/gitee.go
new file mode 100644
index 00000000..e4faa69f
--- /dev/null
+++ b/plugins/gitee/gitee.go
@@ -0,0 +1,27 @@
+package main
+
+import (
+ "github.com/apache/incubator-devlake/plugins/gitee/impl"
+ "github.com/apache/incubator-devlake/runner"
+ "github.com/spf13/cobra"
+)
+
+var PluginEntry impl.Gitee //nolint
+
+func main() {
+ giteeCmd := &cobra.Command{Use: "gitee"}
+ owner := giteeCmd.Flags().StringP("owner", "o", "", "gitee owner")
+ repo := giteeCmd.Flags().StringP("repo", "r", "", "gitee repo")
+ token := giteeCmd.Flags().StringP("auth", "a", "", "access token")
+ _ = giteeCmd.MarkFlagRequired("owner")
+ _ = giteeCmd.MarkFlagRequired("repo")
+
+ giteeCmd.Run = func(cmd *cobra.Command, args []string) {
+ runner.DirectRun(cmd, args, PluginEntry, map[string]interface{}{
+ "owner": *owner,
+ "repo": *repo,
+ "token": *token,
+ })
+ }
+ runner.RunCmd(giteeCmd)
+}
diff --git a/plugins/gitee/impl/impl.go b/plugins/gitee/impl/impl.go
new file mode 100644
index 00000000..b452a44d
--- /dev/null
+++ b/plugins/gitee/impl/impl.go
@@ -0,0 +1,104 @@
+package impl
+
+import (
+ "github.com/apache/incubator-devlake/migration"
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/gitee/api"
+ "github.com/apache/incubator-devlake/plugins/gitee/models/migrationscripts"
+ "github.com/apache/incubator-devlake/plugins/gitee/tasks"
+ "github.com/mitchellh/mapstructure"
+ "github.com/spf13/viper"
+ "gorm.io/gorm"
+)
+
+var _ core.PluginMeta = (*Gitee)(nil)
+var _ core.PluginInit = (*Gitee)(nil)
+var _ core.PluginTask = (*Gitee)(nil)
+var _ core.PluginApi = (*Gitee)(nil)
+var _ core.Migratable = (*Gitee)(nil)
+
+type Gitee string
+
+func (plugin Gitee) Init(config *viper.Viper, logger core.Logger, db *gorm.DB) error {
+ return nil
+}
+
+func (plugin Gitee) Description() string {
+ return "To collect and enrich data from Gitee"
+}
+
+func (plugin Gitee) SubTaskMetas() []core.SubTaskMeta {
+ return []core.SubTaskMeta{
+ tasks.CollectApiRepoMeta,
+ tasks.ExtractApiRepoMeta,
+ tasks.CollectApiIssuesMeta,
+ tasks.ExtractApiIssuesMeta,
+ tasks.CollectCommitsMeta,
+ tasks.ExtractCommitsMeta,
+ tasks.CollectApiPullRequestsMeta,
+ tasks.ExtractApiPullRequestsMeta,
+ tasks.CollectApiIssueCommentsMeta,
+ tasks.ExtractApiIssueCommentsMeta,
+ tasks.CollectApiPullRequestCommitsMeta,
+ tasks.ExtractApiPullRequestCommitsMeta,
+ //tasks.CollectApiPullRequestReviewsMeta,
+ //tasks.ExtractApiPullRequestReviewsMeta,
+ tasks.CollectApiCommitStatsMeta,
+ tasks.ExtractApiCommitStatsMeta,
+ tasks.EnrichPullRequestIssuesMeta,
+ tasks.ConvertRepoMeta,
+ tasks.ConvertIssuesMeta,
+ tasks.ConvertCommitsMeta,
+ tasks.ConvertIssueLabelsMeta,
+ tasks.ConvertPullRequestCommitsMeta,
+ tasks.ConvertPullRequestsMeta,
+ tasks.ConvertPullRequestLabelsMeta,
+ tasks.ConvertPullRequestIssuesMeta,
+ tasks.ConvertUsersMeta,
+ tasks.ConvertIssueCommentsMeta,
+ tasks.ConvertPullRequestCommentsMeta,
+ tasks.ConvertPullRequestsMeta,
+ }
+}
+
+func (plugin Gitee) PrepareTaskData(taskCtx core.TaskContext, options map[string]interface{}) (interface{}, error) {
+ var op tasks.GiteeOptions
+ var err error
+ err = mapstructure.Decode(options, &op)
+ if err != nil {
+ return nil, err
+ }
+
+ apiClient, err := tasks.NewGiteeApiClient(taskCtx)
+ if err != nil {
+ return nil, err
+ }
+
+ return &tasks.GiteeTaskData{
+ Options: &op,
+ ApiClient: apiClient,
+ }, nil
+}
+
+func (plugin Gitee) RootPkgPath() string {
+ return "github.com/apache/incubator-devlake/plugins/gitee"
+}
+
+func (plugin Gitee) MigrationScripts() []migration.Script {
+ return []migration.Script{new(migrationscripts.InitSchemas), new(migrationscripts.InitSchemas)}
+}
+
+func (plugin Gitee) ApiResources() map[string]map[string]core.ApiResourceHandler {
+ return map[string]map[string]core.ApiResourceHandler{
+ "test": {
+ "POST": api.TestConnection,
+ },
+ "connections": {
+ "GET": api.ListConnections,
+ },
+ "connections/:connectionId": {
+ "GET": api.GetConnection,
+ "PATCH": api.PatchConnection,
+ },
+ }
+}
diff --git a/plugins/gitee/models/commit.go b/plugins/gitee/models/commit.go
new file mode 100644
index 00000000..4699d549
--- /dev/null
+++ b/plugins/gitee/models/commit.go
@@ -0,0 +1,47 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package models
+
+import (
+ "time"
+
+ "github.com/apache/incubator-devlake/models/common"
+)
+
+type GiteeCommit struct {
+ Sha string `gorm:"primaryKey;type:varchar(40)"`
+ CommentsUrl string `gorm:"type:varchar(255)"`
+ Message string
+ AuthorId int
+ AuthorName string `gorm:"type:varchar(255)"`
+ AuthorEmail string `gorm:"type:varchar(255)"`
+ AuthoredDate time.Time
+ CommitterId int
+ CommitterName string `gorm:"type:varchar(255)"`
+ CommitterEmail string `gorm:"type:varchar(255)"`
+ CommittedDate time.Time
+ WebUrl string `gorm:"type:varchar(255)"`
+ Additions int `gorm:"comment:Added lines of code"`
+ Deletions int `gorm:"comment:Deleted lines of code"`
+ Total int `gorm:"comment:Sum of added/deleted lines of code"`
+ common.NoPKModel
+}
+
+func (GiteeCommit) TableName() string {
+ return "_tool_gitee_commits"
+}
diff --git a/plugins/gitee/models/commit_stats.go b/plugins/gitee/models/commit_stats.go
new file mode 100644
index 00000000..d4992490
--- /dev/null
+++ b/plugins/gitee/models/commit_stats.go
@@ -0,0 +1,36 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package models
+
+import (
+ "time"
+
+ "github.com/apache/incubator-devlake/models/common"
+)
+
+type GiteeCommitStat struct {
+ Sha string `gorm:"primaryKey;type:varchar(40)"`
+ Additions int `gorm:"comment:Added lines of code"`
+ Deletions int `gorm:"comment:Deleted lines of code"`
+ CommittedDate time.Time `gorm:"index"`
+ common.NoPKModel
+}
+
+func (GiteeCommitStat) TableName() string {
+ return "_tool_gitee_commit_stats"
+}
diff --git a/plugins/gitee/models/connection.go b/plugins/gitee/models/connection.go
new file mode 100644
index 00000000..24ad1b6d
--- /dev/null
+++ b/plugins/gitee/models/connection.go
@@ -0,0 +1,64 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package models
+
+// This object conforms to what the frontend currently sends.
+type GiteeConnection struct {
+ Endpoint string `mapstructure:"endpoint" validate:"required" env:"GITEE_ENDPOINT" json:"endpoint"`
+ Auth string `mapstructure:"auth" validate:"required" env:"GITEE_AUTH" json:"auth"`
+ Proxy string `mapstructure:"proxy" env:"GITEE_PROXY" json:"proxy"`
+}
+
+// This object conforms to what the frontend currently expects.
+type GiteeResponse struct {
+ Name string `json:"name"`
+ ID int `json:"id"`
+ GiteeConnection
+}
+
+// Using User because it requires authentication.
+type ApiUserResponse struct {
+ Id int
+ Name string `json:"name"`
+}
+
+type TestConnectionRequest struct {
+ Endpoint string `json:"endpoint" validate:"required"`
+ Auth string `json:"auth" validate:"required"`
+ Proxy string `json:"proxy"`
+}
+
+type Config struct {
+ PrType string `mapstructure:"prType" env:"GITEE_PR_TYPE" json:"prType"`
+ PrComponent string `mapstructure:"prComponent" env:"GITEE_PR_COMPONENT" json:"prComponent"`
+ IssueSeverity string `mapstructure:"issueSeverity" env:"GITEE_ISSUE_SEVERITY" json:"issueSeverity"`
+ IssuePriority string `mapstructure:"issuePriority" env:"GITEE_ISSUE_PRIORITY" json:"issuePriority"`
+ IssueComponent string `mapstructure:"issueComponent" env:"GITEE_ISSUE_COMPONENT" json:"issueComponent"`
+ IssueTypeBug string `mapstructure:"issueTypeBug" env:"GITEE_ISSUE_TYPE_BUG" json:"issueTypeBug"`
+ IssueTypeIncident string `mapstructure:"issueTypeIncident" env:"GITEE_ISSUE_TYPE_INCIDENT" json:"issueTypeIncident"`
+ IssueTypeRequirement string `mapstructure:"issueTypeRequirement" env:"GITEE_ISSUE_TYPE_REQUIREMENT" json:"issueTypeRequirement"`
+}
+
+// Using Public Email because it requires authentication, and it is public information anyway.
+// We're not using email information for anything here.
+type PublicEmail struct {
+ Email string
+ Primary bool
+ Verified bool
+ Visibility string
+}
diff --git a/plugins/gitee/models/issue.go b/plugins/gitee/models/issue.go
new file mode 100644
index 00000000..aa2ea170
--- /dev/null
+++ b/plugins/gitee/models/issue.go
@@ -0,0 +1,52 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package models
+
+import (
+ "time"
+
+ "github.com/apache/incubator-devlake/models/common"
+)
+
+type GiteeIssue struct {
+ GiteeId int `gorm:"primaryKey"`
+ RepoId int `gorm:"index"`
+ Number string `gorm:"index;comment:Used in API requests ex. api/repo/1/issue/<THIS_NUMBER>"`
+ State string `gorm:"type:varchar(255)"`
+ Title string
+ Body string
+ Priority string `gorm:"type:varchar(255)"`
+ Type string `gorm:"type:varchar(100)"`
+ Status string `gorm:"type:varchar(255)"`
+ AuthorId int
+ AuthorName string `gorm:"type:varchar(255)"`
+ AssigneeId int
+ AssigneeName string `gorm:"type:varchar(255)"`
+ LeadTimeMinutes uint
+ Url string `gorm:"type:varchar(255)"`
+ ClosedAt *time.Time
+ GiteeCreatedAt time.Time
+ GiteeUpdatedAt time.Time `gorm:"index"`
+ Severity string `gorm:"type:varchar(255)"`
+ Component string `gorm:"type:varchar(255)"`
+ common.NoPKModel
+}
+
+func (GiteeIssue) TableName() string {
+ return "_tool_gitee_issues"
+}
diff --git a/plugins/gitee/models/issue_comment.go b/plugins/gitee/models/issue_comment.go
new file mode 100644
index 00000000..9d170040
--- /dev/null
+++ b/plugins/gitee/models/issue_comment.go
@@ -0,0 +1,39 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package models
+
+import (
+ "time"
+
+ "github.com/apache/incubator-devlake/models/common"
+)
+
+type GiteeIssueComment struct {
+ GiteeId int `gorm:"primaryKey"`
+ IssueId int `gorm:"index;comment:References the Issue"`
+ Body string
+ AuthorUsername string `gorm:"type:varchar(255)"`
+ AuthorUserId int
+ GiteeCreatedAt time.Time
+ GiteeUpdatedAt time.Time `gorm:"index"`
+ common.NoPKModel
+}
+
+func (GiteeIssueComment) TableName() string {
+ return "_tool_gitee_issue_comments"
+}
diff --git a/plugins/gitee/models/issue_label.go b/plugins/gitee/models/issue_label.go
new file mode 100644
index 00000000..77f910f9
--- /dev/null
+++ b/plugins/gitee/models/issue_label.go
@@ -0,0 +1,35 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package models
+
+import (
+ "github.com/apache/incubator-devlake/models/common"
+)
+
+// Please note that Issue Labels can also apply to Pull Requests.
+// Pull Requests are considered Issues in Gitee.
+
+type GiteeIssueLabel struct {
+ IssueId int `gorm:"primaryKey;autoIncrement:false"`
+ LabelName string `gorm:"primaryKey;type:varchar(255)"`
+ common.NoPKModel
+}
+
+func (GiteeIssueLabel) TableName() string {
+ return "_tool_gitee_issue_labels"
+}
diff --git a/plugins/gitee/models/migrationscripts/archived/commit.go b/plugins/gitee/models/migrationscripts/archived/commit.go
new file mode 100644
index 00000000..82c2c570
--- /dev/null
+++ b/plugins/gitee/models/migrationscripts/archived/commit.go
@@ -0,0 +1,47 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package archived
+
+import (
+ "time"
+
+ "github.com/apache/incubator-devlake/models/common"
+)
+
+type GiteeCommit struct {
+ Sha string `gorm:"primaryKey;type:varchar(40)"`
+ CommentsUrl string `gorm:"type:varchar(255)"`
+ Message string
+ AuthorId int
+ AuthorName string `gorm:"type:varchar(255)"`
+ AuthorEmail string `gorm:"type:varchar(255)"`
+ AuthoredDate time.Time
+ CommitterId int
+ CommitterName string `gorm:"type:varchar(255)"`
+ CommitterEmail string `gorm:"type:varchar(255)"`
+ CommittedDate time.Time
+ WebUrl string `gorm:"type:varchar(255)"`
+ Additions int `gorm:"comment:Added lines of code"`
+ Deletions int `gorm:"comment:Deleted lines of code"`
+ Total int `gorm:"comment:Sum of added/deleted lines of code"`
+ common.NoPKModel
+}
+
+func (GiteeCommit) TableName() string {
+ return "_tool_gitee_commits"
+}
diff --git a/plugins/gitee/models/migrationscripts/archived/commit_stat.go b/plugins/gitee/models/migrationscripts/archived/commit_stat.go
new file mode 100644
index 00000000..77a9e0a4
--- /dev/null
+++ b/plugins/gitee/models/migrationscripts/archived/commit_stat.go
@@ -0,0 +1,36 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package archived
+
+import (
+ "time"
+
+ "github.com/apache/incubator-devlake/models/migrationscripts/archived"
+)
+
+type GiteeCommitStat struct {
+ Sha string `gorm:"primaryKey;type:varchar(40)"`
+ Additions int `gorm:"comment:Added lines of code"`
+ Deletions int `gorm:"comment:Deleted lines of code"`
+ CommittedDate time.Time `gorm:"index"`
+ archived.NoPKModel
+}
+
+func (GiteeCommitStat) TableName() string {
+ return "_tool_gitee_commit_stats"
+}
diff --git a/plugins/gitee/models/migrationscripts/archived/issue.go b/plugins/gitee/models/migrationscripts/archived/issue.go
new file mode 100644
index 00000000..5d9e6240
--- /dev/null
+++ b/plugins/gitee/models/migrationscripts/archived/issue.go
@@ -0,0 +1,52 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package archived
+
+import (
+ "time"
+
+ "github.com/apache/incubator-devlake/models/migrationscripts/archived"
+)
+
+type GiteeIssue struct {
+ GiteeId int `gorm:"primaryKey"`
+ RepoId int `gorm:"index"`
+ Number string `gorm:"index;comment:Used in API requests ex. api/repo/1/issue/<THIS_NUMBER>"`
+ State string `gorm:"type:varchar(255)"`
+ Title string
+ Body string
+ Priority string `gorm:"type:varchar(255)"`
+ Type string `gorm:"type:varchar(100)"`
+ Status string `gorm:"type:varchar(255)"`
+ AuthorId int
+ AuthorName string `gorm:"type:varchar(255)"`
+ AssigneeId int
+ AssigneeName string `gorm:"type:varchar(255)"`
+ LeadTimeMinutes uint
+ Url string `gorm:"type:varchar(255)"`
+ ClosedAt *time.Time
+ GiteeCreatedAt time.Time
+ GiteeUpdatedAt time.Time `gorm:"index"`
+ Severity string `gorm:"type:varchar(255)"`
+ Component string `gorm:"type:varchar(255)"`
+ archived.NoPKModel
+}
+
+func (GiteeIssue) TableName() string {
+ return "_tool_gitee_issues"
+}
diff --git a/plugins/gitee/models/migrationscripts/archived/issue_comment.go b/plugins/gitee/models/migrationscripts/archived/issue_comment.go
new file mode 100644
index 00000000..ea40c2d9
--- /dev/null
+++ b/plugins/gitee/models/migrationscripts/archived/issue_comment.go
@@ -0,0 +1,39 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package archived
+
+import (
+ "time"
+
+ "github.com/apache/incubator-devlake/models/migrationscripts/archived"
+)
+
+type GiteeIssueComment struct {
+ GiteeId int `gorm:"primaryKey"`
+ IssueId int `gorm:"index;comment:References the Issue"`
+ Body string
+ AuthorUsername string `gorm:"type:varchar(255)"`
+ AuthorUserId int
+ GiteeCreatedAt time.Time
+ GiteeUpdatedAt time.Time `gorm:"index"`
+ archived.NoPKModel
+}
+
+func (GiteeIssueComment) TableName() string {
+ return "_tool_gitee_issue_comments"
+}
diff --git a/plugins/gitee/models/migrationscripts/archived/issue_label.go b/plugins/gitee/models/migrationscripts/archived/issue_label.go
new file mode 100644
index 00000000..9db73278
--- /dev/null
+++ b/plugins/gitee/models/migrationscripts/archived/issue_label.go
@@ -0,0 +1,33 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package archived
+
+import "github.com/apache/incubator-devlake/models/migrationscripts/archived"
+
+// Please note that Issue Labels can also apply to Pull Requests.
+// Pull Requests are considered Issues in Gitee.
+
+type GiteeIssueLabel struct {
+ IssueId int `gorm:"primaryKey;autoIncrement:false"`
+ LabelName string `gorm:"primaryKey;type:varchar(255)"`
+ archived.NoPKModel
+}
+
+func (GiteeIssueLabel) TableName() string {
+ return "_tool_gitee_issue_labels"
+}
diff --git a/plugins/gitee/models/migrationscripts/archived/pull_request.go b/plugins/gitee/models/migrationscripts/archived/pull_request.go
new file mode 100644
index 00000000..39ced959
--- /dev/null
+++ b/plugins/gitee/models/migrationscripts/archived/pull_request.go
@@ -0,0 +1,59 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package archived
+
+import (
+ "time"
+
+ "github.com/apache/incubator-devlake/models/migrationscripts/archived"
+)
+
+type GiteePullRequest struct {
+ GiteeId int `gorm:"primaryKey"`
+ RepoId int `gorm:"index"`
+ Number int `gorm:"index"` // This number is used in GET requests to the API associated to reviewers / comments / etc.
+ State string `gorm:"type:varchar(255)"`
+ Title string `gorm:"type:varchar(255)"`
+ GiteeCreatedAt time.Time
+ GiteeUpdatedAt time.Time `gorm:"index"`
+ ClosedAt *time.Time
+ // In order to get the following fields, we need to collect PRs individually from Gitee
+ Additions int
+ Deletions int
+ Comments int
+ Commits int
+ ReviewComments int
+ Merged bool
+ MergedAt *time.Time
+ Body string
+ Type string `gorm:"type:varchar(255)"`
+ Component string `gorm:"type:varchar(255)"`
+ MergeCommitSha string `gorm:"type:varchar(40)"`
+ HeadRef string `gorm:"type:varchar(255)"`
+ BaseRef string `gorm:"type:varchar(255)"`
+ BaseCommitSha string `gorm:"type:varchar(255)"`
+ HeadCommitSha string `gorm:"type:varchar(255)"`
+ Url string `gorm:"type:varchar(255)"`
+ AuthorName string `gorm:"type:varchar(100)"`
+ AuthorId int
+ archived.NoPKModel
+}
+
+func (GiteePullRequest) TableName() string {
+ return "_tool_gitee_pull_requests"
+}
diff --git a/plugins/gitee/models/migrationscripts/archived/pull_request_comment.go b/plugins/gitee/models/migrationscripts/archived/pull_request_comment.go
new file mode 100644
index 00000000..79bf54be
--- /dev/null
+++ b/plugins/gitee/models/migrationscripts/archived/pull_request_comment.go
@@ -0,0 +1,39 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package archived
+
+import (
+ "time"
+
+ "github.com/apache/incubator-devlake/models/migrationscripts/archived"
+)
+
+type GiteePullRequestComment struct {
+ GiteeId int `gorm:"primaryKey"`
+ PullRequestId int `gorm:"index"`
+ Body string
+ AuthorUsername string `gorm:"type:varchar(255)"`
+ AuthorUserId int
+ GiteeCreatedAt time.Time
+ GiteeUpdatedAt time.Time `gorm:"index"`
+ archived.NoPKModel
+}
+
+func (GiteePullRequestComment) TableName() string {
+ return "_tool_gitee_pull_request_comments"
+}
diff --git a/plugins/gitee/models/migrationscripts/archived/pull_request_commit.go b/plugins/gitee/models/migrationscripts/archived/pull_request_commit.go
new file mode 100644
index 00000000..54b3617f
--- /dev/null
+++ b/plugins/gitee/models/migrationscripts/archived/pull_request_commit.go
@@ -0,0 +1,30 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package archived
+
+import "github.com/apache/incubator-devlake/models/migrationscripts/archived"
+
+type GiteePullRequestCommit struct {
+ CommitSha string `gorm:"primaryKey;type:varchar(40)"`
+ PullRequestId int `gorm:"primaryKey;autoIncrement:false"`
+ archived.NoPKModel
+}
+
+func (GiteePullRequestCommit) TableName() string {
+ return "_tool_gitee_pull_request_commits"
+}
diff --git a/plugins/gitee/models/migrationscripts/archived/pull_request_issue.go b/plugins/gitee/models/migrationscripts/archived/pull_request_issue.go
new file mode 100644
index 00000000..20e3d13e
--- /dev/null
+++ b/plugins/gitee/models/migrationscripts/archived/pull_request_issue.go
@@ -0,0 +1,32 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package archived
+
+import "github.com/apache/incubator-devlake/models/migrationscripts/archived"
+
+type GiteePullRequestIssue struct {
+ PullRequestId int `gorm:"primaryKey"`
+ IssueId int `gorm:"primaryKey"`
+ PullRequestNumber int
+ IssueNumber int
+ archived.NoPKModel
+}
+
+func (GiteePullRequestIssue) TableName() string {
+ return "_tool_gitee_pull_request_issues"
+}
diff --git a/plugins/gitee/models/migrationscripts/archived/pull_request_label.go b/plugins/gitee/models/migrationscripts/archived/pull_request_label.go
new file mode 100644
index 00000000..69f25c7e
--- /dev/null
+++ b/plugins/gitee/models/migrationscripts/archived/pull_request_label.go
@@ -0,0 +1,30 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package archived
+
+import "github.com/apache/incubator-devlake/models/migrationscripts/archived"
+
+type GiteePullRequestLabel struct {
+ PullId int `gorm:"primaryKey;autoIncrement:false"`
+ LabelName string `gorm:"primaryKey;type:varchar(255)"`
+ archived.NoPKModel
+}
+
+func (GiteePullRequestLabel) TableName() string {
+ return "_tool_gitee_pull_request_labels"
+}
diff --git a/plugins/gitee/models/migrationscripts/archived/repo.go b/plugins/gitee/models/migrationscripts/archived/repo.go
new file mode 100644
index 00000000..989aeae2
--- /dev/null
+++ b/plugins/gitee/models/migrationscripts/archived/repo.go
@@ -0,0 +1,43 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package archived
+
+import (
+ "time"
+
+ "github.com/apache/incubator-devlake/models/migrationscripts/archived"
+)
+
+type GiteeRepo struct {
+ GiteeId int `gorm:"primaryKey"`
+ Name string `gorm:"type:varchar(255)"`
+ HTMLUrl string `gorm:"type:varchar(255)"`
+ Description string
+ OwnerId int `json:"ownerId"`
+ OwnerLogin string `json:"ownerLogin" gorm:"type:varchar(255)"`
+ Language string `json:"language" gorm:"type:varchar(255)"`
+ ParentGiteeId int `json:"parentId"`
+ ParentHTMLUrl string `json:"parentHtmlUrl"`
+ CreatedDate time.Time `json:"createdDate"`
+ UpdatedDate *time.Time `json:"updatedDate"`
+ archived.NoPKModel
+}
+
+func (GiteeRepo) TableName() string {
+ return "_tool_gitee_repo"
+}
diff --git a/plugins/gitee/models/migrationscripts/archived/repo_commit.go b/plugins/gitee/models/migrationscripts/archived/repo_commit.go
new file mode 100644
index 00000000..fa70dfb2
--- /dev/null
+++ b/plugins/gitee/models/migrationscripts/archived/repo_commit.go
@@ -0,0 +1,32 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package archived
+
+import (
+ "github.com/apache/incubator-devlake/models/migrationscripts/archived"
+)
+
+type GiteeRepoCommit struct {
+ RepoId int `gorm:"primaryKey"`
+ CommitSha string `gorm:"primaryKey;type:varchar(40)"`
+ archived.NoPKModel
+}
+
+func (GiteeRepoCommit) TableName() string {
+ return "_tool_gitee_repo_commits"
+}
diff --git a/plugins/gitee/models/migrationscripts/archived/reviewer.go b/plugins/gitee/models/migrationscripts/archived/reviewer.go
new file mode 100644
index 00000000..69aeee6b
--- /dev/null
+++ b/plugins/gitee/models/migrationscripts/archived/reviewer.go
@@ -0,0 +1,32 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package archived
+
+import "github.com/apache/incubator-devlake/models/migrationscripts/archived"
+
+type GiteeReviewer struct {
+ GiteeId int `gorm:"primaryKey"`
+ Login string `gorm:"type:varchar(255)"`
+ PullRequestId int `gorm:"primaryKey"`
+
+ archived.NoPKModel
+}
+
+func (GiteeReviewer) TableName() string {
+ return "_tool_gitee_reviewers"
+}
diff --git a/plugins/gitee/models/migrationscripts/archived/user.go b/plugins/gitee/models/migrationscripts/archived/user.go
new file mode 100644
index 00000000..729b484d
--- /dev/null
+++ b/plugins/gitee/models/migrationscripts/archived/user.go
@@ -0,0 +1,47 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package archived
+
+import (
+ "github.com/apache/incubator-devlake/models/migrationscripts/archived"
+)
+
+type GiteeUser struct {
+ Id int `json:"id" gorm:"primaryKey"`
+ Login string `json:"login" gorm:"type:varchar(255)"`
+ Name string `json:"name" gorm:"type:varchar(255)"`
+ AvatarUrl string `json:"avatar_url" gorm:"type:varchar(255)"`
+ EventsUrl string `json:"events_url" gorm:"type:varchar(255)"`
+ FollowersUrl string `json:"followers_url" gorm:"type:varchar(255)"`
+ FollowingUrl string `json:"following_url" gorm:"type:varchar(255)"`
+ GistsUrl string `json:"gists_url" gorm:"type:varchar(255)"`
+ HtmlUrl string `json:"html_url" gorm:"type:varchar(255)"`
+ OrganizationsUrl string `json:"organizations_url" gorm:"type:varchar(255)"`
+ ReceivedEventsUrl string `json:"received_events_url" gorm:"type:varchar(255)"`
+ Remark string `json:"remark" gorm:"type:varchar(255)"`
+ ReposUrl string `json:"repos_url" gorm:"type:varchar(255)"`
+ StarredUrl string `json:"starred_url" gorm:"type:varchar(255)"`
+ SubscriptionsUrl string `json:"subscriptions_url" gorm:"type:varchar(255)"`
+ Url string `json:"url" gorm:"type:varchar(255)"`
+ Type string `json:"type" gorm:"type:varchar(255)"`
+ archived.NoPKModel
+}
+
+func (GiteeUser) TableName() string {
+ return "_tool_gitee_users"
+}
diff --git a/plugins/gitee/models/migrationscripts/init_schema.go b/plugins/gitee/models/migrationscripts/init_schema.go
new file mode 100644
index 00000000..5eceb089
--- /dev/null
+++ b/plugins/gitee/models/migrationscripts/init_schema.go
@@ -0,0 +1,54 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+ "context"
+
+ "github.com/apache/incubator-devlake/plugins/gitee/models/migrationscripts/archived"
+ "gorm.io/gorm"
+)
+
+type InitSchemas struct{}
+
+func (*InitSchemas) Up(ctx context.Context, db *gorm.DB) error {
+ return db.Migrator().AutoMigrate(
+ &archived.GiteeRepo{},
+ &archived.GiteeCommit{},
+ &archived.GiteeRepoCommit{},
+ &archived.GiteePullRequest{},
+ &archived.GiteePullRequestLabel{},
+ &archived.GiteeUser{},
+ &archived.GiteePullRequestComment{},
+ &archived.GiteeIssue{},
+ &archived.GiteeIssueComment{},
+ &archived.GiteeCommitStat{},
+ &archived.GiteeIssueLabel{},
+ &archived.GiteePullRequestCommit{},
+ &archived.GiteePullRequestIssue{},
+ &archived.GiteeReviewer{},
+ )
+}
+
+func (*InitSchemas) Version() uint64 {
+ return 20220407201202
+}
+
+func (*InitSchemas) Name() string {
+ return "Gitee init schemas"
+}
diff --git a/plugins/gitee/models/pull_request.go b/plugins/gitee/models/pull_request.go
new file mode 100644
index 00000000..7fb1caaf
--- /dev/null
+++ b/plugins/gitee/models/pull_request.go
@@ -0,0 +1,59 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package models
+
+import (
+ "time"
+
+ "github.com/apache/incubator-devlake/models/common"
+)
+
+type GiteePullRequest struct {
+ GiteeId int `gorm:"primaryKey"`
+ RepoId int `gorm:"index"`
+ Number int `gorm:"index"` // This number is used in GET requests to the API associated to reviewers / comments / etc.
+ State string `gorm:"type:varchar(255)"`
+ Title string `gorm:"type:varchar(255)"`
+ GiteeCreatedAt time.Time
+ GiteeUpdatedAt time.Time `gorm:"index"`
+ ClosedAt *time.Time
+ // In order to get the following fields, we need to collect PRs individually from Gitee
+ Additions int
+ Deletions int
+ Comments int
+ Commits int
+ ReviewComments int
+ Merged bool
+ MergedAt *time.Time
+ Body string
+ Type string `gorm:"type:varchar(255)"`
+ Component string `gorm:"type:varchar(255)"`
+ MergeCommitSha string `gorm:"type:varchar(40)"`
+ HeadRef string `gorm:"type:varchar(255)"`
+ BaseRef string `gorm:"type:varchar(255)"`
+ BaseCommitSha string `gorm:"type:varchar(255)"`
+ HeadCommitSha string `gorm:"type:varchar(255)"`
+ Url string `gorm:"type:varchar(255)"`
+ AuthorName string `gorm:"type:varchar(100)"`
+ AuthorId int
+ common.NoPKModel
+}
+
+func (GiteePullRequest) TableName() string {
+ return "_tool_gitee_pull_requests"
+}
diff --git a/plugins/gitee/models/pull_request_comment.go b/plugins/gitee/models/pull_request_comment.go
new file mode 100644
index 00000000..6e0b804a
--- /dev/null
+++ b/plugins/gitee/models/pull_request_comment.go
@@ -0,0 +1,39 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package models
+
+import (
+ "time"
+
+ "github.com/apache/incubator-devlake/models/common"
+)
+
+type GiteePullRequestComment struct {
+ GiteeId int `gorm:"primaryKey"`
+ PullRequestId int `gorm:"index"`
+ Body string
+ AuthorUsername string `gorm:"type:varchar(255)"`
+ AuthorUserId int
+ GiteeCreatedAt time.Time
+ GiteeUpdatedAt time.Time `gorm:"index"`
+ common.NoPKModel
+}
+
+func (GiteePullRequestComment) TableName() string {
+ return "_tool_gitee_pull_request_comments"
+}
diff --git a/plugins/gitee/models/pull_request_commit.go b/plugins/gitee/models/pull_request_commit.go
new file mode 100644
index 00000000..6705b665
--- /dev/null
+++ b/plugins/gitee/models/pull_request_commit.go
@@ -0,0 +1,32 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package models
+
+import (
+ "github.com/apache/incubator-devlake/models/common"
+)
+
+type GiteePullRequestCommit struct {
+ CommitSha string `gorm:"primaryKey;type:varchar(40)"`
+ PullRequestId int `gorm:"primaryKey;autoIncrement:false"`
+ common.NoPKModel
+}
+
+func (GiteePullRequestCommit) TableName() string {
+ return "_tool_gitee_pull_request_commits"
+}
diff --git a/plugins/gitee/models/pull_request_issue.go b/plugins/gitee/models/pull_request_issue.go
new file mode 100644
index 00000000..8527ff4d
--- /dev/null
+++ b/plugins/gitee/models/pull_request_issue.go
@@ -0,0 +1,32 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package models
+
+import "github.com/apache/incubator-devlake/models/common"
+
+type GiteePullRequestIssue struct {
+ PullRequestId int `gorm:"primaryKey"`
+ IssueId int `gorm:"primaryKey"`
+ PullRequestNumber int
+ IssueNumber string
+ common.NoPKModel
+}
+
+func (GiteePullRequestIssue) TableName() string {
+ return "_tool_gitee_pull_request_issues"
+}
diff --git a/plugins/gitee/models/pull_request_label.go b/plugins/gitee/models/pull_request_label.go
new file mode 100644
index 00000000..619eeb0d
--- /dev/null
+++ b/plugins/gitee/models/pull_request_label.go
@@ -0,0 +1,32 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package models
+
+import (
+ "github.com/apache/incubator-devlake/models/common"
+)
+
+type GiteePullRequestLabel struct {
+ PullId int `gorm:"primaryKey;autoIncrement:false"`
+ LabelName string `gorm:"primaryKey;type:varchar(255)"`
+ common.NoPKModel
+}
+
+func (GiteePullRequestLabel) TableName() string {
+ return "_tool_gitee_pull_request_labels"
+}
diff --git a/plugins/gitee/models/repo.go b/plugins/gitee/models/repo.go
new file mode 100644
index 00000000..84a4b493
--- /dev/null
+++ b/plugins/gitee/models/repo.go
@@ -0,0 +1,43 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package models
+
+import (
+ "time"
+
+ "github.com/apache/incubator-devlake/models/common"
+)
+
+type GiteeRepo struct {
+ GiteeId int `gorm:"primaryKey"`
+ Name string `gorm:"type:varchar(255)"`
+ HTMLUrl string `gorm:"type:varchar(255)"`
+ Description string
+ OwnerId int `json:"ownerId"`
+ OwnerLogin string `json:"ownerLogin" gorm:"type:varchar(255)"`
+ Language string `json:"language" gorm:"type:varchar(255)"`
+ ParentGiteeId int `json:"parentId"`
+ ParentHTMLUrl string `json:"parentHtmlUrl"`
+ CreatedDate time.Time `json:"createdDate"`
+ UpdatedDate *time.Time `json:"updatedDate"`
+ common.NoPKModel
+}
+
+func (GiteeRepo) TableName() string {
+ return "_tool_gitee_repo"
+}
diff --git a/plugins/gitee/models/repo_commit.go b/plugins/gitee/models/repo_commit.go
new file mode 100644
index 00000000..7e66077e
--- /dev/null
+++ b/plugins/gitee/models/repo_commit.go
@@ -0,0 +1,32 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package models
+
+import (
+ "github.com/apache/incubator-devlake/models/common"
+)
+
+type GiteeRepoCommit struct {
+ RepoId int `gorm:"primaryKey"`
+ CommitSha string `gorm:"primaryKey;type:varchar(40)"`
+ common.NoPKModel
+}
+
+func (GiteeRepoCommit) TableName() string {
+ return "_tool_gitee_repo_commits"
+}
diff --git a/plugins/gitee/models/reviewer.go b/plugins/gitee/models/reviewer.go
new file mode 100644
index 00000000..fc2c6a1d
--- /dev/null
+++ b/plugins/gitee/models/reviewer.go
@@ -0,0 +1,34 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package models
+
+import (
+ "github.com/apache/incubator-devlake/models/common"
+)
+
+type GiteeReviewer struct {
+ GiteeId int `gorm:"primaryKey"`
+ Login string `gorm:"type:varchar(255)"`
+ PullRequestId int `gorm:"primaryKey"`
+
+ common.NoPKModel
+}
+
+func (GiteeReviewer) TableName() string {
+ return "_tool_gitee_reviewers"
+}
diff --git a/plugins/gitee/models/user.go b/plugins/gitee/models/user.go
new file mode 100644
index 00000000..acf6309e
--- /dev/null
+++ b/plugins/gitee/models/user.go
@@ -0,0 +1,45 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package models
+
+import "github.com/apache/incubator-devlake/models/common"
+
+type GiteeUser struct {
+ Id int `json:"id" gorm:"primaryKey"`
+ Login string `json:"login" gorm:"type:varchar(255)"`
+ Name string `json:"name" gorm:"type:varchar(255)"`
+ AvatarUrl string `json:"avatar_url" gorm:"type:varchar(255)"`
+ EventsUrl string `json:"events_url" gorm:"type:varchar(255)"`
+ FollowersUrl string `json:"followers_url" gorm:"type:varchar(255)"`
+ FollowingUrl string `json:"following_url" gorm:"type:varchar(255)"`
+ GistsUrl string `json:"gists_url" gorm:"type:varchar(255)"`
+ HtmlUrl string `json:"html_url" gorm:"type:varchar(255)"`
+ OrganizationsUrl string `json:"organizations_url" gorm:"type:varchar(255)"`
+ ReceivedEventsUrl string `json:"received_events_url" gorm:"type:varchar(255)"`
+ Remark string `json:"remark" gorm:"type:varchar(255)"`
+ ReposUrl string `json:"repos_url" gorm:"type:varchar(255)"`
+ StarredUrl string `json:"starred_url" gorm:"type:varchar(255)"`
+ SubscriptionsUrl string `json:"subscriptions_url" gorm:"type:varchar(255)"`
+ Url string `json:"url" gorm:"type:varchar(255)"`
+ Type string `json:"type" gorm:"type:varchar(255)"`
+ common.NoPKModel
+}
+
+func (GiteeUser) TableName() string {
+ return "_tool_gitee_users"
+}
diff --git a/plugins/gitee/tasks/api_client.go b/plugins/gitee/tasks/api_client.go
new file mode 100644
index 00000000..94c63256
--- /dev/null
+++ b/plugins/gitee/tasks/api_client.go
@@ -0,0 +1,84 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "fmt"
+ "net/http"
+ "strconv"
+ "time"
+
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/helper"
+ "github.com/apache/incubator-devlake/utils"
+)
+
+func NewGiteeApiClient(taskCtx core.TaskContext) (*helper.ApiAsyncClient, error) {
+ endpoint := taskCtx.GetConfig("GITEE_ENDPOINT")
+ if endpoint == "" {
+ return nil, fmt.Errorf("endpint is required")
+ }
+ userRateLimit, err := utils.StrToIntOr(taskCtx.GetConfig("GITEE_API_REQUESTS_PER_HOUR"), 0)
+ if err != nil {
+ return nil, err
+ }
+ auth := taskCtx.GetConfig("GITEE_AUTH")
+ if auth == "" {
+ return nil, fmt.Errorf("GITEE_AUTH is required")
+ }
+ proxy := taskCtx.GetConfig("GITEE_PROXY")
+
+ headers := map[string]string{}
+
+ apiClient, err := helper.NewApiClient(endpoint, headers, 0, proxy, taskCtx.GetContext())
+ if err != nil {
+ return nil, err
+ }
+ apiClient.SetAfterFunction(func(res *http.Response) error {
+ if res.StatusCode == http.StatusUnauthorized {
+ return fmt.Errorf("authentication failed, please check your Basic Auth Token")
+ }
+ return nil
+ })
+
+ rateLimiter := &helper.ApiRateLimitCalculator{
+ UserRateLimitPerHour: userRateLimit,
+ DynamicRateLimit: func(res *http.Response) (int, time.Duration, error) {
+ rateLimitHeader := res.Header.Get("RateLimit-Limit")
+ if rateLimitHeader == "" {
+ // use default
+ return 0, 0, nil
+ }
+ rateLimit, err := strconv.Atoi(rateLimitHeader)
+ if err != nil {
+ return 0, 0, fmt.Errorf("failed to parse RateLimit-Limit header: %w", err)
+ }
+ // seems like gitlab rate limit is on minute basis
+ return rateLimit, 1 * time.Minute, nil
+ },
+ }
+ asyncApiClient, err := helper.CreateAsyncApiClient(
+ taskCtx,
+ apiClient,
+ rateLimiter,
+ )
+ if err != nil {
+ return nil, err
+ }
+ return asyncApiClient, nil
+}
diff --git a/plugins/gitee/tasks/commit_collector.go b/plugins/gitee/tasks/commit_collector.go
new file mode 100644
index 00000000..dc834e53
--- /dev/null
+++ b/plugins/gitee/tasks/commit_collector.go
@@ -0,0 +1,64 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "net/url"
+ "strconv"
+
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/helper"
+)
+
+const RAW_COMMIT_TABLE = "gitee_api_commit"
+
+var CollectCommitsMeta = core.SubTaskMeta{
+ Name: "collectApiCommits",
+ EntryPoint: CollectApiCommits,
+ EnabledByDefault: true,
+ Description: "Collect commit data from gitee api",
+}
+
+func CollectApiCommits(taskCtx core.SubTaskContext) error {
+ rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_COMMIT_TABLE)
+
+ collector, err := helper.NewApiCollector(helper.ApiCollectorArgs{
+ RawDataSubTaskArgs: *rawDataSubTaskArgs,
+ ApiClient: data.ApiClient,
+ PageSize: 100,
+ Incremental: false,
+ UrlTemplate: "repos/{{ .Params.Owner }}/{{ .Params.Repo }}/commits",
+ Query: func(reqData *helper.RequestData) (url.Values, error) {
+ query := url.Values{}
+ query.Set("access_token", data.Options.Token)
+ query.Set("with_stats", "true")
+ query.Set("sort", "asc")
+ query.Set("page", strconv.Itoa(reqData.Pager.Page))
+ query.Set("per_page", strconv.Itoa(reqData.Pager.Size))
+ return query, nil
+ },
+ Concurrency: 20,
+ ResponseParser: GetRawMessageFromResponse,
+ })
+
+ if err != nil {
+ return err
+ }
+
+ return collector.Execute()
+}
diff --git a/plugins/gitee/tasks/commit_convertor.go b/plugins/gitee/tasks/commit_convertor.go
new file mode 100644
index 00000000..d9405edc
--- /dev/null
+++ b/plugins/gitee/tasks/commit_convertor.go
@@ -0,0 +1,102 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "reflect"
+
+ "github.com/apache/incubator-devlake/models/domainlayer/code"
+ "github.com/apache/incubator-devlake/models/domainlayer/didgen"
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/gitee/models"
+ giteeModels "github.com/apache/incubator-devlake/plugins/gitee/models"
+ "github.com/apache/incubator-devlake/plugins/helper"
+)
+
+var ConvertCommitsMeta = core.SubTaskMeta{
+ Name: "convertApiCommits",
+ EntryPoint: ConvertCommits,
+ EnabledByDefault: true,
+ Description: "Convert tool layer table gitee_commits into domain layer table commits",
+}
+
+func ConvertCommits(taskCtx core.SubTaskContext) error {
+
+ rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_COMMIT_TABLE)
+ db := taskCtx.GetDb()
+ repoId := data.Repo.GiteeId
+
+ // select all commits belongs to the project
+ cursor, err := db.Table("_tool_gitee_commits gc").
+ Joins(`left join _tool_gitee_repo_commits gpc on (
+ gpc.commit_sha = gc.sha
+ )`).
+ Select("gc.*").
+ Where("gpc.repo_id = ?", repoId).
+ Rows()
+ if err != nil {
+ return err
+ }
+ defer cursor.Close()
+
+ // TODO: adopt batch indate operation
+ userDidGen := didgen.NewDomainIdGenerator(&models.GiteeUser{})
+ repoDidGen := didgen.NewDomainIdGenerator(&giteeModels.GiteeRepo{})
+ domainRepoId := repoDidGen.Generate(repoId)
+
+ converter, err := helper.NewDataConverter(helper.DataConverterArgs{
+ RawDataSubTaskArgs: *rawDataSubTaskArgs,
+ InputRowType: reflect.TypeOf(models.GiteeCommit{}),
+ Input: cursor,
+
+ Convert: func(inputRow interface{}) ([]interface{}, error) {
+ giteeCommit := inputRow.(*models.GiteeCommit)
+
+ // convert commit
+ commit := &code.Commit{}
+ commit.Sha = giteeCommit.Sha
+ commit.Message = giteeCommit.Message
+ commit.Additions = giteeCommit.Additions
+ commit.Deletions = giteeCommit.Deletions
+ commit.AuthorId = userDidGen.Generate(giteeCommit.AuthorId)
+ commit.AuthorName = giteeCommit.AuthorName
+ commit.AuthorEmail = giteeCommit.AuthorEmail
+ commit.AuthoredDate = giteeCommit.AuthoredDate
+ commit.CommitterName = giteeCommit.CommitterName
+ commit.CommitterEmail = giteeCommit.CommitterEmail
+ commit.CommittedDate = giteeCommit.CommittedDate
+ commit.CommitterId = userDidGen.Generate(giteeCommit.CommitterId)
+
+ // convert repo / commits relationship
+ repoCommit := &code.RepoCommit{
+ RepoId: domainRepoId,
+ CommitSha: giteeCommit.Sha,
+ }
+
+ return []interface{}{
+ commit,
+ repoCommit,
+ }, nil
+ },
+ })
+ if err != nil {
+ return err
+ }
+
+ return converter.Execute()
+}
diff --git a/plugins/gitee/tasks/commit_extractor.go b/plugins/gitee/tasks/commit_extractor.go
new file mode 100644
index 00000000..484a6230
--- /dev/null
+++ b/plugins/gitee/tasks/commit_extractor.go
@@ -0,0 +1,111 @@
+package tasks
+
+import (
+ "encoding/json"
+
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/gitee/models"
+ "github.com/apache/incubator-devlake/plugins/helper"
+)
+
+var ExtractCommitsMeta = core.SubTaskMeta{
+ Name: "extractApiCommits",
+ EntryPoint: ExtractApiCommits,
+ EnabledByDefault: true,
+ Description: "Extract raw commit data into tool layer table GiteeCommit,GiteeUser and GiteeRepoCommit",
+}
+
+type GiteeCommit struct {
+ Author struct {
+ Date helper.Iso8601Time `json:"date"`
+ Email string `json:"email"`
+ Name string `json:"name"`
+ }
+ Committer struct {
+ Date helper.Iso8601Time `json:"date"`
+ Email string `json:"email"`
+ Name string `json:"name"`
+ }
+ Message string `json:"message"`
+}
+
+type GiteeApiCommitResponse struct {
+ Author *models.GiteeUser `json:"author"`
+ AuthorId int
+ CommentsUrl string `json:"comments_url"`
+ Commit GiteeCommit `json:"commit"`
+ Committer *models.GiteeUser `json:"committer"`
+ HtmlUrl string `json:"html_url"`
+ Sha string `json:"sha"`
+ Url string `json:"url"`
+}
+
+func ExtractApiCommits(taskCtx core.SubTaskContext) error {
+ rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_COMMIT_TABLE)
+
+ extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{
+ RawDataSubTaskArgs: *rawDataSubTaskArgs,
+ Extract: func(row *helper.RawData) ([]interface{}, error) {
+ results := make([]interface{}, 0, 4)
+
+ commit := &GiteeApiCommitResponse{}
+
+ err := json.Unmarshal(row.Data, commit)
+
+ if err != nil {
+ return nil, err
+ }
+
+ if commit.Sha == "" {
+ return nil, nil
+ }
+
+ giteeCommit, err := ConvertCommit(commit)
+
+ if err != nil {
+ return nil, err
+ }
+
+ if commit.Author != nil {
+ giteeCommit.AuthorId = commit.Author.Id
+ results = append(results, commit.Author)
+ }
+ if commit.Committer != nil {
+ giteeCommit.CommitterId = commit.Committer.Id
+ results = append(results, commit.Committer)
+
+ }
+
+ giteeRepoCommit := &models.GiteeRepoCommit{
+ RepoId: data.Repo.GiteeId,
+ CommitSha: commit.Sha,
+ }
+ results = append(results, giteeCommit)
+ results = append(results, giteeRepoCommit)
+ return results, nil
+ },
+ })
+
+ if err != nil {
+ return err
+ }
+
+ return extractor.Execute()
+}
+
+// ConvertCommit Convert the API response to our DB model instance
+func ConvertCommit(commit *GiteeApiCommitResponse) (*models.GiteeCommit, error) {
+ giteeCommit := &models.GiteeCommit{
+ Sha: commit.Sha,
+ AuthorId: commit.Author.Id,
+ Message: commit.Commit.Message,
+ AuthorName: commit.Commit.Author.Name,
+ AuthorEmail: commit.Commit.Author.Email,
+ AuthoredDate: commit.Commit.Author.Date.ToTime(),
+ CommitterName: commit.Commit.Author.Name,
+ CommitterEmail: commit.Commit.Author.Email,
+ CommittedDate: commit.Commit.Author.Date.ToTime(),
+ WebUrl: commit.Url,
+ }
+ return giteeCommit, nil
+}
diff --git a/plugins/gitee/tasks/commit_stats_collector.go b/plugins/gitee/tasks/commit_stats_collector.go
new file mode 100644
index 00000000..cdc23c1d
--- /dev/null
+++ b/plugins/gitee/tasks/commit_stats_collector.go
@@ -0,0 +1,112 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "encoding/json"
+ "fmt"
+ "io/ioutil"
+ "net/http"
+ "net/url"
+ "reflect"
+
+ "github.com/apache/incubator-devlake/plugins/helper"
+
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/gitee/models"
+)
+
+const RAW_COMMIT_STATS_TABLE = "gitee_api_commit_stats"
+
+var CollectApiCommitStatsMeta = core.SubTaskMeta{
+ Name: "collectApiCommitStats",
+ EntryPoint: CollectApiCommitStats,
+ EnabledByDefault: false,
+ Description: "Collect commitStats data from Gitee api",
+}
+
+func CollectApiCommitStats(taskCtx core.SubTaskContext) error {
+ db := taskCtx.GetDb()
+ rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_COMMIT_STATS_TABLE)
+
+ var latestUpdated models.GiteeCommitStat
+ err := db.Model(&latestUpdated).Joins("left join _tool_gitee_repo_commits on _tool_gitee_commit_stats.sha = _tool_gitee_repo_commits.commit_sha").
+ Where("_tool_gitee_repo_commits.repo_id = ?", data.Repo.GiteeId).
+ Order("committed_date DESC").Limit(1).Find(&latestUpdated).Error
+ if err != nil {
+ return fmt.Errorf("failed to get latest gitee commit record: %w", err)
+ }
+
+ cursor, err := db.Model(&models.GiteeCommit{}).
+ Joins("left join _tool_gitee_repo_commits on _tool_gitee_commits.sha = _tool_gitee_repo_commits.commit_sha").
+ Where("_tool_gitee_repo_commits.repo_id = ? and _tool_gitee_commits.committed_date >= ?",
+ data.Repo.GiteeId, latestUpdated.CommittedDate.String()).
+ Rows()
+ if err != nil {
+ return err
+ }
+ iterator, err := helper.NewCursorIterator(db, cursor, reflect.TypeOf(models.GiteeCommit{}))
+ if err != nil {
+ return err
+ }
+
+ collector, err := helper.NewApiCollector(helper.ApiCollectorArgs{
+ RawDataSubTaskArgs: *rawDataSubTaskArgs,
+ ApiClient: data.ApiClient,
+ PageSize: 100,
+ Input: iterator,
+ /*
+ url may use arbitrary variables from different source in any order, we need GoTemplate to allow more
+ flexible for all kinds of possibility.
+ Pager contains information for a particular page, calculated by ApiCollector, and will be passed into
+ GoTemplate to generate a url for that page.
+ We want to do page-fetching in ApiCollector, because the logic are highly similar, by doing so, we can
+ avoid duplicate logic for every tasks, and when we have a better idea like improving performance, we can
+ do it in one place
+ */
+ UrlTemplate: "repos/{{ .Params.Owner }}/{{ .Params.Repo }}/commits/{{ .Input.Sha }}",
+ /*
+ (Optional) Return query string for request, or you can plug them into UrlTemplate directly
+ */
+ Query: func(reqData *helper.RequestData) (url.Values, error) {
+ query := url.Values{}
+ query.Set("access_token", data.Options.Token)
+ query.Set("state", "all")
+ query.Set("direction", "asc")
+ query.Set("page", fmt.Sprintf("%v", reqData.Pager.Page))
+ query.Set("per_page", fmt.Sprintf("%v", reqData.Pager.Size))
+
+ return query, nil
+ },
+
+ ResponseParser: func(res *http.Response) ([]json.RawMessage, error) {
+ body, err := ioutil.ReadAll(res.Body)
+ res.Body.Close()
+ if err != nil {
+ return nil, err
+ }
+ return []json.RawMessage{body}, nil
+ },
+ })
+
+ if err != nil {
+ return err
+ }
+
+ return collector.Execute()
+}
diff --git a/plugins/gitee/tasks/commit_stats_extractor.go b/plugins/gitee/tasks/commit_stats_extractor.go
new file mode 100644
index 00000000..680b01a6
--- /dev/null
+++ b/plugins/gitee/tasks/commit_stats_extractor.go
@@ -0,0 +1,98 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "encoding/json"
+
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/gitee/models"
+ "github.com/apache/incubator-devlake/plugins/helper"
+)
+
+var ExtractApiCommitStatsMeta = core.SubTaskMeta{
+ Name: "extractApiCommitStats",
+ EntryPoint: ExtractApiCommitStats,
+ EnabledByDefault: false,
+ Description: "Extract raw commit stats data into tool layer table gitee_commit_stats",
+}
+
+type ApiSingleCommitResponse struct {
+ Sha string
+ Stats struct {
+ id string
+ Additions int
+ Deletions int
+ total int
+ }
+ Commit struct {
+ Committer struct {
+ Name string
+ Email string
+ Date helper.Iso8601Time
+ }
+ }
+}
+
+func ExtractApiCommitStats(taskCtx core.SubTaskContext) error {
+ rawDataSubTaskArgs, _ := CreateRawDataSubTaskArgs(taskCtx, RAW_COMMIT_STATS_TABLE)
+
+ extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{
+ RawDataSubTaskArgs: *rawDataSubTaskArgs,
+ Extract: func(row *helper.RawData) ([]interface{}, error) {
+ body := &ApiSingleCommitResponse{}
+ err := json.Unmarshal(row.Data, body)
+ if err != nil {
+ return nil, err
+ }
+ if body.Sha == "" {
+ return nil, nil
+ }
+
+ db := taskCtx.GetDb()
+ commit := &models.GiteeCommit{}
+ err = db.Model(commit).Where("sha = ?", body.Sha).Limit(1).Find(commit).Error
+ if err != nil {
+ return nil, err
+ }
+
+ commit.Additions = body.Stats.Additions
+ commit.Deletions = body.Stats.Deletions
+
+ commitStat := &models.GiteeCommitStat{
+ Additions: body.Stats.Additions,
+ Deletions: body.Stats.Deletions,
+ CommittedDate: body.Commit.Committer.Date.ToTime(),
+ Sha: body.Sha,
+ }
+
+ results := make([]interface{}, 0, 2)
+
+ results = append(results, commit)
+ results = append(results, commitStat)
+
+ return results, nil
+ },
+ })
+
+ if err != nil {
+ return err
+ }
+
+ return extractor.Execute()
+}
diff --git a/plugins/gitee/tasks/issue_collector.go b/plugins/gitee/tasks/issue_collector.go
new file mode 100644
index 00000000..4b882233
--- /dev/null
+++ b/plugins/gitee/tasks/issue_collector.go
@@ -0,0 +1,97 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "encoding/json"
+ "fmt"
+ "net/http"
+ "net/url"
+
+ "github.com/apache/incubator-devlake/plugins/helper"
+
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/gitee/models"
+)
+
+const RAW_ISSUE_TABLE = "gitee_api_issues"
+
+var CollectApiIssuesMeta = core.SubTaskMeta{
+ Name: "collectApiIssues",
+ EntryPoint: CollectApiIssues,
+ EnabledByDefault: true,
+ Description: "Collect issues data from Gitee api",
+}
+
+func CollectApiIssues(taskCtx core.SubTaskContext) error {
+ db := taskCtx.GetDb()
+ rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_ISSUE_TABLE)
+
+ since := data.Since
+ incremental := false
+ // user didn't specify a time range to sync, try load from database
+ if since == nil {
+ var latestUpdated models.GiteeIssue
+ err := db.Model(&latestUpdated).
+ Where("repo_id = ?", data.Repo.GiteeId).
+ Order("gitee_updated_at DESC").Limit(1).Find(&latestUpdated).Error
+ if err != nil {
+ return fmt.Errorf("failed to get latest gitee issue record: %w", err)
+ }
+ if latestUpdated.GiteeId > 0 {
+ since = &latestUpdated.GiteeUpdatedAt
+ incremental = true
+ }
+ }
+
+ collector, err := helper.NewApiCollector(helper.ApiCollectorArgs{
+ RawDataSubTaskArgs: *rawDataSubTaskArgs,
+ ApiClient: data.ApiClient,
+ PageSize: 100,
+ Incremental: incremental,
+ UrlTemplate: "repos/{{ .Params.Owner }}/{{ .Params.Repo }}/issues",
+ Query: func(reqData *helper.RequestData) (url.Values, error) {
+ query := url.Values{}
+ query.Set("access_token", data.Options.Token)
+ query.Set("state", "all")
+ if since != nil {
+ query.Set("since", since.String())
+ }
+ query.Set("direction", "asc")
+ query.Set("page", fmt.Sprintf("%v", reqData.Pager.Page))
+ query.Set("per_page", fmt.Sprintf("%v", reqData.Pager.Size))
+
+ return query, nil
+ },
+ GetTotalPages: GetTotalPagesFromResponse,
+ ResponseParser: func(res *http.Response) ([]json.RawMessage, error) {
+ var items []json.RawMessage
+ err := helper.UnmarshalResponse(res, &items)
+ if err != nil {
+ return nil, err
+ }
+ return items, nil
+ },
+ })
+
+ if err != nil {
+ return err
+ }
+
+ return collector.Execute()
+}
diff --git a/plugins/gitee/tasks/issue_comment_collector.go b/plugins/gitee/tasks/issue_comment_collector.go
new file mode 100644
index 00000000..a74b08fb
--- /dev/null
+++ b/plugins/gitee/tasks/issue_comment_collector.go
@@ -0,0 +1,110 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "fmt"
+ "net/url"
+
+ "github.com/apache/incubator-devlake/plugins/helper"
+
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/gitee/models"
+)
+
+const RAW_COMMENTS_TABLE = "gitee_issue_comments"
+
+var CollectApiIssueCommentsMeta = core.SubTaskMeta{
+ Name: "collectApiIssueComments",
+ EntryPoint: CollectApiIssueComments,
+ EnabledByDefault: true,
+ Description: "Collect comments data from Gitee api",
+}
+
+func CollectApiIssueComments(taskCtx core.SubTaskContext) error {
+ db := taskCtx.GetDb()
+ rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_COMMENTS_TABLE)
+
+ since := data.Since
+ incremental := false
+ // user didn't specify a time range to sync, try load from database
+ // actually, for gitee pull, since doesn't make any sense, gitee pull api doesn't support it
+ if since == nil {
+ var latestUpdatedIssueComment models.GiteeIssueComment
+ err := db.Model(&latestUpdatedIssueComment).
+ Joins("left join _tool_gitee_issues on _tool_gitee_issues.gitee_id = _tool_gitee_issue_comments.issue_id").
+ Where("_tool_gitee_issues.repo_id = ?", data.Repo.GiteeId).
+ Order("gitee_updated_at DESC").Limit(1).Find(&latestUpdatedIssueComment).Error
+ if err != nil {
+ return fmt.Errorf("failed to get latest gitee issue record: %w", err)
+ }
+ var latestUpdatedPrComt models.GiteePullRequestComment
+ err = db.Model(&latestUpdatedPrComt).
+ Joins("left join _tool_gitee_pull_requests on _tool_gitee_pull_requests.gitee_id = _tool_gitee_pull_request_comments.pull_request_id").
+ Where("_tool_gitee_pull_requests.repo_id = ?", data.Repo.GiteeId).
+ Order("gitee_updated_at DESC").Limit(1).Find(&latestUpdatedPrComt).Error
+ if err != nil {
+ return fmt.Errorf("failed to get latest gitee issue record: %w", err)
+ }
+ if latestUpdatedIssueComment.GiteeId > 0 && latestUpdatedPrComt.GiteeId > 0 {
+ if latestUpdatedIssueComment.GiteeUpdatedAt.Before(latestUpdatedPrComt.GiteeUpdatedAt) {
+ since = &latestUpdatedPrComt.GiteeUpdatedAt
+ } else {
+ since = &latestUpdatedIssueComment.GiteeUpdatedAt
+ }
+ incremental = true
+ } else if latestUpdatedIssueComment.GiteeId > 0 {
+ since = &latestUpdatedIssueComment.GiteeUpdatedAt
+ incremental = true
+ } else if latestUpdatedPrComt.GiteeId > 0 {
+ since = &latestUpdatedPrComt.GiteeUpdatedAt
+ incremental = true
+ }
+
+ }
+
+ collector, err := helper.NewApiCollector(helper.ApiCollectorArgs{
+ RawDataSubTaskArgs: *rawDataSubTaskArgs,
+ ApiClient: data.ApiClient,
+ PageSize: 100,
+ Incremental: incremental,
+
+ UrlTemplate: "repos/{{ .Params.Owner }}/{{ .Params.Repo }}/issues/comments",
+ Query: func(reqData *helper.RequestData) (url.Values, error) {
+ query := url.Values{}
+ query.Set("access_token", data.Options.Token)
+ query.Set("state", "all")
+ if since != nil {
+ query.Set("since", since.String())
+ }
+ query.Set("page", fmt.Sprintf("%v", reqData.Pager.Page))
+ query.Set("direction", "asc")
+ query.Set("per_page", fmt.Sprintf("%v", reqData.Pager.Size))
+
+ return query, nil
+ },
+ GetTotalPages: GetTotalPagesFromResponse,
+ ResponseParser: GetRawMessageFromResponse,
+ })
+
+ if err != nil {
+ return err
+ }
+
+ return collector.Execute()
+}
diff --git a/plugins/gitee/tasks/issue_comment_convertor.go b/plugins/gitee/tasks/issue_comment_convertor.go
new file mode 100644
index 00000000..279d6846
--- /dev/null
+++ b/plugins/gitee/tasks/issue_comment_convertor.go
@@ -0,0 +1,80 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "reflect"
+
+ "github.com/apache/incubator-devlake/models/domainlayer"
+ "github.com/apache/incubator-devlake/models/domainlayer/didgen"
+ "github.com/apache/incubator-devlake/models/domainlayer/ticket"
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/gitee/models"
+ "github.com/apache/incubator-devlake/plugins/helper"
+)
+
+var ConvertIssueCommentsMeta = core.SubTaskMeta{
+ Name: "convertIssueComments",
+ EntryPoint: ConvertIssueComments,
+ EnabledByDefault: true,
+ Description: "ConvertIssueComments data from Gitee api",
+}
+
+func ConvertIssueComments(taskCtx core.SubTaskContext) error {
+ db := taskCtx.GetDb()
+ rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_COMMENTS_TABLE)
+ repoId := data.Repo.GiteeId
+
+ cursor, err := db.Model(&models.GiteeIssueComment{}).
+ Joins("left join _tool_gitee_issues "+
+ "on _tool_gitee_issues.gitee_id = _tool_gitee_issue_comments.issue_id").
+ Where("repo_id = ?", repoId).Rows()
+ if err != nil {
+ return err
+ }
+ defer cursor.Close()
+
+ issueIdGen := didgen.NewDomainIdGenerator(&models.GiteeIssue{})
+ userIdGen := didgen.NewDomainIdGenerator(&models.GiteeUser{})
+
+ converter, err := helper.NewDataConverter(helper.DataConverterArgs{
+ InputRowType: reflect.TypeOf(models.GiteeIssueComment{}),
+ Input: cursor,
+ RawDataSubTaskArgs: *rawDataSubTaskArgs,
+ Convert: func(inputRow interface{}) ([]interface{}, error) {
+ giteeIssueComment := inputRow.(*models.GiteeIssueComment)
+ domainIssueComment := &ticket.IssueComment{
+ DomainEntity: domainlayer.DomainEntity{
+ Id: issueIdGen.Generate(giteeIssueComment.GiteeId),
+ },
+ IssueId: issueIdGen.Generate(giteeIssueComment.IssueId),
+ Body: giteeIssueComment.Body,
+ UserId: userIdGen.Generate(giteeIssueComment.AuthorUserId),
+ CreatedDate: giteeIssueComment.GiteeCreatedAt,
+ }
+ return []interface{}{
+ domainIssueComment,
+ }, nil
+ },
+ })
+ if err != nil {
+ return err
+ }
+
+ return converter.Execute()
+}
diff --git a/plugins/gitee/tasks/issue_comment_extractor.go b/plugins/gitee/tasks/issue_comment_extractor.go
new file mode 100644
index 00000000..8e321b05
--- /dev/null
+++ b/plugins/gitee/tasks/issue_comment_extractor.go
@@ -0,0 +1,119 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "encoding/json"
+
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/gitee/models"
+ "github.com/apache/incubator-devlake/plugins/helper"
+)
+
+var ExtractApiIssueCommentsMeta = core.SubTaskMeta{
+ Name: "extractApiIssueComments",
+ EntryPoint: ExtractApiIssueComments,
+ EnabledByDefault: true,
+ Description: "Extract raw comment data into tool layer table gitee_pull_request_comments" +
+ "and gitee_issue_comments",
+}
+
+type IssueComment struct {
+ GiteeId int `json:"id"`
+ Body string
+ User struct {
+ Login string
+ Id int
+ }
+ IssueUrl string `json:"issue_url"`
+ GiteeCreatedAt helper.Iso8601Time `json:"created_at"`
+ GiteeUpdatedAt helper.Iso8601Time `json:"updated_at"`
+}
+
+func ExtractApiIssueComments(taskCtx core.SubTaskContext) error {
+ data := taskCtx.GetData().(*GiteeTaskData)
+
+ extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{
+ RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
+ Ctx: taskCtx,
+ Params: GiteeApiParams{
+ Owner: data.Options.Owner,
+ Repo: data.Options.Repo,
+ },
+ Table: RAW_COMMENTS_TABLE,
+ },
+ Extract: func(row *helper.RawData) ([]interface{}, error) {
+ apiComment := &IssueComment{}
+ err := json.Unmarshal(row.Data, apiComment)
+ if err != nil {
+ return nil, err
+ }
+ // need to extract 2 kinds of entities here
+ results := make([]interface{}, 0, 2)
+ if apiComment.GiteeId == 0 {
+ return nil, nil
+ }
+ //If this is a pr, ignore
+ issueINumber, err := GetIssueIdByIssueUrl(apiComment.IssueUrl)
+ if err != nil {
+ return nil, err
+ }
+ issue := &models.GiteeIssue{}
+ err = taskCtx.GetDb().Where("number = ? and repo_id = ?", issueINumber, data.Repo.GiteeId).Limit(1).Find(issue).Error
+ if err != nil {
+ return nil, err
+ }
+ //if we can not find issues with issue number above, move the comments to gitee_pull_request_comments
+ if issue.GiteeId == 0 {
+ pr := &models.GiteePullRequest{}
+ err = taskCtx.GetDb().Where("number = ? and repo_id = ?", issueINumber, data.Repo.GiteeId).Limit(1).Find(pr).Error
+ if err != nil {
+ return nil, err
+ }
+ giteePrComment := &models.GiteePullRequestComment{
+ GiteeId: apiComment.GiteeId,
+ PullRequestId: pr.GiteeId,
+ Body: apiComment.Body,
+ AuthorUsername: apiComment.User.Login,
+ AuthorUserId: apiComment.User.Id,
+ GiteeCreatedAt: apiComment.GiteeCreatedAt.ToTime(),
+ GiteeUpdatedAt: apiComment.GiteeUpdatedAt.ToTime(),
+ }
+ results = append(results, giteePrComment)
+ } else {
+ giteeIssueComment := &models.GiteeIssueComment{
+ GiteeId: apiComment.GiteeId,
+ IssueId: issue.GiteeId,
+ Body: apiComment.Body,
+ AuthorUsername: apiComment.User.Login,
+ AuthorUserId: apiComment.User.Id,
+ GiteeCreatedAt: apiComment.GiteeCreatedAt.ToTime(),
+ GiteeUpdatedAt: apiComment.GiteeUpdatedAt.ToTime(),
+ }
+ results = append(results, giteeIssueComment)
+ }
+ return results, nil
+ },
+ })
+
+ if err != nil {
+ return err
+ }
+
+ return extractor.Execute()
+}
diff --git a/plugins/gitee/tasks/issue_convertor.go b/plugins/gitee/tasks/issue_convertor.go
new file mode 100644
index 00000000..0fe2d3a1
--- /dev/null
+++ b/plugins/gitee/tasks/issue_convertor.go
@@ -0,0 +1,101 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "reflect"
+
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/helper"
+
+ "github.com/apache/incubator-devlake/models/domainlayer"
+ "github.com/apache/incubator-devlake/models/domainlayer/didgen"
+ "github.com/apache/incubator-devlake/models/domainlayer/ticket"
+ giteeModels "github.com/apache/incubator-devlake/plugins/gitee/models"
+)
+
+var ConvertIssuesMeta = core.SubTaskMeta{
+ Name: "convertIssues",
+ EntryPoint: ConvertIssues,
+ EnabledByDefault: true,
+ Description: "Convert tool layer table gitee_issues into domain layer table issues",
+}
+
+func ConvertIssues(taskCtx core.SubTaskContext) error {
+ db := taskCtx.GetDb()
+ rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_ISSUE_TABLE)
+ repoId := data.Repo.GiteeId
+
+ issue := &giteeModels.GiteeIssue{}
+ cursor, err := db.Model(issue).Where("repo_id = ?", repoId).Rows()
+
+ if err != nil {
+ return err
+ }
+ defer cursor.Close()
+
+ issueIdGen := didgen.NewDomainIdGenerator(&giteeModels.GiteeIssue{})
+ userIdGen := didgen.NewDomainIdGenerator(&giteeModels.GiteeUser{})
+ boardIdGen := didgen.NewDomainIdGenerator(&giteeModels.GiteeRepo{})
+
+ converter, err := helper.NewDataConverter(helper.DataConverterArgs{
+ RawDataSubTaskArgs: *rawDataSubTaskArgs,
+ InputRowType: reflect.TypeOf(giteeModels.GiteeIssue{}),
+ Input: cursor,
+ Convert: func(inputRow interface{}) ([]interface{}, error) {
+ issue := inputRow.(*giteeModels.GiteeIssue)
+ domainIssue := &ticket.Issue{
+ DomainEntity: domainlayer.DomainEntity{Id: issueIdGen.Generate(issue.GiteeId)},
+ IssueKey: issue.Number,
+ Title: issue.Title,
+ Description: issue.Body,
+ Priority: issue.Priority,
+ Type: issue.Type,
+ AssigneeId: userIdGen.Generate(issue.AssigneeId),
+ AssigneeName: issue.AssigneeName,
+ CreatorId: userIdGen.Generate(issue.AuthorId),
+ CreatorName: issue.AuthorName,
+ LeadTimeMinutes: issue.LeadTimeMinutes,
+ Url: issue.Url,
+ CreatedDate: &issue.GiteeCreatedAt,
+ UpdatedDate: &issue.GiteeUpdatedAt,
+ ResolutionDate: issue.ClosedAt,
+ Severity: issue.Severity,
+ Component: issue.Component,
+ }
+ if issue.State == "closed" {
+ domainIssue.Status = ticket.DONE
+ } else {
+ domainIssue.Status = ticket.TODO
+ }
+ boardIssue := &ticket.BoardIssue{
+ BoardId: boardIdGen.Generate(repoId),
+ IssueId: domainIssue.Id,
+ }
+ return []interface{}{
+ domainIssue,
+ boardIssue,
+ }, nil
+ },
+ })
+ if err != nil {
+ return err
+ }
+
+ return converter.Execute()
+}
diff --git a/plugins/gitee/tasks/issue_extractor.go b/plugins/gitee/tasks/issue_extractor.go
new file mode 100644
index 00000000..69cb7a57
--- /dev/null
+++ b/plugins/gitee/tasks/issue_extractor.go
@@ -0,0 +1,253 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "encoding/json"
+ "regexp"
+
+ "github.com/apache/incubator-devlake/models/domainlayer/ticket"
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/gitee/models"
+ "github.com/apache/incubator-devlake/plugins/helper"
+)
+
+var ExtractApiIssuesMeta = core.SubTaskMeta{
+ Name: "extractApiIssues",
+ EntryPoint: ExtractApiIssues,
+ EnabledByDefault: true,
+ Description: "Extract raw Issues data into tool layer table gitee_issues",
+}
+
+type IssuesResponse struct {
+ GiteeId int `json:"id"`
+ Url string `json:"url"`
+ RepositoryUrl string `json:"repository_url"`
+ Number string `json:"number"`
+ State string `json:"state"`
+ Title string
+ Body string
+ HtmlUrl string `json:"html_url"`
+ CommentsUrl string `json:"comments_url"`
+ PullRequest struct {
+ Url string `json:"url"`
+ HtmlUrl string `json:"html_url"`
+ } `json:"pull_request"`
+ Labels []struct {
+ Id int
+ RepositoryId int `json:"repository_id"`
+ Name string `json:"name"`
+ CreatedAt helper.Iso8601Time `json:"created_at"`
+ UpdatedAt helper.Iso8601Time `json:"updated_at"`
+ } `json:"labels"`
+ Repository struct {
+ Id int
+ FullName string `json:"full_name"`
+ Url string `json:"url"`
+ } `json:"repository"`
+ Assignee *struct {
+ Login string
+ Id int
+ }
+ User *struct {
+ Login string
+ Id int
+ Name string
+ }
+ Comments int `json:"comments"`
+ Priority int `json:"priority"`
+ IssueType string `json:"issue_type"`
+ SecurityHole bool `json:"security_hole"`
+ IssueState string `json:"issue_state"`
+ Branch string `json:"branch"`
+ FinishAt *helper.Iso8601Time `json:"finished_at"`
+ GiteeCreatedAt helper.Iso8601Time `json:"created_at"`
+ GiteeUpdatedAt helper.Iso8601Time `json:"updated_at"`
+ IssueTypeDetail struct {
+ Id int
+ Title string
+ Ident string
+ CreatedAt helper.Iso8601Time `json:"created_at"`
+ UpdatedAt helper.Iso8601Time `json:"updated_at"`
+ }
+ IssueStateDetail struct {
+ Id int
+ Title string
+ Serial string
+ CreatedAt helper.Iso8601Time `json:"created_at"`
+ UpdatedAt helper.Iso8601Time `json:"updated_at"`
+ }
+}
+
+func ExtractApiIssues(taskCtx core.SubTaskContext) error {
+ rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_ISSUE_TABLE)
+ config := data.Options.Config
+ var issueSeverityRegex *regexp.Regexp
+ var issueComponentRegex *regexp.Regexp
+ var issuePriorityRegex *regexp.Regexp
+ var issueTypeBugRegex *regexp.Regexp
+ var issueTypeRequirementRegex *regexp.Regexp
+ var issueTypeIncidentRegex *regexp.Regexp
+ var issueSeverity = config.IssueSeverity
+ if issueSeverity == "" {
+ issueSeverity = taskCtx.GetConfig("GITEE_ISSUE_SEVERITY")
+ }
+ var issueComponent = config.IssueComponent
+ if issueComponent == "" {
+ issueComponent = taskCtx.GetConfig("GITEE_ISSUE_COMPONENT")
+ }
+ var issuePriority = config.IssuePriority
+ if issuePriority == "" {
+ issuePriority = taskCtx.GetConfig("GITEE_ISSUE_PRIORITY")
+ }
+ var issueTypeBug = config.IssueTypeBug
+ if issueTypeBug == "" {
+ issueTypeBug = taskCtx.GetConfig("GITEE_ISSUE_TYPE_BUG")
+ }
+ var issueTypeRequirement = config.IssueTypeRequirement
+ if issueTypeRequirement == "" {
+ issueTypeRequirement = taskCtx.GetConfig("GITEE_ISSUE_TYPE_REQUIREMENT")
+ }
+ var issueTypeIncident = config.IssueTypeIncident
+ if issueTypeIncident == "" {
+ issueTypeIncident = taskCtx.GetConfig("GITEE_ISSUE_TYPE_INCIDENT")
+ }
+ if len(issueSeverity) > 0 {
+ issueSeverityRegex = regexp.MustCompile(issueSeverity)
+ }
+ if len(issueComponent) > 0 {
+ issueComponentRegex = regexp.MustCompile(issueComponent)
+ }
+ if len(issuePriority) > 0 {
+ issuePriorityRegex = regexp.MustCompile(issuePriority)
+ }
+ if len(issueTypeBug) > 0 {
+ issueTypeBugRegex = regexp.MustCompile(issueTypeBug)
+ }
+ if len(issueTypeRequirement) > 0 {
+ issueTypeRequirementRegex = regexp.MustCompile(issueTypeRequirement)
+ }
+ if len(issueTypeIncident) > 0 {
+ issueTypeIncidentRegex = regexp.MustCompile(issueTypeIncident)
+ }
+
+ extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{
+ RawDataSubTaskArgs: *rawDataSubTaskArgs,
+ Extract: func(row *helper.RawData) ([]interface{}, error) {
+ body := &IssuesResponse{}
+ err := json.Unmarshal(row.Data, body)
+ if err != nil {
+ return nil, err
+ }
+ // need to extract 2 kinds of entities here
+ if body.GiteeId == 0 {
+ return nil, nil
+ }
+ //If this is a pr, ignore
+ if body.PullRequest.Url != "" {
+ return nil, nil
+ }
+ results := make([]interface{}, 0, 2)
+ giteeIssue, err := convertGiteeIssue(body, data.Repo.GiteeId)
+ if err != nil {
+ return nil, err
+ }
+ for _, label := range body.Labels {
+ results = append(results, &models.GiteeIssueLabel{
+ IssueId: giteeIssue.GiteeId,
+ LabelName: label.Name,
+ })
+ if issueSeverityRegex != nil {
+ groups := issueSeverityRegex.FindStringSubmatch(label.Name)
+ if len(groups) > 0 {
+ giteeIssue.Severity = groups[1]
+ }
+ }
+
+ if issueComponentRegex != nil {
+ groups := issueComponentRegex.FindStringSubmatch(label.Name)
+ if len(groups) > 0 {
+ giteeIssue.Component = groups[1]
+ }
+ }
+
+ if issuePriorityRegex != nil {
+ groups := issuePriorityRegex.FindStringSubmatch(label.Name)
+ if len(groups) > 0 {
+ giteeIssue.Priority = groups[1]
+ }
+ }
+
+ if issueTypeBugRegex != nil {
+ if ok := issueTypeBugRegex.MatchString(label.Name); ok {
+ giteeIssue.Type = ticket.BUG
+ }
+ }
+
+ if issueTypeRequirementRegex != nil {
+ if ok := issueTypeRequirementRegex.MatchString(label.Name); ok {
+ giteeIssue.Type = ticket.REQUIREMENT
+ }
+ }
+
+ if issueTypeIncidentRegex != nil {
+ if ok := issueTypeIncidentRegex.MatchString(label.Name); ok {
+ giteeIssue.Type = ticket.INCIDENT
+ }
+ }
+ }
+ results = append(results, giteeIssue)
+
+ return results, nil
+ },
+ })
+
+ if err != nil {
+ return err
+ }
+
+ return extractor.Execute()
+}
+func convertGiteeIssue(issue *IssuesResponse, repositoryId int) (*models.GiteeIssue, error) {
+ giteeIssue := &models.GiteeIssue{
+ GiteeId: issue.GiteeId,
+ RepoId: repositoryId,
+ Number: issue.Number,
+ State: issue.State,
+ Title: issue.Title,
+ Body: issue.Body,
+ Url: issue.HtmlUrl,
+ ClosedAt: helper.Iso8601TimeToTime(issue.FinishAt),
+ GiteeCreatedAt: issue.GiteeCreatedAt.ToTime(),
+ GiteeUpdatedAt: issue.GiteeUpdatedAt.ToTime(),
+ }
+
+ if issue.Assignee != nil {
+ giteeIssue.AssigneeId = issue.Assignee.Id
+ giteeIssue.AssigneeName = issue.Assignee.Login
+ }
+ if issue.User != nil {
+ giteeIssue.AuthorId = issue.User.Id
+ giteeIssue.AuthorName = issue.User.Login
+ }
+ if issue.FinishAt != nil {
+ giteeIssue.LeadTimeMinutes = uint(issue.FinishAt.ToTime().Sub(issue.GiteeCreatedAt.ToTime()).Minutes())
+ }
+
+ return giteeIssue, nil
+}
diff --git a/plugins/gitee/tasks/issue_label_convertor.go b/plugins/gitee/tasks/issue_label_convertor.go
new file mode 100644
index 00000000..d017cffa
--- /dev/null
+++ b/plugins/gitee/tasks/issue_label_convertor.go
@@ -0,0 +1,73 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "reflect"
+
+ "github.com/apache/incubator-devlake/models/domainlayer/didgen"
+ "github.com/apache/incubator-devlake/models/domainlayer/ticket"
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/gitee/models"
+ "github.com/apache/incubator-devlake/plugins/helper"
+)
+
+var ConvertIssueLabelsMeta = core.SubTaskMeta{
+ Name: "convertIssueLabels",
+ EntryPoint: ConvertIssueLabels,
+ EnabledByDefault: true,
+ Description: "Convert tool layer table gitee_issue_labels into domain layer table issue_labels",
+}
+
+func ConvertIssueLabels(taskCtx core.SubTaskContext) error {
+ db := taskCtx.GetDb()
+ rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_ISSUE_TABLE)
+ repoId := data.Repo.GiteeId
+
+ cursor, err := db.Model(&models.GiteeIssueLabel{}).
+ Joins(`left join _tool_gitee_issues on _tool_gitee_issues.gitee_id = _tool_gitee_issue_labels.issue_id`).
+ Where("_tool_gitee_issues.repo_id = ?", repoId).
+ Order("issue_id ASC").
+ Rows()
+ if err != nil {
+ return err
+ }
+ defer cursor.Close()
+ issueIdGen := didgen.NewDomainIdGenerator(&models.GiteeIssue{})
+
+ converter, err := helper.NewDataConverter(helper.DataConverterArgs{
+ RawDataSubTaskArgs: *rawDataSubTaskArgs,
+ InputRowType: reflect.TypeOf(models.GiteeIssueLabel{}),
+ Input: cursor,
+ Convert: func(inputRow interface{}) ([]interface{}, error) {
+ issueLabel := inputRow.(*models.GiteeIssueLabel)
+ domainIssueLabel := &ticket.IssueLabel{
+ IssueId: issueIdGen.Generate(issueLabel.IssueId),
+ LabelName: issueLabel.LabelName,
+ }
+ return []interface{}{
+ domainIssueLabel,
+ }, nil
+ },
+ })
+ if err != nil {
+ return err
+ }
+
+ return converter.Execute()
+}
diff --git a/plugins/gitee/tasks/pr_collector.go b/plugins/gitee/tasks/pr_collector.go
new file mode 100644
index 00000000..31f80ddc
--- /dev/null
+++ b/plugins/gitee/tasks/pr_collector.go
@@ -0,0 +1,99 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "encoding/json"
+ "fmt"
+ "net/http"
+ "net/url"
+
+ "github.com/apache/incubator-devlake/plugins/gitee/models"
+
+ "github.com/apache/incubator-devlake/plugins/helper"
+
+ "github.com/apache/incubator-devlake/plugins/core"
+)
+
+const RAW_PULL_REQUEST_TABLE = "gitee_api_pull_requests"
+
+var CollectApiPullRequestsMeta = core.SubTaskMeta{
+ Name: "collectApiPullRequests",
+ EntryPoint: CollectApiPullRequests,
+ EnabledByDefault: true,
+ Description: "Collect PullRequests data from Gitee api",
+}
+
+func CollectApiPullRequests(taskCtx core.SubTaskContext) error {
+ db := taskCtx.GetDb()
+ rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_PULL_REQUEST_TABLE)
+ since := data.Since
+ incremental := false
+ if since == nil {
+ var latestUpdated models.GiteePullRequest
+ err := db.Model(&latestUpdated).
+ Where("repo_id = ?", data.Repo.GiteeId).
+ Order("gitee_updated_at DESC").Limit(1).Find(&latestUpdated).Error
+ if err != nil {
+ return fmt.Errorf("failed to get latest gitee issue record: %w", err)
+ }
+ if latestUpdated.GiteeId > 0 {
+ since = &latestUpdated.GiteeUpdatedAt
+ incremental = true
+ }
+ }
+
+ collector, err := helper.NewApiCollector(helper.ApiCollectorArgs{
+ RawDataSubTaskArgs: *rawDataSubTaskArgs,
+ ApiClient: data.ApiClient,
+ PageSize: 100,
+ Incremental: incremental,
+
+ UrlTemplate: "repos/{{ .Params.Owner }}/{{ .Params.Repo }}/pulls",
+
+ Query: func(reqData *helper.RequestData) (url.Values, error) {
+ query := url.Values{}
+ query.Set("access_token", data.Options.Token)
+ query.Set("state", "all")
+ if since != nil {
+ query.Set("since", since.String())
+ }
+ query.Set("page", fmt.Sprintf("%v", reqData.Pager.Page))
+ query.Set("direction", "asc")
+ query.Set("per_page", fmt.Sprintf("%v", reqData.Pager.Size))
+
+ return query, nil
+ },
+
+ GetTotalPages: GetTotalPagesFromResponse,
+ ResponseParser: func(res *http.Response) ([]json.RawMessage, error) {
+ var items []json.RawMessage
+ err := helper.UnmarshalResponse(res, &items)
+ if err != nil {
+ return nil, err
+ }
+ return items, nil
+ },
+ })
+
+ if err != nil {
+ return err
+ }
+
+ return collector.Execute()
+}
diff --git a/plugins/gitee/tasks/pr_comment_convertor.go b/plugins/gitee/tasks/pr_comment_convertor.go
new file mode 100644
index 00000000..03aff955
--- /dev/null
+++ b/plugins/gitee/tasks/pr_comment_convertor.go
@@ -0,0 +1,82 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "reflect"
+
+ "github.com/apache/incubator-devlake/models/domainlayer"
+ "github.com/apache/incubator-devlake/models/domainlayer/code"
+ "github.com/apache/incubator-devlake/models/domainlayer/didgen"
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/gitee/models"
+ "github.com/apache/incubator-devlake/plugins/helper"
+)
+
+var ConvertPullRequestCommentsMeta = core.SubTaskMeta{
+ Name: "convertPullRequestComments",
+ EntryPoint: ConvertPullRequestComments,
+ EnabledByDefault: true,
+ Description: "ConvertPullRequestComments data from Gitee api",
+}
+
+func ConvertPullRequestComments(taskCtx core.SubTaskContext) error {
+ db := taskCtx.GetDb()
+ rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_COMMENTS_TABLE)
+ repoId := data.Repo.GiteeId
+
+ cursor, err := db.Model(&models.GiteePullRequestComment{}).
+ Joins("left join _tool_gitee_pull_requests "+
+ "on _tool_gitee_pull_requests.gitee_id = _tool_gitee_pull_request_comments.pull_request_id").
+ Where("repo_id = ?", repoId).Rows()
+ if err != nil {
+ return err
+ }
+ defer cursor.Close()
+
+ prIdGen := didgen.NewDomainIdGenerator(&models.GiteePullRequest{})
+ userIdGen := didgen.NewDomainIdGenerator(&models.GiteeUser{})
+
+ converter, err := helper.NewDataConverter(helper.DataConverterArgs{
+ InputRowType: reflect.TypeOf(models.GiteePullRequestComment{}),
+ Input: cursor,
+ RawDataSubTaskArgs: *rawDataSubTaskArgs,
+ Convert: func(inputRow interface{}) ([]interface{}, error) {
+ giteePullRequestComment := inputRow.(*models.GiteePullRequestComment)
+ domainPrComment := &code.PullRequestComment{
+ DomainEntity: domainlayer.DomainEntity{
+ Id: prIdGen.Generate(giteePullRequestComment.GiteeId),
+ },
+ PullRequestId: prIdGen.Generate(giteePullRequestComment.PullRequestId),
+ Body: giteePullRequestComment.Body,
+ UserId: userIdGen.Generate(giteePullRequestComment.AuthorUserId),
+ CreatedDate: giteePullRequestComment.GiteeCreatedAt,
+ CommitSha: "",
+ Position: 0,
+ }
+ return []interface{}{
+ domainPrComment,
+ }, nil
+ },
+ })
+ if err != nil {
+ return err
+ }
+
+ return converter.Execute()
+}
diff --git a/plugins/gitee/tasks/pr_commit_collector.go b/plugins/gitee/tasks/pr_commit_collector.go
new file mode 100644
index 00000000..8a5a1388
--- /dev/null
+++ b/plugins/gitee/tasks/pr_commit_collector.go
@@ -0,0 +1,96 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "encoding/json"
+ "fmt"
+ "net/http"
+ "net/url"
+ "reflect"
+
+ "github.com/apache/incubator-devlake/plugins/helper"
+
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/gitee/models"
+)
+
+const RAW_PULL_REQUEST_COMMIT_TABLE = "gitee_api_pull_request_commits"
+
+var CollectApiPullRequestCommitsMeta = core.SubTaskMeta{
+ Name: "collectApiPullRequestCommits",
+ EntryPoint: CollectApiPullRequestCommits,
+ EnabledByDefault: true,
+ Description: "Collect PullRequestCommits data from Gitee api",
+}
+
+type SimplePr struct {
+ Number int
+ GiteeId int
+}
+
+func CollectApiPullRequestCommits(taskCtx core.SubTaskContext) error {
+ db := taskCtx.GetDb()
+ rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_PULL_REQUEST_COMMIT_TABLE)
+
+ incremental := false
+
+ cursor, err := db.Model(&models.GiteePullRequest{}).Select("number, gitee_id").
+ Where("repo_id = ?", data.Repo.GiteeId).
+ Rows()
+ if err != nil {
+ return err
+ }
+ iterator, err := helper.NewCursorIterator(db, cursor, reflect.TypeOf(SimplePr{}))
+ if err != nil {
+ return err
+ }
+ collector, err := helper.NewApiCollector(helper.ApiCollectorArgs{
+ RawDataSubTaskArgs: *rawDataSubTaskArgs,
+ ApiClient: data.ApiClient,
+ PageSize: 100,
+ Incremental: incremental,
+ Input: iterator,
+
+ UrlTemplate: "repos/{{ .Params.Owner }}/{{ .Params.Repo }}/pulls/{{ .Input.Number }}/commits",
+
+ Query: func(reqData *helper.RequestData) (url.Values, error) {
+ query := url.Values{}
+ query.Set("access_token", data.Options.Token)
+ query.Set("state", "all")
+ query.Set("page", fmt.Sprintf("%v", reqData.Pager.Page))
+ query.Set("direction", "asc")
+ query.Set("per_page", fmt.Sprintf("%v", reqData.Pager.Size))
+
+ return query, nil
+ },
+ ResponseParser: func(res *http.Response) ([]json.RawMessage, error) {
+ var items []json.RawMessage
+ err := helper.UnmarshalResponse(res, &items)
+ if err != nil {
+ return nil, err
+ }
+ return items, nil
+ },
+ })
+
+ if err != nil {
+ return err
+ }
+ return collector.Execute()
+}
diff --git a/plugins/gitee/tasks/pr_commit_convertor.go b/plugins/gitee/tasks/pr_commit_convertor.go
new file mode 100644
index 00000000..e0370135
--- /dev/null
+++ b/plugins/gitee/tasks/pr_commit_convertor.go
@@ -0,0 +1,73 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "reflect"
+
+ "github.com/apache/incubator-devlake/models/domainlayer/code"
+ "github.com/apache/incubator-devlake/models/domainlayer/didgen"
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/gitee/models"
+ "github.com/apache/incubator-devlake/plugins/helper"
+)
+
+var ConvertPullRequestCommitsMeta = core.SubTaskMeta{
+ Name: "convertPullRequestCommits",
+ EntryPoint: ConvertPullRequestCommits,
+ EnabledByDefault: true,
+ Description: "Convert tool layer table gitee_pull_request_commits into domain layer table pull_request_commits",
+}
+
+func ConvertPullRequestCommits(taskCtx core.SubTaskContext) (err error) {
+ db := taskCtx.GetDb()
+ rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_PULL_REQUEST_COMMIT_TABLE)
+ repoId := data.Repo.GiteeId
+
+ pullIdGen := didgen.NewDomainIdGenerator(&models.GiteePullRequest{})
+
+ cursor, err := db.Model(&models.GiteePullRequestCommit{}).
+ Joins(`left join _tool_gitee_pull_requests on _tool_gitee_pull_requests.gitee_id = _tool_gitee_pull_request_commits.pull_request_id`).
+ Where("_tool_gitee_pull_requests.repo_id = ?", repoId).
+ Order("pull_request_id ASC").Rows()
+ if err != nil {
+ return err
+ }
+ defer cursor.Close()
+
+ converter, err := helper.NewDataConverter(helper.DataConverterArgs{
+ InputRowType: reflect.TypeOf(models.GiteePullRequestCommit{}),
+ Input: cursor,
+ RawDataSubTaskArgs: *rawDataSubTaskArgs,
+ Convert: func(inputRow interface{}) ([]interface{}, error) {
+ giteePullRequestCommit := inputRow.(*models.GiteePullRequestCommit)
+ domainPrCommit := &code.PullRequestCommit{
+ CommitSha: giteePullRequestCommit.CommitSha,
+ PullRequestId: pullIdGen.Generate(giteePullRequestCommit.PullRequestId),
+ }
+ return []interface{}{
+ domainPrCommit,
+ }, nil
+ },
+ })
+ if err != nil {
+ return err
+ }
+
+ return converter.Execute()
+}
diff --git a/plugins/gitee/tasks/pr_commit_extractor.go b/plugins/gitee/tasks/pr_commit_extractor.go
new file mode 100644
index 00000000..30befb6e
--- /dev/null
+++ b/plugins/gitee/tasks/pr_commit_extractor.go
@@ -0,0 +1,127 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "encoding/json"
+ "strings"
+
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/gitee/models"
+ "github.com/apache/incubator-devlake/plugins/helper"
+)
+
+var ExtractApiPullRequestCommitsMeta = core.SubTaskMeta{
+ Name: "extractApiPullRequestCommits",
+ EntryPoint: ExtractApiPullRequestCommits,
+ EnabledByDefault: true,
+ Description: "Extract raw PullRequestCommits data into tool layer table gitee_commits",
+}
+
+type PrCommitsResponse struct {
+ Sha string `json:"sha"`
+ Commit PullRequestCommit
+ Url string
+ Author struct {
+ Id int
+ Login string
+ Name string
+ }
+ Committer struct {
+ Id int
+ Login string
+ Name string
+ }
+}
+
+type PullRequestCommit struct {
+ Author struct {
+ Name string
+ Email string
+ Date helper.Iso8601Time
+ }
+ Committer struct {
+ Name string
+ Email string
+ Date helper.Iso8601Time
+ }
+ Message string
+ CommentCount int `json:"comment_count"`
+}
+
+func ExtractApiPullRequestCommits(taskCtx core.SubTaskContext) error {
+ rawDataSubTaskArgs, _ := CreateRawDataSubTaskArgs(taskCtx, RAW_PULL_REQUEST_COMMIT_TABLE)
+ extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{
+ RawDataSubTaskArgs: *rawDataSubTaskArgs,
+ Extract: func(row *helper.RawData) ([]interface{}, error) {
+ apiPullRequestCommit := &PrCommitsResponse{}
+ if strings.HasPrefix(string(row.Data), "{\"message\": \"Not Found\"") {
+ return nil, nil
+ }
+ err := json.Unmarshal(row.Data, apiPullRequestCommit)
+ if err != nil {
+ return nil, err
+ }
+ pull := &SimplePr{}
+ err = json.Unmarshal(row.Input, pull)
+ if err != nil {
+ return nil, err
+ }
+ // need to extract 2 kinds of entities here
+ results := make([]interface{}, 0, 2)
+
+ giteeCommit, err := convertPullRequestCommit(apiPullRequestCommit)
+ if err != nil {
+ return nil, err
+ }
+ results = append(results, giteeCommit)
+
+ giteePullRequestCommit := &models.GiteePullRequestCommit{
+ CommitSha: apiPullRequestCommit.Sha,
+ PullRequestId: pull.GiteeId,
+ }
+ if err != nil {
+ return nil, err
+ }
+ results = append(results, giteePullRequestCommit)
+ return results, nil
+ },
+ })
+
+ if err != nil {
+ return err
+ }
+
+ return extractor.Execute()
+}
+
+func convertPullRequestCommit(prCommit *PrCommitsResponse) (*models.GiteeCommit, error) {
+ giteeCommit := &models.GiteeCommit{
+ Sha: prCommit.Sha,
+ Message: prCommit.Commit.Message,
+ AuthorId: prCommit.Author.Id,
+ AuthorName: prCommit.Commit.Author.Name,
+ AuthorEmail: prCommit.Commit.Author.Email,
+ AuthoredDate: prCommit.Commit.Author.Date.ToTime(),
+ CommitterName: prCommit.Commit.Committer.Name,
+ CommitterEmail: prCommit.Commit.Committer.Email,
+ CommittedDate: prCommit.Commit.Committer.Date.ToTime(),
+ WebUrl: prCommit.Url,
+ }
+ return giteeCommit, nil
+}
diff --git a/plugins/gitee/tasks/pr_convertor.go b/plugins/gitee/tasks/pr_convertor.go
new file mode 100644
index 00000000..28d4d4de
--- /dev/null
+++ b/plugins/gitee/tasks/pr_convertor.go
@@ -0,0 +1,92 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "reflect"
+
+ "github.com/apache/incubator-devlake/models/domainlayer"
+ "github.com/apache/incubator-devlake/models/domainlayer/code"
+ "github.com/apache/incubator-devlake/models/domainlayer/didgen"
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/gitee/models"
+ "github.com/apache/incubator-devlake/plugins/helper"
+)
+
+var ConvertPullRequestsMeta = core.SubTaskMeta{
+ Name: "convertPullRequests",
+ EntryPoint: ConvertPullRequests,
+ EnabledByDefault: true,
+ Description: "ConvertPullRequests data from Gitee api",
+}
+
+func ConvertPullRequests(taskCtx core.SubTaskContext) error {
+ db := taskCtx.GetDb()
+ rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_PULL_REQUEST_TABLE)
+ repoId := data.Repo.GiteeId
+
+ cursor, err := db.Model(&models.GiteePullRequest{}).Where("repo_id = ?", repoId).Rows()
+ if err != nil {
+ return err
+ }
+ defer cursor.Close()
+
+ prIdGen := didgen.NewDomainIdGenerator(&models.GiteePullRequest{})
+ repoIdGen := didgen.NewDomainIdGenerator(&models.GiteeRepo{})
+ userIdGen := didgen.NewDomainIdGenerator(&models.GiteeUser{})
+
+ converter, err := helper.NewDataConverter(helper.DataConverterArgs{
+ InputRowType: reflect.TypeOf(models.GiteePullRequest{}),
+ Input: cursor,
+ RawDataSubTaskArgs: *rawDataSubTaskArgs,
+ Convert: func(inputRow interface{}) ([]interface{}, error) {
+ pr := inputRow.(*models.GiteePullRequest)
+ domainPr := &code.PullRequest{
+ DomainEntity: domainlayer.DomainEntity{
+ Id: prIdGen.Generate(pr.GiteeId),
+ },
+ BaseRepoId: repoIdGen.Generate(pr.RepoId),
+ Status: pr.State,
+ Title: pr.Title,
+ Url: pr.Url,
+ AuthorId: userIdGen.Generate(pr.AuthorId),
+ AuthorName: pr.AuthorName,
+ Description: pr.Body,
+ CreatedDate: pr.GiteeCreatedAt,
+ MergedDate: pr.MergedAt,
+ ClosedDate: pr.ClosedAt,
+ PullRequestKey: pr.Number,
+ Type: pr.Type,
+ Component: pr.Component,
+ MergeCommitSha: pr.MergeCommitSha,
+ BaseRef: pr.BaseRef,
+ BaseCommitSha: pr.BaseCommitSha,
+ HeadRef: pr.HeadRef,
+ HeadCommitSha: pr.HeadCommitSha,
+ }
+ return []interface{}{
+ domainPr,
+ }, nil
+ },
+ })
+ if err != nil {
+ return err
+ }
+
+ return converter.Execute()
+}
diff --git a/plugins/gitee/tasks/pr_extractor.go b/plugins/gitee/tasks/pr_extractor.go
new file mode 100644
index 00000000..3a8f0540
--- /dev/null
+++ b/plugins/gitee/tasks/pr_extractor.go
@@ -0,0 +1,171 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "encoding/json"
+ "regexp"
+
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/gitee/models"
+ "github.com/apache/incubator-devlake/plugins/helper"
+)
+
+var ExtractApiPullRequestsMeta = core.SubTaskMeta{
+ Name: "extractApiPullRequests",
+ EntryPoint: ExtractApiPullRequests,
+ EnabledByDefault: true,
+ Description: "Extract raw PullRequests data into tool layer table gitee_pull_requests",
+}
+
+type GiteeApiPullResponse struct {
+ GiteeId int `json:"id"`
+ Number int
+ State string
+ Title string
+ Body json.RawMessage
+ HtmlUrl string `json:"html_url"`
+ Labels []struct {
+ Name string `json:"name"`
+ } `json:"labels"`
+ Assignee *struct {
+ Id int
+ Login string
+ Name string
+ }
+ User *struct {
+ Id int
+ Login string
+ Name string
+ }
+ ClosedAt *helper.Iso8601Time `json:"closed_at"`
+ MergedAt *helper.Iso8601Time `json:"merged_at"`
+ GiteeCreatedAt helper.Iso8601Time `json:"created_at"`
+ GiteeUpdatedAt helper.Iso8601Time `json:"updated_at"`
+ MergeCommitSha string `json:"merge_commit_sha"`
+ Head struct {
+ Ref string
+ Sha string
+ }
+ Base struct {
+ Ref string
+ Sha string
+ Repo struct {
+ Id int
+ Name string
+ Url string
+ HtmlUrl string
+ SshUrl string `json:"ssh_url"`
+ }
+ }
+}
+
+func ExtractApiPullRequests(taskCtx core.SubTaskContext) error {
+ rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_PULL_REQUEST_TABLE)
+ config := data.Options.Config
+ var labelTypeRegex *regexp.Regexp
+ var labelComponentRegex *regexp.Regexp
+ var prType = config.PrType
+ if prType == "" {
+ prType = taskCtx.GetConfig("GITEE_PR_TYPE")
+ }
+ var prComponent = config.PrComponent
+ if prComponent == "" {
+ prComponent = taskCtx.GetConfig("GITEE_PR_COMPONENT")
+ }
+ if len(prType) > 0 {
+ labelTypeRegex = regexp.MustCompile(prType)
+ }
+ if len(prComponent) > 0 {
+ labelComponentRegex = regexp.MustCompile(prComponent)
+ }
+
+ extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{
+ RawDataSubTaskArgs: *rawDataSubTaskArgs,
+ Extract: func(row *helper.RawData) ([]interface{}, error) {
+ pullResponse := &GiteeApiPullResponse{}
+ err := json.Unmarshal(row.Data, pullResponse)
+ if err != nil {
+ return nil, err
+ }
+
+ // need to extract 2 kinds of entities here
+ results := make([]interface{}, 0, 1)
+ if pullResponse.GiteeId == 0 {
+ return nil, nil
+ }
+ giteePr, err := convertGiteePullRequest(pullResponse, data.Repo.GiteeId)
+ if err != nil {
+ return nil, err
+ }
+ for _, label := range pullResponse.Labels {
+ results = append(results, &models.GiteePullRequestLabel{
+ PullId: giteePr.GiteeId,
+ LabelName: label.Name,
+ })
+ // if pr.Type has not been set and prType is set in .env, process the below
+ if labelTypeRegex != nil {
+ groups := labelTypeRegex.FindStringSubmatch(label.Name)
+ if len(groups) > 0 {
+ giteePr.Type = groups[1]
+ }
+ }
+
+ // if pr.Component has not been set and prComponent is set in .env, process
+ if labelComponentRegex != nil {
+ groups := labelComponentRegex.FindStringSubmatch(label.Name)
+ if len(groups) > 0 {
+ giteePr.Component = groups[1]
+ }
+ }
+ }
+ results = append(results, giteePr)
+
+ return results, nil
+ },
+ })
+
+ if err != nil {
+ return err
+ }
+
+ return extractor.Execute()
+}
+func convertGiteePullRequest(pull *GiteeApiPullResponse, repoId int) (*models.GiteePullRequest, error) {
+ giteePull := &models.GiteePullRequest{
+ GiteeId: pull.GiteeId,
+ RepoId: repoId,
+ Number: pull.Number,
+ State: pull.State,
+ Title: pull.Title,
+ Url: pull.HtmlUrl,
+ AuthorName: pull.User.Login,
+ AuthorId: pull.User.Id,
+ GiteeCreatedAt: pull.GiteeCreatedAt.ToTime(),
+ GiteeUpdatedAt: pull.GiteeUpdatedAt.ToTime(),
+ ClosedAt: helper.Iso8601TimeToTime(pull.ClosedAt),
+ MergedAt: helper.Iso8601TimeToTime(pull.MergedAt),
+ MergeCommitSha: pull.MergeCommitSha,
+ Body: string(pull.Body),
+ BaseRef: pull.Base.Ref,
+ BaseCommitSha: pull.Base.Sha,
+ HeadRef: pull.Head.Ref,
+ HeadCommitSha: pull.Head.Sha,
+ }
+ return giteePull, nil
+}
diff --git a/plugins/gitee/tasks/pr_issue_convertor.go b/plugins/gitee/tasks/pr_issue_convertor.go
new file mode 100644
index 00000000..07a24300
--- /dev/null
+++ b/plugins/gitee/tasks/pr_issue_convertor.go
@@ -0,0 +1,78 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "reflect"
+ "strconv"
+
+ "github.com/apache/incubator-devlake/models/domainlayer/crossdomain"
+ "github.com/apache/incubator-devlake/models/domainlayer/didgen"
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/gitee/models"
+ "github.com/apache/incubator-devlake/plugins/helper"
+)
+
+var ConvertPullRequestIssuesMeta = core.SubTaskMeta{
+ Name: "convertPullRequestIssues",
+ EntryPoint: ConvertPullRequestIssues,
+ EnabledByDefault: true,
+ Description: "Convert tool layer table gitee_pull_request_issues into domain layer table pull_request_issues",
+}
+
+func ConvertPullRequestIssues(taskCtx core.SubTaskContext) error {
+ db := taskCtx.GetDb()
+ rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_PULL_REQUEST_TABLE)
+ repoId := data.Repo.GiteeId
+
+ cursor, err := db.Model(&models.GiteePullRequestIssue{}).
+ Joins(`left join _tool_gitee_pull_requests on _tool_gitee_pull_requests.gitee_id = _tool_gitee_pull_request_issues.pull_request_id`).
+ Where("_tool_gitee_pull_requests.repo_id = ?", repoId).
+ Order("pull_request_id ASC").
+ Rows()
+ if err != nil {
+ return err
+ }
+ defer cursor.Close()
+ prIdGen := didgen.NewDomainIdGenerator(&models.GiteePullRequest{})
+ issueIdGen := didgen.NewDomainIdGenerator(&models.GiteeIssue{})
+
+ converter, err := helper.NewDataConverter(helper.DataConverterArgs{
+ InputRowType: reflect.TypeOf(models.GiteePullRequestIssue{}),
+ Input: cursor,
+ RawDataSubTaskArgs: *rawDataSubTaskArgs,
+ Convert: func(inputRow interface{}) ([]interface{}, error) {
+ giteePrIssue := inputRow.(*models.GiteePullRequestIssue)
+ issueNum, _ := strconv.Atoi(giteePrIssue.IssueNumber)
+ pullRequestIssue := &crossdomain.PullRequestIssue{
+ PullRequestId: prIdGen.Generate(giteePrIssue.PullRequestId),
+ IssueId: issueIdGen.Generate(giteePrIssue.IssueId),
+ IssueNumber: issueNum,
+ PullRequestNumber: giteePrIssue.PullRequestNumber,
+ }
+ return []interface{}{
+ pullRequestIssue,
+ }, nil
+ },
+ })
+ if err != nil {
+ return err
+ }
+
+ return converter.Execute()
+}
diff --git a/plugins/gitee/tasks/pr_issue_enricher.go b/plugins/gitee/tasks/pr_issue_enricher.go
new file mode 100644
index 00000000..3fa136cf
--- /dev/null
+++ b/plugins/gitee/tasks/pr_issue_enricher.go
@@ -0,0 +1,116 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "reflect"
+ "regexp"
+ "strconv"
+ "strings"
+
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/gitee/models"
+ "github.com/apache/incubator-devlake/plugins/helper"
+)
+
+var EnrichPullRequestIssuesMeta = core.SubTaskMeta{
+ Name: "enrichPullRequestIssues",
+ EntryPoint: EnrichPullRequestIssues,
+ EnabledByDefault: true,
+ Description: "Create tool layer table gitee_pull_request_issues from gitee_pull_reqeusts",
+}
+
+func EnrichPullRequestIssues(taskCtx core.SubTaskContext) (err error) {
+ db := taskCtx.GetDb()
+ rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_PULL_REQUEST_TABLE)
+ repoId := data.Repo.GiteeId
+
+ var prBodyCloseRegex *regexp.Regexp
+ prBodyClosePattern := taskCtx.GetConfig("GITEE_PR_BODY_CLOSE_PATTERN")
+ //the pattern before the issue number, sometimes, the issue number is #1098, sometimes it is https://xxx/#1098
+ prBodyClosePattern = strings.Replace(prBodyClosePattern, "%s", data.Options.Owner, 1)
+ prBodyClosePattern = strings.Replace(prBodyClosePattern, "%s", data.Options.Repo, 1)
+ if len(prBodyClosePattern) > 0 {
+ prBodyCloseRegex = regexp.MustCompile(prBodyClosePattern)
+ }
+ charPattern := regexp.MustCompile(`[a-zA-Z\s,]+`)
+ cursor, err := db.Model(&models.GiteePullRequest{}).
+ Where("repo_id = ?", repoId).
+ Rows()
+ if err != nil {
+ return err
+ }
+ defer cursor.Close()
+ // iterate all rows
+
+ converter, err := helper.NewDataConverter(helper.DataConverterArgs{
+ InputRowType: reflect.TypeOf(models.GiteePullRequest{}),
+ Input: cursor,
+ RawDataSubTaskArgs: *rawDataSubTaskArgs,
+ Convert: func(inputRow interface{}) ([]interface{}, error) {
+ giteePullRequst := inputRow.(*models.GiteePullRequest)
+ results := make([]interface{}, 0, 1)
+
+ //find the matched string in body
+ issueNumberListStr := ""
+
+ if prBodyCloseRegex != nil {
+ issueNumberListStr = prBodyCloseRegex.FindString(giteePullRequst.Body)
+ }
+
+ if issueNumberListStr == "" {
+ return nil, nil
+ }
+
+ issueNumberListStr = charPattern.ReplaceAllString(issueNumberListStr, "#")
+ //split the string by '#'
+ issueNumberList := strings.Split(issueNumberListStr, "#")
+
+ for _, issueNumberStr := range issueNumberList {
+ issue := &models.GiteeIssue{}
+ issueNumberStr = strings.TrimSpace(issueNumberStr)
+ //change the issueNumberStr to int, if cannot be changed, just continue
+ issueNumber, numFormatErr := strconv.Atoi(issueNumberStr)
+ if numFormatErr != nil {
+ continue
+ }
+ err = db.Where("number = ? and repo_id = ?", issueNumber, repoId).
+ Limit(1).Find(issue).Error
+ if err != nil {
+ return nil, err
+ }
+ if issue.Number == "" {
+ continue
+ }
+ giteePullRequstIssue := &models.GiteePullRequestIssue{
+ PullRequestId: giteePullRequst.GiteeId,
+ IssueId: issue.GiteeId,
+ PullRequestNumber: giteePullRequst.Number,
+ IssueNumber: issue.Number,
+ }
+ results = append(results, giteePullRequstIssue)
+ }
+ return results, nil
+ },
+ })
+ if err != nil {
+ return err
+ }
+
+ return converter.Execute()
+}
diff --git a/plugins/gitee/tasks/pr_label_convertor.go b/plugins/gitee/tasks/pr_label_convertor.go
new file mode 100644
index 00000000..14b87161
--- /dev/null
+++ b/plugins/gitee/tasks/pr_label_convertor.go
@@ -0,0 +1,73 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "reflect"
+
+ "github.com/apache/incubator-devlake/models/domainlayer/code"
+ "github.com/apache/incubator-devlake/models/domainlayer/didgen"
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/gitee/models"
+ "github.com/apache/incubator-devlake/plugins/helper"
+)
+
+var ConvertPullRequestLabelsMeta = core.SubTaskMeta{
+ Name: "convertPullRequestLabels",
+ EntryPoint: ConvertPullRequestLabels,
+ EnabledByDefault: true,
+ Description: "Convert tool layer table gitee_pull_request_labels into domain layer table pull_request_labels",
+}
+
+func ConvertPullRequestLabels(taskCtx core.SubTaskContext) error {
+ db := taskCtx.GetDb()
+ rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_PULL_REQUEST_TABLE)
+ repoId := data.Repo.GiteeId
+
+ cursor, err := db.Model(&models.GiteePullRequestLabel{}).
+ Joins(`left join _tool_gitee_pull_requests on _tool_gitee_pull_requests.gitee_id = _tool_gitee_pull_request_labels.pull_id`).
+ Where("_tool_gitee_pull_requests.repo_id = ?", repoId).
+ Order("pull_id ASC").
+ Rows()
+ if err != nil {
+ return err
+ }
+ defer cursor.Close()
+ prIdGen := didgen.NewDomainIdGenerator(&models.GiteePullRequest{})
+
+ converter, err := helper.NewDataConverter(helper.DataConverterArgs{
+ InputRowType: reflect.TypeOf(models.GiteePullRequestLabel{}),
+ Input: cursor,
+ RawDataSubTaskArgs: *rawDataSubTaskArgs,
+ Convert: func(inputRow interface{}) ([]interface{}, error) {
+ prLabel := inputRow.(*models.GiteePullRequestLabel)
+ domainPrLabel := &code.PullRequestLabel{
+ PullRequestId: prIdGen.Generate(prLabel.PullId),
+ LabelName: prLabel.LabelName,
+ }
+ return []interface{}{
+ domainPrLabel,
+ }, nil
+ },
+ })
+ if err != nil {
+ return err
+ }
+
+ return converter.Execute()
+}
diff --git a/plugins/gitee/tasks/pr_review_collector.go b/plugins/gitee/tasks/pr_review_collector.go
new file mode 100644
index 00000000..0a64752e
--- /dev/null
+++ b/plugins/gitee/tasks/pr_review_collector.go
@@ -0,0 +1,89 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "encoding/json"
+ "fmt"
+ "net/http"
+ "net/url"
+ "reflect"
+
+ "github.com/apache/incubator-devlake/plugins/helper"
+
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/gitee/models"
+)
+
+const RAW_PULL_REQUEST_REVIEW_TABLE = "gitee_api_pull_request_reviews"
+
+var CollectApiPullRequestReviewsMeta = core.SubTaskMeta{
+ Name: "collectApiPullRequestReviews",
+ EntryPoint: CollectApiPullRequestReviews,
+ EnabledByDefault: true,
+ Description: "Collect PullRequestReviews data from Gitee api",
+}
+
+func CollectApiPullRequestReviews(taskCtx core.SubTaskContext) error {
+ db := taskCtx.GetDb()
+ rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_PULL_REQUEST_REVIEW_TABLE)
+
+ incremental := false
+
+ cursor, err := db.Model(&models.GiteePullRequest{}).Select("number, gitee_id").
+ Where("repo_id = ?", data.Repo.GiteeId).
+ Rows()
+ if err != nil {
+ return err
+ }
+ iterator, err := helper.NewCursorIterator(db, cursor, reflect.TypeOf(SimplePr{}))
+ if err != nil {
+ return err
+ }
+ collector, err := helper.NewApiCollector(helper.ApiCollectorArgs{
+ RawDataSubTaskArgs: *rawDataSubTaskArgs,
+ ApiClient: data.ApiClient,
+ PageSize: 100,
+ Incremental: incremental,
+ Input: iterator,
+
+ UrlTemplate: "repos/{{ .Params.Owner }}/{{ .Params.Repo }}/pulls/3/review",
+
+ Query: func(reqData *helper.RequestData) (url.Values, error) {
+ query := url.Values{}
+ query.Set("access_token", data.Options.Token)
+ query.Set("page", fmt.Sprintf("%v", reqData.Pager.Page))
+ query.Set("per_page", fmt.Sprintf("%v", reqData.Pager.Size))
+
+ return query, nil
+ },
+ ResponseParser: func(res *http.Response) ([]json.RawMessage, error) {
+ var items []json.RawMessage
+ err := helper.UnmarshalResponse(res, &items)
+ if err != nil {
+ return nil, err
+ }
+ return items, nil
+ },
+ })
+
+ if err != nil {
+ return err
+ }
+ return collector.Execute()
+}
diff --git a/plugins/gitee/tasks/pr_review_extractor.go b/plugins/gitee/tasks/pr_review_extractor.go
new file mode 100644
index 00000000..fd4c0499
--- /dev/null
+++ b/plugins/gitee/tasks/pr_review_extractor.go
@@ -0,0 +1,84 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "encoding/json"
+ "strings"
+
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/gitee/models"
+ "github.com/apache/incubator-devlake/plugins/helper"
+)
+
+var ExtractApiPullRequestReviewsMeta = core.SubTaskMeta{
+ Name: "extractApiPullRequestReviews",
+ EntryPoint: ExtractApiPullRequestReviews,
+ EnabledByDefault: true,
+ Description: "Extract raw PullRequestReviews data into tool layer table gitee_reviewers",
+}
+
+type PullRequestReview struct {
+ GiteeId int `json:"id"`
+ User struct {
+ Id int
+ Login string
+ }
+ Body string
+ State string
+ SubmittedAt helper.Iso8601Time `json:"submitted_at"`
+}
+
+func ExtractApiPullRequestReviews(taskCtx core.SubTaskContext) error {
+ rawDataSubTaskArgs, _ := CreateRawDataSubTaskArgs(taskCtx, RAW_PULL_REQUEST_REVIEW_TABLE)
+ extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{
+ RawDataSubTaskArgs: *rawDataSubTaskArgs,
+ Extract: func(row *helper.RawData) ([]interface{}, error) {
+ apiPullRequestReview := &PullRequestReview{}
+ if strings.HasPrefix(string(row.Data), "{\"message\": \"Not Found\"") {
+ return nil, nil
+ }
+ err := json.Unmarshal(row.Data, apiPullRequestReview)
+ if err != nil {
+ return nil, err
+ }
+ pull := &SimplePr{}
+ err = json.Unmarshal(row.Input, pull)
+ if err != nil {
+ return nil, err
+ }
+ // need to extract 2 kinds of entities here
+ results := make([]interface{}, 0, 1)
+
+ giteeReviewer := &models.GiteeReviewer{
+ GiteeId: apiPullRequestReview.User.Id,
+ Login: apiPullRequestReview.User.Login,
+ PullRequestId: pull.GiteeId,
+ }
+ results = append(results, giteeReviewer)
+
+ return results, nil
+ },
+ })
+
+ if err != nil {
+ return err
+ }
+
+ return extractor.Execute()
+}
diff --git a/plugins/gitee/tasks/repo_collector.go b/plugins/gitee/tasks/repo_collector.go
new file mode 100644
index 00000000..15111300
--- /dev/null
+++ b/plugins/gitee/tasks/repo_collector.go
@@ -0,0 +1,72 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "encoding/json"
+ "fmt"
+ "io/ioutil"
+ "net/http"
+ "net/url"
+
+ "github.com/apache/incubator-devlake/plugins/helper"
+
+ "github.com/apache/incubator-devlake/plugins/core"
+)
+
+const RAW_REPOSITORIES_TABLE = "gitee_api_repo"
+
+var CollectApiRepoMeta = core.SubTaskMeta{
+ Name: "collectApiRepo",
+ EntryPoint: CollectApiRepositories,
+ Required: true,
+ Description: "Collect repositories data from Gitee api",
+}
+
+func CollectApiRepositories(taskCtx core.SubTaskContext) error {
+ rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_REPOSITORIES_TABLE)
+
+ collector, err := helper.NewApiCollector(helper.ApiCollectorArgs{
+ RawDataSubTaskArgs: *rawDataSubTaskArgs,
+ ApiClient: data.ApiClient,
+ UrlTemplate: "repos/{{ .Params.Owner }}/{{ .Params.Repo }}",
+ Query: func(reqData *helper.RequestData) (url.Values, error) {
+ query := url.Values{}
+ query.Set("access_token", data.Options.Token)
+ query.Set("state", "all")
+ query.Set("page", fmt.Sprintf("%v", reqData.Pager.Page))
+ query.Set("direction", "asc")
+ query.Set("per_page", fmt.Sprintf("%v", reqData.Pager.Size))
+ return query, nil
+ },
+ ResponseParser: func(res *http.Response) ([]json.RawMessage, error) {
+ body, err := ioutil.ReadAll(res.Body)
+ res.Body.Close()
+ if err != nil {
+ return nil, err
+ }
+ return []json.RawMessage{body}, nil
+ },
+ })
+
+ if err != nil {
+ return err
+ }
+
+ return collector.Execute()
+}
diff --git a/plugins/gitee/tasks/repo_convertor.go b/plugins/gitee/tasks/repo_convertor.go
new file mode 100644
index 00000000..21cc8b82
--- /dev/null
+++ b/plugins/gitee/tasks/repo_convertor.go
@@ -0,0 +1,95 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "fmt"
+ "reflect"
+
+ "github.com/apache/incubator-devlake/models/domainlayer"
+ "github.com/apache/incubator-devlake/models/domainlayer/code"
+ "github.com/apache/incubator-devlake/models/domainlayer/didgen"
+ "github.com/apache/incubator-devlake/models/domainlayer/ticket"
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/gitee/models"
+ "github.com/apache/incubator-devlake/plugins/helper"
+)
+
+var ConvertRepoMeta = core.SubTaskMeta{
+ Name: "convertRepo",
+ EntryPoint: ConvertRepo,
+ EnabledByDefault: true,
+ Description: "Convert tool layer table gitee_repos into domain layer table repos and boards",
+}
+
+func ConvertRepo(taskCtx core.SubTaskContext) error {
+ rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_REPOSITORIES_TABLE)
+ db := taskCtx.GetDb()
+ repoId := data.Repo.GiteeId
+
+ cursor, err := db.Model(&models.GiteeRepo{}).
+ Where("gitee_id = ?", repoId).
+ Rows()
+ if err != nil {
+ return err
+ }
+ defer cursor.Close()
+
+ repoIdGen := didgen.NewDomainIdGenerator(&models.GiteeRepo{})
+
+ converter, err := helper.NewDataConverter(helper.DataConverterArgs{
+ InputRowType: reflect.TypeOf(models.GiteeRepo{}),
+ Input: cursor,
+ RawDataSubTaskArgs: *rawDataSubTaskArgs,
+ Convert: func(inputRow interface{}) ([]interface{}, error) {
+ repository := inputRow.(*models.GiteeRepo)
+ domainRepository := &code.Repo{
+ DomainEntity: domainlayer.DomainEntity{
+ Id: repoIdGen.Generate(repository.GiteeId),
+ },
+ Name: fmt.Sprintf("%s/%s", repository.OwnerLogin, repository.Name),
+ Url: repository.HTMLUrl,
+ Description: repository.Description,
+ ForkedFrom: repository.ParentHTMLUrl,
+ Language: repository.Language,
+ CreatedDate: repository.CreatedDate,
+ UpdatedDate: repository.UpdatedDate,
+ }
+
+ domainBoard := &ticket.Board{
+ DomainEntity: domainlayer.DomainEntity{
+ Id: repoIdGen.Generate(repository.GiteeId),
+ },
+ Name: fmt.Sprintf("%s/%s", repository.OwnerLogin, repository.Name),
+ Url: fmt.Sprintf("%s/%s", repository.HTMLUrl, "issues"),
+ Description: repository.Description,
+ CreatedDate: &repository.CreatedDate,
+ }
+
+ return []interface{}{
+ domainRepository,
+ domainBoard,
+ }, nil
+ },
+ })
+ if err != nil {
+ return err
+ }
+
+ return converter.Execute()
+}
diff --git a/plugins/gitee/tasks/repo_extractor.go b/plugins/gitee/tasks/repo_extractor.go
new file mode 100644
index 00000000..02ce4a7c
--- /dev/null
+++ b/plugins/gitee/tasks/repo_extractor.go
@@ -0,0 +1,90 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "encoding/json"
+ "fmt"
+
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/gitee/models"
+ "github.com/apache/incubator-devlake/plugins/helper"
+)
+
+var ExtractApiRepoMeta = core.SubTaskMeta{
+ Name: "extractApiRepo",
+ EntryPoint: ExtractApiRepositories,
+ Required: true,
+ Description: "Extract raw Repositories data into tool layer table gitee_repos",
+}
+
+type GiteeApiRepoResponse struct {
+ Name string `json:"name"`
+ GiteeId int `json:"id"`
+ HTMLUrl string `json:"html_url"`
+ Language string `json:"language"`
+ Description string `json:"description"`
+ Owner models.GiteeUser `json:"owner"`
+ Parent *GiteeApiRepoResponse `json:"parent"`
+ CreatedAt helper.Iso8601Time `json:"created_at"`
+ UpdatedAt *helper.Iso8601Time `json:"updated_at"`
+}
+
+func ExtractApiRepositories(taskCtx core.SubTaskContext) error {
+ rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_REPOSITORIES_TABLE)
+ extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{
+ RawDataSubTaskArgs: *rawDataSubTaskArgs,
+ Extract: func(row *helper.RawData) ([]interface{}, error) {
+ repo := &GiteeApiRepoResponse{}
+ err := json.Unmarshal(row.Data, repo)
+ if err != nil {
+ return nil, err
+ }
+ if repo.GiteeId == 0 {
+ return nil, fmt.Errorf("repo %s/%s not found", data.Options.Owner, data.Options.Repo)
+ }
+ results := make([]interface{}, 0, 1)
+ giteeRepository := &models.GiteeRepo{
+ GiteeId: repo.GiteeId,
+ Name: repo.Name,
+ HTMLUrl: repo.HTMLUrl,
+ Description: repo.Description,
+ OwnerId: repo.Owner.Id,
+ OwnerLogin: repo.Owner.Login,
+ Language: repo.Language,
+ CreatedDate: repo.CreatedAt.ToTime(),
+ UpdatedDate: helper.Iso8601TimeToTime(repo.UpdatedAt),
+ }
+ data.Repo = giteeRepository
+
+ if repo.Parent != nil {
+ giteeRepository.ParentGiteeId = repo.Parent.GiteeId
+ giteeRepository.ParentHTMLUrl = repo.Parent.HTMLUrl
+ }
+ results = append(results, giteeRepository)
+ taskCtx.TaskContext().GetData().(*GiteeTaskData).Repo = giteeRepository
+ return results, nil
+ },
+ })
+
+ if err != nil {
+ return err
+ }
+
+ return extractor.Execute()
+}
diff --git a/plugins/gitee/tasks/shared.go b/plugins/gitee/tasks/shared.go
new file mode 100644
index 00000000..48238dcb
--- /dev/null
+++ b/plugins/gitee/tasks/shared.go
@@ -0,0 +1,206 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io/ioutil"
+ "net/http"
+ "regexp"
+ "strconv"
+ "strings"
+ "time"
+
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/helper"
+)
+
+type PagingInfo struct {
+ Next int
+ Last int
+ First int
+ Prev int
+}
+
+type RateLimitInfo struct {
+ Date time.Time
+ ResetTime time.Time
+ Remaining int
+}
+
+type GiteeApiParams struct {
+ Repo string
+ Owner string
+ Token string
+}
+
+type GiteeInput struct {
+ Repo string
+ Owner string
+ Iid int
+}
+
+func GetTotalPagesFromResponse(res *http.Response, args *helper.ApiCollectorArgs) (int, error) {
+ total := res.Header.Get("X-Total-Pages")
+ if total == "" {
+ return 0, nil
+ }
+ totalInt, err := strconv.Atoi(total)
+ if err != nil {
+ return 0, err
+ }
+ return totalInt, nil
+}
+
+func GetRawMessageFromResponse(res *http.Response) ([]json.RawMessage, error) {
+ var rawMessages []json.RawMessage
+
+ if res == nil {
+ return nil, fmt.Errorf("res is nil")
+ }
+ defer res.Body.Close()
+ resBody, err := ioutil.ReadAll(res.Body)
+ if err != nil {
+ return nil, fmt.Errorf("%w %s", err, res.Request.URL.String())
+ }
+
+ err = json.Unmarshal(resBody, &rawMessages)
+ if err != nil {
+ return nil, fmt.Errorf("%w %s %s", err, res.Request.URL.String(), string(resBody))
+ }
+
+ return rawMessages, nil
+}
+
+func CreateRawDataSubTaskArgs(taskCtx core.SubTaskContext, Table string) (*helper.RawDataSubTaskArgs, *GiteeTaskData) {
+ data := taskCtx.GetData().(*GiteeTaskData)
+ RawDataSubTaskArgs := &helper.RawDataSubTaskArgs{
+ Ctx: taskCtx,
+ Params: GiteeApiParams{
+ Repo: data.Options.Repo,
+ Owner: data.Options.Owner,
+ Token: data.Options.Token,
+ },
+ Table: Table,
+ }
+ return RawDataSubTaskArgs, data
+}
+
+func ConvertRateLimitInfo(date string, resetTime string, remaining string) (RateLimitInfo, error) {
+ var rateLimitInfo RateLimitInfo
+ var err error
+ if date != "" {
+ rateLimitInfo.Date, err = http.ParseTime(date)
+ if err != nil {
+ return rateLimitInfo, err
+ }
+ } else {
+ return rateLimitInfo, errors.New("rate limit date was an empty string")
+ }
+ if resetTime != "" {
+ resetInt, err := strconv.ParseInt(resetTime, 10, 64)
+ if err != nil {
+ return rateLimitInfo, err
+ }
+ rateLimitInfo.ResetTime = time.Unix(resetInt, 0)
+ } else {
+ return rateLimitInfo, errors.New("rate limit reset time was an empty string")
+ }
+ if remaining != "" {
+ rateLimitInfo.Remaining, err = strconv.Atoi(remaining)
+ if err != nil {
+ return rateLimitInfo, err
+ }
+ } else {
+ return rateLimitInfo, errors.New("rate remaining was an empty string")
+ }
+ return rateLimitInfo, nil
+}
+
+func GetRateLimitPerSecond(info RateLimitInfo) int {
+ unixResetTime := info.ResetTime.Unix()
+ unixNow := info.Date.Unix()
+ timeBetweenNowAndReset := unixResetTime - unixNow
+ // Adjust the remaining to be less then actual to avoid hitting the limit exactly.
+ multiplier := 0.98
+ adjustedRemaining := float64(info.Remaining) * multiplier
+ return int(adjustedRemaining / float64(timeBetweenNowAndReset)) //* multiplier
+}
+func ConvertStringToInt(input string) (int, error) {
+ return strconv.Atoi(input)
+}
+func GetPagingFromLinkHeader(link string) (PagingInfo, error) {
+ result := PagingInfo{
+ Next: 1,
+ Last: 1,
+ Prev: 1,
+ First: 1,
+ }
+ linksArray := strings.Split(link, ",")
+ pattern1 := regexp.MustCompile(`page=*[0-9]+`)
+ pattern2 := regexp.MustCompile(`rel="*[a-z]+`)
+ if len(linksArray) >= 2 {
+ for i := 0; i < len(linksArray); i++ {
+ content := []byte(linksArray[i])
+ loc1 := pattern1.FindIndex(content)
+ loc2 := pattern2.FindIndex(content)
+ if len(loc1) >= 2 && len(loc2) >= 2 {
+ pageNumberSubstring := string(content[loc1[0]:loc1[1]])
+ pageNumberString := strings.Replace(pageNumberSubstring, `page=`, ``, 1)
+ pageNameSubstring := string(content[loc2[0]:loc2[1]])
+ pageNameString := strings.Replace(pageNameSubstring, `rel="`, ``, 1)
+
+ pageNumberInt, convertErr := ConvertStringToInt(pageNumberString)
+ if convertErr != nil {
+ return result, convertErr
+ }
+ switch pageNameString {
+ case "next":
+ result.Next = pageNumberInt
+
+ case "first":
+ result.First = pageNumberInt
+
+ case "last":
+ result.Last = pageNumberInt
+
+ case "prev":
+ result.Prev = pageNumberInt
+ }
+
+ } else {
+ return result, errors.New("parsed string values aren't long enough")
+ }
+ }
+ return result, nil
+ } else {
+ return result, errors.New("the link string provided is invalid. There is likely no next page of data to fetch")
+ }
+}
+
+func GetIssueIdByIssueUrl(s string) (int, error) {
+ regex := regexp.MustCompile(`.*/issues/(\d+)`)
+ groups := regex.FindStringSubmatch(s)
+ if len(groups) > 0 {
+ return strconv.Atoi(groups[1])
+ } else {
+ return 0, errors.New("invalid issue url")
+ }
+}
diff --git a/plugins/gitee/tasks/task_data.go b/plugins/gitee/tasks/task_data.go
new file mode 100644
index 00000000..76956641
--- /dev/null
+++ b/plugins/gitee/tasks/task_data.go
@@ -0,0 +1,41 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "time"
+
+ "github.com/apache/incubator-devlake/plugins/gitee/models"
+ "github.com/apache/incubator-devlake/plugins/helper"
+)
+
+type GiteeOptions struct {
+ Tasks []string `json:"tasks,omitempty"`
+ Since string
+ Owner string
+ Repo string
+ Token string
+ models.Config
+}
+
+type GiteeTaskData struct {
+ Options *GiteeOptions
+ ApiClient *helper.ApiAsyncClient
+ Repo *models.GiteeRepo
+ Since *time.Time
+}
diff --git a/plugins/gitee/tasks/user_convertor.go b/plugins/gitee/tasks/user_convertor.go
new file mode 100644
index 00000000..4c149f4e
--- /dev/null
+++ b/plugins/gitee/tasks/user_convertor.go
@@ -0,0 +1,72 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "reflect"
+
+ "github.com/apache/incubator-devlake/models/domainlayer"
+ "github.com/apache/incubator-devlake/models/domainlayer/didgen"
+ "github.com/apache/incubator-devlake/models/domainlayer/user"
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/gitee/models"
+ "github.com/apache/incubator-devlake/plugins/helper"
+)
+
+var ConvertUsersMeta = core.SubTaskMeta{
+ Name: "convertUsers",
+ EntryPoint: ConvertUsers,
+ EnabledByDefault: true,
+ Description: "Convert tool layer table gitee_users into domain layer table users",
+}
+
+func ConvertUsers(taskCtx core.SubTaskContext) error {
+ db := taskCtx.GetDb()
+ rawDataSubTaskArgs, _ := CreateRawDataSubTaskArgs(taskCtx, RAW_COMMIT_TABLE)
+
+ cursor, err := db.Model(&models.GiteeUser{}).
+ Rows()
+ if err != nil {
+ return err
+ }
+ defer cursor.Close()
+
+ userIdGen := didgen.NewDomainIdGenerator(&models.GiteeUser{})
+
+ converter, err := helper.NewDataConverter(helper.DataConverterArgs{
+ InputRowType: reflect.TypeOf(models.GiteeUser{}),
+ Input: cursor,
+ RawDataSubTaskArgs: *rawDataSubTaskArgs,
+ Convert: func(inputRow interface{}) ([]interface{}, error) {
+ GiteeUser := inputRow.(*models.GiteeUser)
+ domainUser := &user.User{
+ DomainEntity: domainlayer.DomainEntity{Id: userIdGen.Generate(GiteeUser.Id)},
+ Name: GiteeUser.Login,
+ AvatarUrl: GiteeUser.AvatarUrl,
+ }
+ return []interface{}{
+ domainUser,
+ }, nil
+ },
+ })
+ if err != nil {
+ return err
+ }
+
+ return converter.Execute()
+}
[incubator-devlake] 03/10: add asf license and env.example
Posted by wa...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
warren pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit a86c5596ee4e65ea624446c6adaa048b4a30be82
Author: Mr.An <42...@users.noreply.github.com>
AuthorDate: Wed Jun 15 12:28:23 2022 +0800
add asf license and env.example
---
.env.example | 24 ++++++++++++++++++++++++
plugins/gitee/api/connection.go | 17 +++++++++++++++++
plugins/gitee/gitee.go | 17 +++++++++++++++++
plugins/gitee/impl/impl.go | 17 +++++++++++++++++
plugins/gitee/tasks/commit_extractor.go | 17 +++++++++++++++++
5 files changed, 92 insertions(+)
diff --git a/.env.example b/.env.example
index 80dc9e36..c5254985 100644
--- a/.env.example
+++ b/.env.example
@@ -48,6 +48,30 @@ GITLAB_AUTH=
GITLAB_PROXY=
GITLAB_API_REQUESTS_PER_HOUR=
+########################
+# Gitee configuration #
+########################
+GITEE_ENDPOINT="https://gitee.com/api/v5/"
+GITEE_AUTH=
+GITEE_PROXY=
+GITEE_API_REQUESTS_PER_HOUR=
+# GITEE_PR_TYPE=type/(.*)$ the program will extract the value in (), in this example, you will get "refactor" from "type/refactor"
+GITEE_PR_TYPE="type/(.*)$"
+# GITEE_PR_COMPONENT=component/(.*)$ the program will extract the value in (), in this example, you will get "plugins" from "component/plugins"
+GITEE_PR_COMPONENT="component/(.*)$"
+# GITEE_ISSUE_SEVERITY=severity/(.*)$ the program will extract the value in (), in this example, you will get "refactor" from "type/refactor"
+GITEE_ISSUE_SEVERITY="severity/(.*)$"
+# GITEE_ISSUE_COMPONENT=component/(.*)$ the program will extract the value in (), in this example, you will get "refactor" from "type/refactor"
+GITEE_ISSUE_COMPONENT="component/(.*)$"
+GITEE_ISSUE_PRIORITY="^(highest|high|medium|low)$"
+GITEE_ISSUE_TYPE_BUG="^(bug|failure|error)$"
+GITEE_ISSUE_TYPE_REQUIREMENT="^(feat|feature|proposal|requirement)$"
+GITEE_ISSUE_TYPE_INCIDENT=""
+# GITEE_PR_BODY_CLOSE_PATTERN='(?mi)(fix|close|resolve|fixes|closes|resolves|fixed|closed|resolved)[\s]*.*(((and )?(#|https:\/\/github.com\/%s\/%s\/issues\/)\d+[ ]*)+)'
+GITEE_PR_BODY_CLOSE_PATTERN="(?mi)(fix|close|resolve|fixes|closes|resolves|fixed|closed|resolved)[\\s]*.*(((and )?(#|https:\\/\\/github.com\\/%s\\/%s\\/issues\\/)\\d+[ ]*)+)"
+# GITEE_PR_TITLE_PATTERN='.*\(#(\d+)\)'
+GITEE_PR_TITLE_PATTERN=".*\\(#(\\d+)\\)"
+
##########################
# Jira <> Gitlab mapping #
##########################
diff --git a/plugins/gitee/api/connection.go b/plugins/gitee/api/connection.go
index 98884b8a..7c912eb9 100644
--- a/plugins/gitee/api/connection.go
+++ b/plugins/gitee/api/connection.go
@@ -1,3 +1,20 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
package api
import (
diff --git a/plugins/gitee/gitee.go b/plugins/gitee/gitee.go
index e4faa69f..c6f19cf9 100644
--- a/plugins/gitee/gitee.go
+++ b/plugins/gitee/gitee.go
@@ -1,3 +1,20 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
package main
import (
diff --git a/plugins/gitee/impl/impl.go b/plugins/gitee/impl/impl.go
index b452a44d..c1e4d799 100644
--- a/plugins/gitee/impl/impl.go
+++ b/plugins/gitee/impl/impl.go
@@ -1,3 +1,20 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
package impl
import (
diff --git a/plugins/gitee/tasks/commit_extractor.go b/plugins/gitee/tasks/commit_extractor.go
index 484a6230..3f280af5 100644
--- a/plugins/gitee/tasks/commit_extractor.go
+++ b/plugins/gitee/tasks/commit_extractor.go
@@ -1,3 +1,20 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
package tasks
import (
[incubator-devlake] 09/10: remove gitee unused configuration information
Posted by wa...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
warren pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit 5d57ed678e7252190558408fbe23d9f82a54d97f
Author: Mr.An <42...@users.noreply.github.com>
AuthorDate: Sat Jun 18 17:03:05 2022 +0800
remove gitee unused configuration information
---
.env.example | 16 ----------------
1 file changed, 16 deletions(-)
diff --git a/.env.example b/.env.example
index c5254985..b8343150 100644
--- a/.env.example
+++ b/.env.example
@@ -55,22 +55,6 @@ GITEE_ENDPOINT="https://gitee.com/api/v5/"
GITEE_AUTH=
GITEE_PROXY=
GITEE_API_REQUESTS_PER_HOUR=
-# GITEE_PR_TYPE=type/(.*)$ the program will extract the value in (), in this example, you will get "refactor" from "type/refactor"
-GITEE_PR_TYPE="type/(.*)$"
-# GITEE_PR_COMPONENT=component/(.*)$ the program will extract the value in (), in this example, you will get "plugins" from "component/plugins"
-GITEE_PR_COMPONENT="component/(.*)$"
-# GITEE_ISSUE_SEVERITY=severity/(.*)$ the program will extract the value in (), in this example, you will get "refactor" from "type/refactor"
-GITEE_ISSUE_SEVERITY="severity/(.*)$"
-# GITEE_ISSUE_COMPONENT=component/(.*)$ the program will extract the value in (), in this example, you will get "refactor" from "type/refactor"
-GITEE_ISSUE_COMPONENT="component/(.*)$"
-GITEE_ISSUE_PRIORITY="^(highest|high|medium|low)$"
-GITEE_ISSUE_TYPE_BUG="^(bug|failure|error)$"
-GITEE_ISSUE_TYPE_REQUIREMENT="^(feat|feature|proposal|requirement)$"
-GITEE_ISSUE_TYPE_INCIDENT=""
-# GITEE_PR_BODY_CLOSE_PATTERN='(?mi)(fix|close|resolve|fixes|closes|resolves|fixed|closed|resolved)[\s]*.*(((and )?(#|https:\/\/github.com\/%s\/%s\/issues\/)\d+[ ]*)+)'
-GITEE_PR_BODY_CLOSE_PATTERN="(?mi)(fix|close|resolve|fixes|closes|resolves|fixed|closed|resolved)[\\s]*.*(((and )?(#|https:\\/\\/github.com\\/%s\\/%s\\/issues\\/)\\d+[ ]*)+)"
-# GITEE_PR_TITLE_PATTERN='.*\(#(\d+)\)'
-GITEE_PR_TITLE_PATTERN=".*\\(#(\\d+)\\)"
##########################
# Jira <> Gitlab mapping #