You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by wa...@apache.org on 2022/06/18 09:24:19 UTC
[incubator-devlake] 01/10: init commite gitee plugin
This is an automated email from the ASF dual-hosted git repository.
warren pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit 9f239e8a8fbdbcd9e4600678533d4aa6773e1307
Author: Mr.An <42...@users.noreply.github.com>
AuthorDate: Wed Jun 15 12:08:55 2022 +0800
init commite gitee plugin
---
plugins/gitee/README.md | 121 ++++++++++
plugins/gitee/api/connection.go | 131 +++++++++++
plugins/gitee/gitee.go | 27 +++
plugins/gitee/impl/impl.go | 104 +++++++++
plugins/gitee/models/commit.go | 47 ++++
plugins/gitee/models/commit_stats.go | 36 +++
plugins/gitee/models/connection.go | 64 ++++++
plugins/gitee/models/issue.go | 52 +++++
plugins/gitee/models/issue_comment.go | 39 ++++
plugins/gitee/models/issue_label.go | 35 +++
.../models/migrationscripts/archived/commit.go | 47 ++++
.../migrationscripts/archived/commit_stat.go | 36 +++
.../models/migrationscripts/archived/issue.go | 52 +++++
.../migrationscripts/archived/issue_comment.go | 39 ++++
.../migrationscripts/archived/issue_label.go | 33 +++
.../migrationscripts/archived/pull_request.go | 59 +++++
.../archived/pull_request_comment.go | 39 ++++
.../archived/pull_request_commit.go | 30 +++
.../archived/pull_request_issue.go | 32 +++
.../archived/pull_request_label.go | 30 +++
.../gitee/models/migrationscripts/archived/repo.go | 43 ++++
.../migrationscripts/archived/repo_commit.go | 32 +++
.../models/migrationscripts/archived/reviewer.go | 32 +++
.../gitee/models/migrationscripts/archived/user.go | 47 ++++
.../gitee/models/migrationscripts/init_schema.go | 54 +++++
plugins/gitee/models/pull_request.go | 59 +++++
plugins/gitee/models/pull_request_comment.go | 39 ++++
plugins/gitee/models/pull_request_commit.go | 32 +++
plugins/gitee/models/pull_request_issue.go | 32 +++
plugins/gitee/models/pull_request_label.go | 32 +++
plugins/gitee/models/repo.go | 43 ++++
plugins/gitee/models/repo_commit.go | 32 +++
plugins/gitee/models/reviewer.go | 34 +++
plugins/gitee/models/user.go | 45 ++++
plugins/gitee/tasks/api_client.go | 84 +++++++
plugins/gitee/tasks/commit_collector.go | 64 ++++++
plugins/gitee/tasks/commit_convertor.go | 102 +++++++++
plugins/gitee/tasks/commit_extractor.go | 111 +++++++++
plugins/gitee/tasks/commit_stats_collector.go | 112 +++++++++
plugins/gitee/tasks/commit_stats_extractor.go | 98 ++++++++
plugins/gitee/tasks/issue_collector.go | 97 ++++++++
plugins/gitee/tasks/issue_comment_collector.go | 110 +++++++++
plugins/gitee/tasks/issue_comment_convertor.go | 80 +++++++
plugins/gitee/tasks/issue_comment_extractor.go | 119 ++++++++++
plugins/gitee/tasks/issue_convertor.go | 101 ++++++++
plugins/gitee/tasks/issue_extractor.go | 253 +++++++++++++++++++++
plugins/gitee/tasks/issue_label_convertor.go | 73 ++++++
plugins/gitee/tasks/pr_collector.go | 99 ++++++++
plugins/gitee/tasks/pr_comment_convertor.go | 82 +++++++
plugins/gitee/tasks/pr_commit_collector.go | 96 ++++++++
plugins/gitee/tasks/pr_commit_convertor.go | 73 ++++++
plugins/gitee/tasks/pr_commit_extractor.go | 127 +++++++++++
plugins/gitee/tasks/pr_convertor.go | 92 ++++++++
plugins/gitee/tasks/pr_extractor.go | 171 ++++++++++++++
plugins/gitee/tasks/pr_issue_convertor.go | 78 +++++++
plugins/gitee/tasks/pr_issue_enricher.go | 116 ++++++++++
plugins/gitee/tasks/pr_label_convertor.go | 73 ++++++
plugins/gitee/tasks/pr_review_collector.go | 89 ++++++++
plugins/gitee/tasks/pr_review_extractor.go | 84 +++++++
plugins/gitee/tasks/repo_collector.go | 72 ++++++
plugins/gitee/tasks/repo_convertor.go | 95 ++++++++
plugins/gitee/tasks/repo_extractor.go | 90 ++++++++
plugins/gitee/tasks/shared.go | 206 +++++++++++++++++
plugins/gitee/tasks/task_data.go | 41 ++++
plugins/gitee/tasks/user_convertor.go | 72 ++++++
65 files changed, 4769 insertions(+)
diff --git a/plugins/gitee/README.md b/plugins/gitee/README.md
new file mode 100644
index 00000000..98a51514
--- /dev/null
+++ b/plugins/gitee/README.md
@@ -0,0 +1,121 @@
+# Gitee Pond
+
+<div align="center">
+
+| [English](README.md) | [中文](README-zh-CN.md) |
+| --- | --- |
+
+</div>
+
+<br>
+
+## Summary
+
+## Configuration
+
+### Provider (Datasource) Connection
+The connection aspect of the configuration screen requires the following key fields to connect to the **Gitee API**. As gitee is a _single-source data provider_ at the moment, the connection name is read-only as there is only one instance to manage. As we continue our development roadmap we may enable _multi-source_ connections for gitee in the future.
+
+- **Connection Name** [`READONLY`]
+ - ⚠️ Defaults to "**Gitee**" and may not be changed.
+- **Endpoint URL** (REST URL, starts with `https://` or `http://`)
+ - This should be a valid REST API Endpoint eg. `https://gitee.com/api/v5/`
+ - ⚠️ URL should end with`/`
+- **Auth Token(s)** (Personal Access Token)
+ - For help on **Creating a personal access token**
+ - Provide at least one token for Authentication with the . This field accepts a comma-separated list of values for multiple tokens. The data collection will take longer for gitee since they have a **rate limit of 2k requests per hour**. You can accelerate the process by configuring _multiple_ personal access tokens.
+
+"For API requests using `Basic Authentication` or `OAuth`
+
+
+If you have a need for more api rate limits, you can set many tokens in the config file and we will use all of your tokens.
+
+For an overview of the **gitee REST API**, please see official [gitee Docs on REST](https://gitee.com/api/v5/swagger)
+
+Click **Save Connection** to update connection settings.
+
+
+### Provider (Datasource) Settings
+Manage additional settings and options for the gitee Datasource Provider. Currently there is only one **optional** setting, *Proxy URL*. If you are behind a corporate firewall or VPN you may need to utilize a proxy server.
+
+**gitee Proxy URL [ `Optional`]**
+Enter a valid proxy server address on your Network, e.g. `http://your-proxy-server.com:1080`
+
+Click **Save Settings** to update additional settings.
+
+### Regular Expression Configuration
+Define regex pattern in .env
+- gitee_PR_BODY_CLOSE_PATTERN: Define key word to associate issue in pr body, please check the example in .env.example
+
+## Sample Request
+In order to collect data, you have to compose a JSON looks like following one, and send it by selecting `Advanced Mode` on `Create Pipeline Run` page:
+1. Configure-UI Mode
+```json
+[
+ [
+ {
+ "plugin": "gitee",
+ "options": {
+ "repo": "lake",
+ "owner": "merico-dev",
+ "token": "xxxx"
+ }
+ }
+ ]
+]
+```
+and if you want to perform certain subtasks.
+```json
+[
+ [
+ {
+ "plugin": "gitee",
+ "subtasks": ["collectXXX", "extractXXX", "convertXXX"],
+ "options": {
+ "repo": "lake",
+ "owner": "merico-dev",
+ "token": "xxxx"
+ }
+ }
+ ]
+]
+```
+
+2. Curl Mode:
+ You can also trigger data collection by making a POST request to `/pipelines`.
+```
+curl --location --request POST 'localhost:8080/pipelines' \
+--header 'Content-Type: application/json' \
+--data-raw '
+{
+ "name": "gitee 20211126",
+ "tasks": [[{
+ "plugin": "gitee",
+ "options": {
+ "repo": "lake",
+ "owner": "merico-dev"
+ "token": "xxxx"
+ }
+ }]]
+}
+'
+```
+and if you want to perform certain subtasks.
+```
+curl --location --request POST 'localhost:8080/pipelines' \
+--header 'Content-Type: application/json' \
+--data-raw '
+{
+ "name": "gitee 20211126",
+ "tasks": [[{
+ "plugin": "gitee",
+ "subtasks": ["collectXXX", "extractXXX", "convertXXX"],
+ "options": {
+ "repo": "lake",
+ "owner": "merico-dev"
+ "token": "xxxx"
+ }
+ }]]
+}
+'
+```
diff --git a/plugins/gitee/api/connection.go b/plugins/gitee/api/connection.go
new file mode 100644
index 00000000..98884b8a
--- /dev/null
+++ b/plugins/gitee/api/connection.go
@@ -0,0 +1,131 @@
+package api
+
+import (
+ "fmt"
+ "net/http"
+ "net/url"
+ "time"
+
+ "github.com/apache/incubator-devlake/config"
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/gitee/models"
+ "github.com/apache/incubator-devlake/plugins/helper"
+
+ "github.com/go-playground/validator/v10"
+ "github.com/mitchellh/mapstructure"
+)
+
+var vld = validator.New()
+
+/*
+POST /plugins/gitee/test
+*/
+func TestConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) {
+ // decode
+ var err error
+ var connection models.TestConnectionRequest
+ err = mapstructure.Decode(input.Body, &connection)
+ if err != nil {
+ return nil, err
+ }
+ // validate
+ err = vld.Struct(connection)
+ if err != nil {
+ return nil, err
+ }
+ // test connection
+ apiClient, err := helper.NewApiClient(
+ connection.Endpoint,
+ nil,
+ 3*time.Second,
+ connection.Proxy,
+ nil,
+ )
+ if err != nil {
+ return nil, err
+ }
+ query := make(url.Values)
+ query["access_token"] = []string{connection.Auth}
+
+ res, err := apiClient.Get("user", query, nil)
+ if err != nil {
+ return nil, err
+ }
+ resBody := &models.ApiUserResponse{}
+ err = helper.UnmarshalResponse(res, resBody)
+ if err != nil {
+ return nil, err
+ }
+
+ if res.StatusCode != http.StatusOK {
+ return nil, fmt.Errorf("unexpected status code: %d", res.StatusCode)
+ }
+ return nil, nil
+}
+
+/*
+PATCH /plugins/gitee/connections/:connectionId
+*/
+func PatchConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) {
+ v := config.GetConfig()
+ connection := &models.GiteeConnection{}
+ err := helper.EncodeStruct(v, connection, "env")
+ if err != nil {
+ return nil, err
+ }
+ // update from request and save to .env
+ err = helper.DecodeStruct(v, connection, input.Body, "env")
+ if err != nil {
+ return nil, err
+ }
+ err = config.WriteConfig(v)
+ if err != nil {
+ return nil, err
+ }
+ response := models.GiteeResponse{
+ GiteeConnection: *connection,
+ Name: "Gitee",
+ ID: 1,
+ }
+ return &core.ApiResourceOutput{Body: response, Status: http.StatusOK}, nil
+}
+
+/*
+GET /plugins/gitee/connections
+*/
+func ListConnections(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) {
+ // RETURN ONLY 1 SOURCE (FROM ENV) until multi-connection is developed.
+ v := config.GetConfig()
+ connection := &models.GiteeConnection{}
+
+ err := helper.EncodeStruct(v, connection, "env")
+ if err != nil {
+ return nil, err
+ }
+ response := models.GiteeResponse{
+ GiteeConnection: *connection,
+ Name: "Gitee",
+ ID: 1,
+ }
+
+ return &core.ApiResourceOutput{Body: []models.GiteeResponse{response}}, nil
+}
+
+/*
+GET /plugins/gitee/connections/:connectionId
+*/
+func GetConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) {
+ // RETURN ONLY 1 SOURCE FROM ENV (Ignore ID until multi-connection is developed.)
+ v := config.GetConfig()
+ connection := &models.GiteeConnection{}
+ err := helper.EncodeStruct(v, connection, "env")
+ if err != nil {
+ return nil, err
+ }
+ response := &models.GiteeResponse{
+ GiteeConnection: *connection,
+ Name: "Gitee",
+ ID: 1,
+ }
+ return &core.ApiResourceOutput{Body: response}, nil
+}
diff --git a/plugins/gitee/gitee.go b/plugins/gitee/gitee.go
new file mode 100644
index 00000000..e4faa69f
--- /dev/null
+++ b/plugins/gitee/gitee.go
@@ -0,0 +1,27 @@
+package main
+
+import (
+ "github.com/apache/incubator-devlake/plugins/gitee/impl"
+ "github.com/apache/incubator-devlake/runner"
+ "github.com/spf13/cobra"
+)
+
+var PluginEntry impl.Gitee //nolint
+
+func main() {
+ giteeCmd := &cobra.Command{Use: "gitee"}
+ owner := giteeCmd.Flags().StringP("owner", "o", "", "gitee owner")
+ repo := giteeCmd.Flags().StringP("repo", "r", "", "gitee repo")
+ token := giteeCmd.Flags().StringP("auth", "a", "", "access token")
+ _ = giteeCmd.MarkFlagRequired("owner")
+ _ = giteeCmd.MarkFlagRequired("repo")
+
+ giteeCmd.Run = func(cmd *cobra.Command, args []string) {
+ runner.DirectRun(cmd, args, PluginEntry, map[string]interface{}{
+ "owner": *owner,
+ "repo": *repo,
+ "token": *token,
+ })
+ }
+ runner.RunCmd(giteeCmd)
+}
diff --git a/plugins/gitee/impl/impl.go b/plugins/gitee/impl/impl.go
new file mode 100644
index 00000000..b452a44d
--- /dev/null
+++ b/plugins/gitee/impl/impl.go
@@ -0,0 +1,104 @@
+package impl
+
+import (
+ "github.com/apache/incubator-devlake/migration"
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/gitee/api"
+ "github.com/apache/incubator-devlake/plugins/gitee/models/migrationscripts"
+ "github.com/apache/incubator-devlake/plugins/gitee/tasks"
+ "github.com/mitchellh/mapstructure"
+ "github.com/spf13/viper"
+ "gorm.io/gorm"
+)
+
+var _ core.PluginMeta = (*Gitee)(nil)
+var _ core.PluginInit = (*Gitee)(nil)
+var _ core.PluginTask = (*Gitee)(nil)
+var _ core.PluginApi = (*Gitee)(nil)
+var _ core.Migratable = (*Gitee)(nil)
+
+type Gitee string
+
+func (plugin Gitee) Init(config *viper.Viper, logger core.Logger, db *gorm.DB) error {
+ return nil
+}
+
+func (plugin Gitee) Description() string {
+ return "To collect and enrich data from Gitee"
+}
+
+func (plugin Gitee) SubTaskMetas() []core.SubTaskMeta {
+ return []core.SubTaskMeta{
+ tasks.CollectApiRepoMeta,
+ tasks.ExtractApiRepoMeta,
+ tasks.CollectApiIssuesMeta,
+ tasks.ExtractApiIssuesMeta,
+ tasks.CollectCommitsMeta,
+ tasks.ExtractCommitsMeta,
+ tasks.CollectApiPullRequestsMeta,
+ tasks.ExtractApiPullRequestsMeta,
+ tasks.CollectApiIssueCommentsMeta,
+ tasks.ExtractApiIssueCommentsMeta,
+ tasks.CollectApiPullRequestCommitsMeta,
+ tasks.ExtractApiPullRequestCommitsMeta,
+ //tasks.CollectApiPullRequestReviewsMeta,
+ //tasks.ExtractApiPullRequestReviewsMeta,
+ tasks.CollectApiCommitStatsMeta,
+ tasks.ExtractApiCommitStatsMeta,
+ tasks.EnrichPullRequestIssuesMeta,
+ tasks.ConvertRepoMeta,
+ tasks.ConvertIssuesMeta,
+ tasks.ConvertCommitsMeta,
+ tasks.ConvertIssueLabelsMeta,
+ tasks.ConvertPullRequestCommitsMeta,
+ tasks.ConvertPullRequestsMeta,
+ tasks.ConvertPullRequestLabelsMeta,
+ tasks.ConvertPullRequestIssuesMeta,
+ tasks.ConvertUsersMeta,
+ tasks.ConvertIssueCommentsMeta,
+ tasks.ConvertPullRequestCommentsMeta,
+ tasks.ConvertPullRequestsMeta,
+ }
+}
+
+func (plugin Gitee) PrepareTaskData(taskCtx core.TaskContext, options map[string]interface{}) (interface{}, error) {
+ var op tasks.GiteeOptions
+ var err error
+ err = mapstructure.Decode(options, &op)
+ if err != nil {
+ return nil, err
+ }
+
+ apiClient, err := tasks.NewGiteeApiClient(taskCtx)
+ if err != nil {
+ return nil, err
+ }
+
+ return &tasks.GiteeTaskData{
+ Options: &op,
+ ApiClient: apiClient,
+ }, nil
+}
+
+func (plugin Gitee) RootPkgPath() string {
+ return "github.com/apache/incubator-devlake/plugins/gitee"
+}
+
+func (plugin Gitee) MigrationScripts() []migration.Script {
+ return []migration.Script{new(migrationscripts.InitSchemas), new(migrationscripts.InitSchemas)}
+}
+
+func (plugin Gitee) ApiResources() map[string]map[string]core.ApiResourceHandler {
+ return map[string]map[string]core.ApiResourceHandler{
+ "test": {
+ "POST": api.TestConnection,
+ },
+ "connections": {
+ "GET": api.ListConnections,
+ },
+ "connections/:connectionId": {
+ "GET": api.GetConnection,
+ "PATCH": api.PatchConnection,
+ },
+ }
+}
diff --git a/plugins/gitee/models/commit.go b/plugins/gitee/models/commit.go
new file mode 100644
index 00000000..4699d549
--- /dev/null
+++ b/plugins/gitee/models/commit.go
@@ -0,0 +1,47 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package models
+
+import (
+ "time"
+
+ "github.com/apache/incubator-devlake/models/common"
+)
+
+type GiteeCommit struct {
+ Sha string `gorm:"primaryKey;type:varchar(40)"`
+ CommentsUrl string `gorm:"type:varchar(255)"`
+ Message string
+ AuthorId int
+ AuthorName string `gorm:"type:varchar(255)"`
+ AuthorEmail string `gorm:"type:varchar(255)"`
+ AuthoredDate time.Time
+ CommitterId int
+ CommitterName string `gorm:"type:varchar(255)"`
+ CommitterEmail string `gorm:"type:varchar(255)"`
+ CommittedDate time.Time
+ WebUrl string `gorm:"type:varchar(255)"`
+ Additions int `gorm:"comment:Added lines of code"`
+ Deletions int `gorm:"comment:Deleted lines of code"`
+ Total int `gorm:"comment:Sum of added/deleted lines of code"`
+ common.NoPKModel
+}
+
+func (GiteeCommit) TableName() string {
+ return "_tool_gitee_commits"
+}
diff --git a/plugins/gitee/models/commit_stats.go b/plugins/gitee/models/commit_stats.go
new file mode 100644
index 00000000..d4992490
--- /dev/null
+++ b/plugins/gitee/models/commit_stats.go
@@ -0,0 +1,36 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package models
+
+import (
+ "time"
+
+ "github.com/apache/incubator-devlake/models/common"
+)
+
+type GiteeCommitStat struct {
+ Sha string `gorm:"primaryKey;type:varchar(40)"`
+ Additions int `gorm:"comment:Added lines of code"`
+ Deletions int `gorm:"comment:Deleted lines of code"`
+ CommittedDate time.Time `gorm:"index"`
+ common.NoPKModel
+}
+
+func (GiteeCommitStat) TableName() string {
+ return "_tool_gitee_commit_stats"
+}
diff --git a/plugins/gitee/models/connection.go b/plugins/gitee/models/connection.go
new file mode 100644
index 00000000..24ad1b6d
--- /dev/null
+++ b/plugins/gitee/models/connection.go
@@ -0,0 +1,64 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package models
+
+// This object conforms to what the frontend currently sends.
+type GiteeConnection struct {
+ Endpoint string `mapstructure:"endpoint" validate:"required" env:"GITEE_ENDPOINT" json:"endpoint"`
+ Auth string `mapstructure:"auth" validate:"required" env:"GITEE_AUTH" json:"auth"`
+ Proxy string `mapstructure:"proxy" env:"GITEE_PROXY" json:"proxy"`
+}
+
+// This object conforms to what the frontend currently expects.
+type GiteeResponse struct {
+ Name string `json:"name"`
+ ID int `json:"id"`
+ GiteeConnection
+}
+
+// Using User because it requires authentication.
+type ApiUserResponse struct {
+ Id int
+ Name string `json:"name"`
+}
+
+type TestConnectionRequest struct {
+ Endpoint string `json:"endpoint" validate:"required"`
+ Auth string `json:"auth" validate:"required"`
+ Proxy string `json:"proxy"`
+}
+
+type Config struct {
+ PrType string `mapstructure:"prType" env:"GITEE_PR_TYPE" json:"prType"`
+ PrComponent string `mapstructure:"prComponent" env:"GITEE_PR_COMPONENT" json:"prComponent"`
+ IssueSeverity string `mapstructure:"issueSeverity" env:"GITEE_ISSUE_SEVERITY" json:"issueSeverity"`
+ IssuePriority string `mapstructure:"issuePriority" env:"GITEE_ISSUE_PRIORITY" json:"issuePriority"`
+ IssueComponent string `mapstructure:"issueComponent" env:"GITEE_ISSUE_COMPONENT" json:"issueComponent"`
+ IssueTypeBug string `mapstructure:"issueTypeBug" env:"GITEE_ISSUE_TYPE_BUG" json:"issueTypeBug"`
+ IssueTypeIncident string `mapstructure:"issueTypeIncident" env:"GITEE_ISSUE_TYPE_INCIDENT" json:"issueTypeIncident"`
+ IssueTypeRequirement string `mapstructure:"issueTypeRequirement" env:"GITEE_ISSUE_TYPE_REQUIREMENT" json:"issueTypeRequirement"`
+}
+
+// Using Public Email because it requires authentication, and it is public information anyway.
+// We're not using email information for anything here.
+type PublicEmail struct {
+ Email string
+ Primary bool
+ Verified bool
+ Visibility string
+}
diff --git a/plugins/gitee/models/issue.go b/plugins/gitee/models/issue.go
new file mode 100644
index 00000000..aa2ea170
--- /dev/null
+++ b/plugins/gitee/models/issue.go
@@ -0,0 +1,52 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package models
+
+import (
+ "time"
+
+ "github.com/apache/incubator-devlake/models/common"
+)
+
+type GiteeIssue struct {
+ GiteeId int `gorm:"primaryKey"`
+ RepoId int `gorm:"index"`
+ Number string `gorm:"index;comment:Used in API requests ex. api/repo/1/issue/<THIS_NUMBER>"`
+ State string `gorm:"type:varchar(255)"`
+ Title string
+ Body string
+ Priority string `gorm:"type:varchar(255)"`
+ Type string `gorm:"type:varchar(100)"`
+ Status string `gorm:"type:varchar(255)"`
+ AuthorId int
+ AuthorName string `gorm:"type:varchar(255)"`
+ AssigneeId int
+ AssigneeName string `gorm:"type:varchar(255)"`
+ LeadTimeMinutes uint
+ Url string `gorm:"type:varchar(255)"`
+ ClosedAt *time.Time
+ GiteeCreatedAt time.Time
+ GiteeUpdatedAt time.Time `gorm:"index"`
+ Severity string `gorm:"type:varchar(255)"`
+ Component string `gorm:"type:varchar(255)"`
+ common.NoPKModel
+}
+
+func (GiteeIssue) TableName() string {
+ return "_tool_gitee_issues"
+}
diff --git a/plugins/gitee/models/issue_comment.go b/plugins/gitee/models/issue_comment.go
new file mode 100644
index 00000000..9d170040
--- /dev/null
+++ b/plugins/gitee/models/issue_comment.go
@@ -0,0 +1,39 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package models
+
+import (
+ "time"
+
+ "github.com/apache/incubator-devlake/models/common"
+)
+
+type GiteeIssueComment struct {
+ GiteeId int `gorm:"primaryKey"`
+ IssueId int `gorm:"index;comment:References the Issue"`
+ Body string
+ AuthorUsername string `gorm:"type:varchar(255)"`
+ AuthorUserId int
+ GiteeCreatedAt time.Time
+ GiteeUpdatedAt time.Time `gorm:"index"`
+ common.NoPKModel
+}
+
+func (GiteeIssueComment) TableName() string {
+ return "_tool_gitee_issue_comments"
+}
diff --git a/plugins/gitee/models/issue_label.go b/plugins/gitee/models/issue_label.go
new file mode 100644
index 00000000..77f910f9
--- /dev/null
+++ b/plugins/gitee/models/issue_label.go
@@ -0,0 +1,35 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package models
+
+import (
+ "github.com/apache/incubator-devlake/models/common"
+)
+
+// Please note that Issue Labels can also apply to Pull Requests.
+// Pull Requests are considered Issues in Gitee.
+
+type GiteeIssueLabel struct {
+ IssueId int `gorm:"primaryKey;autoIncrement:false"`
+ LabelName string `gorm:"primaryKey;type:varchar(255)"`
+ common.NoPKModel
+}
+
+func (GiteeIssueLabel) TableName() string {
+ return "_tool_gitee_issue_labels"
+}
diff --git a/plugins/gitee/models/migrationscripts/archived/commit.go b/plugins/gitee/models/migrationscripts/archived/commit.go
new file mode 100644
index 00000000..82c2c570
--- /dev/null
+++ b/plugins/gitee/models/migrationscripts/archived/commit.go
@@ -0,0 +1,47 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package archived
+
+import (
+ "time"
+
+ "github.com/apache/incubator-devlake/models/common"
+)
+
+type GiteeCommit struct {
+ Sha string `gorm:"primaryKey;type:varchar(40)"`
+ CommentsUrl string `gorm:"type:varchar(255)"`
+ Message string
+ AuthorId int
+ AuthorName string `gorm:"type:varchar(255)"`
+ AuthorEmail string `gorm:"type:varchar(255)"`
+ AuthoredDate time.Time
+ CommitterId int
+ CommitterName string `gorm:"type:varchar(255)"`
+ CommitterEmail string `gorm:"type:varchar(255)"`
+ CommittedDate time.Time
+ WebUrl string `gorm:"type:varchar(255)"`
+ Additions int `gorm:"comment:Added lines of code"`
+ Deletions int `gorm:"comment:Deleted lines of code"`
+ Total int `gorm:"comment:Sum of added/deleted lines of code"`
+ common.NoPKModel
+}
+
+func (GiteeCommit) TableName() string {
+ return "_tool_gitee_commits"
+}
diff --git a/plugins/gitee/models/migrationscripts/archived/commit_stat.go b/plugins/gitee/models/migrationscripts/archived/commit_stat.go
new file mode 100644
index 00000000..77a9e0a4
--- /dev/null
+++ b/plugins/gitee/models/migrationscripts/archived/commit_stat.go
@@ -0,0 +1,36 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package archived
+
+import (
+ "time"
+
+ "github.com/apache/incubator-devlake/models/migrationscripts/archived"
+)
+
+type GiteeCommitStat struct {
+ Sha string `gorm:"primaryKey;type:varchar(40)"`
+ Additions int `gorm:"comment:Added lines of code"`
+ Deletions int `gorm:"comment:Deleted lines of code"`
+ CommittedDate time.Time `gorm:"index"`
+ archived.NoPKModel
+}
+
+func (GiteeCommitStat) TableName() string {
+ return "_tool_gitee_commit_stats"
+}
diff --git a/plugins/gitee/models/migrationscripts/archived/issue.go b/plugins/gitee/models/migrationscripts/archived/issue.go
new file mode 100644
index 00000000..5d9e6240
--- /dev/null
+++ b/plugins/gitee/models/migrationscripts/archived/issue.go
@@ -0,0 +1,52 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package archived
+
+import (
+ "time"
+
+ "github.com/apache/incubator-devlake/models/migrationscripts/archived"
+)
+
+type GiteeIssue struct {
+ GiteeId int `gorm:"primaryKey"`
+ RepoId int `gorm:"index"`
+ Number string `gorm:"index;comment:Used in API requests ex. api/repo/1/issue/<THIS_NUMBER>"`
+ State string `gorm:"type:varchar(255)"`
+ Title string
+ Body string
+ Priority string `gorm:"type:varchar(255)"`
+ Type string `gorm:"type:varchar(100)"`
+ Status string `gorm:"type:varchar(255)"`
+ AuthorId int
+ AuthorName string `gorm:"type:varchar(255)"`
+ AssigneeId int
+ AssigneeName string `gorm:"type:varchar(255)"`
+ LeadTimeMinutes uint
+ Url string `gorm:"type:varchar(255)"`
+ ClosedAt *time.Time
+ GiteeCreatedAt time.Time
+ GiteeUpdatedAt time.Time `gorm:"index"`
+ Severity string `gorm:"type:varchar(255)"`
+ Component string `gorm:"type:varchar(255)"`
+ archived.NoPKModel
+}
+
+func (GiteeIssue) TableName() string {
+ return "_tool_gitee_issues"
+}
diff --git a/plugins/gitee/models/migrationscripts/archived/issue_comment.go b/plugins/gitee/models/migrationscripts/archived/issue_comment.go
new file mode 100644
index 00000000..ea40c2d9
--- /dev/null
+++ b/plugins/gitee/models/migrationscripts/archived/issue_comment.go
@@ -0,0 +1,39 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package archived
+
+import (
+ "time"
+
+ "github.com/apache/incubator-devlake/models/migrationscripts/archived"
+)
+
+type GiteeIssueComment struct {
+ GiteeId int `gorm:"primaryKey"`
+ IssueId int `gorm:"index;comment:References the Issue"`
+ Body string
+ AuthorUsername string `gorm:"type:varchar(255)"`
+ AuthorUserId int
+ GiteeCreatedAt time.Time
+ GiteeUpdatedAt time.Time `gorm:"index"`
+ archived.NoPKModel
+}
+
+func (GiteeIssueComment) TableName() string {
+ return "_tool_gitee_issue_comments"
+}
diff --git a/plugins/gitee/models/migrationscripts/archived/issue_label.go b/plugins/gitee/models/migrationscripts/archived/issue_label.go
new file mode 100644
index 00000000..9db73278
--- /dev/null
+++ b/plugins/gitee/models/migrationscripts/archived/issue_label.go
@@ -0,0 +1,33 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package archived
+
+import "github.com/apache/incubator-devlake/models/migrationscripts/archived"
+
+// Please note that Issue Labels can also apply to Pull Requests.
+// Pull Requests are considered Issues in Gitee.
+
+type GiteeIssueLabel struct {
+ IssueId int `gorm:"primaryKey;autoIncrement:false"`
+ LabelName string `gorm:"primaryKey;type:varchar(255)"`
+ archived.NoPKModel
+}
+
+func (GiteeIssueLabel) TableName() string {
+ return "_tool_gitee_issue_labels"
+}
diff --git a/plugins/gitee/models/migrationscripts/archived/pull_request.go b/plugins/gitee/models/migrationscripts/archived/pull_request.go
new file mode 100644
index 00000000..39ced959
--- /dev/null
+++ b/plugins/gitee/models/migrationscripts/archived/pull_request.go
@@ -0,0 +1,59 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package archived
+
+import (
+ "time"
+
+ "github.com/apache/incubator-devlake/models/migrationscripts/archived"
+)
+
+type GiteePullRequest struct {
+ GiteeId int `gorm:"primaryKey"`
+ RepoId int `gorm:"index"`
+ Number int `gorm:"index"` // This number is used in GET requests to the API associated to reviewers / comments / etc.
+ State string `gorm:"type:varchar(255)"`
+ Title string `gorm:"type:varchar(255)"`
+ GiteeCreatedAt time.Time
+ GiteeUpdatedAt time.Time `gorm:"index"`
+ ClosedAt *time.Time
+ // In order to get the following fields, we need to collect PRs individually from Gitee
+ Additions int
+ Deletions int
+ Comments int
+ Commits int
+ ReviewComments int
+ Merged bool
+ MergedAt *time.Time
+ Body string
+ Type string `gorm:"type:varchar(255)"`
+ Component string `gorm:"type:varchar(255)"`
+ MergeCommitSha string `gorm:"type:varchar(40)"`
+ HeadRef string `gorm:"type:varchar(255)"`
+ BaseRef string `gorm:"type:varchar(255)"`
+ BaseCommitSha string `gorm:"type:varchar(255)"`
+ HeadCommitSha string `gorm:"type:varchar(255)"`
+ Url string `gorm:"type:varchar(255)"`
+ AuthorName string `gorm:"type:varchar(100)"`
+ AuthorId int
+ archived.NoPKModel
+}
+
+func (GiteePullRequest) TableName() string {
+ return "_tool_gitee_pull_requests"
+}
diff --git a/plugins/gitee/models/migrationscripts/archived/pull_request_comment.go b/plugins/gitee/models/migrationscripts/archived/pull_request_comment.go
new file mode 100644
index 00000000..79bf54be
--- /dev/null
+++ b/plugins/gitee/models/migrationscripts/archived/pull_request_comment.go
@@ -0,0 +1,39 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package archived
+
+import (
+ "time"
+
+ "github.com/apache/incubator-devlake/models/migrationscripts/archived"
+)
+
+type GiteePullRequestComment struct {
+ GiteeId int `gorm:"primaryKey"`
+ PullRequestId int `gorm:"index"`
+ Body string
+ AuthorUsername string `gorm:"type:varchar(255)"`
+ AuthorUserId int
+ GiteeCreatedAt time.Time
+ GiteeUpdatedAt time.Time `gorm:"index"`
+ archived.NoPKModel
+}
+
+func (GiteePullRequestComment) TableName() string {
+ return "_tool_gitee_pull_request_comments"
+}
diff --git a/plugins/gitee/models/migrationscripts/archived/pull_request_commit.go b/plugins/gitee/models/migrationscripts/archived/pull_request_commit.go
new file mode 100644
index 00000000..54b3617f
--- /dev/null
+++ b/plugins/gitee/models/migrationscripts/archived/pull_request_commit.go
@@ -0,0 +1,30 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package archived
+
+import "github.com/apache/incubator-devlake/models/migrationscripts/archived"
+
+type GiteePullRequestCommit struct {
+ CommitSha string `gorm:"primaryKey;type:varchar(40)"`
+ PullRequestId int `gorm:"primaryKey;autoIncrement:false"`
+ archived.NoPKModel
+}
+
+func (GiteePullRequestCommit) TableName() string {
+ return "_tool_gitee_pull_request_commits"
+}
diff --git a/plugins/gitee/models/migrationscripts/archived/pull_request_issue.go b/plugins/gitee/models/migrationscripts/archived/pull_request_issue.go
new file mode 100644
index 00000000..20e3d13e
--- /dev/null
+++ b/plugins/gitee/models/migrationscripts/archived/pull_request_issue.go
@@ -0,0 +1,32 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package archived
+
+import "github.com/apache/incubator-devlake/models/migrationscripts/archived"
+
+type GiteePullRequestIssue struct {
+ PullRequestId int `gorm:"primaryKey"`
+ IssueId int `gorm:"primaryKey"`
+ PullRequestNumber int
+ IssueNumber int
+ archived.NoPKModel
+}
+
+func (GiteePullRequestIssue) TableName() string {
+ return "_tool_gitee_pull_request_issues"
+}
diff --git a/plugins/gitee/models/migrationscripts/archived/pull_request_label.go b/plugins/gitee/models/migrationscripts/archived/pull_request_label.go
new file mode 100644
index 00000000..69f25c7e
--- /dev/null
+++ b/plugins/gitee/models/migrationscripts/archived/pull_request_label.go
@@ -0,0 +1,30 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package archived
+
+import "github.com/apache/incubator-devlake/models/migrationscripts/archived"
+
+type GiteePullRequestLabel struct {
+ PullId int `gorm:"primaryKey;autoIncrement:false"`
+ LabelName string `gorm:"primaryKey;type:varchar(255)"`
+ archived.NoPKModel
+}
+
+func (GiteePullRequestLabel) TableName() string {
+ return "_tool_gitee_pull_request_labels"
+}
diff --git a/plugins/gitee/models/migrationscripts/archived/repo.go b/plugins/gitee/models/migrationscripts/archived/repo.go
new file mode 100644
index 00000000..989aeae2
--- /dev/null
+++ b/plugins/gitee/models/migrationscripts/archived/repo.go
@@ -0,0 +1,43 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package archived
+
+import (
+ "time"
+
+ "github.com/apache/incubator-devlake/models/migrationscripts/archived"
+)
+
+type GiteeRepo struct {
+ GiteeId int `gorm:"primaryKey"`
+ Name string `gorm:"type:varchar(255)"`
+ HTMLUrl string `gorm:"type:varchar(255)"`
+ Description string
+ OwnerId int `json:"ownerId"`
+ OwnerLogin string `json:"ownerLogin" gorm:"type:varchar(255)"`
+ Language string `json:"language" gorm:"type:varchar(255)"`
+ ParentGiteeId int `json:"parentId"`
+ ParentHTMLUrl string `json:"parentHtmlUrl"`
+ CreatedDate time.Time `json:"createdDate"`
+ UpdatedDate *time.Time `json:"updatedDate"`
+ archived.NoPKModel
+}
+
+func (GiteeRepo) TableName() string {
+ return "_tool_gitee_repo"
+}
diff --git a/plugins/gitee/models/migrationscripts/archived/repo_commit.go b/plugins/gitee/models/migrationscripts/archived/repo_commit.go
new file mode 100644
index 00000000..fa70dfb2
--- /dev/null
+++ b/plugins/gitee/models/migrationscripts/archived/repo_commit.go
@@ -0,0 +1,32 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package archived
+
+import (
+ "github.com/apache/incubator-devlake/models/migrationscripts/archived"
+)
+
+type GiteeRepoCommit struct {
+ RepoId int `gorm:"primaryKey"`
+ CommitSha string `gorm:"primaryKey;type:varchar(40)"`
+ archived.NoPKModel
+}
+
+func (GiteeRepoCommit) TableName() string {
+ return "_tool_gitee_repo_commits"
+}
diff --git a/plugins/gitee/models/migrationscripts/archived/reviewer.go b/plugins/gitee/models/migrationscripts/archived/reviewer.go
new file mode 100644
index 00000000..69aeee6b
--- /dev/null
+++ b/plugins/gitee/models/migrationscripts/archived/reviewer.go
@@ -0,0 +1,32 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package archived
+
+import "github.com/apache/incubator-devlake/models/migrationscripts/archived"
+
+type GiteeReviewer struct {
+ GiteeId int `gorm:"primaryKey"`
+ Login string `gorm:"type:varchar(255)"`
+ PullRequestId int `gorm:"primaryKey"`
+
+ archived.NoPKModel
+}
+
+func (GiteeReviewer) TableName() string {
+ return "_tool_gitee_reviewers"
+}
diff --git a/plugins/gitee/models/migrationscripts/archived/user.go b/plugins/gitee/models/migrationscripts/archived/user.go
new file mode 100644
index 00000000..729b484d
--- /dev/null
+++ b/plugins/gitee/models/migrationscripts/archived/user.go
@@ -0,0 +1,47 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package archived
+
+import (
+ "github.com/apache/incubator-devlake/models/migrationscripts/archived"
+)
+
+type GiteeUser struct {
+ Id int `json:"id" gorm:"primaryKey"`
+ Login string `json:"login" gorm:"type:varchar(255)"`
+ Name string `json:"name" gorm:"type:varchar(255)"`
+ AvatarUrl string `json:"avatar_url" gorm:"type:varchar(255)"`
+ EventsUrl string `json:"events_url" gorm:"type:varchar(255)"`
+ FollowersUrl string `json:"followers_url" gorm:"type:varchar(255)"`
+ FollowingUrl string `json:"following_url" gorm:"type:varchar(255)"`
+ GistsUrl string `json:"gists_url" gorm:"type:varchar(255)"`
+ HtmlUrl string `json:"html_url" gorm:"type:varchar(255)"`
+ OrganizationsUrl string `json:"organizations_url" gorm:"type:varchar(255)"`
+ ReceivedEventsUrl string `json:"received_events_url" gorm:"type:varchar(255)"`
+ Remark string `json:"remark" gorm:"type:varchar(255)"`
+ ReposUrl string `json:"repos_url" gorm:"type:varchar(255)"`
+ StarredUrl string `json:"starred_url" gorm:"type:varchar(255)"`
+ SubscriptionsUrl string `json:"subscriptions_url" gorm:"type:varchar(255)"`
+ Url string `json:"url" gorm:"type:varchar(255)"`
+ Type string `json:"type" gorm:"type:varchar(255)"`
+ archived.NoPKModel
+}
+
+func (GiteeUser) TableName() string {
+ return "_tool_gitee_users"
+}
diff --git a/plugins/gitee/models/migrationscripts/init_schema.go b/plugins/gitee/models/migrationscripts/init_schema.go
new file mode 100644
index 00000000..5eceb089
--- /dev/null
+++ b/plugins/gitee/models/migrationscripts/init_schema.go
@@ -0,0 +1,54 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+ "context"
+
+ "github.com/apache/incubator-devlake/plugins/gitee/models/migrationscripts/archived"
+ "gorm.io/gorm"
+)
+
+type InitSchemas struct{}
+
+func (*InitSchemas) Up(ctx context.Context, db *gorm.DB) error {
+ return db.Migrator().AutoMigrate(
+ &archived.GiteeRepo{},
+ &archived.GiteeCommit{},
+ &archived.GiteeRepoCommit{},
+ &archived.GiteePullRequest{},
+ &archived.GiteePullRequestLabel{},
+ &archived.GiteeUser{},
+ &archived.GiteePullRequestComment{},
+ &archived.GiteeIssue{},
+ &archived.GiteeIssueComment{},
+ &archived.GiteeCommitStat{},
+ &archived.GiteeIssueLabel{},
+ &archived.GiteePullRequestCommit{},
+ &archived.GiteePullRequestIssue{},
+ &archived.GiteeReviewer{},
+ )
+}
+
+func (*InitSchemas) Version() uint64 {
+ return 20220407201202
+}
+
+func (*InitSchemas) Name() string {
+ return "Gitee init schemas"
+}
diff --git a/plugins/gitee/models/pull_request.go b/plugins/gitee/models/pull_request.go
new file mode 100644
index 00000000..7fb1caaf
--- /dev/null
+++ b/plugins/gitee/models/pull_request.go
@@ -0,0 +1,59 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package models
+
+import (
+ "time"
+
+ "github.com/apache/incubator-devlake/models/common"
+)
+
+type GiteePullRequest struct {
+ GiteeId int `gorm:"primaryKey"`
+ RepoId int `gorm:"index"`
+ Number int `gorm:"index"` // This number is used in GET requests to the API associated to reviewers / comments / etc.
+ State string `gorm:"type:varchar(255)"`
+ Title string `gorm:"type:varchar(255)"`
+ GiteeCreatedAt time.Time
+ GiteeUpdatedAt time.Time `gorm:"index"`
+ ClosedAt *time.Time
+ // In order to get the following fields, we need to collect PRs individually from Gitee
+ Additions int
+ Deletions int
+ Comments int
+ Commits int
+ ReviewComments int
+ Merged bool
+ MergedAt *time.Time
+ Body string
+ Type string `gorm:"type:varchar(255)"`
+ Component string `gorm:"type:varchar(255)"`
+ MergeCommitSha string `gorm:"type:varchar(40)"`
+ HeadRef string `gorm:"type:varchar(255)"`
+ BaseRef string `gorm:"type:varchar(255)"`
+ BaseCommitSha string `gorm:"type:varchar(255)"`
+ HeadCommitSha string `gorm:"type:varchar(255)"`
+ Url string `gorm:"type:varchar(255)"`
+ AuthorName string `gorm:"type:varchar(100)"`
+ AuthorId int
+ common.NoPKModel
+}
+
+func (GiteePullRequest) TableName() string {
+ return "_tool_gitee_pull_requests"
+}
diff --git a/plugins/gitee/models/pull_request_comment.go b/plugins/gitee/models/pull_request_comment.go
new file mode 100644
index 00000000..6e0b804a
--- /dev/null
+++ b/plugins/gitee/models/pull_request_comment.go
@@ -0,0 +1,39 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package models
+
+import (
+ "time"
+
+ "github.com/apache/incubator-devlake/models/common"
+)
+
+type GiteePullRequestComment struct {
+ GiteeId int `gorm:"primaryKey"`
+ PullRequestId int `gorm:"index"`
+ Body string
+ AuthorUsername string `gorm:"type:varchar(255)"`
+ AuthorUserId int
+ GiteeCreatedAt time.Time
+ GiteeUpdatedAt time.Time `gorm:"index"`
+ common.NoPKModel
+}
+
+func (GiteePullRequestComment) TableName() string {
+ return "_tool_gitee_pull_request_comments"
+}
diff --git a/plugins/gitee/models/pull_request_commit.go b/plugins/gitee/models/pull_request_commit.go
new file mode 100644
index 00000000..6705b665
--- /dev/null
+++ b/plugins/gitee/models/pull_request_commit.go
@@ -0,0 +1,32 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package models
+
+import (
+ "github.com/apache/incubator-devlake/models/common"
+)
+
+type GiteePullRequestCommit struct {
+ CommitSha string `gorm:"primaryKey;type:varchar(40)"`
+ PullRequestId int `gorm:"primaryKey;autoIncrement:false"`
+ common.NoPKModel
+}
+
+func (GiteePullRequestCommit) TableName() string {
+ return "_tool_gitee_pull_request_commits"
+}
diff --git a/plugins/gitee/models/pull_request_issue.go b/plugins/gitee/models/pull_request_issue.go
new file mode 100644
index 00000000..8527ff4d
--- /dev/null
+++ b/plugins/gitee/models/pull_request_issue.go
@@ -0,0 +1,32 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package models
+
+import "github.com/apache/incubator-devlake/models/common"
+
+type GiteePullRequestIssue struct {
+ PullRequestId int `gorm:"primaryKey"`
+ IssueId int `gorm:"primaryKey"`
+ PullRequestNumber int
+ IssueNumber string
+ common.NoPKModel
+}
+
+func (GiteePullRequestIssue) TableName() string {
+ return "_tool_gitee_pull_request_issues"
+}
diff --git a/plugins/gitee/models/pull_request_label.go b/plugins/gitee/models/pull_request_label.go
new file mode 100644
index 00000000..619eeb0d
--- /dev/null
+++ b/plugins/gitee/models/pull_request_label.go
@@ -0,0 +1,32 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package models
+
+import (
+ "github.com/apache/incubator-devlake/models/common"
+)
+
+type GiteePullRequestLabel struct {
+ PullId int `gorm:"primaryKey;autoIncrement:false"`
+ LabelName string `gorm:"primaryKey;type:varchar(255)"`
+ common.NoPKModel
+}
+
+func (GiteePullRequestLabel) TableName() string {
+ return "_tool_gitee_pull_request_labels"
+}
diff --git a/plugins/gitee/models/repo.go b/plugins/gitee/models/repo.go
new file mode 100644
index 00000000..84a4b493
--- /dev/null
+++ b/plugins/gitee/models/repo.go
@@ -0,0 +1,43 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package models
+
+import (
+ "time"
+
+ "github.com/apache/incubator-devlake/models/common"
+)
+
+type GiteeRepo struct {
+ GiteeId int `gorm:"primaryKey"`
+ Name string `gorm:"type:varchar(255)"`
+ HTMLUrl string `gorm:"type:varchar(255)"`
+ Description string
+ OwnerId int `json:"ownerId"`
+ OwnerLogin string `json:"ownerLogin" gorm:"type:varchar(255)"`
+ Language string `json:"language" gorm:"type:varchar(255)"`
+ ParentGiteeId int `json:"parentId"`
+ ParentHTMLUrl string `json:"parentHtmlUrl"`
+ CreatedDate time.Time `json:"createdDate"`
+ UpdatedDate *time.Time `json:"updatedDate"`
+ common.NoPKModel
+}
+
+func (GiteeRepo) TableName() string {
+ return "_tool_gitee_repo"
+}
diff --git a/plugins/gitee/models/repo_commit.go b/plugins/gitee/models/repo_commit.go
new file mode 100644
index 00000000..7e66077e
--- /dev/null
+++ b/plugins/gitee/models/repo_commit.go
@@ -0,0 +1,32 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package models
+
+import (
+ "github.com/apache/incubator-devlake/models/common"
+)
+
+type GiteeRepoCommit struct {
+ RepoId int `gorm:"primaryKey"`
+ CommitSha string `gorm:"primaryKey;type:varchar(40)"`
+ common.NoPKModel
+}
+
+func (GiteeRepoCommit) TableName() string {
+ return "_tool_gitee_repo_commits"
+}
diff --git a/plugins/gitee/models/reviewer.go b/plugins/gitee/models/reviewer.go
new file mode 100644
index 00000000..fc2c6a1d
--- /dev/null
+++ b/plugins/gitee/models/reviewer.go
@@ -0,0 +1,34 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package models
+
+import (
+ "github.com/apache/incubator-devlake/models/common"
+)
+
+type GiteeReviewer struct {
+ GiteeId int `gorm:"primaryKey"`
+ Login string `gorm:"type:varchar(255)"`
+ PullRequestId int `gorm:"primaryKey"`
+
+ common.NoPKModel
+}
+
+func (GiteeReviewer) TableName() string {
+ return "_tool_gitee_reviewers"
+}
diff --git a/plugins/gitee/models/user.go b/plugins/gitee/models/user.go
new file mode 100644
index 00000000..acf6309e
--- /dev/null
+++ b/plugins/gitee/models/user.go
@@ -0,0 +1,45 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package models
+
+import "github.com/apache/incubator-devlake/models/common"
+
+type GiteeUser struct {
+ Id int `json:"id" gorm:"primaryKey"`
+ Login string `json:"login" gorm:"type:varchar(255)"`
+ Name string `json:"name" gorm:"type:varchar(255)"`
+ AvatarUrl string `json:"avatar_url" gorm:"type:varchar(255)"`
+ EventsUrl string `json:"events_url" gorm:"type:varchar(255)"`
+ FollowersUrl string `json:"followers_url" gorm:"type:varchar(255)"`
+ FollowingUrl string `json:"following_url" gorm:"type:varchar(255)"`
+ GistsUrl string `json:"gists_url" gorm:"type:varchar(255)"`
+ HtmlUrl string `json:"html_url" gorm:"type:varchar(255)"`
+ OrganizationsUrl string `json:"organizations_url" gorm:"type:varchar(255)"`
+ ReceivedEventsUrl string `json:"received_events_url" gorm:"type:varchar(255)"`
+ Remark string `json:"remark" gorm:"type:varchar(255)"`
+ ReposUrl string `json:"repos_url" gorm:"type:varchar(255)"`
+ StarredUrl string `json:"starred_url" gorm:"type:varchar(255)"`
+ SubscriptionsUrl string `json:"subscriptions_url" gorm:"type:varchar(255)"`
+ Url string `json:"url" gorm:"type:varchar(255)"`
+ Type string `json:"type" gorm:"type:varchar(255)"`
+ common.NoPKModel
+}
+
+func (GiteeUser) TableName() string {
+ return "_tool_gitee_users"
+}
diff --git a/plugins/gitee/tasks/api_client.go b/plugins/gitee/tasks/api_client.go
new file mode 100644
index 00000000..94c63256
--- /dev/null
+++ b/plugins/gitee/tasks/api_client.go
@@ -0,0 +1,84 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "fmt"
+ "net/http"
+ "strconv"
+ "time"
+
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/helper"
+ "github.com/apache/incubator-devlake/utils"
+)
+
+func NewGiteeApiClient(taskCtx core.TaskContext) (*helper.ApiAsyncClient, error) {
+ endpoint := taskCtx.GetConfig("GITEE_ENDPOINT")
+ if endpoint == "" {
+ return nil, fmt.Errorf("endpint is required")
+ }
+ userRateLimit, err := utils.StrToIntOr(taskCtx.GetConfig("GITEE_API_REQUESTS_PER_HOUR"), 0)
+ if err != nil {
+ return nil, err
+ }
+ auth := taskCtx.GetConfig("GITEE_AUTH")
+ if auth == "" {
+ return nil, fmt.Errorf("GITEE_AUTH is required")
+ }
+ proxy := taskCtx.GetConfig("GITEE_PROXY")
+
+ headers := map[string]string{}
+
+ apiClient, err := helper.NewApiClient(endpoint, headers, 0, proxy, taskCtx.GetContext())
+ if err != nil {
+ return nil, err
+ }
+ apiClient.SetAfterFunction(func(res *http.Response) error {
+ if res.StatusCode == http.StatusUnauthorized {
+ return fmt.Errorf("authentication failed, please check your Basic Auth Token")
+ }
+ return nil
+ })
+
+ rateLimiter := &helper.ApiRateLimitCalculator{
+ UserRateLimitPerHour: userRateLimit,
+ DynamicRateLimit: func(res *http.Response) (int, time.Duration, error) {
+ rateLimitHeader := res.Header.Get("RateLimit-Limit")
+ if rateLimitHeader == "" {
+ // use default
+ return 0, 0, nil
+ }
+ rateLimit, err := strconv.Atoi(rateLimitHeader)
+ if err != nil {
+ return 0, 0, fmt.Errorf("failed to parse RateLimit-Limit header: %w", err)
+ }
+ // seems like gitlab rate limit is on minute basis
+ return rateLimit, 1 * time.Minute, nil
+ },
+ }
+ asyncApiClient, err := helper.CreateAsyncApiClient(
+ taskCtx,
+ apiClient,
+ rateLimiter,
+ )
+ if err != nil {
+ return nil, err
+ }
+ return asyncApiClient, nil
+}
diff --git a/plugins/gitee/tasks/commit_collector.go b/plugins/gitee/tasks/commit_collector.go
new file mode 100644
index 00000000..dc834e53
--- /dev/null
+++ b/plugins/gitee/tasks/commit_collector.go
@@ -0,0 +1,64 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "net/url"
+ "strconv"
+
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/helper"
+)
+
+const RAW_COMMIT_TABLE = "gitee_api_commit"
+
+var CollectCommitsMeta = core.SubTaskMeta{
+ Name: "collectApiCommits",
+ EntryPoint: CollectApiCommits,
+ EnabledByDefault: true,
+ Description: "Collect commit data from gitee api",
+}
+
+func CollectApiCommits(taskCtx core.SubTaskContext) error {
+ rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_COMMIT_TABLE)
+
+ collector, err := helper.NewApiCollector(helper.ApiCollectorArgs{
+ RawDataSubTaskArgs: *rawDataSubTaskArgs,
+ ApiClient: data.ApiClient,
+ PageSize: 100,
+ Incremental: false,
+ UrlTemplate: "repos/{{ .Params.Owner }}/{{ .Params.Repo }}/commits",
+ Query: func(reqData *helper.RequestData) (url.Values, error) {
+ query := url.Values{}
+ query.Set("access_token", data.Options.Token)
+ query.Set("with_stats", "true")
+ query.Set("sort", "asc")
+ query.Set("page", strconv.Itoa(reqData.Pager.Page))
+ query.Set("per_page", strconv.Itoa(reqData.Pager.Size))
+ return query, nil
+ },
+ Concurrency: 20,
+ ResponseParser: GetRawMessageFromResponse,
+ })
+
+ if err != nil {
+ return err
+ }
+
+ return collector.Execute()
+}
diff --git a/plugins/gitee/tasks/commit_convertor.go b/plugins/gitee/tasks/commit_convertor.go
new file mode 100644
index 00000000..d9405edc
--- /dev/null
+++ b/plugins/gitee/tasks/commit_convertor.go
@@ -0,0 +1,102 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "reflect"
+
+ "github.com/apache/incubator-devlake/models/domainlayer/code"
+ "github.com/apache/incubator-devlake/models/domainlayer/didgen"
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/gitee/models"
+ giteeModels "github.com/apache/incubator-devlake/plugins/gitee/models"
+ "github.com/apache/incubator-devlake/plugins/helper"
+)
+
+var ConvertCommitsMeta = core.SubTaskMeta{
+ Name: "convertApiCommits",
+ EntryPoint: ConvertCommits,
+ EnabledByDefault: true,
+ Description: "Convert tool layer table gitee_commits into domain layer table commits",
+}
+
+func ConvertCommits(taskCtx core.SubTaskContext) error {
+
+ rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_COMMIT_TABLE)
+ db := taskCtx.GetDb()
+ repoId := data.Repo.GiteeId
+
+ // select all commits belongs to the project
+ cursor, err := db.Table("_tool_gitee_commits gc").
+ Joins(`left join _tool_gitee_repo_commits gpc on (
+ gpc.commit_sha = gc.sha
+ )`).
+ Select("gc.*").
+ Where("gpc.repo_id = ?", repoId).
+ Rows()
+ if err != nil {
+ return err
+ }
+ defer cursor.Close()
+
+ // TODO: adopt batch indate operation
+ userDidGen := didgen.NewDomainIdGenerator(&models.GiteeUser{})
+ repoDidGen := didgen.NewDomainIdGenerator(&giteeModels.GiteeRepo{})
+ domainRepoId := repoDidGen.Generate(repoId)
+
+ converter, err := helper.NewDataConverter(helper.DataConverterArgs{
+ RawDataSubTaskArgs: *rawDataSubTaskArgs,
+ InputRowType: reflect.TypeOf(models.GiteeCommit{}),
+ Input: cursor,
+
+ Convert: func(inputRow interface{}) ([]interface{}, error) {
+ giteeCommit := inputRow.(*models.GiteeCommit)
+
+ // convert commit
+ commit := &code.Commit{}
+ commit.Sha = giteeCommit.Sha
+ commit.Message = giteeCommit.Message
+ commit.Additions = giteeCommit.Additions
+ commit.Deletions = giteeCommit.Deletions
+ commit.AuthorId = userDidGen.Generate(giteeCommit.AuthorId)
+ commit.AuthorName = giteeCommit.AuthorName
+ commit.AuthorEmail = giteeCommit.AuthorEmail
+ commit.AuthoredDate = giteeCommit.AuthoredDate
+ commit.CommitterName = giteeCommit.CommitterName
+ commit.CommitterEmail = giteeCommit.CommitterEmail
+ commit.CommittedDate = giteeCommit.CommittedDate
+ commit.CommitterId = userDidGen.Generate(giteeCommit.CommitterId)
+
+ // convert repo / commits relationship
+ repoCommit := &code.RepoCommit{
+ RepoId: domainRepoId,
+ CommitSha: giteeCommit.Sha,
+ }
+
+ return []interface{}{
+ commit,
+ repoCommit,
+ }, nil
+ },
+ })
+ if err != nil {
+ return err
+ }
+
+ return converter.Execute()
+}
diff --git a/plugins/gitee/tasks/commit_extractor.go b/plugins/gitee/tasks/commit_extractor.go
new file mode 100644
index 00000000..484a6230
--- /dev/null
+++ b/plugins/gitee/tasks/commit_extractor.go
@@ -0,0 +1,111 @@
+package tasks
+
+import (
+ "encoding/json"
+
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/gitee/models"
+ "github.com/apache/incubator-devlake/plugins/helper"
+)
+
+var ExtractCommitsMeta = core.SubTaskMeta{
+ Name: "extractApiCommits",
+ EntryPoint: ExtractApiCommits,
+ EnabledByDefault: true,
+ Description: "Extract raw commit data into tool layer table GiteeCommit,GiteeUser and GiteeRepoCommit",
+}
+
+type GiteeCommit struct {
+ Author struct {
+ Date helper.Iso8601Time `json:"date"`
+ Email string `json:"email"`
+ Name string `json:"name"`
+ }
+ Committer struct {
+ Date helper.Iso8601Time `json:"date"`
+ Email string `json:"email"`
+ Name string `json:"name"`
+ }
+ Message string `json:"message"`
+}
+
+type GiteeApiCommitResponse struct {
+ Author *models.GiteeUser `json:"author"`
+ AuthorId int
+ CommentsUrl string `json:"comments_url"`
+ Commit GiteeCommit `json:"commit"`
+ Committer *models.GiteeUser `json:"committer"`
+ HtmlUrl string `json:"html_url"`
+ Sha string `json:"sha"`
+ Url string `json:"url"`
+}
+
+func ExtractApiCommits(taskCtx core.SubTaskContext) error {
+ rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_COMMIT_TABLE)
+
+ extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{
+ RawDataSubTaskArgs: *rawDataSubTaskArgs,
+ Extract: func(row *helper.RawData) ([]interface{}, error) {
+ results := make([]interface{}, 0, 4)
+
+ commit := &GiteeApiCommitResponse{}
+
+ err := json.Unmarshal(row.Data, commit)
+
+ if err != nil {
+ return nil, err
+ }
+
+ if commit.Sha == "" {
+ return nil, nil
+ }
+
+ giteeCommit, err := ConvertCommit(commit)
+
+ if err != nil {
+ return nil, err
+ }
+
+ if commit.Author != nil {
+ giteeCommit.AuthorId = commit.Author.Id
+ results = append(results, commit.Author)
+ }
+ if commit.Committer != nil {
+ giteeCommit.CommitterId = commit.Committer.Id
+ results = append(results, commit.Committer)
+
+ }
+
+ giteeRepoCommit := &models.GiteeRepoCommit{
+ RepoId: data.Repo.GiteeId,
+ CommitSha: commit.Sha,
+ }
+ results = append(results, giteeCommit)
+ results = append(results, giteeRepoCommit)
+ return results, nil
+ },
+ })
+
+ if err != nil {
+ return err
+ }
+
+ return extractor.Execute()
+}
+
+// ConvertCommit Convert the API response to our DB model instance
+func ConvertCommit(commit *GiteeApiCommitResponse) (*models.GiteeCommit, error) {
+ giteeCommit := &models.GiteeCommit{
+ Sha: commit.Sha,
+ AuthorId: commit.Author.Id,
+ Message: commit.Commit.Message,
+ AuthorName: commit.Commit.Author.Name,
+ AuthorEmail: commit.Commit.Author.Email,
+ AuthoredDate: commit.Commit.Author.Date.ToTime(),
+ CommitterName: commit.Commit.Author.Name,
+ CommitterEmail: commit.Commit.Author.Email,
+ CommittedDate: commit.Commit.Author.Date.ToTime(),
+ WebUrl: commit.Url,
+ }
+ return giteeCommit, nil
+}
diff --git a/plugins/gitee/tasks/commit_stats_collector.go b/plugins/gitee/tasks/commit_stats_collector.go
new file mode 100644
index 00000000..cdc23c1d
--- /dev/null
+++ b/plugins/gitee/tasks/commit_stats_collector.go
@@ -0,0 +1,112 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "encoding/json"
+ "fmt"
+ "io/ioutil"
+ "net/http"
+ "net/url"
+ "reflect"
+
+ "github.com/apache/incubator-devlake/plugins/helper"
+
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/gitee/models"
+)
+
+const RAW_COMMIT_STATS_TABLE = "gitee_api_commit_stats"
+
+var CollectApiCommitStatsMeta = core.SubTaskMeta{
+ Name: "collectApiCommitStats",
+ EntryPoint: CollectApiCommitStats,
+ EnabledByDefault: false,
+ Description: "Collect commitStats data from Gitee api",
+}
+
+func CollectApiCommitStats(taskCtx core.SubTaskContext) error {
+ db := taskCtx.GetDb()
+ rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_COMMIT_STATS_TABLE)
+
+ var latestUpdated models.GiteeCommitStat
+ err := db.Model(&latestUpdated).Joins("left join _tool_gitee_repo_commits on _tool_gitee_commit_stats.sha = _tool_gitee_repo_commits.commit_sha").
+ Where("_tool_gitee_repo_commits.repo_id = ?", data.Repo.GiteeId).
+ Order("committed_date DESC").Limit(1).Find(&latestUpdated).Error
+ if err != nil {
+ return fmt.Errorf("failed to get latest gitee commit record: %w", err)
+ }
+
+ cursor, err := db.Model(&models.GiteeCommit{}).
+ Joins("left join _tool_gitee_repo_commits on _tool_gitee_commits.sha = _tool_gitee_repo_commits.commit_sha").
+ Where("_tool_gitee_repo_commits.repo_id = ? and _tool_gitee_commits.committed_date >= ?",
+ data.Repo.GiteeId, latestUpdated.CommittedDate.String()).
+ Rows()
+ if err != nil {
+ return err
+ }
+ iterator, err := helper.NewCursorIterator(db, cursor, reflect.TypeOf(models.GiteeCommit{}))
+ if err != nil {
+ return err
+ }
+
+ collector, err := helper.NewApiCollector(helper.ApiCollectorArgs{
+ RawDataSubTaskArgs: *rawDataSubTaskArgs,
+ ApiClient: data.ApiClient,
+ PageSize: 100,
+ Input: iterator,
+ /*
+ url may use arbitrary variables from different source in any order, we need GoTemplate to allow more
+ flexible for all kinds of possibility.
+ Pager contains information for a particular page, calculated by ApiCollector, and will be passed into
+ GoTemplate to generate a url for that page.
+ We want to do page-fetching in ApiCollector, because the logic are highly similar, by doing so, we can
+ avoid duplicate logic for every tasks, and when we have a better idea like improving performance, we can
+ do it in one place
+ */
+ UrlTemplate: "repos/{{ .Params.Owner }}/{{ .Params.Repo }}/commits/{{ .Input.Sha }}",
+ /*
+ (Optional) Return query string for request, or you can plug them into UrlTemplate directly
+ */
+ Query: func(reqData *helper.RequestData) (url.Values, error) {
+ query := url.Values{}
+ query.Set("access_token", data.Options.Token)
+ query.Set("state", "all")
+ query.Set("direction", "asc")
+ query.Set("page", fmt.Sprintf("%v", reqData.Pager.Page))
+ query.Set("per_page", fmt.Sprintf("%v", reqData.Pager.Size))
+
+ return query, nil
+ },
+
+ ResponseParser: func(res *http.Response) ([]json.RawMessage, error) {
+ body, err := ioutil.ReadAll(res.Body)
+ res.Body.Close()
+ if err != nil {
+ return nil, err
+ }
+ return []json.RawMessage{body}, nil
+ },
+ })
+
+ if err != nil {
+ return err
+ }
+
+ return collector.Execute()
+}
diff --git a/plugins/gitee/tasks/commit_stats_extractor.go b/plugins/gitee/tasks/commit_stats_extractor.go
new file mode 100644
index 00000000..680b01a6
--- /dev/null
+++ b/plugins/gitee/tasks/commit_stats_extractor.go
@@ -0,0 +1,98 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "encoding/json"
+
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/gitee/models"
+ "github.com/apache/incubator-devlake/plugins/helper"
+)
+
+var ExtractApiCommitStatsMeta = core.SubTaskMeta{
+ Name: "extractApiCommitStats",
+ EntryPoint: ExtractApiCommitStats,
+ EnabledByDefault: false,
+ Description: "Extract raw commit stats data into tool layer table gitee_commit_stats",
+}
+
+type ApiSingleCommitResponse struct {
+ Sha string
+ Stats struct {
+ id string
+ Additions int
+ Deletions int
+ total int
+ }
+ Commit struct {
+ Committer struct {
+ Name string
+ Email string
+ Date helper.Iso8601Time
+ }
+ }
+}
+
+func ExtractApiCommitStats(taskCtx core.SubTaskContext) error {
+ rawDataSubTaskArgs, _ := CreateRawDataSubTaskArgs(taskCtx, RAW_COMMIT_STATS_TABLE)
+
+ extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{
+ RawDataSubTaskArgs: *rawDataSubTaskArgs,
+ Extract: func(row *helper.RawData) ([]interface{}, error) {
+ body := &ApiSingleCommitResponse{}
+ err := json.Unmarshal(row.Data, body)
+ if err != nil {
+ return nil, err
+ }
+ if body.Sha == "" {
+ return nil, nil
+ }
+
+ db := taskCtx.GetDb()
+ commit := &models.GiteeCommit{}
+ err = db.Model(commit).Where("sha = ?", body.Sha).Limit(1).Find(commit).Error
+ if err != nil {
+ return nil, err
+ }
+
+ commit.Additions = body.Stats.Additions
+ commit.Deletions = body.Stats.Deletions
+
+ commitStat := &models.GiteeCommitStat{
+ Additions: body.Stats.Additions,
+ Deletions: body.Stats.Deletions,
+ CommittedDate: body.Commit.Committer.Date.ToTime(),
+ Sha: body.Sha,
+ }
+
+ results := make([]interface{}, 0, 2)
+
+ results = append(results, commit)
+ results = append(results, commitStat)
+
+ return results, nil
+ },
+ })
+
+ if err != nil {
+ return err
+ }
+
+ return extractor.Execute()
+}
diff --git a/plugins/gitee/tasks/issue_collector.go b/plugins/gitee/tasks/issue_collector.go
new file mode 100644
index 00000000..4b882233
--- /dev/null
+++ b/plugins/gitee/tasks/issue_collector.go
@@ -0,0 +1,97 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "encoding/json"
+ "fmt"
+ "net/http"
+ "net/url"
+
+ "github.com/apache/incubator-devlake/plugins/helper"
+
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/gitee/models"
+)
+
+const RAW_ISSUE_TABLE = "gitee_api_issues"
+
+var CollectApiIssuesMeta = core.SubTaskMeta{
+ Name: "collectApiIssues",
+ EntryPoint: CollectApiIssues,
+ EnabledByDefault: true,
+ Description: "Collect issues data from Gitee api",
+}
+
+func CollectApiIssues(taskCtx core.SubTaskContext) error {
+ db := taskCtx.GetDb()
+ rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_ISSUE_TABLE)
+
+ since := data.Since
+ incremental := false
+ // user didn't specify a time range to sync, try load from database
+ if since == nil {
+ var latestUpdated models.GiteeIssue
+ err := db.Model(&latestUpdated).
+ Where("repo_id = ?", data.Repo.GiteeId).
+ Order("gitee_updated_at DESC").Limit(1).Find(&latestUpdated).Error
+ if err != nil {
+ return fmt.Errorf("failed to get latest gitee issue record: %w", err)
+ }
+ if latestUpdated.GiteeId > 0 {
+ since = &latestUpdated.GiteeUpdatedAt
+ incremental = true
+ }
+ }
+
+ collector, err := helper.NewApiCollector(helper.ApiCollectorArgs{
+ RawDataSubTaskArgs: *rawDataSubTaskArgs,
+ ApiClient: data.ApiClient,
+ PageSize: 100,
+ Incremental: incremental,
+ UrlTemplate: "repos/{{ .Params.Owner }}/{{ .Params.Repo }}/issues",
+ Query: func(reqData *helper.RequestData) (url.Values, error) {
+ query := url.Values{}
+ query.Set("access_token", data.Options.Token)
+ query.Set("state", "all")
+ if since != nil {
+ query.Set("since", since.String())
+ }
+ query.Set("direction", "asc")
+ query.Set("page", fmt.Sprintf("%v", reqData.Pager.Page))
+ query.Set("per_page", fmt.Sprintf("%v", reqData.Pager.Size))
+
+ return query, nil
+ },
+ GetTotalPages: GetTotalPagesFromResponse,
+ ResponseParser: func(res *http.Response) ([]json.RawMessage, error) {
+ var items []json.RawMessage
+ err := helper.UnmarshalResponse(res, &items)
+ if err != nil {
+ return nil, err
+ }
+ return items, nil
+ },
+ })
+
+ if err != nil {
+ return err
+ }
+
+ return collector.Execute()
+}
diff --git a/plugins/gitee/tasks/issue_comment_collector.go b/plugins/gitee/tasks/issue_comment_collector.go
new file mode 100644
index 00000000..a74b08fb
--- /dev/null
+++ b/plugins/gitee/tasks/issue_comment_collector.go
@@ -0,0 +1,110 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "fmt"
+ "net/url"
+
+ "github.com/apache/incubator-devlake/plugins/helper"
+
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/gitee/models"
+)
+
+const RAW_COMMENTS_TABLE = "gitee_issue_comments"
+
+var CollectApiIssueCommentsMeta = core.SubTaskMeta{
+ Name: "collectApiIssueComments",
+ EntryPoint: CollectApiIssueComments,
+ EnabledByDefault: true,
+ Description: "Collect comments data from Gitee api",
+}
+
+func CollectApiIssueComments(taskCtx core.SubTaskContext) error {
+ db := taskCtx.GetDb()
+ rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_COMMENTS_TABLE)
+
+ since := data.Since
+ incremental := false
+ // user didn't specify a time range to sync, try load from database
+ // actually, for gitee pull, since doesn't make any sense, gitee pull api doesn't support it
+ if since == nil {
+ var latestUpdatedIssueComment models.GiteeIssueComment
+ err := db.Model(&latestUpdatedIssueComment).
+ Joins("left join _tool_gitee_issues on _tool_gitee_issues.gitee_id = _tool_gitee_issue_comments.issue_id").
+ Where("_tool_gitee_issues.repo_id = ?", data.Repo.GiteeId).
+ Order("gitee_updated_at DESC").Limit(1).Find(&latestUpdatedIssueComment).Error
+ if err != nil {
+ return fmt.Errorf("failed to get latest gitee issue record: %w", err)
+ }
+ var latestUpdatedPrComt models.GiteePullRequestComment
+ err = db.Model(&latestUpdatedPrComt).
+ Joins("left join _tool_gitee_pull_requests on _tool_gitee_pull_requests.gitee_id = _tool_gitee_pull_request_comments.pull_request_id").
+ Where("_tool_gitee_pull_requests.repo_id = ?", data.Repo.GiteeId).
+ Order("gitee_updated_at DESC").Limit(1).Find(&latestUpdatedPrComt).Error
+ if err != nil {
+ return fmt.Errorf("failed to get latest gitee issue record: %w", err)
+ }
+ if latestUpdatedIssueComment.GiteeId > 0 && latestUpdatedPrComt.GiteeId > 0 {
+ if latestUpdatedIssueComment.GiteeUpdatedAt.Before(latestUpdatedPrComt.GiteeUpdatedAt) {
+ since = &latestUpdatedPrComt.GiteeUpdatedAt
+ } else {
+ since = &latestUpdatedIssueComment.GiteeUpdatedAt
+ }
+ incremental = true
+ } else if latestUpdatedIssueComment.GiteeId > 0 {
+ since = &latestUpdatedIssueComment.GiteeUpdatedAt
+ incremental = true
+ } else if latestUpdatedPrComt.GiteeId > 0 {
+ since = &latestUpdatedPrComt.GiteeUpdatedAt
+ incremental = true
+ }
+
+ }
+
+ collector, err := helper.NewApiCollector(helper.ApiCollectorArgs{
+ RawDataSubTaskArgs: *rawDataSubTaskArgs,
+ ApiClient: data.ApiClient,
+ PageSize: 100,
+ Incremental: incremental,
+
+ UrlTemplate: "repos/{{ .Params.Owner }}/{{ .Params.Repo }}/issues/comments",
+ Query: func(reqData *helper.RequestData) (url.Values, error) {
+ query := url.Values{}
+ query.Set("access_token", data.Options.Token)
+ query.Set("state", "all")
+ if since != nil {
+ query.Set("since", since.String())
+ }
+ query.Set("page", fmt.Sprintf("%v", reqData.Pager.Page))
+ query.Set("direction", "asc")
+ query.Set("per_page", fmt.Sprintf("%v", reqData.Pager.Size))
+
+ return query, nil
+ },
+ GetTotalPages: GetTotalPagesFromResponse,
+ ResponseParser: GetRawMessageFromResponse,
+ })
+
+ if err != nil {
+ return err
+ }
+
+ return collector.Execute()
+}
diff --git a/plugins/gitee/tasks/issue_comment_convertor.go b/plugins/gitee/tasks/issue_comment_convertor.go
new file mode 100644
index 00000000..279d6846
--- /dev/null
+++ b/plugins/gitee/tasks/issue_comment_convertor.go
@@ -0,0 +1,80 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "reflect"
+
+ "github.com/apache/incubator-devlake/models/domainlayer"
+ "github.com/apache/incubator-devlake/models/domainlayer/didgen"
+ "github.com/apache/incubator-devlake/models/domainlayer/ticket"
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/gitee/models"
+ "github.com/apache/incubator-devlake/plugins/helper"
+)
+
+var ConvertIssueCommentsMeta = core.SubTaskMeta{
+ Name: "convertIssueComments",
+ EntryPoint: ConvertIssueComments,
+ EnabledByDefault: true,
+ Description: "ConvertIssueComments data from Gitee api",
+}
+
+func ConvertIssueComments(taskCtx core.SubTaskContext) error {
+ db := taskCtx.GetDb()
+ rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_COMMENTS_TABLE)
+ repoId := data.Repo.GiteeId
+
+ cursor, err := db.Model(&models.GiteeIssueComment{}).
+ Joins("left join _tool_gitee_issues "+
+ "on _tool_gitee_issues.gitee_id = _tool_gitee_issue_comments.issue_id").
+ Where("repo_id = ?", repoId).Rows()
+ if err != nil {
+ return err
+ }
+ defer cursor.Close()
+
+ issueIdGen := didgen.NewDomainIdGenerator(&models.GiteeIssue{})
+ userIdGen := didgen.NewDomainIdGenerator(&models.GiteeUser{})
+
+ converter, err := helper.NewDataConverter(helper.DataConverterArgs{
+ InputRowType: reflect.TypeOf(models.GiteeIssueComment{}),
+ Input: cursor,
+ RawDataSubTaskArgs: *rawDataSubTaskArgs,
+ Convert: func(inputRow interface{}) ([]interface{}, error) {
+ giteeIssueComment := inputRow.(*models.GiteeIssueComment)
+ domainIssueComment := &ticket.IssueComment{
+ DomainEntity: domainlayer.DomainEntity{
+ Id: issueIdGen.Generate(giteeIssueComment.GiteeId),
+ },
+ IssueId: issueIdGen.Generate(giteeIssueComment.IssueId),
+ Body: giteeIssueComment.Body,
+ UserId: userIdGen.Generate(giteeIssueComment.AuthorUserId),
+ CreatedDate: giteeIssueComment.GiteeCreatedAt,
+ }
+ return []interface{}{
+ domainIssueComment,
+ }, nil
+ },
+ })
+ if err != nil {
+ return err
+ }
+
+ return converter.Execute()
+}
diff --git a/plugins/gitee/tasks/issue_comment_extractor.go b/plugins/gitee/tasks/issue_comment_extractor.go
new file mode 100644
index 00000000..8e321b05
--- /dev/null
+++ b/plugins/gitee/tasks/issue_comment_extractor.go
@@ -0,0 +1,119 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "encoding/json"
+
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/gitee/models"
+ "github.com/apache/incubator-devlake/plugins/helper"
+)
+
+var ExtractApiIssueCommentsMeta = core.SubTaskMeta{
+ Name: "extractApiIssueComments",
+ EntryPoint: ExtractApiIssueComments,
+ EnabledByDefault: true,
+ Description: "Extract raw comment data into tool layer table gitee_pull_request_comments" +
+ "and gitee_issue_comments",
+}
+
+type IssueComment struct {
+ GiteeId int `json:"id"`
+ Body string
+ User struct {
+ Login string
+ Id int
+ }
+ IssueUrl string `json:"issue_url"`
+ GiteeCreatedAt helper.Iso8601Time `json:"created_at"`
+ GiteeUpdatedAt helper.Iso8601Time `json:"updated_at"`
+}
+
+func ExtractApiIssueComments(taskCtx core.SubTaskContext) error {
+ data := taskCtx.GetData().(*GiteeTaskData)
+
+ extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{
+ RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
+ Ctx: taskCtx,
+ Params: GiteeApiParams{
+ Owner: data.Options.Owner,
+ Repo: data.Options.Repo,
+ },
+ Table: RAW_COMMENTS_TABLE,
+ },
+ Extract: func(row *helper.RawData) ([]interface{}, error) {
+ apiComment := &IssueComment{}
+ err := json.Unmarshal(row.Data, apiComment)
+ if err != nil {
+ return nil, err
+ }
+ // need to extract 2 kinds of entities here
+ results := make([]interface{}, 0, 2)
+ if apiComment.GiteeId == 0 {
+ return nil, nil
+ }
+ //If this is a pr, ignore
+ issueINumber, err := GetIssueIdByIssueUrl(apiComment.IssueUrl)
+ if err != nil {
+ return nil, err
+ }
+ issue := &models.GiteeIssue{}
+ err = taskCtx.GetDb().Where("number = ? and repo_id = ?", issueINumber, data.Repo.GiteeId).Limit(1).Find(issue).Error
+ if err != nil {
+ return nil, err
+ }
+ //if we can not find issues with issue number above, move the comments to gitee_pull_request_comments
+ if issue.GiteeId == 0 {
+ pr := &models.GiteePullRequest{}
+ err = taskCtx.GetDb().Where("number = ? and repo_id = ?", issueINumber, data.Repo.GiteeId).Limit(1).Find(pr).Error
+ if err != nil {
+ return nil, err
+ }
+ giteePrComment := &models.GiteePullRequestComment{
+ GiteeId: apiComment.GiteeId,
+ PullRequestId: pr.GiteeId,
+ Body: apiComment.Body,
+ AuthorUsername: apiComment.User.Login,
+ AuthorUserId: apiComment.User.Id,
+ GiteeCreatedAt: apiComment.GiteeCreatedAt.ToTime(),
+ GiteeUpdatedAt: apiComment.GiteeUpdatedAt.ToTime(),
+ }
+ results = append(results, giteePrComment)
+ } else {
+ giteeIssueComment := &models.GiteeIssueComment{
+ GiteeId: apiComment.GiteeId,
+ IssueId: issue.GiteeId,
+ Body: apiComment.Body,
+ AuthorUsername: apiComment.User.Login,
+ AuthorUserId: apiComment.User.Id,
+ GiteeCreatedAt: apiComment.GiteeCreatedAt.ToTime(),
+ GiteeUpdatedAt: apiComment.GiteeUpdatedAt.ToTime(),
+ }
+ results = append(results, giteeIssueComment)
+ }
+ return results, nil
+ },
+ })
+
+ if err != nil {
+ return err
+ }
+
+ return extractor.Execute()
+}
diff --git a/plugins/gitee/tasks/issue_convertor.go b/plugins/gitee/tasks/issue_convertor.go
new file mode 100644
index 00000000..0fe2d3a1
--- /dev/null
+++ b/plugins/gitee/tasks/issue_convertor.go
@@ -0,0 +1,101 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "reflect"
+
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/helper"
+
+ "github.com/apache/incubator-devlake/models/domainlayer"
+ "github.com/apache/incubator-devlake/models/domainlayer/didgen"
+ "github.com/apache/incubator-devlake/models/domainlayer/ticket"
+ giteeModels "github.com/apache/incubator-devlake/plugins/gitee/models"
+)
+
+var ConvertIssuesMeta = core.SubTaskMeta{
+ Name: "convertIssues",
+ EntryPoint: ConvertIssues,
+ EnabledByDefault: true,
+ Description: "Convert tool layer table gitee_issues into domain layer table issues",
+}
+
+func ConvertIssues(taskCtx core.SubTaskContext) error {
+ db := taskCtx.GetDb()
+ rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_ISSUE_TABLE)
+ repoId := data.Repo.GiteeId
+
+ issue := &giteeModels.GiteeIssue{}
+ cursor, err := db.Model(issue).Where("repo_id = ?", repoId).Rows()
+
+ if err != nil {
+ return err
+ }
+ defer cursor.Close()
+
+ issueIdGen := didgen.NewDomainIdGenerator(&giteeModels.GiteeIssue{})
+ userIdGen := didgen.NewDomainIdGenerator(&giteeModels.GiteeUser{})
+ boardIdGen := didgen.NewDomainIdGenerator(&giteeModels.GiteeRepo{})
+
+ converter, err := helper.NewDataConverter(helper.DataConverterArgs{
+ RawDataSubTaskArgs: *rawDataSubTaskArgs,
+ InputRowType: reflect.TypeOf(giteeModels.GiteeIssue{}),
+ Input: cursor,
+ Convert: func(inputRow interface{}) ([]interface{}, error) {
+ issue := inputRow.(*giteeModels.GiteeIssue)
+ domainIssue := &ticket.Issue{
+ DomainEntity: domainlayer.DomainEntity{Id: issueIdGen.Generate(issue.GiteeId)},
+ IssueKey: issue.Number,
+ Title: issue.Title,
+ Description: issue.Body,
+ Priority: issue.Priority,
+ Type: issue.Type,
+ AssigneeId: userIdGen.Generate(issue.AssigneeId),
+ AssigneeName: issue.AssigneeName,
+ CreatorId: userIdGen.Generate(issue.AuthorId),
+ CreatorName: issue.AuthorName,
+ LeadTimeMinutes: issue.LeadTimeMinutes,
+ Url: issue.Url,
+ CreatedDate: &issue.GiteeCreatedAt,
+ UpdatedDate: &issue.GiteeUpdatedAt,
+ ResolutionDate: issue.ClosedAt,
+ Severity: issue.Severity,
+ Component: issue.Component,
+ }
+ if issue.State == "closed" {
+ domainIssue.Status = ticket.DONE
+ } else {
+ domainIssue.Status = ticket.TODO
+ }
+ boardIssue := &ticket.BoardIssue{
+ BoardId: boardIdGen.Generate(repoId),
+ IssueId: domainIssue.Id,
+ }
+ return []interface{}{
+ domainIssue,
+ boardIssue,
+ }, nil
+ },
+ })
+ if err != nil {
+ return err
+ }
+
+ return converter.Execute()
+}
diff --git a/plugins/gitee/tasks/issue_extractor.go b/plugins/gitee/tasks/issue_extractor.go
new file mode 100644
index 00000000..69cb7a57
--- /dev/null
+++ b/plugins/gitee/tasks/issue_extractor.go
@@ -0,0 +1,253 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "encoding/json"
+ "regexp"
+
+ "github.com/apache/incubator-devlake/models/domainlayer/ticket"
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/gitee/models"
+ "github.com/apache/incubator-devlake/plugins/helper"
+)
+
+var ExtractApiIssuesMeta = core.SubTaskMeta{
+ Name: "extractApiIssues",
+ EntryPoint: ExtractApiIssues,
+ EnabledByDefault: true,
+ Description: "Extract raw Issues data into tool layer table gitee_issues",
+}
+
+type IssuesResponse struct {
+ GiteeId int `json:"id"`
+ Url string `json:"url"`
+ RepositoryUrl string `json:"repository_url"`
+ Number string `json:"number"`
+ State string `json:"state"`
+ Title string
+ Body string
+ HtmlUrl string `json:"html_url"`
+ CommentsUrl string `json:"comments_url"`
+ PullRequest struct {
+ Url string `json:"url"`
+ HtmlUrl string `json:"html_url"`
+ } `json:"pull_request"`
+ Labels []struct {
+ Id int
+ RepositoryId int `json:"repository_id"`
+ Name string `json:"name"`
+ CreatedAt helper.Iso8601Time `json:"created_at"`
+ UpdatedAt helper.Iso8601Time `json:"updated_at"`
+ } `json:"labels"`
+ Repository struct {
+ Id int
+ FullName string `json:"full_name"`
+ Url string `json:"url"`
+ } `json:"repository"`
+ Assignee *struct {
+ Login string
+ Id int
+ }
+ User *struct {
+ Login string
+ Id int
+ Name string
+ }
+ Comments int `json:"comments"`
+ Priority int `json:"priority"`
+ IssueType string `json:"issue_type"`
+ SecurityHole bool `json:"security_hole"`
+ IssueState string `json:"issue_state"`
+ Branch string `json:"branch"`
+ FinishAt *helper.Iso8601Time `json:"finished_at"`
+ GiteeCreatedAt helper.Iso8601Time `json:"created_at"`
+ GiteeUpdatedAt helper.Iso8601Time `json:"updated_at"`
+ IssueTypeDetail struct {
+ Id int
+ Title string
+ Ident string
+ CreatedAt helper.Iso8601Time `json:"created_at"`
+ UpdatedAt helper.Iso8601Time `json:"updated_at"`
+ }
+ IssueStateDetail struct {
+ Id int
+ Title string
+ Serial string
+ CreatedAt helper.Iso8601Time `json:"created_at"`
+ UpdatedAt helper.Iso8601Time `json:"updated_at"`
+ }
+}
+
+func ExtractApiIssues(taskCtx core.SubTaskContext) error {
+ rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_ISSUE_TABLE)
+ config := data.Options.Config
+ var issueSeverityRegex *regexp.Regexp
+ var issueComponentRegex *regexp.Regexp
+ var issuePriorityRegex *regexp.Regexp
+ var issueTypeBugRegex *regexp.Regexp
+ var issueTypeRequirementRegex *regexp.Regexp
+ var issueTypeIncidentRegex *regexp.Regexp
+ var issueSeverity = config.IssueSeverity
+ if issueSeverity == "" {
+ issueSeverity = taskCtx.GetConfig("GITEE_ISSUE_SEVERITY")
+ }
+ var issueComponent = config.IssueComponent
+ if issueComponent == "" {
+ issueComponent = taskCtx.GetConfig("GITEE_ISSUE_COMPONENT")
+ }
+ var issuePriority = config.IssuePriority
+ if issuePriority == "" {
+ issuePriority = taskCtx.GetConfig("GITEE_ISSUE_PRIORITY")
+ }
+ var issueTypeBug = config.IssueTypeBug
+ if issueTypeBug == "" {
+ issueTypeBug = taskCtx.GetConfig("GITEE_ISSUE_TYPE_BUG")
+ }
+ var issueTypeRequirement = config.IssueTypeRequirement
+ if issueTypeRequirement == "" {
+ issueTypeRequirement = taskCtx.GetConfig("GITEE_ISSUE_TYPE_REQUIREMENT")
+ }
+ var issueTypeIncident = config.IssueTypeIncident
+ if issueTypeIncident == "" {
+ issueTypeIncident = taskCtx.GetConfig("GITEE_ISSUE_TYPE_INCIDENT")
+ }
+ if len(issueSeverity) > 0 {
+ issueSeverityRegex = regexp.MustCompile(issueSeverity)
+ }
+ if len(issueComponent) > 0 {
+ issueComponentRegex = regexp.MustCompile(issueComponent)
+ }
+ if len(issuePriority) > 0 {
+ issuePriorityRegex = regexp.MustCompile(issuePriority)
+ }
+ if len(issueTypeBug) > 0 {
+ issueTypeBugRegex = regexp.MustCompile(issueTypeBug)
+ }
+ if len(issueTypeRequirement) > 0 {
+ issueTypeRequirementRegex = regexp.MustCompile(issueTypeRequirement)
+ }
+ if len(issueTypeIncident) > 0 {
+ issueTypeIncidentRegex = regexp.MustCompile(issueTypeIncident)
+ }
+
+ extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{
+ RawDataSubTaskArgs: *rawDataSubTaskArgs,
+ Extract: func(row *helper.RawData) ([]interface{}, error) {
+ body := &IssuesResponse{}
+ err := json.Unmarshal(row.Data, body)
+ if err != nil {
+ return nil, err
+ }
+ // need to extract 2 kinds of entities here
+ if body.GiteeId == 0 {
+ return nil, nil
+ }
+ //If this is a pr, ignore
+ if body.PullRequest.Url != "" {
+ return nil, nil
+ }
+ results := make([]interface{}, 0, 2)
+ giteeIssue, err := convertGiteeIssue(body, data.Repo.GiteeId)
+ if err != nil {
+ return nil, err
+ }
+ for _, label := range body.Labels {
+ results = append(results, &models.GiteeIssueLabel{
+ IssueId: giteeIssue.GiteeId,
+ LabelName: label.Name,
+ })
+ if issueSeverityRegex != nil {
+ groups := issueSeverityRegex.FindStringSubmatch(label.Name)
+ if len(groups) > 0 {
+ giteeIssue.Severity = groups[1]
+ }
+ }
+
+ if issueComponentRegex != nil {
+ groups := issueComponentRegex.FindStringSubmatch(label.Name)
+ if len(groups) > 0 {
+ giteeIssue.Component = groups[1]
+ }
+ }
+
+ if issuePriorityRegex != nil {
+ groups := issuePriorityRegex.FindStringSubmatch(label.Name)
+ if len(groups) > 0 {
+ giteeIssue.Priority = groups[1]
+ }
+ }
+
+ if issueTypeBugRegex != nil {
+ if ok := issueTypeBugRegex.MatchString(label.Name); ok {
+ giteeIssue.Type = ticket.BUG
+ }
+ }
+
+ if issueTypeRequirementRegex != nil {
+ if ok := issueTypeRequirementRegex.MatchString(label.Name); ok {
+ giteeIssue.Type = ticket.REQUIREMENT
+ }
+ }
+
+ if issueTypeIncidentRegex != nil {
+ if ok := issueTypeIncidentRegex.MatchString(label.Name); ok {
+ giteeIssue.Type = ticket.INCIDENT
+ }
+ }
+ }
+ results = append(results, giteeIssue)
+
+ return results, nil
+ },
+ })
+
+ if err != nil {
+ return err
+ }
+
+ return extractor.Execute()
+}
+func convertGiteeIssue(issue *IssuesResponse, repositoryId int) (*models.GiteeIssue, error) {
+ giteeIssue := &models.GiteeIssue{
+ GiteeId: issue.GiteeId,
+ RepoId: repositoryId,
+ Number: issue.Number,
+ State: issue.State,
+ Title: issue.Title,
+ Body: issue.Body,
+ Url: issue.HtmlUrl,
+ ClosedAt: helper.Iso8601TimeToTime(issue.FinishAt),
+ GiteeCreatedAt: issue.GiteeCreatedAt.ToTime(),
+ GiteeUpdatedAt: issue.GiteeUpdatedAt.ToTime(),
+ }
+
+ if issue.Assignee != nil {
+ giteeIssue.AssigneeId = issue.Assignee.Id
+ giteeIssue.AssigneeName = issue.Assignee.Login
+ }
+ if issue.User != nil {
+ giteeIssue.AuthorId = issue.User.Id
+ giteeIssue.AuthorName = issue.User.Login
+ }
+ if issue.FinishAt != nil {
+ giteeIssue.LeadTimeMinutes = uint(issue.FinishAt.ToTime().Sub(issue.GiteeCreatedAt.ToTime()).Minutes())
+ }
+
+ return giteeIssue, nil
+}
diff --git a/plugins/gitee/tasks/issue_label_convertor.go b/plugins/gitee/tasks/issue_label_convertor.go
new file mode 100644
index 00000000..d017cffa
--- /dev/null
+++ b/plugins/gitee/tasks/issue_label_convertor.go
@@ -0,0 +1,73 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "reflect"
+
+ "github.com/apache/incubator-devlake/models/domainlayer/didgen"
+ "github.com/apache/incubator-devlake/models/domainlayer/ticket"
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/gitee/models"
+ "github.com/apache/incubator-devlake/plugins/helper"
+)
+
+var ConvertIssueLabelsMeta = core.SubTaskMeta{
+ Name: "convertIssueLabels",
+ EntryPoint: ConvertIssueLabels,
+ EnabledByDefault: true,
+ Description: "Convert tool layer table gitee_issue_labels into domain layer table issue_labels",
+}
+
+func ConvertIssueLabels(taskCtx core.SubTaskContext) error {
+ db := taskCtx.GetDb()
+ rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_ISSUE_TABLE)
+ repoId := data.Repo.GiteeId
+
+ cursor, err := db.Model(&models.GiteeIssueLabel{}).
+ Joins(`left join _tool_gitee_issues on _tool_gitee_issues.gitee_id = _tool_gitee_issue_labels.issue_id`).
+ Where("_tool_gitee_issues.repo_id = ?", repoId).
+ Order("issue_id ASC").
+ Rows()
+ if err != nil {
+ return err
+ }
+ defer cursor.Close()
+ issueIdGen := didgen.NewDomainIdGenerator(&models.GiteeIssue{})
+
+ converter, err := helper.NewDataConverter(helper.DataConverterArgs{
+ RawDataSubTaskArgs: *rawDataSubTaskArgs,
+ InputRowType: reflect.TypeOf(models.GiteeIssueLabel{}),
+ Input: cursor,
+ Convert: func(inputRow interface{}) ([]interface{}, error) {
+ issueLabel := inputRow.(*models.GiteeIssueLabel)
+ domainIssueLabel := &ticket.IssueLabel{
+ IssueId: issueIdGen.Generate(issueLabel.IssueId),
+ LabelName: issueLabel.LabelName,
+ }
+ return []interface{}{
+ domainIssueLabel,
+ }, nil
+ },
+ })
+ if err != nil {
+ return err
+ }
+
+ return converter.Execute()
+}
diff --git a/plugins/gitee/tasks/pr_collector.go b/plugins/gitee/tasks/pr_collector.go
new file mode 100644
index 00000000..31f80ddc
--- /dev/null
+++ b/plugins/gitee/tasks/pr_collector.go
@@ -0,0 +1,99 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "encoding/json"
+ "fmt"
+ "net/http"
+ "net/url"
+
+ "github.com/apache/incubator-devlake/plugins/gitee/models"
+
+ "github.com/apache/incubator-devlake/plugins/helper"
+
+ "github.com/apache/incubator-devlake/plugins/core"
+)
+
+const RAW_PULL_REQUEST_TABLE = "gitee_api_pull_requests"
+
+var CollectApiPullRequestsMeta = core.SubTaskMeta{
+ Name: "collectApiPullRequests",
+ EntryPoint: CollectApiPullRequests,
+ EnabledByDefault: true,
+ Description: "Collect PullRequests data from Gitee api",
+}
+
+func CollectApiPullRequests(taskCtx core.SubTaskContext) error {
+ db := taskCtx.GetDb()
+ rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_PULL_REQUEST_TABLE)
+ since := data.Since
+ incremental := false
+ if since == nil {
+ var latestUpdated models.GiteePullRequest
+ err := db.Model(&latestUpdated).
+ Where("repo_id = ?", data.Repo.GiteeId).
+ Order("gitee_updated_at DESC").Limit(1).Find(&latestUpdated).Error
+ if err != nil {
+ return fmt.Errorf("failed to get latest gitee issue record: %w", err)
+ }
+ if latestUpdated.GiteeId > 0 {
+ since = &latestUpdated.GiteeUpdatedAt
+ incremental = true
+ }
+ }
+
+ collector, err := helper.NewApiCollector(helper.ApiCollectorArgs{
+ RawDataSubTaskArgs: *rawDataSubTaskArgs,
+ ApiClient: data.ApiClient,
+ PageSize: 100,
+ Incremental: incremental,
+
+ UrlTemplate: "repos/{{ .Params.Owner }}/{{ .Params.Repo }}/pulls",
+
+ Query: func(reqData *helper.RequestData) (url.Values, error) {
+ query := url.Values{}
+ query.Set("access_token", data.Options.Token)
+ query.Set("state", "all")
+ if since != nil {
+ query.Set("since", since.String())
+ }
+ query.Set("page", fmt.Sprintf("%v", reqData.Pager.Page))
+ query.Set("direction", "asc")
+ query.Set("per_page", fmt.Sprintf("%v", reqData.Pager.Size))
+
+ return query, nil
+ },
+
+ GetTotalPages: GetTotalPagesFromResponse,
+ ResponseParser: func(res *http.Response) ([]json.RawMessage, error) {
+ var items []json.RawMessage
+ err := helper.UnmarshalResponse(res, &items)
+ if err != nil {
+ return nil, err
+ }
+ return items, nil
+ },
+ })
+
+ if err != nil {
+ return err
+ }
+
+ return collector.Execute()
+}
diff --git a/plugins/gitee/tasks/pr_comment_convertor.go b/plugins/gitee/tasks/pr_comment_convertor.go
new file mode 100644
index 00000000..03aff955
--- /dev/null
+++ b/plugins/gitee/tasks/pr_comment_convertor.go
@@ -0,0 +1,82 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "reflect"
+
+ "github.com/apache/incubator-devlake/models/domainlayer"
+ "github.com/apache/incubator-devlake/models/domainlayer/code"
+ "github.com/apache/incubator-devlake/models/domainlayer/didgen"
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/gitee/models"
+ "github.com/apache/incubator-devlake/plugins/helper"
+)
+
+var ConvertPullRequestCommentsMeta = core.SubTaskMeta{
+ Name: "convertPullRequestComments",
+ EntryPoint: ConvertPullRequestComments,
+ EnabledByDefault: true,
+ Description: "ConvertPullRequestComments data from Gitee api",
+}
+
+func ConvertPullRequestComments(taskCtx core.SubTaskContext) error {
+ db := taskCtx.GetDb()
+ rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_COMMENTS_TABLE)
+ repoId := data.Repo.GiteeId
+
+ cursor, err := db.Model(&models.GiteePullRequestComment{}).
+ Joins("left join _tool_gitee_pull_requests "+
+ "on _tool_gitee_pull_requests.gitee_id = _tool_gitee_pull_request_comments.pull_request_id").
+ Where("repo_id = ?", repoId).Rows()
+ if err != nil {
+ return err
+ }
+ defer cursor.Close()
+
+ prIdGen := didgen.NewDomainIdGenerator(&models.GiteePullRequest{})
+ userIdGen := didgen.NewDomainIdGenerator(&models.GiteeUser{})
+
+ converter, err := helper.NewDataConverter(helper.DataConverterArgs{
+ InputRowType: reflect.TypeOf(models.GiteePullRequestComment{}),
+ Input: cursor,
+ RawDataSubTaskArgs: *rawDataSubTaskArgs,
+ Convert: func(inputRow interface{}) ([]interface{}, error) {
+ giteePullRequestComment := inputRow.(*models.GiteePullRequestComment)
+ domainPrComment := &code.PullRequestComment{
+ DomainEntity: domainlayer.DomainEntity{
+ Id: prIdGen.Generate(giteePullRequestComment.GiteeId),
+ },
+ PullRequestId: prIdGen.Generate(giteePullRequestComment.PullRequestId),
+ Body: giteePullRequestComment.Body,
+ UserId: userIdGen.Generate(giteePullRequestComment.AuthorUserId),
+ CreatedDate: giteePullRequestComment.GiteeCreatedAt,
+ CommitSha: "",
+ Position: 0,
+ }
+ return []interface{}{
+ domainPrComment,
+ }, nil
+ },
+ })
+ if err != nil {
+ return err
+ }
+
+ return converter.Execute()
+}
diff --git a/plugins/gitee/tasks/pr_commit_collector.go b/plugins/gitee/tasks/pr_commit_collector.go
new file mode 100644
index 00000000..8a5a1388
--- /dev/null
+++ b/plugins/gitee/tasks/pr_commit_collector.go
@@ -0,0 +1,96 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "encoding/json"
+ "fmt"
+ "net/http"
+ "net/url"
+ "reflect"
+
+ "github.com/apache/incubator-devlake/plugins/helper"
+
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/gitee/models"
+)
+
+const RAW_PULL_REQUEST_COMMIT_TABLE = "gitee_api_pull_request_commits"
+
+var CollectApiPullRequestCommitsMeta = core.SubTaskMeta{
+ Name: "collectApiPullRequestCommits",
+ EntryPoint: CollectApiPullRequestCommits,
+ EnabledByDefault: true,
+ Description: "Collect PullRequestCommits data from Gitee api",
+}
+
+type SimplePr struct {
+ Number int
+ GiteeId int
+}
+
+func CollectApiPullRequestCommits(taskCtx core.SubTaskContext) error {
+ db := taskCtx.GetDb()
+ rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_PULL_REQUEST_COMMIT_TABLE)
+
+ incremental := false
+
+ cursor, err := db.Model(&models.GiteePullRequest{}).Select("number, gitee_id").
+ Where("repo_id = ?", data.Repo.GiteeId).
+ Rows()
+ if err != nil {
+ return err
+ }
+ iterator, err := helper.NewCursorIterator(db, cursor, reflect.TypeOf(SimplePr{}))
+ if err != nil {
+ return err
+ }
+ collector, err := helper.NewApiCollector(helper.ApiCollectorArgs{
+ RawDataSubTaskArgs: *rawDataSubTaskArgs,
+ ApiClient: data.ApiClient,
+ PageSize: 100,
+ Incremental: incremental,
+ Input: iterator,
+
+ UrlTemplate: "repos/{{ .Params.Owner }}/{{ .Params.Repo }}/pulls/{{ .Input.Number }}/commits",
+
+ Query: func(reqData *helper.RequestData) (url.Values, error) {
+ query := url.Values{}
+ query.Set("access_token", data.Options.Token)
+ query.Set("state", "all")
+ query.Set("page", fmt.Sprintf("%v", reqData.Pager.Page))
+ query.Set("direction", "asc")
+ query.Set("per_page", fmt.Sprintf("%v", reqData.Pager.Size))
+
+ return query, nil
+ },
+ ResponseParser: func(res *http.Response) ([]json.RawMessage, error) {
+ var items []json.RawMessage
+ err := helper.UnmarshalResponse(res, &items)
+ if err != nil {
+ return nil, err
+ }
+ return items, nil
+ },
+ })
+
+ if err != nil {
+ return err
+ }
+ return collector.Execute()
+}
diff --git a/plugins/gitee/tasks/pr_commit_convertor.go b/plugins/gitee/tasks/pr_commit_convertor.go
new file mode 100644
index 00000000..e0370135
--- /dev/null
+++ b/plugins/gitee/tasks/pr_commit_convertor.go
@@ -0,0 +1,73 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "reflect"
+
+ "github.com/apache/incubator-devlake/models/domainlayer/code"
+ "github.com/apache/incubator-devlake/models/domainlayer/didgen"
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/gitee/models"
+ "github.com/apache/incubator-devlake/plugins/helper"
+)
+
+var ConvertPullRequestCommitsMeta = core.SubTaskMeta{
+ Name: "convertPullRequestCommits",
+ EntryPoint: ConvertPullRequestCommits,
+ EnabledByDefault: true,
+ Description: "Convert tool layer table gitee_pull_request_commits into domain layer table pull_request_commits",
+}
+
+func ConvertPullRequestCommits(taskCtx core.SubTaskContext) (err error) {
+ db := taskCtx.GetDb()
+ rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_PULL_REQUEST_COMMIT_TABLE)
+ repoId := data.Repo.GiteeId
+
+ pullIdGen := didgen.NewDomainIdGenerator(&models.GiteePullRequest{})
+
+ cursor, err := db.Model(&models.GiteePullRequestCommit{}).
+ Joins(`left join _tool_gitee_pull_requests on _tool_gitee_pull_requests.gitee_id = _tool_gitee_pull_request_commits.pull_request_id`).
+ Where("_tool_gitee_pull_requests.repo_id = ?", repoId).
+ Order("pull_request_id ASC").Rows()
+ if err != nil {
+ return err
+ }
+ defer cursor.Close()
+
+ converter, err := helper.NewDataConverter(helper.DataConverterArgs{
+ InputRowType: reflect.TypeOf(models.GiteePullRequestCommit{}),
+ Input: cursor,
+ RawDataSubTaskArgs: *rawDataSubTaskArgs,
+ Convert: func(inputRow interface{}) ([]interface{}, error) {
+ giteePullRequestCommit := inputRow.(*models.GiteePullRequestCommit)
+ domainPrCommit := &code.PullRequestCommit{
+ CommitSha: giteePullRequestCommit.CommitSha,
+ PullRequestId: pullIdGen.Generate(giteePullRequestCommit.PullRequestId),
+ }
+ return []interface{}{
+ domainPrCommit,
+ }, nil
+ },
+ })
+ if err != nil {
+ return err
+ }
+
+ return converter.Execute()
+}
diff --git a/plugins/gitee/tasks/pr_commit_extractor.go b/plugins/gitee/tasks/pr_commit_extractor.go
new file mode 100644
index 00000000..30befb6e
--- /dev/null
+++ b/plugins/gitee/tasks/pr_commit_extractor.go
@@ -0,0 +1,127 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "encoding/json"
+ "strings"
+
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/gitee/models"
+ "github.com/apache/incubator-devlake/plugins/helper"
+)
+
+var ExtractApiPullRequestCommitsMeta = core.SubTaskMeta{
+ Name: "extractApiPullRequestCommits",
+ EntryPoint: ExtractApiPullRequestCommits,
+ EnabledByDefault: true,
+ Description: "Extract raw PullRequestCommits data into tool layer table gitee_commits",
+}
+
+type PrCommitsResponse struct {
+ Sha string `json:"sha"`
+ Commit PullRequestCommit
+ Url string
+ Author struct {
+ Id int
+ Login string
+ Name string
+ }
+ Committer struct {
+ Id int
+ Login string
+ Name string
+ }
+}
+
+type PullRequestCommit struct {
+ Author struct {
+ Name string
+ Email string
+ Date helper.Iso8601Time
+ }
+ Committer struct {
+ Name string
+ Email string
+ Date helper.Iso8601Time
+ }
+ Message string
+ CommentCount int `json:"comment_count"`
+}
+
+func ExtractApiPullRequestCommits(taskCtx core.SubTaskContext) error {
+ rawDataSubTaskArgs, _ := CreateRawDataSubTaskArgs(taskCtx, RAW_PULL_REQUEST_COMMIT_TABLE)
+ extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{
+ RawDataSubTaskArgs: *rawDataSubTaskArgs,
+ Extract: func(row *helper.RawData) ([]interface{}, error) {
+ apiPullRequestCommit := &PrCommitsResponse{}
+ if strings.HasPrefix(string(row.Data), "{\"message\": \"Not Found\"") {
+ return nil, nil
+ }
+ err := json.Unmarshal(row.Data, apiPullRequestCommit)
+ if err != nil {
+ return nil, err
+ }
+ pull := &SimplePr{}
+ err = json.Unmarshal(row.Input, pull)
+ if err != nil {
+ return nil, err
+ }
+ // need to extract 2 kinds of entities here
+ results := make([]interface{}, 0, 2)
+
+ giteeCommit, err := convertPullRequestCommit(apiPullRequestCommit)
+ if err != nil {
+ return nil, err
+ }
+ results = append(results, giteeCommit)
+
+ giteePullRequestCommit := &models.GiteePullRequestCommit{
+ CommitSha: apiPullRequestCommit.Sha,
+ PullRequestId: pull.GiteeId,
+ }
+ if err != nil {
+ return nil, err
+ }
+ results = append(results, giteePullRequestCommit)
+ return results, nil
+ },
+ })
+
+ if err != nil {
+ return err
+ }
+
+ return extractor.Execute()
+}
+
+func convertPullRequestCommit(prCommit *PrCommitsResponse) (*models.GiteeCommit, error) {
+ giteeCommit := &models.GiteeCommit{
+ Sha: prCommit.Sha,
+ Message: prCommit.Commit.Message,
+ AuthorId: prCommit.Author.Id,
+ AuthorName: prCommit.Commit.Author.Name,
+ AuthorEmail: prCommit.Commit.Author.Email,
+ AuthoredDate: prCommit.Commit.Author.Date.ToTime(),
+ CommitterName: prCommit.Commit.Committer.Name,
+ CommitterEmail: prCommit.Commit.Committer.Email,
+ CommittedDate: prCommit.Commit.Committer.Date.ToTime(),
+ WebUrl: prCommit.Url,
+ }
+ return giteeCommit, nil
+}
diff --git a/plugins/gitee/tasks/pr_convertor.go b/plugins/gitee/tasks/pr_convertor.go
new file mode 100644
index 00000000..28d4d4de
--- /dev/null
+++ b/plugins/gitee/tasks/pr_convertor.go
@@ -0,0 +1,92 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "reflect"
+
+ "github.com/apache/incubator-devlake/models/domainlayer"
+ "github.com/apache/incubator-devlake/models/domainlayer/code"
+ "github.com/apache/incubator-devlake/models/domainlayer/didgen"
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/gitee/models"
+ "github.com/apache/incubator-devlake/plugins/helper"
+)
+
+var ConvertPullRequestsMeta = core.SubTaskMeta{
+ Name: "convertPullRequests",
+ EntryPoint: ConvertPullRequests,
+ EnabledByDefault: true,
+ Description: "ConvertPullRequests data from Gitee api",
+}
+
+func ConvertPullRequests(taskCtx core.SubTaskContext) error {
+ db := taskCtx.GetDb()
+ rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_PULL_REQUEST_TABLE)
+ repoId := data.Repo.GiteeId
+
+ cursor, err := db.Model(&models.GiteePullRequest{}).Where("repo_id = ?", repoId).Rows()
+ if err != nil {
+ return err
+ }
+ defer cursor.Close()
+
+ prIdGen := didgen.NewDomainIdGenerator(&models.GiteePullRequest{})
+ repoIdGen := didgen.NewDomainIdGenerator(&models.GiteeRepo{})
+ userIdGen := didgen.NewDomainIdGenerator(&models.GiteeUser{})
+
+ converter, err := helper.NewDataConverter(helper.DataConverterArgs{
+ InputRowType: reflect.TypeOf(models.GiteePullRequest{}),
+ Input: cursor,
+ RawDataSubTaskArgs: *rawDataSubTaskArgs,
+ Convert: func(inputRow interface{}) ([]interface{}, error) {
+ pr := inputRow.(*models.GiteePullRequest)
+ domainPr := &code.PullRequest{
+ DomainEntity: domainlayer.DomainEntity{
+ Id: prIdGen.Generate(pr.GiteeId),
+ },
+ BaseRepoId: repoIdGen.Generate(pr.RepoId),
+ Status: pr.State,
+ Title: pr.Title,
+ Url: pr.Url,
+ AuthorId: userIdGen.Generate(pr.AuthorId),
+ AuthorName: pr.AuthorName,
+ Description: pr.Body,
+ CreatedDate: pr.GiteeCreatedAt,
+ MergedDate: pr.MergedAt,
+ ClosedDate: pr.ClosedAt,
+ PullRequestKey: pr.Number,
+ Type: pr.Type,
+ Component: pr.Component,
+ MergeCommitSha: pr.MergeCommitSha,
+ BaseRef: pr.BaseRef,
+ BaseCommitSha: pr.BaseCommitSha,
+ HeadRef: pr.HeadRef,
+ HeadCommitSha: pr.HeadCommitSha,
+ }
+ return []interface{}{
+ domainPr,
+ }, nil
+ },
+ })
+ if err != nil {
+ return err
+ }
+
+ return converter.Execute()
+}
diff --git a/plugins/gitee/tasks/pr_extractor.go b/plugins/gitee/tasks/pr_extractor.go
new file mode 100644
index 00000000..3a8f0540
--- /dev/null
+++ b/plugins/gitee/tasks/pr_extractor.go
@@ -0,0 +1,171 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "encoding/json"
+ "regexp"
+
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/gitee/models"
+ "github.com/apache/incubator-devlake/plugins/helper"
+)
+
+var ExtractApiPullRequestsMeta = core.SubTaskMeta{
+ Name: "extractApiPullRequests",
+ EntryPoint: ExtractApiPullRequests,
+ EnabledByDefault: true,
+ Description: "Extract raw PullRequests data into tool layer table gitee_pull_requests",
+}
+
+type GiteeApiPullResponse struct {
+ GiteeId int `json:"id"`
+ Number int
+ State string
+ Title string
+ Body json.RawMessage
+ HtmlUrl string `json:"html_url"`
+ Labels []struct {
+ Name string `json:"name"`
+ } `json:"labels"`
+ Assignee *struct {
+ Id int
+ Login string
+ Name string
+ }
+ User *struct {
+ Id int
+ Login string
+ Name string
+ }
+ ClosedAt *helper.Iso8601Time `json:"closed_at"`
+ MergedAt *helper.Iso8601Time `json:"merged_at"`
+ GiteeCreatedAt helper.Iso8601Time `json:"created_at"`
+ GiteeUpdatedAt helper.Iso8601Time `json:"updated_at"`
+ MergeCommitSha string `json:"merge_commit_sha"`
+ Head struct {
+ Ref string
+ Sha string
+ }
+ Base struct {
+ Ref string
+ Sha string
+ Repo struct {
+ Id int
+ Name string
+ Url string
+ HtmlUrl string
+ SshUrl string `json:"ssh_url"`
+ }
+ }
+}
+
+func ExtractApiPullRequests(taskCtx core.SubTaskContext) error {
+ rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_PULL_REQUEST_TABLE)
+ config := data.Options.Config
+ var labelTypeRegex *regexp.Regexp
+ var labelComponentRegex *regexp.Regexp
+ var prType = config.PrType
+ if prType == "" {
+ prType = taskCtx.GetConfig("GITEE_PR_TYPE")
+ }
+ var prComponent = config.PrComponent
+ if prComponent == "" {
+ prComponent = taskCtx.GetConfig("GITEE_PR_COMPONENT")
+ }
+ if len(prType) > 0 {
+ labelTypeRegex = regexp.MustCompile(prType)
+ }
+ if len(prComponent) > 0 {
+ labelComponentRegex = regexp.MustCompile(prComponent)
+ }
+
+ extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{
+ RawDataSubTaskArgs: *rawDataSubTaskArgs,
+ Extract: func(row *helper.RawData) ([]interface{}, error) {
+ pullResponse := &GiteeApiPullResponse{}
+ err := json.Unmarshal(row.Data, pullResponse)
+ if err != nil {
+ return nil, err
+ }
+
+ // need to extract 2 kinds of entities here
+ results := make([]interface{}, 0, 1)
+ if pullResponse.GiteeId == 0 {
+ return nil, nil
+ }
+ giteePr, err := convertGiteePullRequest(pullResponse, data.Repo.GiteeId)
+ if err != nil {
+ return nil, err
+ }
+ for _, label := range pullResponse.Labels {
+ results = append(results, &models.GiteePullRequestLabel{
+ PullId: giteePr.GiteeId,
+ LabelName: label.Name,
+ })
+ // if pr.Type has not been set and prType is set in .env, process the below
+ if labelTypeRegex != nil {
+ groups := labelTypeRegex.FindStringSubmatch(label.Name)
+ if len(groups) > 0 {
+ giteePr.Type = groups[1]
+ }
+ }
+
+ // if pr.Component has not been set and prComponent is set in .env, process
+ if labelComponentRegex != nil {
+ groups := labelComponentRegex.FindStringSubmatch(label.Name)
+ if len(groups) > 0 {
+ giteePr.Component = groups[1]
+ }
+ }
+ }
+ results = append(results, giteePr)
+
+ return results, nil
+ },
+ })
+
+ if err != nil {
+ return err
+ }
+
+ return extractor.Execute()
+}
+func convertGiteePullRequest(pull *GiteeApiPullResponse, repoId int) (*models.GiteePullRequest, error) {
+ giteePull := &models.GiteePullRequest{
+ GiteeId: pull.GiteeId,
+ RepoId: repoId,
+ Number: pull.Number,
+ State: pull.State,
+ Title: pull.Title,
+ Url: pull.HtmlUrl,
+ AuthorName: pull.User.Login,
+ AuthorId: pull.User.Id,
+ GiteeCreatedAt: pull.GiteeCreatedAt.ToTime(),
+ GiteeUpdatedAt: pull.GiteeUpdatedAt.ToTime(),
+ ClosedAt: helper.Iso8601TimeToTime(pull.ClosedAt),
+ MergedAt: helper.Iso8601TimeToTime(pull.MergedAt),
+ MergeCommitSha: pull.MergeCommitSha,
+ Body: string(pull.Body),
+ BaseRef: pull.Base.Ref,
+ BaseCommitSha: pull.Base.Sha,
+ HeadRef: pull.Head.Ref,
+ HeadCommitSha: pull.Head.Sha,
+ }
+ return giteePull, nil
+}
diff --git a/plugins/gitee/tasks/pr_issue_convertor.go b/plugins/gitee/tasks/pr_issue_convertor.go
new file mode 100644
index 00000000..07a24300
--- /dev/null
+++ b/plugins/gitee/tasks/pr_issue_convertor.go
@@ -0,0 +1,78 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "reflect"
+ "strconv"
+
+ "github.com/apache/incubator-devlake/models/domainlayer/crossdomain"
+ "github.com/apache/incubator-devlake/models/domainlayer/didgen"
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/gitee/models"
+ "github.com/apache/incubator-devlake/plugins/helper"
+)
+
+var ConvertPullRequestIssuesMeta = core.SubTaskMeta{
+ Name: "convertPullRequestIssues",
+ EntryPoint: ConvertPullRequestIssues,
+ EnabledByDefault: true,
+ Description: "Convert tool layer table gitee_pull_request_issues into domain layer table pull_request_issues",
+}
+
+func ConvertPullRequestIssues(taskCtx core.SubTaskContext) error {
+ db := taskCtx.GetDb()
+ rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_PULL_REQUEST_TABLE)
+ repoId := data.Repo.GiteeId
+
+ cursor, err := db.Model(&models.GiteePullRequestIssue{}).
+ Joins(`left join _tool_gitee_pull_requests on _tool_gitee_pull_requests.gitee_id = _tool_gitee_pull_request_issues.pull_request_id`).
+ Where("_tool_gitee_pull_requests.repo_id = ?", repoId).
+ Order("pull_request_id ASC").
+ Rows()
+ if err != nil {
+ return err
+ }
+ defer cursor.Close()
+ prIdGen := didgen.NewDomainIdGenerator(&models.GiteePullRequest{})
+ issueIdGen := didgen.NewDomainIdGenerator(&models.GiteeIssue{})
+
+ converter, err := helper.NewDataConverter(helper.DataConverterArgs{
+ InputRowType: reflect.TypeOf(models.GiteePullRequestIssue{}),
+ Input: cursor,
+ RawDataSubTaskArgs: *rawDataSubTaskArgs,
+ Convert: func(inputRow interface{}) ([]interface{}, error) {
+ giteePrIssue := inputRow.(*models.GiteePullRequestIssue)
+ issueNum, _ := strconv.Atoi(giteePrIssue.IssueNumber)
+ pullRequestIssue := &crossdomain.PullRequestIssue{
+ PullRequestId: prIdGen.Generate(giteePrIssue.PullRequestId),
+ IssueId: issueIdGen.Generate(giteePrIssue.IssueId),
+ IssueNumber: issueNum,
+ PullRequestNumber: giteePrIssue.PullRequestNumber,
+ }
+ return []interface{}{
+ pullRequestIssue,
+ }, nil
+ },
+ })
+ if err != nil {
+ return err
+ }
+
+ return converter.Execute()
+}
diff --git a/plugins/gitee/tasks/pr_issue_enricher.go b/plugins/gitee/tasks/pr_issue_enricher.go
new file mode 100644
index 00000000..3fa136cf
--- /dev/null
+++ b/plugins/gitee/tasks/pr_issue_enricher.go
@@ -0,0 +1,116 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "reflect"
+ "regexp"
+ "strconv"
+ "strings"
+
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/gitee/models"
+ "github.com/apache/incubator-devlake/plugins/helper"
+)
+
+var EnrichPullRequestIssuesMeta = core.SubTaskMeta{
+ Name: "enrichPullRequestIssues",
+ EntryPoint: EnrichPullRequestIssues,
+ EnabledByDefault: true,
+ Description: "Create tool layer table gitee_pull_request_issues from gitee_pull_reqeusts",
+}
+
+func EnrichPullRequestIssues(taskCtx core.SubTaskContext) (err error) {
+ db := taskCtx.GetDb()
+ rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_PULL_REQUEST_TABLE)
+ repoId := data.Repo.GiteeId
+
+ var prBodyCloseRegex *regexp.Regexp
+ prBodyClosePattern := taskCtx.GetConfig("GITEE_PR_BODY_CLOSE_PATTERN")
+ //the pattern before the issue number, sometimes, the issue number is #1098, sometimes it is https://xxx/#1098
+ prBodyClosePattern = strings.Replace(prBodyClosePattern, "%s", data.Options.Owner, 1)
+ prBodyClosePattern = strings.Replace(prBodyClosePattern, "%s", data.Options.Repo, 1)
+ if len(prBodyClosePattern) > 0 {
+ prBodyCloseRegex = regexp.MustCompile(prBodyClosePattern)
+ }
+ charPattern := regexp.MustCompile(`[a-zA-Z\s,]+`)
+ cursor, err := db.Model(&models.GiteePullRequest{}).
+ Where("repo_id = ?", repoId).
+ Rows()
+ if err != nil {
+ return err
+ }
+ defer cursor.Close()
+ // iterate all rows
+
+ converter, err := helper.NewDataConverter(helper.DataConverterArgs{
+ InputRowType: reflect.TypeOf(models.GiteePullRequest{}),
+ Input: cursor,
+ RawDataSubTaskArgs: *rawDataSubTaskArgs,
+ Convert: func(inputRow interface{}) ([]interface{}, error) {
+ giteePullRequst := inputRow.(*models.GiteePullRequest)
+ results := make([]interface{}, 0, 1)
+
+ //find the matched string in body
+ issueNumberListStr := ""
+
+ if prBodyCloseRegex != nil {
+ issueNumberListStr = prBodyCloseRegex.FindString(giteePullRequst.Body)
+ }
+
+ if issueNumberListStr == "" {
+ return nil, nil
+ }
+
+ issueNumberListStr = charPattern.ReplaceAllString(issueNumberListStr, "#")
+ //split the string by '#'
+ issueNumberList := strings.Split(issueNumberListStr, "#")
+
+ for _, issueNumberStr := range issueNumberList {
+ issue := &models.GiteeIssue{}
+ issueNumberStr = strings.TrimSpace(issueNumberStr)
+ //change the issueNumberStr to int, if cannot be changed, just continue
+ issueNumber, numFormatErr := strconv.Atoi(issueNumberStr)
+ if numFormatErr != nil {
+ continue
+ }
+ err = db.Where("number = ? and repo_id = ?", issueNumber, repoId).
+ Limit(1).Find(issue).Error
+ if err != nil {
+ return nil, err
+ }
+ if issue.Number == "" {
+ continue
+ }
+ giteePullRequstIssue := &models.GiteePullRequestIssue{
+ PullRequestId: giteePullRequst.GiteeId,
+ IssueId: issue.GiteeId,
+ PullRequestNumber: giteePullRequst.Number,
+ IssueNumber: issue.Number,
+ }
+ results = append(results, giteePullRequstIssue)
+ }
+ return results, nil
+ },
+ })
+ if err != nil {
+ return err
+ }
+
+ return converter.Execute()
+}
diff --git a/plugins/gitee/tasks/pr_label_convertor.go b/plugins/gitee/tasks/pr_label_convertor.go
new file mode 100644
index 00000000..14b87161
--- /dev/null
+++ b/plugins/gitee/tasks/pr_label_convertor.go
@@ -0,0 +1,73 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "reflect"
+
+ "github.com/apache/incubator-devlake/models/domainlayer/code"
+ "github.com/apache/incubator-devlake/models/domainlayer/didgen"
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/gitee/models"
+ "github.com/apache/incubator-devlake/plugins/helper"
+)
+
+var ConvertPullRequestLabelsMeta = core.SubTaskMeta{
+ Name: "convertPullRequestLabels",
+ EntryPoint: ConvertPullRequestLabels,
+ EnabledByDefault: true,
+ Description: "Convert tool layer table gitee_pull_request_labels into domain layer table pull_request_labels",
+}
+
+func ConvertPullRequestLabels(taskCtx core.SubTaskContext) error {
+ db := taskCtx.GetDb()
+ rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_PULL_REQUEST_TABLE)
+ repoId := data.Repo.GiteeId
+
+ cursor, err := db.Model(&models.GiteePullRequestLabel{}).
+ Joins(`left join _tool_gitee_pull_requests on _tool_gitee_pull_requests.gitee_id = _tool_gitee_pull_request_labels.pull_id`).
+ Where("_tool_gitee_pull_requests.repo_id = ?", repoId).
+ Order("pull_id ASC").
+ Rows()
+ if err != nil {
+ return err
+ }
+ defer cursor.Close()
+ prIdGen := didgen.NewDomainIdGenerator(&models.GiteePullRequest{})
+
+ converter, err := helper.NewDataConverter(helper.DataConverterArgs{
+ InputRowType: reflect.TypeOf(models.GiteePullRequestLabel{}),
+ Input: cursor,
+ RawDataSubTaskArgs: *rawDataSubTaskArgs,
+ Convert: func(inputRow interface{}) ([]interface{}, error) {
+ prLabel := inputRow.(*models.GiteePullRequestLabel)
+ domainPrLabel := &code.PullRequestLabel{
+ PullRequestId: prIdGen.Generate(prLabel.PullId),
+ LabelName: prLabel.LabelName,
+ }
+ return []interface{}{
+ domainPrLabel,
+ }, nil
+ },
+ })
+ if err != nil {
+ return err
+ }
+
+ return converter.Execute()
+}
diff --git a/plugins/gitee/tasks/pr_review_collector.go b/plugins/gitee/tasks/pr_review_collector.go
new file mode 100644
index 00000000..0a64752e
--- /dev/null
+++ b/plugins/gitee/tasks/pr_review_collector.go
@@ -0,0 +1,89 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "encoding/json"
+ "fmt"
+ "net/http"
+ "net/url"
+ "reflect"
+
+ "github.com/apache/incubator-devlake/plugins/helper"
+
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/gitee/models"
+)
+
+const RAW_PULL_REQUEST_REVIEW_TABLE = "gitee_api_pull_request_reviews"
+
+var CollectApiPullRequestReviewsMeta = core.SubTaskMeta{
+ Name: "collectApiPullRequestReviews",
+ EntryPoint: CollectApiPullRequestReviews,
+ EnabledByDefault: true,
+ Description: "Collect PullRequestReviews data from Gitee api",
+}
+
+func CollectApiPullRequestReviews(taskCtx core.SubTaskContext) error {
+ db := taskCtx.GetDb()
+ rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_PULL_REQUEST_REVIEW_TABLE)
+
+ incremental := false
+
+ cursor, err := db.Model(&models.GiteePullRequest{}).Select("number, gitee_id").
+ Where("repo_id = ?", data.Repo.GiteeId).
+ Rows()
+ if err != nil {
+ return err
+ }
+ iterator, err := helper.NewCursorIterator(db, cursor, reflect.TypeOf(SimplePr{}))
+ if err != nil {
+ return err
+ }
+ collector, err := helper.NewApiCollector(helper.ApiCollectorArgs{
+ RawDataSubTaskArgs: *rawDataSubTaskArgs,
+ ApiClient: data.ApiClient,
+ PageSize: 100,
+ Incremental: incremental,
+ Input: iterator,
+
+ UrlTemplate: "repos/{{ .Params.Owner }}/{{ .Params.Repo }}/pulls/3/review",
+
+ Query: func(reqData *helper.RequestData) (url.Values, error) {
+ query := url.Values{}
+ query.Set("access_token", data.Options.Token)
+ query.Set("page", fmt.Sprintf("%v", reqData.Pager.Page))
+ query.Set("per_page", fmt.Sprintf("%v", reqData.Pager.Size))
+
+ return query, nil
+ },
+ ResponseParser: func(res *http.Response) ([]json.RawMessage, error) {
+ var items []json.RawMessage
+ err := helper.UnmarshalResponse(res, &items)
+ if err != nil {
+ return nil, err
+ }
+ return items, nil
+ },
+ })
+
+ if err != nil {
+ return err
+ }
+ return collector.Execute()
+}
diff --git a/plugins/gitee/tasks/pr_review_extractor.go b/plugins/gitee/tasks/pr_review_extractor.go
new file mode 100644
index 00000000..fd4c0499
--- /dev/null
+++ b/plugins/gitee/tasks/pr_review_extractor.go
@@ -0,0 +1,84 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "encoding/json"
+ "strings"
+
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/gitee/models"
+ "github.com/apache/incubator-devlake/plugins/helper"
+)
+
+var ExtractApiPullRequestReviewsMeta = core.SubTaskMeta{
+ Name: "extractApiPullRequestReviews",
+ EntryPoint: ExtractApiPullRequestReviews,
+ EnabledByDefault: true,
+ Description: "Extract raw PullRequestReviews data into tool layer table gitee_reviewers",
+}
+
+type PullRequestReview struct {
+ GiteeId int `json:"id"`
+ User struct {
+ Id int
+ Login string
+ }
+ Body string
+ State string
+ SubmittedAt helper.Iso8601Time `json:"submitted_at"`
+}
+
+func ExtractApiPullRequestReviews(taskCtx core.SubTaskContext) error {
+ rawDataSubTaskArgs, _ := CreateRawDataSubTaskArgs(taskCtx, RAW_PULL_REQUEST_REVIEW_TABLE)
+ extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{
+ RawDataSubTaskArgs: *rawDataSubTaskArgs,
+ Extract: func(row *helper.RawData) ([]interface{}, error) {
+ apiPullRequestReview := &PullRequestReview{}
+ if strings.HasPrefix(string(row.Data), "{\"message\": \"Not Found\"") {
+ return nil, nil
+ }
+ err := json.Unmarshal(row.Data, apiPullRequestReview)
+ if err != nil {
+ return nil, err
+ }
+ pull := &SimplePr{}
+ err = json.Unmarshal(row.Input, pull)
+ if err != nil {
+ return nil, err
+ }
+ // need to extract 2 kinds of entities here
+ results := make([]interface{}, 0, 1)
+
+ giteeReviewer := &models.GiteeReviewer{
+ GiteeId: apiPullRequestReview.User.Id,
+ Login: apiPullRequestReview.User.Login,
+ PullRequestId: pull.GiteeId,
+ }
+ results = append(results, giteeReviewer)
+
+ return results, nil
+ },
+ })
+
+ if err != nil {
+ return err
+ }
+
+ return extractor.Execute()
+}
diff --git a/plugins/gitee/tasks/repo_collector.go b/plugins/gitee/tasks/repo_collector.go
new file mode 100644
index 00000000..15111300
--- /dev/null
+++ b/plugins/gitee/tasks/repo_collector.go
@@ -0,0 +1,72 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "encoding/json"
+ "fmt"
+ "io/ioutil"
+ "net/http"
+ "net/url"
+
+ "github.com/apache/incubator-devlake/plugins/helper"
+
+ "github.com/apache/incubator-devlake/plugins/core"
+)
+
+const RAW_REPOSITORIES_TABLE = "gitee_api_repo"
+
+var CollectApiRepoMeta = core.SubTaskMeta{
+ Name: "collectApiRepo",
+ EntryPoint: CollectApiRepositories,
+ Required: true,
+ Description: "Collect repositories data from Gitee api",
+}
+
+func CollectApiRepositories(taskCtx core.SubTaskContext) error {
+ rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_REPOSITORIES_TABLE)
+
+ collector, err := helper.NewApiCollector(helper.ApiCollectorArgs{
+ RawDataSubTaskArgs: *rawDataSubTaskArgs,
+ ApiClient: data.ApiClient,
+ UrlTemplate: "repos/{{ .Params.Owner }}/{{ .Params.Repo }}",
+ Query: func(reqData *helper.RequestData) (url.Values, error) {
+ query := url.Values{}
+ query.Set("access_token", data.Options.Token)
+ query.Set("state", "all")
+ query.Set("page", fmt.Sprintf("%v", reqData.Pager.Page))
+ query.Set("direction", "asc")
+ query.Set("per_page", fmt.Sprintf("%v", reqData.Pager.Size))
+ return query, nil
+ },
+ ResponseParser: func(res *http.Response) ([]json.RawMessage, error) {
+ body, err := ioutil.ReadAll(res.Body)
+ res.Body.Close()
+ if err != nil {
+ return nil, err
+ }
+ return []json.RawMessage{body}, nil
+ },
+ })
+
+ if err != nil {
+ return err
+ }
+
+ return collector.Execute()
+}
diff --git a/plugins/gitee/tasks/repo_convertor.go b/plugins/gitee/tasks/repo_convertor.go
new file mode 100644
index 00000000..21cc8b82
--- /dev/null
+++ b/plugins/gitee/tasks/repo_convertor.go
@@ -0,0 +1,95 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "fmt"
+ "reflect"
+
+ "github.com/apache/incubator-devlake/models/domainlayer"
+ "github.com/apache/incubator-devlake/models/domainlayer/code"
+ "github.com/apache/incubator-devlake/models/domainlayer/didgen"
+ "github.com/apache/incubator-devlake/models/domainlayer/ticket"
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/gitee/models"
+ "github.com/apache/incubator-devlake/plugins/helper"
+)
+
+var ConvertRepoMeta = core.SubTaskMeta{
+ Name: "convertRepo",
+ EntryPoint: ConvertRepo,
+ EnabledByDefault: true,
+ Description: "Convert tool layer table gitee_repos into domain layer table repos and boards",
+}
+
+func ConvertRepo(taskCtx core.SubTaskContext) error {
+ rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_REPOSITORIES_TABLE)
+ db := taskCtx.GetDb()
+ repoId := data.Repo.GiteeId
+
+ cursor, err := db.Model(&models.GiteeRepo{}).
+ Where("gitee_id = ?", repoId).
+ Rows()
+ if err != nil {
+ return err
+ }
+ defer cursor.Close()
+
+ repoIdGen := didgen.NewDomainIdGenerator(&models.GiteeRepo{})
+
+ converter, err := helper.NewDataConverter(helper.DataConverterArgs{
+ InputRowType: reflect.TypeOf(models.GiteeRepo{}),
+ Input: cursor,
+ RawDataSubTaskArgs: *rawDataSubTaskArgs,
+ Convert: func(inputRow interface{}) ([]interface{}, error) {
+ repository := inputRow.(*models.GiteeRepo)
+ domainRepository := &code.Repo{
+ DomainEntity: domainlayer.DomainEntity{
+ Id: repoIdGen.Generate(repository.GiteeId),
+ },
+ Name: fmt.Sprintf("%s/%s", repository.OwnerLogin, repository.Name),
+ Url: repository.HTMLUrl,
+ Description: repository.Description,
+ ForkedFrom: repository.ParentHTMLUrl,
+ Language: repository.Language,
+ CreatedDate: repository.CreatedDate,
+ UpdatedDate: repository.UpdatedDate,
+ }
+
+ domainBoard := &ticket.Board{
+ DomainEntity: domainlayer.DomainEntity{
+ Id: repoIdGen.Generate(repository.GiteeId),
+ },
+ Name: fmt.Sprintf("%s/%s", repository.OwnerLogin, repository.Name),
+ Url: fmt.Sprintf("%s/%s", repository.HTMLUrl, "issues"),
+ Description: repository.Description,
+ CreatedDate: &repository.CreatedDate,
+ }
+
+ return []interface{}{
+ domainRepository,
+ domainBoard,
+ }, nil
+ },
+ })
+ if err != nil {
+ return err
+ }
+
+ return converter.Execute()
+}
diff --git a/plugins/gitee/tasks/repo_extractor.go b/plugins/gitee/tasks/repo_extractor.go
new file mode 100644
index 00000000..02ce4a7c
--- /dev/null
+++ b/plugins/gitee/tasks/repo_extractor.go
@@ -0,0 +1,90 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "encoding/json"
+ "fmt"
+
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/gitee/models"
+ "github.com/apache/incubator-devlake/plugins/helper"
+)
+
+var ExtractApiRepoMeta = core.SubTaskMeta{
+ Name: "extractApiRepo",
+ EntryPoint: ExtractApiRepositories,
+ Required: true,
+ Description: "Extract raw Repositories data into tool layer table gitee_repos",
+}
+
+type GiteeApiRepoResponse struct {
+ Name string `json:"name"`
+ GiteeId int `json:"id"`
+ HTMLUrl string `json:"html_url"`
+ Language string `json:"language"`
+ Description string `json:"description"`
+ Owner models.GiteeUser `json:"owner"`
+ Parent *GiteeApiRepoResponse `json:"parent"`
+ CreatedAt helper.Iso8601Time `json:"created_at"`
+ UpdatedAt *helper.Iso8601Time `json:"updated_at"`
+}
+
+func ExtractApiRepositories(taskCtx core.SubTaskContext) error {
+ rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_REPOSITORIES_TABLE)
+ extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{
+ RawDataSubTaskArgs: *rawDataSubTaskArgs,
+ Extract: func(row *helper.RawData) ([]interface{}, error) {
+ repo := &GiteeApiRepoResponse{}
+ err := json.Unmarshal(row.Data, repo)
+ if err != nil {
+ return nil, err
+ }
+ if repo.GiteeId == 0 {
+ return nil, fmt.Errorf("repo %s/%s not found", data.Options.Owner, data.Options.Repo)
+ }
+ results := make([]interface{}, 0, 1)
+ giteeRepository := &models.GiteeRepo{
+ GiteeId: repo.GiteeId,
+ Name: repo.Name,
+ HTMLUrl: repo.HTMLUrl,
+ Description: repo.Description,
+ OwnerId: repo.Owner.Id,
+ OwnerLogin: repo.Owner.Login,
+ Language: repo.Language,
+ CreatedDate: repo.CreatedAt.ToTime(),
+ UpdatedDate: helper.Iso8601TimeToTime(repo.UpdatedAt),
+ }
+ data.Repo = giteeRepository
+
+ if repo.Parent != nil {
+ giteeRepository.ParentGiteeId = repo.Parent.GiteeId
+ giteeRepository.ParentHTMLUrl = repo.Parent.HTMLUrl
+ }
+ results = append(results, giteeRepository)
+ taskCtx.TaskContext().GetData().(*GiteeTaskData).Repo = giteeRepository
+ return results, nil
+ },
+ })
+
+ if err != nil {
+ return err
+ }
+
+ return extractor.Execute()
+}
diff --git a/plugins/gitee/tasks/shared.go b/plugins/gitee/tasks/shared.go
new file mode 100644
index 00000000..48238dcb
--- /dev/null
+++ b/plugins/gitee/tasks/shared.go
@@ -0,0 +1,206 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io/ioutil"
+ "net/http"
+ "regexp"
+ "strconv"
+ "strings"
+ "time"
+
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/helper"
+)
+
+type PagingInfo struct {
+ Next int
+ Last int
+ First int
+ Prev int
+}
+
+type RateLimitInfo struct {
+ Date time.Time
+ ResetTime time.Time
+ Remaining int
+}
+
+type GiteeApiParams struct {
+ Repo string
+ Owner string
+ Token string
+}
+
+type GiteeInput struct {
+ Repo string
+ Owner string
+ Iid int
+}
+
+func GetTotalPagesFromResponse(res *http.Response, args *helper.ApiCollectorArgs) (int, error) {
+ total := res.Header.Get("X-Total-Pages")
+ if total == "" {
+ return 0, nil
+ }
+ totalInt, err := strconv.Atoi(total)
+ if err != nil {
+ return 0, err
+ }
+ return totalInt, nil
+}
+
+func GetRawMessageFromResponse(res *http.Response) ([]json.RawMessage, error) {
+ var rawMessages []json.RawMessage
+
+ if res == nil {
+ return nil, fmt.Errorf("res is nil")
+ }
+ defer res.Body.Close()
+ resBody, err := ioutil.ReadAll(res.Body)
+ if err != nil {
+ return nil, fmt.Errorf("%w %s", err, res.Request.URL.String())
+ }
+
+ err = json.Unmarshal(resBody, &rawMessages)
+ if err != nil {
+ return nil, fmt.Errorf("%w %s %s", err, res.Request.URL.String(), string(resBody))
+ }
+
+ return rawMessages, nil
+}
+
+func CreateRawDataSubTaskArgs(taskCtx core.SubTaskContext, Table string) (*helper.RawDataSubTaskArgs, *GiteeTaskData) {
+ data := taskCtx.GetData().(*GiteeTaskData)
+ RawDataSubTaskArgs := &helper.RawDataSubTaskArgs{
+ Ctx: taskCtx,
+ Params: GiteeApiParams{
+ Repo: data.Options.Repo,
+ Owner: data.Options.Owner,
+ Token: data.Options.Token,
+ },
+ Table: Table,
+ }
+ return RawDataSubTaskArgs, data
+}
+
+func ConvertRateLimitInfo(date string, resetTime string, remaining string) (RateLimitInfo, error) {
+ var rateLimitInfo RateLimitInfo
+ var err error
+ if date != "" {
+ rateLimitInfo.Date, err = http.ParseTime(date)
+ if err != nil {
+ return rateLimitInfo, err
+ }
+ } else {
+ return rateLimitInfo, errors.New("rate limit date was an empty string")
+ }
+ if resetTime != "" {
+ resetInt, err := strconv.ParseInt(resetTime, 10, 64)
+ if err != nil {
+ return rateLimitInfo, err
+ }
+ rateLimitInfo.ResetTime = time.Unix(resetInt, 0)
+ } else {
+ return rateLimitInfo, errors.New("rate limit reset time was an empty string")
+ }
+ if remaining != "" {
+ rateLimitInfo.Remaining, err = strconv.Atoi(remaining)
+ if err != nil {
+ return rateLimitInfo, err
+ }
+ } else {
+ return rateLimitInfo, errors.New("rate remaining was an empty string")
+ }
+ return rateLimitInfo, nil
+}
+
+func GetRateLimitPerSecond(info RateLimitInfo) int {
+ unixResetTime := info.ResetTime.Unix()
+ unixNow := info.Date.Unix()
+ timeBetweenNowAndReset := unixResetTime - unixNow
+ // Adjust the remaining to be less then actual to avoid hitting the limit exactly.
+ multiplier := 0.98
+ adjustedRemaining := float64(info.Remaining) * multiplier
+ return int(adjustedRemaining / float64(timeBetweenNowAndReset)) //* multiplier
+}
+func ConvertStringToInt(input string) (int, error) {
+ return strconv.Atoi(input)
+}
+func GetPagingFromLinkHeader(link string) (PagingInfo, error) {
+ result := PagingInfo{
+ Next: 1,
+ Last: 1,
+ Prev: 1,
+ First: 1,
+ }
+ linksArray := strings.Split(link, ",")
+ pattern1 := regexp.MustCompile(`page=*[0-9]+`)
+ pattern2 := regexp.MustCompile(`rel="*[a-z]+`)
+ if len(linksArray) >= 2 {
+ for i := 0; i < len(linksArray); i++ {
+ content := []byte(linksArray[i])
+ loc1 := pattern1.FindIndex(content)
+ loc2 := pattern2.FindIndex(content)
+ if len(loc1) >= 2 && len(loc2) >= 2 {
+ pageNumberSubstring := string(content[loc1[0]:loc1[1]])
+ pageNumberString := strings.Replace(pageNumberSubstring, `page=`, ``, 1)
+ pageNameSubstring := string(content[loc2[0]:loc2[1]])
+ pageNameString := strings.Replace(pageNameSubstring, `rel="`, ``, 1)
+
+ pageNumberInt, convertErr := ConvertStringToInt(pageNumberString)
+ if convertErr != nil {
+ return result, convertErr
+ }
+ switch pageNameString {
+ case "next":
+ result.Next = pageNumberInt
+
+ case "first":
+ result.First = pageNumberInt
+
+ case "last":
+ result.Last = pageNumberInt
+
+ case "prev":
+ result.Prev = pageNumberInt
+ }
+
+ } else {
+ return result, errors.New("parsed string values aren't long enough")
+ }
+ }
+ return result, nil
+ } else {
+ return result, errors.New("the link string provided is invalid. There is likely no next page of data to fetch")
+ }
+}
+
+func GetIssueIdByIssueUrl(s string) (int, error) {
+ regex := regexp.MustCompile(`.*/issues/(\d+)`)
+ groups := regex.FindStringSubmatch(s)
+ if len(groups) > 0 {
+ return strconv.Atoi(groups[1])
+ } else {
+ return 0, errors.New("invalid issue url")
+ }
+}
diff --git a/plugins/gitee/tasks/task_data.go b/plugins/gitee/tasks/task_data.go
new file mode 100644
index 00000000..76956641
--- /dev/null
+++ b/plugins/gitee/tasks/task_data.go
@@ -0,0 +1,41 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "time"
+
+ "github.com/apache/incubator-devlake/plugins/gitee/models"
+ "github.com/apache/incubator-devlake/plugins/helper"
+)
+
+type GiteeOptions struct {
+ Tasks []string `json:"tasks,omitempty"`
+ Since string
+ Owner string
+ Repo string
+ Token string
+ models.Config
+}
+
+type GiteeTaskData struct {
+ Options *GiteeOptions
+ ApiClient *helper.ApiAsyncClient
+ Repo *models.GiteeRepo
+ Since *time.Time
+}
diff --git a/plugins/gitee/tasks/user_convertor.go b/plugins/gitee/tasks/user_convertor.go
new file mode 100644
index 00000000..4c149f4e
--- /dev/null
+++ b/plugins/gitee/tasks/user_convertor.go
@@ -0,0 +1,72 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "reflect"
+
+ "github.com/apache/incubator-devlake/models/domainlayer"
+ "github.com/apache/incubator-devlake/models/domainlayer/didgen"
+ "github.com/apache/incubator-devlake/models/domainlayer/user"
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/gitee/models"
+ "github.com/apache/incubator-devlake/plugins/helper"
+)
+
+var ConvertUsersMeta = core.SubTaskMeta{
+ Name: "convertUsers",
+ EntryPoint: ConvertUsers,
+ EnabledByDefault: true,
+ Description: "Convert tool layer table gitee_users into domain layer table users",
+}
+
+func ConvertUsers(taskCtx core.SubTaskContext) error {
+ db := taskCtx.GetDb()
+ rawDataSubTaskArgs, _ := CreateRawDataSubTaskArgs(taskCtx, RAW_COMMIT_TABLE)
+
+ cursor, err := db.Model(&models.GiteeUser{}).
+ Rows()
+ if err != nil {
+ return err
+ }
+ defer cursor.Close()
+
+ userIdGen := didgen.NewDomainIdGenerator(&models.GiteeUser{})
+
+ converter, err := helper.NewDataConverter(helper.DataConverterArgs{
+ InputRowType: reflect.TypeOf(models.GiteeUser{}),
+ Input: cursor,
+ RawDataSubTaskArgs: *rawDataSubTaskArgs,
+ Convert: func(inputRow interface{}) ([]interface{}, error) {
+ GiteeUser := inputRow.(*models.GiteeUser)
+ domainUser := &user.User{
+ DomainEntity: domainlayer.DomainEntity{Id: userIdGen.Generate(GiteeUser.Id)},
+ Name: GiteeUser.Login,
+ AvatarUrl: GiteeUser.AvatarUrl,
+ }
+ return []interface{}{
+ domainUser,
+ }, nil
+ },
+ })
+ if err != nil {
+ return err
+ }
+
+ return converter.Execute()
+}