You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by ab...@apache.org on 2023/03/16 11:15:00 UTC
[incubator-devlake] branch main updated: feat: enhance timeFilter/diffSync support for jenkins and github plugin (#4674)
This is an automated email from the ASF dual-hosted git repository.
abeizn pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 916f6f5e8 feat: enhance timeFilter/diffSync support for jenkins and github plugin (#4674)
916f6f5e8 is described below
commit 916f6f5e8cb98ef29dafc86cb1b7ad5b7ffc1bf1
Author: abeizn <zi...@merico.dev>
AuthorDate: Thu Mar 16 19:14:55 2023 +0800
feat: enhance timeFilter/diffSync support for jenkins and github plugin (#4674)
* feat: enhance timeFilter/diffSync support for jenkins and github plugin
* feat: enhance timeFilter/diffSync support for jenkins and github plugin and format
* feat: enhance timeFilter/diffSync support for jenkins and github plugin and format
* feat: enhance github events timeFilter/diffSync support
---------
Co-authored-by: Klesh Wong <zh...@merico.dev>
---
backend/plugins/github/tasks/account_collector.go | 9 +-
.../plugins/github/tasks/account_org_collector.go | 2 +-
backend/plugins/github/tasks/cicd_job_collector.go | 2 +-
backend/plugins/github/tasks/cicd_run_collector.go | 205 ++++++++-------------
backend/plugins/github/tasks/comment_collector.go | 2 +-
backend/plugins/github/tasks/commit_collector.go | 2 +-
.../plugins/github/tasks/commit_stats_collector.go | 11 +-
backend/plugins/github/tasks/event_collector.go | 91 ++++++---
backend/plugins/github/tasks/issue_collector.go | 2 +-
.../plugins/github/tasks/milestone_collector.go | 7 +-
backend/plugins/github/tasks/pr_collector.go | 2 +-
.../plugins/github/tasks/pr_commit_collector.go | 2 +-
.../plugins/github/tasks/pr_review_collector.go | 2 +-
.../github/tasks/pr_review_comment_collector.go | 2 +-
.../github_graphql/tasks/account_collector.go | 5 +-
.../github_graphql/tasks/issue_collector.go | 2 +-
.../plugins/github_graphql/tasks/job_collector.go | 2 +-
.../plugins/github_graphql/tasks/pr_collector.go | 2 +-
backend/plugins/jenkins/tasks/build_collector.go | 7 +-
backend/plugins/jenkins/tasks/stage_collector.go | 2 +-
20 files changed, 181 insertions(+), 180 deletions(-)
diff --git a/backend/plugins/github/tasks/account_collector.go b/backend/plugins/github/tasks/account_collector.go
index 3195adc89..2aa8beada 100644
--- a/backend/plugins/github/tasks/account_collector.go
+++ b/backend/plugins/github/tasks/account_collector.go
@@ -19,14 +19,15 @@ package tasks
import (
"encoding/json"
+ "io"
+ "net/http"
+ "reflect"
+
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/github/models"
- "io"
- "net/http"
- "reflect"
)
const RAW_ACCOUNT_TABLE = "github_api_accounts"
@@ -89,6 +90,6 @@ var CollectAccountsMeta = plugin.SubTaskMeta{
Name: "collectAccounts",
EntryPoint: CollectAccounts,
EnabledByDefault: true,
- Description: "Collect accounts data from Github api",
+ Description: "Collect accounts data from Github api, does not support either timeFilter or diffSync.",
DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS},
}
diff --git a/backend/plugins/github/tasks/account_org_collector.go b/backend/plugins/github/tasks/account_org_collector.go
index af6b38a44..ae8c569ad 100644
--- a/backend/plugins/github/tasks/account_org_collector.go
+++ b/backend/plugins/github/tasks/account_org_collector.go
@@ -90,6 +90,6 @@ var CollectAccountOrgMeta = plugin.SubTaskMeta{
Name: "collectAccountOrg",
EntryPoint: CollectAccountOrg,
EnabledByDefault: true,
- Description: "Collect accounts org data from Github api",
+ Description: "Collect accounts org data from Github api, does not support either timeFilter or diffSync.",
DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS},
}
diff --git a/backend/plugins/github/tasks/cicd_job_collector.go b/backend/plugins/github/tasks/cicd_job_collector.go
index 5db40eebc..4dd95c5c5 100644
--- a/backend/plugins/github/tasks/cicd_job_collector.go
+++ b/backend/plugins/github/tasks/cicd_job_collector.go
@@ -37,7 +37,7 @@ var CollectJobsMeta = plugin.SubTaskMeta{
Name: "collectJobs",
EntryPoint: CollectJobs,
EnabledByDefault: true,
- Description: "Collect Jobs data from Github action api",
+ Description: "Collect Jobs data from Github action api, supports both timeFilter and diffSync.",
DomainTypes: []string{plugin.DOMAIN_TYPE_CICD},
}
diff --git a/backend/plugins/github/tasks/cicd_run_collector.go b/backend/plugins/github/tasks/cicd_run_collector.go
index 598967a5e..3af4a66ce 100644
--- a/backend/plugins/github/tasks/cicd_run_collector.go
+++ b/backend/plugins/github/tasks/cicd_run_collector.go
@@ -40,132 +40,28 @@ const RAW_RUN_TABLE = "github_api_runs"
// https://github.com/apache/incubator-devlake/issues/3199
const PAGE_SIZE = 30
+type GithubRawRunsResult struct {
+ TotalCount int64 `json:"total_count"`
+ GithubWorkflowRuns []json.RawMessage `json:"workflow_runs"`
+}
+
+type SimpleGithubApiJob struct {
+ ID int64
+ CreatedAt helper.Iso8601Time `json:"created_at"`
+}
+
var CollectRunsMeta = plugin.SubTaskMeta{
Name: "collectRuns",
EntryPoint: CollectRuns,
EnabledByDefault: true,
- Description: "Collect Runs data from Github action api",
+ Description: "Collect Runs data from Github action api, supports both timeFilter and diffSync.",
DomainTypes: []string{plugin.DOMAIN_TYPE_CICD},
}
func CollectRuns(taskCtx plugin.SubTaskContext) errors.Error {
- data := taskCtx.GetData().(*GithubTaskData)
- collectorWithState, err := helper.NewStatefulApiCollector(helper.RawDataSubTaskArgs{
- Ctx: taskCtx,
- Params: GithubApiParams{
- ConnectionId: data.Options.ConnectionId,
- Name: data.Options.Name,
- },
- Table: RAW_RUN_TABLE,
- }, data.TimeAfter)
- if err != nil {
- return err
- }
-
- incremental := collectorWithState.IsIncremental()
-
- // step 1: fetch records created after createdAfter
- var createdAfter *time.Time
- if incremental {
- createdAfter = collectorWithState.LatestState.LatestSuccessStart
- } else {
- createdAfter = data.TimeAfter
- }
-
- err = collectorWithState.InitCollector(helper.ApiCollectorArgs{
- ApiClient: data.ApiClient,
- PageSize: PAGE_SIZE,
- Incremental: incremental,
- UrlTemplate: "repos/{{ .Params.Name }}/actions/runs",
- Query: func(reqData *helper.RequestData) (url.Values, errors.Error) {
- query := url.Values{}
- query.Set("page", fmt.Sprintf("%v", reqData.Pager.Page))
- query.Set("per_page", fmt.Sprintf("%v", reqData.Pager.Size))
- return query, nil
- },
- // use Undetermined strategy so we can stop fetching further pages by using
- // ErrFinishCollect
- Concurrency: 10,
- ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) {
- body := &GithubRawRunsResult{}
- err := helper.UnmarshalResponse(res, body)
- if err != nil {
- return nil, err
- }
-
- if len(body.GithubWorkflowRuns) == 0 {
- return nil, nil
- }
-
- // time filter or diff sync
- if createdAfter != nil {
- // if the first record of the page was created before minCreated, return emtpy set and stop
- firstRun := &models.GithubRun{}
- if e := json.Unmarshal(body.GithubWorkflowRuns[0], firstRun); e != nil {
- return nil, errors.Default.Wrap(e, "failed to unmarshal first run")
- }
- if firstRun.GithubCreatedAt.Before(*createdAfter) {
- return nil, helper.ErrFinishCollect
- }
- // if the last record was created before minCreated, return records and stop
- lastRun := &models.GithubRun{}
- if e := json.Unmarshal(body.GithubWorkflowRuns[len(body.GithubWorkflowRuns)-1], lastRun); e != nil {
- return nil, errors.Default.Wrap(e, "failed to unmarshal last run")
- }
- if lastRun.GithubCreatedAt.Before(*createdAfter) {
- err = helper.ErrFinishCollect
- }
- }
-
- return body.GithubWorkflowRuns, err
- },
- })
-
- if err != nil {
- return err
- }
-
- err = collectorWithState.Execute()
- if err != nil {
- return err
- }
-
- // step 2: for incremental collection, we have to update previous collected data which status is unfinished
- if incremental {
- // update existing data by collecting unfinished runs prior to LatestState.LatestSuccessStart
- return collectUnfinishedRuns(taskCtx)
- }
- return nil
-}
-
-type GithubRawRunsResult struct {
- TotalCount int64 `json:"total_count"`
- GithubWorkflowRuns []json.RawMessage `json:"workflow_runs"`
-}
-
-func collectUnfinishedRuns(taskCtx plugin.SubTaskContext) errors.Error {
data := taskCtx.GetData().(*GithubTaskData)
db := taskCtx.GetDal()
-
- // load unfinished runs from the database
- cursor, err := db.Cursor(
- dal.Select("id"),
- dal.From(&models.GithubRun{}),
- dal.Where(
- "repo_id = ? AND connection_id = ? AND status IN ('ACTION_REQUIRED', 'STALE', 'IN_PROGRESS', 'QUEUED', 'REQUESTED', 'WAITING', 'PENDING')",
- data.Options.GithubId, data.Options.ConnectionId,
- ),
- )
- if err != nil {
- return err
- }
- iterator, err := helper.NewDalCursorIterator(db, cursor, reflect.TypeOf(SimpleGithubRun{}))
- if err != nil {
- return err
- }
-
- // collect details from api
- collector, err := helper.NewApiCollector(helper.ApiCollectorArgs{
+ collector, err := helper.NewStatefulApiCollectorForFinalizableEntity(helper.FinalizableApiCollectorArgs{
RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
Ctx: taskCtx,
Params: GithubApiParams{
@@ -174,23 +70,76 @@ func collectUnfinishedRuns(taskCtx plugin.SubTaskContext) errors.Error {
},
Table: RAW_RUN_TABLE,
},
- ApiClient: data.ApiClient,
- Input: iterator,
- Incremental: true,
- UrlTemplate: "repos/{{ .Params.Name }}/actions/runs/{{ .Input.ID }}",
- ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) {
- body, err := io.ReadAll(res.Body)
- if err != nil {
- return nil, errors.Convert(err)
- }
- res.Body.Close()
- return []json.RawMessage{body}, nil
+ ApiClient: data.ApiClient,
+ TimeAfter: data.TimeAfter,
+ CollectNewRecordsByList: helper.FinalizableApiCollectorListArgs{
+ PageSize: PAGE_SIZE,
+ Concurrency: 10,
+ FinalizableApiCollectorCommonArgs: helper.FinalizableApiCollectorCommonArgs{
+ UrlTemplate: "repos/{{ .Params.Name }}/actions/runs",
+ Query: func(reqData *helper.RequestData, createdAfter *time.Time) (url.Values, errors.Error) {
+ query := url.Values{}
+ query.Set("page", fmt.Sprintf("%v", reqData.Pager.Page))
+ query.Set("per_page", fmt.Sprintf("%v", reqData.Pager.Size))
+ return query, nil
+ },
+ ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) {
+ body := &GithubRawRunsResult{}
+ err := helper.UnmarshalResponse(res, body)
+ if err != nil {
+ return nil, err
+ }
+ if len(body.GithubWorkflowRuns) == 0 {
+ return nil, nil
+ }
+ return body.GithubWorkflowRuns, nil
+ },
+ },
+ GetCreated: func(item json.RawMessage) (time.Time, errors.Error) {
+ pj := &SimpleGithubApiJob{}
+ err := json.Unmarshal(item, pj)
+ if err != nil {
+ return time.Time{}, errors.BadInput.Wrap(err, "failed to unmarshal github run")
+ }
+ return pj.CreatedAt.ToTime(), nil
+ },
+ },
+ CollectUnfinishedDetails: helper.FinalizableApiCollectorDetailArgs{
+ BuildInputIterator: func() (helper.Iterator, errors.Error) {
+ // load unfinished runs from the database
+ cursor, err := db.Cursor(
+ dal.Select("id"),
+ dal.From(&models.GithubRun{}),
+ dal.Where(
+ "repo_id = ? AND connection_id = ? AND status IN ('ACTION_REQUIRED', 'STALE', 'IN_PROGRESS', 'QUEUED', 'REQUESTED', 'WAITING', 'PENDING')",
+ data.Options.GithubId, data.Options.ConnectionId,
+ ),
+ )
+ if err != nil {
+ return nil, err
+ }
+ return helper.NewDalCursorIterator(db, cursor, reflect.TypeOf(SimpleGithubApiJob{}))
+ },
+
+ FinalizableApiCollectorCommonArgs: helper.FinalizableApiCollectorCommonArgs{
+ UrlTemplate: "repos/{{ .Params.Name }}/actions/runs/{{ .Input.ID }}",
+ ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) {
+ body, err := io.ReadAll(res.Body)
+ if err != nil {
+ return nil, errors.Convert(err)
+ }
+ res.Body.Close()
+ return []json.RawMessage{body}, nil
+ },
+ AfterResponse: ignoreHTTPStatus404,
+ },
},
- AfterResponse: ignoreHTTPStatus404,
})
if err != nil {
return err
}
+
return collector.Execute()
+
}
diff --git a/backend/plugins/github/tasks/comment_collector.go b/backend/plugins/github/tasks/comment_collector.go
index f57c733a1..709a7dc35 100644
--- a/backend/plugins/github/tasks/comment_collector.go
+++ b/backend/plugins/github/tasks/comment_collector.go
@@ -92,6 +92,6 @@ var CollectApiCommentsMeta = plugin.SubTaskMeta{
Name: "collectApiComments",
EntryPoint: CollectApiComments,
EnabledByDefault: true,
- Description: "Collect comments data from Github api",
+ Description: "Collect comments data from Github api, supports both timeFilter and diffSync.",
DomainTypes: []string{plugin.DOMAIN_TYPE_CODE_REVIEW, plugin.DOMAIN_TYPE_TICKET},
}
diff --git a/backend/plugins/github/tasks/commit_collector.go b/backend/plugins/github/tasks/commit_collector.go
index 3136a4c98..35f7e95da 100644
--- a/backend/plugins/github/tasks/commit_collector.go
+++ b/backend/plugins/github/tasks/commit_collector.go
@@ -34,7 +34,7 @@ var CollectApiCommitsMeta = plugin.SubTaskMeta{
Name: "collectApiCommits",
EntryPoint: CollectApiCommits,
EnabledByDefault: false,
- Description: "Collect commits data from Github api",
+ Description: "Collect commits data from Github api, supports both timeFilter and diffSync.",
DomainTypes: []string{plugin.DOMAIN_TYPE_CODE},
}
diff --git a/backend/plugins/github/tasks/commit_stats_collector.go b/backend/plugins/github/tasks/commit_stats_collector.go
index 577880507..6d2cbc410 100644
--- a/backend/plugins/github/tasks/commit_stats_collector.go
+++ b/backend/plugins/github/tasks/commit_stats_collector.go
@@ -20,15 +20,16 @@ package tasks
import (
"encoding/json"
"fmt"
+ "io"
+ "net/http"
+ "net/url"
+ "reflect"
+
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/github/models"
- "io"
- "net/http"
- "net/url"
- "reflect"
)
const RAW_COMMIT_STATS_TABLE = "github_api_commit_stats"
@@ -37,7 +38,7 @@ var CollectApiCommitStatsMeta = plugin.SubTaskMeta{
Name: "collectApiCommitStats",
EntryPoint: CollectApiCommitStats,
EnabledByDefault: false,
- Description: "Collect commitStats data from Github api",
+ Description: "Collect commitStats data from Github api, does not support either timeFilter or diffSync.",
DomainTypes: []string{plugin.DOMAIN_TYPE_CODE},
}
diff --git a/backend/plugins/github/tasks/event_collector.go b/backend/plugins/github/tasks/event_collector.go
index 574cff1ab..f1f6831c5 100644
--- a/backend/plugins/github/tasks/event_collector.go
+++ b/backend/plugins/github/tasks/event_collector.go
@@ -20,30 +20,39 @@ package tasks
import (
"encoding/json"
"fmt"
+ "io"
"net/http"
"net/url"
+ "reflect"
+ "time"
+ "github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "github.com/apache/incubator-devlake/plugins/github/models"
)
const RAW_EVENTS_TABLE = "github_api_events"
-// this struct should be moved to `gitub_api_common.go`
+type SimpleGithubApiEvents struct {
+ GithubId int64
+ CreatedAt helper.Iso8601Time `json:"created_at"`
+}
var CollectApiEventsMeta = plugin.SubTaskMeta{
Name: "collectApiEvents",
EntryPoint: CollectApiEvents,
EnabledByDefault: true,
- Description: "Collect Events data from Github api",
+ Description: "Collect Events data from Github api, supports both timeFilter and diffSync.",
DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET},
}
func CollectApiEvents(taskCtx plugin.SubTaskContext) errors.Error {
data := taskCtx.GetData().(*GithubTaskData)
+ db := taskCtx.GetDal()
- collector, err := helper.NewApiCollector(helper.ApiCollectorArgs{
+ collector, err := helper.NewStatefulApiCollectorForFinalizableEntity(helper.FinalizableApiCollectorArgs{
RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
Ctx: taskCtx,
Params: GithubApiParams{
@@ -52,26 +61,64 @@ func CollectApiEvents(taskCtx plugin.SubTaskContext) errors.Error {
},
Table: RAW_EVENTS_TABLE,
},
- ApiClient: data.ApiClient,
- PageSize: 100,
- Incremental: false,
-
- UrlTemplate: "repos/{{ .Params.Name }}/issues/events",
- Query: func(reqData *helper.RequestData) (url.Values, errors.Error) {
- query := url.Values{}
- query.Set("page", fmt.Sprintf("%v", reqData.Pager.Page))
- query.Set("per_page", fmt.Sprintf("%v", reqData.Pager.Size))
-
- return query, nil
+ ApiClient: data.ApiClient,
+ TimeAfter: data.TimeAfter, // set to nil to disable timeFilter
+ CollectNewRecordsByList: helper.FinalizableApiCollectorListArgs{
+ PageSize: 100,
+ Concurrency: 10,
+ //GetTotalPages: GetTotalPagesFromResponse,
+ FinalizableApiCollectorCommonArgs: helper.FinalizableApiCollectorCommonArgs{
+ UrlTemplate: "repos/{{ .Params.Name }}/issues/events",
+ Query: func(reqData *helper.RequestData, createdAfter *time.Time) (url.Values, errors.Error) {
+ query := url.Values{}
+ query.Set("page", fmt.Sprintf("%v", reqData.Pager.Page))
+ query.Set("per_page", fmt.Sprintf("%v", reqData.Pager.Size))
+ return query, nil
+ },
+ ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) {
+ var items []json.RawMessage
+ err := helper.UnmarshalResponse(res, &items)
+ if err != nil {
+ return nil, err
+ }
+ return items, nil
+ },
+ },
+ GetCreated: func(item json.RawMessage) (time.Time, errors.Error) {
+ e := &SimpleGithubApiEvents{}
+ err := json.Unmarshal(item, e)
+ if err != nil {
+ return time.Time{}, errors.BadInput.Wrap(err, "failed to unmarshal github events")
+ }
+ return e.CreatedAt.ToTime(), nil
+ },
},
- GetTotalPages: GetTotalPagesFromResponse,
- ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) {
- var items []json.RawMessage
- err := helper.UnmarshalResponse(res, &items)
- if err != nil {
- return nil, err
- }
- return items, nil
+ CollectUnfinishedDetails: helper.FinalizableApiCollectorDetailArgs{
+ BuildInputIterator: func() (helper.Iterator, errors.Error) {
+ cursor, err := db.Cursor(
+ dal.Select("github_id"),
+ dal.From(&models.GithubIssueEvent{}),
+ dal.Where(
+ "github_id = ? AND connection_id = ? and type NOT IN ('closed', 'merged')",
+ data.Options.GithubId, data.Options.ConnectionId,
+ ),
+ )
+ if err != nil {
+ return nil, err
+ }
+ return helper.NewDalCursorIterator(db, cursor, reflect.TypeOf(SimpleGithubApiEvents{}))
+ },
+ FinalizableApiCollectorCommonArgs: helper.FinalizableApiCollectorCommonArgs{
+ UrlTemplate: "repos/{{ .Params.Name }}/issues/events/{{ .Input.GithubId }}",
+ ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) {
+ body, err := io.ReadAll(res.Body)
+ if err != nil {
+ return nil, errors.Convert(err)
+ }
+ res.Body.Close()
+ return []json.RawMessage{body}, nil
+ },
+ },
},
})
if err != nil {
diff --git a/backend/plugins/github/tasks/issue_collector.go b/backend/plugins/github/tasks/issue_collector.go
index 89e07fc2d..e646f7e08 100644
--- a/backend/plugins/github/tasks/issue_collector.go
+++ b/backend/plugins/github/tasks/issue_collector.go
@@ -34,7 +34,7 @@ var CollectApiIssuesMeta = plugin.SubTaskMeta{
Name: "collectApiIssues",
EntryPoint: CollectApiIssues,
EnabledByDefault: true,
- Description: "Collect issues data from Github api",
+ Description: "Collect issues data from Github api, supports both timeFilter and diffSync.",
DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET},
}
diff --git a/backend/plugins/github/tasks/milestone_collector.go b/backend/plugins/github/tasks/milestone_collector.go
index 2eef4777c..51d4d3439 100644
--- a/backend/plugins/github/tasks/milestone_collector.go
+++ b/backend/plugins/github/tasks/milestone_collector.go
@@ -20,11 +20,12 @@ package tasks
import (
"encoding/json"
"fmt"
+ "net/http"
+ "net/url"
+
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
- "net/http"
- "net/url"
)
const RAW_MILESTONE_TABLE = "github_milestones"
@@ -33,7 +34,7 @@ var CollectMilestonesMeta = plugin.SubTaskMeta{
Name: "collectApiMilestones",
EntryPoint: CollectApiMilestones,
EnabledByDefault: true,
- Description: "Collect milestone data from Github api",
+ Description: "Collect milestone data from Github api, does not support either timeFilter or diffSync.",
DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET},
}
diff --git a/backend/plugins/github/tasks/pr_collector.go b/backend/plugins/github/tasks/pr_collector.go
index d22a47d77..d15bf3db1 100644
--- a/backend/plugins/github/tasks/pr_collector.go
+++ b/backend/plugins/github/tasks/pr_collector.go
@@ -39,7 +39,7 @@ var CollectApiPullRequestsMeta = plugin.SubTaskMeta{
Name: "collectApiPullRequests",
EntryPoint: CollectApiPullRequests,
EnabledByDefault: true,
- Description: "Collect PullRequests data from Github api",
+ Description: "Collect PullRequests data from Github api, supports both timeFilter and diffSync.",
DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS, plugin.DOMAIN_TYPE_CODE_REVIEW},
}
diff --git a/backend/plugins/github/tasks/pr_commit_collector.go b/backend/plugins/github/tasks/pr_commit_collector.go
index 0c876936a..2ee08cf76 100644
--- a/backend/plugins/github/tasks/pr_commit_collector.go
+++ b/backend/plugins/github/tasks/pr_commit_collector.go
@@ -42,7 +42,7 @@ var CollectApiPullRequestCommitsMeta = plugin.SubTaskMeta{
Name: "collectApiPullRequestCommits",
EntryPoint: CollectApiPullRequestCommits,
EnabledByDefault: true,
- Description: "Collect PullRequestCommits data from Github api",
+ Description: "Collect PullRequestCommits data from Github api, supports both timeFilter and diffSync.",
DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS, plugin.DOMAIN_TYPE_CODE_REVIEW},
}
diff --git a/backend/plugins/github/tasks/pr_review_collector.go b/backend/plugins/github/tasks/pr_review_collector.go
index e5928cf9f..888186cea 100644
--- a/backend/plugins/github/tasks/pr_review_collector.go
+++ b/backend/plugins/github/tasks/pr_review_collector.go
@@ -39,7 +39,7 @@ var CollectApiPullRequestReviewsMeta = plugin.SubTaskMeta{
Name: "collectApiPullRequestReviews",
EntryPoint: CollectApiPullRequestReviews,
EnabledByDefault: true,
- Description: "Collect PullRequestReviews data from Github api",
+ Description: "Collect PullRequestReviews data from Github api, supports both timeFilter and diffSync.",
DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS, plugin.DOMAIN_TYPE_CODE_REVIEW},
}
diff --git a/backend/plugins/github/tasks/pr_review_comment_collector.go b/backend/plugins/github/tasks/pr_review_comment_collector.go
index e19349761..2468b1b93 100644
--- a/backend/plugins/github/tasks/pr_review_comment_collector.go
+++ b/backend/plugins/github/tasks/pr_review_comment_collector.go
@@ -93,6 +93,6 @@ var CollectApiPrReviewCommentsMeta = plugin.SubTaskMeta{
Name: "collectApiPrReviewCommentsMeta",
EntryPoint: CollectPrReviewComments,
EnabledByDefault: true,
- Description: "Collect pr review comments data from Github api",
+ Description: "Collect pr review comments data from Github api, supports both timeFilter and diffSync.",
DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS, plugin.DOMAIN_TYPE_CODE_REVIEW},
}
diff --git a/backend/plugins/github_graphql/tasks/account_collector.go b/backend/plugins/github_graphql/tasks/account_collector.go
index 6e98d0dd3..c7b2924a6 100644
--- a/backend/plugins/github_graphql/tasks/account_collector.go
+++ b/backend/plugins/github_graphql/tasks/account_collector.go
@@ -18,6 +18,8 @@ limitations under the License.
package tasks
import (
+ "reflect"
+
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
@@ -25,7 +27,6 @@ import (
"github.com/apache/incubator-devlake/plugins/github/models"
githubTasks "github.com/apache/incubator-devlake/plugins/github/tasks"
"github.com/merico-dev/graphql"
- "reflect"
)
const RAW_ACCOUNTS_TABLE = "github_graphql_accounts"
@@ -60,7 +61,7 @@ var CollectAccountMeta = plugin.SubTaskMeta{
Name: "CollectAccount",
EntryPoint: CollectAccount,
EnabledByDefault: true,
- Description: "Collect Account data from GithubGraphql api",
+ Description: "Collect Account data from GithubGraphql api, does not support either timeFilter or diffSync.",
DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS},
}
diff --git a/backend/plugins/github_graphql/tasks/issue_collector.go b/backend/plugins/github_graphql/tasks/issue_collector.go
index 5cddf35d3..53aafe66d 100644
--- a/backend/plugins/github_graphql/tasks/issue_collector.go
+++ b/backend/plugins/github_graphql/tasks/issue_collector.go
@@ -77,7 +77,7 @@ var CollectIssueMeta = plugin.SubTaskMeta{
Name: "CollectIssue",
EntryPoint: CollectIssue,
EnabledByDefault: true,
- Description: "Collect Issue data from GithubGraphql api",
+ Description: "Collect Issue data from GithubGraphql api, supports both timeFilter and diffSync.",
DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET},
}
diff --git a/backend/plugins/github_graphql/tasks/job_collector.go b/backend/plugins/github_graphql/tasks/job_collector.go
index 8d3a1d285..7c992f4d6 100644
--- a/backend/plugins/github_graphql/tasks/job_collector.go
+++ b/backend/plugins/github_graphql/tasks/job_collector.go
@@ -92,7 +92,7 @@ var CollectGraphqlJobsMeta = plugin.SubTaskMeta{
Name: "CollectGraphqlJobs",
EntryPoint: CollectGraphqlJobs,
EnabledByDefault: true,
- Description: "Collect Jobs(CheckRun) data from GithubGraphql api",
+ Description: "Collect Jobs(CheckRun) data from GithubGraphql api, supports both timeFilter and diffSync.",
DomainTypes: []string{plugin.DOMAIN_TYPE_CICD},
}
diff --git a/backend/plugins/github_graphql/tasks/pr_collector.go b/backend/plugins/github_graphql/tasks/pr_collector.go
index c9e605b2a..510bde4e9 100644
--- a/backend/plugins/github_graphql/tasks/pr_collector.go
+++ b/backend/plugins/github_graphql/tasks/pr_collector.go
@@ -123,7 +123,7 @@ var CollectPrMeta = plugin.SubTaskMeta{
Name: "CollectPr",
EntryPoint: CollectPr,
EnabledByDefault: true,
- Description: "Collect Pr data from GithubGraphql api",
+ Description: "Collect Pr data from GithubGraphql api, supports both timeFilter and diffSync.",
DomainTypes: []string{plugin.DOMAIN_TYPE_CODE_REVIEW},
}
diff --git a/backend/plugins/jenkins/tasks/build_collector.go b/backend/plugins/jenkins/tasks/build_collector.go
index 241b02b10..3a64b0086 100644
--- a/backend/plugins/jenkins/tasks/build_collector.go
+++ b/backend/plugins/jenkins/tasks/build_collector.go
@@ -20,11 +20,12 @@ package tasks
import (
"encoding/json"
"fmt"
+ "net/http"
+ "net/url"
+
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
- "net/http"
- "net/url"
)
const RAW_BUILD_TABLE = "jenkins_api_builds"
@@ -33,7 +34,7 @@ var CollectApiBuildsMeta = plugin.SubTaskMeta{
Name: "collectApiBuilds",
EntryPoint: CollectApiBuilds,
EnabledByDefault: true,
- Description: "Collect builds data from jenkins api",
+ Description: "Collect builds data from jenkins api, supports both timeFilter and diffSync.",
DomainTypes: []string{plugin.DOMAIN_TYPE_CICD},
}
diff --git a/backend/plugins/jenkins/tasks/stage_collector.go b/backend/plugins/jenkins/tasks/stage_collector.go
index 41c926352..e6c4d1b30 100644
--- a/backend/plugins/jenkins/tasks/stage_collector.go
+++ b/backend/plugins/jenkins/tasks/stage_collector.go
@@ -36,7 +36,7 @@ var CollectApiStagesMeta = plugin.SubTaskMeta{
Name: "collectApiStages",
EntryPoint: CollectApiStages,
EnabledByDefault: true,
- Description: "Collect stages data from jenkins api",
+ Description: "Collect stages data from jenkins api, supports timeFilter but not diffSync.",
DomainTypes: []string{plugin.DOMAIN_TYPE_CICD},
}