You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by ab...@apache.org on 2023/12/08 06:50:28 UTC
(incubator-devlake) branch main updated: fix: github grapahql increment collector (#6601)
This is an automated email from the ASF dual-hosted git repository.
abeizn pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new a438debb7 fix: github grapahql increment collector (#6601)
a438debb7 is described below
commit a438debb70f640bc0be3ac5f3587897844571fde
Author: abeizn <zi...@merico.dev>
AuthorDate: Fri Dec 8 14:50:24 2023 +0800
fix: github grapahql increment collector (#6601)
* fix: github grapahql increment collector
* fix: issue updated_at filter
---
.../helpers/pluginhelper/api/api_collector_with_state.go | 1 +
.../plugins/github_graphql/tasks/deployment_collector.go | 2 +-
backend/plugins/github_graphql/tasks/issue_collector.go | 16 +++++++++-------
backend/plugins/github_graphql/tasks/pr_collector.go | 16 ++++------------
4 files changed, 15 insertions(+), 20 deletions(-)
diff --git a/backend/helpers/pluginhelper/api/api_collector_with_state.go b/backend/helpers/pluginhelper/api/api_collector_with_state.go
index 51c9d7ec3..47f653ccd 100644
--- a/backend/helpers/pluginhelper/api/api_collector_with_state.go
+++ b/backend/helpers/pluginhelper/api/api_collector_with_state.go
@@ -156,6 +156,7 @@ func (m *ApiCollectorStateManager) InitCollector(args ApiCollectorArgs) errors.E
// InitGraphQLCollector init the embedded collector
func (m *ApiCollectorStateManager) InitGraphQLCollector(args GraphqlCollectorArgs) errors.Error {
args.RawDataSubTaskArgs = m.RawDataSubTaskArgs
+ args.Incremental = args.Incremental || m.IsIncremental
graphqlCollector, err := NewGraphqlCollector(args)
if err != nil {
return err
diff --git a/backend/plugins/github_graphql/tasks/deployment_collector.go b/backend/plugins/github_graphql/tasks/deployment_collector.go
index 7ad553dd4..6de272f4f 100644
--- a/backend/plugins/github_graphql/tasks/deployment_collector.go
+++ b/backend/plugins/github_graphql/tasks/deployment_collector.go
@@ -125,7 +125,7 @@ func CollectDeployments(taskCtx plugin.SubTaskContext) errors.Error {
deployments := query.Repository.Deployments.Deployments
for _, rawL := range deployments {
if collectorWithState.Since != nil && !collectorWithState.Since.Before(rawL.UpdatedAt) {
- break
+ return nil, helper.ErrFinishCollect
}
}
return nil, nil
diff --git a/backend/plugins/github_graphql/tasks/issue_collector.go b/backend/plugins/github_graphql/tasks/issue_collector.go
index b73f3a5a3..5bd27267f 100644
--- a/backend/plugins/github_graphql/tasks/issue_collector.go
+++ b/backend/plugins/github_graphql/tasks/issue_collector.go
@@ -39,7 +39,7 @@ type GraphqlQueryIssueWrapper struct {
TotalCount graphql.Int
Issues []GraphqlQueryIssue `graphql:"nodes"`
PageInfo *helper.GraphqlQueryPageInfo
- } `graphql:"issues(first: $pageSize, after: $skipCursor, orderBy: {field: UPDATED_AT, direction: DESC}, filterBy: {since: $since})"`
+ } `graphql:"issues(first: $pageSize, after: $skipCursor, orderBy: {field: UPDATED_AT, direction: DESC})"`
} `graphql:"repository(owner: $owner, name: $name)"`
}
@@ -96,19 +96,14 @@ func CollectIssues(taskCtx plugin.SubTaskContext) errors.Error {
err = collectorWithState.InitGraphQLCollector(helper.GraphqlCollectorArgs{
GraphqlClient: data.GraphqlClient,
- PageSize: 100,
+ PageSize: 10,
BuildQuery: func(reqData *helper.GraphqlRequestData) (interface{}, map[string]interface{}, error) {
query := &GraphqlQueryIssueWrapper{}
if reqData == nil {
return query, map[string]interface{}{}, nil
}
- since := helper.DateTime{}
- if collectorWithState.Since != nil {
- since = helper.DateTime{Time: *collectorWithState.Since}
- }
ownerName := strings.Split(data.Options.Name, "/")
variables := map[string]interface{}{
- "since": since,
"pageSize": graphql.Int(reqData.Pager.Size),
"skipCursor": (*graphql.String)(reqData.Pager.SkipCursor),
"owner": graphql.String(ownerName[0]),
@@ -121,6 +116,13 @@ func CollectIssues(taskCtx plugin.SubTaskContext) errors.Error {
return query.Repository.IssueList.PageInfo, nil
},
ResponseParser: func(iQuery interface{}, variables map[string]interface{}) ([]interface{}, error) {
+ query := iQuery.(*GraphqlQueryIssueWrapper)
+ issues := query.Repository.IssueList.Issues
+ for _, rawL := range issues {
+ if collectorWithState.Since != nil && !collectorWithState.Since.Before(rawL.UpdatedAt) {
+ return nil, helper.ErrFinishCollect
+ }
+ }
return nil, nil
},
})
diff --git a/backend/plugins/github_graphql/tasks/pr_collector.go b/backend/plugins/github_graphql/tasks/pr_collector.go
index 152ea27b2..66e007a8b 100644
--- a/backend/plugins/github_graphql/tasks/pr_collector.go
+++ b/backend/plugins/github_graphql/tasks/pr_collector.go
@@ -43,7 +43,7 @@ type GraphqlQueryPrWrapper struct {
PageInfo *api.GraphqlQueryPageInfo
Prs []GraphqlQueryPr `graphql:"nodes"`
TotalCount graphql.Int
- } `graphql:"pullRequests(first: $pageSize, after: $skipCursor, orderBy: {field: UPDATED_AT, direction: DESC})"`
+ } `graphql:"pullRequests(first: $pageSize, after: $skipCursor, orderBy: {field: CREATED_AT, direction: DESC})"`
} `graphql:"repository(owner: $owner, name: $name)"`
}
@@ -169,20 +169,12 @@ func CollectPrs(taskCtx plugin.SubTaskContext) errors.Error {
ResponseParser: func(iQuery interface{}, variables map[string]interface{}) ([]interface{}, error) {
query := iQuery.(*GraphqlQueryPrWrapper)
prs := query.Repository.PullRequests.Prs
-
- isFinish := false
for _, rawL := range prs {
- // collect data even though in increment mode because of updating existing data
- if collectorWithState.Since != nil && !collectorWithState.Since.Before(rawL.UpdatedAt) {
- isFinish = true
- break
+ if collectorWithState.Since != nil && !collectorWithState.Since.Before(rawL.CreatedAt) {
+ return nil, api.ErrFinishCollect
}
}
- if isFinish {
- return nil, api.ErrFinishCollect
- } else {
- return nil, nil
- }
+ return nil, nil
},
})
if err != nil {