You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by ab...@apache.org on 2023/04/13 13:20:12 UTC
[incubator-devlake] branch release-v0.16 updated: fix: move extract existing graphql data to general position (#4921)
This is an automated email from the ASF dual-hosted git repository.
abeizn pushed a commit to branch release-v0.16
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/release-v0.16 by this push:
new 382a08e53 fix: move extract existing graphql data to general position (#4921)
382a08e53 is described below
commit 382a08e5351df4a520605aba0b4b6f74ab95c6cf
Author: Likyh <ya...@meri.co>
AuthorDate: Thu Apr 13 21:17:15 2023 +0800
fix: move extract existing graphql data to general position (#4921)
Co-authored-by: abeizn <zi...@merico.dev>
---
backend/helpers/pluginhelper/api/api_collector.go | 2 +-
.../helpers/pluginhelper/api/graphql_collector.go | 22 ++++++++++++----------
.../github_graphql/tasks/account_collector.go | 5 ++++-
.../github_graphql/tasks/issue_collector.go | 5 ++++-
.../plugins/github_graphql/tasks/job_collector.go | 5 ++++-
.../plugins/github_graphql/tasks/pr_collector.go | 6 ++++--
6 files changed, 29 insertions(+), 16 deletions(-)
diff --git a/backend/helpers/pluginhelper/api/api_collector.go b/backend/helpers/pluginhelper/api/api_collector.go
index 27b6b8c5c..845658bef 100644
--- a/backend/helpers/pluginhelper/api/api_collector.go
+++ b/backend/helpers/pluginhelper/api/api_collector.go
@@ -425,7 +425,7 @@ func (collector *ApiCollector) fetchAsync(reqData *RequestData, handler func(int
items, err := collector.args.ResponseParser(res)
if err != nil {
if errors.Is(err, ErrFinishCollect) {
- logger.Info("a fetch stop by parser, reqInput: #%d", reqData.Params)
+ logger.Info("a fetch stop by parser, reqInput: #%s", reqData.Params)
handler = nil
} else {
return errors.Default.Wrap(err, fmt.Sprintf("error parsing response from %s", apiUrl))
diff --git a/backend/helpers/pluginhelper/api/graphql_collector.go b/backend/helpers/pluginhelper/api/graphql_collector.go
index de364cfd6..67e3eca80 100644
--- a/backend/helpers/pluginhelper/api/graphql_collector.go
+++ b/backend/helpers/pluginhelper/api/graphql_collector.go
@@ -134,16 +134,23 @@ func (collector *GraphqlCollector) Execute() errors.Error {
if err != nil {
return errors.Default.Wrap(err, "error running auto-migrate")
}
+
+ divider := NewBatchSaveDivider(collector.args.Ctx, collector.args.BatchSize, collector.table, collector.params)
+
// flush data if not incremental collection
- if !collector.args.Incremental {
+ if collector.args.Incremental {
+ // re extract data for new transformation rules
+ err = collector.ExtractExistRawData(divider)
+ if err != nil {
+ collector.checkError(err)
+ }
+ } else {
err = db.Delete(&RawData{}, dal.From(collector.table), dal.Where("params = ?", collector.params))
if err != nil {
return errors.Default.Wrap(err, "error deleting data from collector")
}
}
- divider := NewBatchSaveDivider(collector.args.Ctx, collector.args.BatchSize, collector.table, collector.params)
-
collector.args.Ctx.SetProgress(0, -1)
if collector.args.Input != nil {
iterator := collector.args.Input
@@ -208,11 +215,6 @@ func (collector *GraphqlCollector) exec(divider *BatchSaveDivider, input interfa
SkipCursor: nil,
Size: collector.args.PageSize,
}
- err = collector.ExtractExistRawData(divider, reqData)
- if err != nil {
- collector.checkError(err)
- return
- }
if collector.args.GetPageInfo != nil {
collector.fetchOneByOne(divider, reqData)
} else {
@@ -280,7 +282,7 @@ func (collector *GraphqlCollector) BatchSaveWithOrigin(divider *BatchSaveDivider
}
// ExtractExistRawData will extract data from existing data from raw layer if increment
-func (collector *GraphqlCollector) ExtractExistRawData(divider *BatchSaveDivider, reqData *GraphqlRequestData) errors.Error {
+func (collector *GraphqlCollector) ExtractExistRawData(divider *BatchSaveDivider) errors.Error {
// load data from database
db := collector.args.Ctx.GetDal()
logger := collector.args.Ctx.GetLogger()
@@ -304,7 +306,7 @@ func (collector *GraphqlCollector) ExtractExistRawData(divider *BatchSaveDivider
row := &RawData{}
// get the type of query and variables
- query, variables, _ := collector.args.BuildQuery(reqData)
+ query, variables, _ := collector.args.BuildQuery(nil)
// prgress
collector.args.Ctx.SetProgress(0, -1)
diff --git a/backend/plugins/github_graphql/tasks/account_collector.go b/backend/plugins/github_graphql/tasks/account_collector.go
index 6e98d0dd3..f8781befc 100644
--- a/backend/plugins/github_graphql/tasks/account_collector.go
+++ b/backend/plugins/github_graphql/tasks/account_collector.go
@@ -100,8 +100,11 @@ func CollectAccount(taskCtx plugin.SubTaskContext) errors.Error {
InputStep: 100,
GraphqlClient: data.GraphqlClient,
BuildQuery: func(reqData *helper.GraphqlRequestData) (interface{}, map[string]interface{}, error) {
- accounts := reqData.Input.([]interface{})
query := &GraphqlQueryAccountWrapper{}
+ if reqData == nil {
+ return query, map[string]interface{}{}, nil
+ }
+ accounts := reqData.Input.([]interface{})
users := []map[string]interface{}{}
for _, iAccount := range accounts {
account := iAccount.(*SimpleAccount)
diff --git a/backend/plugins/github_graphql/tasks/issue_collector.go b/backend/plugins/github_graphql/tasks/issue_collector.go
index 5cddf35d3..b65673ca5 100644
--- a/backend/plugins/github_graphql/tasks/issue_collector.go
+++ b/backend/plugins/github_graphql/tasks/issue_collector.go
@@ -116,13 +116,16 @@ func CollectIssue(taskCtx plugin.SubTaskContext) errors.Error {
PageSize: 100,
Incremental: incremental,
BuildQuery: func(reqData *helper.GraphqlRequestData) (interface{}, map[string]interface{}, error) {
+ query := &GraphqlQueryIssueWrapper{}
+ if reqData == nil {
+ return query, map[string]interface{}{}, nil
+ }
since := helper.DateTime{}
if incremental {
since = helper.DateTime{Time: *collectorWithState.LatestState.LatestSuccessStart}
} else if collectorWithState.TimeAfter != nil {
since = helper.DateTime{Time: *collectorWithState.TimeAfter}
}
- query := &GraphqlQueryIssueWrapper{}
ownerName := strings.Split(data.Options.Name, "/")
variables := map[string]interface{}{
"since": since,
diff --git a/backend/plugins/github_graphql/tasks/job_collector.go b/backend/plugins/github_graphql/tasks/job_collector.go
index 8d3a1d285..f46bedf23 100644
--- a/backend/plugins/github_graphql/tasks/job_collector.go
+++ b/backend/plugins/github_graphql/tasks/job_collector.go
@@ -144,8 +144,11 @@ func CollectGraphqlJobs(taskCtx plugin.SubTaskContext) errors.Error {
Incremental: incremental,
GraphqlClient: data.GraphqlClient,
BuildQuery: func(reqData *helper.GraphqlRequestData) (interface{}, map[string]interface{}, error) {
- workflowRuns := reqData.Input.([]interface{})
query := &GraphqlQueryCheckRunWrapper{}
+ if reqData == nil {
+ return query, map[string]interface{}{}, nil
+ }
+ workflowRuns := reqData.Input.([]interface{})
checkSuiteIds := []map[string]interface{}{}
for _, iWorkflowRuns := range workflowRuns {
workflowRun := iWorkflowRuns.(*SimpleWorkflowRun)
diff --git a/backend/plugins/github_graphql/tasks/pr_collector.go b/backend/plugins/github_graphql/tasks/pr_collector.go
index c9e605b2a..b723e0a5a 100644
--- a/backend/plugins/github_graphql/tasks/pr_collector.go
+++ b/backend/plugins/github_graphql/tasks/pr_collector.go
@@ -171,6 +171,9 @@ func CollectPr(taskCtx plugin.SubTaskContext) errors.Error {
*/
BuildQuery: func(reqData *api.GraphqlRequestData) (interface{}, map[string]interface{}, error) {
query := &GraphqlQueryPrWrapper{}
+ if reqData == nil {
+ return query, map[string]interface{}{}, nil
+ }
ownerName := strings.Split(data.Options.Name, "/")
variables := map[string]interface{}{
"pageSize": graphql.Int(reqData.Pager.Size),
@@ -191,12 +194,11 @@ func CollectPr(taskCtx plugin.SubTaskContext) errors.Error {
results := make([]interface{}, 0, 1)
isFinish := false
for _, rawL := range prs {
- // collect all data even though in increment mode because of existing data extracting
+ // collect data even though in increment mode because of updating existing data
if collectorWithState.TimeAfter != nil && !collectorWithState.TimeAfter.Before(rawL.UpdatedAt) {
isFinish = true
break
}
- //If this is a pr, ignore
githubPr, err := convertGithubPullRequest(rawL, data.Options.ConnectionId, data.Options.GithubId)
if err != nil {
return nil, err