You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by li...@apache.org on 2022/11/03 09:30:38 UTC
[incubator-devlake] branch release-v0.14 updated: cherry-pick #3664 to 0.14 (#3665)
This is an automated email from the ASF dual-hosted git repository.
likyh pushed a commit to branch release-v0.14
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/release-v0.14 by this push:
new afc8d13f cherry-pick #3664 to 0.14 (#3665)
afc8d13f is described below
commit afc8d13f9c0722e13cca74602109561af2abe9ad
Author: Likyh <l...@likyh.com>
AuthorDate: Thu Nov 3 17:30:34 2022 +0800
cherry-pick #3664 to 0.14 (#3665)
* fix: replace number with databaseId in issue.id
* fix: deal 30/page for pr in github graphql
* fix: fix a wrong err check
Co-authored-by: linyh <ya...@meri.co>
---
plugins/core/dal/dal.go | 6 +--
plugins/github_graphql/plugin_main.go | 6 ++-
plugins/github_graphql/tasks/issue_collector.go | 50 ++++++++++++++++++-------
plugins/github_graphql/tasks/pr_collector.go | 2 +-
plugins/helper/graphql_collector.go | 4 +-
5 files changed, 48 insertions(+), 20 deletions(-)
diff --git a/plugins/core/dal/dal.go b/plugins/core/dal/dal.go
index 3a31e1d1..5e820829 100644
--- a/plugins/core/dal/dal.go
+++ b/plugins/core/dal/dal.go
@@ -62,11 +62,11 @@ type Dal interface {
Cursor(clauses ...Clause) (*sql.Rows, errors.Error)
// Fetch loads row data from `cursor` into `dst`
Fetch(cursor *sql.Rows, dst interface{}) errors.Error
- // All loads matched rows from database to `dst`, USE IT WITH COUTIOUS!!
+ // All loads matched rows from database to `dst`, USE IT WITH CAUTIOUS!!
All(dst interface{}, clauses ...Clause) errors.Error
// First loads first matched row from database to `dst`, error will be returned if no records were found
First(dst interface{}, clauses ...Clause) errors.Error
- // All loads matched rows from database to `dst`, USE IT WITH COUTIOUS!!
+ // Count matched rows from database
Count(clauses ...Clause) (int64, errors.Error)
// Pluck used to query single column
Pluck(column string, dest interface{}, clauses ...Clause) errors.Error
@@ -86,7 +86,7 @@ type Dal interface {
AllTables() ([]string, errors.Error)
// GetColumns returns table columns in database
GetColumns(dst schema.Tabler, filter func(columnMeta ColumnMeta) bool) (cms []ColumnMeta, err errors.Error)
- // GetPrimarykeyFields get the PrimaryKey from `gorm` tag
+ // GetPrimaryKeyFields get the PrimaryKey from `gorm` tag
GetPrimaryKeyFields(t reflect.Type) []reflect.StructField
// Dialect returns the dialect of current database
Dialect() string
diff --git a/plugins/github_graphql/plugin_main.go b/plugins/github_graphql/plugin_main.go
index a2a8b84a..85fc2aa8 100644
--- a/plugins/github_graphql/plugin_main.go
+++ b/plugins/github_graphql/plugin_main.go
@@ -59,6 +59,10 @@ func (plugin GithubGraphql) Init(config *viper.Viper, logger core.Logger, db *go
func (plugin GithubGraphql) SubTaskMetas() []core.SubTaskMeta {
return []core.SubTaskMeta{
tasks.CollectRepoMeta,
+
+ githubTasks.CollectMilestonesMeta,
+ githubTasks.ExtractMilestonesMeta,
+
tasks.CollectIssueMeta,
tasks.CollectPrMeta,
@@ -66,8 +70,6 @@ func (plugin GithubGraphql) SubTaskMetas() []core.SubTaskMeta {
githubTasks.ExtractApiCommentsMeta,
githubTasks.CollectApiEventsMeta,
githubTasks.ExtractApiEventsMeta,
- githubTasks.CollectMilestonesMeta,
- githubTasks.ExtractMilestonesMeta,
githubTasks.CollectApiPrReviewCommentsMeta,
githubTasks.ExtractApiPrReviewCommentsMeta,
diff --git a/plugins/github_graphql/tasks/issue_collector.go b/plugins/github_graphql/tasks/issue_collector.go
index 9ceb85a3..7315ee8f 100644
--- a/plugins/github_graphql/tasks/issue_collector.go
+++ b/plugins/github_graphql/tasks/issue_collector.go
@@ -21,6 +21,7 @@ import (
"github.com/apache/incubator-devlake/errors"
"github.com/apache/incubator-devlake/models/domainlayer/ticket"
"github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/core/dal"
"github.com/apache/incubator-devlake/plugins/github/models"
githubTasks "github.com/apache/incubator-devlake/plugins/github/tasks"
"github.com/apache/incubator-devlake/plugins/helper"
@@ -61,12 +62,12 @@ type GraphqlQueryIssue struct {
Assignees []GraphqlInlineAccountQuery `graphql:"nodes"`
} `graphql:"assignees(first: 1)"`
Milestone *struct {
- Number int `json:"number"`
+ Number int
} `json:"milestone"`
Labels struct {
Nodes []struct {
- Id string `json:"id"`
- Name string `json:"name"`
+ Id string
+ Name string
}
} `graphql:"labels(first: 100)"`
}
@@ -81,6 +82,7 @@ var CollectIssueMeta = core.SubTaskMeta{
var _ core.SubTaskEntryPoint = CollectIssue
func CollectIssue(taskCtx core.SubTaskContext) errors.Error {
+ db := taskCtx.GetDal()
data := taskCtx.GetData().(*githubTasks.GithubTaskData)
config := data.Options.TransformationRules
issueRegexes, err := githubTasks.NewIssueRegexes(config)
@@ -88,13 +90,14 @@ func CollectIssue(taskCtx core.SubTaskContext) errors.Error {
return nil
}
+ milestoneMap, err := getMilestoneMap(db, data.Repo.GithubId, data.Repo.ConnectionId)
+ if err != nil {
+ return nil
+ }
+
collector, err := helper.NewGraphqlCollector(helper.GraphqlCollectorArgs{
RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
Ctx: taskCtx,
- /*
- This struct will be JSONEncoded and stored into database along with raw data itself, to identity minimal
- set of data to be process, for example, we process JiraIssues by Board
- */
Params: githubTasks.GithubApiParams{
ConnectionId: data.Options.ConnectionId,
Owner: data.Options.Owner,
@@ -104,9 +107,6 @@ func CollectIssue(taskCtx core.SubTaskContext) errors.Error {
},
GraphqlClient: data.GraphqlClient,
PageSize: 100,
- /*
- (Optional) Return query string for request, or you can plug them into UrlTemplate directly
- */
BuildQuery: func(reqData *helper.GraphqlRequestData) (interface{}, map[string]interface{}, error) {
query := &GraphqlQueryIssueWrapper{}
variables := map[string]interface{}{
@@ -127,7 +127,7 @@ func CollectIssue(taskCtx core.SubTaskContext) errors.Error {
results := make([]interface{}, 0, 1)
for _, issue := range issues {
- githubIssue, err := convertGithubIssue(issue, data.Options.ConnectionId, data.Repo.GithubId)
+ githubIssue, err := convertGithubIssue(milestoneMap, issue, data.Options.ConnectionId, data.Repo.GithubId)
if err != nil {
return nil, err
}
@@ -163,7 +163,29 @@ func CollectIssue(taskCtx core.SubTaskContext) errors.Error {
return collector.Execute()
}
-func convertGithubIssue(issue GraphqlQueryIssue, connectionId uint64, repositoryId int) (*models.GithubIssue, errors.Error) {
+// create a milestone map for numberId to databaseId
+func getMilestoneMap(db dal.Dal, repoId int, connectionId uint64) (map[int]int, errors.Error) {
+ milestoneMap := map[int]int{}
+ var milestones []struct {
+ MilestoneId int
+ RepoId int
+ Number int
+ }
+ err := db.All(
+ &milestones,
+ dal.From(&models.GithubMilestone{}),
+ dal.Where("repo_id = ? and connection_id = ?", repoId, connectionId),
+ )
+ if err != nil {
+ return nil, err
+ }
+ for _, milestone := range milestones {
+ milestoneMap[milestone.Number] = milestone.MilestoneId
+ }
+ return milestoneMap, nil
+}
+
+func convertGithubIssue(milestoneMap map[int]int, issue GraphqlQueryIssue, connectionId uint64, repositoryId int) (*models.GithubIssue, errors.Error) {
githubIssue := &models.GithubIssue{
ConnectionId: connectionId,
GithubId: issue.DatabaseId,
@@ -189,7 +211,9 @@ func convertGithubIssue(issue GraphqlQueryIssue, connectionId uint64, repository
githubIssue.LeadTimeMinutes = uint(issue.ClosedAt.Sub(issue.CreatedAt).Minutes())
}
if issue.Milestone != nil {
- githubIssue.MilestoneId = issue.Milestone.Number
+ if milestoneId, ok := milestoneMap[issue.Milestone.Number]; ok {
+ githubIssue.MilestoneId = milestoneId
+ }
}
return githubIssue, nil
}
diff --git a/plugins/github_graphql/tasks/pr_collector.go b/plugins/github_graphql/tasks/pr_collector.go
index a56a33bc..fc0be696 100644
--- a/plugins/github_graphql/tasks/pr_collector.go
+++ b/plugins/github_graphql/tasks/pr_collector.go
@@ -154,7 +154,7 @@ func CollectPr(taskCtx core.SubTaskContext) errors.Error {
Table: RAW_PRS_TABLE,
},
GraphqlClient: data.GraphqlClient,
- PageSize: 100,
+ PageSize: 30,
/*
(Optional) Return query string for request, or you can plug them into UrlTemplate directly
*/
diff --git a/plugins/helper/graphql_collector.go b/plugins/helper/graphql_collector.go
index 1091e199..c03a9eec 100644
--- a/plugins/helper/graphql_collector.go
+++ b/plugins/helper/graphql_collector.go
@@ -264,7 +264,9 @@ func (collector *GraphqlCollector) fetchAsync(divider *BatchSaveDivider, reqData
}
if len(dataErrors) > 0 {
if collector.args.ResponseParserWithDataErrors == nil {
- collector.checkError(errors.Default.Wrap(err, `graphql query got error`))
+ for _, dataError := range dataErrors {
+ collector.checkError(errors.Default.Wrap(dataError, `graphql query got error`))
+ }
return
}
// else: error will deal by ResponseParserWithDataErrors