You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by ab...@apache.org on 2024/07/11 04:19:57 UTC
(incubator-devlake) branch add-github-request-reviewer updated: feat: github add pull request reviewer
This is an automated email from the ASF dual-hosted git repository.
abeizn pushed a commit to branch add-github-request-reviewer
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/add-github-request-reviewer by this push:
new a858fa5bb feat: github add pull request reviewer
a858fa5bb is described below
commit a858fa5bb8bfb5a0866cb8434fb0e06a40b8df55
Author: abeizn <zi...@merico.dev>
AuthorDate: Thu Jul 11 12:19:45 2024 +0800
feat: github add pull request reviewer
---
.../20240711_restruct_github_reviewers.go | 65 ++++++++++++++
.../github/models/migrationscripts/register.go | 1 +
backend/plugins/github/models/reviewer.go | 8 +-
.../plugins/github/tasks/pr_review_extractor.go | 4 +-
backend/plugins/github/tasks/review_convertor.go | 98 ++++++++++++++++++++++
backend/plugins/github_graphql/impl/impl.go | 1 +
.../plugins/github_graphql/tasks/pr_collector.go | 27 +++++-
.../plugins/github_graphql/tasks/pr_extractor.go | 9 ++
8 files changed, 208 insertions(+), 5 deletions(-)
diff --git a/backend/plugins/github/models/migrationscripts/20240711_restruct_github_reviewers.go b/backend/plugins/github/models/migrationscripts/20240711_restruct_github_reviewers.go
new file mode 100644
index 000000000..af5a95a34
--- /dev/null
+++ b/backend/plugins/github/models/migrationscripts/20240711_restruct_github_reviewers.go
@@ -0,0 +1,65 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+ "github.com/apache/incubator-devlake/core/context"
+ "github.com/apache/incubator-devlake/core/errors"
+ coreArchived "github.com/apache/incubator-devlake/core/models/migrationscripts/archived"
+ "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/plugins/github/models/migrationscripts/archived"
+)
+
+var _ plugin.MigrationScript = (*restructReviewer)(nil)
+
+type reviewer20240711 struct {
+ ConnectionId uint64 `gorm:"primaryKey"`
+ ReviewerId int `gorm:"primaryKey"`
+ PullRequestId int `gorm:"primaryKey"`
+ Name string `gorm:"type:varchar(255)"`
+ Username string `gorm:"type:varchar(255)"`
+ State string `gorm:"type:varchar(255)"`
+ AvatarUrl string `gorm:"type:varchar(255)"`
+ WebUrl string `gorm:"type:varchar(255)"`
+ coreArchived.NoPKModel
+}
+
+func (reviewer20240711) TableName() string {
+ return "_tool_github_reviewers"
+}
+
+type restructReviewer struct{}
+
+func (*restructReviewer) Up(basicRes context.BasicRes) errors.Error {
+ db := basicRes.GetDal()
+ if err := db.DropTables(&archived.GithubReviewer{}); err != nil {
+ return err
+ }
+ if err := db.AutoMigrate(&reviewer20240711{}); err != nil {
+ return err
+ }
+ return nil
+}
+
+func (*restructReviewer) Version() uint64 {
+ return 20240710142104
+}
+
+func (*restructReviewer) Name() string {
+ return "restruct reviewer table"
+}
diff --git a/backend/plugins/github/models/migrationscripts/register.go b/backend/plugins/github/models/migrationscripts/register.go
index 35ad1b1dc..4a4b95069 100644
--- a/backend/plugins/github/models/migrationscripts/register.go
+++ b/backend/plugins/github/models/migrationscripts/register.go
@@ -51,5 +51,6 @@ func All() []plugin.MigrationScript {
new(addReleaseTable),
new(addReleaseCommitSha),
new(addMergedByToPr),
+ new(restructReviewer),
}
}
diff --git a/backend/plugins/github/models/reviewer.go b/backend/plugins/github/models/reviewer.go
index c8af4e181..7d82a88e6 100644
--- a/backend/plugins/github/models/reviewer.go
+++ b/backend/plugins/github/models/reviewer.go
@@ -23,9 +23,13 @@ import (
type GithubReviewer struct {
ConnectionId uint64 `gorm:"primaryKey"`
- GithubId int `gorm:"primaryKey"`
- Login string `gorm:"type:varchar(255)"`
+ ReviewerId int `gorm:"primaryKey"`
PullRequestId int `gorm:"primaryKey"`
+ Name string `gorm:"type:varchar(255)"`
+ Username string `gorm:"type:varchar(255)"`
+ State string `gorm:"type:varchar(255)"`
+ AvatarUrl string `gorm:"type:varchar(255)"`
+ WebUrl string `gorm:"type:varchar(255)"`
common.NoPKModel
}
diff --git a/backend/plugins/github/tasks/pr_review_extractor.go b/backend/plugins/github/tasks/pr_review_extractor.go
index c14a179e8..2fba9a955 100644
--- a/backend/plugins/github/tasks/pr_review_extractor.go
+++ b/backend/plugins/github/tasks/pr_review_extractor.go
@@ -108,8 +108,8 @@ func ExtractApiPullRequestReviews(taskCtx plugin.SubTaskContext) errors.Error {
}
if apiPullRequestReview.User != nil {
- githubReviewer.GithubId = apiPullRequestReview.User.Id
- githubReviewer.Login = apiPullRequestReview.User.Login
+ githubReviewer.ReviewerId = apiPullRequestReview.User.Id
+ githubReviewer.Username = apiPullRequestReview.User.Login
githubPrReview.AuthorUserId = apiPullRequestReview.User.Id
githubPrReview.AuthorUsername = apiPullRequestReview.User.Login
diff --git a/backend/plugins/github/tasks/review_convertor.go b/backend/plugins/github/tasks/review_convertor.go
new file mode 100644
index 000000000..d4d101042
--- /dev/null
+++ b/backend/plugins/github/tasks/review_convertor.go
@@ -0,0 +1,98 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "reflect"
+
+ "github.com/apache/incubator-devlake/core/dal"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/models/domainlayer/code"
+ "github.com/apache/incubator-devlake/core/models/domainlayer/didgen"
+ "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "github.com/apache/incubator-devlake/plugins/github/models"
+)
+
+func init() {
+ RegisterSubtaskMeta(&ConvertReviewsMeta)
+}
+
+var ConvertReviewsMeta = plugin.SubTaskMeta{
+ Name: "Convert Reviews",
+ EntryPoint: ConvertReviews,
+ EnabledByDefault: true,
+ Description: "ConvertReviews data from Github api",
+ DomainTypes: []string{plugin.DOMAIN_TYPE_CODE_REVIEW},
+ DependencyTables: []string{
+ models.GithubReviewer{}.TableName(), // cursor and id generator
+ models.GithubPullRequest{}.TableName(), // cursor and id generator
+ models.GithubAccount{}.TableName(), // id generator
+ },
+ ProductTables: []string{code.PullRequestReviewer{}.TableName()},
+}
+
+func ConvertReviews(taskCtx plugin.SubTaskContext) errors.Error {
+ db := taskCtx.GetDal()
+ data := taskCtx.GetData().(*GithubTaskData)
+ repoId := data.Options.GithubId
+
+ cursor, err := db.Cursor(
+ dal.From(&models.GithubReviewer{}),
+ dal.Join("left join _tool_github_pull_requests "+
+ "on _tool_github_pull_requests.github_id = _tool_github_reviewers.pull_request_id"),
+ dal.Where("repo_id = ? and _tool_github_pull_requests.connection_id = ?", repoId, data.Options.ConnectionId),
+ )
+ if err != nil {
+ return err
+ }
+ defer cursor.Close()
+
+ prIdGen := didgen.NewDomainIdGenerator(&models.GithubPullRequest{})
+ accountIdGen := didgen.NewDomainIdGenerator(&models.GithubAccount{})
+
+ converter, err := api.NewDataConverter(api.DataConverterArgs{
+ InputRowType: reflect.TypeOf(models.GithubReviewer{}),
+ Input: cursor,
+ RawDataSubTaskArgs: api.RawDataSubTaskArgs{
+ Ctx: taskCtx,
+ Params: GithubApiParams{
+ ConnectionId: data.Options.ConnectionId,
+ Name: data.Options.Name,
+ },
+ Table: RAW_PR_REVIEW_TABLE,
+ },
+ Convert: func(inputRow interface{}) ([]interface{}, errors.Error) {
+ githubReview := inputRow.(*models.GithubReviewer)
+ domainReview := &code.PullRequestReviewer{
+ PullRequestId: prIdGen.Generate(data.Options.ConnectionId, githubReview.PullRequestId),
+ ReviewerId: accountIdGen.Generate(data.Options.ConnectionId, githubReview.ReviewerId),
+ Name: githubReview.Name,
+ UserName: githubReview.Username,
+ }
+ return []interface{}{
+ domainReview,
+ }, nil
+ },
+ })
+ if err != nil {
+ return err
+ }
+
+ return converter.Execute()
+}
diff --git a/backend/plugins/github_graphql/impl/impl.go b/backend/plugins/github_graphql/impl/impl.go
index 8045cd145..66433699c 100644
--- a/backend/plugins/github_graphql/impl/impl.go
+++ b/backend/plugins/github_graphql/impl/impl.go
@@ -125,6 +125,7 @@ func (p GithubGraphql) SubTaskMetas() []plugin.SubTaskMeta {
githubTasks.ConvertIssueAssigneeMeta,
githubTasks.ConvertIssueCommentsMeta,
githubTasks.ConvertPullRequestCommentsMeta,
+ githubTasks.ConvertReviewsMeta,
githubTasks.ConvertMilestonesMeta,
githubTasks.ConvertAccountsMeta,
diff --git a/backend/plugins/github_graphql/tasks/pr_collector.go b/backend/plugins/github_graphql/tasks/pr_collector.go
index 82e5da69a..2cfdb36a9 100644
--- a/backend/plugins/github_graphql/tasks/pr_collector.go
+++ b/backend/plugins/github_graphql/tasks/pr_collector.go
@@ -18,6 +18,7 @@ limitations under the License.
package tasks
import (
+ "fmt"
"strings"
"time"
@@ -85,9 +86,32 @@ type GraphqlQueryPr struct {
TotalCount graphql.Int
Nodes []GraphqlQueryReview `graphql:"nodes"`
} `graphql:"reviews(first: 100)"`
- MergedBy *GraphqlInlineAccountQuery
+ MergedBy *GraphqlInlineAccountQuery
+ ReviewRequests struct {
+ Nodes []ReviewRequestNode `graphql:"nodes"`
+ } `graphql:"reviewRequests(first: 10)"`
}
+type ReviewRequestNode struct {
+ RequestedReviewer RequestedReviewer `graphql:"requestedReviewer"`
+}
+
+type RequestedReviewer struct {
+ User User `graphql:"... on User"`
+ Team Team `graphql:"... on Team"`
+}
+
+type User struct {
+ Id int `graphql:"databaseId"`
+ Login string `graphql:"login"`
+ Name string `graphql:"name"`
+}
+
+type Team struct {
+ Id int `graphql:"databaseId"`
+ Name string `graphql:"name"`
+ Slug string `graphql:"slug"`
+}
type GraphqlQueryReview struct {
Body string
Author *GraphqlInlineAccountQuery
@@ -161,6 +185,7 @@ func CollectPrs(taskCtx plugin.SubTaskContext) errors.Error {
"owner": graphql.String(ownerName[0]),
"name": graphql.String(ownerName[1]),
}
+ fmt.Println(query)
return query, variables, nil
},
GetPageInfo: func(iQuery interface{}, args *api.GraphqlCollectorArgs) (*api.GraphqlQueryPageInfo, error) {
diff --git a/backend/plugins/github_graphql/tasks/pr_extractor.go b/backend/plugins/github_graphql/tasks/pr_extractor.go
index 163fa4c84..cc26d0cc9 100644
--- a/backend/plugins/github_graphql/tasks/pr_extractor.go
+++ b/backend/plugins/github_graphql/tasks/pr_extractor.go
@@ -121,6 +121,15 @@ func ExtractPrs(taskCtx plugin.SubTaskContext) errors.Error {
results = append(results, githubPrReview)
}
}
+ for _, apiReviewRequests := range rawL.ReviewRequests.Nodes {
+ githubReviewRequests := &models.GithubReviewer{
+ ConnectionId: data.Options.ConnectionId,
+ PullRequestId: githubPr.GithubId,
+ ReviewerId: apiReviewRequests.RequestedReviewer.User.Id,
+ Username: apiReviewRequests.RequestedReviewer.User.Login,
+ }
+ results = append(results, githubReviewRequests)
+ }
for _, apiPullRequestCommit := range rawL.Commits.Nodes {
githubCommit, err := convertPullRequestCommit(apiPullRequestCommit)