You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by li...@apache.org on 2022/10/20 01:48:28 UTC
[incubator-devlake] branch release-v0.14 updated (f691100e -> 96559007)
This is an automated email from the ASF dual-hosted git repository.
likyh pushed a change to branch release-v0.14
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
from f691100e fix(tapd): set sprint status to std
new d5e1f213 feat: enable converter for github_graphql
new cb92da02 fix: fix bug for github_graphql when language is empty
new 992c0744 fix: confirm all graphql task finish
new d415e7a2 feat: save query vars into raw_table.input
new 36e9d99d fix: set username/id to empty when github user is nil
new 96559007 fix: skip when enrichGithubPrComment not found PR
The 6 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
plugins/github/tasks/comment_extractor.go | 22 ++++++++-----
plugins/github/tasks/event_extractor.go | 36 +++++++++++-----------
.../github/tasks/pr_review_comment_extractor.go | 30 +++++++++++-------
plugins/github/tasks/pr_review_extractor.go | 24 +++++++++------
plugins/github_graphql/plugin_main.go | 16 ++++++++++
plugins/github_graphql/tasks/pr_collector.go | 25 +++++++++------
plugins/github_graphql/tasks/repo_collector.go | 6 +++-
plugins/helper/graphql_async_client.go | 5 ++-
plugins/helper/graphql_collector.go | 6 +++-
services/pipeline.go | 4 +--
10 files changed, 112 insertions(+), 62 deletions(-)
[incubator-devlake] 04/06: feat: save query vars into raw_table.input
Posted by li...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
likyh pushed a commit to branch release-v0.14
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit d415e7a26f180bb29e7fc35187d7d46d9a25ccf6
Author: linyh <ya...@meri.co>
AuthorDate: Wed Oct 19 19:42:36 2022 +0800
feat: save query vars into raw_table.input
---
plugins/helper/graphql_collector.go | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/plugins/helper/graphql_collector.go b/plugins/helper/graphql_collector.go
index 261009b6..07999394 100644
--- a/plugins/helper/graphql_collector.go
+++ b/plugins/helper/graphql_collector.go
@@ -259,11 +259,15 @@ func (collector *GraphqlCollector) fetchAsync(divider *BatchSaveDivider, reqData
}
db := collector.args.Ctx.GetDal()
queryStr, _ := graphql.ConstructQuery(query, variables)
+ variablesJson, err := json.Marshal(variables)
+ if err != nil {
+ panic(err)
+ }
row := &RawData{
Params: collector.params,
Data: paramsBytes,
Url: queryStr,
- Input: reqData.InputJSON,
+ Input: variablesJson,
}
err = db.Create(row, dal.From(collector.table))
if err != nil {
[incubator-devlake] 03/06: fix: confirm all graphql task finish
Posted by li...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
likyh pushed a commit to branch release-v0.14
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit 992c0744160299914836899e8ed377eab51208e6
Author: linyh <ya...@meri.co>
AuthorDate: Wed Oct 19 19:42:02 2022 +0800
fix: confirm all graphql task finish
---
plugins/helper/graphql_async_client.go | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/plugins/helper/graphql_async_client.go b/plugins/helper/graphql_async_client.go
index 91be1728..fcd46aef 100644
--- a/plugins/helper/graphql_async_client.go
+++ b/plugins/helper/graphql_async_client.go
@@ -131,12 +131,15 @@ func (apiClient *GraphqlAsyncClient) NextTick(task func() errors.Error) {
// to make sure task will be enqueued
apiClient.waitGroup.Add(1)
go func() {
- defer apiClient.waitGroup.Done()
select {
case <-apiClient.ctx.Done():
return
default:
go func() {
+ // if set waitGroup done here, a serial of goruntine will block until son goruntine finish.
+ // But if done out of this go func, so task will run after waitGroup finish
+ // I have no idea about this now...
+ defer apiClient.waitGroup.Done()
apiClient.checkError(task())
}()
}
[incubator-devlake] 05/06: fix: set username/id to empty when github user is nil
Posted by li...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
likyh pushed a commit to branch release-v0.14
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit 36e9d99da13698f3b032950a6335f205a57afb93
Author: linyh <ya...@meri.co>
AuthorDate: Wed Oct 19 20:51:25 2022 +0800
fix: set username/id to empty when github user is nil
---
plugins/github/tasks/comment_extractor.go | 22 ++++++++-----
plugins/github/tasks/event_extractor.go | 36 +++++++++++-----------
.../github/tasks/pr_review_comment_extractor.go | 11 +++++--
plugins/github/tasks/pr_review_extractor.go | 24 +++++++++------
plugins/github_graphql/tasks/pr_collector.go | 25 +++++++++------
5 files changed, 69 insertions(+), 49 deletions(-)
diff --git a/plugins/github/tasks/comment_extractor.go b/plugins/github/tasks/comment_extractor.go
index ffcef550..b237d53a 100644
--- a/plugins/github/tasks/comment_extractor.go
+++ b/plugins/github/tasks/comment_extractor.go
@@ -95,12 +95,14 @@ func ExtractApiComments(taskCtx core.SubTaskContext) errors.Error {
GithubId: apiComment.GithubId,
PullRequestId: pr.GithubId,
Body: string(apiComment.Body),
- AuthorUsername: apiComment.User.Login,
- AuthorUserId: apiComment.User.Id,
GithubCreatedAt: apiComment.GithubCreatedAt.ToTime(),
GithubUpdatedAt: apiComment.GithubUpdatedAt.ToTime(),
Type: "NORMAL",
}
+ if apiComment.User != nil {
+ githubPrComment.AuthorUsername = apiComment.User.Login
+ githubPrComment.AuthorUserId = apiComment.User.Id
+ }
results = append(results, githubPrComment)
} else {
githubIssueComment := &models.GithubIssueComment{
@@ -108,18 +110,22 @@ func ExtractApiComments(taskCtx core.SubTaskContext) errors.Error {
GithubId: apiComment.GithubId,
IssueId: issue.GithubId,
Body: string(apiComment.Body),
- AuthorUsername: apiComment.User.Login,
- AuthorUserId: apiComment.User.Id,
GithubCreatedAt: apiComment.GithubCreatedAt.ToTime(),
GithubUpdatedAt: apiComment.GithubUpdatedAt.ToTime(),
}
+ if apiComment.User != nil {
+ githubIssueComment.AuthorUsername = apiComment.User.Login
+ githubIssueComment.AuthorUserId = apiComment.User.Id
+ }
results = append(results, githubIssueComment)
}
- githubAccount, err := convertAccount(apiComment.User, data.Repo.GithubId, data.Options.ConnectionId)
- if err != nil {
- return nil, err
+ if apiComment.User != nil {
+ githubAccount, err := convertAccount(apiComment.User, data.Repo.GithubId, data.Options.ConnectionId)
+ if err != nil {
+ return nil, err
+ }
+ results = append(results, githubAccount)
}
- results = append(results, githubAccount)
return results, nil
},
})
diff --git a/plugins/github/tasks/event_extractor.go b/plugins/github/tasks/event_extractor.go
index a130271f..1cb6a106 100644
--- a/plugins/github/tasks/event_extractor.go
+++ b/plugins/github/tasks/event_extractor.go
@@ -66,16 +66,28 @@ func ExtractApiEvents(taskCtx core.SubTaskContext) errors.Error {
if body.GithubId == 0 || body.Actor == nil {
return nil, nil
}
- githubIssueEvent, err := convertGithubEvent(body, data.Options.ConnectionId)
- if err != nil {
- return nil, err
+ githubIssueEvent := &models.GithubIssueEvent{
+ ConnectionId: data.Options.ConnectionId,
+ GithubId: body.GithubId,
+ IssueId: body.Issue.Id,
+ Type: body.Event,
+ GithubCreatedAt: body.GithubCreatedAt.ToTime(),
}
- results = append(results, githubIssueEvent)
- githubAccount, err := convertAccount(body.Actor, data.Repo.GithubId, data.Options.ConnectionId)
+
+ if body.Actor != nil {
+ githubIssueEvent.AuthorUsername = body.Actor.Login
+
+ githubAccount, err := convertAccount(body.Actor, data.Repo.GithubId, data.Options.ConnectionId)
+ if err != nil {
+ return nil, err
+ }
+ results = append(results, githubAccount)
+ }
+
if err != nil {
return nil, err
}
- results = append(results, githubAccount)
+ results = append(results, githubIssueEvent)
return results, nil
},
@@ -87,15 +99,3 @@ func ExtractApiEvents(taskCtx core.SubTaskContext) errors.Error {
return extractor.Execute()
}
-
-func convertGithubEvent(event *IssueEvent, connId uint64) (*models.GithubIssueEvent, errors.Error) {
- githubEvent := &models.GithubIssueEvent{
- ConnectionId: connId,
- GithubId: event.GithubId,
- IssueId: event.Issue.Id,
- Type: event.Event,
- AuthorUsername: event.Actor.Login,
- GithubCreatedAt: event.GithubCreatedAt.ToTime(),
- }
- return githubEvent, nil
-}
diff --git a/plugins/github/tasks/pr_review_comment_extractor.go b/plugins/github/tasks/pr_review_comment_extractor.go
index 3ef12736..fae44a83 100644
--- a/plugins/github/tasks/pr_review_comment_extractor.go
+++ b/plugins/github/tasks/pr_review_comment_extractor.go
@@ -78,8 +78,6 @@ func ExtractApiPrReviewComments(taskCtx core.SubTaskContext) errors.Error {
Body: string(prReviewComment.Body),
CommitSha: prReviewComment.CommitId,
ReviewId: prReviewComment.PrReviewId,
- AuthorUsername: prReviewComment.User.Login,
- AuthorUserId: prReviewComment.User.Id,
GithubCreatedAt: prReviewComment.GithubCreatedAt.ToTime(),
GithubUpdatedAt: prReviewComment.GithubUpdatedAt.ToTime(),
Type: "DIFF",
@@ -96,12 +94,19 @@ func ExtractApiPrReviewComments(taskCtx core.SubTaskContext) errors.Error {
if prId != 0 {
githubPrComment.PullRequestId = prId
}
- results = append(results, githubPrComment)
+
+ if prReviewComment.User != nil {
+ githubPrComment.AuthorUserId = prReviewComment.User.Id
+ githubPrComment.AuthorUsername = prReviewComment.User.Login
+
githubAccount, err := convertAccount(prReviewComment.User, data.Repo.GithubId, data.Options.ConnectionId)
if err != nil {
return nil, err
}
results = append(results, githubAccount)
+ }
+
+ results = append(results, githubPrComment)
return results, nil
},
})
diff --git a/plugins/github/tasks/pr_review_extractor.go b/plugins/github/tasks/pr_review_extractor.go
index c8de960b..f199c4a4 100644
--- a/plugins/github/tasks/pr_review_extractor.go
+++ b/plugins/github/tasks/pr_review_extractor.go
@@ -85,8 +85,6 @@ func ExtractApiPullRequestReviews(taskCtx core.SubTaskContext) errors.Error {
githubReviewer := &models.GithubReviewer{
ConnectionId: data.Options.ConnectionId,
- GithubId: apiPullRequestReview.User.Id,
- Login: apiPullRequestReview.User.Login,
PullRequestId: pull.GithubId,
}
@@ -97,19 +95,25 @@ func ExtractApiPullRequestReviews(taskCtx core.SubTaskContext) errors.Error {
State: apiPullRequestReview.State,
CommitSha: apiPullRequestReview.CommitId,
GithubSubmitAt: apiPullRequestReview.SubmittedAt.ToNullableTime(),
-
PullRequestId: pull.GithubId,
- AuthorUsername: apiPullRequestReview.User.Login,
- AuthorUserId: apiPullRequestReview.User.Id,
+ }
+
+ if apiPullRequestReview.User != nil {
+ githubReviewer.GithubId = apiPullRequestReview.User.Id
+ githubReviewer.Login = apiPullRequestReview.User.Login
+
+ githubPrReview.AuthorUserId = apiPullRequestReview.User.Id
+ githubPrReview.AuthorUsername = apiPullRequestReview.User.Login
+
+ githubUser, err := convertAccount(apiPullRequestReview.User, data.Repo.GithubId, data.Options.ConnectionId)
+ if err != nil {
+ return nil, err
+ }
+ results = append(results, githubUser)
}
results = append(results, githubReviewer)
results = append(results, githubPrReview)
- githubUser, err := convertAccount(apiPullRequestReview.User, data.Repo.GithubId, data.Options.ConnectionId)
- if err != nil {
- return nil, err
- }
- results = append(results, githubUser)
return results, nil
},
diff --git a/plugins/github_graphql/tasks/pr_collector.go b/plugins/github_graphql/tasks/pr_collector.go
index fd0a3b35..a56a33bc 100644
--- a/plugins/github_graphql/tasks/pr_collector.go
+++ b/plugins/github_graphql/tasks/pr_collector.go
@@ -218,8 +218,6 @@ func CollectPr(taskCtx core.SubTaskContext) errors.Error {
if apiPullRequestReview.State != "PENDING" {
githubReviewer := &models.GithubReviewer{
ConnectionId: data.Options.ConnectionId,
- GithubId: apiPullRequestReview.Author.Id,
- Login: apiPullRequestReview.Author.Login,
PullRequestId: githubPr.GithubId,
}
@@ -231,18 +229,25 @@ func CollectPr(taskCtx core.SubTaskContext) errors.Error {
CommitSha: apiPullRequestReview.Commit.Oid,
GithubSubmitAt: apiPullRequestReview.SubmittedAt,
- PullRequestId: githubPr.GithubId,
- AuthorUsername: apiPullRequestReview.Author.Login,
- AuthorUserId: apiPullRequestReview.Author.Id,
+ PullRequestId: githubPr.GithubId,
+ }
+
+ if apiPullRequestReview.Author != nil {
+ githubReviewer.GithubId = apiPullRequestReview.Author.Id
+ githubReviewer.Login = apiPullRequestReview.Author.Login
+
+ githubPrReview.AuthorUserId = apiPullRequestReview.Author.Id
+ githubPrReview.AuthorUsername = apiPullRequestReview.Author.Login
+
+ githubUser, err := convertGraphqlPreAccount(*apiPullRequestReview.Author, data.Repo.GithubId, data.Options.ConnectionId)
+ if err != nil {
+ return nil, err
+ }
+ results = append(results, githubUser)
}
results = append(results, githubReviewer)
results = append(results, githubPrReview)
- githubUser, err := convertGraphqlPreAccount(*apiPullRequestReview.Author, data.Repo.GithubId, data.Options.ConnectionId)
- if err != nil {
- return nil, err
- }
- results = append(results, githubUser)
}
}
[incubator-devlake] 02/06: fix: fix bug for github_graphql when language is empty
Posted by li...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
likyh pushed a commit to branch release-v0.14
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit cb92da02cc7d85f6475db879a70ef2c307b1f343
Author: linyh <ya...@meri.co>
AuthorDate: Wed Oct 19 19:41:41 2022 +0800
fix: fix bug for github_graphql when language is empty
---
plugins/github_graphql/tasks/repo_collector.go | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/plugins/github_graphql/tasks/repo_collector.go b/plugins/github_graphql/tasks/repo_collector.go
index 516dad2c..e96ad410 100644
--- a/plugins/github_graphql/tasks/repo_collector.go
+++ b/plugins/github_graphql/tasks/repo_collector.go
@@ -96,6 +96,10 @@ func CollectRepo(taskCtx core.SubTaskContext) errors.Error {
query := iQuery.(*GraphqlQueryRepo)
repository := query.Repository
results := make([]interface{}, 0, 1)
+ language := ``
+ if len(repository.Languages.Nodes) > 0 {
+ language = repository.Languages.Nodes[0].Name
+ }
githubRepository := &models.GithubRepo{
ConnectionId: data.Options.ConnectionId,
GithubId: repository.GithubId,
@@ -104,7 +108,7 @@ func CollectRepo(taskCtx core.SubTaskContext) errors.Error {
Description: repository.Description,
OwnerId: repository.Owner.Id,
OwnerLogin: repository.Owner.Login,
- Language: repository.Languages.Nodes[0].Name,
+ Language: language,
CreatedDate: repository.CreatedDate,
UpdatedDate: repository.UpdatedDate,
}
[incubator-devlake] 06/06: fix: skip when enrichGithubPrComment not found PR
Posted by li...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
likyh pushed a commit to branch release-v0.14
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit 96559007fbc27996d28381399a364b6cc36a99d1
Author: linyh <ya...@meri.co>
AuthorDate: Wed Oct 19 20:54:33 2022 +0800
fix: skip when enrichGithubPrComment not found PR
---
plugins/github/tasks/pr_review_comment_extractor.go | 21 ++++++++++++---------
services/pipeline.go | 4 ++--
2 files changed, 14 insertions(+), 11 deletions(-)
diff --git a/plugins/github/tasks/pr_review_comment_extractor.go b/plugins/github/tasks/pr_review_comment_extractor.go
index fae44a83..0307f720 100644
--- a/plugins/github/tasks/pr_review_comment_extractor.go
+++ b/plugins/github/tasks/pr_review_comment_extractor.go
@@ -19,14 +19,15 @@ package tasks
import (
"encoding/json"
+ goerror "errors"
"fmt"
- "github.com/apache/incubator-devlake/errors"
- "github.com/apache/incubator-devlake/plugins/core/dal"
"gorm.io/gorm"
"regexp"
"strconv"
+ "github.com/apache/incubator-devlake/errors"
"github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/core/dal"
"github.com/apache/incubator-devlake/plugins/github/models"
"github.com/apache/incubator-devlake/plugins/helper"
)
@@ -99,11 +100,11 @@ func ExtractApiPrReviewComments(taskCtx core.SubTaskContext) errors.Error {
githubPrComment.AuthorUserId = prReviewComment.User.Id
githubPrComment.AuthorUsername = prReviewComment.User.Login
- githubAccount, err := convertAccount(prReviewComment.User, data.Repo.GithubId, data.Options.ConnectionId)
- if err != nil {
- return nil, err
- }
- results = append(results, githubAccount)
+ githubAccount, err := convertAccount(prReviewComment.User, data.Repo.GithubId, data.Options.ConnectionId)
+ if err != nil {
+ return nil, err
+ }
+ results = append(results, githubAccount)
}
results = append(results, githubPrComment)
@@ -127,8 +128,10 @@ func enrichGithubPrComment(data *GithubTaskData, db dal.Dal, prUrlRegex *regexp.
}
pr := &models.GithubPullRequest{}
err = db.First(pr, dal.Where("connection_id = ? and number = ? and repo_id = ?", data.Options.ConnectionId, prNumber, data.Repo.GithubId))
- if err != nil && err != gorm.ErrRecordNotFound {
- return 0, errors.NotFound.Wrap(err, "github pull request not found in DB")
+ if goerror.Is(err, gorm.ErrRecordNotFound) {
+ return 0, nil
+ } else if err != nil {
+ return 0, errors.NotFound.Wrap(err, "github pull request parse failed ")
}
return pr.GithubId, nil
}
diff --git a/services/pipeline.go b/services/pipeline.go
index 17c1a65b..54806fce 100644
--- a/services/pipeline.go
+++ b/services/pipeline.go
@@ -157,14 +157,14 @@ func RunPipelineInQueue(pipelineMaxParallel int64) {
sema := semaphore.NewWeighted(pipelineMaxParallel)
startedPipelineIds := []uint64{}
for {
- globalPipelineLog.Info("wait for new pipeline")
+ globalPipelineLog.Info("acquire lock")
// start goroutine when sema lock ready and pipeline exist.
// to avoid read old pipeline, acquire lock before read exist pipeline
err := sema.Acquire(context.TODO(), 1)
if err != nil {
panic(err)
}
- globalPipelineLog.Info("get lock and wait pipeline")
+ globalPipelineLog.Info("get lock and wait next pipeline")
dbPipeline := &models.DbPipeline{}
for {
cronLocker.Lock()
[incubator-devlake] 01/06: feat: enable converter for github_graphql
Posted by li...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
likyh pushed a commit to branch release-v0.14
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit d5e1f213f6462bc5493c3cb2c4c2344fd53722de
Author: linyh <ya...@meri.co>
AuthorDate: Wed Oct 19 19:41:12 2022 +0800
feat: enable converter for github_graphql
---
plugins/github_graphql/plugin_main.go | 16 ++++++++++++++++
1 file changed, 16 insertions(+)
diff --git a/plugins/github_graphql/plugin_main.go b/plugins/github_graphql/plugin_main.go
index edf537b3..937db7e4 100644
--- a/plugins/github_graphql/plugin_main.go
+++ b/plugins/github_graphql/plugin_main.go
@@ -71,6 +71,22 @@ func (plugin GithubGraphql) SubTaskMetas() []core.SubTaskMeta {
githubTasks.ExtractApiPrReviewCommentsMeta,
tasks.CollectAccountMeta,
+
+ githubTasks.ConvertJobsMeta,
+ githubTasks.EnrichPullRequestIssuesMeta,
+ githubTasks.ConvertRepoMeta,
+ githubTasks.ConvertIssuesMeta,
+ githubTasks.ConvertCommitsMeta,
+ githubTasks.ConvertIssueLabelsMeta,
+ githubTasks.ConvertPullRequestCommitsMeta,
+ githubTasks.ConvertPullRequestsMeta,
+ githubTasks.ConvertPullRequestReviewsMeta,
+ githubTasks.ConvertPullRequestLabelsMeta,
+ githubTasks.ConvertPullRequestIssuesMeta,
+ githubTasks.ConvertIssueCommentsMeta,
+ githubTasks.ConvertPullRequestCommentsMeta,
+ githubTasks.ConvertMilestonesMeta,
+ githubTasks.ConvertAccountsMeta,
}
}