You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by zh...@apache.org on 2022/07/18 03:10:44 UTC
[incubator-devlake] branch main updated: feat(github): collect comments/reviews (#2484)
This is an automated email from the ASF dual-hosted git repository.
zhangliang2022 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 3e59c99d feat(github): collect comments/reviews (#2484)
3e59c99d is described below
commit 3e59c99d50c6899572d683b08ede8a62acb47ebd
Author: Warren Chen <yi...@merico.dev>
AuthorDate: Mon Jul 18 11:10:40 2022 +0800
feat(github): collect comments/reviews (#2484)
* feat(github): collect comments/reviews
closes #2452
* feat(github): collect comments/reviews
closes #2452
---
generator/cmd/init_migration.go | 2 +-
models/domainlayer/code/pull_request_comment.go | 2 +
models/migrationscripts/archived/pull_request.go | 2 +
.../{init_schema.go => init_domain_schemas.go} | 10 +-
.../{init_pre_schema.go => init_lake_schemas.go} | 10 +-
models/migrationscripts/register.go | 4 +-
.../models/migrationscripts/archived/connection.go | 10 +-
plugins/github/e2e/comment_test.go | 18 +++-
plugins/github/e2e/pr_commit_test.go | 4 +-
plugins/github/e2e/pr_enrich_issue_test.go | 4 +-
.../e2e/{pr_reviewer_test.go => pr_review_test.go} | 27 ++++-
plugins/github/e2e/pr_test.go | 6 +-
...raw_github_api_pull_request_review_comments.csv | 17 +++
.../_tool_github_accounts_in_comment.csv | 2 +
.../_tool_github_accounts_in_pr.csv | 16 +++
.../_tool_github_accounts_in_review.csv | 19 ++++
.../snapshot_tables/_tool_github_milestones.csv | 4 +-
.../_tool_github_pull_request_comments.csv | 110 +++++++++++---------
.../_tool_github_pull_request_reviews.csv | 31 ++++++
plugins/github/e2e/snapshot_tables/boards.csv | 2 +-
.../e2e/snapshot_tables/pull_request_comments.csv | 94 ++++++++---------
plugins/github/impl/impl.go | 5 +-
.../models/migrationscripts/archived/account.go | 3 +
.../{pull_request_issue.go => account_org.go} | 15 ++-
.../models/migrationscripts/archived/connection.go | 15 ++-
.../{pull_request_comment.go => pr_comment.go} | 10 +-
.../{pull_request_commit.go => pr_commit.go} | 4 +-
.../{pull_request_issue.go => pr_issue.go} | 4 +-
.../{pull_request_label.go => pr_label.go} | 4 +-
.../{pull_request_comment.go => pr_review.go} | 26 ++---
.../{pull_request_issue.go => repo_account.go} | 15 ++-
.../github/models/migrationscripts/init_schema.go | 54 ++++------
plugins/github/models/migrationscripts/register.go | 2 -
.../updateSchemas20220713000003.go | 68 ------------
.../updateSchemas20220713000004.go | 50 ---------
plugins/github/models/pr_comment.go | 7 +-
plugins/github/models/pr_commit.go | 4 +-
plugins/github/models/pr_issue.go | 4 +-
plugins/github/models/pr_label.go | 4 +-
.../github/models/{pr_label.go => pr_review.go} | 22 ++--
plugins/github/tasks/comment_collector.go | 2 +-
plugins/github/tasks/comment_extractor.go | 7 +-
plugins/github/tasks/issue_collector.go | 1 -
plugins/github/tasks/issue_extractor.go | 4 +-
plugins/github/tasks/pr_comment_convertor.go | 16 +--
plugins/github/tasks/pr_commit_collector.go | 4 +-
plugins/github/tasks/pr_commit_convertor.go | 8 +-
plugins/github/tasks/pr_commit_extractor.go | 4 +-
plugins/github/tasks/pr_extractor.go | 2 +-
plugins/github/tasks/pr_issue_convertor.go | 6 +-
plugins/github/tasks/pr_issue_enricher.go | 2 +-
plugins/github/tasks/pr_label_convertor.go | 6 +-
plugins/github/tasks/pr_review_collector.go | 47 +++++++--
...collector.go => pr_review_comment_collector.go} | 90 ++++++++--------
.../github/tasks/pr_review_comment_extractor.go | 114 +++++++++++++++++++++
...comment_convertor.go => pr_review_convertor.go} | 42 ++++----
plugins/github/tasks/pr_review_extractor.go | 29 +++++-
plugins/gitlab/impl/impl.go | 4 -
plugins/helper/api_async_client.go | 2 +-
plugins/helper/api_collector.go | 2 +-
60 files changed, 644 insertions(+), 458 deletions(-)
diff --git a/generator/cmd/init_migration.go b/generator/cmd/init_migration.go
index 8bc735bc..58e2a462 100644
--- a/generator/cmd/init_migration.go
+++ b/generator/cmd/init_migration.go
@@ -63,7 +63,7 @@ Type in which plugin do you want init migrations in, then generator will create
// read template
templates := map[string]string{
- `initSchema.go`: util.ReadTemplate("generator/template/migrationscripts/init_schema.go-template"),
+ `initSchema.go`: util.ReadTemplate("generator/template/migrationscripts/init_domain_schemas.go-template"),
`register.go`: util.ReadTemplate("generator/template/migrationscripts/register.go-template"),
`archived/.gitkeep`: ``,
}
diff --git a/models/domainlayer/code/pull_request_comment.go b/models/domainlayer/code/pull_request_comment.go
index 74b1cabe..80667418 100644
--- a/models/domainlayer/code/pull_request_comment.go
+++ b/models/domainlayer/code/pull_request_comment.go
@@ -31,6 +31,8 @@ type PullRequestComment struct {
CommitSha string `gorm:"type:varchar(255)"`
Position int
Type string `gorm:"type:varchar(255)"`
+ ReviewId string `gorm:"type:varchar(255)"`
+ Status string `gorm:"type:varchar(255)"`
}
func (PullRequestComment) TableName() string {
diff --git a/models/migrationscripts/archived/pull_request.go b/models/migrationscripts/archived/pull_request.go
index d244f0f8..5fd99779 100644
--- a/models/migrationscripts/archived/pull_request.go
+++ b/models/migrationscripts/archived/pull_request.go
@@ -82,6 +82,8 @@ type PullRequestComment struct {
CommitSha string `gorm:"type:varchar(255)"`
Position int
Type string `gorm:"type:varchar(255)"`
+ ReviewId string `gorm:"type:varchar(255)"`
+ Status string `gorm:"type:varchar(255)"`
}
func (PullRequestComment) TableName() string {
diff --git a/models/migrationscripts/init_schema.go b/models/migrationscripts/init_domain_schemas.go
similarity index 92%
rename from models/migrationscripts/init_schema.go
rename to models/migrationscripts/init_domain_schemas.go
index 3c40e0b8..e0120359 100644
--- a/models/migrationscripts/init_schema.go
+++ b/models/migrationscripts/init_domain_schemas.go
@@ -24,9 +24,9 @@ import (
"gorm.io/gorm"
)
-type initSchemas struct{}
+type initDomainSchemas struct{}
-func (*initSchemas) Up(ctx context.Context, db *gorm.DB) error {
+func (*initDomainSchemas) Up(ctx context.Context, db *gorm.DB) error {
err := db.Migrator().DropTable(
"issue_assignee_history",
"issue_status_history",
@@ -106,14 +106,14 @@ func (*initSchemas) Up(ctx context.Context, db *gorm.DB) error {
)
}
-func (*initSchemas) Version() uint64 {
+func (*initDomainSchemas) Version() uint64 {
return 20220707232344
}
-func (*initSchemas) Owner() string {
+func (*initDomainSchemas) Owner() string {
return "Framework"
}
-func (*initSchemas) Name() string {
+func (*initDomainSchemas) Name() string {
return "create init schemas"
}
diff --git a/models/migrationscripts/init_pre_schema.go b/models/migrationscripts/init_lake_schemas.go
similarity index 83%
rename from models/migrationscripts/init_pre_schema.go
rename to models/migrationscripts/init_lake_schemas.go
index eacab277..b85593ca 100644
--- a/models/migrationscripts/init_pre_schema.go
+++ b/models/migrationscripts/init_lake_schemas.go
@@ -24,9 +24,9 @@ import (
"gorm.io/gorm"
)
-type initPreSchemas struct{}
+type initLakeSchemas struct{}
-func (*initPreSchemas) Up(ctx context.Context, db *gorm.DB) error {
+func (*initLakeSchemas) Up(ctx context.Context, db *gorm.DB) error {
return db.Migrator().AutoMigrate(
&archived.Task{},
&archived.Notification{},
@@ -35,14 +35,14 @@ func (*initPreSchemas) Up(ctx context.Context, db *gorm.DB) error {
)
}
-func (*initPreSchemas) Version() uint64 {
+func (*initLakeSchemas) Version() uint64 {
return 20220406212344
}
-func (*initPreSchemas) Owner() string {
+func (*initLakeSchemas) Owner() string {
return "Framework"
}
-func (*initPreSchemas) Name() string {
+func (*initLakeSchemas) Name() string {
return "create init pre-schemas"
}
diff --git a/models/migrationscripts/register.go b/models/migrationscripts/register.go
index b9fc3a6e..49b9922f 100644
--- a/models/migrationscripts/register.go
+++ b/models/migrationscripts/register.go
@@ -22,11 +22,11 @@ import "github.com/apache/incubator-devlake/migration"
// All return all the migration scripts of framework
func All() []migration.Script {
return []migration.Script{
- new(initPreSchemas),
+ new(initLakeSchemas),
new(updateSchemas20220505),
new(updateSchemas20220601),
new(updateSchemas20220616),
new(blueprintNormalMode),
- new(initSchemas),
+ new(initDomainSchemas),
}
}
diff --git a/plugins/feishu/models/migrationscripts/archived/connection.go b/plugins/feishu/models/migrationscripts/archived/connection.go
index 9715f006..b8b93923 100644
--- a/plugins/feishu/models/migrationscripts/archived/connection.go
+++ b/plugins/feishu/models/migrationscripts/archived/connection.go
@@ -23,12 +23,12 @@ import (
type FeishuConnection struct {
commonArchived.Model
- Name string `gorm:"type:varchar(100);uniqueIndex" json:"name" validate:"required"`
- Endpoint string `mapstructure:"endpoint" env:"GITHUB_ENDPOINT" validate:"required"`
- Proxy string `mapstructure:"proxy" env:"GITHUB_PROXY"`
+ Name string `gorm:"type:varchar(100);uniqueIndex" json:"name" validate:"required"`
+ Endpoint string `gorm:"type:varchar(255)"`
+ Proxy string `json:"proxy" gorm:"type:varchar(255)"`
RateLimitPerHour int `comment:"api request rate limit per hour"`
- AppId string `mapstructure:"app_id" validate:"required" json:"app_id"`
- SecretKey string `mapstructure:"secret_key" validate:"required" json:"secret_key" encrypt:"yes"`
+ AppId string `gorm:"type:varchar(255)"`
+ SecretKey string `gorm:"type:varchar(255)"`
}
func (FeishuConnection) TableName() string {
diff --git a/plugins/github/e2e/comment_test.go b/plugins/github/e2e/comment_test.go
index 7c2d935c..c9257d09 100644
--- a/plugins/github/e2e/comment_test.go
+++ b/plugins/github/e2e/comment_test.go
@@ -59,15 +59,17 @@ func TestCommentDataFlow(t *testing.T) {
// import raw data table
dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_github_api_comments.csv", "_raw_github_api_comments")
+ dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_github_api_pull_request_review_comments.csv", "_raw_github_api_pull_request_review_comments")
dataflowTester.ImportCsvIntoTabler("./snapshot_tables/_tool_github_issues.csv", &models.GithubIssue{})
dataflowTester.ImportCsvIntoTabler("./snapshot_tables/_tool_github_issue_labels.csv", &models.GithubIssueLabel{})
dataflowTester.ImportCsvIntoTabler("./snapshot_tables/_tool_github_pull_requests.csv", models.GithubPullRequest{})
// verify extraction
dataflowTester.FlushTabler(&models.GithubIssueComment{})
- dataflowTester.FlushTabler(&models.GithubPullRequestComment{})
+ dataflowTester.FlushTabler(&models.GithubPrComment{})
dataflowTester.FlushTabler(&models.GithubRepoAccount{})
dataflowTester.Subtask(tasks.ExtractApiCommentsMeta, taskData)
+ dataflowTester.Subtask(tasks.ExtractApiPrReviewCommentsMeta, taskData)
dataflowTester.VerifyTable(
models.GithubIssueComment{},
fmt.Sprintf("./snapshot_tables/%s.csv", models.GithubIssueComment{}.TableName()),
@@ -87,8 +89,8 @@ func TestCommentDataFlow(t *testing.T) {
},
)
dataflowTester.VerifyTable(
- models.GithubPullRequestComment{},
- fmt.Sprintf("./snapshot_tables/%s.csv", models.GithubPullRequestComment{}.TableName()),
+ models.GithubPrComment{},
+ fmt.Sprintf("./snapshot_tables/%s.csv", models.GithubPrComment{}.TableName()),
[]string{
"connection_id",
"github_id",
@@ -96,8 +98,11 @@ func TestCommentDataFlow(t *testing.T) {
"body",
"author_username",
"author_user_id",
+ "commit_sha",
"github_created_at",
"github_updated_at",
+ "review_id",
+ "type",
"_raw_data_params",
"_raw_data_table",
"_raw_data_id",
@@ -138,12 +143,19 @@ func TestCommentDataFlow(t *testing.T) {
fmt.Sprintf("./snapshot_tables/%s.csv", code.PullRequestComment{}.TableName()),
[]string{
"id",
+ "_raw_data_params",
+ "_raw_data_table",
+ "_raw_data_id",
+ "_raw_data_remark",
"pull_request_id",
"body",
"user_id",
"created_date",
"commit_sha",
"position",
+ "type",
+ "review_id",
+ "status",
},
)
}
diff --git a/plugins/github/e2e/pr_commit_test.go b/plugins/github/e2e/pr_commit_test.go
index fbd7307d..1f064c06 100644
--- a/plugins/github/e2e/pr_commit_test.go
+++ b/plugins/github/e2e/pr_commit_test.go
@@ -49,7 +49,7 @@ func TestPrCommitDataFlow(t *testing.T) {
// verify extraction
dataflowTester.FlushTabler(&models.GithubCommit{})
- dataflowTester.FlushTabler(&models.GithubPullRequestCommit{})
+ dataflowTester.FlushTabler(&models.GithubPrCommit{})
dataflowTester.Subtask(tasks.ExtractApiPullRequestCommitsMeta, taskData)
dataflowTester.VerifyTable(
models.GithubCommit{},
@@ -76,7 +76,7 @@ func TestPrCommitDataFlow(t *testing.T) {
)
dataflowTester.VerifyTable(
- models.GithubPullRequestCommit{},
+ models.GithubPrCommit{},
"./snapshot_tables/_tool_github_pull_request_commits.csv",
[]string{
"connection_id",
diff --git a/plugins/github/e2e/pr_enrich_issue_test.go b/plugins/github/e2e/pr_enrich_issue_test.go
index c63d73fa..577ee812 100644
--- a/plugins/github/e2e/pr_enrich_issue_test.go
+++ b/plugins/github/e2e/pr_enrich_issue_test.go
@@ -59,10 +59,10 @@ func TestPrEnrichIssueDataFlow(t *testing.T) {
dataflowTester.ImportCsvIntoTabler("./snapshot_tables/_tool_github_pull_requests.csv", models.GithubPullRequest{})
// verify extraction
- dataflowTester.FlushTabler(&models.GithubPullRequestIssue{})
+ dataflowTester.FlushTabler(&models.GithubPrIssue{})
dataflowTester.Subtask(tasks.EnrichPullRequestIssuesMeta, taskData)
dataflowTester.VerifyTable(
- models.GithubPullRequestIssue{},
+ models.GithubPrIssue{},
"./snapshot_tables/_tool_github_pull_request_issues.csv",
[]string{
"connection_id",
diff --git a/plugins/github/e2e/pr_reviewer_test.go b/plugins/github/e2e/pr_review_test.go
similarity index 77%
rename from plugins/github/e2e/pr_reviewer_test.go
rename to plugins/github/e2e/pr_review_test.go
index c2742802..e38c2bb1 100644
--- a/plugins/github/e2e/pr_reviewer_test.go
+++ b/plugins/github/e2e/pr_review_test.go
@@ -18,6 +18,7 @@ limitations under the License.
package e2e
import (
+ "fmt"
"testing"
"github.com/apache/incubator-devlake/plugins/github/models"
@@ -27,7 +28,7 @@ import (
"github.com/apache/incubator-devlake/plugins/github/tasks"
)
-func TestPrReviewerDataFlow(t *testing.T) {
+func TestPrReviewDataFlow(t *testing.T) {
var plugin impl.Github
dataflowTester := e2ehelper.NewDataFlowTester(t, "github", plugin)
@@ -49,8 +50,28 @@ func TestPrReviewerDataFlow(t *testing.T) {
// verify extraction
dataflowTester.FlushTabler(&models.GithubReviewer{})
- dataflowTester.FlushTabler(&models.GithubRepoAccount{})
- dataflowTester.Subtask(tasks.ExtractApiPullRequestReviewersMeta, taskData)
+ dataflowTester.FlushTabler(&models.GithubAccount{})
+ dataflowTester.FlushTabler(&models.GithubPrReview{})
+ dataflowTester.Subtask(tasks.ExtractApiPullRequestReviewsMeta, taskData)
+ dataflowTester.VerifyTable(
+ models.GithubPrReview{},
+ fmt.Sprintf("./snapshot_tables/%s.csv", models.GithubPrReview{}.TableName()),
+ []string{
+ "connection_id",
+ "github_id",
+ "pull_request_id",
+ "body",
+ "author_username",
+ "author_user_id",
+ "github_submit_at",
+ "commit_sha",
+ "state",
+ "_raw_data_params",
+ "_raw_data_table",
+ "_raw_data_id",
+ "_raw_data_remark",
+ },
+ )
dataflowTester.VerifyTable(
models.GithubReviewer{},
"./snapshot_tables/_tool_github_reviewers.csv",
diff --git a/plugins/github/e2e/pr_test.go b/plugins/github/e2e/pr_test.go
index 7a75691f..b6bc2b7c 100644
--- a/plugins/github/e2e/pr_test.go
+++ b/plugins/github/e2e/pr_test.go
@@ -54,8 +54,8 @@ func TestPrDataFlow(t *testing.T) {
// verify pr extraction
dataflowTester.FlushTabler(&models.GithubPullRequest{})
- dataflowTester.FlushTabler(&models.GithubPullRequestLabel{})
- dataflowTester.FlushTabler(&models.GithubRepoAccount{})
+ dataflowTester.FlushTabler(&models.GithubPrLabel{})
+ dataflowTester.FlushTabler(&models.GithubAccount{})
dataflowTester.Subtask(tasks.ExtractApiPullRequestsMeta, taskData)
dataflowTester.VerifyTable(
models.GithubPullRequest{},
@@ -96,7 +96,7 @@ func TestPrDataFlow(t *testing.T) {
)
dataflowTester.VerifyTable(
- models.GithubPullRequestLabel{},
+ models.GithubPrLabel{},
"./snapshot_tables/_tool_github_pull_request_labels.csv",
[]string{"connection_id", "pull_id", "label_name"},
)
diff --git a/plugins/github/e2e/raw_tables/_raw_github_api_pull_request_review_comments.csv b/plugins/github/e2e/raw_tables/_raw_github_api_pull_request_review_comments.csv
new file mode 100644
index 00000000..5d4fc661
--- /dev/null
+++ b/plugins/github/e2e/raw_tables/_raw_github_api_pull_request_review_comments.csv
@@ -0,0 +1,17 @@
+id,params,data,url,input,created_at
+1,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/pulls/comments/300245643"",""pull_request_review_id"":257849164,""id"":300245643,""node_id"":""MDI0OlB1bGxSZXF1ZXN0UmV2aWV3Q29tbWVudDMwMDI0NTY0Mw=="",""diff_hunk"":""@@ -46,27 +45,28 @@ type Worker struct {\n func (w *Worker) run() {\n \tw.pool.incRunning()\n \tgo func() {\n-\t\tdefer func() {\n-\t\t\tif p := recover(); p != nil {\n-\t\t\t\tw.pool.decRunning()\n-\t\t [...]
+2,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/pulls/comments/300246398"",""pull_request_review_id"":257849164,""id"":300246398,""node_id"":""MDI0OlB1bGxSZXF1ZXN0UmV2aWV3Q29tbWVudDMwMDI0NjM5OA=="",""diff_hunk"":""@@ -46,27 +45,28 @@ type Worker struct {\n func (w *Worker) run() {\n \tw.pool.incRunning()\n \tgo func() {\n-\t\tdefer func() {\n-\t\t\tif p := recover(); p != nil {\n-\t\t\t\tw.pool.decRunning()\n-\t\t [...]
+3,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/pulls/comments/307667007"",""pull_request_review_id"":267089136,""id"":307667007,""node_id"":""MDI0OlB1bGxSZXF1ZXN0UmV2aWV3Q29tbWVudDMwNzY2NzAwNw=="",""diff_hunk"":""@@ -33,7 +33,7 @@ const (\n \tDEFAULT_ANTS_POOL_SIZE = math.MaxInt32\n \n \t// DEFAULT_CLEAN_INTERVAL_TIME is the interval time to clean up goroutines.\n-\tDEFAULT_CLEAN_INTERVAL_TIME = 1\n+\tDEFAULT_CLE [...]
+4,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/pulls/comments/307667451"",""pull_request_review_id"":267089136,""id"":307667451,""node_id"":""MDI0OlB1bGxSZXF1ZXN0UmV2aWV3Q29tbWVudDMwNzY2NzQ1MQ=="",""diff_hunk"":""@@ -268,13 +268,13 @@ func TestPurge(t *testing.T) {\n }\n \n func TestRestCodeCoverage(t *testing.T) {"",""path"":""ants_test.go"",""position"":188,""original_position"":3,""commit_id"":""5dc8b9a71737eb [...]
+5,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/pulls/comments/307669178"",""pull_request_review_id"":267089136,""id"":307669178,""node_id"":""MDI0OlB1bGxSZXF1ZXN0UmV2aWV3Q29tbWVudDMwNzY2OTE3OA=="",""diff_hunk"":""@@ -24,6 +24,7 @@ package ants\n \n import ("",""path"":""worker.go"",""position"":2,""original_position"":2,""commit_id"":""5dc8b9a71737eb57dc03fbbe3eb9010ff6c3fbb6"",""original_commit_id"":""807ff757e5 [...]
+6,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/pulls/comments/307671676"",""pull_request_review_id"":267094793,""id"":307671676,""node_id"":""MDI0OlB1bGxSZXF1ZXN0UmV2aWV3Q29tbWVudDMwNzY3MTY3Ng=="",""diff_hunk"":""@@ -24,6 +24,7 @@ package ants\n \n import ("",""path"":""worker.go"",""position"":2,""original_position"":2,""commit_id"":""5dc8b9a71737eb57dc03fbbe3eb9010ff6c3fbb6"",""original_commit_id"":""807ff757e5 [...]
+7,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/pulls/comments/307672146"",""pull_request_review_id"":267095424,""id"":307672146,""node_id"":""MDI0OlB1bGxSZXF1ZXN0UmV2aWV3Q29tbWVudDMwNzY3MjE0Ng=="",""diff_hunk"":""@@ -268,13 +268,13 @@ func TestPurge(t *testing.T) {\n }\n \n func TestRestCodeCoverage(t *testing.T) {"",""path"":""ants_test.go"",""position"":188,""original_position"":3,""commit_id"":""5dc8b9a71737eb [...]
+8,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/pulls/comments/307672417"",""pull_request_review_id"":267095785,""id"":307672417,""node_id"":""MDI0OlB1bGxSZXF1ZXN0UmV2aWV3Q29tbWVudDMwNzY3MjQxNw=="",""diff_hunk"":""@@ -33,7 +33,7 @@ const (\n \tDEFAULT_ANTS_POOL_SIZE = math.MaxInt32\n \n \t// DEFAULT_CLEAN_INTERVAL_TIME is the interval time to clean up goroutines.\n-\tDEFAULT_CLEAN_INTERVAL_TIME = 1\n+\tDEFAULT_CLE [...]
+9,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/pulls/comments/307675200"",""pull_request_review_id"":267099413,""id"":307675200,""node_id"":""MDI0OlB1bGxSZXF1ZXN0UmV2aWV3Q29tbWVudDMwNzY3NTIwMA=="",""diff_hunk"":""@@ -33,7 +33,7 @@ const (\n \tDEFAULT_ANTS_POOL_SIZE = math.MaxInt32\n \n \t// DEFAULT_CLEAN_INTERVAL_TIME is the interval time to clean up goroutines.\n-\tDEFAULT_CLEAN_INTERVAL_TIME = 1\n+\tDEFAULT_CLE [...]
+10,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/pulls/comments/307686223"",""pull_request_review_id"":267113847,""id"":307686223,""node_id"":""MDI0OlB1bGxSZXF1ZXN0UmV2aWV3Q29tbWVudDMwNzY4NjIyMw=="",""diff_hunk"":""@@ -33,7 +33,7 @@ const (\n \tDEFAULT_ANTS_POOL_SIZE = math.MaxInt32\n \n \t// DEFAULT_CLEAN_INTERVAL_TIME is the interval time to clean up goroutines.\n-\tDEFAULT_CLEAN_INTERVAL_TIME = 1\n+\tDEFAULT_CL [...]
+11,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/pulls/comments/315493235"",""pull_request_review_id"":276914383,""id"":315493235,""node_id"":""MDI0OlB1bGxSZXF1ZXN0UmV2aWV3Q29tbWVudDMxNTQ5MzIzNQ=="",""diff_hunk"":""@@ -416,6 +416,78 @@ func TestPurgePreMalloc(t *testing.T) {\n \t}\n }\n \n+func TestNonblockingSubmit(t *testing.T) {\n+\tpoolSize := 10\n+\tp, err := ants.NewPool(poolSize)\n+\tif err != nil {\n+\t\tt [...]
+12,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/pulls/comments/315497949"",""pull_request_review_id"":276920486,""id"":315497949,""node_id"":""MDI0OlB1bGxSZXF1ZXN0UmV2aWV3Q29tbWVudDMxNTQ5Nzk0OQ=="",""diff_hunk"":""@@ -236,8 +253,18 @@ func (p *Pool) retrieveWorker() *Worker {\n \t\tp.lock.Unlock()\n \t\tspawnWorker()\n \t} else {\n+\t\tif p.NonblockingSubmit {\n+\t\t\tp.lock.Unlock()\n+\t\t\treturn nil\n+\t\t}\n [...]
+13,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/pulls/comments/315498393"",""pull_request_review_id"":276920486,""id"":315498393,""node_id"":""MDI0OlB1bGxSZXF1ZXN0UmV2aWV3Q29tbWVudDMxNTQ5ODM5Mw=="",""diff_hunk"":""@@ -151,7 +164,11 @@ func (p *Pool) Submit(task func()) error {\n \tif atomic.LoadInt32(&p.release) == CLOSED {\n \t\treturn ErrPoolClosed\n \t}\n-\tp.retrieveWorker().task <- task\n+\tw := p.retrieveWo [...]
+14,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/pulls/comments/315498823"",""pull_request_review_id"":276920486,""id"":315498823,""node_id"":""MDI0OlB1bGxSZXF1ZXN0UmV2aWV3Q29tbWVudDMxNTQ5ODgyMw=="",""diff_hunk"":""@@ -51,6 +51,9 @@ var (\n \n \t// ErrPoolClosed will be returned when submitting task to a closed pool.\n \tErrPoolClosed = errors.New(\""this pool has been closed\"")\n+\n+\t// ErrPoolBlocked will be r [...]
+15,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/pulls/comments/315499067"",""pull_request_review_id"":276920486,""id"":315499067,""node_id"":""MDI0OlB1bGxSZXF1ZXN0UmV2aWV3Q29tbWVudDMxNTQ5OTA2Nw=="",""diff_hunk"":""@@ -60,6 +60,19 @@ type Pool struct {\n \t// PanicHandler is used to handle panics from each worker goroutine.\n \t// if nil, panics will be thrown out again from worker goroutines.\n \tPanicHandler fun [...]
+16,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/pulls/comments/315499281"",""pull_request_review_id"":276920486,""id"":315499281,""node_id"":""MDI0OlB1bGxSZXF1ZXN0UmV2aWV3Q29tbWVudDMxNTQ5OTI4MQ=="",""diff_hunk"":""@@ -60,6 +60,19 @@ type Pool struct {\n \t// PanicHandler is used to handle panics from each worker goroutine.\n \t// if nil, panics will be thrown out again from worker goroutines.\n \tPanicHandler fun [...]
\ No newline at end of file
diff --git a/plugins/github/e2e/snapshot_tables/_tool_github_accounts_in_comment.csv b/plugins/github/e2e/snapshot_tables/_tool_github_accounts_in_comment.csv
index 3300ad15..39634bc0 100644
--- a/plugins/github/e2e/snapshot_tables/_tool_github_accounts_in_comment.csv
+++ b/plugins/github/e2e/snapshot_tables/_tool_github_accounts_in_comment.csv
@@ -11,8 +11,10 @@ connection_id,account_id,repo_github_id,login
1,12890888,134018330,liyonglion
1,13118848,134018330,lovelly
1,16658738,134018330,barryz
+1,17495446,134018330,Anteoy
1,22429695,134018330,codecov[bot]
1,29241786,134018330,ylwangs
1,29243953,134018330,naiba
1,29452204,134018330,edcismybrother
1,32898629,134018330,hawklin2017
+1,42910462,134018330,golangcibot
diff --git a/plugins/github/e2e/snapshot_tables/_tool_github_accounts_in_pr.csv b/plugins/github/e2e/snapshot_tables/_tool_github_accounts_in_pr.csv
index 173e342e..7d37bc1d 100644
--- a/plugins/github/e2e/snapshot_tables/_tool_github_accounts_in_pr.csv
+++ b/plugins/github/e2e/snapshot_tables/_tool_github_accounts_in_pr.csv
@@ -2,27 +2,39 @@ connection_id,account_id,repo_github_id,login
1,12317,134018330,jdamick
1,21979,134018330,appleboy
1,192964,134018330,egonelbre
+1,720086,134018330,huangjunwen
1,964542,134018330,sarathsp06
+1,1284892,134018330,jjeffcaii
1,1290360,134018330,wwjiang
1,1940588,134018330,bingoohuang
1,2813260,134018330,KevinBaiSg
1,2832687,134018330,kaiiak
1,3794113,134018330,shanhuhai5739
1,3814966,134018330,lilien1010
+1,4555057,134018330,huiwq1990
+1,5044825,134018330,tenfyzhong
1,5227289,134018330,lntotk
+1,5244267,134018330,jiashiwen
1,5268051,134018330,lord63
+1,5668717,134018330,kklinan
1,5715152,134018330,RealLiuSha
1,7411249,134018330,imxyb
1,7496278,134018330,panjf2000
1,7931755,134018330,zplzpl
+1,8509898,134018330,prprprus
1,8518239,134018330,gitter-badger
1,8597823,134018330,hongli-my
+1,8605102,134018330,pathbox
1,8644923,134018330,polar9527
1,8923413,134018330,choleraehyq
+1,10361713,134018330,Ainiroad
+1,11763614,134018330,Moonlight-Zhao
1,11977524,134018330,arjunmahishi
1,12420699,134018330,shanghai-Jerry
1,12890888,134018330,liyonglion
1,12907474,134018330,wreulicke
+1,13118848,134018330,lovelly
+1,13944100,134018330,LinuxForYQH
1,15234973,134018330,lucafmarques
1,15699766,134018330,TwiN
1,16658738,134018330,barryz
@@ -34,13 +46,17 @@ connection_id,account_id,repo_github_id,login
1,22676124,134018330,liu-song
1,24841832,134018330,rikewang
1,27898261,134018330,qmdx00
+1,29243953,134018330,naiba
+1,29452204,134018330,edcismybrother
1,29589055,134018330,Mutated1994
1,32893410,134018330,zhangyuanxue
1,35493957,134018330,codingfanlt
+1,38367404,134018330,liliang8858
1,38849208,134018330,king526
1,41562937,134018330,yddeng
1,42808204,134018330,Nonnnnnnnnn
1,47921612,134018330,Comolli
+1,48135919,134018330,tsatke
1,49174849,134018330,thinkgos
1,58211133,134018330,icecube092
1,64823610,134018330,bright2227
diff --git a/plugins/github/e2e/snapshot_tables/_tool_github_accounts_in_review.csv b/plugins/github/e2e/snapshot_tables/_tool_github_accounts_in_review.csv
index 3b40ebbf..cfc55e1a 100644
--- a/plugins/github/e2e/snapshot_tables/_tool_github_accounts_in_review.csv
+++ b/plugins/github/e2e/snapshot_tables/_tool_github_accounts_in_review.csv
@@ -1,4 +1,23 @@
connection_id,account_id,repo_github_id,login
+1,720086,134018330,huangjunwen
+1,1284892,134018330,jjeffcaii
+1,1290360,134018330,wwjiang
1,2813260,134018330,KevinBaiSg
+1,4555057,134018330,huiwq1990
+1,5044825,134018330,tenfyzhong
+1,5244267,134018330,jiashiwen
+1,5668717,134018330,kklinan
1,7496278,134018330,panjf2000
+1,7931755,134018330,zplzpl
+1,8509898,134018330,prprprus
+1,8605102,134018330,pathbox
1,8923413,134018330,choleraehyq
+1,10361713,134018330,Ainiroad
+1,11763614,134018330,Moonlight-Zhao
+1,13118848,134018330,lovelly
+1,13944100,134018330,LinuxForYQH
+1,29243953,134018330,naiba
+1,29452204,134018330,edcismybrother
+1,38367404,134018330,liliang8858
+1,38849208,134018330,king526
+1,48135919,134018330,tsatke
diff --git a/plugins/github/e2e/snapshot_tables/_tool_github_milestones.csv b/plugins/github/e2e/snapshot_tables/_tool_github_milestones.csv
index 75dfe942..03eeeb06 100644
--- a/plugins/github/e2e/snapshot_tables/_tool_github_milestones.csv
+++ b/plugins/github/e2e/snapshot_tables/_tool_github_milestones.csv
@@ -1,2 +1,2 @@
-connection_id,milestone_id,repo_id,number,url,title,open_issues,closed_issues,state,created_at,updated_at,closed_at,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
-1,7856149,134018330,7,https://api.github.com/repos/apache/incubator-devlake/milestones/7,v0.11.0,2,118,open,2022-04-08T02:05:35.000+00:00,2022-06-24T01:34:37.000+00:00,,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_milestones,109,
+connection_id,milestone_id,repo_id,number,url,title,open_issues,closed_issues,state,closed_at
+1,7856149,134018330,7,https://api.github.com/repos/apache/incubator-devlake/milestones/7,v0.11.0,2,118,open,
diff --git a/plugins/github/e2e/snapshot_tables/_tool_github_pull_request_comments.csv b/plugins/github/e2e/snapshot_tables/_tool_github_pull_request_comments.csv
index f66d21c2..ba942f34 100644
--- a/plugins/github/e2e/snapshot_tables/_tool_github_pull_request_comments.csv
+++ b/plugins/github/e2e/snapshot_tables/_tool_github_pull_request_comments.csv
@@ -1,47 +1,63 @@
-connection_id,github_id,pull_request_id,body,author_username,author_user_id,github_created_at,github_updated_at,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
-1,407675431,203756736,"""# [Codecov](https://codecov.io/gh/panjf2000/ants/pull/4?src=pr&el=h1) Report\n> Merging [#4](https://codecov.io/gh/panjf2000/ants/pull/4?src=pr&el=desc) into [master](https://codecov.io/gh/panjf2000/ants/commit/f5b37d0798a8e4c6780a1e08270fa50e979aa1d7?src=pr&el=desc) will **not change** coverage.\n> The diff coverage is `100%`.\n\n[ [...]
-1,407714136,203756736,"""@barryz 改的不错，不过有一点需要商榷，那个预分配的大小，之前benchmark我记得1000w也只需要用到7w多，这样的话预分配5w对于大部分场景其实有点浪费，你有没有用修改后的代码benchmark一下，每次allocs/op应该是变大了吧""",panjf2000,7496278,2018-07-25T10:49:44.000+00:00,2018-07-25T10:49:44.000+00:00,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3163,
-1,407755528,203756736,"""@panjf2000 因为从压测结果来看， 1000W的场景下goroutine池的作用很明显， 尤其是在节省内存开销方面，所以在选择默认worker数量时，取了一个相对较大的值。 下面是benchmark 对比\r\n\r\nmaster:\r\n\r\n```\r\ngoos: linux\r\ngoarch: amd64\r\nBenchmarkGoroutineWithFunc-4 1 15079030287 ns/op 723120296 B/op 10176159 allocs/op\r\nPASS\r\nok command-line-arguments 15.106s\r\n```\r\n\r\nPR:\r\n```\r\ngoos: linux\r\ngoarch: amd64\r\nBenchmarkGoroutineWithFunc-4 1 15443376244 ns/op 72096288 [...]
-1,407764945,203756736,"""@barryz 这样看起来预分配似乎没有太大的内存优势，相反在其他数量的任务量场景下可能还会有点浪费，这样吧，要不你先把预分配内存这一块的暂时移除，然后我合一下优化代码的部分，至于预分配内存这一块，后续再继续讨论下看看有没有能兼顾的办法""",panjf2000,7496278,2018-07-25T14:02:41.000+00:00,2018-07-25T14:02:41.000+00:00,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3165,
-1,416794440,211603583,"""@hongli-my 不好意思，我没太懂你的意图？能麻烦说详细点吗？""",panjf2000,7496278,2018-08-29T01:39:23.000+00:00,2018-08-29T01:39:23.000+00:00,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3180,
-1,416794871,211603583,"""# [Codecov](https://codecov.io/gh/panjf2000/ants/pull/8?src=pr&el=h1) Report\n> Merging [#8](https://codecov.io/gh/panjf2000/ants/pull/8?src=pr&el=desc) into [master](https://codecov.io/gh/panjf2000/ants/commit/666635c65d8d3bb1223b819325e0bd23c81f2733?src=pr&el=desc) will **decrease** coverage by `1.24%`.\n> The diff coverage is `100%`.\n\n[![Impacted file tree graph](https://codecov.io/gh/panjf2000/ants/pull/8/graphs/tree.svg?width=650&token=JFa3laaGKq&height=15 [...]
-1,416803651,211603583,"""```n := 0\r\n\t\tfor i, w := range idleWorkers {\r\n\t\t\tif currentTime.Sub(w.recycleTime) <= p.expiryDuration {\r\n\t\t\t\tbreak\r\n\t\t\t}\r\n\t\t\tn = i\r\n\t\t\tw.args <- nil\r\n\t\t\tidleWorkers[i] = nil\r\n\t\t}\r\n\t\tn++\r\n\t\tif n >= len(idleWorkers) {\r\n\t\t\tp.workers = idleWorkers[:0]\r\n\t\t} else {\r\n\t\t\tp.workers = idleWorkers[n:]\r\n\t\t}```\r\nfor 循环中，如果for 循环没有满足条件，n 并没有复制，此时n ++, n 变为1， p.workers = idleWorkers[1:], idleWorkers[0] 丢弃了？ [...]
-1,416818094,211603583,"""@hongli-my 嗯，我知道了，但是你的修改似乎不完整，我已经在develop分支改了一版，要不你看下那个分支，然后按照那个再改一下，然后我再merge""",panjf2000,7496278,2018-08-29T04:11:17.000+00:00,2018-08-29T04:11:17.000+00:00,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3183,
-1,421884048,211603583,"""对于worker中的chan应该也需要close吧，当协程数量增大时，这种也是一种消耗""",ylwangs,29241786,2018-09-17T03:37:44.000+00:00,2018-09-17T03:37:44.000+00:00,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3188,
-1,425423544,218939809,"""@liyonglion 我看修改的代码应该是正确的，但是有两个问题：\r\n1. 要正确测试你pr，你要把ants_test.go里import ants的路径改成你自己的路径；\r\n2. 你现在只修改了pool.go的代码，麻烦把pool_func.go里相应的地方也优化一下。\r\n\r\n你重新修改一下pr，然后通过测试，我就merge了。""",panjf2000,7496278,2018-09-28T12:43:30.000+00:00,2018-09-28T12:44:42.000+00:00,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3199,
-1,425428914,218939809,"""我又仔细地想了想，在putWorker方法中，调用p.cond.Signal()之前，p.lock已经解锁了，如果这时候被\r\n\r\n这里获得锁，则空闲队列中的刚刚放入的worker会被取出，这时候p.cond.Wait()再获得锁去队列中取worker的时候就会取不到，len(p.workers) - 1就会是-1，会报错：index out of range，和https://github.com/panjf2000/ants/issues/6 相似的问题，所以p.cond.Signal()应该放在putWorker中的p.lock.Lock()和p.lock.Unlock()之间。\r\n""",panjf2000,7496278,2018-09-28T13:03:48.000+00:00,201 [...]
-1,425431081,218939809,"""我明天仔细研究下。如果这个地方的死锁不好解决，其实可以使用另外一把锁，专门给条件变量使用。""",liyonglion,12890888,2018-09-28T13:12:08.000+00:00,2018-09-28T13:12:08.000+00:00,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3201,
-1,425432498,218939809,"""单一操作变量最好不要有多个锁，这样引发死锁的概率会非常高。""",panjf2000,7496278,2018-09-28T13:17:44.000+00:00,2018-09-28T13:17:57.000+00:00,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3202,
-1,425438107,218939809,"""新的锁只给条件变量使用，其他任何地方都不会使用。如果怕死锁，明天我封装一层，pool层只有一把锁。""",liyonglion,12890888,2018-09-28T13:37:48.000+00:00,2018-09-28T13:37:48.000+00:00,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3203,
-1,425439137,218939809,"""封装一层后，按照你的思路修改应该就可以了。""",liyonglion,12890888,2018-09-28T13:41:20.000+00:00,2018-09-28T13:41:20.000+00:00,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3204,
-1,425608265,218939809,"""看了下代码，你说的没错。我在测试下性能方面""",liyonglion,12890888,2018-09-29T02:22:39.000+00:00,2018-09-29T02:22:39.000+00:00,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3205,
-1,425609411,218939809,"""\r\n你这改的有问题，取出来后没有对workers队列缩容，现在Travis CI整个卡住了""",panjf2000,7496278,2018-09-29T02:41:56.000+00:00,2018-09-29T02:43:26.000+00:00,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3206,
-1,425609608,218939809,"""嗯，到时候我会补上。容我再想想""",liyonglion,12890888,2018-09-29T02:45:23.000+00:00,2018-09-29T02:45:23.000+00:00,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3207,
-1,425622339,218939809,"""性能测试如下：\r\n100w的并发，5w的goroutine\r\n\r\n\r\n\r\n""",liyonglion,12890888,2018-09-29T07:02:56.000+00:00,2018-09-29T07:02:56.000+00:00,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3208,
-1,425622437,218939809,"""# [Codecov](https://codecov.io/gh/panjf2000/ants/pull/13?src=pr&el=h1) Report\n> Merging [#13](https://codecov.io/gh/panjf2000/ants/pull/13?src=pr&el=desc) into [master](https://codecov.io/gh/panjf2000/ants/commit/af376f1b7b59dc488458bcecd4273f0fcde33c55?src=pr&el=desc) will **decrease** coverage by `90.38%`.\n> The diff coverage is `5%`.\n\n[![Impacted file tree graph](https://codecov.io/gh/panjf2000/ants/pull/13/graphs/tree.svg?width=650&token=JFa3laaGKq&height [...]
-1,425623081,218939809,"""@panjf2000 这个覆盖率怎么看？""",liyonglion,12890888,2018-09-29T07:16:02.000+00:00,2018-09-29T07:16:02.000+00:00,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3210,
-1,425632550,218939809,"""怎么会有负数。。。""",panjf2000,7496278,2018-09-29T09:53:47.000+00:00,2018-09-29T09:53:47.000+00:00,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3211,
-1,425632613,218939809,"""而且你那个取出worker之后还是没有对队列进行缩容吧""",panjf2000,7496278,2018-09-29T09:54:48.000+00:00,2018-09-29T09:54:48.000+00:00,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3212,
-1,425632632,218939809,"""这个我没玩过。。。。""",liyonglion,12890888,2018-09-29T09:55:07.000+00:00,2018-09-29T09:55:07.000+00:00,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3213,
-1,425632777,218939809,"""@panjf2000 我缩容了，具体哪个地方？""",liyonglion,12890888,2018-09-29T09:57:34.000+00:00,2018-09-29T09:57:34.000+00:00,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3214,
-1,425637776,218939809,"""@liyonglion 我看错了，缩容的补上了，这块没什么问题了，关于覆盖率这块我明天再查查。。。""",panjf2000,7496278,2018-09-29T11:25:56.000+00:00,2018-09-29T11:25:56.000+00:00,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3215,
-1,425637938,218939809,"""哦，我知道了，应该是那个import路径的问题导致的，那我先merge一下代码""",panjf2000,7496278,2018-09-29T11:29:28.000+00:00,2018-09-29T11:29:28.000+00:00,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3216,
-1,425896318,219363161,"""# [Codecov](https://codecov.io/gh/panjf2000/ants/pull/14?src=pr&el=h1) Report\n> Merging [#14](https://codecov.io/gh/panjf2000/ants/pull/14?src=pr&el=desc) into [master](https://codecov.io/gh/panjf2000/ants/commit/1b62696050b7030106291980d5220f886b017eff?src=pr&el=desc) will **decrease** coverage by `2.12%`.\n> The diff coverage is `100%`.\n\n[![Impacted file tree graph](https://codecov.io/gh/panjf2000/ants/pull/14/graphs/tree.svg?width=650&token=JFa3laaGKq&heigh [...]
-1,425906970,219363161,"""thanks for your correction and could you also modify the pool_func.go cuz there is also the same useless statement in that file. i will accept this pr after your modification is done, thanks.""",panjf2000,7496278,2018-10-01T13:27:04.000+00:00,2018-10-02T13:52:03.000+00:00,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3218,
-1,426537013,219936521,"""# [Codecov](https://codecov.io/gh/panjf2000/ants/pull/15?src=pr&el=h1) Report\n> Merging [#15](https://codecov.io/gh/panjf2000/ants/pull/15?src=pr&el=desc) into [master](https://codecov.io/gh/panjf2000/ants/commit/29730bb70343924a2f56a13a9799611dd1cd27fd?src=pr&el=desc) will **decrease** coverage by `0.84%`.\n> The diff coverage is `n/a`.\n\n[![Impacted file tree graph](https://codecov.io/gh/panjf2000/ants/pull/15/graphs/tree.svg?width=650&token=JFa3laaGKq&height [...]
-1,426596957,219936521,"""@egonelbre thanks for your interest in this repository and i got some puzzles.\r\nfirstly, i don't understand why you use both the chan struct{} and sync.WaitGroup at once, that seems a bit excessive cuz waitgroup and chan did the same thing.\r\nsecondly, method BenchmarkGoroutine was design to test throughput so it should run without waitgroup.""",panjf2000,7496278,2018-10-03T11:05:45.000+00:00,2018-10-03T12:02:17.000+00:00,"{""ConnectionId"":1,""Owner"":""panjf [...]
-1,426604764,219936521,"""Hi, semaphore acts as protection to avoid starting too many goroutines at once. `WaitGroup` is used to wait for everything to complete.\r\n\r\n`BenchmarkGoroutine` doesn't wait for all the goroutines to finish, so you are testing goroutine startup speed rather than throughput. It's somewhat unclear what you mean by \""throughput\"" in this case, but either way you would need to wait for all goroutines to finish, otherwise they will start affecting all other bench [...]
-1,429609579,222703171,"""# [Codecov](https://codecov.io/gh/panjf2000/ants/pull/16?src=pr&el=h1) Report\n> Merging [#16](https://codecov.io/gh/panjf2000/ants/pull/16?src=pr&el=desc) into [master](https://codecov.io/gh/panjf2000/ants/commit/711dbdb7a222771ce15aaee1bb7b7c6e9731f208?src=pr&el=desc) will **decrease** coverage by `2.03%`.\n> The diff coverage is `0%`.\n\n[![Impacted file tree graph](https://codecov.io/gh/panjf2000/ants/pull/16/graphs/tree.svg?width=650&token=JFa3laaGKq&height= [...]
-1,433821356,211603583,"""这种bugfix是不是应该赶紧修了？""",choleraehyq,8923413,2018-10-29T08:10:59.000+00:00,2018-10-29T08:10:59.000+00:00,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3223,
-1,434126048,211603583,"""@choleraehyq 这个很早就已经修复了这个pr一直忘了关。""",panjf2000,7496278,2018-10-30T00:10:32.000+00:00,2018-10-30T00:10:32.000+00:00,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3224,
-1,434126349,211603583,"""@MrDragon1122 一般来说channel是不需要显式close的，只要没有goroutine持有channel，相关资源会自动释放。""",panjf2000,7496278,2018-10-30T00:12:13.000+00:00,2018-10-30T00:12:13.000+00:00,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3225,
-1,439758381,231840723,"""# [Codecov](https://codecov.io/gh/panjf2000/ants/pull/19?src=pr&el=h1) Report\n> Merging [#19](https://codecov.io/gh/panjf2000/ants/pull/19?src=pr&el=desc) into [develop](https://codecov.io/gh/panjf2000/ants/commit/92acf74bb71c1dc1758c61346c88325284978b3e?src=pr&el=desc) will **increase** coverage by `0.02%`.\n> The diff coverage is `100%`.\n\n[![Impacted file tree graph](https://codecov.io/gh/panjf2000/ants/pull/19/graphs/tree.svg?width=650&token=JFa3laaGKq&heig [...]
-1,439758480,231840723,"""我自己写了测试示例\r\n重复\r\nants.NewPoolWithFunc()\r\n然后\r\n进行Resize(0) && Release\r\n但仍发现内存泄漏现象，追溯到了NewTicker没有进行Stop可能导致\r\n\r\nflat flat% sum% cum cum%\r\n\r\n 7168.57kB 77.60% 77.60% 7168.57kB 77.60% time.NewTicker\r\n\r\n\r\n\r\n""",zplzpl,7931755,2018-11-19T03:06:50.000+00:00,2018-11-19T03:07:36.000+00:00,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3227,
-1,439781607,231840723,"""突然看到一个PR有更完善的解决方式，不知道为啥没有PR成功""",zplzpl,7931755,2018-11-19T06:07:43.000+00:00,2018-11-19T06:07:43.000+00:00,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3228,
-1,439793352,231840723,"""另外一个pr跟你大概是完成类似的功能，不过那个failed之后作者就没再更新了，所以就没有merge，你要是有时间不如参考下他的代码再优化下？就放到这个pr里。""",panjf2000,7496278,2018-11-19T07:14:58.000+00:00,2018-11-19T07:14:58.000+00:00,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3230,
-1,456033956,246250598,"""# [Codecov](https://codecov.io/gh/panjf2000/ants/pull/23?src=pr&el=h1) Report\n> Merging [#23](https://codecov.io/gh/panjf2000/ants/pull/23?src=pr&el=desc) into [master](https://codecov.io/gh/panjf2000/ants/commit/812dd4e01075be3cf97429a43abaf6837908cdcd?src=pr&el=desc) will **decrease** coverage by `1.02%`.\n> The diff coverage is `71.42%`.\n\n[![Impacted file tree graph](https://codecov.io/gh/panjf2000/ants/pull/23/graphs/tree.svg?width=650&token=JFa3laaGKq&hei [...]
-1,456071084,246250598,"""@choleraehyq \r\nbuild失败了，解决一下？""",panjf2000,7496278,2019-01-21T13:18:19.000+00:00,2019-01-21T13:18:38.000+00:00,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3247,
-1,456261758,246250598,"""@panjf2000 这是测试覆盖率的问题吧，我已经测了设置 PanicHandler 的情况了，不设置的情况一测就 panic 了，而且是在一个 worker 里 panic，没法 recover""",choleraehyq,8923413,2019-01-22T03:45:04.000+00:00,2019-01-22T03:45:04.000+00:00,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3248,
-1,456270386,246250598,"""@choleraehyq \r\n三点：\r\n1. CI failed是因为TestPanicHandler测试失败了\r\n\r\n2. 在Worker的recover里面，需要再加一个w.pool.decRunning()，因为panic之后这个goroutine就销毁了，所以要更新pool中的可用worker数量\r\n3. 麻烦在pool_func.go和worker_func.go里面也相应地加上PanicHandler的逻辑以及对应的unit tests\r\n\r\nThanks.""",panjf2000,7496278,2019-01-22T04:50:28.000+00:00,2019-01-22T04:54:54.000+00:00,"{""ConnectionId"":1,""Ow [...]
-1,456273052,246250598,"""@panjf2000 thanks，修了""",choleraehyq,8923413,2019-01-22T05:10:00.000+00:00,2019-01-22T05:10:00.000+00:00,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3250,
-1,456277710,246250598,"""Thanks for your contributions to `ants`!""",panjf2000,7496278,2019-01-22T05:41:10.000+00:00,2019-01-22T05:41:10.000+00:00,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3251,
+connection_id,github_id,pull_request_id,body,author_username,author_user_id,commit_sha,github_created_at,github_updated_at,review_id,type,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
+1,300245643,36,"""这里我不太明白，这么写能让这个goroutine不结束？""",panjf2000,7496278,39f04c6e65b76b5f20abd3ca0606db4cd038e5c2,2019-07-04T06:32:11.000+00:00,2019-07-18T15:26:41.000+00:00,257849164,DIFF,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_pull_request_review_comments,1,
+1,300246398,36,"""`f()`应该放到这个匿名函数里面吧？""",panjf2000,7496278,39f04c6e65b76b5f20abd3ca0606db4cd038e5c2,2019-07-04T06:35:07.000+00:00,2019-07-18T15:26:41.000+00:00,257849164,DIFF,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_pull_request_review_comments,2,
+1,307667007,40,"""I don't think assigning this variable to a oversize value is a good idea, threre are different scenarios which should have different interval durations to clean up goroutines, so could you please make `DEFAULT_CLEAN_INTERVAL_TIME ` configurable? Maybe adding a new function that accepts this variable as its argument.""",panjf2000,7496278,5dc8b9a71737eb57dc03fbbe3eb9010ff6c3fbb6,2019-07-26T09:38:21.000+00:00,2019-07-26T14:21:07.000+00:00,267089136,DIFF,"{""ConnectionId"": [...]
+1,307667451,40,"""I believe you should add new unit tests for your new feature.""",panjf2000,7496278,5dc8b9a71737eb57dc03fbbe3eb9010ff6c3fbb6,2019-07-26T09:39:24.000+00:00,2019-07-26T14:21:07.000+00:00,267089136,DIFF,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_pull_request_review_comments,4,
+1,307669178,40,"""You omitted the `worker_func.go`, please update it.""",panjf2000,7496278,5dc8b9a71737eb57dc03fbbe3eb9010ff6c3fbb6,2019-07-26T09:43:56.000+00:00,2019-07-26T14:21:07.000+00:00,267089136,DIFF,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_pull_request_review_comments,5,
+1,307671676,40,"""got it""",Anteoy,17495446,5dc8b9a71737eb57dc03fbbe3eb9010ff6c3fbb6,2019-07-26T09:50:06.000+00:00,2019-07-26T14:21:07.000+00:00,267094793,DIFF,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_pull_request_review_comments,6,
+1,307672146,40,"""sure, I will add.""",Anteoy,17495446,5dc8b9a71737eb57dc03fbbe3eb9010ff6c3fbb6,2019-07-26T09:51:31.000+00:00,2019-07-26T14:21:07.000+00:00,267095424,DIFF,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_pull_request_review_comments,7,
+1,307672417,40,"""I will think about it""",Anteoy,17495446,5dc8b9a71737eb57dc03fbbe3eb9010ff6c3fbb6,2019-07-26T09:52:20.000+00:00,2019-07-26T14:21:07.000+00:00,267095785,DIFF,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_pull_request_review_comments,8,
+1,307675200,40,""" When the pool size is too large , Performance will drop by the lock . Locking at high concurrency is like an atomic bomb. So I think it should be changed. Because it's the default pool value. Many user will use self-defined pool by NewPool. Of course, it is ok to not change.\r\n""",Anteoy,17495446,5dc8b9a71737eb57dc03fbbe3eb9010ff6c3fbb6,2019-07-26T09:59:42.000+00:00,2019-07-26T14:21:07.000+00:00,267099413,DIFF,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""a [...]
+1,307686223,40,"""I think we can modify the New method and periodicallyPurge, it can be used as the next enhancement. I restored it for this pr. May it can be another pr for this.""",Anteoy,17495446,5dc8b9a71737eb57dc03fbbe3eb9010ff6c3fbb6,2019-07-26T10:33:56.000+00:00,2019-07-26T14:21:07.000+00:00,267113847,DIFF,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_pull_request_review_comments,10,
+1,315493235,41,"""SA2002: the goroutine calls T.Fatalf, which must be called in the same goroutine as the test (from `staticcheck`)""",golangcibot,42910462,58466b12b03a603d9f0331bbcc64a7557b27865d,2019-08-20T03:25:26.000+00:00,2019-08-20T10:31:55.000+00:00,276914383,DIFF,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_pull_request_review_comments,11,
+1,315497949,41,"""Are these two fields of `MaxBlockingSubmit` and `blockingNum` really necessary? From where I stand, the throttle of your business traffics should be the capacity of pool, if you've set up `nonblocking` as `true`, it returns nil once the amount of all running workers reaches the pool capacity, otherwise, it is blocked until there is worker available.\r\n\r\nWhy did you add these two fields? is there any specific scenario?""",panjf2000,7496278,58466b12b03a603d9f0331bbcc64 [...]
+1,315498393,41,"""This section could be refined as:\r\n```go\r\nif w := p.retrieveWorker(); w == nil {\r\n return ErrPoolBlocked\r\n} else {\r\n w.task <- task\r\n}\r\n```""",panjf2000,7496278,58466b12b03a603d9f0331bbcc64a7557b27865d,2019-08-20T04:02:19.000+00:00,2019-08-20T10:31:55.000+00:00,276920486,DIFF,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_pull_request_review_comments,13,
+1,315498823,41,"""The name of `ErrPoolBlocked` is blurry and unable to tell why it is being blocked, I would suggest `ErrPoolOverload` or `ErrPoolOverflow`.""",panjf2000,7496278,58466b12b03a603d9f0331bbcc64a7557b27865d,2019-08-20T04:05:11.000+00:00,2019-08-20T10:31:55.000+00:00,276920486,DIFF,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_pull_request_review_comments,14,
+1,315499067,41,"""If this field is necessary, rename it to `MaxBlockingTasks`.""",panjf2000,7496278,58466b12b03a603d9f0331bbcc64a7557b27865d,2019-08-20T04:06:45.000+00:00,2019-08-20T10:31:55.000+00:00,276920486,DIFF,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_pull_request_review_comments,15,
+1,315499281,41,"""I think `Nonblocking` is enough.""",panjf2000,7496278,58466b12b03a603d9f0331bbcc64a7557b27865d,2019-08-20T04:08:21.000+00:00,2019-08-20T10:31:55.000+00:00,276920486,DIFF,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_pull_request_review_comments,16,
+1,407675431,203756736,"""# [Codecov](https://codecov.io/gh/panjf2000/ants/pull/4?src=pr&el=h1) Report\n> Merging [#4](https://codecov.io/gh/panjf2000/ants/pull/4?src=pr&el=desc) into [master](https://codecov.io/gh/panjf2000/ants/commit/f5b37d0798a8e4c6780a1e08270fa50e979aa1d7?src=pr&el=desc) will **not change** coverage.\n> The diff coverage is `100%`.\n\n[ [...]
+1,407714136,203756736,"""@barryz 改的不错，不过有一点需要商榷，那个预分配的大小，之前benchmark我记得1000w也只需要用到7w多，这样的话预分配5w对于大部分场景其实有点浪费，你有没有用修改后的代码benchmark一下，每次allocs/op应该是变大了吧""",panjf2000,7496278,,2018-07-25T10:49:44.000+00:00,2018-07-25T10:49:44.000+00:00,0,NORMAL,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3163,
+1,407755528,203756736,"""@panjf2000 因为从压测结果来看， 1000W的场景下goroutine池的作用很明显， 尤其是在节省内存开销方面，所以在选择默认worker数量时，取了一个相对较大的值。 下面是benchmark 对比\r\n\r\nmaster:\r\n\r\n```\r\ngoos: linux\r\ngoarch: amd64\r\nBenchmarkGoroutineWithFunc-4 1 15079030287 ns/op 723120296 B/op 10176159 allocs/op\r\nPASS\r\nok command-line-arguments 15.106s\r\n```\r\n\r\nPR:\r\n```\r\ngoos: linux\r\ngoarch: amd64\r\nBenchmarkGoroutineWithFunc-4 1 15443376244 ns/op 72096288 [...]
+1,407764945,203756736,"""@barryz 这样看起来预分配似乎没有太大的内存优势，相反在其他数量的任务量场景下可能还会有点浪费，这样吧，要不你先把预分配内存这一块的暂时移除，然后我合一下优化代码的部分，至于预分配内存这一块，后续再继续讨论下看看有没有能兼顾的办法""",panjf2000,7496278,,2018-07-25T14:02:41.000+00:00,2018-07-25T14:02:41.000+00:00,0,NORMAL,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3165,
+1,416794440,211603583,"""@hongli-my 不好意思，我没太懂你的意图？能麻烦说详细点吗？""",panjf2000,7496278,,2018-08-29T01:39:23.000+00:00,2018-08-29T01:39:23.000+00:00,0,NORMAL,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3180,
+1,416794871,211603583,"""# [Codecov](https://codecov.io/gh/panjf2000/ants/pull/8?src=pr&el=h1) Report\n> Merging [#8](https://codecov.io/gh/panjf2000/ants/pull/8?src=pr&el=desc) into [master](https://codecov.io/gh/panjf2000/ants/commit/666635c65d8d3bb1223b819325e0bd23c81f2733?src=pr&el=desc) will **decrease** coverage by `1.24%`.\n> The diff coverage is `100%`.\n\n[![Impacted file tree graph](https://codecov.io/gh/panjf2000/ants/pull/8/graphs/tree.svg?width=650&token=JFa3laaGKq&height=15 [...]
+1,416803651,211603583,"""```n := 0\r\n\t\tfor i, w := range idleWorkers {\r\n\t\t\tif currentTime.Sub(w.recycleTime) <= p.expiryDuration {\r\n\t\t\t\tbreak\r\n\t\t\t}\r\n\t\t\tn = i\r\n\t\t\tw.args <- nil\r\n\t\t\tidleWorkers[i] = nil\r\n\t\t}\r\n\t\tn++\r\n\t\tif n >= len(idleWorkers) {\r\n\t\t\tp.workers = idleWorkers[:0]\r\n\t\t} else {\r\n\t\t\tp.workers = idleWorkers[n:]\r\n\t\t}```\r\nfor 循环中，如果for 循环没有满足条件，n 并没有复制，此时n ++, n 变为1， p.workers = idleWorkers[1:], idleWorkers[0] 丢弃了？ [...]
+1,416818094,211603583,"""@hongli-my 嗯，我知道了，但是你的修改似乎不完整，我已经在develop分支改了一版，要不你看下那个分支，然后按照那个再改一下，然后我再merge""",panjf2000,7496278,,2018-08-29T04:11:17.000+00:00,2018-08-29T04:11:17.000+00:00,0,NORMAL,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3183,
+1,421884048,211603583,"""对于worker中的chan应该也需要close吧，当协程数量增大时，这种也是一种消耗""",ylwangs,29241786,,2018-09-17T03:37:44.000+00:00,2018-09-17T03:37:44.000+00:00,0,NORMAL,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3188,
+1,425423544,218939809,"""@liyonglion 我看修改的代码应该是正确的，但是有两个问题：\r\n1. 要正确测试你pr，你要把ants_test.go里import ants的路径改成你自己的路径；\r\n2. 你现在只修改了pool.go的代码，麻烦把pool_func.go里相应的地方也优化一下。\r\n\r\n你重新修改一下pr，然后通过测试，我就merge了。""",panjf2000,7496278,,2018-09-28T12:43:30.000+00:00,2018-09-28T12:44:42.000+00:00,0,NORMAL,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3199,
+1,425428914,218939809,"""我又仔细地想了想，在putWorker方法中，调用p.cond.Signal()之前，p.lock已经解锁了，如果这时候被\r\n\r\n这里获得锁，则空闲队列中的刚刚放入的worker会被取出，这时候p.cond.Wait()再获得锁去队列中取worker的时候就会取不到，len(p.workers) - 1就会是-1，会报错：index out of range，和https://github.com/panjf2000/ants/issues/6 相似的问题，所以p.cond.Signal()应该放在putWorker中的p.lock.Lock()和p.lock.Unlock()之间。\r\n""",panjf2000,7496278,,2018-09-28T13:03:48.000+00:00,20 [...]
+1,425431081,218939809,"""我明天仔细研究下。如果这个地方的死锁不好解决，其实可以使用另外一把锁，专门给条件变量使用。""",liyonglion,12890888,,2018-09-28T13:12:08.000+00:00,2018-09-28T13:12:08.000+00:00,0,NORMAL,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3201,
+1,425432498,218939809,"""单一操作变量最好不要有多个锁，这样引发死锁的概率会非常高。""",panjf2000,7496278,,2018-09-28T13:17:44.000+00:00,2018-09-28T13:17:57.000+00:00,0,NORMAL,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3202,
+1,425438107,218939809,"""新的锁只给条件变量使用，其他任何地方都不会使用。如果怕死锁，明天我封装一层，pool层只有一把锁。""",liyonglion,12890888,,2018-09-28T13:37:48.000+00:00,2018-09-28T13:37:48.000+00:00,0,NORMAL,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3203,
+1,425439137,218939809,"""封装一层后，按照你的思路修改应该就可以了。""",liyonglion,12890888,,2018-09-28T13:41:20.000+00:00,2018-09-28T13:41:20.000+00:00,0,NORMAL,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3204,
+1,425608265,218939809,"""看了下代码，你说的没错。我在测试下性能方面""",liyonglion,12890888,,2018-09-29T02:22:39.000+00:00,2018-09-29T02:22:39.000+00:00,0,NORMAL,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3205,
+1,425609411,218939809,"""\r\n你这改的有问题，取出来后没有对workers队列缩容，现在Travis CI整个卡住了""",panjf2000,7496278,,2018-09-29T02:41:56.000+00:00,2018-09-29T02:43:26.000+00:00,0,NORMAL,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3206,
+1,425609608,218939809,"""嗯，到时候我会补上。容我再想想""",liyonglion,12890888,,2018-09-29T02:45:23.000+00:00,2018-09-29T02:45:23.000+00:00,0,NORMAL,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3207,
+1,425622339,218939809,"""性能测试如下：\r\n100w的并发，5w的goroutine\r\n\r\n\r\n\r\n""",liyonglion,12890888,,2018-09-29T07:02:56.000+00:00,2018-09-29T07:02:56.000+00:00,0,NORMAL,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3208,
+1,425622437,218939809,"""# [Codecov](https://codecov.io/gh/panjf2000/ants/pull/13?src=pr&el=h1) Report\n> Merging [#13](https://codecov.io/gh/panjf2000/ants/pull/13?src=pr&el=desc) into [master](https://codecov.io/gh/panjf2000/ants/commit/af376f1b7b59dc488458bcecd4273f0fcde33c55?src=pr&el=desc) will **decrease** coverage by `90.38%`.\n> The diff coverage is `5%`.\n\n[![Impacted file tree graph](https://codecov.io/gh/panjf2000/ants/pull/13/graphs/tree.svg?width=650&token=JFa3laaGKq&height [...]
+1,425623081,218939809,"""@panjf2000 这个覆盖率怎么看？""",liyonglion,12890888,,2018-09-29T07:16:02.000+00:00,2018-09-29T07:16:02.000+00:00,0,NORMAL,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3210,
+1,425632550,218939809,"""怎么会有负数。。。""",panjf2000,7496278,,2018-09-29T09:53:47.000+00:00,2018-09-29T09:53:47.000+00:00,0,NORMAL,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3211,
+1,425632613,218939809,"""而且你那个取出worker之后还是没有对队列进行缩容吧""",panjf2000,7496278,,2018-09-29T09:54:48.000+00:00,2018-09-29T09:54:48.000+00:00,0,NORMAL,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3212,
+1,425632632,218939809,"""这个我没玩过。。。。""",liyonglion,12890888,,2018-09-29T09:55:07.000+00:00,2018-09-29T09:55:07.000+00:00,0,NORMAL,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3213,
+1,425632777,218939809,"""@panjf2000 我缩容了，具体哪个地方？""",liyonglion,12890888,,2018-09-29T09:57:34.000+00:00,2018-09-29T09:57:34.000+00:00,0,NORMAL,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3214,
+1,425637776,218939809,"""@liyonglion 我看错了，缩容的补上了，这块没什么问题了，关于覆盖率这块我明天再查查。。。""",panjf2000,7496278,,2018-09-29T11:25:56.000+00:00,2018-09-29T11:25:56.000+00:00,0,NORMAL,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3215,
+1,425637938,218939809,"""哦，我知道了，应该是那个import路径的问题导致的，那我先merge一下代码""",panjf2000,7496278,,2018-09-29T11:29:28.000+00:00,2018-09-29T11:29:28.000+00:00,0,NORMAL,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3216,
+1,425896318,219363161,"""# [Codecov](https://codecov.io/gh/panjf2000/ants/pull/14?src=pr&el=h1) Report\n> Merging [#14](https://codecov.io/gh/panjf2000/ants/pull/14?src=pr&el=desc) into [master](https://codecov.io/gh/panjf2000/ants/commit/1b62696050b7030106291980d5220f886b017eff?src=pr&el=desc) will **decrease** coverage by `2.12%`.\n> The diff coverage is `100%`.\n\n[![Impacted file tree graph](https://codecov.io/gh/panjf2000/ants/pull/14/graphs/tree.svg?width=650&token=JFa3laaGKq&heigh [...]
+1,425906970,219363161,"""thanks for your correction and could you also modify the pool_func.go cuz there is also the same useless statement in that file. i will accept this pr after your modification is done, thanks.""",panjf2000,7496278,,2018-10-01T13:27:04.000+00:00,2018-10-02T13:52:03.000+00:00,0,NORMAL,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3218,
+1,426537013,219936521,"""# [Codecov](https://codecov.io/gh/panjf2000/ants/pull/15?src=pr&el=h1) Report\n> Merging [#15](https://codecov.io/gh/panjf2000/ants/pull/15?src=pr&el=desc) into [master](https://codecov.io/gh/panjf2000/ants/commit/29730bb70343924a2f56a13a9799611dd1cd27fd?src=pr&el=desc) will **decrease** coverage by `0.84%`.\n> The diff coverage is `n/a`.\n\n[![Impacted file tree graph](https://codecov.io/gh/panjf2000/ants/pull/15/graphs/tree.svg?width=650&token=JFa3laaGKq&height [...]
+1,426596957,219936521,"""@egonelbre thanks for your interest in this repository and i got some puzzles.\r\nfirstly, i don't understand why you use both the chan struct{} and sync.WaitGroup at once, that seems a bit excessive cuz waitgroup and chan did the same thing.\r\nsecondly, method BenchmarkGoroutine was design to test throughput so it should run without waitgroup.""",panjf2000,7496278,,2018-10-03T11:05:45.000+00:00,2018-10-03T12:02:17.000+00:00,0,NORMAL,"{""ConnectionId"":1,""Owner [...]
+1,426604764,219936521,"""Hi, semaphore acts as protection to avoid starting too many goroutines at once. `WaitGroup` is used to wait for everything to complete.\r\n\r\n`BenchmarkGoroutine` doesn't wait for all the goroutines to finish, so you are testing goroutine startup speed rather than throughput. It's somewhat unclear what you mean by \""throughput\"" in this case, but either way you would need to wait for all goroutines to finish, otherwise they will start affecting all other bench [...]
+1,429609579,222703171,"""# [Codecov](https://codecov.io/gh/panjf2000/ants/pull/16?src=pr&el=h1) Report\n> Merging [#16](https://codecov.io/gh/panjf2000/ants/pull/16?src=pr&el=desc) into [master](https://codecov.io/gh/panjf2000/ants/commit/711dbdb7a222771ce15aaee1bb7b7c6e9731f208?src=pr&el=desc) will **decrease** coverage by `2.03%`.\n> The diff coverage is `0%`.\n\n[![Impacted file tree graph](https://codecov.io/gh/panjf2000/ants/pull/16/graphs/tree.svg?width=650&token=JFa3laaGKq&height= [...]
+1,433821356,211603583,"""这种bugfix是不是应该赶紧修了？""",choleraehyq,8923413,,2018-10-29T08:10:59.000+00:00,2018-10-29T08:10:59.000+00:00,0,NORMAL,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3223,
+1,434126048,211603583,"""@choleraehyq 这个很早就已经修复了这个pr一直忘了关。""",panjf2000,7496278,,2018-10-30T00:10:32.000+00:00,2018-10-30T00:10:32.000+00:00,0,NORMAL,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3224,
+1,434126349,211603583,"""@MrDragon1122 一般来说channel是不需要显式close的，只要没有goroutine持有channel，相关资源会自动释放。""",panjf2000,7496278,,2018-10-30T00:12:13.000+00:00,2018-10-30T00:12:13.000+00:00,0,NORMAL,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3225,
+1,439758381,231840723,"""# [Codecov](https://codecov.io/gh/panjf2000/ants/pull/19?src=pr&el=h1) Report\n> Merging [#19](https://codecov.io/gh/panjf2000/ants/pull/19?src=pr&el=desc) into [develop](https://codecov.io/gh/panjf2000/ants/commit/92acf74bb71c1dc1758c61346c88325284978b3e?src=pr&el=desc) will **increase** coverage by `0.02%`.\n> The diff coverage is `100%`.\n\n[![Impacted file tree graph](https://codecov.io/gh/panjf2000/ants/pull/19/graphs/tree.svg?width=650&token=JFa3laaGKq&heig [...]
+1,439758480,231840723,"""我自己写了测试示例\r\n重复\r\nants.NewPoolWithFunc()\r\n然后\r\n进行Resize(0) && Release\r\n但仍发现内存泄漏现象，追溯到了NewTicker没有进行Stop可能导致\r\n\r\nflat flat% sum% cum cum%\r\n\r\n 7168.57kB 77.60% 77.60% 7168.57kB 77.60% time.NewTicker\r\n\r\n\r\n\r\n""",zplzpl,7931755,,2018-11-19T03:06:50.000+00:00,2018-11-19T03:07:36.000+00:00,0,NORMAL,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3227,
+1,439781607,231840723,"""突然看到一个PR有更完善的解决方式，不知道为啥没有PR成功""",zplzpl,7931755,,2018-11-19T06:07:43.000+00:00,2018-11-19T06:07:43.000+00:00,0,NORMAL,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3228,
+1,439793352,231840723,"""另外一个pr跟你大概是完成类似的功能，不过那个failed之后作者就没再更新了，所以就没有merge，你要是有时间不如参考下他的代码再优化下？就放到这个pr里。""",panjf2000,7496278,,2018-11-19T07:14:58.000+00:00,2018-11-19T07:14:58.000+00:00,0,NORMAL,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3230,
+1,456033956,246250598,"""# [Codecov](https://codecov.io/gh/panjf2000/ants/pull/23?src=pr&el=h1) Report\n> Merging [#23](https://codecov.io/gh/panjf2000/ants/pull/23?src=pr&el=desc) into [master](https://codecov.io/gh/panjf2000/ants/commit/812dd4e01075be3cf97429a43abaf6837908cdcd?src=pr&el=desc) will **decrease** coverage by `1.02%`.\n> The diff coverage is `71.42%`.\n\n[![Impacted file tree graph](https://codecov.io/gh/panjf2000/ants/pull/23/graphs/tree.svg?width=650&token=JFa3laaGKq&hei [...]
+1,456071084,246250598,"""@choleraehyq \r\nbuild失败了，解决一下？""",panjf2000,7496278,,2019-01-21T13:18:19.000+00:00,2019-01-21T13:18:38.000+00:00,0,NORMAL,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3247,
+1,456261758,246250598,"""@panjf2000 这是测试覆盖率的问题吧，我已经测了设置 PanicHandler 的情况了，不设置的情况一测就 panic 了，而且是在一个 worker 里 panic，没法 recover""",choleraehyq,8923413,,2019-01-22T03:45:04.000+00:00,2019-01-22T03:45:04.000+00:00,0,NORMAL,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3248,
+1,456270386,246250598,"""@choleraehyq \r\n三点：\r\n1. CI failed是因为TestPanicHandler测试失败了\r\n\r\n2. 在Worker的recover里面，需要再加一个w.pool.decRunning()，因为panic之后这个goroutine就销毁了，所以要更新pool中的可用worker数量\r\n3. 麻烦在pool_func.go和worker_func.go里面也相应地加上PanicHandler的逻辑以及对应的unit tests\r\n\r\nThanks.""",panjf2000,7496278,,2019-01-22T04:50:28.000+00:00,2019-01-22T04:54:54.000+00:00,0,NORMAL,"{""ConnectionI [...]
+1,456273052,246250598,"""@panjf2000 thanks，修了""",choleraehyq,8923413,,2019-01-22T05:10:00.000+00:00,2019-01-22T05:10:00.000+00:00,0,NORMAL,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3250,
+1,456277710,246250598,"""Thanks for your contributions to `ants`!""",panjf2000,7496278,,2019-01-22T05:41:10.000+00:00,2019-01-22T05:41:10.000+00:00,0,NORMAL,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3251,
diff --git a/plugins/github/e2e/snapshot_tables/_tool_github_pull_request_reviews.csv b/plugins/github/e2e/snapshot_tables/_tool_github_pull_request_reviews.csv
new file mode 100644
index 00000000..417d3465
--- /dev/null
+++ b/plugins/github/e2e/snapshot_tables/_tool_github_pull_request_reviews.csv
@@ -0,0 +1,31 @@
+connection_id,github_id,pull_request_id,body,author_username,author_user_id,github_submit_at,commit_sha,state,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
+1,277027723,308859272,,choleraehyq,8923413,2019-08-20T09:05:02.000+00:00,ee1ad289e5af90bd7ee573f2ff1c968e57e3d69b,COMMENTED,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_pull_request_reviews,10,
+1,277036116,308859272,,panjf2000,7496278,2019-08-20T09:19:55.000+00:00,ee1ad289e5af90bd7ee573f2ff1c968e57e3d69b,COMMENTED,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_pull_request_reviews,11,
+1,277042368,308859272,,choleraehyq,8923413,2019-08-20T09:30:46.000+00:00,6fcab0b1193daa957b660215ee5b243f69098c83,COMMENTED,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_pull_request_reviews,12,
+1,277042525,308859272,,choleraehyq,8923413,2019-08-20T09:31:02.000+00:00,6fcab0b1193daa957b660215ee5b243f69098c83,COMMENTED,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_pull_request_reviews,13,
+1,277042640,308859272,,choleraehyq,8923413,2019-08-20T09:31:13.000+00:00,6fcab0b1193daa957b660215ee5b243f69098c83,COMMENTED,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_pull_request_reviews,14,
+1,277042704,308859272,,choleraehyq,8923413,2019-08-20T09:31:20.000+00:00,6fcab0b1193daa957b660215ee5b243f69098c83,COMMENTED,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_pull_request_reviews,15,
+1,277042799,308859272,,choleraehyq,8923413,2019-08-20T09:31:30.000+00:00,6fcab0b1193daa957b660215ee5b243f69098c83,COMMENTED,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_pull_request_reviews,16,
+1,277048557,308859272,,panjf2000,7496278,2019-08-20T09:41:40.000+00:00,ee1ad289e5af90bd7ee573f2ff1c968e57e3d69b,COMMENTED,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_pull_request_reviews,17,
+1,277049738,308859272,,panjf2000,7496278,2019-08-20T09:44:27.000+00:00,73aae70ec916a654a7f20ab89eda872b826292ba,COMMENTED,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_pull_request_reviews,18,
+1,277062848,308859272,"Besides, MaxBlockingTasks and Nonblocking are under race risk , please make them goroutine-safe.",panjf2000,7496278,2019-08-20T10:08:17.000+00:00,73aae70ec916a654a7f20ab89eda872b826292ba,COMMENTED,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_pull_request_reviews,19,
+1,277073714,308859272,,choleraehyq,8923413,2019-08-20T10:31:22.000+00:00,73aae70ec916a654a7f20ab89eda872b826292ba,COMMENTED,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_pull_request_reviews,20,
+1,277081346,308859272,,panjf2000,7496278,2019-08-20T10:48:36.000+00:00,58466b12b03a603d9f0331bbcc64a7557b27865d,COMMENTED,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_pull_request_reviews,21,
+1,286687335,316337433,有时间看下我的 comment,panjf2000,7496278,2019-09-11T12:07:36.000+00:00,2cef9f56273b2015a0c9a6b96430a6fb695cda94,COMMENTED,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_pull_request_reviews,22,
+1,286772978,316337433,,choleraehyq,8923413,2019-09-11T13:03:23.000+00:00,2cef9f56273b2015a0c9a6b96430a6fb695cda94,COMMENTED,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_pull_request_reviews,23,
+1,286774552,316337433,,choleraehyq,8923413,2019-09-11T13:05:57.000+00:00,2cef9f56273b2015a0c9a6b96430a6fb695cda94,COMMENTED,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_pull_request_reviews,24,
+1,298229495,325179595,我已经 review 过你的 PR 了，留了一些 comments，你有时间看一下。,panjf2000,7496278,2019-10-07T16:56:26.000+00:00,2950e919e72eb230b2b12f797f379ca6f04fe80c,CHANGES_REQUESTED,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_pull_request_reviews,25,
+1,298484051,325179595,,KevinBaiSg,2813260,2019-10-08T00:45:44.000+00:00,2950e919e72eb230b2b12f797f379ca6f04fe80c,COMMENTED,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_pull_request_reviews,26,
+1,298484184,325179595,,KevinBaiSg,2813260,2019-10-08T00:46:21.000+00:00,2950e919e72eb230b2b12f797f379ca6f04fe80c,COMMENTED,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_pull_request_reviews,27,
+1,298499211,325179595,,KevinBaiSg,2813260,2019-10-08T01:59:39.000+00:00,2950e919e72eb230b2b12f797f379ca6f04fe80c,COMMENTED,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_pull_request_reviews,28,
+1,298503749,325179595,,panjf2000,7496278,2019-10-08T02:22:38.000+00:00,2950e919e72eb230b2b12f797f379ca6f04fe80c,COMMENTED,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_pull_request_reviews,29,
+1,298544685,325179595,,panjf2000,7496278,2019-10-08T05:53:30.000+00:00,2950e919e72eb230b2b12f797f379ca6f04fe80c,COMMENTED,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_pull_request_reviews,30,
+1,298546417,325179595,,panjf2000,7496278,2019-10-08T06:00:53.000+00:00,2950e919e72eb230b2b12f797f379ca6f04fe80c,COMMENTED,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_pull_request_reviews,31,
+1,298546957,325179595,留了一些新的 comments，有时间再看看~~,panjf2000,7496278,2019-10-08T06:07:20.000+00:00,2950e919e72eb230b2b12f797f379ca6f04fe80c,CHANGES_REQUESTED,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_pull_request_reviews,33,
+1,298547873,325179595,,KevinBaiSg,2813260,2019-10-08T06:06:51.000+00:00,2950e919e72eb230b2b12f797f379ca6f04fe80c,COMMENTED,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_pull_request_reviews,32,
+1,298549409,325179595,,panjf2000,7496278,2019-10-08T06:13:03.000+00:00,2950e919e72eb230b2b12f797f379ca6f04fe80c,COMMENTED,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_pull_request_reviews,34,
+1,298549661,325179595,,panjf2000,7496278,2019-10-08T06:14:00.000+00:00,2950e919e72eb230b2b12f797f379ca6f04fe80c,COMMENTED,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_pull_request_reviews,35,
+1,298789411,325179595,,KevinBaiSg,2813260,2019-10-08T14:06:55.000+00:00,2516ada077dcec8ee338a5409d74fec35b6c8d98,COMMENTED,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_pull_request_reviews,36,
+1,298791860,325179595,,KevinBaiSg,2813260,2019-10-08T14:10:13.000+00:00,2516ada077dcec8ee338a5409d74fec35b6c8d98,COMMENTED,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_pull_request_reviews,37,
+1,299362850,325179595,代码逻辑还有点问题。,panjf2000,7496278,2019-10-09T12:58:59.000+00:00,460fea75691aa2dee5d0737fa07e442db465b6b0,CHANGES_REQUESTED,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_pull_request_reviews,38,
+1,299446893,325179595,,KevinBaiSg,2813260,2019-10-09T14:18:28.000+00:00,460fea75691aa2dee5d0737fa07e442db465b6b0,COMMENTED,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_pull_request_reviews,39,
diff --git a/plugins/github/e2e/snapshot_tables/boards.csv b/plugins/github/e2e/snapshot_tables/boards.csv
index e98f6bfa..7201be87 100644
--- a/plugins/github/e2e/snapshot_tables/boards.csv
+++ b/plugins/github/e2e/snapshot_tables/boards.csv
@@ -1,2 +1,2 @@
id,name,description,url,created_date
-github:GithubRepo:1:134018330,panjf2000/ants,"🐜🐜🐜 ants is a high-performance and low-cost goroutine pool in Go, inspired by fasthttp./ ants 是一个高性能且低损耗的 goroutine 池。",https://github.com/panjf2000/ants/issues,2018-05-19T01:13:38.000+00:00
\ No newline at end of file
+github:GithubRepo:1:134018330,panjf2000/ants,"🐜🐜🐜 ants is a high-performance and low-cost goroutine pool in Go, inspired by fasthttp./ ants 是一个高性能且低损耗的 goroutine 池。",https://github.com/panjf2000/ants/issues,2018-05-19T01:13:38.000+00:00
diff --git a/plugins/github/e2e/snapshot_tables/pull_request_comments.csv b/plugins/github/e2e/snapshot_tables/pull_request_comments.csv
index 9ed31a62..a3a3a226 100644
--- a/plugins/github/e2e/snapshot_tables/pull_request_comments.csv
+++ b/plugins/github/e2e/snapshot_tables/pull_request_comments.csv
@@ -1,47 +1,47 @@
-id,pull_request_id,body,user_id,created_date,commit_sha,position
-github:GithubPullRequest:1:407675431,github:GithubPullRequest:1:203756736,"""# [Codecov](https://codecov.io/gh/panjf2000/ants/pull/4?src=pr&el=h1) Report\n> Merging [#4](https://codecov.io/gh/panjf2000/ants/pull/4?src=pr&el=desc) into [master](https://codecov.io/gh/panjf2000/ants/commit/f5b37d0798a8e4c6780a1e08270fa50e979aa1d7?src=pr&el=desc) will **not change** coverage.\n> The diff coverage is `100%`.\n\n[![Impacted file tree graph](https://codecov.io/gh/panjf2000/ants/pull/4/graphs/tr [...]
-github:GithubPullRequest:1:407714136,github:GithubPullRequest:1:203756736,"""@barryz 改的不错，不过有一点需要商榷，那个预分配的大小，之前benchmark我记得1000w也只需要用到7w多，这样的话预分配5w对于大部分场景其实有点浪费，你有没有用修改后的代码benchmark一下，每次allocs/op应该是变大了吧""",github:GithubAccount:1:7496278,2018-07-25T10:49:44.000+00:00,,0
-github:GithubPullRequest:1:407755528,github:GithubPullRequest:1:203756736,"""@panjf2000 因为从压测结果来看， 1000W的场景下goroutine池的作用很明显， 尤其是在节省内存开销方面，所以在选择默认worker数量时，取了一个相对较大的值。 下面是benchmark 对比\r\n\r\nmaster:\r\n\r\n```\r\ngoos: linux\r\ngoarch: amd64\r\nBenchmarkGoroutineWithFunc-4 1 15079030287 ns/op 723120296 B/op 10176159 allocs/op\r\nPASS\r\nok command-line-arguments 15.106s\r\n```\r\n\r\nPR:\r\n```\r\ngoos: linux\r\ngoarch: amd64\r\nBenchmarkGoroutineWithFunc-4 [...]
-github:GithubPullRequest:1:407764945,github:GithubPullRequest:1:203756736,"""@barryz 这样看起来预分配似乎没有太大的内存优势，相反在其他数量的任务量场景下可能还会有点浪费，这样吧，要不你先把预分配内存这一块的暂时移除，然后我合一下优化代码的部分，至于预分配内存这一块，后续再继续讨论下看看有没有能兼顾的办法""",github:GithubAccount:1:7496278,2018-07-25T14:02:41.000+00:00,,0
-github:GithubPullRequest:1:416794440,github:GithubPullRequest:1:211603583,"""@hongli-my 不好意思，我没太懂你的意图？能麻烦说详细点吗？""",github:GithubAccount:1:7496278,2018-08-29T01:39:23.000+00:00,,0
-github:GithubPullRequest:1:416794871,github:GithubPullRequest:1:211603583,"""# [Codecov](https://codecov.io/gh/panjf2000/ants/pull/8?src=pr&el=h1) Report\n> Merging [#8](https://codecov.io/gh/panjf2000/ants/pull/8?src=pr&el=desc) into [master](https://codecov.io/gh/panjf2000/ants/commit/666635c65d8d3bb1223b819325e0bd23c81f2733?src=pr&el=desc) will **decrease** coverage by `1.24%`.\n> The diff coverage is `100%`.\n\n[![Impacted file tree graph](https://codecov.io/gh/panjf2000/ants/pull/8/ [...]
-github:GithubPullRequest:1:416803651,github:GithubPullRequest:1:211603583,"""```n := 0\r\n\t\tfor i, w := range idleWorkers {\r\n\t\t\tif currentTime.Sub(w.recycleTime) <= p.expiryDuration {\r\n\t\t\t\tbreak\r\n\t\t\t}\r\n\t\t\tn = i\r\n\t\t\tw.args <- nil\r\n\t\t\tidleWorkers[i] = nil\r\n\t\t}\r\n\t\tn++\r\n\t\tif n >= len(idleWorkers) {\r\n\t\t\tp.workers = idleWorkers[:0]\r\n\t\t} else {\r\n\t\t\tp.workers = idleWorkers[n:]\r\n\t\t}```\r\nfor 循环中，如果for 循环没有满足条件，n 并没有复制，此时n ++, n 变为1 [...]
-github:GithubPullRequest:1:416818094,github:GithubPullRequest:1:211603583,"""@hongli-my 嗯，我知道了，但是你的修改似乎不完整，我已经在develop分支改了一版，要不你看下那个分支，然后按照那个再改一下，然后我再merge""",github:GithubAccount:1:7496278,2018-08-29T04:11:17.000+00:00,,0
-github:GithubPullRequest:1:421884048,github:GithubPullRequest:1:211603583,"""对于worker中的chan应该也需要close吧，当协程数量增大时，这种也是一种消耗""",github:GithubAccount:1:29241786,2018-09-17T03:37:44.000+00:00,,0
-github:GithubPullRequest:1:425423544,github:GithubPullRequest:1:218939809,"""@liyonglion 我看修改的代码应该是正确的，但是有两个问题：\r\n1. 要正确测试你pr，你要把ants_test.go里import ants的路径改成你自己的路径；\r\n2. 你现在只修改了pool.go的代码，麻烦把pool_func.go里相应的地方也优化一下。\r\n\r\n你重新修改一下pr，然后通过测试，我就merge了。""",github:GithubAccount:1:7496278,2018-09-28T12:43:30.000+00:00,,0
-github:GithubPullRequest:1:425428914,github:GithubPullRequest:1:218939809,"""我又仔细地想了想，在putWorker方法中，调用p.cond.Signal()之前，p.lock已经解锁了，如果这时候被\r\n\r\n这里获得锁，则空闲队列中的刚刚放入的worker会被取出，这时候p.cond.Wait()再获得锁去队列中取worker的时候就会取不到，len(p.workers) - 1就会是-1，会报错：index out of range，和https://github.com/panjf2000/ants/issues/6 相似的问题，所以p.cond.Signal()应该放在putWorker中的p.lock.Lock()和p.lock.Unlock()之间。\r\n""" [...]
-github:GithubPullRequest:1:425431081,github:GithubPullRequest:1:218939809,"""我明天仔细研究下。如果这个地方的死锁不好解决，其实可以使用另外一把锁，专门给条件变量使用。""",github:GithubAccount:1:12890888,2018-09-28T13:12:08.000+00:00,,0
-github:GithubPullRequest:1:425432498,github:GithubPullRequest:1:218939809,"""单一操作变量最好不要有多个锁，这样引发死锁的概率会非常高。""",github:GithubAccount:1:7496278,2018-09-28T13:17:44.000+00:00,,0
-github:GithubPullRequest:1:425438107,github:GithubPullRequest:1:218939809,"""新的锁只给条件变量使用，其他任何地方都不会使用。如果怕死锁，明天我封装一层，pool层只有一把锁。""",github:GithubAccount:1:12890888,2018-09-28T13:37:48.000+00:00,,0
-github:GithubPullRequest:1:425439137,github:GithubPullRequest:1:218939809,"""封装一层后，按照你的思路修改应该就可以了。""",github:GithubAccount:1:12890888,2018-09-28T13:41:20.000+00:00,,0
-github:GithubPullRequest:1:425608265,github:GithubPullRequest:1:218939809,"""看了下代码，你说的没错。我在测试下性能方面""",github:GithubAccount:1:12890888,2018-09-29T02:22:39.000+00:00,,0
-github:GithubPullRequest:1:425609411,github:GithubPullRequest:1:218939809,"""\r\n你这改的有问题，取出来后没有对workers队列缩容，现在Travis CI整个卡住了""",github:GithubAccount:1:7496278,2018-09-29T02:41:56.000+00:00,,0
-github:GithubPullRequest:1:425609608,github:GithubPullRequest:1:218939809,"""嗯，到时候我会补上。容我再想想""",github:GithubAccount:1:12890888,2018-09-29T02:45:23.000+00:00,,0
-github:GithubPullRequest:1:425622339,github:GithubPullRequest:1:218939809,"""性能测试如下：\r\n100w的并发，5w的goroutine\r\n\r\n\r\n\r\n""",github:GithubAccount:1:12890888,2018-09-29T07:02:56.000+00:00,,0
-github:GithubPullRequest:1:425622437,github:GithubPullRequest:1:218939809,"""# [Codecov](https://codecov.io/gh/panjf2000/ants/pull/13?src=pr&el=h1) Report\n> Merging [#13](https://codecov.io/gh/panjf2000/ants/pull/13?src=pr&el=desc) into [master](https://codecov.io/gh/panjf2000/ants/commit/af376f1b7b59dc488458bcecd4273f0fcde33c55?src=pr&el=desc) will **decrease** coverage by `90.38%`.\n> The diff coverage is `5%`.\n\n[![Impacted file tree graph](https://codecov.io/gh/panjf2000/ants/pull/ [...]
-github:GithubPullRequest:1:425623081,github:GithubPullRequest:1:218939809,"""@panjf2000 这个覆盖率怎么看？""",github:GithubAccount:1:12890888,2018-09-29T07:16:02.000+00:00,,0
-github:GithubPullRequest:1:425632550,github:GithubPullRequest:1:218939809,"""怎么会有负数。。。""",github:GithubAccount:1:7496278,2018-09-29T09:53:47.000+00:00,,0
-github:GithubPullRequest:1:425632613,github:GithubPullRequest:1:218939809,"""而且你那个取出worker之后还是没有对队列进行缩容吧""",github:GithubAccount:1:7496278,2018-09-29T09:54:48.000+00:00,,0
-github:GithubPullRequest:1:425632632,github:GithubPullRequest:1:218939809,"""这个我没玩过。。。。""",github:GithubAccount:1:12890888,2018-09-29T09:55:07.000+00:00,,0
-github:GithubPullRequest:1:425632777,github:GithubPullRequest:1:218939809,"""@panjf2000 我缩容了，具体哪个地方？""",github:GithubAccount:1:12890888,2018-09-29T09:57:34.000+00:00,,0
-github:GithubPullRequest:1:425637776,github:GithubPullRequest:1:218939809,"""@liyonglion 我看错了，缩容的补上了，这块没什么问题了，关于覆盖率这块我明天再查查。。。""",github:GithubAccount:1:7496278,2018-09-29T11:25:56.000+00:00,,0
-github:GithubPullRequest:1:425637938,github:GithubPullRequest:1:218939809,"""哦，我知道了，应该是那个import路径的问题导致的，那我先merge一下代码""",github:GithubAccount:1:7496278,2018-09-29T11:29:28.000+00:00,,0
-github:GithubPullRequest:1:425896318,github:GithubPullRequest:1:219363161,"""# [Codecov](https://codecov.io/gh/panjf2000/ants/pull/14?src=pr&el=h1) Report\n> Merging [#14](https://codecov.io/gh/panjf2000/ants/pull/14?src=pr&el=desc) into [master](https://codecov.io/gh/panjf2000/ants/commit/1b62696050b7030106291980d5220f886b017eff?src=pr&el=desc) will **decrease** coverage by `2.12%`.\n> The diff coverage is `100%`.\n\n[![Impacted file tree graph](https://codecov.io/gh/panjf2000/ants/pull [...]
-github:GithubPullRequest:1:425906970,github:GithubPullRequest:1:219363161,"""thanks for your correction and could you also modify the pool_func.go cuz there is also the same useless statement in that file. i will accept this pr after your modification is done, thanks.""",github:GithubAccount:1:7496278,2018-10-01T13:27:04.000+00:00,,0
-github:GithubPullRequest:1:426537013,github:GithubPullRequest:1:219936521,"""# [Codecov](https://codecov.io/gh/panjf2000/ants/pull/15?src=pr&el=h1) Report\n> Merging [#15](https://codecov.io/gh/panjf2000/ants/pull/15?src=pr&el=desc) into [master](https://codecov.io/gh/panjf2000/ants/commit/29730bb70343924a2f56a13a9799611dd1cd27fd?src=pr&el=desc) will **decrease** coverage by `0.84%`.\n> The diff coverage is `n/a`.\n\n[![Impacted file tree graph](https://codecov.io/gh/panjf2000/ants/pull/ [...]
-github:GithubPullRequest:1:426596957,github:GithubPullRequest:1:219936521,"""@egonelbre thanks for your interest in this repository and i got some puzzles.\r\nfirstly, i don't understand why you use both the chan struct{} and sync.WaitGroup at once, that seems a bit excessive cuz waitgroup and chan did the same thing.\r\nsecondly, method BenchmarkGoroutine was design to test throughput so it should run without waitgroup.""",github:GithubAccount:1:7496278,2018-10-03T11:05:45.000+00:00,,0
-github:GithubPullRequest:1:426604764,github:GithubPullRequest:1:219936521,"""Hi, semaphore acts as protection to avoid starting too many goroutines at once. `WaitGroup` is used to wait for everything to complete.\r\n\r\n`BenchmarkGoroutine` doesn't wait for all the goroutines to finish, so you are testing goroutine startup speed rather than throughput. It's somewhat unclear what you mean by \""throughput\"" in this case, but either way you would need to wait for all goroutines to finish, [...]
-github:GithubPullRequest:1:429609579,github:GithubPullRequest:1:222703171,"""# [Codecov](https://codecov.io/gh/panjf2000/ants/pull/16?src=pr&el=h1) Report\n> Merging [#16](https://codecov.io/gh/panjf2000/ants/pull/16?src=pr&el=desc) into [master](https://codecov.io/gh/panjf2000/ants/commit/711dbdb7a222771ce15aaee1bb7b7c6e9731f208?src=pr&el=desc) will **decrease** coverage by `2.03%`.\n> The diff coverage is `0%`.\n\n[![Impacted file tree graph](https://codecov.io/gh/panjf2000/ants/pull/1 [...]
-github:GithubPullRequest:1:433821356,github:GithubPullRequest:1:211603583,"""这种bugfix是不是应该赶紧修了？""",github:GithubAccount:1:8923413,2018-10-29T08:10:59.000+00:00,,0
-github:GithubPullRequest:1:434126048,github:GithubPullRequest:1:211603583,"""@choleraehyq 这个很早就已经修复了这个pr一直忘了关。""",github:GithubAccount:1:7496278,2018-10-30T00:10:32.000+00:00,,0
-github:GithubPullRequest:1:434126349,github:GithubPullRequest:1:211603583,"""@MrDragon1122 一般来说channel是不需要显式close的，只要没有goroutine持有channel，相关资源会自动释放。""",github:GithubAccount:1:7496278,2018-10-30T00:12:13.000+00:00,,0
-github:GithubPullRequest:1:439758381,github:GithubPullRequest:1:231840723,"""# [Codecov](https://codecov.io/gh/panjf2000/ants/pull/19?src=pr&el=h1) Report\n> Merging [#19](https://codecov.io/gh/panjf2000/ants/pull/19?src=pr&el=desc) into [develop](https://codecov.io/gh/panjf2000/ants/commit/92acf74bb71c1dc1758c61346c88325284978b3e?src=pr&el=desc) will **increase** coverage by `0.02%`.\n> The diff coverage is `100%`.\n\n[![Impacted file tree graph](https://codecov.io/gh/panjf2000/ants/pul [...]
-github:GithubPullRequest:1:439758480,github:GithubPullRequest:1:231840723,"""我自己写了测试示例\r\n重复\r\nants.NewPoolWithFunc()\r\n然后\r\n进行Resize(0) && Release\r\n但仍发现内存泄漏现象，追溯到了NewTicker没有进行Stop可能导致\r\n\r\nflat flat% sum% cum cum%\r\n\r\n 7168.57kB 77.60% 77.60% 7168.57kB 77.60% time.NewTicker\r\n\r\n\r\n\r\n""",github:GithubAccount:1:7931755,2018-11-19T03:06:50.000+00:00,,0
-github:GithubPullRequest:1:439781607,github:GithubPullRequest:1:231840723,"""突然看到一个PR有更完善的解决方式，不知道为啥没有PR成功""",github:GithubAccount:1:7931755,2018-11-19T06:07:43.000+00:00,,0
-github:GithubPullRequest:1:439793352,github:GithubPullRequest:1:231840723,"""另外一个pr跟你大概是完成类似的功能，不过那个failed之后作者就没再更新了，所以就没有merge，你要是有时间不如参考下他的代码再优化下？就放到这个pr里。""",github:GithubAccount:1:7496278,2018-11-19T07:14:58.000+00:00,,0
-github:GithubPullRequest:1:456033956,github:GithubPullRequest:1:246250598,"""# [Codecov](https://codecov.io/gh/panjf2000/ants/pull/23?src=pr&el=h1) Report\n> Merging [#23](https://codecov.io/gh/panjf2000/ants/pull/23?src=pr&el=desc) into [master](https://codecov.io/gh/panjf2000/ants/commit/812dd4e01075be3cf97429a43abaf6837908cdcd?src=pr&el=desc) will **decrease** coverage by `1.02%`.\n> The diff coverage is `71.42%`.\n\n[![Impacted file tree graph](https://codecov.io/gh/panjf2000/ants/pu [...]
-github:GithubPullRequest:1:456071084,github:GithubPullRequest:1:246250598,"""@choleraehyq \r\nbuild失败了，解决一下？""",github:GithubAccount:1:7496278,2019-01-21T13:18:19.000+00:00,,0
-github:GithubPullRequest:1:456261758,github:GithubPullRequest:1:246250598,"""@panjf2000 这是测试覆盖率的问题吧，我已经测了设置 PanicHandler 的情况了，不设置的情况一测就 panic 了，而且是在一个 worker 里 panic，没法 recover""",github:GithubAccount:1:8923413,2019-01-22T03:45:04.000+00:00,,0
-github:GithubPullRequest:1:456270386,github:GithubPullRequest:1:246250598,"""@choleraehyq \r\n三点：\r\n1. CI failed是因为TestPanicHandler测试失败了\r\n\r\n2. 在Worker的recover里面，需要再加一个w.pool.decRunning()，因为panic之后这个goroutine就销毁了，所以要更新pool中的可用worker数量\r\n3. 麻烦在pool_func.go和worker_func.go里面也相应地加上PanicHandler的逻辑以及对应的unit tests\r\n\r\nThanks.""",github:GithubAccount:1:7496278,2019-01-22T04:50:28. [...]
-github:GithubPullRequest:1:456273052,github:GithubPullRequest:1:246250598,"""@panjf2000 thanks，修了""",github:GithubAccount:1:8923413,2019-01-22T05:10:00.000+00:00,,0
-github:GithubPullRequest:1:456277710,github:GithubPullRequest:1:246250598,"""Thanks for your contributions to `ants`!""",github:GithubAccount:1:7496278,2019-01-22T05:41:10.000+00:00,,0
+id,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark,pull_request_id,body,user_id,created_date,commit_sha,position,type,review_id,status
+github:GithubPrComment:1:407675431,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3162,,github:GithubPullRequest:1:203756736,"""# [Codecov](https://codecov.io/gh/panjf2000/ants/pull/4?src=pr&el=h1) Report\n> Merging [#4](https://codecov.io/gh/panjf2000/ants/pull/4?src=pr&el=desc) into [master](https://codecov.io/gh/panjf2000/ants/commit/f5b37d0798a8e4c6780a1e08270fa50e979aa1d7?src=pr&el=desc) will **not change** coverage.\n> The diff coverage is [...]
+github:GithubPrComment:1:407714136,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3163,,github:GithubPullRequest:1:203756736,"""@barryz 改的不错，不过有一点需要商榷，那个预分配的大小，之前benchmark我记得1000w也只需要用到7w多，这样的话预分配5w对于大部分场景其实有点浪费，你有没有用修改后的代码benchmark一下，每次allocs/op应该是变大了吧""",github:GithubAccount:1:7496278,2018-07-25T10:49:44.000+00:00,,0,NORMAL,github:GithubPrReview:1:0,
+github:GithubPrComment:1:407755528,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3164,,github:GithubPullRequest:1:203756736,"""@panjf2000 因为从压测结果来看， 1000W的场景下goroutine池的作用很明显， 尤其是在节省内存开销方面，所以在选择默认worker数量时，取了一个相对较大的值。 下面是benchmark 对比\r\n\r\nmaster:\r\n\r\n```\r\ngoos: linux\r\ngoarch: amd64\r\nBenchmarkGoroutineWithFunc-4 1 15079030287 ns/op 723120296 B/op 10176159 allocs/op\r\nPASS\r\nok command-line-arguments 15 [...]
+github:GithubPrComment:1:407764945,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3165,,github:GithubPullRequest:1:203756736,"""@barryz 这样看起来预分配似乎没有太大的内存优势，相反在其他数量的任务量场景下可能还会有点浪费，这样吧，要不你先把预分配内存这一块的暂时移除，然后我合一下优化代码的部分，至于预分配内存这一块，后续再继续讨论下看看有没有能兼顾的办法""",github:GithubAccount:1:7496278,2018-07-25T14:02:41.000+00:00,,0,NORMAL,github:GithubPrReview:1:0,
+github:GithubPrComment:1:416794440,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3180,,github:GithubPullRequest:1:211603583,"""@hongli-my 不好意思，我没太懂你的意图？能麻烦说详细点吗？""",github:GithubAccount:1:7496278,2018-08-29T01:39:23.000+00:00,,0,NORMAL,github:GithubPrReview:1:0,
+github:GithubPrComment:1:416794871,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3181,,github:GithubPullRequest:1:211603583,"""# [Codecov](https://codecov.io/gh/panjf2000/ants/pull/8?src=pr&el=h1) Report\n> Merging [#8](https://codecov.io/gh/panjf2000/ants/pull/8?src=pr&el=desc) into [master](https://codecov.io/gh/panjf2000/ants/commit/666635c65d8d3bb1223b819325e0bd23c81f2733?src=pr&el=desc) will **decrease** coverage by `1.24%`.\n> The diff co [...]
+github:GithubPrComment:1:416803651,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3182,,github:GithubPullRequest:1:211603583,"""```n := 0\r\n\t\tfor i, w := range idleWorkers {\r\n\t\t\tif currentTime.Sub(w.recycleTime) <= p.expiryDuration {\r\n\t\t\t\tbreak\r\n\t\t\t}\r\n\t\t\tn = i\r\n\t\t\tw.args <- nil\r\n\t\t\tidleWorkers[i] = nil\r\n\t\t}\r\n\t\tn++\r\n\t\tif n >= len(idleWorkers) {\r\n\t\t\tp.workers = idleWorkers[:0]\r\n\t\t} else {\r\n\ [...]
+github:GithubPrComment:1:416818094,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3183,,github:GithubPullRequest:1:211603583,"""@hongli-my 嗯，我知道了，但是你的修改似乎不完整，我已经在develop分支改了一版，要不你看下那个分支，然后按照那个再改一下，然后我再merge""",github:GithubAccount:1:7496278,2018-08-29T04:11:17.000+00:00,,0,NORMAL,github:GithubPrReview:1:0,
+github:GithubPrComment:1:421884048,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3188,,github:GithubPullRequest:1:211603583,"""对于worker中的chan应该也需要close吧，当协程数量增大时，这种也是一种消耗""",github:GithubAccount:1:29241786,2018-09-17T03:37:44.000+00:00,,0,NORMAL,github:GithubPrReview:1:0,
+github:GithubPrComment:1:425423544,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3199,,github:GithubPullRequest:1:218939809,"""@liyonglion 我看修改的代码应该是正确的，但是有两个问题：\r\n1. 要正确测试你pr，你要把ants_test.go里import ants的路径改成你自己的路径；\r\n2. 你现在只修改了pool.go的代码，麻烦把pool_func.go里相应的地方也优化一下。\r\n\r\n你重新修改一下pr，然后通过测试，我就merge了。""",github:GithubAccount:1:7496278,2018-09-28T12:43:30.000+00:00,,0,NORMAL,github:GithubPrReview:1:0,
+github:GithubPrComment:1:425428914,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3200,,github:GithubPullRequest:1:218939809,"""我又仔细地想了想，在putWorker方法中，调用p.cond.Signal()之前，p.lock已经解锁了，如果这时候被\r\n\r\n这里获得锁，则空闲队列中的刚刚放入的worker会被取出，这时候p.cond.Wait()再获得锁去队列中取worker的时候就会取不到，len(p.workers) - 1就会是-1，会报错：index out of range，和https://github.com/panjf20 [...]
+github:GithubPrComment:1:425431081,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3201,,github:GithubPullRequest:1:218939809,"""我明天仔细研究下。如果这个地方的死锁不好解决，其实可以使用另外一把锁，专门给条件变量使用。""",github:GithubAccount:1:12890888,2018-09-28T13:12:08.000+00:00,,0,NORMAL,github:GithubPrReview:1:0,
+github:GithubPrComment:1:425432498,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3202,,github:GithubPullRequest:1:218939809,"""单一操作变量最好不要有多个锁，这样引发死锁的概率会非常高。""",github:GithubAccount:1:7496278,2018-09-28T13:17:44.000+00:00,,0,NORMAL,github:GithubPrReview:1:0,
+github:GithubPrComment:1:425438107,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3203,,github:GithubPullRequest:1:218939809,"""新的锁只给条件变量使用，其他任何地方都不会使用。如果怕死锁，明天我封装一层，pool层只有一把锁。""",github:GithubAccount:1:12890888,2018-09-28T13:37:48.000+00:00,,0,NORMAL,github:GithubPrReview:1:0,
+github:GithubPrComment:1:425439137,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3204,,github:GithubPullRequest:1:218939809,"""封装一层后，按照你的思路修改应该就可以了。""",github:GithubAccount:1:12890888,2018-09-28T13:41:20.000+00:00,,0,NORMAL,github:GithubPrReview:1:0,
+github:GithubPrComment:1:425608265,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3205,,github:GithubPullRequest:1:218939809,"""看了下代码，你说的没错。我在测试下性能方面""",github:GithubAccount:1:12890888,2018-09-29T02:22:39.000+00:00,,0,NORMAL,github:GithubPrReview:1:0,
+github:GithubPrComment:1:425609411,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3206,,github:GithubPullRequest:1:218939809,"""\r\n你这改的有问题，取出来后没有对workers队列缩容，现在Travis CI整个卡住了""",github:GithubAccount:1:7496278,2018-09-29T02:41:56.000+00:00,,0,NORMAL,github:GithubPrReview:1:0,
+github:GithubPrComment:1:425609608,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3207,,github:GithubPullRequest:1:218939809,"""嗯，到时候我会补上。容我再想想""",github:GithubAccount:1:12890888,2018-09-29T02:45:23.000+00:00,,0,NORMAL,github:GithubPrReview:1:0,
+github:GithubPrComment:1:425622339,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3208,,github:GithubPullRequest:1:218939809,"""性能测试如下：\r\n100w的并发，5w的goroutine\r\n\r\n\r\n\r\n""",github:GithubAccount:1:12890888, [...]
+github:GithubPrComment:1:425622437,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3209,,github:GithubPullRequest:1:218939809,"""# [Codecov](https://codecov.io/gh/panjf2000/ants/pull/13?src=pr&el=h1) Report\n> Merging [#13](https://codecov.io/gh/panjf2000/ants/pull/13?src=pr&el=desc) into [master](https://codecov.io/gh/panjf2000/ants/commit/af376f1b7b59dc488458bcecd4273f0fcde33c55?src=pr&el=desc) will **decrease** coverage by `90.38%`.\n> The dif [...]
+github:GithubPrComment:1:425623081,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3210,,github:GithubPullRequest:1:218939809,"""@panjf2000 这个覆盖率怎么看？""",github:GithubAccount:1:12890888,2018-09-29T07:16:02.000+00:00,,0,NORMAL,github:GithubPrReview:1:0,
+github:GithubPrComment:1:425632550,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3211,,github:GithubPullRequest:1:218939809,"""怎么会有负数。。。""",github:GithubAccount:1:7496278,2018-09-29T09:53:47.000+00:00,,0,NORMAL,github:GithubPrReview:1:0,
+github:GithubPrComment:1:425632613,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3212,,github:GithubPullRequest:1:218939809,"""而且你那个取出worker之后还是没有对队列进行缩容吧""",github:GithubAccount:1:7496278,2018-09-29T09:54:48.000+00:00,,0,NORMAL,github:GithubPrReview:1:0,
+github:GithubPrComment:1:425632632,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3213,,github:GithubPullRequest:1:218939809,"""这个我没玩过。。。。""",github:GithubAccount:1:12890888,2018-09-29T09:55:07.000+00:00,,0,NORMAL,github:GithubPrReview:1:0,
+github:GithubPrComment:1:425632777,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3214,,github:GithubPullRequest:1:218939809,"""@panjf2000 我缩容了，具体哪个地方？""",github:GithubAccount:1:12890888,2018-09-29T09:57:34.000+00:00,,0,NORMAL,github:GithubPrReview:1:0,
+github:GithubPrComment:1:425637776,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3215,,github:GithubPullRequest:1:218939809,"""@liyonglion 我看错了，缩容的补上了，这块没什么问题了，关于覆盖率这块我明天再查查。。。""",github:GithubAccount:1:7496278,2018-09-29T11:25:56.000+00:00,,0,NORMAL,github:GithubPrReview:1:0,
+github:GithubPrComment:1:425637938,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3216,,github:GithubPullRequest:1:218939809,"""哦，我知道了，应该是那个import路径的问题导致的，那我先merge一下代码""",github:GithubAccount:1:7496278,2018-09-29T11:29:28.000+00:00,,0,NORMAL,github:GithubPrReview:1:0,
+github:GithubPrComment:1:425896318,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3217,,github:GithubPullRequest:1:219363161,"""# [Codecov](https://codecov.io/gh/panjf2000/ants/pull/14?src=pr&el=h1) Report\n> Merging [#14](https://codecov.io/gh/panjf2000/ants/pull/14?src=pr&el=desc) into [master](https://codecov.io/gh/panjf2000/ants/commit/1b62696050b7030106291980d5220f886b017eff?src=pr&el=desc) will **decrease** coverage by `2.12%`.\n> The diff [...]
+github:GithubPrComment:1:425906970,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3218,,github:GithubPullRequest:1:219363161,"""thanks for your correction and could you also modify the pool_func.go cuz there is also the same useless statement in that file. i will accept this pr after your modification is done, thanks.""",github:GithubAccount:1:7496278,2018-10-01T13:27:04.000+00:00,,0,NORMAL,github:GithubPrReview:1:0,
+github:GithubPrComment:1:426537013,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3219,,github:GithubPullRequest:1:219936521,"""# [Codecov](https://codecov.io/gh/panjf2000/ants/pull/15?src=pr&el=h1) Report\n> Merging [#15](https://codecov.io/gh/panjf2000/ants/pull/15?src=pr&el=desc) into [master](https://codecov.io/gh/panjf2000/ants/commit/29730bb70343924a2f56a13a9799611dd1cd27fd?src=pr&el=desc) will **decrease** coverage by `0.84%`.\n> The diff [...]
+github:GithubPrComment:1:426596957,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3220,,github:GithubPullRequest:1:219936521,"""@egonelbre thanks for your interest in this repository and i got some puzzles.\r\nfirstly, i don't understand why you use both the chan struct{} and sync.WaitGroup at once, that seems a bit excessive cuz waitgroup and chan did the same thing.\r\nsecondly, method BenchmarkGoroutine was design to test throughput so it sho [...]
+github:GithubPrComment:1:426604764,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3221,,github:GithubPullRequest:1:219936521,"""Hi, semaphore acts as protection to avoid starting too many goroutines at once. `WaitGroup` is used to wait for everything to complete.\r\n\r\n`BenchmarkGoroutine` doesn't wait for all the goroutines to finish, so you are testing goroutine startup speed rather than throughput. It's somewhat unclear what you mean by \""t [...]
+github:GithubPrComment:1:429609579,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3222,,github:GithubPullRequest:1:222703171,"""# [Codecov](https://codecov.io/gh/panjf2000/ants/pull/16?src=pr&el=h1) Report\n> Merging [#16](https://codecov.io/gh/panjf2000/ants/pull/16?src=pr&el=desc) into [master](https://codecov.io/gh/panjf2000/ants/commit/711dbdb7a222771ce15aaee1bb7b7c6e9731f208?src=pr&el=desc) will **decrease** coverage by `2.03%`.\n> The diff [...]
+github:GithubPrComment:1:433821356,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3223,,github:GithubPullRequest:1:211603583,"""这种bugfix是不是应该赶紧修了？""",github:GithubAccount:1:8923413,2018-10-29T08:10:59.000+00:00,,0,NORMAL,github:GithubPrReview:1:0,
+github:GithubPrComment:1:434126048,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3224,,github:GithubPullRequest:1:211603583,"""@choleraehyq 这个很早就已经修复了这个pr一直忘了关。""",github:GithubAccount:1:7496278,2018-10-30T00:10:32.000+00:00,,0,NORMAL,github:GithubPrReview:1:0,
+github:GithubPrComment:1:434126349,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3225,,github:GithubPullRequest:1:211603583,"""@MrDragon1122 一般来说channel是不需要显式close的，只要没有goroutine持有channel，相关资源会自动释放。""",github:GithubAccount:1:7496278,2018-10-30T00:12:13.000+00:00,,0,NORMAL,github:GithubPrReview:1:0,
+github:GithubPrComment:1:439758381,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3226,,github:GithubPullRequest:1:231840723,"""# [Codecov](https://codecov.io/gh/panjf2000/ants/pull/19?src=pr&el=h1) Report\n> Merging [#19](https://codecov.io/gh/panjf2000/ants/pull/19?src=pr&el=desc) into [develop](https://codecov.io/gh/panjf2000/ants/commit/92acf74bb71c1dc1758c61346c88325284978b3e?src=pr&el=desc) will **increase** coverage by `0.02%`.\n> The dif [...]
+github:GithubPrComment:1:439758480,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3227,,github:GithubPullRequest:1:231840723,"""我自己写了测试示例\r\n重复\r\nants.NewPoolWithFunc()\r\n然后\r\n进行Resize(0) && Release\r\n但仍发现内存泄漏现象，追溯到了NewTicker没有进行Stop可能导致\r\n\r\nflat flat% sum% cum cum%\r\n\r\n 7168.57kB 77.60% 77.60% 7168.57kB 77.60% time.NewTicker\r\n\r\n\r\n\r\n""",github:GithubAccount:1:7931755,2018-11-19T03:06:50.000+00:00,,0,NORMAL,github: [...]
+github:GithubPrComment:1:439781607,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3228,,github:GithubPullRequest:1:231840723,"""突然看到一个PR有更完善的解决方式，不知道为啥没有PR成功""",github:GithubAccount:1:7931755,2018-11-19T06:07:43.000+00:00,,0,NORMAL,github:GithubPrReview:1:0,
+github:GithubPrComment:1:439793352,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3230,,github:GithubPullRequest:1:231840723,"""另外一个pr跟你大概是完成类似的功能，不过那个failed之后作者就没再更新了，所以就没有merge，你要是有时间不如参考下他的代码再优化下？就放到这个pr里。""",github:GithubAccount:1:7496278,2018-11-19T07:14:58.000+00:00,,0,NORMAL,github:GithubPrReview:1:0,
+github:GithubPrComment:1:456033956,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3246,,github:GithubPullRequest:1:246250598,"""# [Codecov](https://codecov.io/gh/panjf2000/ants/pull/23?src=pr&el=h1) Report\n> Merging [#23](https://codecov.io/gh/panjf2000/ants/pull/23?src=pr&el=desc) into [master](https://codecov.io/gh/panjf2000/ants/commit/812dd4e01075be3cf97429a43abaf6837908cdcd?src=pr&el=desc) will **decrease** coverage by `1.02%`.\n> The diff [...]
+github:GithubPrComment:1:456071084,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3247,,github:GithubPullRequest:1:246250598,"""@choleraehyq \r\nbuild失败了，解决一下？""",github:GithubAccount:1:7496278,2019-01-21T13:18:19.000+00:00,,0,NORMAL,github:GithubPrReview:1:0,
+github:GithubPrComment:1:456261758,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3248,,github:GithubPullRequest:1:246250598,"""@panjf2000 这是测试覆盖率的问题吧，我已经测了设置 PanicHandler 的情况了，不设置的情况一测就 panic 了，而且是在一个 worker 里 panic，没法 recover""",github:GithubAccount:1:8923413,2019-01-22T03:45:04.000+00:00,,0,NORMAL,github:GithubPrReview:1:0,
+github:GithubPrComment:1:456270386,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3249,,github:GithubPullRequest:1:246250598,"""@choleraehyq \r\n三点：\r\n1. CI failed是因为TestPanicHandler测试失败了\r\n\r\n2. 在Worker的recover里面，需要再加一个w.pool.decRunning()，因为panic之后这个goroutine就销毁了，所以要更新pool中的可用worker数量\r\n3. 麻烦在pool_func.go和worker_func.go里面也相应地加上PanicH [...]
+github:GithubPrComment:1:456273052,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3250,,github:GithubPullRequest:1:246250598,"""@panjf2000 thanks，修了""",github:GithubAccount:1:8923413,2019-01-22T05:10:00.000+00:00,,0,NORMAL,github:GithubPrReview:1:0,
+github:GithubPrComment:1:456277710,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3251,,github:GithubPullRequest:1:246250598,"""Thanks for your contributions to `ants`!""",github:GithubAccount:1:7496278,2019-01-22T05:41:10.000+00:00,,0,NORMAL,github:GithubPrReview:1:0,
diff --git a/plugins/github/impl/impl.go b/plugins/github/impl/impl.go
index 9eb826c5..35c8a42c 100644
--- a/plugins/github/impl/impl.go
+++ b/plugins/github/impl/impl.go
@@ -65,7 +65,9 @@ func (plugin Github) SubTaskMetas() []core.SubTaskMeta {
tasks.CollectApiPullRequestCommitsMeta,
tasks.ExtractApiPullRequestCommitsMeta,
tasks.CollectApiPullRequestReviewsMeta,
- tasks.ExtractApiPullRequestReviewersMeta,
+ tasks.ExtractApiPullRequestReviewsMeta,
+ tasks.CollectApiPrReviewCommentsMeta,
+ tasks.ExtractApiPrReviewCommentsMeta,
tasks.CollectApiCommitsMeta,
tasks.ExtractApiCommitsMeta,
tasks.CollectApiCommitStatsMeta,
@@ -83,6 +85,7 @@ func (plugin Github) SubTaskMetas() []core.SubTaskMeta {
tasks.ConvertIssueLabelsMeta,
tasks.ConvertPullRequestCommitsMeta,
tasks.ConvertPullRequestsMeta,
+ tasks.ConvertPullRequestReviewsMeta,
tasks.ConvertPullRequestLabelsMeta,
tasks.ConvertPullRequestIssuesMeta,
tasks.ConvertIssueCommentsMeta,
diff --git a/plugins/github/models/migrationscripts/archived/account.go b/plugins/github/models/migrationscripts/archived/account.go
index 41ff5ab4..61fd20bf 100644
--- a/plugins/github/models/migrationscripts/archived/account.go
+++ b/plugins/github/models/migrationscripts/archived/account.go
@@ -27,6 +27,9 @@ type GithubAccount struct {
Url string `json:"url" gorm:"type:varchar(255)"`
HtmlUrl string `json:"html_url" gorm:"type:varchar(255)"`
Type string `json:"type" gorm:"type:varchar(255)"`
+ Name string `json:"name" gorm:"type:varchar(255)"`
+ Company string `json:"company" gorm:"type:varchar(255)"`
+ Email string `json:"Email" gorm:"type:varchar(255)"`
archived.NoPKModel
}
diff --git a/plugins/github/models/migrationscripts/archived/pull_request_issue.go b/plugins/github/models/migrationscripts/archived/account_org.go
similarity index 72%
copy from plugins/github/models/migrationscripts/archived/pull_request_issue.go
copy to plugins/github/models/migrationscripts/archived/account_org.go
index 0f22a55a..9556fad4 100644
--- a/plugins/github/models/migrationscripts/archived/pull_request_issue.go
+++ b/plugins/github/models/migrationscripts/archived/account_org.go
@@ -19,15 +19,14 @@ package archived
import "github.com/apache/incubator-devlake/models/migrationscripts/archived"
-type GithubPullRequestIssue struct {
- ConnectionId uint64 `gorm:"primaryKey"`
- PullRequestId int `gorm:"primaryKey"`
- IssueId int `gorm:"primaryKey"`
- PullRequestNumber int
- IssueNumber int
+type GithubAccountOrg struct {
+ ConnectionId uint64 `gorm:"primaryKey"`
+ AccountId int `gorm:"primaryKey;autoIncrement:false"`
+ OrgId int `gorm:"primaryKey;autoIncrement:false"`
+ OrgLogin string `json:"org_login" gorm:"type:varchar(255)"`
archived.NoPKModel
}
-func (GithubPullRequestIssue) TableName() string {
- return "_tool_github_pull_request_issues"
+func (GithubAccountOrg) TableName() string {
+ return "_tool_github_account_orgs"
}
diff --git a/plugins/feishu/models/migrationscripts/archived/connection.go b/plugins/github/models/migrationscripts/archived/connection.go
similarity index 72%
copy from plugins/feishu/models/migrationscripts/archived/connection.go
copy to plugins/github/models/migrationscripts/archived/connection.go
index 9715f006..1221fead 100644
--- a/plugins/feishu/models/migrationscripts/archived/connection.go
+++ b/plugins/github/models/migrationscripts/archived/connection.go
@@ -17,20 +17,17 @@ limitations under the License.
package archived
-import (
- commonArchived "github.com/apache/incubator-devlake/models/migrationscripts/archived"
-)
+import "github.com/apache/incubator-devlake/models/migrationscripts/archived"
-type FeishuConnection struct {
- commonArchived.Model
+type GithubConnection struct {
+ archived.Model
Name string `gorm:"type:varchar(100);uniqueIndex" json:"name" validate:"required"`
Endpoint string `mapstructure:"endpoint" env:"GITHUB_ENDPOINT" validate:"required"`
Proxy string `mapstructure:"proxy" env:"GITHUB_PROXY"`
RateLimitPerHour int `comment:"api request rate limit per hour"`
- AppId string `mapstructure:"app_id" validate:"required" json:"app_id"`
- SecretKey string `mapstructure:"secret_key" validate:"required" json:"secret_key" encrypt:"yes"`
+ Token string `mapstructure:"token" env:"GITHUB_AUTH" validate:"required" encrypt:"yes"`
}
-func (FeishuConnection) TableName() string {
- return "_tool_feishu_connections"
+func (GithubConnection) TableName() string {
+ return "_tool_github_connections"
}
diff --git a/plugins/github/models/migrationscripts/archived/pull_request_comment.go b/plugins/github/models/migrationscripts/archived/pr_comment.go
similarity index 85%
copy from plugins/github/models/migrationscripts/archived/pull_request_comment.go
copy to plugins/github/models/migrationscripts/archived/pr_comment.go
index c88387c8..62d7bfc0 100644
--- a/plugins/github/models/migrationscripts/archived/pull_request_comment.go
+++ b/plugins/github/models/migrationscripts/archived/pr_comment.go
@@ -18,23 +18,25 @@ limitations under the License.
package archived
import (
- "time"
-
"github.com/apache/incubator-devlake/models/migrationscripts/archived"
+ "time"
)
-type GithubPullRequestComment struct {
+type GithubPrComment struct {
ConnectionId uint64 `gorm:"primaryKey"`
GithubId int `gorm:"primaryKey"`
PullRequestId int `gorm:"index"`
Body string
AuthorUsername string `gorm:"type:varchar(255)"`
AuthorUserId int
+ CommitSha string `gorm:"type:varchar(255)"`
GithubCreatedAt time.Time
GithubUpdatedAt time.Time `gorm:"index"`
+ ReviewId int
+ Type string `gorm:"index;type:varchar(255)"`
archived.NoPKModel
}
-func (GithubPullRequestComment) TableName() string {
+func (GithubPrComment) TableName() string {
return "_tool_github_pull_request_comments"
}
diff --git a/plugins/github/models/migrationscripts/archived/pull_request_commit.go b/plugins/github/models/migrationscripts/archived/pr_commit.go
similarity index 92%
rename from plugins/github/models/migrationscripts/archived/pull_request_commit.go
rename to plugins/github/models/migrationscripts/archived/pr_commit.go
index 3700d230..5109527b 100644
--- a/plugins/github/models/migrationscripts/archived/pull_request_commit.go
+++ b/plugins/github/models/migrationscripts/archived/pr_commit.go
@@ -19,13 +19,13 @@ package archived
import "github.com/apache/incubator-devlake/models/migrationscripts/archived"
-type GithubPullRequestCommit struct {
+type GithubPrCommit struct {
ConnectionId uint64 `gorm:"primaryKey"`
CommitSha string `gorm:"primaryKey;type:varchar(40)"`
PullRequestId int `gorm:"primaryKey;autoIncrement:false"`
archived.NoPKModel
}
-func (GithubPullRequestCommit) TableName() string {
+func (GithubPrCommit) TableName() string {
return "_tool_github_pull_request_commits"
}
diff --git a/plugins/github/models/migrationscripts/archived/pull_request_issue.go b/plugins/github/models/migrationscripts/archived/pr_issue.go
similarity index 92%
copy from plugins/github/models/migrationscripts/archived/pull_request_issue.go
copy to plugins/github/models/migrationscripts/archived/pr_issue.go
index 0f22a55a..10c7c20d 100644
--- a/plugins/github/models/migrationscripts/archived/pull_request_issue.go
+++ b/plugins/github/models/migrationscripts/archived/pr_issue.go
@@ -19,7 +19,7 @@ package archived
import "github.com/apache/incubator-devlake/models/migrationscripts/archived"
-type GithubPullRequestIssue struct {
+type GithubPrIssue struct {
ConnectionId uint64 `gorm:"primaryKey"`
PullRequestId int `gorm:"primaryKey"`
IssueId int `gorm:"primaryKey"`
@@ -28,6 +28,6 @@ type GithubPullRequestIssue struct {
archived.NoPKModel
}
-func (GithubPullRequestIssue) TableName() string {
+func (GithubPrIssue) TableName() string {
return "_tool_github_pull_request_issues"
}
diff --git a/plugins/github/models/migrationscripts/archived/pull_request_label.go b/plugins/github/models/migrationscripts/archived/pr_label.go
similarity index 93%
rename from plugins/github/models/migrationscripts/archived/pull_request_label.go
rename to plugins/github/models/migrationscripts/archived/pr_label.go
index b96fb424..dc68dc4e 100644
--- a/plugins/github/models/migrationscripts/archived/pull_request_label.go
+++ b/plugins/github/models/migrationscripts/archived/pr_label.go
@@ -22,13 +22,13 @@ import "github.com/apache/incubator-devlake/models/migrationscripts/archived"
// Please note that Issue Labels can also apply to Pull Requests.
// Pull Requests are considered Issues in GitHub.
-type GithubPullRequestLabel struct {
+type GithubPrLabel struct {
ConnectionId uint64 `gorm:"primaryKey"`
PullId int `gorm:"primaryKey;autoIncrement:false"`
LabelName string `gorm:"primaryKey;type:varchar(255)"`
archived.NoPKModel
}
-func (GithubPullRequestLabel) TableName() string {
+func (GithubPrLabel) TableName() string {
return "_tool_github_pull_request_labels"
}
diff --git a/plugins/github/models/migrationscripts/archived/pull_request_comment.go b/plugins/github/models/migrationscripts/archived/pr_review.go
similarity index 66%
rename from plugins/github/models/migrationscripts/archived/pull_request_comment.go
rename to plugins/github/models/migrationscripts/archived/pr_review.go
index c88387c8..0f200915 100644
--- a/plugins/github/models/migrationscripts/archived/pull_request_comment.go
+++ b/plugins/github/models/migrationscripts/archived/pr_review.go
@@ -18,23 +18,23 @@ limitations under the License.
package archived
import (
- "time"
-
"github.com/apache/incubator-devlake/models/migrationscripts/archived"
+ "time"
)
-type GithubPullRequestComment struct {
- ConnectionId uint64 `gorm:"primaryKey"`
- GithubId int `gorm:"primaryKey"`
- PullRequestId int `gorm:"index"`
- Body string
- AuthorUsername string `gorm:"type:varchar(255)"`
- AuthorUserId int
- GithubCreatedAt time.Time
- GithubUpdatedAt time.Time `gorm:"index"`
+type GithubPrReview struct {
+ ConnectionId uint64 `gorm:"primaryKey"`
+ GithubId int `gorm:"primaryKey"`
+ PullRequestId int `gorm:"index"`
+ Body string
+ AuthorUsername string `gorm:"type:varchar(255)"`
+ AuthorUserId int
+ GithubSubmitAt time.Time
+ CommitSha string `gorm:"type:varchar(255)"`
+ State string `gorm:"type:varchar(255)"`
archived.NoPKModel
}
-func (GithubPullRequestComment) TableName() string {
- return "_tool_github_pull_request_comments"
+func (GithubPrReview) TableName() string {
+ return "_tool_github_pull_request_reviews"
}
diff --git a/plugins/github/models/migrationscripts/archived/pull_request_issue.go b/plugins/github/models/migrationscripts/archived/repo_account.go
similarity index 72%
rename from plugins/github/models/migrationscripts/archived/pull_request_issue.go
rename to plugins/github/models/migrationscripts/archived/repo_account.go
index 0f22a55a..5aac9b99 100644
--- a/plugins/github/models/migrationscripts/archived/pull_request_issue.go
+++ b/plugins/github/models/migrationscripts/archived/repo_account.go
@@ -19,15 +19,14 @@ package archived
import "github.com/apache/incubator-devlake/models/migrationscripts/archived"
-type GithubPullRequestIssue struct {
- ConnectionId uint64 `gorm:"primaryKey"`
- PullRequestId int `gorm:"primaryKey"`
- IssueId int `gorm:"primaryKey"`
- PullRequestNumber int
- IssueNumber int
+type GithubRepoAccount struct {
+ ConnectionId uint64 `gorm:"primaryKey"`
+ AccountId int `gorm:"primaryKey;autoIncrement:false"`
+ RepoGithubId int `gorm:"primaryKey;autoIncrement:false"`
+ Login string `json:"login" gorm:"type:varchar(255)"`
archived.NoPKModel
}
-func (GithubPullRequestIssue) TableName() string {
- return "_tool_github_pull_request_issues"
+func (GithubRepoAccount) TableName() string {
+ return "_tool_github_repo_accounts"
}
diff --git a/plugins/github/models/migrationscripts/init_schema.go b/plugins/github/models/migrationscripts/init_schema.go
index 19d5fcce..553f86ed 100644
--- a/plugins/github/models/migrationscripts/init_schema.go
+++ b/plugins/github/models/migrationscripts/init_schema.go
@@ -19,28 +19,13 @@ package migrationscripts
import (
"context"
-
"github.com/apache/incubator-devlake/plugins/core"
"github.com/apache/incubator-devlake/plugins/helper"
- commonArchived "github.com/apache/incubator-devlake/models/migrationscripts/archived"
"github.com/apache/incubator-devlake/plugins/github/models/migrationscripts/archived"
"gorm.io/gorm"
)
-type GithubConnection struct {
- commonArchived.Model
- Name string `gorm:"type:varchar(100);uniqueIndex" json:"name" validate:"required"`
- Endpoint string `mapstructure:"endpoint" env:"GITHUB_ENDPOINT" validate:"required"`
- Proxy string `mapstructure:"proxy" env:"GITHUB_PROXY"`
- RateLimitPerHour int `comment:"api request rate limit per hour"`
- Token string `mapstructure:"token" env:"GITHUB_AUTH" validate:"required" encrypt:"yes"`
-}
-
-func (GithubConnection) TableName() string {
- return "_tool_github_connections"
-}
-
type initSchemas struct {
config core.ConfigGetter
}
@@ -50,27 +35,26 @@ func (u *initSchemas) SetConfigGetter(config core.ConfigGetter) {
}
func (u *initSchemas) Up(ctx context.Context, db *gorm.DB) error {
- if db.Migrator().HasTable(GithubConnection{}) {
- err := db.Migrator().DropTable(GithubConnection{})
- if err != nil {
- return err
- }
- }
err := db.Migrator().DropTable(
&archived.GithubRepo{},
+ &archived.GithubConnection{},
&archived.GithubCommit{},
&archived.GithubRepoCommit{},
&archived.GithubPullRequest{},
&archived.GithubReviewer{},
- &archived.GithubPullRequestComment{},
- &archived.GithubPullRequestCommit{},
- &archived.GithubPullRequestLabel{},
+ &archived.GithubPrCommit{},
+ &archived.GithubPrLabel{},
&archived.GithubIssue{},
&archived.GithubIssueComment{},
&archived.GithubIssueEvent{},
&archived.GithubIssueLabel{},
- &archived.GithubPullRequestIssue{},
+ &archived.GithubPrIssue{},
&archived.GithubCommitStat{},
+ &archived.GithubPrComment{},
+ &archived.GithubPrReview{},
+ &archived.GithubRepoAccount{},
+ &archived.GithubAccountOrg{},
+ &archived.GithubAccount{},
"_tool_github_users",
"_tool_github_milestones",
"_raw_github_api_issues",
@@ -83,18 +67,20 @@ func (u *initSchemas) Up(ctx context.Context, db *gorm.DB) error {
"_raw_github_api_pull_request_commits",
"_raw_github_api_pull_request_reviews",
"_raw_github_api_repositories",
+ "_raw_github_api_reviews",
)
// create connection
if err != nil {
return err
}
- err = db.Migrator().CreateTable(GithubConnection{})
+
+ err = db.Migrator().CreateTable(archived.GithubConnection{})
if err != nil {
return err
}
encodeKey := u.config.GetString(core.EncodeKeyEnvStr)
- connection := &GithubConnection{}
+ connection := &archived.GithubConnection{}
connection.Endpoint = u.config.GetString(`GITHUB_ENDPOINT`)
connection.Proxy = u.config.GetString(`GITHUB_PROXY`)
connection.Token = u.config.GetString(`GITHUB_AUTH`)
@@ -117,22 +103,26 @@ func (u *initSchemas) Up(ctx context.Context, db *gorm.DB) error {
&archived.GithubRepoCommit{},
&archived.GithubPullRequest{},
&archived.GithubReviewer{},
- &archived.GithubPullRequestComment{},
- &archived.GithubPullRequestCommit{},
- &archived.GithubPullRequestLabel{},
+ &archived.GithubPrComment{},
+ &archived.GithubPrCommit{},
+ &archived.GithubPrLabel{},
&archived.GithubIssue{},
&archived.GithubIssueComment{},
&archived.GithubIssueEvent{},
&archived.GithubIssueLabel{},
&archived.GithubAccount{},
- &archived.GithubPullRequestIssue{},
+ &archived.GithubPrIssue{},
&archived.GithubCommitStat{},
&archived.GithubMilestone{},
+ &archived.GithubPrReview{},
+ &archived.GithubRepoAccount{},
+ &archived.GithubAccountOrg{},
+ &archived.GithubAccount{},
)
}
func (*initSchemas) Version() uint64 {
- return 20220714000001
+ return 20220715000001
}
func (*initSchemas) Name() string {
diff --git a/plugins/github/models/migrationscripts/register.go b/plugins/github/models/migrationscripts/register.go
index 5afe3dfe..f7da9f74 100644
--- a/plugins/github/models/migrationscripts/register.go
+++ b/plugins/github/models/migrationscripts/register.go
@@ -25,7 +25,5 @@ import (
func All() []migration.Script {
return []migration.Script{
new(initSchemas),
- new(updateSchemas20220713000003),
- new(updateSchemas20220713000004),
}
}
diff --git a/plugins/github/models/migrationscripts/updateSchemas20220713000003.go b/plugins/github/models/migrationscripts/updateSchemas20220713000003.go
deleted file mode 100644
index 0c5521df..00000000
--- a/plugins/github/models/migrationscripts/updateSchemas20220713000003.go
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package migrationscripts
-
-import (
- "context"
- "github.com/apache/incubator-devlake/models/migrationscripts/archived"
- "gorm.io/gorm"
-)
-
-type GithubAccount20220713 struct {
- Name string `json:"name" gorm:"type:varchar(255)"`
- Company string `json:"company" gorm:"type:varchar(255)"`
- Email string `json:"Email" gorm:"type:varchar(255)"`
-}
-
-func (GithubAccount20220713) TableName() string {
- return "_tool_github_accounts"
-}
-
-type GithubRepoAccount20220713 struct {
- ConnectionId uint64 `gorm:"primaryKey"`
- AccountId int `gorm:"primaryKey;autoIncrement:false"`
- RepoGithubId int `gorm:"primaryKey;autoIncrement:false"`
- Login string `json:"login" gorm:"type:varchar(255)"`
- archived.NoPKModel
-}
-
-func (GithubRepoAccount20220713) TableName() string {
- return "_tool_github_repo_accounts"
-}
-
-type updateSchemas20220713000003 struct{}
-
-func (*updateSchemas20220713000003) Up(ctx context.Context, db *gorm.DB) error {
- err := db.Migrator().AutoMigrate(&GithubAccount20220713{})
- if err != nil {
- return err
- }
- err = db.Migrator().AutoMigrate(&GithubRepoAccount20220713{})
- if err != nil {
- return err
- }
- return nil
-}
-
-func (*updateSchemas20220713000003) Version() uint64 {
- return 20220713000003
-}
-
-func (*updateSchemas20220713000003) Name() string {
- return "UpdateSchemas for extend account in 20220713"
-}
diff --git a/plugins/github/models/migrationscripts/updateSchemas20220713000004.go b/plugins/github/models/migrationscripts/updateSchemas20220713000004.go
deleted file mode 100644
index 09cc11f3..00000000
--- a/plugins/github/models/migrationscripts/updateSchemas20220713000004.go
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package migrationscripts
-
-import (
- "context"
- "github.com/apache/incubator-devlake/models/migrationscripts/archived"
- "gorm.io/gorm"
-)
-
-type GithubAccountOrg20220713 struct {
- ConnectionId uint64 `gorm:"primaryKey"`
- AccountId int `gorm:"primaryKey;autoIncrement:false"`
- OrgId int `gorm:"primaryKey;autoIncrement:false"`
- OrgLogin string `json:"org_login" gorm:"type:varchar(255)"`
- archived.NoPKModel
-}
-
-func (GithubAccountOrg20220713) TableName() string {
- return "_tool_github_account_orgs"
-}
-
-type updateSchemas20220713000004 struct{}
-
-func (*updateSchemas20220713000004) Up(ctx context.Context, db *gorm.DB) error {
- return db.Migrator().AutoMigrate(&GithubAccountOrg20220713{})
-}
-
-func (*updateSchemas20220713000004) Version() uint64 {
- return 20220713000004
-}
-
-func (*updateSchemas20220713000004) Name() string {
- return "UpdateSchemas for add org in 20220713"
-}
diff --git a/plugins/github/models/pr_comment.go b/plugins/github/models/pr_comment.go
index bea0a5c0..8fceda41 100644
--- a/plugins/github/models/pr_comment.go
+++ b/plugins/github/models/pr_comment.go
@@ -22,18 +22,21 @@ import (
"time"
)
-type GithubPullRequestComment struct {
+type GithubPrComment struct {
ConnectionId uint64 `gorm:"primaryKey"`
GithubId int `gorm:"primaryKey"`
PullRequestId int `gorm:"index"`
Body string
AuthorUsername string `gorm:"type:varchar(255)"`
AuthorUserId int
+ CommitSha string `gorm:"type:varchar(255)"`
GithubCreatedAt time.Time
GithubUpdatedAt time.Time `gorm:"index"`
+ ReviewId int
+ Type string `gorm:"index;type:varchar(255)"`
common.NoPKModel
}
-func (GithubPullRequestComment) TableName() string {
+func (GithubPrComment) TableName() string {
return "_tool_github_pull_request_comments"
}
diff --git a/plugins/github/models/pr_commit.go b/plugins/github/models/pr_commit.go
index 99068da0..ae41f007 100644
--- a/plugins/github/models/pr_commit.go
+++ b/plugins/github/models/pr_commit.go
@@ -21,13 +21,13 @@ import (
"github.com/apache/incubator-devlake/models/common"
)
-type GithubPullRequestCommit struct {
+type GithubPrCommit struct {
ConnectionId uint64 `gorm:"primaryKey"`
CommitSha string `gorm:"primaryKey;type:varchar(40)"`
PullRequestId int `gorm:"primaryKey;autoIncrement:false"`
common.NoPKModel
}
-func (GithubPullRequestCommit) TableName() string {
+func (GithubPrCommit) TableName() string {
return "_tool_github_pull_request_commits"
}
diff --git a/plugins/github/models/pr_issue.go b/plugins/github/models/pr_issue.go
index e1efcf32..53378774 100644
--- a/plugins/github/models/pr_issue.go
+++ b/plugins/github/models/pr_issue.go
@@ -19,7 +19,7 @@ package models
import "github.com/apache/incubator-devlake/models/common"
-type GithubPullRequestIssue struct {
+type GithubPrIssue struct {
ConnectionId uint64 `gorm:"primaryKey"`
PullRequestId int `gorm:"primaryKey"`
IssueId int `gorm:"primaryKey"`
@@ -28,6 +28,6 @@ type GithubPullRequestIssue struct {
common.NoPKModel
}
-func (GithubPullRequestIssue) TableName() string {
+func (GithubPrIssue) TableName() string {
return "_tool_github_pull_request_issues"
}
diff --git a/plugins/github/models/pr_label.go b/plugins/github/models/pr_label.go
index caf58e18..728cb311 100644
--- a/plugins/github/models/pr_label.go
+++ b/plugins/github/models/pr_label.go
@@ -24,13 +24,13 @@ import (
// Please note that Issue Labels can also apply to Pull Requests.
// Pull Requests are considered Issues in GitHub.
-type GithubPullRequestLabel struct {
+type GithubPrLabel struct {
ConnectionId uint64 `gorm:"primaryKey"`
PullId int `gorm:"primaryKey;autoIncrement:false"`
LabelName string `gorm:"primaryKey;type:varchar(255)"`
common.NoPKModel
}
-func (GithubPullRequestLabel) TableName() string {
+func (GithubPrLabel) TableName() string {
return "_tool_github_pull_request_labels"
}
diff --git a/plugins/github/models/pr_label.go b/plugins/github/models/pr_review.go
similarity index 67%
copy from plugins/github/models/pr_label.go
copy to plugins/github/models/pr_review.go
index caf58e18..e69e85ec 100644
--- a/plugins/github/models/pr_label.go
+++ b/plugins/github/models/pr_review.go
@@ -19,18 +19,22 @@ package models
import (
"github.com/apache/incubator-devlake/models/common"
+ "time"
)
-// Please note that Issue Labels can also apply to Pull Requests.
-// Pull Requests are considered Issues in GitHub.
-
-type GithubPullRequestLabel struct {
- ConnectionId uint64 `gorm:"primaryKey"`
- PullId int `gorm:"primaryKey;autoIncrement:false"`
- LabelName string `gorm:"primaryKey;type:varchar(255)"`
+type GithubPrReview struct {
+ ConnectionId uint64 `gorm:"primaryKey"`
+ GithubId int `gorm:"primaryKey"`
+ PullRequestId int `gorm:"index"`
+ Body string
+ AuthorUsername string `gorm:"type:varchar(255)"`
+ AuthorUserId int
+ CommitSha string
+ GithubSubmitAt *time.Time
+ State string
common.NoPKModel
}
-func (GithubPullRequestLabel) TableName() string {
- return "_tool_github_pull_request_labels"
+func (GithubPrReview) TableName() string {
+ return "_tool_github_pull_request_reviews"
}
diff --git a/plugins/github/tasks/comment_collector.go b/plugins/github/tasks/comment_collector.go
index 74c66545..80da9ca4 100644
--- a/plugins/github/tasks/comment_collector.go
+++ b/plugins/github/tasks/comment_collector.go
@@ -53,7 +53,7 @@ func CollectApiComments(taskCtx core.SubTaskContext) error {
if err != nil {
return fmt.Errorf("failed to get latest github issue record: %w", err)
}
- var latestUpdatedPrComt models.GithubPullRequestComment
+ var latestUpdatedPrComt models.GithubPrComment
err = db.All(
&latestUpdatedPrComt,
dal.Join("left join _tool_github_pull_requests on _tool_github_pull_requests.github_id = _tool_github_pull_request_comments.pull_request_id"),
diff --git a/plugins/github/tasks/comment_extractor.go b/plugins/github/tasks/comment_extractor.go
index c3712201..8e8b6993 100644
--- a/plugins/github/tasks/comment_extractor.go
+++ b/plugins/github/tasks/comment_extractor.go
@@ -19,9 +19,9 @@ package tasks
import (
"encoding/json"
+ "gorm.io/gorm"
"github.com/apache/incubator-devlake/plugins/core/dal"
- "gorm.io/gorm"
"github.com/apache/incubator-devlake/plugins/core"
"github.com/apache/incubator-devlake/plugins/github/models"
@@ -78,7 +78,7 @@ func ExtractApiComments(taskCtx core.SubTaskContext) error {
}
issue := &models.GithubIssue{}
err = taskCtx.GetDal().All(issue, dal.Where("connection_id = ? and number = ? and repo_id = ?", data.Options.ConnectionId, issueINumber, data.Repo.GithubId))
- if err != nil {
+ if err != nil && err != gorm.ErrRecordNotFound {
return nil, err
}
//if we can not find issues with issue number above, move the comments to github_pull_request_comments
@@ -88,7 +88,7 @@ func ExtractApiComments(taskCtx core.SubTaskContext) error {
if err != nil && err != gorm.ErrRecordNotFound {
return nil, err
}
- githubPrComment := &models.GithubPullRequestComment{
+ githubPrComment := &models.GithubPrComment{
ConnectionId: data.Options.ConnectionId,
GithubId: apiComment.GithubId,
PullRequestId: pr.GithubId,
@@ -97,6 +97,7 @@ func ExtractApiComments(taskCtx core.SubTaskContext) error {
AuthorUserId: apiComment.User.Id,
GithubCreatedAt: apiComment.GithubCreatedAt.ToTime(),
GithubUpdatedAt: apiComment.GithubUpdatedAt.ToTime(),
+ Type: "NORMAL",
}
results = append(results, githubPrComment)
} else {
diff --git a/plugins/github/tasks/issue_collector.go b/plugins/github/tasks/issue_collector.go
index ae735f12..508ff437 100644
--- a/plugins/github/tasks/issue_collector.go
+++ b/plugins/github/tasks/issue_collector.go
@@ -114,7 +114,6 @@ func CollectApiIssues(taskCtx core.SubTaskContext) error {
query.Set("direction", "asc")
query.Set("page", fmt.Sprintf("%v", reqData.Pager.Page))
query.Set("per_page", fmt.Sprintf("%v", reqData.Pager.Size))
- query.Set("milestone", "*")
return query, nil
},
/*
diff --git a/plugins/github/tasks/issue_extractor.go b/plugins/github/tasks/issue_extractor.go
index 2097e2ac..9dd8ea70 100644
--- a/plugins/github/tasks/issue_extractor.go
+++ b/plugins/github/tasks/issue_extractor.go
@@ -159,7 +159,6 @@ func convertGithubIssue(issue *IssuesResponse, connectionId uint64, repositoryId
Title: issue.Title,
Body: string(issue.Body),
Url: issue.HtmlUrl,
- MilestoneId: issue.Milestone.Id,
ClosedAt: helper.Iso8601TimeToTime(issue.ClosedAt),
GithubCreatedAt: issue.GithubCreatedAt.ToTime(),
GithubUpdatedAt: issue.GithubUpdatedAt.ToTime(),
@@ -175,6 +174,9 @@ func convertGithubIssue(issue *IssuesResponse, connectionId uint64, repositoryId
if issue.ClosedAt != nil {
githubIssue.LeadTimeMinutes = uint(issue.ClosedAt.ToTime().Sub(issue.GithubCreatedAt.ToTime()).Minutes())
}
+ if issue.Milestone != nil {
+ githubIssue.MilestoneId = issue.Milestone.Id
+ }
return githubIssue, nil
}
diff --git a/plugins/github/tasks/pr_comment_convertor.go b/plugins/github/tasks/pr_comment_convertor.go
index 23004a34..a915721f 100644
--- a/plugins/github/tasks/pr_comment_convertor.go
+++ b/plugins/github/tasks/pr_comment_convertor.go
@@ -43,7 +43,7 @@ func ConvertPullRequestComments(taskCtx core.SubTaskContext) error {
repoId := data.Repo.GithubId
cursor, err := db.Cursor(
- dal.From(&githubModels.GithubPullRequestComment{}),
+ dal.From(&githubModels.GithubPrComment{}),
dal.Join("left join _tool_github_pull_requests "+
"on _tool_github_pull_requests.github_id = _tool_github_pull_request_comments.pull_request_id"),
dal.Where("repo_id = ? and _tool_github_pull_requests.connection_id = ?", repoId, data.Options.ConnectionId),
@@ -53,11 +53,12 @@ func ConvertPullRequestComments(taskCtx core.SubTaskContext) error {
}
defer cursor.Close()
+ prCommentIdGen := didgen.NewDomainIdGenerator(&githubModels.GithubPrComment{})
prIdGen := didgen.NewDomainIdGenerator(&githubModels.GithubPullRequest{})
accountIdGen := didgen.NewDomainIdGenerator(&githubModels.GithubAccount{})
-
+ prReviewIdGen := didgen.NewDomainIdGenerator(&githubModels.GithubPrReview{})
converter, err := helper.NewDataConverter(helper.DataConverterArgs{
- InputRowType: reflect.TypeOf(githubModels.GithubPullRequestComment{}),
+ InputRowType: reflect.TypeOf(githubModels.GithubPrComment{}),
Input: cursor,
RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
Ctx: taskCtx,
@@ -69,17 +70,18 @@ func ConvertPullRequestComments(taskCtx core.SubTaskContext) error {
Table: RAW_COMMENTS_TABLE,
},
Convert: func(inputRow interface{}) ([]interface{}, error) {
- githubPullRequestComment := inputRow.(*githubModels.GithubPullRequestComment)
+ githubPullRequestComment := inputRow.(*githubModels.GithubPrComment)
domainPrComment := &code.PullRequestComment{
DomainEntity: domainlayer.DomainEntity{
- Id: prIdGen.Generate(data.Options.ConnectionId, githubPullRequestComment.GithubId),
+ Id: prCommentIdGen.Generate(data.Options.ConnectionId, githubPullRequestComment.GithubId),
},
PullRequestId: prIdGen.Generate(data.Options.ConnectionId, githubPullRequestComment.PullRequestId),
Body: githubPullRequestComment.Body,
UserId: accountIdGen.Generate(data.Options.ConnectionId, githubPullRequestComment.AuthorUserId),
CreatedDate: githubPullRequestComment.GithubCreatedAt,
- CommitSha: "",
- Position: 0,
+ CommitSha: githubPullRequestComment.CommitSha,
+ Type: githubPullRequestComment.Type,
+ ReviewId: prReviewIdGen.Generate(data.Options.ConnectionId, githubPullRequestComment.ReviewId),
}
return []interface{}{
domainPrComment,
diff --git a/plugins/github/tasks/pr_commit_collector.go b/plugins/github/tasks/pr_commit_collector.go
index faf39137..c832aa99 100644
--- a/plugins/github/tasks/pr_commit_collector.go
+++ b/plugins/github/tasks/pr_commit_collector.go
@@ -32,7 +32,7 @@ import (
"github.com/apache/incubator-devlake/plugins/github/models"
)
-const RAW_PULL_REQUEST_COMMIT_TABLE = "github_api_pull_request_commits"
+const RAW_PR_COMMIT_TABLE = "github_api_pull_request_commits"
// this struct should be moved to `gitub_api_common.go`
@@ -83,7 +83,7 @@ func CollectApiPullRequestCommits(taskCtx core.SubTaskContext) error {
/*
Table store raw data
*/
- Table: RAW_PULL_REQUEST_COMMIT_TABLE,
+ Table: RAW_PR_COMMIT_TABLE,
},
ApiClient: data.ApiClient,
PageSize: 100,
diff --git a/plugins/github/tasks/pr_commit_convertor.go b/plugins/github/tasks/pr_commit_convertor.go
index c306e4e9..a6aea439 100644
--- a/plugins/github/tasks/pr_commit_convertor.go
+++ b/plugins/github/tasks/pr_commit_convertor.go
@@ -45,7 +45,7 @@ func ConvertPullRequestCommits(taskCtx core.SubTaskContext) (err error) {
pullIdGen := didgen.NewDomainIdGenerator(&githubModels.GithubPullRequest{})
cursor, err := db.Cursor(
- dal.From(&githubModels.GithubPullRequestCommit{}),
+ dal.From(&githubModels.GithubPrCommit{}),
dal.Join(`left join _tool_github_pull_requests on _tool_github_pull_requests.github_id = _tool_github_pull_request_commits.pull_request_id`),
dal.Where("_tool_github_pull_requests.repo_id = ? and _tool_github_pull_requests.connection_id = ?", repoId, data.Options.ConnectionId),
dal.Orderby("pull_request_id ASC"),
@@ -56,7 +56,7 @@ func ConvertPullRequestCommits(taskCtx core.SubTaskContext) (err error) {
defer cursor.Close()
converter, err := helper.NewDataConverter(helper.DataConverterArgs{
- InputRowType: reflect.TypeOf(githubModels.GithubPullRequestCommit{}),
+ InputRowType: reflect.TypeOf(githubModels.GithubPrCommit{}),
Input: cursor,
RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
Ctx: taskCtx,
@@ -65,10 +65,10 @@ func ConvertPullRequestCommits(taskCtx core.SubTaskContext) (err error) {
Owner: data.Options.Owner,
Repo: data.Options.Repo,
},
- Table: RAW_PULL_REQUEST_COMMIT_TABLE,
+ Table: RAW_PR_COMMIT_TABLE,
},
Convert: func(inputRow interface{}) ([]interface{}, error) {
- githubPullRequestCommit := inputRow.(*githubModels.GithubPullRequestCommit)
+ githubPullRequestCommit := inputRow.(*githubModels.GithubPrCommit)
domainPrCommit := &code.PullRequestCommit{
CommitSha: githubPullRequestCommit.CommitSha,
PullRequestId: pullIdGen.Generate(data.Options.ConnectionId, githubPullRequestCommit.PullRequestId),
diff --git a/plugins/github/tasks/pr_commit_extractor.go b/plugins/github/tasks/pr_commit_extractor.go
index 74cb4dc5..b3e6b4d3 100644
--- a/plugins/github/tasks/pr_commit_extractor.go
+++ b/plugins/github/tasks/pr_commit_extractor.go
@@ -72,7 +72,7 @@ func ExtractApiPullRequestCommits(taskCtx core.SubTaskContext) error {
/*
Table store raw data
*/
- Table: RAW_PULL_REQUEST_COMMIT_TABLE,
+ Table: RAW_PR_COMMIT_TABLE,
},
Extract: func(row *helper.RawData) ([]interface{}, error) {
apiPullRequestCommit := &PrCommitsResponse{}
@@ -97,7 +97,7 @@ func ExtractApiPullRequestCommits(taskCtx core.SubTaskContext) error {
}
results = append(results, githubCommit)
- githubPullRequestCommit := &models.GithubPullRequestCommit{
+ githubPullRequestCommit := &models.GithubPrCommit{
ConnectionId: data.Options.ConnectionId,
CommitSha: apiPullRequestCommit.Sha,
PullRequestId: pull.GithubId,
diff --git a/plugins/github/tasks/pr_extractor.go b/plugins/github/tasks/pr_extractor.go
index 54efc163..3e83ef58 100644
--- a/plugins/github/tasks/pr_extractor.go
+++ b/plugins/github/tasks/pr_extractor.go
@@ -127,7 +127,7 @@ func ExtractApiPullRequests(taskCtx core.SubTaskContext) error {
githubPr.AuthorId = githubUser.AccountId
}
for _, label := range rawL.Labels {
- results = append(results, &models.GithubPullRequestLabel{
+ results = append(results, &models.GithubPrLabel{
ConnectionId: data.Options.ConnectionId,
PullId: githubPr.GithubId,
LabelName: label.Name,
diff --git a/plugins/github/tasks/pr_issue_convertor.go b/plugins/github/tasks/pr_issue_convertor.go
index 10b9cac8..3de1db86 100644
--- a/plugins/github/tasks/pr_issue_convertor.go
+++ b/plugins/github/tasks/pr_issue_convertor.go
@@ -43,7 +43,7 @@ func ConvertPullRequestIssues(taskCtx core.SubTaskContext) error {
repoId := data.Repo.GithubId
cursor, err := db.Cursor(
- dal.From(&githubModels.GithubPullRequestIssue{}),
+ dal.From(&githubModels.GithubPrIssue{}),
dal.Join(`left join _tool_github_pull_requests on _tool_github_pull_requests.github_id = _tool_github_pull_request_issues.pull_request_id`),
dal.Where("_tool_github_pull_requests.repo_id = ? and _tool_github_pull_requests.connection_id = ?", repoId, data.Options.ConnectionId),
dal.Orderby("pull_request_id ASC"),
@@ -56,7 +56,7 @@ func ConvertPullRequestIssues(taskCtx core.SubTaskContext) error {
issueIdGen := didgen.NewDomainIdGenerator(&githubModels.GithubIssue{})
converter, err := helper.NewDataConverter(helper.DataConverterArgs{
- InputRowType: reflect.TypeOf(githubModels.GithubPullRequestIssue{}),
+ InputRowType: reflect.TypeOf(githubModels.GithubPrIssue{}),
Input: cursor,
RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
Ctx: taskCtx,
@@ -68,7 +68,7 @@ func ConvertPullRequestIssues(taskCtx core.SubTaskContext) error {
Table: RAW_PULL_REQUEST_TABLE,
},
Convert: func(inputRow interface{}) ([]interface{}, error) {
- githubPrIssue := inputRow.(*githubModels.GithubPullRequestIssue)
+ githubPrIssue := inputRow.(*githubModels.GithubPrIssue)
pullRequestIssue := &crossdomain.PullRequestIssue{
PullRequestId: prIdGen.Generate(data.Options.ConnectionId, githubPrIssue.PullRequestId),
IssueId: issueIdGen.Generate(data.Options.ConnectionId, githubPrIssue.IssueId),
diff --git a/plugins/github/tasks/pr_issue_enricher.go b/plugins/github/tasks/pr_issue_enricher.go
index 0cc66285..8c743027 100644
--- a/plugins/github/tasks/pr_issue_enricher.go
+++ b/plugins/github/tasks/pr_issue_enricher.go
@@ -113,7 +113,7 @@ func EnrichPullRequestIssues(taskCtx core.SubTaskContext) (err error) {
if issue.Number == 0 {
continue
}
- githubPullRequstIssue := &githubModels.GithubPullRequestIssue{
+ githubPullRequstIssue := &githubModels.GithubPrIssue{
ConnectionId: data.Options.ConnectionId,
PullRequestId: githubPullRequst.GithubId,
IssueId: issue.GithubId,
diff --git a/plugins/github/tasks/pr_label_convertor.go b/plugins/github/tasks/pr_label_convertor.go
index bfa6f109..4a0bf10e 100644
--- a/plugins/github/tasks/pr_label_convertor.go
+++ b/plugins/github/tasks/pr_label_convertor.go
@@ -43,7 +43,7 @@ func ConvertPullRequestLabels(taskCtx core.SubTaskContext) error {
repoId := data.Repo.GithubId
cursor, err := db.Cursor(
- dal.From(&githubModels.GithubPullRequestLabel{}),
+ dal.From(&githubModels.GithubPrLabel{}),
dal.Join(`left join _tool_github_pull_requests on _tool_github_pull_requests.github_id = _tool_github_pull_request_labels.pull_id`),
dal.Where("_tool_github_pull_requests.repo_id = ? and _tool_github_pull_requests.connection_id = ?", repoId, data.Options.ConnectionId),
dal.Orderby("pull_id ASC"),
@@ -55,7 +55,7 @@ func ConvertPullRequestLabels(taskCtx core.SubTaskContext) error {
prIdGen := didgen.NewDomainIdGenerator(&githubModels.GithubPullRequest{})
converter, err := helper.NewDataConverter(helper.DataConverterArgs{
- InputRowType: reflect.TypeOf(githubModels.GithubPullRequestLabel{}),
+ InputRowType: reflect.TypeOf(githubModels.GithubPrLabel{}),
Input: cursor,
RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
Ctx: taskCtx,
@@ -67,7 +67,7 @@ func ConvertPullRequestLabels(taskCtx core.SubTaskContext) error {
Table: RAW_PULL_REQUEST_TABLE,
},
Convert: func(inputRow interface{}) ([]interface{}, error) {
- prLabel := inputRow.(*githubModels.GithubPullRequestLabel)
+ prLabel := inputRow.(*githubModels.GithubPrLabel)
domainPrLabel := &code.PullRequestLabel{
PullRequestId: prIdGen.Generate(data.Options.ConnectionId, prLabel.PullId),
LabelName: prLabel.LabelName,
diff --git a/plugins/github/tasks/pr_review_collector.go b/plugins/github/tasks/pr_review_collector.go
index 3364b251..38727472 100644
--- a/plugins/github/tasks/pr_review_collector.go
+++ b/plugins/github/tasks/pr_review_collector.go
@@ -20,19 +20,18 @@ package tasks
import (
"encoding/json"
"fmt"
+ "github.com/apache/incubator-devlake/plugins/core/dal"
+ "github.com/apache/incubator-devlake/plugins/github/models"
"net/http"
"net/url"
"reflect"
- "github.com/apache/incubator-devlake/plugins/core/dal"
- "github.com/apache/incubator-devlake/plugins/github/models"
-
"github.com/apache/incubator-devlake/plugins/helper"
"github.com/apache/incubator-devlake/plugins/core"
)
-const RAW_PULL_REQUEST_REVIEW_TABLE = "github_api_pull_request_reviews"
+const RAW_PR_REVIEW_TABLE = "github_api_pull_request_reviews"
// this struct should be moved to `gitub_api_common.go`
@@ -47,19 +46,50 @@ var CollectApiPullRequestReviewsMeta = core.SubTaskMeta{
func CollectApiPullRequestReviews(taskCtx core.SubTaskContext) error {
db := taskCtx.GetDal()
data := taskCtx.GetData().(*GithubTaskData)
-
- cursor, err := db.Cursor(
+ since := data.Since
+
+ incremental := false
+ if since == nil {
+ var latestUpdatedPrReview models.GithubPrReview
+ err := db.All(
+ &latestUpdatedPrReview,
+ dal.Join(`left join _tool_github_pull_requests on
+ _tool_github_pull_requests.github_id = _tool_github_pull_request_reviews.pull_request_id
+ and _tool_github_pull_requests.connection_id = _tool_github_pull_request_reviews.connection_id`),
+ dal.Where(
+ "_tool_github_pull_requests.repo_id = ? AND _tool_github_pull_requests.connection_id = ?", data.Repo.GithubId, data.Repo.ConnectionId,
+ ),
+ dal.Orderby("github_updated_at DESC"),
+ dal.Limit(1),
+ )
+ if err != nil {
+ return fmt.Errorf("failed to get latest github issue record: %w", err)
+ }
+ if latestUpdatedPrReview.GithubId > 0 {
+ since = latestUpdatedPrReview.GithubSubmitAt
+ incremental = true
+ }
+ }
+ clauses := []dal.Clause{
dal.Select("number, github_id"),
dal.From(models.GithubPullRequest{}.TableName()),
dal.Where("repo_id = ? and connection_id=?", data.Repo.GithubId, data.Options.ConnectionId),
+ }
+ if since != nil {
+ clauses = append(clauses, dal.Where("github_updated_at > ?", *since))
+ }
+ cursor, err := db.Cursor(
+ clauses...,
)
if err != nil {
return err
}
+
iterator, err := helper.NewDalCursorIterator(db, cursor, reflect.TypeOf(SimplePr{}))
if err != nil {
return err
}
+
collector, err := helper.NewApiCollector(helper.ApiCollectorArgs{
RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
Ctx: taskCtx,
@@ -76,18 +106,17 @@ func CollectApiPullRequestReviews(taskCtx core.SubTaskContext) error {
/*
Table store raw data
*/
- Table: RAW_PULL_REQUEST_REVIEW_TABLE,
+ Table: RAW_PR_REVIEW_TABLE,
},
ApiClient: data.ApiClient,
PageSize: 100,
- Incremental: false,
+ Incremental: incremental,
Input: iterator,
UrlTemplate: "repos/{{ .Params.Owner }}/{{ .Params.Repo }}/pulls/{{ .Input.Number }}/reviews",
Query: func(reqData *helper.RequestData) (url.Values, error) {
query := url.Values{}
- query.Set("state", "all")
query.Set("page", fmt.Sprintf("%v", reqData.Pager.Page))
query.Set("direction", "asc")
query.Set("per_page", fmt.Sprintf("%v", reqData.Pager.Size))
diff --git a/plugins/github/tasks/pr_review_collector.go b/plugins/github/tasks/pr_review_comment_collector.go
similarity index 58%
copy from plugins/github/tasks/pr_review_collector.go
copy to plugins/github/tasks/pr_review_comment_collector.go
index 3364b251..7551d98b 100644
--- a/plugins/github/tasks/pr_review_collector.go
+++ b/plugins/github/tasks/pr_review_comment_collector.go
@@ -20,80 +20,73 @@ package tasks
import (
"encoding/json"
"fmt"
- "net/http"
- "net/url"
- "reflect"
-
+ "github.com/apache/incubator-devlake/plugins/core"
"github.com/apache/incubator-devlake/plugins/core/dal"
"github.com/apache/incubator-devlake/plugins/github/models"
-
"github.com/apache/incubator-devlake/plugins/helper"
-
- "github.com/apache/incubator-devlake/plugins/core"
+ "net/http"
+ "net/url"
)
-const RAW_PULL_REQUEST_REVIEW_TABLE = "github_api_pull_request_reviews"
+const RAW_PR_REVIEW_COMMENTS_TABLE = "github_api_pull_request_review_comments"
-// this struct should be moved to `gitub_api_common.go`
+// this struct should be moved to `github_api_common.go`
-var CollectApiPullRequestReviewsMeta = core.SubTaskMeta{
- Name: "collectApiPullRequestReviews",
- EntryPoint: CollectApiPullRequestReviews,
- EnabledByDefault: true,
- Description: "Collect PullRequestReviews data from Github api",
- DomainTypes: []string{core.DOMAIN_TYPE_CODE},
-}
-
-func CollectApiPullRequestReviews(taskCtx core.SubTaskContext) error {
+func CollectPrReviewComments(taskCtx core.SubTaskContext) error {
db := taskCtx.GetDal()
data := taskCtx.GetData().(*GithubTaskData)
- cursor, err := db.Cursor(
- dal.Select("number, github_id"),
- dal.From(models.GithubPullRequest{}.TableName()),
- dal.Where("repo_id = ? and connection_id=?", data.Repo.GithubId, data.Options.ConnectionId),
- )
- if err != nil {
- return err
- }
- iterator, err := helper.NewDalCursorIterator(db, cursor, reflect.TypeOf(SimplePr{}))
- if err != nil {
- return err
+ since := data.Since
+ incremental := false
+ if since == nil {
+ var latestUpdatedPrReviewComt models.GithubPrComment
+ err := db.All(
+ &latestUpdatedPrReviewComt,
+ dal.Join(`left join _tool_github_pull_requests on
+ _tool_github_pull_requests.github_id = _tool_github_pull_request_comments.pull_request_id
+ and _tool_github_pull_requests.connection_id = _tool_github_pull_request_comments.connection_id`),
+ dal.Where(
+ "_tool_github_pull_requests.repo_id = ? AND _tool_github_pull_requests.connection_id = ? AND _tool_github_pull_request_comments.type = ?",
+ data.Repo.GithubId, data.Repo.ConnectionId, "DIFF",
+ ),
+ dal.Orderby("github_updated_at DESC"),
+ dal.Limit(1),
+ )
+ if err != nil {
+ return fmt.Errorf("failed to get latest github issue record: %w", err)
+ }
+ if latestUpdatedPrReviewComt.GithubId > 0 {
+ since = &latestUpdatedPrReviewComt.GithubUpdatedAt
+ incremental = true
+ }
}
+
collector, err := helper.NewApiCollector(helper.ApiCollectorArgs{
RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
Ctx: taskCtx,
- /*
- This struct will be JSONEncoded and stored into database along with raw data itself, to identity minimal
- set of data to be process, for example, we process JiraIssues by Board
- */
Params: GithubApiParams{
ConnectionId: data.Options.ConnectionId,
Owner: data.Options.Owner,
Repo: data.Options.Repo,
},
-
- /*
- Table store raw data
- */
- Table: RAW_PULL_REQUEST_REVIEW_TABLE,
+ Table: RAW_PR_REVIEW_COMMENTS_TABLE,
},
ApiClient: data.ApiClient,
PageSize: 100,
- Incremental: false,
- Input: iterator,
-
- UrlTemplate: "repos/{{ .Params.Owner }}/{{ .Params.Repo }}/pulls/{{ .Input.Number }}/reviews",
+ Incremental: incremental,
+ UrlTemplate: "repos/{{ .Params.Owner }}/{{ .Params.Repo }}/pulls/comments",
Query: func(reqData *helper.RequestData) (url.Values, error) {
query := url.Values{}
- query.Set("state", "all")
+ if since != nil {
+ query.Set("since", since.String())
+ }
query.Set("page", fmt.Sprintf("%v", reqData.Pager.Page))
query.Set("direction", "asc")
query.Set("per_page", fmt.Sprintf("%v", reqData.Pager.Size))
-
return query, nil
},
+ GetTotalPages: GetTotalPagesFromResponse,
ResponseParser: func(res *http.Response) ([]json.RawMessage, error) {
var items []json.RawMessage
err := helper.UnmarshalResponse(res, &items)
@@ -107,5 +100,14 @@ func CollectApiPullRequestReviews(taskCtx core.SubTaskContext) error {
if err != nil {
return err
}
+
return collector.Execute()
}
+
+var CollectApiPrReviewCommentsMeta = core.SubTaskMeta{
+ Name: "CollectApiPrReviewCommentsMeta",
+ EntryPoint: CollectPrReviewComments,
+ EnabledByDefault: true,
+ Description: "Collect pr review comments data from Github api",
+ DomainTypes: []string{core.DOMAIN_TYPE_CODE},
+}
diff --git a/plugins/github/tasks/pr_review_comment_extractor.go b/plugins/github/tasks/pr_review_comment_extractor.go
new file mode 100644
index 00000000..e15ccf6f
--- /dev/null
+++ b/plugins/github/tasks/pr_review_comment_extractor.go
@@ -0,0 +1,114 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "encoding/json"
+ "fmt"
+ "regexp"
+ "runtime/debug"
+ "strconv"
+
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/github/models"
+ "github.com/apache/incubator-devlake/plugins/helper"
+)
+
+var ExtractApiPrReviewCommentsMeta = core.SubTaskMeta{
+ Name: "extractApiPrReviewComments",
+ EntryPoint: ExtractApiPrReviewComments,
+ EnabledByDefault: true,
+ Description: "Extract raw comment data into tool layer table github_pull_request_comments" +
+ "and github_issue_comments",
+ DomainTypes: []string{core.DOMAIN_TYPE_CODE},
+}
+
+func ExtractApiPrReviewComments(taskCtx core.SubTaskContext) error {
+ data := taskCtx.GetData().(*GithubTaskData)
+ prUrlPattern := fmt.Sprintf(`https\:\/\/api\.github\.com\/repos\/%s\/%s\/pulls\/(\d+)`, data.Options.Owner, data.Options.Repo)
+
+ extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{
+ RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
+ Ctx: taskCtx,
+ Params: GithubApiParams{
+ ConnectionId: data.Options.ConnectionId,
+ Owner: data.Options.Owner,
+ Repo: data.Options.Repo,
+ },
+ Table: RAW_PR_REVIEW_COMMENTS_TABLE,
+ },
+ Extract: func(row *helper.RawData) ([]interface{}, error) {
+ var prReviewComment struct {
+ GithubId int `json:"id"`
+ Body json.RawMessage
+ User *GithubAccountResponse
+ PrUrl string `json:"pull_request_url"`
+ GithubCreatedAt helper.Iso8601Time `json:"created_at"`
+ GithubUpdatedAt helper.Iso8601Time `json:"updated_at"`
+ CommitId string `json:"commit_id"`
+ PrReviewId int `json:"pull_request_review_id"`
+ }
+ err := json.Unmarshal(row.Data, &prReviewComment)
+ if err != nil {
+ return nil, err
+ }
+
+ results := make([]interface{}, 0, 1)
+ githubPrComment := &models.GithubPrComment{
+ ConnectionId: data.Options.ConnectionId,
+ GithubId: prReviewComment.GithubId,
+ Body: string(prReviewComment.Body),
+ CommitSha: prReviewComment.CommitId,
+ ReviewId: prReviewComment.PrReviewId,
+ AuthorUsername: prReviewComment.User.Login,
+ AuthorUserId: prReviewComment.User.Id,
+ GithubCreatedAt: prReviewComment.GithubCreatedAt.ToTime(),
+ GithubUpdatedAt: prReviewComment.GithubUpdatedAt.ToTime(),
+ Type: "DIFF",
+ }
+
+ prUrlRegex, err := regexp.Compile(prUrlPattern)
+ if err != nil {
+ return nil, fmt.Errorf("regexp Compile prUrlPattern failed:[%s] stack:[%s]", err.Error(), debug.Stack())
+ }
+ if prUrlRegex != nil {
+ groups := prUrlRegex.FindStringSubmatch(prReviewComment.PrUrl)
+ if len(groups) > 0 {
+ prId, err := strconv.Atoi(groups[1])
+ if err != nil {
+ return nil, fmt.Errorf("parse prId failed:[%s] stack:[%s]", err.Error(), debug.Stack())
+ }
+ githubPrComment.PullRequestId = prId
+ }
+ }
+ results = append(results, githubPrComment)
+ githubAccount, err := convertAccount(prReviewComment.User, data.Repo.GithubId, data.Options.ConnectionId)
+ if err != nil {
+ return nil, err
+ }
+ results = append(results, githubAccount)
+ return results, nil
+ },
+ })
+
+ if err != nil {
+ return err
+ }
+
+ return extractor.Execute()
+}
diff --git a/plugins/github/tasks/pr_comment_convertor.go b/plugins/github/tasks/pr_review_convertor.go
similarity index 68%
copy from plugins/github/tasks/pr_comment_convertor.go
copy to plugins/github/tasks/pr_review_convertor.go
index 23004a34..4b1e7bd6 100644
--- a/plugins/github/tasks/pr_comment_convertor.go
+++ b/plugins/github/tasks/pr_review_convertor.go
@@ -29,23 +29,23 @@ import (
"github.com/apache/incubator-devlake/plugins/helper"
)
-var ConvertPullRequestCommentsMeta = core.SubTaskMeta{
- Name: "convertPullRequestComments",
- EntryPoint: ConvertPullRequestComments,
+var ConvertPullRequestReviewsMeta = core.SubTaskMeta{
+ Name: "convertPullRequestReviews",
+ EntryPoint: ConvertPullRequestReviews,
EnabledByDefault: true,
- Description: "ConvertPullRequestComments data from Github api",
+ Description: "ConvertPullRequestReviews data from Github api",
DomainTypes: []string{core.DOMAIN_TYPE_CODE},
}
-func ConvertPullRequestComments(taskCtx core.SubTaskContext) error {
+func ConvertPullRequestReviews(taskCtx core.SubTaskContext) error {
db := taskCtx.GetDal()
data := taskCtx.GetData().(*GithubTaskData)
repoId := data.Repo.GithubId
cursor, err := db.Cursor(
- dal.From(&githubModels.GithubPullRequestComment{}),
+ dal.From(&githubModels.GithubPrReview{}),
dal.Join("left join _tool_github_pull_requests "+
- "on _tool_github_pull_requests.github_id = _tool_github_pull_request_comments.pull_request_id"),
+ "on _tool_github_pull_requests.github_id = _tool_github_pull_request_reviews.pull_request_id"),
dal.Where("repo_id = ? and _tool_github_pull_requests.connection_id = ?", repoId, data.Options.ConnectionId),
)
if err != nil {
@@ -53,11 +53,12 @@ func ConvertPullRequestComments(taskCtx core.SubTaskContext) error {
}
defer cursor.Close()
+ prReviewUIdGen := didgen.NewDomainIdGenerator(&githubModels.GithubPrReview{})
prIdGen := didgen.NewDomainIdGenerator(&githubModels.GithubPullRequest{})
accountIdGen := didgen.NewDomainIdGenerator(&githubModels.GithubAccount{})
converter, err := helper.NewDataConverter(helper.DataConverterArgs{
- InputRowType: reflect.TypeOf(githubModels.GithubPullRequestComment{}),
+ InputRowType: reflect.TypeOf(githubModels.GithubPrReview{}),
Input: cursor,
RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
Ctx: taskCtx,
@@ -66,23 +67,26 @@ func ConvertPullRequestComments(taskCtx core.SubTaskContext) error {
Owner: data.Options.Owner,
Repo: data.Options.Repo,
},
- Table: RAW_COMMENTS_TABLE,
+ Table: RAW_PR_REVIEW_TABLE,
},
Convert: func(inputRow interface{}) ([]interface{}, error) {
- githubPullRequestComment := inputRow.(*githubModels.GithubPullRequestComment)
- domainPrComment := &code.PullRequestComment{
+ githubPullRequestReview := inputRow.(*githubModels.GithubPrReview)
+ domainPrReview := &code.PullRequestComment{
DomainEntity: domainlayer.DomainEntity{
- Id: prIdGen.Generate(data.Options.ConnectionId, githubPullRequestComment.GithubId),
+ Id: prReviewUIdGen.Generate(data.Options.ConnectionId, githubPullRequestReview.GithubId),
},
- PullRequestId: prIdGen.Generate(data.Options.ConnectionId, githubPullRequestComment.PullRequestId),
- Body: githubPullRequestComment.Body,
- UserId: accountIdGen.Generate(data.Options.ConnectionId, githubPullRequestComment.AuthorUserId),
- CreatedDate: githubPullRequestComment.GithubCreatedAt,
- CommitSha: "",
- Position: 0,
+ PullRequestId: prIdGen.Generate(data.Options.ConnectionId, githubPullRequestReview.PullRequestId),
+ Body: githubPullRequestReview.Body,
+ UserId: accountIdGen.Generate(data.Options.ConnectionId, githubPullRequestReview.AuthorUserId),
+ CommitSha: githubPullRequestReview.CommitSha,
+ Type: "REVIEW",
+ Status: githubPullRequestReview.State,
+ }
+ if githubPullRequestReview.GithubSubmitAt != nil {
+ domainPrReview.CreatedDate = *githubPullRequestReview.GithubSubmitAt
}
return []interface{}{
- domainPrComment,
+ domainPrReview,
}, nil
},
})
diff --git a/plugins/github/tasks/pr_review_extractor.go b/plugins/github/tasks/pr_review_extractor.go
index d958d2ab..c1357c63 100644
--- a/plugins/github/tasks/pr_review_extractor.go
+++ b/plugins/github/tasks/pr_review_extractor.go
@@ -26,9 +26,9 @@ import (
"github.com/apache/incubator-devlake/plugins/helper"
)
-var ExtractApiPullRequestReviewersMeta = core.SubTaskMeta{
- Name: "extractApiPullRequestReviewers",
- EntryPoint: ExtractApiPullRequestReviewers,
+var ExtractApiPullRequestReviewsMeta = core.SubTaskMeta{
+ Name: "extractApiPullRequestReviews",
+ EntryPoint: ExtractApiPullRequestReviews,
EnabledByDefault: true,
Description: "Extract raw PullRequestReviewers data into tool layer table github_reviewers",
DomainTypes: []string{core.DOMAIN_TYPE_CODE},
@@ -39,10 +39,11 @@ type PullRequestReview struct {
User *GithubAccountResponse
Body string
State string
+ CommitId string `json:"commit_id"`
SubmittedAt helper.Iso8601Time `json:"submitted_at"`
}
-func ExtractApiPullRequestReviewers(taskCtx core.SubTaskContext) error {
+func ExtractApiPullRequestReviews(taskCtx core.SubTaskContext) error {
data := taskCtx.GetData().(*GithubTaskData)
extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{
RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
@@ -59,7 +60,7 @@ func ExtractApiPullRequestReviewers(taskCtx core.SubTaskContext) error {
/*
Table store raw data
*/
- Table: RAW_PULL_REQUEST_REVIEW_TABLE,
+ Table: RAW_PR_REVIEW_TABLE,
},
Extract: func(row *helper.RawData) ([]interface{}, error) {
apiPullRequestReview := &PullRequestReview{}
@@ -70,6 +71,9 @@ func ExtractApiPullRequestReviewers(taskCtx core.SubTaskContext) error {
if err != nil {
return nil, err
}
+ if apiPullRequestReview.State == "PENDING" {
+ return nil, nil
+ }
pull := &SimplePr{}
err = json.Unmarshal(row.Input, pull)
if err != nil {
@@ -84,7 +88,22 @@ func ExtractApiPullRequestReviewers(taskCtx core.SubTaskContext) error {
Login: apiPullRequestReview.User.Login,
PullRequestId: pull.GithubId,
}
+
+ githubPrReview := &models.GithubPrReview{
+ ConnectionId: data.Options.ConnectionId,
+ GithubId: apiPullRequestReview.GithubId,
+ Body: apiPullRequestReview.Body,
+ State: apiPullRequestReview.State,
+ CommitSha: apiPullRequestReview.CommitId,
+ GithubSubmitAt: apiPullRequestReview.SubmittedAt.ToNullableTime(),
+
+ PullRequestId: pull.GithubId,
+ AuthorUsername: apiPullRequestReview.User.Login,
+ AuthorUserId: apiPullRequestReview.User.Id,
+ }
+
results = append(results, githubReviewer)
+ results = append(results, githubPrReview)
githubUser, err := convertAccount(apiPullRequestReview.User, data.Repo.GithubId, data.Options.ConnectionId)
if err != nil {
return nil, err
diff --git a/plugins/gitlab/impl/impl.go b/plugins/gitlab/impl/impl.go
index b6bf0113..fe21fc65 100644
--- a/plugins/gitlab/impl/impl.go
+++ b/plugins/gitlab/impl/impl.go
@@ -54,10 +54,6 @@ func (plugin Gitlab) SubTaskMetas() []core.SubTaskMeta {
return []core.SubTaskMeta{
tasks.CollectProjectMeta,
tasks.ExtractProjectMeta,
- //tasks.CollectApiCommitsMeta,
- //tasks.ExtractApiCommitsMeta,
- //tasks.CollectTagMeta,
- //tasks.ExtractTagMeta,
tasks.CollectApiIssuesMeta,
tasks.ExtractApiIssuesMeta,
tasks.CollectApiMergeRequestsMeta,
diff --git a/plugins/helper/api_async_client.go b/plugins/helper/api_async_client.go
index 899bde74..8a060df4 100644
--- a/plugins/helper/api_async_client.go
+++ b/plugins/helper/api_async_client.go
@@ -128,7 +128,7 @@ func (apiClient *ApiAsyncClient) SetMaxRetry(
apiClient.maxRetry = maxRetry
}
-// DoAsync would carry out a asynchronous request
+// DoAsync would carry out an asynchronous request
func (apiClient *ApiAsyncClient) DoAsync(
method string,
path string,
diff --git a/plugins/helper/api_collector.go b/plugins/helper/api_collector.go
index a9b7390d..eadff643 100644
--- a/plugins/helper/api_collector.go
+++ b/plugins/helper/api_collector.go
@@ -178,7 +178,7 @@ func (collector *ApiCollector) Execute() error {
if err != nil {
return err
}
- logger.Debug("wait for all async api to finished")
+ logger.Debug("wait for all async api to be finished")
err = collector.args.ApiClient.WaitAsync()
if err != nil {
logger.Info("end api collection error: %w", err)