You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by kl...@apache.org on 2022/06/16 09:58:17 UTC
[incubator-devlake] branch main updated: github issue task refactor (#2202)
This is an automated email from the ASF dual-hosted git repository.
klesh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new ea6542a0 github issue task refactor (#2202)
ea6542a0 is described below
commit ea6542a02ff1fa63aa08ffd52423a90ace7c174c
Author: likyh <l...@likyh.com>
AuthorDate: Thu Jun 16 17:58:14 2022 +0800
github issue task refactor (#2202)
* fix repo
* append ConnectionId for repo in github
* add repo raw table
* add csv
* update csv for connectionId
* update csv for connectionId 2
* add issue extract and some old e2e test
* update connection id for issue in github
* append
* fix for err check
Co-authored-by: linyh <ya...@meri.co>
---
helpers/e2ehelper/data_flow_tester.go | 17 ++-
plugins/github/e2e/issue_test.go | 160 +++++++++++++++++++++
.../e2e/raw_tables/_raw_github_api_issues.csv | 42 ++++++
.../snapshot_tables/_tool_github_issue_labels.csv | 12 ++
.../e2e/snapshot_tables/_tool_github_issues.csv | 27 ++++
.../github/e2e/snapshot_tables/board_issues.csv | 27 ++++
.../github/e2e/snapshot_tables/issue_labels.csv | 12 ++
plugins/github/e2e/snapshot_tables/issues.csv | 27 ++++
plugins/github/models/issue.go | 1 +
plugins/github/models/issue_label.go | 5 +-
.../models/migrationscripts/archived/issue.go | 1 +
.../migrationscripts/archived/issue_label.go | 5 +-
..._schema_20220610.go => init_schema_20220611.go} | 4 +-
plugins/github/tasks/issue_collector.go | 17 ++-
plugins/github/tasks/issue_convertor.go | 18 ++-
plugins/github/tasks/issue_extractor.go | 19 +--
plugins/github/tasks/issue_label_convertor.go | 21 +--
17 files changed, 376 insertions(+), 39 deletions(-)
diff --git a/helpers/e2ehelper/data_flow_tester.go b/helpers/e2ehelper/data_flow_tester.go
index 4f30f681..53bc83c9 100644
--- a/helpers/e2ehelper/data_flow_tester.go
+++ b/helpers/e2ehelper/data_flow_tester.go
@@ -226,7 +226,7 @@ func (t *DataFlowTester) VerifyTable(dst schema.Tabler, csvRelPath string, pkfie
location, _ := time.LoadLocation(`UTC`)
defer csvIter.Close()
- var expectedTotal int64
+ expectedTotal := int64(0)
for csvIter.HasNext() {
expected := csvIter.Fetch()
pkvalues := make([]interface{}, 0, len(pkfields))
@@ -238,7 +238,18 @@ func (t *DataFlowTester) VerifyTable(dst schema.Tabler, csvRelPath string, pkfie
for _, field := range pkfields {
where = append(where, fmt.Sprintf(" %s = ?", field))
}
- err := t.Db.Table(dst.TableName()).Where(strings.Join(where, ` AND `), pkvalues...).Find(actual).Error
+
+ var actualCount int64
+ err := t.Db.Table(dst.TableName()).Where(strings.Join(where, ` AND `), pkvalues...).Count(&actualCount).Error
+ if err != nil {
+ panic(err)
+ }
+ assert.Equal(t.T, int64(1), actualCount, fmt.Sprintf(`%s found not eq 1 but %d (with params from csv %s)`, dst.TableName(), actualCount, pkvalues))
+ if actualCount != 1 {
+ continue
+ }
+
+ err = t.Db.Table(dst.TableName()).Where(strings.Join(where, ` AND `), pkvalues...).Find(actual).Error
if err != nil {
panic(err)
}
@@ -260,7 +271,7 @@ func (t *DataFlowTester) VerifyTable(dst schema.Tabler, csvRelPath string, pkfie
actualValue = fmt.Sprint(actual[field])
}
}
- assert.Equal(t.T, expected[field], actualValue, fmt.Sprintf(`%s.%s not match`, dst.TableName(), field))
+ assert.Equal(t.T, expected[field], actualValue, fmt.Sprintf(`%s.%s not match (with params from csv %s)`, dst.TableName(), field, pkvalues))
}
expectedTotal++
}
diff --git a/plugins/github/e2e/issue_test.go b/plugins/github/e2e/issue_test.go
new file mode 100644
index 00000000..068d1128
--- /dev/null
+++ b/plugins/github/e2e/issue_test.go
@@ -0,0 +1,160 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+ "fmt"
+ "github.com/apache/incubator-devlake/models/domainlayer/ticket"
+ "github.com/apache/incubator-devlake/plugins/github/models"
+ "testing"
+
+ "github.com/apache/incubator-devlake/helpers/e2ehelper"
+ "github.com/apache/incubator-devlake/plugins/github/impl"
+ "github.com/apache/incubator-devlake/plugins/github/tasks"
+)
+
+func TestIssueDataFlow(t *testing.T) {
+ var plugin impl.Github
+ dataflowTester := e2ehelper.NewDataFlowTester(t, "gitlab", plugin)
+
+ githubRepository := &models.GithubRepo{
+ GithubId: 134018330,
+ }
+ taskData := &tasks.GithubTaskData{
+ Options: &tasks.GithubOptions{
+ ConnectionId: 1,
+ Owner: "panjf2000",
+ Repo: "ants",
+ Config: models.Config{
+ PrType: "type/(.*)$",
+ PrComponent: "component/(.*)$",
+ PrBodyClosePattern: "(?mi)(fix|close|resolve|fixes|closes|resolves|fixed|closed|resolved)[\\s]*.*(((and )?(#|https:\\/\\/github.com\\/%s\\/%s\\/issues\\/)\\d+[ ]*)+)",
+ IssueSeverity: "severity/(.*)$",
+ IssuePriority: "^(highest|high|medium|low)$",
+ IssueComponent: "component/(.*)$",
+ IssueTypeBug: "^(bug|failure|error)$",
+ IssueTypeIncident: "",
+ IssueTypeRequirement: "^(feat|feature|proposal|requirement)$",
+ },
+ },
+ Repo: githubRepository,
+ }
+
+ // import raw data table
+ dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_github_api_issues.csv", "_raw_github_api_issues")
+
+ // verify extraction
+ dataflowTester.FlushTabler(&models.GithubIssue{})
+ dataflowTester.FlushTabler(&models.GithubIssueLabel{})
+ dataflowTester.Subtask(tasks.ExtractApiIssuesMeta, taskData)
+ dataflowTester.VerifyTable(
+ models.GithubIssue{},
+ fmt.Sprintf("./snapshot_tables/%s.csv", models.GithubIssue{}.TableName()),
+ []string{"connection_id", "github_id", "repo_id"},
+ []string{
+ "number",
+ "state",
+ "title",
+ "body",
+ "priority",
+ "type",
+ "status",
+ "author_id",
+ "author_name",
+ "assignee_id",
+ "assignee_name",
+ "lead_time_minutes",
+ "url",
+ "closed_at",
+ "github_created_at",
+ "github_updated_at",
+ "severity",
+ "component",
+ "_raw_data_params",
+ "_raw_data_table",
+ "_raw_data_id",
+ "_raw_data_remark",
+ },
+ )
+
+ // verify extraction
+ dataflowTester.VerifyTable(
+ models.GithubIssueLabel{},
+ fmt.Sprintf("./snapshot_tables/%s.csv", models.GithubIssueLabel{}.TableName()),
+ []string{"connection_id", "issue_id", "label_name"},
+ []string{
+ "_raw_data_params",
+ "_raw_data_table",
+ "_raw_data_id",
+ "_raw_data_remark",
+ },
+ )
+
+ // verify extraction
+ dataflowTester.FlushTabler(&ticket.Issue{})
+ dataflowTester.FlushTabler(&ticket.BoardIssue{})
+ dataflowTester.Subtask(tasks.ConvertIssuesMeta, taskData)
+ dataflowTester.VerifyTable(
+ ticket.Issue{},
+ fmt.Sprintf("./snapshot_tables/%s.csv", ticket.Issue{}.TableName()),
+ []string{"id"},
+ []string{
+ "url",
+ "icon_url",
+ "issue_key",
+ "title",
+ "description",
+ "epic_key",
+ "type",
+ "status",
+ "original_status",
+ "story_point",
+ "resolution_date",
+ "created_date",
+ "updated_date",
+ "lead_time_minutes",
+ "parent_issue_id",
+ "priority",
+ "original_estimate_minutes",
+ "time_spent_minutes",
+ "time_remaining_minutes",
+ "creator_id",
+ "creator_name",
+ "assignee_id",
+ "assignee_name",
+ "severity",
+ "component",
+ },
+ )
+ dataflowTester.VerifyTable(
+ ticket.BoardIssue{},
+ fmt.Sprintf("./snapshot_tables/%s.csv", ticket.BoardIssue{}.TableName()),
+ []string{"board_id", "issue_id"},
+ []string{},
+ )
+
+ // verify extraction
+ dataflowTester.FlushTabler(&ticket.IssueLabel{})
+ dataflowTester.Subtask(tasks.ConvertIssueLabelsMeta, taskData)
+ dataflowTester.VerifyTable(
+ ticket.IssueLabel{},
+ fmt.Sprintf("./snapshot_tables/%s.csv", ticket.IssueLabel{}.TableName()),
+ []string{"issue_id", "label_name"},
+ []string{},
+ )
+}
diff --git a/plugins/github/e2e/raw_tables/_raw_github_api_issues.csv b/plugins/github/e2e/raw_tables/_raw_github_api_issues.csv
new file mode 100644
index 00000000..bbce55ef
--- /dev/null
+++ b/plugins/github/e2e/raw_tables/_raw_github_api_issues.csv
@@ -0,0 +1,42 @@
+id,params,data,url,input,created_at
+10,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/4"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/4/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/4/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/4/events"",""html_url"":""https://github.com/panjf200 [...]
+11,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/5"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/5/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/5/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/5/events"",""html_url"":""https://github.com/panjf200 [...]
+12,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/6"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/6/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/6/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/6/events"",""html_url"":""https://github.com/panjf200 [...]
+13,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/7"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/7/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/7/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/7/events"",""html_url"":""https://github.com/panjf200 [...]
+14,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/8"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/8/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/8/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/8/events"",""html_url"":""https://github.com/panjf200 [...]
+15,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/9"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/9/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/9/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/9/events"",""html_url"":""https://github.com/panjf200 [...]
+16,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/10"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/10/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/10/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/10/events"",""html_url"":""https://github.com/panj [...]
+17,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/11"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/11/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/11/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/11/events"",""html_url"":""https://github.com/panj [...]
+18,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/12"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/12/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/12/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/12/events"",""html_url"":""https://github.com/panj [...]
+19,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/13"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/13/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/13/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/13/events"",""html_url"":""https://github.com/panj [...]
+20,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/14"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/14/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/14/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/14/events"",""html_url"":""https://github.com/panj [...]
+21,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/15"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/15/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/15/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/15/events"",""html_url"":""https://github.com/panj [...]
+22,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/16"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/16/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/16/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/16/events"",""html_url"":""https://github.com/panj [...]
+23,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/17"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/17/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/17/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/17/events"",""html_url"":""https://github.com/panj [...]
+24,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/18"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/18/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/18/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/18/events"",""html_url"":""https://github.com/panj [...]
+25,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/19"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/19/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/19/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/19/events"",""html_url"":""https://github.com/panj [...]
+26,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/20"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/20/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/20/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/20/events"",""html_url"":""https://github.com/panj [...]
+27,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/21"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/21/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/21/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/21/events"",""html_url"":""https://github.com/panj [...]
+28,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/22"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/22/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/22/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/22/events"",""html_url"":""https://github.com/panj [...]
+29,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/23"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/23/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/23/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/23/events"",""html_url"":""https://github.com/panj [...]
+30,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/24"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/24/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/24/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/24/events"",""html_url"":""https://github.com/panj [...]
+31,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/25"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/25/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/25/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/25/events"",""html_url"":""https://github.com/panj [...]
+32,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/26"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/26/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/26/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/26/events"",""html_url"":""https://github.com/panj [...]
+33,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/27"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/27/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/27/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/27/events"",""html_url"":""https://github.com/panj [...]
+34,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/28"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/28/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/28/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/28/events"",""html_url"":""https://github.com/panj [...]
+35,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/29"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/29/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/29/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/29/events"",""html_url"":""https://github.com/panj [...]
+36,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/30"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/30/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/30/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/30/events"",""html_url"":""https://github.com/panj [...]
+37,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/31"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/31/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/31/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/31/events"",""html_url"":""https://github.com/panj [...]
+38,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/32"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/32/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/32/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/32/events"",""html_url"":""https://github.com/panj [...]
+39,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/33"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/33/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/33/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/33/events"",""html_url"":""https://github.com/panj [...]
+40,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/34"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/34/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/34/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/34/events"",""html_url"":""https://github.com/panj [...]
+41,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/35"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/35/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/35/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/35/events"",""html_url"":""https://github.com/panj [...]
+42,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/36"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/36/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/36/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/36/events"",""html_url"":""https://github.com/panj [...]
+43,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/37"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/37/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/37/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/37/events"",""html_url"":""https://github.com/panj [...]
+44,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/38"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/38/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/38/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/38/events"",""html_url"":""https://github.com/panj [...]
+45,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/39"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/39/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/39/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/39/events"",""html_url"":""https://github.com/panj [...]
+46,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/40"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/40/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/40/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/40/events"",""html_url"":""https://github.com/panj [...]
+47,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/41"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/41/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/41/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/41/events"",""html_url"":""https://github.com/panj [...]
+48,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/42"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/42/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/42/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/42/events"",""html_url"":""https://github.com/panj [...]
+49,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/43"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/43/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/43/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/43/events"",""html_url"":""https://github.com/panj [...]
+50,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""url"":""https://api.github.com/repos/panjf2000/ants/issues/44"",""repository_url"":""https://api.github.com/repos/panjf2000/ants"",""labels_url"":""https://api.github.com/repos/panjf2000/ants/issues/44/labels{/name}"",""comments_url"":""https://api.github.com/repos/panjf2000/ants/issues/44/comments"",""events_url"":""https://api.github.com/repos/panjf2000/ants/issues/44/events"",""html_url"":""https://github.com/panj [...]
diff --git a/plugins/github/e2e/snapshot_tables/_tool_github_issue_labels.csv b/plugins/github/e2e/snapshot_tables/_tool_github_issue_labels.csv
new file mode 100644
index 00000000..44b54924
--- /dev/null
+++ b/plugins/github/e2e/snapshot_tables/_tool_github_issue_labels.csv
@@ -0,0 +1,12 @@
+connection_id,issue_id,label_name,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
+1,346842831,question,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_issues,11,
+1,347255859,bug,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_issues,12,
+1,348630179,bug,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_issues,13,
+1,356703393,question,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_issues,16,
+1,381941219,enhancement,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_issues,23,
+1,382039050,question,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_issues,24,
+1,382574800,enhancement,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_issues,26,
+1,401277739,enhancement,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_issues,28,
+1,402513849,enhancement,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_issues,30,
+1,433564955,enhancement,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_issues,38,
+1,434069015,enhancement,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_issues,39,
diff --git a/plugins/github/e2e/snapshot_tables/_tool_github_issues.csv b/plugins/github/e2e/snapshot_tables/_tool_github_issues.csv
new file mode 100644
index 00000000..801fc2dd
--- /dev/null
+++ b/plugins/github/e2e/snapshot_tables/_tool_github_issues.csv
@@ -0,0 +1,27 @@
+connection_id,github_id,repo_id,number,state,title,body,priority,type,status,author_id,author_name,assignee_id,assignee_name,lead_time_minutes,url,closed_at,github_created_at,github_updated_at,severity,component,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
+1,346842831,134018330,5,closed,关于 <-p.freeSignal 的疑惑,"""Hi,\r\n 我阅读了源码，对 `<-p.freeSignal` 这句代码有疑惑。 这句代码出现在了多个地方，freeSignal 的作用英文注释我是理解的，并且知道在 `putWorker` 中才进行 `p.freeSignal <- sig{}`\r\n\r\n对于下面的代码\r\n```\r\nfunc (p *Pool) getWorker() *Worker {\r\n\tvar w *Worker\r\n\twaiting := false\r\n\r\n\tp.lock.Lock()\r\n\tidleWorkers := p.workers\r\n\tn := len(idleWorkers) - 1\r\n\tif n < 0 { // 说明 pool中没有worker了\r\n\t\twaiting = p.Running() >= p.Cap()\r\n\t} else { // 说明pool中有worker\r\n\t\t<-p [...]
+1,347255859,134018330,6,closed,死锁bug,"""func (p *Pool) getWorker() *Worker 这个函数的 199行 \r\n必须先解锁在加锁， 要不然会产生死锁\r\n\r\n\tp.lock.Unlock()\r\n\t\t<-p.freeSignal\r\n\t\tp.lock.Lock()""",,BUG,,13118848,lovelly,0,,1786,https://github.com/panjf2000/ants/issues/6,2018-08-04T10:18:41.000+00:00,2018-08-03T04:32:28.000+00:00,2018-08-04T10:18:41.000+00:00,,,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_issues,12,
+1,348630179,134018330,7,closed,清理过期协程报错,"""你好，非常感谢提供这么好用的工具包。我在使用ants时，发现报异常。结果见下图\r\n\r\n日志是我在periodicallyPurge里加的调试信息\r\n\r\n\r\n### 原因分析\r\n\r\n我认为可能原因是没有处理n==0的情况\r\n```\r\nif n > 0 {\r\n\tn++\r\n\tp.workers = idleWorkers[n:]\r\n}\r\n```\r\n\r\n\r\n### 测试代码\r\n```\r\npa [...]
+1,356703393,134018330,10,closed,高并发下设定较小的worker数量问题,"""会存在cpu飚升的问题吧？""",,,,11763614,Moonlight-Zhao,0,,36198,https://github.com/panjf2000/ants/issues/10,2018-09-29T11:45:00.000+00:00,2018-09-04T08:26:55.000+00:00,2018-09-29T11:45:00.000+00:00,,,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_issues,16,
+1,364361014,134018330,12,closed,潘少，更新下tag吧,"""鄙人现在在弄dep依赖管理，有用到你写的ants项目，可是你好像忘记打最新的tag了。最新的tag 3.6是指向ed55924这个提交，git上的最新代码是af376f1b这次提交，两次提交都隔了快5个月了，看到的话，麻烦打一个最新的tag吧。（手动可怜）""",,,,29452204,edcismybrother,0,,1293,https://github.com/panjf2000/ants/issues/12,2018-09-28T06:05:58.000+00:00,2018-09-27T08:32:25.000+00:00,2019-04-21T08:19:58.000+00:00,,,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_issues,18,
+1,381941219,134018330,17,closed,关于优雅退出的问题,"""关于这个package优雅退出的问题，我看了一下Release的代码：\r\n\r\n`\r\n\t// Release Closed this pool.\r\n\tfunc (p *PoolWithFunc) Release() error {\r\n\t\tp.once.Do(func() {\r\n\t\t\tp.release <- sig{}\r\n\t\t\tp.lock.Lock()\r\n\t\t\tidleWorkers := p.workers\r\n\t\t\tfor i, w := range idleWorkers {\r\n\t\t\t\tw.args <- nil\r\n\t\t\t\tidleWorkers[i] = nil\r\n\t\t\t}\r\n\t\t\tp.workers = nil\r\n\t\t\tp.lock.Unlock()\r\n\t\t})\r\n\t\treturn nil\r\n\t}\r\n`\r\n\r\nrelea [...]
+1,382039050,134018330,18,closed,go协程的理解,"""你好楼主，向您请教一个协程和线程的问题，协程基于go进程调度，线程基于系统内核调度，调度协程的过程是先调度线程后获得资源再去调度协程。\""官方解释： GOMAXPROCS sets the maximum number of CPUs that can be executing simultaneously。限制cpu数，本质上是什么，限制并行数?，并行数即同时执行数量?，执行单元即线程?，即限制最大并行线程数量?\""""",,,,13944100,LinuxForYQH,0,,20213,https://github.com/panjf2000/ants/issues/18,2018-12-03T03:53:50.000+00:00,2018-11-19T02:59:53.000+00:00,2018-12-03T03:53:50.000+00:00,,,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""} [...]
+1,382574800,134018330,20,closed,是否考虑任务支持回调函数处理失败的逻辑和任务依赖,"""#""",,,,5668717,kklinan,0,,95398,https://github.com/panjf2000/ants/issues/20,2019-01-25T15:34:03.000+00:00,2018-11-20T09:36:02.000+00:00,2019-01-25T15:34:03.000+00:00,,,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_issues,26,
+1,388907811,134018330,21,closed,Benchmark 下直接使用 Semaphore 似乎更快呢？,"""简单跑了一下 benchmark，Semaphore 更快且很简单\r\n\r\n```bash\r\n$ go test -bench .\r\ngoos: darwin\r\ngoarch: amd64\r\npkg: github.com/panjf2000/ants\r\nBenchmarkGoroutineWithFunc-4 \t 1\t3445631705 ns/op\r\nBenchmarkSemaphoreWithFunc-4 \t 1\t1037219073 ns/op\r\nBenchmarkAntsPoolWithFunc-4 \t 1\t1138053222 ns/op\r\nBenchmarkGoroutine-4 \t 2\t 731850771 ns/op\r\nBenchmarkSemaphore-4 [...]
+1,401277739,134018330,22,closed,是否考虑 worker 中添加 PanicHandler ？,"""比方说在创建 Pool 的时候传入一个 PanicHandler，然后在每个 worker 创建的时候 recover 之后传给 PanicHandler 处理。否则池子里如果发生 panic 会直接挂掉整个进程。""",,,,8923413,choleraehyq,0,,1174,https://github.com/panjf2000/ants/issues/22,2019-01-22T05:41:34.000+00:00,2019-01-21T10:06:56.000+00:00,2019-01-22T05:41:34.000+00:00,,,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_issues,28,
+1,402513849,134018330,24,closed,提交任务不阻塞,"""`Pool.Submit`和`PoolWithFunc.Server`提交任务，如果没有空的worker，会一直阻塞。建议增加不阻塞的接口，当前失败时直接返回错误。""",,,,5044825,tenfyzhong,0,,300032,https://github.com/panjf2000/ants/issues/24,2019-08-20T10:56:30.000+00:00,2019-01-24T02:24:13.000+00:00,2019-08-20T10:56:30.000+00:00,,,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_issues,30,
+1,405951301,134018330,25,closed,use example errors,"""./antstest.go:37:14: cannot use syncCalculateSum (type func()) as type ants.f in argument to ants.Submit\r\n./antstest.go:45:35: cannot use func literal (type func(interface {})) as type ants.pf in argument to ants.NewPoolWithFunc\r\n""",,,,5244267,jiashiwen,0,,3088,https://github.com/panjf2000/ants/issues/25,2019-02-04T09:11:52.000+00:00,2019-02-02T05:43:38.000+00:00,2019-02-04T09:11:52.000+00:00,,,"{""ConnectionId"":1,""Owner"":""pa [...]
+1,413968505,134018330,26,closed,running可能大于cap的问题,"""running与cap的比较判断与incRuning分开执行的, 可能会出现running大于cap的问题?\r\n`func (p *Pool) retrieveWorker() *Worker {\r\n\tvar w *Worker\r\n\r\n\tp.lock.Lock()\r\n\tidleWorkers := p.workers\r\n\tn := len(idleWorkers) - 1\r\n\tif n >= 0 {\r\n\t\tw = idleWorkers[n]\r\n\t\tidleWorkers[n] = nil\r\n\t\tp.workers = idleWorkers[:n]\r\n\t\tp.lock.Unlock()\r\n\t} else if p.Running() < p.Cap() {\r\n\t\tp.lock.Unlock()\r\n\t\tif cacheWorker := p.workerCache.Get() [...]
+1,419183961,134018330,27,closed,为何goroutine一直上不去，用户量也打不上去,"""为何goroutine一直上不去，用户量也打不上去\r\n是我用的有问题吗？\r\n\r\nwebsocket server\r\nhttps://github.com/im-ai/pushm/blob/master/learn/goroutine/goroutinepoolwebsocket.go\r\n\r\nwebsocket cient\r\nhttps://github.com/im-ai/pushm/blob/master/learn/goroutine/goroutinepoolwebsocketclient.go\r\n""",,,,38367404,liliang8858,0,,37496,https://github.com/panjf2000/ants/issues/27,2019-04-05T14:05:20.000+00:00,2019-03-10T13:08:52.000+00:00,2019-04-05T14:05:20 [...]
+1,419268851,134018330,28,closed,cap 和 running 比较的问题,"""这是我在 Playground 上面的代码 https://play.golang.org/p/D94YUU3FnX6\r\natomic 只能保证自增自减时的原子操作，在比较过程中，其他线程对变量进行了操作 比较过程并无感知，所以这个比较结果 不是完全正确的，想要实现 比较的数量完全正确，只能在修改和比较两个值的地方加锁\r\n像 #26 说的是对的""",,,,29243953,naiba,0,,237002,https://github.com/panjf2000/ants/issues/28,2019-08-22T16:27:37.000+00:00,2019-03-11T02:24:41.000+00:00,2019-08-22T16:27:37.000+00:00,,,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_issues,34,
+1,424634533,134018330,29,closed,任务传参,"""你好，你的项目太酷了👍\r\n\r\nhttps://github.com/panjf2000/ants/blob/master/pool.go#L124 貌似不支持带参数的任务, 请问传参是用闭包的方式吗?\r\n""",,,,8509898,prprprus,0,,999,https://github.com/panjf2000/ants/issues/29,2019-03-25T09:32:11.000+00:00,2019-03-24T16:52:21.000+00:00,2019-03-25T09:45:05.000+00:00,,,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_issues,35,
+1,429972115,134018330,31,closed,Add go.mod,"""""",,,,48135919,tsatke,0,,3474,https://github.com/panjf2000/ants/issues/31,2019-04-08T09:45:31.000+00:00,2019-04-05T23:50:36.000+00:00,2019-10-17T03:12:19.000+00:00,,,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_issues,37,
+1,433564955,134018330,32,closed,关于版本问题，我发现小版本(0.0.x)这种更新就会不向下兼容？,"""如题，我感觉这样不好。\r\n\r\n功能版本号不向下兼容能理解\r\n\r\n修复问题的版本号也不向下兼容，难以理解。""",,,,7931755,zplzpl,0,,7440,https://github.com/panjf2000/ants/issues/32,2019-04-21T07:16:26.000+00:00,2019-04-16T03:16:02.000+00:00,2019-04-21T07:16:26.000+00:00,,,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_issues,38,
+1,434069015,134018330,33,closed,support semantic versioning.,"""建议将发布的tag兼容为semantic versioning，vX.Y.Z。go modules对此支持比较良好。\r\nhttps://semver.org/\r\nhttps://research.swtch.com/vgo-import""",,,,1284892,jjeffcaii,0,,6090,https://github.com/panjf2000/ants/issues/33,2019-04-21T08:25:20.000+00:00,2019-04-17T02:55:11.000+00:00,2019-04-21T08:25:20.000+00:00,,,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_issues,39,
+1,435486645,134018330,34,closed,Important announcement about <ants> from author !!!,"""**Dear users of `ants`:**\r\nI am apologetically telling you that I have to dump all tags which already presents in `ants` repository.\r\n\r\nThe reason why I'm doing so is to standardize the version management with `Semantic Versioning`, which will make a formal and clear dependency management in go, for go modules, godep, or glide, etc. So I decide to start over the tag sequence, you could find more [...]
+1,461280653,134018330,35,closed,worker exit on panic,"""个人认为PanicHandler设计不妥。\r\n1.无PanicHandler时，抛出给外面的不是panic,外层感受不到。\r\n2.无论有没有PanicHandler，都会导致worker退出，最终pool阻塞住全部任务。""",,,,38849208,king526,0,,74481,https://github.com/panjf2000/ants/issues/35,2019-08-17T20:33:10.000+00:00,2019-06-27T03:11:49.000+00:00,2019-08-17T20:33:10.000+00:00,,,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_issues,41,
+1,462631417,134018330,37,closed,请不要再随意变更版本号了。。。,"""之前用的是 3.9.9，结果今天构建出了问题，一看发现这个版本没了，变成 1.0.0。这种变更完全不考虑现有用户的情况。希望以后不要随意变更了""",,,,8923413,choleraehyq,0,,140,https://github.com/panjf2000/ants/issues/37,2019-07-01T12:37:55.000+00:00,2019-07-01T10:17:15.000+00:00,2019-07-02T10:17:31.000+00:00,,,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_issues,43,
+1,472125082,134018330,38,closed,retrieveWorker与revertWorker之间会导致死锁,"""func (p *Pool) retrieveWorker() *Worker {\r\n\tvar w *Worker\r\n\r\n\t**p.lock.Lock()**\r\n\tidleWorkers := p.workers\r\n\tn := len(idleWorkers) - 1\r\n\tif n >= 0 {\r\n\t\tw = idleWorkers[n]\r\n\t\tidleWorkers[n] = nil\r\n\t\tp.workers = idleWorkers[:n]\r\n\t\tp.lock.Unlock()\r\n\t} else if p.Running() < p.Cap() {\r\n\t\tp.lock.Unlock()\r\n\t\tif cacheWorker := p.workerCache.Get(); cacheWorker != nil {\r\n\t\t\tw = ca [...]
+1,483164833,134018330,42,closed,带选项的初始化函数，我觉得用 functional options 更好一点,"""以下是示意代码\r\n如果用 functional options，原来的写法是\r\n```\r\nants.NewPool(10)\r\n```\r\n新的写法，如果不加 option，写法是不变的，因为 options 是作为可变参数传进去的。如果要加 option，只需要改成\r\n```\r\nants.NewPool(10, ants.WithNonblocking(true))\r\n```\r\n这样。\r\n\r\n现在是直接传一个 Option 结构体进去，所有的地方都要改，感觉很不优雅。\r\n具体 functional options 的设计可以看 rob pike 的一篇博客 https://commandcenter.blogspot.com/2014/01/self-referential-functions-and-design.html""",,,,8923413,choleraehyq,0 [...]
+1,483736247,134018330,43,closed,1.3.0 是不兼容更新,"""Pool 里那些暴露出来的字段（PanicHandler 之类的）都没了，这是一个不兼容更新，根据语义化版本的要求要发大版本。""",,,,8923413,choleraehyq,0,,652,https://github.com/panjf2000/ants/issues/43,2019-08-22T13:22:10.000+00:00,2019-08-22T02:29:34.000+00:00,2019-08-22T13:22:10.000+00:00,,,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_issues,49,
+1,484311063,134018330,44,closed,1.1.1 -> 1.2.0 也是不兼容更新,"""Pool.Release 的返回值没了""",,,,8923413,choleraehyq,0,,3068,https://github.com/panjf2000/ants/issues/44,2019-08-25T06:36:14.000+00:00,2019-08-23T03:27:38.000+00:00,2019-08-25T06:36:14.000+00:00,,,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_issues,50,
diff --git a/plugins/github/e2e/snapshot_tables/board_issues.csv b/plugins/github/e2e/snapshot_tables/board_issues.csv
new file mode 100644
index 00000000..9b2423b7
--- /dev/null
+++ b/plugins/github/e2e/snapshot_tables/board_issues.csv
@@ -0,0 +1,27 @@
+board_id,issue_id
+gitlab:GithubRepo:1:134018330,gitlab:GithubIssue:1:346842831
+gitlab:GithubRepo:1:134018330,gitlab:GithubIssue:1:347255859
+gitlab:GithubRepo:1:134018330,gitlab:GithubIssue:1:348630179
+gitlab:GithubRepo:1:134018330,gitlab:GithubIssue:1:356703393
+gitlab:GithubRepo:1:134018330,gitlab:GithubIssue:1:364361014
+gitlab:GithubRepo:1:134018330,gitlab:GithubIssue:1:381941219
+gitlab:GithubRepo:1:134018330,gitlab:GithubIssue:1:382039050
+gitlab:GithubRepo:1:134018330,gitlab:GithubIssue:1:382574800
+gitlab:GithubRepo:1:134018330,gitlab:GithubIssue:1:388907811
+gitlab:GithubRepo:1:134018330,gitlab:GithubIssue:1:401277739
+gitlab:GithubRepo:1:134018330,gitlab:GithubIssue:1:402513849
+gitlab:GithubRepo:1:134018330,gitlab:GithubIssue:1:405951301
+gitlab:GithubRepo:1:134018330,gitlab:GithubIssue:1:413968505
+gitlab:GithubRepo:1:134018330,gitlab:GithubIssue:1:419183961
+gitlab:GithubRepo:1:134018330,gitlab:GithubIssue:1:419268851
+gitlab:GithubRepo:1:134018330,gitlab:GithubIssue:1:424634533
+gitlab:GithubRepo:1:134018330,gitlab:GithubIssue:1:429972115
+gitlab:GithubRepo:1:134018330,gitlab:GithubIssue:1:433564955
+gitlab:GithubRepo:1:134018330,gitlab:GithubIssue:1:434069015
+gitlab:GithubRepo:1:134018330,gitlab:GithubIssue:1:435486645
+gitlab:GithubRepo:1:134018330,gitlab:GithubIssue:1:461280653
+gitlab:GithubRepo:1:134018330,gitlab:GithubIssue:1:462631417
+gitlab:GithubRepo:1:134018330,gitlab:GithubIssue:1:472125082
+gitlab:GithubRepo:1:134018330,gitlab:GithubIssue:1:483164833
+gitlab:GithubRepo:1:134018330,gitlab:GithubIssue:1:483736247
+gitlab:GithubRepo:1:134018330,gitlab:GithubIssue:1:484311063
diff --git a/plugins/github/e2e/snapshot_tables/issue_labels.csv b/plugins/github/e2e/snapshot_tables/issue_labels.csv
new file mode 100644
index 00000000..c324c08c
--- /dev/null
+++ b/plugins/github/e2e/snapshot_tables/issue_labels.csv
@@ -0,0 +1,12 @@
+issue_id,label_name
+gitlab:GithubIssue:1:346842831,question
+gitlab:GithubIssue:1:347255859,bug
+gitlab:GithubIssue:1:348630179,bug
+gitlab:GithubIssue:1:356703393,question
+gitlab:GithubIssue:1:381941219,enhancement
+gitlab:GithubIssue:1:382039050,question
+gitlab:GithubIssue:1:382574800,enhancement
+gitlab:GithubIssue:1:401277739,enhancement
+gitlab:GithubIssue:1:402513849,enhancement
+gitlab:GithubIssue:1:433564955,enhancement
+gitlab:GithubIssue:1:434069015,enhancement
diff --git a/plugins/github/e2e/snapshot_tables/issues.csv b/plugins/github/e2e/snapshot_tables/issues.csv
new file mode 100644
index 00000000..477c87e7
--- /dev/null
+++ b/plugins/github/e2e/snapshot_tables/issues.csv
@@ -0,0 +1,27 @@
+id,url,icon_url,issue_key,title,description,epic_key,type,status,original_status,story_point,resolution_date,created_date,updated_date,lead_time_minutes,parent_issue_id,priority,original_estimate_minutes,time_spent_minutes,time_remaining_minutes,creator_id,creator_name,assignee_id,assignee_name,severity,component
+gitlab:GithubIssue:1:346842831,https://github.com/panjf2000/ants/issues/5,,5,关于 <-p.freeSignal 的疑惑,"""Hi,\r\n 我阅读了源码，对 `<-p.freeSignal` 这句代码有疑惑。 这句代码出现在了多个地方，freeSignal 的作用英文注释我是理解的，并且知道在 `putWorker` 中才进行 `p.freeSignal <- sig{}`\r\n\r\n对于下面的代码\r\n```\r\nfunc (p *Pool) getWorker() *Worker {\r\n\tvar w *Worker\r\n\twaiting := false\r\n\r\n\tp.lock.Lock()\r\n\tidleWorkers := p.workers\r\n\tn := len(idleWorkers) - 1\r\n\tif n < 0 { // 说明 pool中没有worker了\r\n\t\twaiting = p.Running() >= p.Ca [...]
+gitlab:GithubIssue:1:347255859,https://github.com/panjf2000/ants/issues/6,,6,死锁bug,"""func (p *Pool) getWorker() *Worker 这个函数的 199行 \r\n必须先解锁在加锁， 要不然会产生死锁\r\n\r\n\tp.lock.Unlock()\r\n\t\t<-p.freeSignal\r\n\t\tp.lock.Lock()""",,BUG,DONE,,0,2018-08-04T10:18:41.000+00:00,2018-08-03T04:32:28.000+00:00,2018-08-04T10:18:41.000+00:00,1786,,,0,0,0,gitlab:GithubUser:13118848,lovelly,gitlab:GithubUser:0,,,
+gitlab:GithubIssue:1:348630179,https://github.com/panjf2000/ants/issues/7,,7,清理过期协程报错,"""你好，非常感谢提供这么好用的工具包。我在使用ants时，发现报异常。结果见下图\r\n\r\n日志是我在periodicallyPurge里加的调试信息\r\n\r\n\r\n### 原因分析\r\n\r\n我认为可能原因是没有处理n==0的情况\r\n```\r\nif n > 0 {\r\n\tn++\r\n\tp.workers = idleWorkers[n: [...]
+gitlab:GithubIssue:1:356703393,https://github.com/panjf2000/ants/issues/10,,10,高并发下设定较小的worker数量问题,"""会存在cpu飚升的问题吧？""",,,DONE,,0,2018-09-29T11:45:00.000+00:00,2018-09-04T08:26:55.000+00:00,2018-09-29T11:45:00.000+00:00,36198,,,0,0,0,gitlab:GithubUser:11763614,Moonlight-Zhao,gitlab:GithubUser:0,,,
+gitlab:GithubIssue:1:364361014,https://github.com/panjf2000/ants/issues/12,,12,潘少，更新下tag吧,"""鄙人现在在弄dep依赖管理，有用到你写的ants项目，可是你好像忘记打最新的tag了。最新的tag 3.6是指向ed55924这个提交，git上的最新代码是af376f1b这次提交，两次提交都隔了快5个月了，看到的话，麻烦打一个最新的tag吧。（手动可怜）""",,,DONE,,0,2018-09-28T06:05:58.000+00:00,2018-09-27T08:32:25.000+00:00,2019-04-21T08:19:58.000+00:00,1293,,,0,0,0,gitlab:GithubUser:29452204,edcismybrother,gitlab:GithubUser:0,,,
+gitlab:GithubIssue:1:381941219,https://github.com/panjf2000/ants/issues/17,,17,关于优雅退出的问题,"""关于这个package优雅退出的问题，我看了一下Release的代码：\r\n\r\n`\r\n\t// Release Closed this pool.\r\n\tfunc (p *PoolWithFunc) Release() error {\r\n\t\tp.once.Do(func() {\r\n\t\t\tp.release <- sig{}\r\n\t\t\tp.lock.Lock()\r\n\t\t\tidleWorkers := p.workers\r\n\t\t\tfor i, w := range idleWorkers {\r\n\t\t\t\tw.args <- nil\r\n\t\t\t\tidleWorkers[i] = nil\r\n\t\t\t}\r\n\t\t\tp.workers = nil\r\n\t\t\tp.lock.Unlock()\r\n\t [...]
+gitlab:GithubIssue:1:382039050,https://github.com/panjf2000/ants/issues/18,,18,go协程的理解,"""你好楼主，向您请教一个协程和线程的问题，协程基于go进程调度，线程基于系统内核调度，调度协程的过程是先调度线程后获得资源再去调度协程。\""官方解释： GOMAXPROCS sets the maximum number of CPUs that can be executing simultaneously。限制cpu数，本质上是什么，限制并行数?，并行数即同时执行数量?，执行单元即线程?，即限制最大并行线程数量?\""""",,,DONE,,0,2018-12-03T03:53:50.000+00:00,2018-11-19T02:59:53.000+00:00,2018-12-03T03:53:50.000+00:00,20213,,,0,0,0,gitlab:GithubUser:13944100,LinuxForYQH,gitlab:GithubUser:0,,,
+gitlab:GithubIssue:1:382574800,https://github.com/panjf2000/ants/issues/20,,20,是否考虑任务支持回调函数处理失败的逻辑和任务依赖,"""#""",,,DONE,,0,2019-01-25T15:34:03.000+00:00,2018-11-20T09:36:02.000+00:00,2019-01-25T15:34:03.000+00:00,95398,,,0,0,0,gitlab:GithubUser:5668717,kklinan,gitlab:GithubUser:0,,,
+gitlab:GithubIssue:1:388907811,https://github.com/panjf2000/ants/issues/21,,21,Benchmark 下直接使用 Semaphore 似乎更快呢？,"""简单跑了一下 benchmark，Semaphore 更快且很简单\r\n\r\n```bash\r\n$ go test -bench .\r\ngoos: darwin\r\ngoarch: amd64\r\npkg: github.com/panjf2000/ants\r\nBenchmarkGoroutineWithFunc-4 \t 1\t3445631705 ns/op\r\nBenchmarkSemaphoreWithFunc-4 \t 1\t1037219073 ns/op\r\nBenchmarkAntsPoolWithFunc-4 \t 1\t1138053222 ns/op\r\nBenchmarkGoroutine-4 \t 2\t 731 [...]
+gitlab:GithubIssue:1:401277739,https://github.com/panjf2000/ants/issues/22,,22,是否考虑 worker 中添加 PanicHandler ？,"""比方说在创建 Pool 的时候传入一个 PanicHandler，然后在每个 worker 创建的时候 recover 之后传给 PanicHandler 处理。否则池子里如果发生 panic 会直接挂掉整个进程。""",,,DONE,,0,2019-01-22T05:41:34.000+00:00,2019-01-21T10:06:56.000+00:00,2019-01-22T05:41:34.000+00:00,1174,,,0,0,0,gitlab:GithubUser:8923413,choleraehyq,gitlab:GithubUser:0,,,
+gitlab:GithubIssue:1:402513849,https://github.com/panjf2000/ants/issues/24,,24,提交任务不阻塞,"""`Pool.Submit`和`PoolWithFunc.Server`提交任务，如果没有空的worker，会一直阻塞。建议增加不阻塞的接口，当前失败时直接返回错误。""",,,DONE,,0,2019-08-20T10:56:30.000+00:00,2019-01-24T02:24:13.000+00:00,2019-08-20T10:56:30.000+00:00,300032,,,0,0,0,gitlab:GithubUser:5044825,tenfyzhong,gitlab:GithubUser:0,,,
+gitlab:GithubIssue:1:405951301,https://github.com/panjf2000/ants/issues/25,,25,use example errors,"""./antstest.go:37:14: cannot use syncCalculateSum (type func()) as type ants.f in argument to ants.Submit\r\n./antstest.go:45:35: cannot use func literal (type func(interface {})) as type ants.pf in argument to ants.NewPoolWithFunc\r\n""",,,DONE,,0,2019-02-04T09:11:52.000+00:00,2019-02-02T05:43:38.000+00:00,2019-02-04T09:11:52.000+00:00,3088,,,0,0,0,gitlab:GithubUser:5244267,jiashiwen,gitl [...]
+gitlab:GithubIssue:1:413968505,https://github.com/panjf2000/ants/issues/26,,26,running可能大于cap的问题,"""running与cap的比较判断与incRuning分开执行的, 可能会出现running大于cap的问题?\r\n`func (p *Pool) retrieveWorker() *Worker {\r\n\tvar w *Worker\r\n\r\n\tp.lock.Lock()\r\n\tidleWorkers := p.workers\r\n\tn := len(idleWorkers) - 1\r\n\tif n >= 0 {\r\n\t\tw = idleWorkers[n]\r\n\t\tidleWorkers[n] = nil\r\n\t\tp.workers = idleWorkers[:n]\r\n\t\tp.lock.Unlock()\r\n\t} else if p.Running() < p.Cap() {\r\n\t\tp.lock.Unlock [...]
+gitlab:GithubIssue:1:419183961,https://github.com/panjf2000/ants/issues/27,,27,为何goroutine一直上不去，用户量也打不上去,"""为何goroutine一直上不去，用户量也打不上去\r\n是我用的有问题吗？\r\n\r\nwebsocket server\r\nhttps://github.com/im-ai/pushm/blob/master/learn/goroutine/goroutinepoolwebsocket.go\r\n\r\nwebsocket cient\r\nhttps://github.com/im-ai/pushm/blob/master/learn/goroutine/goroutinepoolwebsocketclient.go\r\n""",,,DONE,,0,2019-04-05T14:05:20.000+00:00,2019-03-10T13:08:52.000+00:00,2019-04-05T14:05:20.000+00:00,37496,,,0 [...]
+gitlab:GithubIssue:1:419268851,https://github.com/panjf2000/ants/issues/28,,28,cap 和 running 比较的问题,"""这是我在 Playground 上面的代码 https://play.golang.org/p/D94YUU3FnX6\r\natomic 只能保证自增自减时的原子操作，在比较过程中，其他线程对变量进行了操作 比较过程并无感知，所以这个比较结果 不是完全正确的，想要实现 比较的数量完全正确，只能在修改和比较两个值的地方加锁\r\n像 #26 说的是对的""",,,DONE,,0,2019-08-22T16:27:37.000+00:00,2019-03-11T02:24:41.000+00:00,2019-08-22T16:27:37.000+00:00,237002,,,0,0,0,gitlab:GithubUser:29243953,naiba,gitlab:GithubUser:0,,,
+gitlab:GithubIssue:1:424634533,https://github.com/panjf2000/ants/issues/29,,29,任务传参,"""你好，你的项目太酷了👍\r\n\r\nhttps://github.com/panjf2000/ants/blob/master/pool.go#L124 貌似不支持带参数的任务, 请问传参是用闭包的方式吗?\r\n""",,,DONE,,0,2019-03-25T09:32:11.000+00:00,2019-03-24T16:52:21.000+00:00,2019-03-25T09:45:05.000+00:00,999,,,0,0,0,gitlab:GithubUser:8509898,prprprus,gitlab:GithubUser:0,,,
+gitlab:GithubIssue:1:429972115,https://github.com/panjf2000/ants/issues/31,,31,Add go.mod,"""""",,,DONE,,0,2019-04-08T09:45:31.000+00:00,2019-04-05T23:50:36.000+00:00,2019-10-17T03:12:19.000+00:00,3474,,,0,0,0,gitlab:GithubUser:48135919,tsatke,gitlab:GithubUser:0,,,
+gitlab:GithubIssue:1:433564955,https://github.com/panjf2000/ants/issues/32,,32,关于版本问题，我发现小版本(0.0.x)这种更新就会不向下兼容？,"""如题，我感觉这样不好。\r\n\r\n功能版本号不向下兼容能理解\r\n\r\n修复问题的版本号也不向下兼容，难以理解。""",,,DONE,,0,2019-04-21T07:16:26.000+00:00,2019-04-16T03:16:02.000+00:00,2019-04-21T07:16:26.000+00:00,7440,,,0,0,0,gitlab:GithubUser:7931755,zplzpl,gitlab:GithubUser:0,,,
+gitlab:GithubIssue:1:434069015,https://github.com/panjf2000/ants/issues/33,,33,support semantic versioning.,"""建议将发布的tag兼容为semantic versioning，vX.Y.Z。go modules对此支持比较良好。\r\nhttps://semver.org/\r\nhttps://research.swtch.com/vgo-import""",,,DONE,,0,2019-04-21T08:25:20.000+00:00,2019-04-17T02:55:11.000+00:00,2019-04-21T08:25:20.000+00:00,6090,,,0,0,0,gitlab:GithubUser:1284892,jjeffcaii,gitlab:GithubUser:0,,,
+gitlab:GithubIssue:1:435486645,https://github.com/panjf2000/ants/issues/34,,34,Important announcement about <ants> from author !!!,"""**Dear users of `ants`:**\r\nI am apologetically telling you that I have to dump all tags which already presents in `ants` repository.\r\n\r\nThe reason why I'm doing so is to standardize the version management with `Semantic Versioning`, which will make a formal and clear dependency management in go, for go modules, godep, or glide, etc. So I decide to st [...]
+gitlab:GithubIssue:1:461280653,https://github.com/panjf2000/ants/issues/35,,35,worker exit on panic,"""个人认为PanicHandler设计不妥。\r\n1.无PanicHandler时，抛出给外面的不是panic,外层感受不到。\r\n2.无论有没有PanicHandler，都会导致worker退出，最终pool阻塞住全部任务。""",,,DONE,,0,2019-08-17T20:33:10.000+00:00,2019-06-27T03:11:49.000+00:00,2019-08-17T20:33:10.000+00:00,74481,,,0,0,0,gitlab:GithubUser:38849208,king526,gitlab:GithubUser:0,,,
+gitlab:GithubIssue:1:462631417,https://github.com/panjf2000/ants/issues/37,,37,请不要再随意变更版本号了。。。,"""之前用的是 3.9.9，结果今天构建出了问题，一看发现这个版本没了，变成 1.0.0。这种变更完全不考虑现有用户的情况。希望以后不要随意变更了""",,,DONE,,0,2019-07-01T12:37:55.000+00:00,2019-07-01T10:17:15.000+00:00,2019-07-02T10:17:31.000+00:00,140,,,0,0,0,gitlab:GithubUser:8923413,choleraehyq,gitlab:GithubUser:0,,,
+gitlab:GithubIssue:1:472125082,https://github.com/panjf2000/ants/issues/38,,38,retrieveWorker与revertWorker之间会导致死锁,"""func (p *Pool) retrieveWorker() *Worker {\r\n\tvar w *Worker\r\n\r\n\t**p.lock.Lock()**\r\n\tidleWorkers := p.workers\r\n\tn := len(idleWorkers) - 1\r\n\tif n >= 0 {\r\n\t\tw = idleWorkers[n]\r\n\t\tidleWorkers[n] = nil\r\n\t\tp.workers = idleWorkers[:n]\r\n\t\tp.lock.Unlock()\r\n\t} else if p.Running() < p.Cap() {\r\n\t\tp.lock.Unlock()\r\n\t\tif cacheWorker := p.workerCa [...]
+gitlab:GithubIssue:1:483164833,https://github.com/panjf2000/ants/issues/42,,42,带选项的初始化函数，我觉得用 functional options 更好一点,"""以下是示意代码\r\n如果用 functional options，原来的写法是\r\n```\r\nants.NewPool(10)\r\n```\r\n新的写法，如果不加 option，写法是不变的，因为 options 是作为可变参数传进去的。如果要加 option，只需要改成\r\n```\r\nants.NewPool(10, ants.WithNonblocking(true))\r\n```\r\n这样。\r\n\r\n现在是直接传一个 Option 结构体进去，所有的地方都要改，感觉很不优雅。\r\n具体 functional options 的设计可以看 rob pike 的一篇博客 https://commandcenter.blogspot.com/2014/01/self-referential-functi [...]
+gitlab:GithubIssue:1:483736247,https://github.com/panjf2000/ants/issues/43,,43,1.3.0 是不兼容更新,"""Pool 里那些暴露出来的字段（PanicHandler 之类的）都没了，这是一个不兼容更新，根据语义化版本的要求要发大版本。""",,,DONE,,0,2019-08-22T13:22:10.000+00:00,2019-08-22T02:29:34.000+00:00,2019-08-22T13:22:10.000+00:00,652,,,0,0,0,gitlab:GithubUser:8923413,choleraehyq,gitlab:GithubUser:0,,,
+gitlab:GithubIssue:1:484311063,https://github.com/panjf2000/ants/issues/44,,44,1.1.1 -> 1.2.0 也是不兼容更新,"""Pool.Release 的返回值没了""",,,DONE,,0,2019-08-25T06:36:14.000+00:00,2019-08-23T03:27:38.000+00:00,2019-08-25T06:36:14.000+00:00,3068,,,0,0,0,gitlab:GithubUser:8923413,choleraehyq,gitlab:GithubUser:0,,,
diff --git a/plugins/github/models/issue.go b/plugins/github/models/issue.go
index b99cfa46..b508e55d 100644
--- a/plugins/github/models/issue.go
+++ b/plugins/github/models/issue.go
@@ -23,6 +23,7 @@ import (
)
type GithubIssue struct {
+ ConnectionId uint64 `gorm:"primaryKey"`
GithubId int `gorm:"primaryKey"`
RepoId int `gorm:"index"`
Number int `gorm:"index;comment:Used in API requests ex. api/repo/1/issue/<THIS_NUMBER>"`
diff --git a/plugins/github/models/issue_label.go b/plugins/github/models/issue_label.go
index 6bd74a8c..5cc960d1 100644
--- a/plugins/github/models/issue_label.go
+++ b/plugins/github/models/issue_label.go
@@ -25,8 +25,9 @@ import (
// Pull Requests are considered Issues in GitHub.
type GithubIssueLabel struct {
- IssueId int `gorm:"primaryKey;autoIncrement:false"`
- LabelName string `gorm:"primaryKey;type:varchar(255)"`
+ ConnectionId uint64 `gorm:"primaryKey"`
+ IssueId int `gorm:"primaryKey;autoIncrement:false"`
+ LabelName string `gorm:"primaryKey;type:varchar(255)"`
common.NoPKModel
}
diff --git a/plugins/github/models/migrationscripts/archived/issue.go b/plugins/github/models/migrationscripts/archived/issue.go
index ff465b8b..09af0e6e 100644
--- a/plugins/github/models/migrationscripts/archived/issue.go
+++ b/plugins/github/models/migrationscripts/archived/issue.go
@@ -24,6 +24,7 @@ import (
)
type GithubIssue struct {
+ ConnectionId uint64 `gorm:"primaryKey"`
GithubId int `gorm:"primaryKey"`
RepoId int `gorm:"index"`
Number int `gorm:"index;comment:Used in API requests ex. api/repo/1/issue/<THIS_NUMBER>"`
diff --git a/plugins/github/models/migrationscripts/archived/issue_label.go b/plugins/github/models/migrationscripts/archived/issue_label.go
index e37b2b34..dc6bc7cd 100644
--- a/plugins/github/models/migrationscripts/archived/issue_label.go
+++ b/plugins/github/models/migrationscripts/archived/issue_label.go
@@ -23,8 +23,9 @@ import "github.com/apache/incubator-devlake/models/migrationscripts/archived"
// Pull Requests are considered Issues in GitHub.
type GithubIssueLabel struct {
- IssueId int `gorm:"primaryKey;autoIncrement:false"`
- LabelName string `gorm:"primaryKey;type:varchar(255)"`
+ ConnectionId uint64 `gorm:"primaryKey"`
+ IssueId int `gorm:"primaryKey;autoIncrement:false"`
+ LabelName string `gorm:"primaryKey;type:varchar(255)"`
archived.NoPKModel
}
diff --git a/plugins/github/models/migrationscripts/init_schema_20220610.go b/plugins/github/models/migrationscripts/init_schema_20220611.go
similarity index 98%
rename from plugins/github/models/migrationscripts/init_schema_20220610.go
rename to plugins/github/models/migrationscripts/init_schema_20220611.go
index d684f767..23acfce8 100644
--- a/plugins/github/models/migrationscripts/init_schema_20220610.go
+++ b/plugins/github/models/migrationscripts/init_schema_20220611.go
@@ -131,9 +131,9 @@ func (u *InitSchemas) Up(ctx context.Context, db *gorm.DB) error {
}
func (*InitSchemas) Version() uint64 {
- return 20220610000001
+ return 20220611000001
}
func (*InitSchemas) Name() string {
- return "Github init schemas 20220610"
+ return "Github init schemas 20220611"
}
diff --git a/plugins/github/tasks/issue_collector.go b/plugins/github/tasks/issue_collector.go
index 89f11cd0..fe378c0b 100644
--- a/plugins/github/tasks/issue_collector.go
+++ b/plugins/github/tasks/issue_collector.go
@@ -20,6 +20,7 @@ package tasks
import (
"encoding/json"
"fmt"
+ "github.com/apache/incubator-devlake/plugins/core/dal"
"net/http"
"net/url"
@@ -46,7 +47,7 @@ var CollectApiIssuesMeta = core.SubTaskMeta{
}
func CollectApiIssues(taskCtx core.SubTaskContext) error {
- db := taskCtx.GetDb()
+ db := taskCtx.GetDal()
data := taskCtx.GetData().(*GithubTaskData)
since := data.Since
@@ -54,9 +55,12 @@ func CollectApiIssues(taskCtx core.SubTaskContext) error {
// user didn't specify a time range to sync, try load from database
if since == nil {
var latestUpdated models.GithubIssue
- err := db.Model(&latestUpdated).
- Where("repo_id = ?", data.Repo.GithubId).
- Order("github_updated_at DESC").Limit(1).Find(&latestUpdated).Error
+ err := db.All(
+ &latestUpdated,
+ dal.Where("repo_id = ? and connection_id = ?", data.Repo.GithubId, data.Repo.ConnectionId),
+ dal.Orderby("github_updated_at DESC"),
+ dal.Limit(1),
+ )
if err != nil {
return fmt.Errorf("failed to get latest github issue record: %w", err)
}
@@ -74,8 +78,9 @@ func CollectApiIssues(taskCtx core.SubTaskContext) error {
set of data to be process, for example, we process JiraIssues by Board
*/
Params: GithubApiParams{
- Owner: data.Options.Owner,
- Repo: data.Options.Repo,
+ ConnectionId: data.Options.ConnectionId,
+ Owner: data.Options.Owner,
+ Repo: data.Options.Repo,
},
/*
Table store raw data
diff --git a/plugins/github/tasks/issue_convertor.go b/plugins/github/tasks/issue_convertor.go
index f7aa338d..477b3fa9 100644
--- a/plugins/github/tasks/issue_convertor.go
+++ b/plugins/github/tasks/issue_convertor.go
@@ -18,6 +18,7 @@ limitations under the License.
package tasks
import (
+ "github.com/apache/incubator-devlake/plugins/core/dal"
"reflect"
"strconv"
@@ -38,13 +39,15 @@ var ConvertIssuesMeta = core.SubTaskMeta{
}
func ConvertIssues(taskCtx core.SubTaskContext) error {
- db := taskCtx.GetDb()
+ db := taskCtx.GetDal()
data := taskCtx.GetData().(*GithubTaskData)
repoId := data.Repo.GithubId
issue := &githubModels.GithubIssue{}
- cursor, err := db.Model(issue).Where("repo_id = ?", repoId).Rows()
-
+ cursor, err := db.Cursor(
+ dal.From(issue),
+ dal.Where("repo_id = ? and connection_id=?", repoId, data.Options.ConnectionId),
+ )
if err != nil {
return err
}
@@ -58,8 +61,9 @@ func ConvertIssues(taskCtx core.SubTaskContext) error {
RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
Ctx: taskCtx,
Params: GithubApiParams{
- Owner: data.Options.Owner,
- Repo: data.Options.Repo,
+ ConnectionId: data.Options.ConnectionId,
+ Owner: data.Options.Owner,
+ Repo: data.Options.Repo,
},
Table: RAW_ISSUE_TABLE,
},
@@ -68,7 +72,7 @@ func ConvertIssues(taskCtx core.SubTaskContext) error {
Convert: func(inputRow interface{}) ([]interface{}, error) {
issue := inputRow.(*githubModels.GithubIssue)
domainIssue := &ticket.Issue{
- DomainEntity: domainlayer.DomainEntity{Id: issueIdGen.Generate(issue.GithubId)},
+ DomainEntity: domainlayer.DomainEntity{Id: issueIdGen.Generate(data.Options.ConnectionId, issue.GithubId)},
IssueKey: strconv.Itoa(issue.Number),
Title: issue.Title,
Description: issue.Body,
@@ -92,7 +96,7 @@ func ConvertIssues(taskCtx core.SubTaskContext) error {
domainIssue.Status = ticket.TODO
}
boardIssue := &ticket.BoardIssue{
- BoardId: boardIdGen.Generate(repoId),
+ BoardId: boardIdGen.Generate(data.Options.ConnectionId, repoId),
IssueId: domainIssue.Id,
}
return []interface{}{
diff --git a/plugins/github/tasks/issue_extractor.go b/plugins/github/tasks/issue_extractor.go
index 8581112f..6abbeffa 100644
--- a/plugins/github/tasks/issue_extractor.go
+++ b/plugins/github/tasks/issue_extractor.go
@@ -39,7 +39,7 @@ type IssuesResponse struct {
Number int
State string
Title string
- Body string
+ Body json.RawMessage
HtmlUrl string `json:"html_url"`
PullRequest struct {
Url string `json:"url"`
@@ -104,8 +104,9 @@ func ExtractApiIssues(taskCtx core.SubTaskContext) error {
set of data to be process, for example, we process JiraIssues by Board
*/
Params: GithubApiParams{
- Owner: data.Options.Owner,
- Repo: data.Options.Repo,
+ ConnectionId: data.Options.ConnectionId,
+ Owner: data.Options.Owner,
+ Repo: data.Options.Repo,
},
/*
Table store raw data
@@ -127,14 +128,15 @@ func ExtractApiIssues(taskCtx core.SubTaskContext) error {
return nil, nil
}
results := make([]interface{}, 0, 2)
- githubIssue, err := convertGithubIssue(body, data.Repo.GithubId)
+ githubIssue, err := convertGithubIssue(body, data.Options.ConnectionId, data.Repo.GithubId)
if err != nil {
return nil, err
}
for _, label := range body.Labels {
results = append(results, &models.GithubIssueLabel{
- IssueId: githubIssue.GithubId,
- LabelName: label.Name,
+ ConnectionId: data.Options.ConnectionId,
+ IssueId: githubIssue.GithubId,
+ LabelName: label.Name,
})
if issueSeverityRegex != nil {
groups := issueSeverityRegex.FindStringSubmatch(label.Name)
@@ -187,14 +189,15 @@ func ExtractApiIssues(taskCtx core.SubTaskContext) error {
return extractor.Execute()
}
-func convertGithubIssue(issue *IssuesResponse, repositoryId int) (*models.GithubIssue, error) {
+func convertGithubIssue(issue *IssuesResponse, connectionId uint64, repositoryId int) (*models.GithubIssue, error) {
githubIssue := &models.GithubIssue{
+ ConnectionId: connectionId,
GithubId: issue.GithubId,
RepoId: repositoryId,
Number: issue.Number,
State: issue.State,
Title: issue.Title,
- Body: issue.Body,
+ Body: string(issue.Body),
Url: issue.HtmlUrl,
ClosedAt: helper.Iso8601TimeToTime(issue.ClosedAt),
GithubCreatedAt: issue.GithubCreatedAt.ToTime(),
diff --git a/plugins/github/tasks/issue_label_convertor.go b/plugins/github/tasks/issue_label_convertor.go
index b2ece5dc..6ba44d77 100644
--- a/plugins/github/tasks/issue_label_convertor.go
+++ b/plugins/github/tasks/issue_label_convertor.go
@@ -18,6 +18,7 @@ limitations under the License.
package tasks
import (
+ "github.com/apache/incubator-devlake/plugins/core/dal"
"reflect"
"github.com/apache/incubator-devlake/models/domainlayer/didgen"
@@ -35,15 +36,16 @@ var ConvertIssueLabelsMeta = core.SubTaskMeta{
}
func ConvertIssueLabels(taskCtx core.SubTaskContext) error {
- db := taskCtx.GetDb()
+ db := taskCtx.GetDal()
data := taskCtx.GetData().(*GithubTaskData)
repoId := data.Repo.GithubId
- cursor, err := db.Model(&githubModels.GithubIssueLabel{}).
- Joins(`left join _tool_github_issues on _tool_github_issues.github_id = _tool_github_issue_labels.issue_id`).
- Where("_tool_github_issues.repo_id = ?", repoId).
- Order("issue_id ASC").
- Rows()
+ cursor, err := db.Cursor(
+ dal.From(&githubModels.GithubIssueLabel{}),
+ dal.Join(`left join _tool_github_issues on _tool_github_issues.github_id = _tool_github_issue_labels.issue_id`),
+ dal.Where("_tool_github_issues.repo_id = ? and _tool_github_issues.connection_id = ?", repoId, data.Options.ConnectionId),
+ dal.Orderby("issue_id ASC"),
+ )
if err != nil {
return err
}
@@ -54,8 +56,9 @@ func ConvertIssueLabels(taskCtx core.SubTaskContext) error {
RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
Ctx: taskCtx,
Params: GithubApiParams{
- Owner: data.Options.Owner,
- Repo: data.Options.Repo,
+ ConnectionId: data.Options.ConnectionId,
+ Owner: data.Options.Owner,
+ Repo: data.Options.Repo,
},
Table: RAW_ISSUE_TABLE,
},
@@ -64,7 +67,7 @@ func ConvertIssueLabels(taskCtx core.SubTaskContext) error {
Convert: func(inputRow interface{}) ([]interface{}, error) {
issueLabel := inputRow.(*githubModels.GithubIssueLabel)
domainIssueLabel := &ticket.IssueLabel{
- IssueId: issueIdGen.Generate(issueLabel.IssueId),
+ IssueId: issueIdGen.Generate(data.Options.ConnectionId, issueLabel.IssueId),
LabelName: issueLabel.LabelName,
}
return []interface{}{