You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by kl...@apache.org on 2022/06/20 07:06:49 UTC
[incubator-devlake] branch main updated: github event&pr_issue task refactor (#2205)
This is an automated email from the ASF dual-hosted git repository.
klesh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 17824b8c github event&pr_issue task refactor (#2205)
17824b8c is described below
commit 17824b8cf158b9b5a914d69486284a3478623f56
Author: likyh <l...@likyh.com>
AuthorDate: Mon Jun 20 15:06:45 2022 +0800
github event&pr_issue task refactor (#2205)
* base csv
* base code
* add connection id
* delete unuse check
* append
* fix comment
* fix for review
* base
* add connection id for event and pr_issue
* append
* fix name
Co-authored-by: linyh <ya...@meri.co>
Co-authored-by: Klesh Wong <zh...@merico.dev>
---
plugins/github/e2e/event_test.go | 67 +++++++++++++++
plugins/github/e2e/pr_enrich_issue_test.go | 95 ++++++++++++++++++++++
.../e2e/raw_tables/_raw_github_api_events.csv | 50 ++++++++++++
.../snapshot_tables/_tool_github_issue_events.csv | 50 ++++++++++++
.../_tool_github_pull_request_issues.csv | 2 +
.../e2e/snapshot_tables/pull_request_issues.csv | 2 +
plugins/github/models/issue_event.go | 1 +
.../migrationscripts/archived/issue_event.go | 1 +
.../archived/pull_request_issue.go | 5 +-
plugins/github/models/pr_issue.go | 5 +-
plugins/github/tasks/event_collector.go | 19 +++--
plugins/github/tasks/event_extractor.go | 10 ++-
plugins/github/tasks/pr_issue_convertor.go | 23 +++---
plugins/github/tasks/pr_issue_enricher.go | 20 +++--
14 files changed, 317 insertions(+), 33 deletions(-)
diff --git a/plugins/github/e2e/event_test.go b/plugins/github/e2e/event_test.go
new file mode 100644
index 00000000..6b8b11b6
--- /dev/null
+++ b/plugins/github/e2e/event_test.go
@@ -0,0 +1,67 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+ "fmt"
+ "github.com/apache/incubator-devlake/plugins/github/models"
+ "testing"
+
+ "github.com/apache/incubator-devlake/helpers/e2ehelper"
+ "github.com/apache/incubator-devlake/plugins/github/impl"
+ "github.com/apache/incubator-devlake/plugins/github/tasks"
+)
+
+func TestEventDataFlow(t *testing.T) {
+ var plugin impl.Github
+ dataflowTester := e2ehelper.NewDataFlowTester(t, "gitlab", plugin)
+
+ githubRepository := &models.GithubRepo{
+ GithubId: 134018330,
+ }
+ taskData := &tasks.GithubTaskData{
+ Options: &tasks.GithubOptions{
+ ConnectionId: 1,
+ Owner: "panjf2000",
+ Repo: "ants",
+ },
+ Repo: githubRepository,
+ }
+
+ // import raw data table
+ dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_github_api_events.csv", "_raw_github_api_events")
+
+ // verify extraction
+ dataflowTester.FlushTabler(&models.GithubIssueEvent{})
+ dataflowTester.Subtask(tasks.ExtractApiEventsMeta, taskData)
+ dataflowTester.VerifyTable(
+ models.GithubIssueEvent{},
+ fmt.Sprintf("./snapshot_tables/%s.csv", models.GithubIssueEvent{}.TableName()),
+ []string{"connection_id", "github_id"},
+ []string{
+ "issue_id",
+ "type",
+ "author_username",
+ "github_created_at",
+ "_raw_data_params",
+ "_raw_data_table",
+ "_raw_data_id",
+ "_raw_data_remark",
+ },
+ )
+}
diff --git a/plugins/github/e2e/pr_enrich_issue_test.go b/plugins/github/e2e/pr_enrich_issue_test.go
new file mode 100644
index 00000000..42ebec8b
--- /dev/null
+++ b/plugins/github/e2e/pr_enrich_issue_test.go
@@ -0,0 +1,95 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+ "fmt"
+ "github.com/apache/incubator-devlake/models/domainlayer/crossdomain"
+ "github.com/apache/incubator-devlake/plugins/github/models"
+ "testing"
+
+ "github.com/apache/incubator-devlake/helpers/e2ehelper"
+ "github.com/apache/incubator-devlake/plugins/github/impl"
+ "github.com/apache/incubator-devlake/plugins/github/tasks"
+)
+
+func TestPrEnrichIssueDataFlow(t *testing.T) {
+ var plugin impl.Github
+ dataflowTester := e2ehelper.NewDataFlowTester(t, "gitlab", plugin)
+
+ githubRepository := &models.GithubRepo{
+ GithubId: 134018330,
+ }
+ taskData := &tasks.GithubTaskData{
+ Options: &tasks.GithubOptions{
+ ConnectionId: 1,
+ Owner: "panjf2000",
+ Repo: "ants",
+ Config: models.Config{
+ PrType: "type/(.*)$",
+ PrComponent: "component/(.*)$",
+ PrBodyClosePattern: "(?mi)(fix|close|resolve|fixes|closes|resolves|fixed|closed|resolved)[\\s]*.*(((and )?(#|https:\\/\\/github.com\\/%s\\/%s\\/issues\\/)\\d+[ ]*)+)",
+ IssueSeverity: "severity/(.*)$",
+ IssuePriority: "^(highest|high|medium|low)$",
+ IssueComponent: "component/(.*)$",
+ IssueTypeBug: "^(bug|failure|error)$",
+ IssueTypeIncident: "",
+ IssueTypeRequirement: "^(feat|feature|proposal|requirement)$",
+ },
+ },
+ Repo: githubRepository,
+ }
+
+ // import raw data table
+ dataflowTester.ImportCsvIntoTabler(fmt.Sprintf("./snapshot_tables/%s.csv", models.GithubIssue{}.TableName()), &models.GithubIssue{})
+ dataflowTester.ImportCsvIntoTabler(fmt.Sprintf("./snapshot_tables/%s.csv", models.GithubPullRequest{}.TableName()), models.GithubPullRequest{})
+
+ // verify extraction
+ dataflowTester.FlushTabler(&models.GithubPullRequestIssue{})
+ dataflowTester.Subtask(tasks.EnrichPullRequestIssuesMeta, taskData)
+ dataflowTester.VerifyTable(
+ models.GithubPullRequestIssue{},
+ fmt.Sprintf("./snapshot_tables/%s.csv", models.GithubPullRequestIssue{}.TableName()),
+ []string{"connection_id", "pull_request_id", "issue_id"},
+ []string{
+ "pull_request_number",
+ "issue_number",
+ "_raw_data_params",
+ "_raw_data_table",
+ "_raw_data_id",
+ "_raw_data_remark",
+ },
+ )
+
+ // verify extraction
+ dataflowTester.FlushTabler(&crossdomain.PullRequestIssue{})
+ dataflowTester.Subtask(tasks.ConvertPullRequestIssuesMeta, taskData)
+ dataflowTester.VerifyTable(
+ crossdomain.PullRequestIssue{},
+ fmt.Sprintf("./snapshot_tables/%s.csv", crossdomain.PullRequestIssue{}.TableName()),
+ []string{"pull_request_id", "issue_id"},
+ []string{
+ "pull_request_number",
+ "issue_number",
+ "_raw_data_params",
+ "_raw_data_table",
+ "_raw_data_id",
+ "_raw_data_remark",
+ },
+ )
+}
diff --git a/plugins/github/e2e/raw_tables/_raw_github_api_events.csv b/plugins/github/e2e/raw_tables/_raw_github_api_events.csv
new file mode 100644
index 00000000..55472f0a
--- /dev/null
+++ b/plugins/github/e2e/raw_tables/_raw_github_api_events.csv
@@ -0,0 +1,50 @@
+id,params,data,url,input,created_at
+1,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""id"":6774117135,""node_id"":""CE_lADOB_z1Gs5K-Ts6zwAAAAGTxNMP"",""url"":""https://api.github.com/repos/panjf2000/ants/issues/events/6774117135"",""actor"":{""login"":""ZhMaio"",""id"":47515663,""node_id"":""MDQ6VXNlcjQ3NTE1NjYz"",""avatar_url"":""https://avatars.githubusercontent.com/u/47515663?v=4"",""gravatar_id"":"""",""url"":""https://api.github.com/users/ZhMaio"",""html_url"":""https://github.com/ZhMaio"",""follo [...]
+2,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""id"":6766052007,""node_id"":""AE_lADOB_z1Gs5LWMEGzwAAAAGTScKn"",""url"":""https://api.github.com/repos/panjf2000/ants/issues/events/6766052007"",""actor"":{""login"":""panjf2000"",""id"":7496278,""node_id"":""MDQ6VXNlcjc0OTYyNzg="",""avatar_url"":""https://avatars.githubusercontent.com/u/7496278?v=4"",""gravatar_id"":"""",""url"":""https://api.github.com/users/panjf2000"",""html_url"":""https://github.com/panjf2000"", [...]
+3,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""id"":6766051996,""node_id"":""LE_lADOB_z1Gs5LWMEGzwAAAAGTScKc"",""url"":""https://api.github.com/repos/panjf2000/ants/issues/events/6766051996"",""actor"":{""login"":""iGen1us"",""id"":14806824,""node_id"":""MDQ6VXNlcjE0ODA2ODI0"",""avatar_url"":""https://avatars.githubusercontent.com/u/14806824?v=4"",""gravatar_id"":"""",""url"":""https://api.github.com/users/iGen1us"",""html_url"":""https://github.com/iGen1us"",""fo [...]
+4,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""id"":6766051995,""node_id"":""LE_lADOB_z1Gs5LWMEGzwAAAAGTScKb"",""url"":""https://api.github.com/repos/panjf2000/ants/issues/events/6766051995"",""actor"":{""login"":""iGen1us"",""id"":14806824,""node_id"":""MDQ6VXNlcjE0ODA2ODI0"",""avatar_url"":""https://avatars.githubusercontent.com/u/14806824?v=4"",""gravatar_id"":"""",""url"":""https://api.github.com/users/iGen1us"",""html_url"":""https://github.com/iGen1us"",""fo [...]
+5,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""id"":6759426948,""node_id"":""AE_lADOB_z1Gs5LR45mzwAAAAGS5KuE"",""url"":""https://api.github.com/repos/panjf2000/ants/issues/events/6759426948"",""actor"":{""login"":""panjf2000"",""id"":7496278,""node_id"":""MDQ6VXNlcjc0OTYyNzg="",""avatar_url"":""https://avatars.githubusercontent.com/u/7496278?v=4"",""gravatar_id"":"""",""url"":""https://api.github.com/users/panjf2000"",""html_url"":""https://github.com/panjf2000"", [...]
+6,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""id"":6759426887,""node_id"":""LE_lADOB_z1Gs5LR45mzwAAAAGS5KtH"",""url"":""https://api.github.com/repos/panjf2000/ants/issues/events/6759426887"",""actor"":{""login"":""imDpeng"",""id"":48267340,""node_id"":""MDQ6VXNlcjQ4MjY3MzQw"",""avatar_url"":""https://avatars.githubusercontent.com/u/48267340?v=4"",""gravatar_id"":"""",""url"":""https://api.github.com/users/imDpeng"",""html_url"":""https://github.com/imDpeng"",""fo [...]
+7,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""id"":6759426885,""node_id"":""LE_lADOB_z1Gs5LR45mzwAAAAGS5KtF"",""url"":""https://api.github.com/repos/panjf2000/ants/issues/events/6759426885"",""actor"":{""login"":""imDpeng"",""id"":48267340,""node_id"":""MDQ6VXNlcjQ4MjY3MzQw"",""avatar_url"":""https://avatars.githubusercontent.com/u/48267340?v=4"",""gravatar_id"":"""",""url"":""https://api.github.com/users/imDpeng"",""html_url"":""https://github.com/imDpeng"",""fo [...]
+73,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""id"":6035033987,""node_id"":""LE_lADOB_z1Gs5DORwszwAAAAFnt0-D"",""url"":""https://api.github.com/repos/panjf2000/ants/issues/events/6035033987"",""actor"":{""login"":""panjf2000"",""id"":7496278,""node_id"":""MDQ6VXNlcjc0OTYyNzg="",""avatar_url"":""https://avatars.githubusercontent.com/u/7496278?v=4"",""gravatar_id"":"""",""url"":""https://api.github.com/users/panjf2000"",""html_url"":""https://github.com/panjf2000"" [...]
+74,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""id"":6031595363,""node_id"":""HRFPE_lADOB_z1Gs5CpQAGzwAAAAFngtdj"",""url"":""https://api.github.com/repos/panjf2000/ants/issues/events/6031595363"",""actor"":{""login"":""codingfanlt"",""id"":35493957,""node_id"":""MDQ6VXNlcjM1NDkzOTU3"",""avatar_url"":""https://avatars.githubusercontent.com/u/35493957?v=4"",""gravatar_id"":"""",""url"":""https://api.github.com/users/codingfanlt"",""html_url"":""https://github.com/co [...]
+75,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""id"":6030205602,""node_id"":""AE_lADOB_z1Gs5DORwszwAAAAFnbaKi"",""url"":""https://api.github.com/repos/panjf2000/ants/issues/events/6030205602"",""actor"":{""login"":""panjf2000"",""id"":7496278,""node_id"":""MDQ6VXNlcjc0OTYyNzg="",""avatar_url"":""https://avatars.githubusercontent.com/u/7496278?v=4"",""gravatar_id"":"""",""url"":""https://api.github.com/users/panjf2000"",""html_url"":""https://github.com/panjf2000"" [...]
+76,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""id"":6030205564,""node_id"":""LE_lADOB_z1Gs5DORwszwAAAAFnbaJ8"",""url"":""https://api.github.com/repos/panjf2000/ants/issues/events/6030205564"",""actor"":{""login"":""ShivanshVij"",""id"":27162109,""node_id"":""MDQ6VXNlcjI3MTYyMTA5"",""avatar_url"":""https://avatars.githubusercontent.com/u/27162109?v=4"",""gravatar_id"":"""",""url"":""https://api.github.com/users/ShivanshVij"",""html_url"":""https://github.com/Shiva [...]
+8,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""id"":6726348886,""node_id"":""RTE_lADOB_z1Gs5K-Ts6zwAAAAGQ6_BW"",""url"":""https://api.github.com/repos/panjf2000/ants/issues/events/6726348886"",""actor"":{""login"":""ZhMaio"",""id"":47515663,""node_id"":""MDQ6VXNlcjQ3NTE1NjYz"",""avatar_url"":""https://avatars.githubusercontent.com/u/47515663?v=4"",""gravatar_id"":"""",""url"":""https://api.github.com/users/ZhMaio"",""html_url"":""https://github.com/ZhMaio"",""foll [...]
+9,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""id"":6726343315,""node_id"":""AE_lADOB_z1Gs5K-Ts6zwAAAAGQ69qT"",""url"":""https://api.github.com/repos/panjf2000/ants/issues/events/6726343315"",""actor"":{""login"":""panjf2000"",""id"":7496278,""node_id"":""MDQ6VXNlcjc0OTYyNzg="",""avatar_url"":""https://avatars.githubusercontent.com/u/7496278?v=4"",""gravatar_id"":"""",""url"":""https://api.github.com/users/panjf2000"",""html_url"":""https://github.com/panjf2000"", [...]
+10,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""id"":6726343287,""node_id"":""LE_lADOB_z1Gs5K-Ts6zwAAAAGQ69p3"",""url"":""https://api.github.com/repos/panjf2000/ants/issues/events/6726343287"",""actor"":{""login"":""ZhMaio"",""id"":47515663,""node_id"":""MDQ6VXNlcjQ3NTE1NjYz"",""avatar_url"":""https://avatars.githubusercontent.com/u/47515663?v=4"",""gravatar_id"":"""",""url"":""https://api.github.com/users/ZhMaio"",""html_url"":""https://github.com/ZhMaio"",""foll [...]
+11,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""id"":6726343285,""node_id"":""LE_lADOB_z1Gs5K-Ts6zwAAAAGQ69p1"",""url"":""https://api.github.com/repos/panjf2000/ants/issues/events/6726343285"",""actor"":{""login"":""ZhMaio"",""id"":47515663,""node_id"":""MDQ6VXNlcjQ3NTE1NjYz"",""avatar_url"":""https://avatars.githubusercontent.com/u/47515663?v=4"",""gravatar_id"":"""",""url"":""https://api.github.com/users/ZhMaio"",""html_url"":""https://github.com/ZhMaio"",""foll [...]
+12,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""id"":6709899522,""node_id"":""AE_lADOB_z1Gs5KvTKOzwAAAAGP8PEC"",""url"":""https://api.github.com/repos/panjf2000/ants/issues/events/6709899522"",""actor"":{""login"":""panjf2000"",""id"":7496278,""node_id"":""MDQ6VXNlcjc0OTYyNzg="",""avatar_url"":""https://avatars.githubusercontent.com/u/7496278?v=4"",""gravatar_id"":"""",""url"":""https://api.github.com/users/panjf2000"",""html_url"":""https://github.com/panjf2000"" [...]
+13,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""id"":6709899424,""node_id"":""LE_lADOB_z1Gs5KvTKOzwAAAAGP8PCg"",""url"":""https://api.github.com/repos/panjf2000/ants/issues/events/6709899424"",""actor"":{""login"":""LiaoPuJian"",""id"":29474400,""node_id"":""MDQ6VXNlcjI5NDc0NDAw"",""avatar_url"":""https://avatars.githubusercontent.com/u/29474400?v=4"",""gravatar_id"":"""",""url"":""https://api.github.com/users/LiaoPuJian"",""html_url"":""https://github.com/LiaoPuJ [...]
+14,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""id"":6709899399,""node_id"":""LE_lADOB_z1Gs5KvTKOzwAAAAGP8PCH"",""url"":""https://api.github.com/repos/panjf2000/ants/issues/events/6709899399"",""actor"":{""login"":""LiaoPuJian"",""id"":29474400,""node_id"":""MDQ6VXNlcjI5NDc0NDAw"",""avatar_url"":""https://avatars.githubusercontent.com/u/29474400?v=4"",""gravatar_id"":"""",""url"":""https://api.github.com/users/LiaoPuJian"",""html_url"":""https://github.com/LiaoPuJ [...]
+15,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""id"":6655982786,""node_id"":""CE_lADOB_z1Gs5KJmWuzwAAAAGMujzC"",""url"":""https://api.github.com/repos/panjf2000/ants/issues/events/6655982786"",""actor"":{""login"":""fufuok"",""id"":4979407,""node_id"":""MDQ6VXNlcjQ5Nzk0MDc="",""avatar_url"":""https://avatars.githubusercontent.com/u/4979407?v=4"",""gravatar_id"":"""",""url"":""https://api.github.com/users/fufuok"",""html_url"":""https://github.com/fufuok"",""follow [...]
+16,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""id"":6655132917,""node_id"":""AE_lADOB_z1Gs5KJmWuzwAAAAGMrUT1"",""url"":""https://api.github.com/repos/panjf2000/ants/issues/events/6655132917"",""actor"":{""login"":""panjf2000"",""id"":7496278,""node_id"":""MDQ6VXNlcjc0OTYyNzg="",""avatar_url"":""https://avatars.githubusercontent.com/u/7496278?v=4"",""gravatar_id"":"""",""url"":""https://api.github.com/users/panjf2000"",""html_url"":""https://github.com/panjf2000"" [...]
+17,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""id"":6655132907,""node_id"":""LE_lADOB_z1Gs5KJmWuzwAAAAGMrUTr"",""url"":""https://api.github.com/repos/panjf2000/ants/issues/events/6655132907"",""actor"":{""login"":""fufuok"",""id"":4979407,""node_id"":""MDQ6VXNlcjQ5Nzk0MDc="",""avatar_url"":""https://avatars.githubusercontent.com/u/4979407?v=4"",""gravatar_id"":"""",""url"":""https://api.github.com/users/fufuok"",""html_url"":""https://github.com/fufuok"",""follow [...]
+77,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""id"":6026454374,""node_id"":""SE_lADOB_z1Gs5DJVidzwAAAAFnNGVm"",""url"":""https://api.github.com/repos/panjf2000/ants/issues/events/6026454374"",""actor"":{""login"":""panjf2000"",""id"":7496278,""node_id"":""MDQ6VXNlcjc0OTYyNzg="",""avatar_url"":""https://avatars.githubusercontent.com/u/7496278?v=4"",""gravatar_id"":"""",""url"":""https://api.github.com/users/panjf2000"",""html_url"":""https://github.com/panjf2000"" [...]
+18,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""id"":6651984465,""node_id"":""AE_lADOB_z1Gs5KHEwFzwAAAAGMfTpR"",""url"":""https://api.github.com/repos/panjf2000/ants/issues/events/6651984465"",""actor"":{""login"":""panjf2000"",""id"":7496278,""node_id"":""MDQ6VXNlcjc0OTYyNzg="",""avatar_url"":""https://avatars.githubusercontent.com/u/7496278?v=4"",""gravatar_id"":"""",""url"":""https://api.github.com/users/panjf2000"",""html_url"":""https://github.com/panjf2000"" [...]
+19,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""id"":6651984442,""node_id"":""LE_lADOB_z1Gs5KHEwFzwAAAAGMfTo6"",""url"":""https://api.github.com/repos/panjf2000/ants/issues/events/6651984442"",""actor"":{""login"":""zqlpaopao"",""id"":43371021,""node_id"":""MDQ6VXNlcjQzMzcxMDIx"",""avatar_url"":""https://avatars.githubusercontent.com/u/43371021?v=4"",""gravatar_id"":"""",""url"":""https://api.github.com/users/zqlpaopao"",""html_url"":""https://github.com/zqlpaopao [...]
+20,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""id"":6651984441,""node_id"":""LE_lADOB_z1Gs5KHEwFzwAAAAGMfTo5"",""url"":""https://api.github.com/repos/panjf2000/ants/issues/events/6651984441"",""actor"":{""login"":""zqlpaopao"",""id"":43371021,""node_id"":""MDQ6VXNlcjQzMzcxMDIx"",""avatar_url"":""https://avatars.githubusercontent.com/u/43371021?v=4"",""gravatar_id"":"""",""url"":""https://api.github.com/users/zqlpaopao"",""html_url"":""https://github.com/zqlpaopao [...]
+21,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""id"":6628985629,""node_id"":""CE_lADOB_z1Gs5J37MlzwAAAAGLHksd"",""url"":""https://api.github.com/repos/panjf2000/ants/issues/events/6628985629"",""actor"":{""login"":""hugh-404"",""id"":39076288,""node_id"":""MDQ6VXNlcjM5MDc2Mjg4"",""avatar_url"":""https://avatars.githubusercontent.com/u/39076288?v=4"",""gravatar_id"":"""",""url"":""https://api.github.com/users/hugh-404"",""html_url"":""https://github.com/hugh-404"", [...]
+78,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""id"":6026454371,""node_id"":""MEE_lADOB_z1Gs5DJVidzwAAAAFnNGVj"",""url"":""https://api.github.com/repos/panjf2000/ants/issues/events/6026454371"",""actor"":{""login"":""panjf2000"",""id"":7496278,""node_id"":""MDQ6VXNlcjc0OTYyNzg="",""avatar_url"":""https://avatars.githubusercontent.com/u/7496278?v=4"",""gravatar_id"":"""",""url"":""https://api.github.com/users/panjf2000"",""html_url"":""https://github.com/panjf2000" [...]
+79,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""id"":6026450677,""node_id"":""HRFPE_lADOB_z1Gs5DJVidzwAAAAFnNFb1"",""url"":""https://api.github.com/repos/panjf2000/ants/issues/events/6026450677"",""actor"":{""login"":""lucafmarques"",""id"":15234973,""node_id"":""MDQ6VXNlcjE1MjM0OTcz"",""avatar_url"":""https://avatars.githubusercontent.com/u/15234973?v=4"",""gravatar_id"":"""",""url"":""https://api.github.com/users/lucafmarques"",""html_url"":""https://github.com/ [...]
+80,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""id"":6026431404,""node_id"":""HRFPE_lADOB_z1Gs5DJVidzwAAAAFnNAus"",""url"":""https://api.github.com/repos/panjf2000/ants/issues/events/6026431404"",""actor"":{""login"":""lucafmarques"",""id"":15234973,""node_id"":""MDQ6VXNlcjE1MjM0OTcz"",""avatar_url"":""https://avatars.githubusercontent.com/u/15234973?v=4"",""gravatar_id"":"""",""url"":""https://api.github.com/users/lucafmarques"",""html_url"":""https://github.com/ [...]
+81,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""id"":6024185240,""node_id"":""CE_lADOB_z1Gs5AQbIGzwAAAAFnEcWY"",""url"":""https://api.github.com/repos/panjf2000/ants/issues/events/6024185240"",""actor"":{""login"":""panjf2000"",""id"":7496278,""node_id"":""MDQ6VXNlcjc0OTYyNzg="",""avatar_url"":""https://avatars.githubusercontent.com/u/7496278?v=4"",""gravatar_id"":"""",""url"":""https://api.github.com/users/panjf2000"",""html_url"":""https://github.com/panjf2000"" [...]
+82,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""id"":6022451048,""node_id"":""HRFPE_lADOB_z1Gs5DJVidzwAAAAFm909o"",""url"":""https://api.github.com/repos/panjf2000/ants/issues/events/6022451048"",""actor"":{""login"":""lucafmarques"",""id"":15234973,""node_id"":""MDQ6VXNlcjE1MjM0OTcz"",""avatar_url"":""https://avatars.githubusercontent.com/u/15234973?v=4"",""gravatar_id"":"""",""url"":""https://api.github.com/users/lucafmarques"",""html_url"":""https://github.com/ [...]
+22,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""id"":6567581911,""node_id"":""CE_lADOB_z1Gs5ABTlvzwAAAAGHdVjX"",""url"":""https://api.github.com/repos/panjf2000/ants/issues/events/6567581911"",""actor"":{""login"":""panjf2000"",""id"":7496278,""node_id"":""MDQ6VXNlcjc0OTYyNzg="",""avatar_url"":""https://avatars.githubusercontent.com/u/7496278?v=4"",""gravatar_id"":"""",""url"":""https://api.github.com/users/panjf2000"",""html_url"":""https://github.com/panjf2000"" [...]
+23,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""id"":6567571146,""node_id"":""CE_lADOB_z1Gs5HWLfGzwAAAAGHdS7K"",""url"":""https://api.github.com/repos/panjf2000/ants/issues/events/6567571146"",""actor"":{""login"":""panjf2000"",""id"":7496278,""node_id"":""MDQ6VXNlcjc0OTYyNzg="",""avatar_url"":""https://avatars.githubusercontent.com/u/7496278?v=4"",""gravatar_id"":"""",""url"":""https://api.github.com/users/panjf2000"",""html_url"":""https://github.com/panjf2000"" [...]
+24,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""id"":6567570821,""node_id"":""CE_lADOB_z1Gs41lT7DzwAAAAGHdS2F"",""url"":""https://api.github.com/repos/panjf2000/ants/issues/events/6567570821"",""actor"":{""login"":""panjf2000"",""id"":7496278,""node_id"":""MDQ6VXNlcjc0OTYyNzg="",""avatar_url"":""https://avatars.githubusercontent.com/u/7496278?v=4"",""gravatar_id"":"""",""url"":""https://api.github.com/users/panjf2000"",""html_url"":""https://github.com/panjf2000"" [...]
+25,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""id"":6567390248,""node_id"":""CE_lADOB_z1Gs5IJOZzzwAAAAGHcmwo"",""url"":""https://api.github.com/repos/panjf2000/ants/issues/events/6567390248"",""actor"":{""login"":""panjf2000"",""id"":7496278,""node_id"":""MDQ6VXNlcjc0OTYyNzg="",""avatar_url"":""https://avatars.githubusercontent.com/u/7496278?v=4"",""gravatar_id"":"""",""url"":""https://api.github.com/users/panjf2000"",""html_url"":""https://github.com/panjf2000"" [...]
+26,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""id"":6553562170,""node_id"":""CE_lADOB_z1Gs5H5AWkzwAAAAGGn2w6"",""url"":""https://api.github.com/repos/panjf2000/ants/issues/events/6553562170"",""actor"":{""login"":""panjf2000"",""id"":7496278,""node_id"":""MDQ6VXNlcjc0OTYyNzg="",""avatar_url"":""https://avatars.githubusercontent.com/u/7496278?v=4"",""gravatar_id"":"""",""url"":""https://api.github.com/users/panjf2000"",""html_url"":""https://github.com/panjf2000"" [...]
+83,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""id"":6022440198,""node_id"":""RFRE_lADOB_z1Gs5DJVidzwAAAAFm9yUG"",""url"":""https://api.github.com/repos/panjf2000/ants/issues/events/6022440198"",""actor"":{""login"":""lucafmarques"",""id"":15234973,""node_id"":""MDQ6VXNlcjE1MjM0OTcz"",""avatar_url"":""https://avatars.githubusercontent.com/u/15234973?v=4"",""gravatar_id"":"""",""url"":""https://api.github.com/users/lucafmarques"",""html_url"":""https://github.com/l [...]
+84,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""id"":6018650885,""node_id"":""HRFPE_lADOB_z1Gs5CpQAGzwAAAAFmvVMF"",""url"":""https://api.github.com/repos/panjf2000/ants/issues/events/6018650885"",""actor"":{""login"":""codingfanlt"",""id"":35493957,""node_id"":""MDQ6VXNlcjM1NDkzOTU3"",""avatar_url"":""https://avatars.githubusercontent.com/u/35493957?v=4"",""gravatar_id"":"""",""url"":""https://api.github.com/users/codingfanlt"",""html_url"":""https://github.com/co [...]
+85,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""id"":6018634131,""node_id"":""HRFPE_lADOB_z1Gs5CpQAGzwAAAAFmvRGT"",""url"":""https://api.github.com/repos/panjf2000/ants/issues/events/6018634131"",""actor"":{""login"":""codingfanlt"",""id"":35493957,""node_id"":""MDQ6VXNlcjM1NDkzOTU3"",""avatar_url"":""https://avatars.githubusercontent.com/u/35493957?v=4"",""gravatar_id"":"""",""url"":""https://api.github.com/users/codingfanlt"",""html_url"":""https://github.com/co [...]
+27,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""id"":6553561086,""node_id"":""CE_lADOB_z1Gs5IcPRKzwAAAAGGn2f-"",""url"":""https://api.github.com/repos/panjf2000/ants/issues/events/6553561086"",""actor"":{""login"":""panjf2000"",""id"":7496278,""node_id"":""MDQ6VXNlcjc0OTYyNzg="",""avatar_url"":""https://avatars.githubusercontent.com/u/7496278?v=4"",""gravatar_id"":"""",""url"":""https://api.github.com/users/panjf2000"",""html_url"":""https://github.com/panjf2000"" [...]
+28,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""id"":6495314656,""node_id"":""AE_lADOB_z1Gs5IcPRKzwAAAAGDJqLg"",""url"":""https://api.github.com/repos/panjf2000/ants/issues/events/6495314656"",""actor"":{""login"":""panjf2000"",""id"":7496278,""node_id"":""MDQ6VXNlcjc0OTYyNzg="",""avatar_url"":""https://avatars.githubusercontent.com/u/7496278?v=4"",""gravatar_id"":"""",""url"":""https://api.github.com/users/panjf2000"",""html_url"":""https://github.com/panjf2000"" [...]
+29,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""id"":6495314644,""node_id"":""LE_lADOB_z1Gs5IcPRKzwAAAAGDJqLU"",""url"":""https://api.github.com/repos/panjf2000/ants/issues/events/6495314644"",""actor"":{""login"":""yxiupei"",""id"":30883503,""node_id"":""MDQ6VXNlcjMwODgzNTAz"",""avatar_url"":""https://avatars.githubusercontent.com/u/30883503?v=4"",""gravatar_id"":"""",""url"":""https://api.github.com/users/yxiupei"",""html_url"":""https://github.com/yxiupei"",""f [...]
+30,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""id"":6495314642,""node_id"":""LE_lADOB_z1Gs5IcPRKzwAAAAGDJqLS"",""url"":""https://api.github.com/repos/panjf2000/ants/issues/events/6495314642"",""actor"":{""login"":""yxiupei"",""id"":30883503,""node_id"":""MDQ6VXNlcjMwODgzNTAz"",""avatar_url"":""https://avatars.githubusercontent.com/u/30883503?v=4"",""gravatar_id"":"""",""url"":""https://api.github.com/users/yxiupei"",""html_url"":""https://github.com/yxiupei"",""f [...]
+31,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""id"":6485990121,""node_id"":""LE_lADOB_z1Gs5IJOZzzwAAAAGCmFrp"",""url"":""https://api.github.com/repos/panjf2000/ants/issues/events/6485990121"",""actor"":{""login"":""panjf2000"",""id"":7496278,""node_id"":""MDQ6VXNlcjc0OTYyNzg="",""avatar_url"":""https://avatars.githubusercontent.com/u/7496278?v=4"",""gravatar_id"":"""",""url"":""https://api.github.com/users/panjf2000"",""html_url"":""https://github.com/panjf2000"" [...]
+32,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""id"":6467254520,""node_id"":""AE_lADOB_z1Gs5IJOZzzwAAAAGBenj4"",""url"":""https://api.github.com/repos/panjf2000/ants/issues/events/6467254520"",""actor"":{""login"":""panjf2000"",""id"":7496278,""node_id"":""MDQ6VXNlcjc0OTYyNzg="",""avatar_url"":""https://avatars.githubusercontent.com/u/7496278?v=4"",""gravatar_id"":"""",""url"":""https://api.github.com/users/panjf2000"",""html_url"":""https://github.com/panjf2000"" [...]
+33,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""id"":6467254512,""node_id"":""LE_lADOB_z1Gs5IJOZzzwAAAAGBenjw"",""url"":""https://api.github.com/repos/panjf2000/ants/issues/events/6467254512"",""actor"":{""login"":""FeurJak"",""id"":77963837,""node_id"":""MDQ6VXNlcjc3OTYzODM3"",""avatar_url"":""https://avatars.githubusercontent.com/u/77963837?v=4"",""gravatar_id"":"""",""url"":""https://api.github.com/users/FeurJak"",""html_url"":""https://github.com/FeurJak"",""f [...]
+34,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""id"":6467254510,""node_id"":""LE_lADOB_z1Gs5IJOZzzwAAAAGBenju"",""url"":""https://api.github.com/repos/panjf2000/ants/issues/events/6467254510"",""actor"":{""login"":""FeurJak"",""id"":77963837,""node_id"":""MDQ6VXNlcjc3OTYzODM3"",""avatar_url"":""https://avatars.githubusercontent.com/u/77963837?v=4"",""gravatar_id"":"""",""url"":""https://api.github.com/users/FeurJak"",""html_url"":""https://github.com/FeurJak"",""f [...]
+35,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""id"":6443467220,""node_id"":""AE_lADOB_z1Gs5H5AWkzwAAAAGAD4HU"",""url"":""https://api.github.com/repos/panjf2000/ants/issues/events/6443467220"",""actor"":{""login"":""panjf2000"",""id"":7496278,""node_id"":""MDQ6VXNlcjc0OTYyNzg="",""avatar_url"":""https://avatars.githubusercontent.com/u/7496278?v=4"",""gravatar_id"":"""",""url"":""https://api.github.com/users/panjf2000"",""html_url"":""https://github.com/panjf2000"" [...]
+86,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}","{""id"":5977768239,""node_id"":""CE_lADOB_z1Gs5BYTRpzwAAAAFkTYEv"",""url"":""https://api.github.com/repos/panjf2000/ants/issues/events/5977768239"",""actor"":{""login"":""panjf2000"",""id"":7496278,""node_id"":""MDQ6VXNlcjc0OTYyNzg="",""avatar_url"":""https://avatars.githubusercontent.com/u/7496278?v=4"",""gravatar_id"":"""",""url"":""https://api.github.com/users/panjf2000"",""html_url"":""https://github.com/panjf2000"" [...]
diff --git a/plugins/github/e2e/snapshot_tables/_tool_github_issue_events.csv b/plugins/github/e2e/snapshot_tables/_tool_github_issue_events.csv
new file mode 100644
index 00000000..b59a3a15
--- /dev/null
+++ b/plugins/github/e2e/snapshot_tables/_tool_github_issue_events.csv
@@ -0,0 +1,50 @@
+connection_id,github_id,issue_id,type,author_username,github_created_at,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
+1,5977768239,1096889449,closed,panjf2000,2022-01-31T02:49:03.000+00:00,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_events,86,
+1,6018634131,1118109702,head_ref_force_pushed,codingfanlt,2022-02-07T11:42:33.000+00:00,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_events,85,
+1,6018650885,1118109702,head_ref_force_pushed,codingfanlt,2022-02-07T11:45:28.000+00:00,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_events,84,
+1,6022440198,1126520989,ready_for_review,lucafmarques,2022-02-07T21:31:56.000+00:00,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_events,83,
+1,6022451048,1126520989,head_ref_force_pushed,lucafmarques,2022-02-07T21:33:55.000+00:00,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_events,82,
+1,6024185240,1078047238,closed,panjf2000,2022-02-08T05:53:49.000+00:00,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_events,81,
+1,6026431404,1126520989,head_ref_force_pushed,lucafmarques,2022-02-08T12:31:54.000+00:00,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_events,80,
+1,6026450677,1126520989,head_ref_force_pushed,lucafmarques,2022-02-08T12:35:17.000+00:00,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_events,79,
+1,6026454371,1126520989,mentioned,panjf2000,2022-02-08T12:35:56.000+00:00,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_events,78,
+1,6026454374,1126520989,subscribed,panjf2000,2022-02-08T12:35:56.000+00:00,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_events,77,
+1,6030205564,1127816236,labeled,ShivanshVij,2022-02-08T22:16:27.000+00:00,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_events,76,
+1,6030205602,1127816236,assigned,panjf2000,2022-02-08T22:16:27.000+00:00,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_events,75,
+1,6031595363,1118109702,head_ref_force_pushed,codingfanlt,2022-02-09T05:13:23.000+00:00,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_events,74,
+1,6035033987,1127816236,labeled,panjf2000,2022-02-09T15:00:46.000+00:00,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_events,73,
+1,6443467220,1206125988,assigned,panjf2000,2022-04-16T14:33:57.000+00:00,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_events,35,
+1,6467254510,1210377843,labeled,FeurJak,2022-04-21T01:11:58.000+00:00,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_events,34,
+1,6467254512,1210377843,labeled,FeurJak,2022-04-21T01:11:58.000+00:00,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_events,33,
+1,6467254520,1210377843,assigned,panjf2000,2022-04-21T01:11:58.000+00:00,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_events,32,
+1,6485990121,1210377843,labeled,panjf2000,2022-04-24T15:19:07.000+00:00,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_events,31,
+1,6495314642,1215362122,labeled,yxiupei,2022-04-26T04:10:53.000+00:00,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_events,30,
+1,6495314644,1215362122,labeled,yxiupei,2022-04-26T04:10:53.000+00:00,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_events,29,
+1,6495314656,1215362122,assigned,panjf2000,2022-04-26T04:10:53.000+00:00,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_events,28,
+1,6553561086,1215362122,closed,panjf2000,2022-05-05T03:25:13.000+00:00,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_events,27,
+1,6553562170,1206125988,closed,panjf2000,2022-05-05T03:25:43.000+00:00,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_events,26,
+1,6567390248,1210377843,closed,panjf2000,2022-05-07T11:29:32.000+00:00,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_events,25,
+1,6567570821,898973379,closed,panjf2000,2022-05-07T14:45:12.000+00:00,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_events,24,
+1,6567571146,1196996550,closed,panjf2000,2022-05-07T14:45:36.000+00:00,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_events,23,
+1,6567581911,1074084207,closed,panjf2000,2022-05-07T14:57:13.000+00:00,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_events,22,
+1,6628985629,1239397157,closed,hugh-404,2022-05-18T04:00:54.000+00:00,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_events,21,
+1,6651984441,1243368453,labeled,zqlpaopao,2022-05-20T16:14:22.000+00:00,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_events,20,
+1,6651984442,1243368453,labeled,zqlpaopao,2022-05-20T16:14:22.000+00:00,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_events,19,
+1,6651984465,1243368453,assigned,panjf2000,2022-05-20T16:14:22.000+00:00,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_events,18,
+1,6655132907,1244030382,labeled,fufuok,2022-05-21T16:39:29.000+00:00,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_events,17,
+1,6655132917,1244030382,assigned,panjf2000,2022-05-21T16:39:30.000+00:00,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_events,16,
+1,6655982786,1244030382,closed,fufuok,2022-05-22T10:46:11.000+00:00,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_events,15,
+1,6709899399,1253913230,labeled,LiaoPuJian,2022-05-31T13:58:47.000+00:00,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_events,14,
+1,6709899424,1253913230,labeled,LiaoPuJian,2022-05-31T13:58:47.000+00:00,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_events,13,
+1,6709899522,1253913230,assigned,panjf2000,2022-05-31T13:58:48.000+00:00,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_events,12,
+1,6726343285,1257847610,labeled,ZhMaio,2022-06-02T08:13:04.000+00:00,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_events,11,
+1,6726343287,1257847610,labeled,ZhMaio,2022-06-02T08:13:04.000+00:00,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_events,10,
+1,6726343315,1257847610,assigned,panjf2000,2022-06-02T08:13:04.000+00:00,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_events,9,
+1,6726348886,1257847610,renamed,ZhMaio,2022-06-02T08:13:57.000+00:00,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_events,8,
+1,6759426885,1262980710,labeled,imDpeng,2022-06-07T09:11:46.000+00:00,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_events,7,
+1,6759426887,1262980710,labeled,imDpeng,2022-06-07T09:11:46.000+00:00,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_events,6,
+1,6759426948,1262980710,assigned,panjf2000,2022-06-07T09:11:47.000+00:00,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_events,5,
+1,6766051995,1264107782,labeled,iGen1us,2022-06-08T02:35:14.000+00:00,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_events,4,
+1,6766051996,1264107782,labeled,iGen1us,2022-06-08T02:35:14.000+00:00,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_events,3,
+1,6766052007,1264107782,assigned,panjf2000,2022-06-08T02:35:14.000+00:00,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_events,2,
+1,6774117135,1257847610,closed,ZhMaio,2022-06-09T01:42:11.000+00:00,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_events,1,
diff --git a/plugins/github/e2e/snapshot_tables/_tool_github_pull_request_issues.csv b/plugins/github/e2e/snapshot_tables/_tool_github_pull_request_issues.csv
new file mode 100644
index 00000000..5e3ff696
--- /dev/null
+++ b/plugins/github/e2e/snapshot_tables/_tool_github_pull_request_issues.csv
@@ -0,0 +1,2 @@
+connection_id,pull_request_id,issue_id,pull_request_number,issue_number,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
+1,246250598,401277739,23,22,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_pull_requests,255,
diff --git a/plugins/github/e2e/snapshot_tables/pull_request_issues.csv b/plugins/github/e2e/snapshot_tables/pull_request_issues.csv
new file mode 100644
index 00000000..24be7f00
--- /dev/null
+++ b/plugins/github/e2e/snapshot_tables/pull_request_issues.csv
@@ -0,0 +1,2 @@
+pull_request_id,issue_id,pull_request_number,issue_number,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
+gitlab:GithubPullRequest:1:246250598,gitlab:GithubIssue:1:401277739,23,22,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_pull_requests,255,
diff --git a/plugins/github/models/issue_event.go b/plugins/github/models/issue_event.go
index 1d47ec6d..a5c5c480 100644
--- a/plugins/github/models/issue_event.go
+++ b/plugins/github/models/issue_event.go
@@ -23,6 +23,7 @@ import (
)
type GithubIssueEvent struct {
+ ConnectionId uint64 `gorm:"primaryKey"`
GithubId int `gorm:"primaryKey"`
IssueId int `gorm:"index;comment:References the Issue"`
Type string `gorm:"type:varchar(255);comment:Events that can occur to an issue, ex. assigned, closed, labeled, etc."`
diff --git a/plugins/github/models/migrationscripts/archived/issue_event.go b/plugins/github/models/migrationscripts/archived/issue_event.go
index eb2d14aa..cc71b251 100644
--- a/plugins/github/models/migrationscripts/archived/issue_event.go
+++ b/plugins/github/models/migrationscripts/archived/issue_event.go
@@ -24,6 +24,7 @@ import (
)
type GithubIssueEvent struct {
+ ConnectionId uint64 `gorm:"primaryKey"`
GithubId int `gorm:"primaryKey"`
IssueId int `gorm:"index;comment:References the Issue"`
Type string `gorm:"type:varchar(255);comment:Events that can occur to an issue, ex. assigned, closed, labeled, etc."`
diff --git a/plugins/github/models/migrationscripts/archived/pull_request_issue.go b/plugins/github/models/migrationscripts/archived/pull_request_issue.go
index 67887fd2..0f22a55a 100644
--- a/plugins/github/models/migrationscripts/archived/pull_request_issue.go
+++ b/plugins/github/models/migrationscripts/archived/pull_request_issue.go
@@ -20,8 +20,9 @@ package archived
import "github.com/apache/incubator-devlake/models/migrationscripts/archived"
type GithubPullRequestIssue struct {
- PullRequestId int `gorm:"primaryKey"`
- IssueId int `gorm:"primaryKey"`
+ ConnectionId uint64 `gorm:"primaryKey"`
+ PullRequestId int `gorm:"primaryKey"`
+ IssueId int `gorm:"primaryKey"`
PullRequestNumber int
IssueNumber int
archived.NoPKModel
diff --git a/plugins/github/models/pr_issue.go b/plugins/github/models/pr_issue.go
index 088065d1..e1efcf32 100644
--- a/plugins/github/models/pr_issue.go
+++ b/plugins/github/models/pr_issue.go
@@ -20,8 +20,9 @@ package models
import "github.com/apache/incubator-devlake/models/common"
type GithubPullRequestIssue struct {
- PullRequestId int `gorm:"primaryKey"`
- IssueId int `gorm:"primaryKey"`
+ ConnectionId uint64 `gorm:"primaryKey"`
+ PullRequestId int `gorm:"primaryKey"`
+ IssueId int `gorm:"primaryKey"`
PullRequestNumber int
IssueNumber int
common.NoPKModel
diff --git a/plugins/github/tasks/event_collector.go b/plugins/github/tasks/event_collector.go
index b040baa7..f02fcc42 100644
--- a/plugins/github/tasks/event_collector.go
+++ b/plugins/github/tasks/event_collector.go
@@ -20,6 +20,7 @@ package tasks
import (
"encoding/json"
"fmt"
+ "github.com/apache/incubator-devlake/plugins/core/dal"
"net/http"
"net/url"
@@ -41,7 +42,7 @@ var CollectApiEventsMeta = core.SubTaskMeta{
}
func CollectApiEvents(taskCtx core.SubTaskContext) error {
- db := taskCtx.GetDb()
+ db := taskCtx.GetDal()
data := taskCtx.GetData().(*GithubTaskData)
since := data.Since
@@ -50,10 +51,13 @@ func CollectApiEvents(taskCtx core.SubTaskContext) error {
// actually, for github pull, since doesn't make any sense, github pull api doesn't support it
if since == nil {
var latestUpdatedIssueEvent models.GithubIssueEvent
- err := db.Model(&latestUpdatedIssueEvent).
- Joins("left join _tool_github_issues on _tool_github_issues.github_id = _tool_github_issue_events.issue_id").
- Where("_tool_github_issues.repo_id = ?", data.Repo.GithubId).
- Order("github_created_at DESC").Limit(1).Find(&latestUpdatedIssueEvent).Error
+ err := db.All(
+ &latestUpdatedIssueEvent,
+ dal.Join("left join _tool_github_issues on _tool_github_issues.github_id = _tool_github_issue_events.issue_id"),
+ dal.Where("_tool_github_issues.repo_id = ? and _tool_github_issues.repo_id = ?", data.Repo.GithubId, data.Repo.ConnectionId),
+ dal.Orderby("github_created_at DESC"),
+ dal.Limit(1),
+ )
if err != nil {
return fmt.Errorf("failed to get latest github issue record: %w", err)
}
@@ -69,8 +73,9 @@ func CollectApiEvents(taskCtx core.SubTaskContext) error {
RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
Ctx: taskCtx,
Params: GithubApiParams{
- Owner: data.Options.Owner,
- Repo: data.Options.Repo,
+ ConnectionId: data.Options.ConnectionId,
+ Owner: data.Options.Owner,
+ Repo: data.Options.Repo,
},
Table: RAW_EVENTS_TABLE,
},
diff --git a/plugins/github/tasks/event_extractor.go b/plugins/github/tasks/event_extractor.go
index 3b209090..d55283de 100644
--- a/plugins/github/tasks/event_extractor.go
+++ b/plugins/github/tasks/event_extractor.go
@@ -51,8 +51,9 @@ func ExtractApiEvents(taskCtx core.SubTaskContext) error {
RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
Ctx: taskCtx,
Params: GithubApiParams{
- Owner: data.Options.Owner,
- Repo: data.Options.Repo,
+ ConnectionId: data.Options.ConnectionId,
+ Owner: data.Options.Owner,
+ Repo: data.Options.Repo,
},
Table: RAW_EVENTS_TABLE,
},
@@ -66,7 +67,7 @@ func ExtractApiEvents(taskCtx core.SubTaskContext) error {
if body.GithubId == 0 {
return nil, nil
}
- githubIssueEvent, err := convertGithubEvent(body)
+ githubIssueEvent, err := convertGithubEvent(body, data.Options.ConnectionId)
if err != nil {
return nil, err
}
@@ -83,8 +84,9 @@ func ExtractApiEvents(taskCtx core.SubTaskContext) error {
return extractor.Execute()
}
-func convertGithubEvent(event *IssueEvent) (*models.GithubIssueEvent, error) {
+func convertGithubEvent(event *IssueEvent, connId uint64) (*models.GithubIssueEvent, error) {
githubEvent := &models.GithubIssueEvent{
+ ConnectionId: connId,
GithubId: event.GithubId,
IssueId: event.Issue.Id,
Type: event.Event,
diff --git a/plugins/github/tasks/pr_issue_convertor.go b/plugins/github/tasks/pr_issue_convertor.go
index 5a9ff729..84b6ed1e 100644
--- a/plugins/github/tasks/pr_issue_convertor.go
+++ b/plugins/github/tasks/pr_issue_convertor.go
@@ -18,6 +18,7 @@ limitations under the License.
package tasks
import (
+ "github.com/apache/incubator-devlake/plugins/core/dal"
"reflect"
"github.com/apache/incubator-devlake/models/domainlayer/crossdomain"
@@ -35,15 +36,16 @@ var ConvertPullRequestIssuesMeta = core.SubTaskMeta{
}
func ConvertPullRequestIssues(taskCtx core.SubTaskContext) error {
- db := taskCtx.GetDb()
+ db := taskCtx.GetDal()
data := taskCtx.GetData().(*GithubTaskData)
repoId := data.Repo.GithubId
- cursor, err := db.Model(&githubModels.GithubPullRequestIssue{}).
- Joins(`left join _tool_github_pull_requests on _tool_github_pull_requests.github_id = _tool_github_pull_request_issues.pull_request_id`).
- Where("_tool_github_pull_requests.repo_id = ?", repoId).
- Order("pull_request_id ASC").
- Rows()
+ cursor, err := db.Cursor(
+ dal.From(&githubModels.GithubPullRequestIssue{}),
+ dal.Join(`left join _tool_github_pull_requests on _tool_github_pull_requests.github_id = _tool_github_pull_request_issues.pull_request_id`),
+ dal.Where("_tool_github_pull_requests.repo_id = ? and _tool_github_pull_requests.connection_id = ?", repoId, data.Options.ConnectionId),
+ dal.Orderby("pull_request_id ASC"),
+ )
if err != nil {
return err
}
@@ -57,16 +59,17 @@ func ConvertPullRequestIssues(taskCtx core.SubTaskContext) error {
RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
Ctx: taskCtx,
Params: GithubApiParams{
- Owner: data.Options.Owner,
- Repo: data.Options.Repo,
+ ConnectionId: data.Options.ConnectionId,
+ Owner: data.Options.Owner,
+ Repo: data.Options.Repo,
},
Table: RAW_PULL_REQUEST_TABLE,
},
Convert: func(inputRow interface{}) ([]interface{}, error) {
githubPrIssue := inputRow.(*githubModels.GithubPullRequestIssue)
pullRequestIssue := &crossdomain.PullRequestIssue{
- PullRequestId: prIdGen.Generate(githubPrIssue.PullRequestId),
- IssueId: issueIdGen.Generate(githubPrIssue.IssueId),
+ PullRequestId: prIdGen.Generate(data.Options.ConnectionId, githubPrIssue.PullRequestId),
+ IssueId: issueIdGen.Generate(data.Options.ConnectionId, githubPrIssue.IssueId),
IssueNumber: githubPrIssue.IssueNumber,
PullRequestNumber: githubPrIssue.PullRequestNumber,
}
diff --git a/plugins/github/tasks/pr_issue_enricher.go b/plugins/github/tasks/pr_issue_enricher.go
index f16beacc..9ce7f25f 100644
--- a/plugins/github/tasks/pr_issue_enricher.go
+++ b/plugins/github/tasks/pr_issue_enricher.go
@@ -19,6 +19,7 @@ package tasks
import (
"github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/core/dal"
githubModels "github.com/apache/incubator-devlake/plugins/github/models"
"github.com/apache/incubator-devlake/plugins/helper"
"reflect"
@@ -35,7 +36,7 @@ var EnrichPullRequestIssuesMeta = core.SubTaskMeta{
}
func EnrichPullRequestIssues(taskCtx core.SubTaskContext) (err error) {
- db := taskCtx.GetDb()
+ db := taskCtx.GetDal()
data := taskCtx.GetData().(*GithubTaskData)
repoId := data.Repo.GithubId
@@ -48,9 +49,7 @@ func EnrichPullRequestIssues(taskCtx core.SubTaskContext) (err error) {
prBodyCloseRegex = regexp.MustCompile(prBodyClosePattern)
}
charPattern := regexp.MustCompile(`[a-zA-Z\s,]+`)
- cursor, err := db.Model(&githubModels.GithubPullRequest{}).
- Where("repo_id = ?", repoId).
- Rows()
+ cursor, err := db.Cursor(dal.From(&githubModels.GithubPullRequest{}), dal.Where("repo_id = ?", repoId))
if err != nil {
return err
}
@@ -63,8 +62,9 @@ func EnrichPullRequestIssues(taskCtx core.SubTaskContext) (err error) {
RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
Ctx: taskCtx,
Params: GithubApiParams{
- Owner: data.Options.Owner,
- Repo: data.Options.Repo,
+ ConnectionId: data.Options.ConnectionId,
+ Owner: data.Options.Owner,
+ Repo: data.Options.Repo,
},
Table: RAW_PULL_REQUEST_TABLE,
},
@@ -95,8 +95,11 @@ func EnrichPullRequestIssues(taskCtx core.SubTaskContext) (err error) {
if numFormatErr != nil {
continue
}
- err = db.Where("number = ? and repo_id = ?", issueNumber, repoId).
- Limit(1).Find(issue).Error
+ err = db.All(
+ issue,
+ dal.Where("number = ? and repo_id = ? and connection_id = ?", issueNumber, repoId, data.Options.ConnectionId),
+ dal.Limit(1),
+ )
if err != nil {
return nil, err
}
@@ -104,6 +107,7 @@ func EnrichPullRequestIssues(taskCtx core.SubTaskContext) (err error) {
continue
}
githubPullRequstIssue := &githubModels.GithubPullRequestIssue{
+ ConnectionId: data.Options.ConnectionId,
PullRequestId: githubPullRequst.GithubId,
IssueId: issue.GithubId,
PullRequestNumber: githubPullRequst.Number,