You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by kl...@apache.org on 2022/07/04 06:57:49 UTC
[incubator-devlake] branch main updated: feat: record user info in github plugin (#2406)
This is an automated email from the ASF dual-hosted git repository.
klesh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 0c62a875 feat: record user info in github plugin (#2406)
0c62a875 is described below
commit 0c62a875bdb80771ab2f0b569238edfafb465b1e
Author: likyh <l...@likyh.com>
AuthorDate: Mon Jul 4 14:57:45 2022 +0800
feat: record user info in github plugin (#2406)
* feat: record user info in github plugin
* fix: fix for ci
Co-authored-by: linyh <ya...@meri.co>
---
plugins/github/e2e/comment_test.go | 14 ++++++
plugins/github/e2e/commit_test.go | 1 -
plugins/github/e2e/event_test.go | 14 ++++++
plugins/github/e2e/issue_test.go | 14 ++++++
plugins/github/e2e/pr_reviewer_test.go | 14 ++++++
plugins/github/e2e/pr_test.go | 15 ++++++
plugins/github/e2e/repo_test.go | 14 ++++++
.../_tool_github_users_in_comment.csv | 18 +++++++
.../_tool_github_users_in_event.csv | 14 ++++++
.../_tool_github_users_in_issue.csv | 22 +++++++++
.../snapshot_tables/_tool_github_users_in_pr.csv | 46 ++++++++++++++++++
.../snapshot_tables/_tool_github_users_in_repo.csv | 2 +
.../_tool_github_users_in_review.csv | 4 ++
plugins/github/tasks/comment_extractor.go | 14 +++---
plugins/github/tasks/event_extractor.go | 11 +++--
plugins/github/tasks/issue_extractor.go | 37 +++++++-------
plugins/github/tasks/pr_extractor.go | 21 ++++----
plugins/github/tasks/pr_review_extractor.go | 12 +++--
plugins/github/tasks/repo_extractor.go | 8 +++-
plugins/github/tasks/user_convertor.go | 2 +-
plugins/github/tasks/user_extractor.go | 56 ++++++++++++++++++++++
21 files changed, 308 insertions(+), 45 deletions(-)
diff --git a/plugins/github/e2e/comment_test.go b/plugins/github/e2e/comment_test.go
index 8b2cc5f6..76d3b872 100644
--- a/plugins/github/e2e/comment_test.go
+++ b/plugins/github/e2e/comment_test.go
@@ -65,6 +65,7 @@ func TestCommentDataFlow(t *testing.T) {
// verify extraction
dataflowTester.FlushTabler(&models.GithubIssueComment{})
dataflowTester.FlushTabler(&models.GithubPullRequestComment{})
+ dataflowTester.FlushTabler(&models.GithubUser{})
dataflowTester.Subtask(tasks.ExtractApiCommentsMeta, taskData)
dataflowTester.VerifyTable(
models.GithubIssueComment{},
@@ -102,6 +103,19 @@ func TestCommentDataFlow(t *testing.T) {
"_raw_data_remark",
},
)
+ dataflowTester.VerifyTable(
+ models.GithubUser{},
+ "./snapshot_tables/_tool_github_users_in_comment.csv",
+ []string{
+ "connection_id",
+ "id",
+ "login",
+ "avatar_url",
+ "url",
+ "html_url",
+ "type",
+ },
+ )
// verify comment conversion
dataflowTester.FlushTabler(&ticket.IssueComment{})
diff --git a/plugins/github/e2e/commit_test.go b/plugins/github/e2e/commit_test.go
index be5a7b33..ecbbf679 100644
--- a/plugins/github/e2e/commit_test.go
+++ b/plugins/github/e2e/commit_test.go
@@ -30,5 +30,4 @@ func TestCommitDataFlow(t *testing.T) {
assert.False(t, tasks.CollectApiCommitStatsMeta.EnabledByDefault)
assert.False(t, tasks.ExtractApiCommitStatsMeta.EnabledByDefault)
assert.False(t, tasks.ConvertCommitsMeta.EnabledByDefault)
- assert.False(t, tasks.ConvertUsersMeta.EnabledByDefault)
}
diff --git a/plugins/github/e2e/event_test.go b/plugins/github/e2e/event_test.go
index cba2b2e6..32d2616d 100644
--- a/plugins/github/e2e/event_test.go
+++ b/plugins/github/e2e/event_test.go
@@ -47,6 +47,7 @@ func TestEventDataFlow(t *testing.T) {
// verify extraction
dataflowTester.FlushTabler(&models.GithubIssueEvent{})
+ dataflowTester.FlushTabler(&models.GithubUser{})
dataflowTester.Subtask(tasks.ExtractApiEventsMeta, taskData)
dataflowTester.VerifyTable(
models.GithubIssueEvent{},
@@ -64,4 +65,17 @@ func TestEventDataFlow(t *testing.T) {
"_raw_data_remark",
},
)
+ dataflowTester.VerifyTable(
+ models.GithubUser{},
+ "./snapshot_tables/_tool_github_users_in_event.csv",
+ []string{
+ "connection_id",
+ "id",
+ "login",
+ "avatar_url",
+ "url",
+ "html_url",
+ "type",
+ },
+ )
}
diff --git a/plugins/github/e2e/issue_test.go b/plugins/github/e2e/issue_test.go
index 593a02ae..1e2339d1 100644
--- a/plugins/github/e2e/issue_test.go
+++ b/plugins/github/e2e/issue_test.go
@@ -60,6 +60,7 @@ func TestIssueDataFlow(t *testing.T) {
// verify issue extraction
dataflowTester.FlushTabler(&models.GithubIssue{})
dataflowTester.FlushTabler(&models.GithubIssueLabel{})
+ dataflowTester.FlushTabler(&models.GithubUser{})
dataflowTester.Subtask(tasks.ExtractApiIssuesMeta, taskData)
dataflowTester.VerifyTable(
models.GithubIssue{},
@@ -105,6 +106,19 @@ func TestIssueDataFlow(t *testing.T) {
"_raw_data_remark",
},
)
+ dataflowTester.VerifyTable(
+ models.GithubUser{},
+ "./snapshot_tables/_tool_github_users_in_issue.csv",
+ []string{
+ "connection_id",
+ "id",
+ "login",
+ "avatar_url",
+ "url",
+ "html_url",
+ "type",
+ },
+ )
// verify issue conversion
dataflowTester.FlushTabler(&ticket.Issue{})
diff --git a/plugins/github/e2e/pr_reviewer_test.go b/plugins/github/e2e/pr_reviewer_test.go
index 0abbec01..7bb8838b 100644
--- a/plugins/github/e2e/pr_reviewer_test.go
+++ b/plugins/github/e2e/pr_reviewer_test.go
@@ -48,6 +48,7 @@ func TestPrReviewerDataFlow(t *testing.T) {
// verify extraction
dataflowTester.FlushTabler(&models.GithubReviewer{})
+ dataflowTester.FlushTabler(&models.GithubUser{})
dataflowTester.Subtask(tasks.ExtractApiPullRequestReviewersMeta, taskData)
dataflowTester.VerifyTable(
models.GithubReviewer{},
@@ -63,4 +64,17 @@ func TestPrReviewerDataFlow(t *testing.T) {
"_raw_data_remark",
},
)
+ dataflowTester.VerifyTable(
+ models.GithubUser{},
+ "./snapshot_tables/_tool_github_users_in_review.csv",
+ []string{
+ "connection_id",
+ "id",
+ "login",
+ "avatar_url",
+ "url",
+ "html_url",
+ "type",
+ },
+ )
}
diff --git a/plugins/github/e2e/pr_test.go b/plugins/github/e2e/pr_test.go
index 7069de81..0dd08dfc 100644
--- a/plugins/github/e2e/pr_test.go
+++ b/plugins/github/e2e/pr_test.go
@@ -54,6 +54,7 @@ func TestPrDataFlow(t *testing.T) {
// verify pr extraction
dataflowTester.FlushTabler(&models.GithubPullRequest{})
dataflowTester.FlushTabler(&models.GithubPullRequestLabel{})
+ dataflowTester.FlushTabler(&models.GithubUser{})
dataflowTester.Subtask(tasks.ExtractApiPullRequestsMeta, taskData)
dataflowTester.VerifyTable(
models.GithubPullRequest{},
@@ -99,6 +100,20 @@ func TestPrDataFlow(t *testing.T) {
[]string{"connection_id", "pull_id", "label_name"},
)
+ dataflowTester.VerifyTable(
+ models.GithubUser{},
+ "./snapshot_tables/_tool_github_users_in_pr.csv",
+ []string{
+ "connection_id",
+ "id",
+ "login",
+ "avatar_url",
+ "url",
+ "html_url",
+ "type",
+ },
+ )
+
// verify pr conversion
dataflowTester.FlushTabler(&code.PullRequest{})
dataflowTester.Subtask(tasks.ConvertPullRequestsMeta, taskData)
diff --git a/plugins/github/e2e/repo_test.go b/plugins/github/e2e/repo_test.go
index cbd5455a..d4512b64 100644
--- a/plugins/github/e2e/repo_test.go
+++ b/plugins/github/e2e/repo_test.go
@@ -53,6 +53,7 @@ func TestRepoDataFlow(t *testing.T) {
// verify extraction
dataflowTester.FlushTabler(&models.GithubRepo{})
+ dataflowTester.FlushTabler(&models.GithubUser{})
dataflowTester.Subtask(tasks.ExtractApiRepoMeta, taskData)
dataflowTester.VerifyTable(
models.GithubRepo{},
@@ -74,6 +75,19 @@ func TestRepoDataFlow(t *testing.T) {
"_raw_data_remark",
},
)
+ dataflowTester.VerifyTable(
+ models.GithubUser{},
+ "./snapshot_tables/_tool_github_users_in_repo.csv",
+ []string{
+ "connection_id",
+ "id",
+ "login",
+ "avatar_url",
+ "url",
+ "html_url",
+ "type",
+ },
+ )
// verify extraction
dataflowTester.FlushTabler(&code.Repo{})
diff --git a/plugins/github/e2e/snapshot_tables/_tool_github_users_in_comment.csv b/plugins/github/e2e/snapshot_tables/_tool_github_users_in_comment.csv
new file mode 100644
index 00000000..570871a2
--- /dev/null
+++ b/plugins/github/e2e/snapshot_tables/_tool_github_users_in_comment.csv
@@ -0,0 +1,18 @@
+connection_id,id,login,avatar_url,url,html_url,type
+1,192964,egonelbre,https://avatars.githubusercontent.com/u/192964?v=4,https://api.github.com/users/egonelbre,https://github.com/egonelbre,User
+1,720086,huangjunwen,https://avatars.githubusercontent.com/u/720086?v=4,https://api.github.com/users/huangjunwen,https://github.com/huangjunwen,User
+1,4555057,huiwq1990,https://avatars.githubusercontent.com/u/4555057?v=4,https://api.github.com/users/huiwq1990,https://github.com/huiwq1990,User
+1,5668717,kklinan,https://avatars.githubusercontent.com/u/5668717?v=4,https://api.github.com/users/kklinan,https://github.com/kklinan,User
+1,7496278,panjf2000,https://avatars.githubusercontent.com/u/7496278?v=4,https://api.github.com/users/panjf2000,https://github.com/panjf2000,User
+1,7931755,zplzpl,https://avatars.githubusercontent.com/u/7931755?v=4,https://api.github.com/users/zplzpl,https://github.com/zplzpl,User
+1,8597823,hongli-my,https://avatars.githubusercontent.com/u/8597823?v=4,https://api.github.com/users/hongli-my,https://github.com/hongli-my,User
+1,8923413,choleraehyq,https://avatars.githubusercontent.com/u/8923413?v=4,https://api.github.com/users/choleraehyq,https://github.com/choleraehyq,User
+1,11763614,Moonlight-Zhao,https://avatars.githubusercontent.com/u/11763614?v=4,https://api.github.com/users/Moonlight-Zhao,https://github.com/Moonlight-Zhao,User
+1,12890888,liyonglion,https://avatars.githubusercontent.com/u/12890888?v=4,https://api.github.com/users/liyonglion,https://github.com/liyonglion,User
+1,13118848,lovelly,https://avatars.githubusercontent.com/u/13118848?v=4,https://api.github.com/users/lovelly,https://github.com/lovelly,User
+1,16658738,barryz,https://avatars.githubusercontent.com/u/16658738?v=4,https://api.github.com/users/barryz,https://github.com/barryz,User
+1,22429695,codecov[bot],https://avatars.githubusercontent.com/in/254?v=4,https://api.github.com/users/codecov%5Bbot%5D,https://github.com/apps/codecov,Bot
+1,29241786,ylwangs,https://avatars.githubusercontent.com/u/29241786?v=4,https://api.github.com/users/ylwangs,https://github.com/ylwangs,User
+1,29243953,naiba,https://avatars.githubusercontent.com/u/29243953?v=4,https://api.github.com/users/naiba,https://github.com/naiba,User
+1,29452204,edcismybrother,https://avatars.githubusercontent.com/u/29452204?v=4,https://api.github.com/users/edcismybrother,https://github.com/edcismybrother,User
+1,32898629,hawklin2017,https://avatars.githubusercontent.com/u/32898629?v=4,https://api.github.com/users/hawklin2017,https://github.com/hawklin2017,User
diff --git a/plugins/github/e2e/snapshot_tables/_tool_github_users_in_event.csv b/plugins/github/e2e/snapshot_tables/_tool_github_users_in_event.csv
new file mode 100644
index 00000000..35dc0a1f
--- /dev/null
+++ b/plugins/github/e2e/snapshot_tables/_tool_github_users_in_event.csv
@@ -0,0 +1,14 @@
+connection_id,id,login,avatar_url,url,html_url,type
+1,4979407,fufuok,https://avatars.githubusercontent.com/u/4979407?v=4,https://api.github.com/users/fufuok,https://github.com/fufuok,User
+1,7496278,panjf2000,https://avatars.githubusercontent.com/u/7496278?v=4,https://api.github.com/users/panjf2000,https://github.com/panjf2000,User
+1,14806824,iGen1us,https://avatars.githubusercontent.com/u/14806824?v=4,https://api.github.com/users/iGen1us,https://github.com/iGen1us,User
+1,15234973,lucafmarques,https://avatars.githubusercontent.com/u/15234973?v=4,https://api.github.com/users/lucafmarques,https://github.com/lucafmarques,User
+1,27162109,ShivanshVij,https://avatars.githubusercontent.com/u/27162109?v=4,https://api.github.com/users/ShivanshVij,https://github.com/ShivanshVij,User
+1,29474400,LiaoPuJian,https://avatars.githubusercontent.com/u/29474400?v=4,https://api.github.com/users/LiaoPuJian,https://github.com/LiaoPuJian,User
+1,30883503,yxiupei,https://avatars.githubusercontent.com/u/30883503?v=4,https://api.github.com/users/yxiupei,https://github.com/yxiupei,User
+1,35493957,codingfanlt,https://avatars.githubusercontent.com/u/35493957?v=4,https://api.github.com/users/codingfanlt,https://github.com/codingfanlt,User
+1,39076288,hugh-404,https://avatars.githubusercontent.com/u/39076288?v=4,https://api.github.com/users/hugh-404,https://github.com/hugh-404,User
+1,43371021,zqlpaopao,https://avatars.githubusercontent.com/u/43371021?v=4,https://api.github.com/users/zqlpaopao,https://github.com/zqlpaopao,User
+1,47515663,ZhMaio,https://avatars.githubusercontent.com/u/47515663?v=4,https://api.github.com/users/ZhMaio,https://github.com/ZhMaio,User
+1,48267340,imDpeng,https://avatars.githubusercontent.com/u/48267340?v=4,https://api.github.com/users/imDpeng,https://github.com/imDpeng,User
+1,77963837,FeurJak,https://avatars.githubusercontent.com/u/77963837?v=4,https://api.github.com/users/FeurJak,https://github.com/FeurJak,User
diff --git a/plugins/github/e2e/snapshot_tables/_tool_github_users_in_issue.csv b/plugins/github/e2e/snapshot_tables/_tool_github_users_in_issue.csv
new file mode 100644
index 00000000..23427a5c
--- /dev/null
+++ b/plugins/github/e2e/snapshot_tables/_tool_github_users_in_issue.csv
@@ -0,0 +1,22 @@
+connection_id,id,login,avatar_url,url,html_url,type
+1,720086,huangjunwen,https://avatars.githubusercontent.com/u/720086?v=4,https://api.github.com/users/huangjunwen,https://github.com/huangjunwen,User
+1,1284892,jjeffcaii,https://avatars.githubusercontent.com/u/1284892?v=4,https://api.github.com/users/jjeffcaii,https://github.com/jjeffcaii,User
+1,1290360,wwjiang,https://avatars.githubusercontent.com/u/1290360?v=4,https://api.github.com/users/wwjiang,https://github.com/wwjiang,User
+1,4555057,huiwq1990,https://avatars.githubusercontent.com/u/4555057?v=4,https://api.github.com/users/huiwq1990,https://github.com/huiwq1990,User
+1,5044825,tenfyzhong,https://avatars.githubusercontent.com/u/5044825?v=4,https://api.github.com/users/tenfyzhong,https://github.com/tenfyzhong,User
+1,5244267,jiashiwen,https://avatars.githubusercontent.com/u/5244267?v=4,https://api.github.com/users/jiashiwen,https://github.com/jiashiwen,User
+1,5668717,kklinan,https://avatars.githubusercontent.com/u/5668717?v=4,https://api.github.com/users/kklinan,https://github.com/kklinan,User
+1,7496278,panjf2000,https://avatars.githubusercontent.com/u/7496278?v=4,https://api.github.com/users/panjf2000,https://github.com/panjf2000,User
+1,7931755,zplzpl,https://avatars.githubusercontent.com/u/7931755?v=4,https://api.github.com/users/zplzpl,https://github.com/zplzpl,User
+1,8509898,prprprus,https://avatars.githubusercontent.com/u/8509898?v=4,https://api.github.com/users/prprprus,https://github.com/prprprus,User
+1,8605102,pathbox,https://avatars.githubusercontent.com/u/8605102?v=4,https://api.github.com/users/pathbox,https://github.com/pathbox,User
+1,8923413,choleraehyq,https://avatars.githubusercontent.com/u/8923413?v=4,https://api.github.com/users/choleraehyq,https://github.com/choleraehyq,User
+1,10361713,Ainiroad,https://avatars.githubusercontent.com/u/10361713?v=4,https://api.github.com/users/Ainiroad,https://github.com/Ainiroad,User
+1,11763614,Moonlight-Zhao,https://avatars.githubusercontent.com/u/11763614?v=4,https://api.github.com/users/Moonlight-Zhao,https://github.com/Moonlight-Zhao,User
+1,13118848,lovelly,https://avatars.githubusercontent.com/u/13118848?v=4,https://api.github.com/users/lovelly,https://github.com/lovelly,User
+1,13944100,LinuxForYQH,https://avatars.githubusercontent.com/u/13944100?v=4,https://api.github.com/users/LinuxForYQH,https://github.com/LinuxForYQH,User
+1,29243953,naiba,https://avatars.githubusercontent.com/u/29243953?v=4,https://api.github.com/users/naiba,https://github.com/naiba,User
+1,29452204,edcismybrother,https://avatars.githubusercontent.com/u/29452204?v=4,https://api.github.com/users/edcismybrother,https://github.com/edcismybrother,User
+1,38367404,liliang8858,https://avatars.githubusercontent.com/u/38367404?v=4,https://api.github.com/users/liliang8858,https://github.com/liliang8858,User
+1,38849208,king526,https://avatars.githubusercontent.com/u/38849208?v=4,https://api.github.com/users/king526,https://github.com/king526,User
+1,48135919,tsatke,https://avatars.githubusercontent.com/u/48135919?v=4,https://api.github.com/users/tsatke,https://github.com/tsatke,User
diff --git a/plugins/github/e2e/snapshot_tables/_tool_github_users_in_pr.csv b/plugins/github/e2e/snapshot_tables/_tool_github_users_in_pr.csv
new file mode 100644
index 00000000..9de30a7a
--- /dev/null
+++ b/plugins/github/e2e/snapshot_tables/_tool_github_users_in_pr.csv
@@ -0,0 +1,46 @@
+connection_id,id,login,avatar_url,url,html_url,type
+1,12317,jdamick,https://avatars.githubusercontent.com/u/12317?v=4,https://api.github.com/users/jdamick,https://github.com/jdamick,User
+1,21979,appleboy,https://avatars.githubusercontent.com/u/21979?v=4,https://api.github.com/users/appleboy,https://github.com/appleboy,User
+1,192964,egonelbre,https://avatars.githubusercontent.com/u/192964?v=4,https://api.github.com/users/egonelbre,https://github.com/egonelbre,User
+1,964542,sarathsp06,https://avatars.githubusercontent.com/u/964542?v=4,https://api.github.com/users/sarathsp06,https://github.com/sarathsp06,User
+1,1290360,wwjiang,https://avatars.githubusercontent.com/u/1290360?v=4,https://api.github.com/users/wwjiang,https://github.com/wwjiang,User
+1,1940588,bingoohuang,https://avatars.githubusercontent.com/u/1940588?v=4,https://api.github.com/users/bingoohuang,https://github.com/bingoohuang,User
+1,2813260,KevinBaiSg,https://avatars.githubusercontent.com/u/2813260?v=4,https://api.github.com/users/KevinBaiSg,https://github.com/KevinBaiSg,User
+1,2832687,kaiiak,https://avatars.githubusercontent.com/u/2832687?v=4,https://api.github.com/users/kaiiak,https://github.com/kaiiak,User
+1,3794113,shanhuhai5739,https://avatars.githubusercontent.com/u/3794113?v=4,https://api.github.com/users/shanhuhai5739,https://github.com/shanhuhai5739,User
+1,3814966,lilien1010,https://avatars.githubusercontent.com/u/3814966?v=4,https://api.github.com/users/lilien1010,https://github.com/lilien1010,User
+1,5227289,lntotk,https://avatars.githubusercontent.com/u/5227289?v=4,https://api.github.com/users/lntotk,https://github.com/lntotk,User
+1,5268051,lord63,https://avatars.githubusercontent.com/u/5268051?v=4,https://api.github.com/users/lord63,https://github.com/lord63,User
+1,5715152,RealLiuSha,https://avatars.githubusercontent.com/u/5715152?v=4,https://api.github.com/users/RealLiuSha,https://github.com/RealLiuSha,User
+1,7411249,imxyb,https://avatars.githubusercontent.com/u/7411249?v=4,https://api.github.com/users/imxyb,https://github.com/imxyb,User
+1,7496278,panjf2000,https://avatars.githubusercontent.com/u/7496278?v=4,https://api.github.com/users/panjf2000,https://github.com/panjf2000,User
+1,7931755,zplzpl,https://avatars.githubusercontent.com/u/7931755?v=4,https://api.github.com/users/zplzpl,https://github.com/zplzpl,User
+1,8518239,gitter-badger,https://avatars.githubusercontent.com/u/8518239?v=4,https://api.github.com/users/gitter-badger,https://github.com/gitter-badger,User
+1,8597823,hongli-my,https://avatars.githubusercontent.com/u/8597823?v=4,https://api.github.com/users/hongli-my,https://github.com/hongli-my,User
+1,8644923,polar9527,https://avatars.githubusercontent.com/u/8644923?v=4,https://api.github.com/users/polar9527,https://github.com/polar9527,User
+1,8923413,choleraehyq,https://avatars.githubusercontent.com/u/8923413?v=4,https://api.github.com/users/choleraehyq,https://github.com/choleraehyq,User
+1,11977524,arjunmahishi,https://avatars.githubusercontent.com/u/11977524?v=4,https://api.github.com/users/arjunmahishi,https://github.com/arjunmahishi,User
+1,12420699,shanghai-Jerry,https://avatars.githubusercontent.com/u/12420699?v=4,https://api.github.com/users/shanghai-Jerry,https://github.com/shanghai-Jerry,User
+1,12890888,liyonglion,https://avatars.githubusercontent.com/u/12890888?v=4,https://api.github.com/users/liyonglion,https://github.com/liyonglion,User
+1,12907474,wreulicke,https://avatars.githubusercontent.com/u/12907474?v=4,https://api.github.com/users/wreulicke,https://github.com/wreulicke,User
+1,15234973,lucafmarques,https://avatars.githubusercontent.com/u/15234973?v=4,https://api.github.com/users/lucafmarques,https://github.com/lucafmarques,User
+1,15699766,TwiN,https://avatars.githubusercontent.com/u/15699766?v=4,https://api.github.com/users/TwiN,https://github.com/TwiN,User
+1,16658738,barryz,https://avatars.githubusercontent.com/u/16658738?v=4,https://api.github.com/users/barryz,https://github.com/barryz,User
+1,17286982,automano,https://avatars.githubusercontent.com/u/17286982?v=4,https://api.github.com/users/automano,https://github.com/automano,User
+1,17495446,Anteoy,https://avatars.githubusercontent.com/u/17495446?v=4,https://api.github.com/users/Anteoy,https://github.com/Anteoy,User
+1,20608155,SimePel,https://avatars.githubusercontent.com/u/20608155?v=4,https://api.github.com/users/SimePel,https://github.com/SimePel,User
+1,22312935,lam2003,https://avatars.githubusercontent.com/u/22312935?v=4,https://api.github.com/users/lam2003,https://github.com/lam2003,User
+1,22613193,exfly,https://avatars.githubusercontent.com/u/22613193?v=4,https://api.github.com/users/exfly,https://github.com/exfly,User
+1,22676124,liu-song,https://avatars.githubusercontent.com/u/22676124?v=4,https://api.github.com/users/liu-song,https://github.com/liu-song,User
+1,24841832,rikewang,https://avatars.githubusercontent.com/u/24841832?v=4,https://api.github.com/users/rikewang,https://github.com/rikewang,User
+1,27898261,qmdx00,https://avatars.githubusercontent.com/u/27898261?v=4,https://api.github.com/users/qmdx00,https://github.com/qmdx00,User
+1,29589055,Mutated1994,https://avatars.githubusercontent.com/u/29589055?v=4,https://api.github.com/users/Mutated1994,https://github.com/Mutated1994,User
+1,32893410,zhangyuanxue,https://avatars.githubusercontent.com/u/32893410?v=4,https://api.github.com/users/zhangyuanxue,https://github.com/zhangyuanxue,User
+1,35493957,codingfanlt,https://avatars.githubusercontent.com/u/35493957?v=4,https://api.github.com/users/codingfanlt,https://github.com/codingfanlt,User
+1,38849208,king526,https://avatars.githubusercontent.com/u/38849208?v=4,https://api.github.com/users/king526,https://github.com/king526,User
+1,41562937,yddeng,https://avatars.githubusercontent.com/u/41562937?v=4,https://api.github.com/users/yddeng,https://github.com/yddeng,User
+1,42808204,Nonnnnnnnnn,https://avatars.githubusercontent.com/u/42808204?v=4,https://api.github.com/users/Nonnnnnnnnn,https://github.com/Nonnnnnnnnn,User
+1,47921612,Comolli,https://avatars.githubusercontent.com/u/47921612?v=4,https://api.github.com/users/Comolli,https://github.com/Comolli,User
+1,49174849,thinkgos,https://avatars.githubusercontent.com/u/49174849?v=4,https://api.github.com/users/thinkgos,https://github.com/thinkgos,User
+1,58211133,icecube092,https://avatars.githubusercontent.com/u/58211133?v=4,https://api.github.com/users/icecube092,https://github.com/icecube092,User
+1,64823610,bright2227,https://avatars.githubusercontent.com/u/64823610?v=4,https://api.github.com/users/bright2227,https://github.com/bright2227,User
diff --git a/plugins/github/e2e/snapshot_tables/_tool_github_users_in_repo.csv b/plugins/github/e2e/snapshot_tables/_tool_github_users_in_repo.csv
new file mode 100644
index 00000000..54ba4369
--- /dev/null
+++ b/plugins/github/e2e/snapshot_tables/_tool_github_users_in_repo.csv
@@ -0,0 +1,2 @@
+connection_id,id,login,avatar_url,url,html_url,type
+1,7496278,panjf2000,https://avatars.githubusercontent.com/u/7496278?v=4,https://api.github.com/users/panjf2000,https://github.com/panjf2000,User
diff --git a/plugins/github/e2e/snapshot_tables/_tool_github_users_in_review.csv b/plugins/github/e2e/snapshot_tables/_tool_github_users_in_review.csv
new file mode 100644
index 00000000..cbcfdae5
--- /dev/null
+++ b/plugins/github/e2e/snapshot_tables/_tool_github_users_in_review.csv
@@ -0,0 +1,4 @@
+connection_id,id,login,avatar_url,url,html_url,type
+1,2813260,KevinBaiSg,https://avatars.githubusercontent.com/u/2813260?u=b36f46765f6e90f5cace0d526e0eee393017f109&v=4,https://api.github.com/users/KevinBaiSg,https://github.com/KevinBaiSg,User
+1,7496278,panjf2000,https://avatars.githubusercontent.com/u/7496278?u=07ba348f0ca79d78e45b1f628723a7c3abf80fac&v=4,https://api.github.com/users/panjf2000,https://github.com/panjf2000,User
+1,8923413,choleraehyq,https://avatars.githubusercontent.com/u/8923413?u=1053453176d6be4d727304fe7b134098c9ecf316&v=4,https://api.github.com/users/choleraehyq,https://github.com/choleraehyq,User
diff --git a/plugins/github/tasks/comment_extractor.go b/plugins/github/tasks/comment_extractor.go
index 6946d349..24164234 100644
--- a/plugins/github/tasks/comment_extractor.go
+++ b/plugins/github/tasks/comment_extractor.go
@@ -38,12 +38,9 @@ var ExtractApiCommentsMeta = core.SubTaskMeta{
}
type IssueComment struct {
- GithubId int `json:"id"`
- Body json.RawMessage
- User struct {
- Login string
- Id int
- }
+ GithubId int `json:"id"`
+ Body json.RawMessage
+ User *GithubUserResponse
IssueUrl string `json:"issue_url"`
GithubCreatedAt helper.Iso8601Time `json:"created_at"`
GithubUpdatedAt helper.Iso8601Time `json:"updated_at"`
@@ -114,6 +111,11 @@ func ExtractApiComments(taskCtx core.SubTaskContext) error {
}
results = append(results, githubIssueComment)
}
+ githubUser, err := convertUser(apiComment.User, data.Options.ConnectionId)
+ if err != nil {
+ return nil, err
+ }
+ results = append(results, githubUser)
return results, nil
},
})
diff --git a/plugins/github/tasks/event_extractor.go b/plugins/github/tasks/event_extractor.go
index 9081bc88..9d3a66a1 100644
--- a/plugins/github/tasks/event_extractor.go
+++ b/plugins/github/tasks/event_extractor.go
@@ -36,10 +36,8 @@ var ExtractApiEventsMeta = core.SubTaskMeta{
type IssueEvent struct {
GithubId int `json:"id"`
Event string
- Actor struct {
- Login string
- }
- Issue struct {
+ Actor *GithubUserResponse
+ Issue struct {
Id int
}
GithubCreatedAt helper.Iso8601Time `json:"created_at"`
@@ -73,6 +71,11 @@ func ExtractApiEvents(taskCtx core.SubTaskContext) error {
return nil, err
}
results = append(results, githubIssueEvent)
+ githubUser, err := convertUser(body.Actor, data.Options.ConnectionId)
+ if err != nil {
+ return nil, err
+ }
+ results = append(results, githubUser)
return results, nil
},
diff --git a/plugins/github/tasks/issue_extractor.go b/plugins/github/tasks/issue_extractor.go
index 4643501b..5888e1f5 100644
--- a/plugins/github/tasks/issue_extractor.go
+++ b/plugins/github/tasks/issue_extractor.go
@@ -50,14 +50,8 @@ type IssuesResponse struct {
Name string `json:"name"`
} `json:"labels"`
- Assignee *struct {
- Login string
- Id int
- }
- User *struct {
- Login string
- Id int
- }
+ Assignee *GithubUserResponse
+ User *GithubUserResponse
ClosedAt *helper.Iso8601Time `json:"closed_at"`
GithubCreatedAt helper.Iso8601Time `json:"created_at"`
GithubUpdatedAt helper.Iso8601Time `json:"updated_at"`
@@ -133,6 +127,24 @@ func ExtractApiIssues(taskCtx core.SubTaskContext) error {
if err != nil {
return nil, err
}
+ if body.Assignee != nil {
+ githubIssue.AssigneeId = body.Assignee.Id
+ githubIssue.AssigneeName = body.Assignee.Login
+ relatedUser, err := convertUser(body.Assignee, data.Options.ConnectionId)
+ if err != nil {
+ return nil, err
+ }
+ results = append(results, relatedUser)
+ }
+ if body.User != nil {
+ githubIssue.AuthorId = body.User.Id
+ githubIssue.AuthorName = body.User.Login
+ relatedUser, err := convertUser(body.User, data.Options.ConnectionId)
+ if err != nil {
+ return nil, err
+ }
+ results = append(results, relatedUser)
+ }
for _, label := range body.Labels {
results = append(results, &models.GithubIssueLabel{
ConnectionId: data.Options.ConnectionId,
@@ -204,15 +216,6 @@ func convertGithubIssue(issue *IssuesResponse, connectionId uint64, repositoryId
GithubCreatedAt: issue.GithubCreatedAt.ToTime(),
GithubUpdatedAt: issue.GithubUpdatedAt.ToTime(),
}
-
- if issue.Assignee != nil {
- githubIssue.AssigneeId = issue.Assignee.Id
- githubIssue.AssigneeName = issue.Assignee.Login
- }
- if issue.User != nil {
- githubIssue.AuthorId = issue.User.Id
- githubIssue.AuthorName = issue.User.Login
- }
if issue.ClosedAt != nil {
githubIssue.LeadTimeMinutes = uint(issue.ClosedAt.ToTime().Sub(issue.GithubCreatedAt.ToTime()).Minutes())
}
diff --git a/plugins/github/tasks/pr_extractor.go b/plugins/github/tasks/pr_extractor.go
index 1790d658..c50f7b0a 100644
--- a/plugins/github/tasks/pr_extractor.go
+++ b/plugins/github/tasks/pr_extractor.go
@@ -44,14 +44,8 @@ type GithubApiPullRequest struct {
Labels []struct {
Name string `json:"name"`
} `json:"labels"`
- Assignee *struct {
- Login string
- Id int
- }
- User *struct {
- Id int
- Login string
- }
+ Assignee *GithubUserResponse
+ User *GithubUserResponse
ClosedAt *helper.Iso8601Time `json:"closed_at"`
MergedAt *helper.Iso8601Time `json:"merged_at"`
GithubCreatedAt helper.Iso8601Time `json:"created_at"`
@@ -114,6 +108,15 @@ func ExtractApiPullRequests(taskCtx core.SubTaskContext) error {
if err != nil {
return nil, err
}
+ if rawL.User != nil {
+ githubUser, err := convertUser(rawL.User, data.Options.ConnectionId)
+ if err != nil {
+ return nil, err
+ }
+ results = append(results, githubUser)
+ githubPr.AuthorName = githubUser.Login
+ githubPr.AuthorId = githubUser.Id
+ }
for _, label := range rawL.Labels {
results = append(results, &models.GithubPullRequestLabel{
ConnectionId: data.Options.ConnectionId,
@@ -157,8 +160,6 @@ func convertGithubPullRequest(pull *GithubApiPullRequest, connId uint64, repoId
State: pull.State,
Title: pull.Title,
Url: pull.HtmlUrl,
- AuthorName: pull.User.Login,
- AuthorId: pull.User.Id,
GithubCreatedAt: pull.GithubCreatedAt.ToTime(),
GithubUpdatedAt: pull.GithubUpdatedAt.ToTime(),
ClosedAt: helper.Iso8601TimeToTime(pull.ClosedAt),
diff --git a/plugins/github/tasks/pr_review_extractor.go b/plugins/github/tasks/pr_review_extractor.go
index 1136ed0d..c56ca23e 100644
--- a/plugins/github/tasks/pr_review_extractor.go
+++ b/plugins/github/tasks/pr_review_extractor.go
@@ -35,11 +35,8 @@ var ExtractApiPullRequestReviewersMeta = core.SubTaskMeta{
}
type PullRequestReview struct {
- GithubId int `json:"id"`
- User struct {
- Id int
- Login string
- }
+ GithubId int `json:"id"`
+ User *GithubUserResponse
Body string
State string
SubmittedAt helper.Iso8601Time `json:"submitted_at"`
@@ -88,6 +85,11 @@ func ExtractApiPullRequestReviewers(taskCtx core.SubTaskContext) error {
PullRequestId: pull.GithubId,
}
results = append(results, githubReviewer)
+ githubUser, err := convertUser(apiPullRequestReview.User, data.Options.ConnectionId)
+ if err != nil {
+ return nil, err
+ }
+ results = append(results, githubUser)
return results, nil
},
diff --git a/plugins/github/tasks/repo_extractor.go b/plugins/github/tasks/repo_extractor.go
index 4446ff8a..dcb33662 100644
--- a/plugins/github/tasks/repo_extractor.go
+++ b/plugins/github/tasks/repo_extractor.go
@@ -42,7 +42,7 @@ type GithubApiRepo struct {
HTMLUrl string `json:"html_url"`
Language string `json:"language"`
Description string `json:"description"`
- Owner models.GithubUser
+ Owner *GithubUserResponse
Parent *GithubApiRepo `json:"parent"`
CreatedAt helper.Iso8601Time `json:"created_at"`
UpdatedAt *helper.Iso8601Time `json:"updated_at"`
@@ -98,6 +98,12 @@ func ExtractApiRepositories(taskCtx core.SubTaskContext) error {
}
results = append(results, githubRepository)
+ githubUser, err := convertUser(body.Owner, data.Options.ConnectionId)
+ if err != nil {
+ return nil, err
+ }
+ results = append(results, githubUser)
+
parentTaskContext := taskCtx.TaskContext()
if parentTaskContext != nil {
parentTaskContext.GetData().(*GithubTaskData).Repo = githubRepository
diff --git a/plugins/github/tasks/user_convertor.go b/plugins/github/tasks/user_convertor.go
index 284746ea..3f1bbdc9 100644
--- a/plugins/github/tasks/user_convertor.go
+++ b/plugins/github/tasks/user_convertor.go
@@ -33,7 +33,7 @@ import (
var ConvertUsersMeta = core.SubTaskMeta{
Name: "convertUsers",
EntryPoint: ConvertUsers,
- EnabledByDefault: false,
+ EnabledByDefault: true,
Description: "Convert tool layer table github_users into domain layer table users",
DomainTypes: []string{core.DOMAIN_TYPE_CROSS},
}
diff --git a/plugins/github/tasks/user_extractor.go b/plugins/github/tasks/user_extractor.go
new file mode 100644
index 00000000..8e227a23
--- /dev/null
+++ b/plugins/github/tasks/user_extractor.go
@@ -0,0 +1,56 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "github.com/apache/incubator-devlake/plugins/github/models"
+)
+
+type GithubUserResponse struct {
+ Login string `json:"login"`
+ Id int `json:"id"`
+ NodeId string `json:"node_id"`
+ AvatarUrl string `json:"avatar_url"`
+ GravatarId string `json:"gravatar_id"`
+ Url string `json:"url"`
+ HtmlUrl string `json:"html_url"`
+ FollowersUrl string `json:"followers_url"`
+ FollowingUrl string `json:"following_url"`
+ GistsUrl string `json:"gists_url"`
+ StarredUrl string `json:"starred_url"`
+ SubscriptionsUrl string `json:"subscriptions_url"`
+ OrganizationsUrl string `json:"organizations_url"`
+ ReposUrl string `json:"repos_url"`
+ EventsUrl string `json:"events_url"`
+ ReceivedEventsUrl string `json:"received_events_url"`
+ Type string `json:"type"`
+ SiteAdmin bool `json:"site_admin"`
+}
+
+func convertUser(res *GithubUserResponse, connId uint64) (*models.GithubUser, error) {
+ githubUser := &models.GithubUser{
+ ConnectionId: connId,
+ Id: res.Id,
+ Login: res.Login,
+ AvatarUrl: res.AvatarUrl,
+ Url: res.Url,
+ HtmlUrl: res.HtmlUrl,
+ Type: res.Type,
+ }
+ return githubUser, nil
+}