You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by kl...@apache.org on 2022/10/31 08:25:46 UTC
[incubator-devlake] branch main updated: fix: fix e2e verify table (#3592)
This is an automated email from the ASF dual-hosted git repository.
klesh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new fd6b8c1f fix: fix e2e verify table (#3592)
fd6b8c1f is described below
commit fd6b8c1f13b12eea704682be82ee65d3e269c9a5
Author: mappjzc <zh...@merico.dev>
AuthorDate: Mon Oct 31 16:25:41 2022 +0800
fix: fix e2e verify table (#3592)
* fix: fix e2e verify table
fix VerifyTableWithOptions error with caculate Total.
Nddtfjiang <zh...@merico.dev>
* fix: fix bitbucket e2e
remove BitbucketAccount VerifyTable on TestRepoDataFlow
Nddtfjiang <zh...@merico.dev>
* fix: github e2e
fix github e2e data wrong.
Nddtfjiang <zh...@merico.dev>
---
helpers/e2ehelper/data_flow_tester.go | 16 ++++++++--------
plugins/bitbucket/e2e/repo_test.go | 16 ----------------
plugins/github/e2e/pr_test.go | 1 +
.../snapshot_tables/_tool_github_accounts_in_pr.csv | 21 ---------------------
4 files changed, 9 insertions(+), 45 deletions(-)
diff --git a/helpers/e2ehelper/data_flow_tester.go b/helpers/e2ehelper/data_flow_tester.go
index f7301744..dc792b4f 100644
--- a/helpers/e2ehelper/data_flow_tester.go
+++ b/helpers/e2ehelper/data_flow_tester.go
@@ -426,9 +426,11 @@ func (t *DataFlowTester) VerifyTableWithOptions(dst schema.Tabler, opts TableOpt
csvIter := pluginhelper.NewCsvFileIterator(opts.CSVRelPath)
defer csvIter.Close()
- expectedTotal := int64(0)
+
+ var expectedTotal int64
csvMap := map[string]map[string]interface{}{}
for csvIter.HasNext() {
+ expectedTotal++
expected := csvIter.Fetch()
pkValues := make([]string, 0, len(pkColumns))
for _, pkc := range pkColumns {
@@ -442,12 +444,15 @@ func (t *DataFlowTester) VerifyTableWithOptions(dst schema.Tabler, opts TableOpt
}
csvMap[pkValueStr] = expected
}
+
+ var actualTotal int64
dbRows := &[]map[string]interface{}{}
err = t.Db.Table(dst.TableName()).Find(dbRows).Error
if err != nil {
panic(err)
}
for _, actual := range *dbRows {
+ actualTotal++
pkValues := make([]string, 0, len(pkColumns))
for _, pkc := range pkColumns {
pkValues = append(pkValues, formatDbValue(actual[pkc.Name()]))
@@ -460,12 +465,7 @@ func (t *DataFlowTester) VerifyTableWithOptions(dst schema.Tabler, opts TableOpt
for _, field := range targetFields {
assert.Equal(t.T, expected[field], formatDbValue(actual[field]), fmt.Sprintf(`%s.%s not match (with params from csv %s)`, dst.TableName(), field, pkValues))
}
- expectedTotal++
}
- var actualTotal int64
- err = t.Db.Table(dst.TableName()).Count(&actualTotal).Error
- if err != nil {
- panic(err)
- }
- assert.Equal(t.T, expectedTotal, actualTotal, fmt.Sprintf(`%s count not match`, dst.TableName()))
+
+ assert.Equal(t.T, expectedTotal, actualTotal, fmt.Sprintf(`%s count not match count,[expected:%d][actual:%d]`, dst.TableName(), expectedTotal, actualTotal))
}
diff --git a/plugins/bitbucket/e2e/repo_test.go b/plugins/bitbucket/e2e/repo_test.go
index 104cf0fd..a2f5d76c 100644
--- a/plugins/bitbucket/e2e/repo_test.go
+++ b/plugins/bitbucket/e2e/repo_test.go
@@ -51,7 +51,6 @@ func TestRepoDataFlow(t *testing.T) {
// verify extraction
dataflowTester.FlushTabler(&models.BitbucketRepo{})
- dataflowTester.FlushTabler(&models.BitbucketAccount{})
dataflowTester.Subtask(tasks.ExtractApiRepoMeta, taskData)
dataflowTester.VerifyTable(
models.BitbucketRepo{},
@@ -66,21 +65,6 @@ func TestRepoDataFlow(t *testing.T) {
"language",
),
)
- dataflowTester.VerifyTable(
- models.BitbucketAccount{},
- "./snapshot_tables/_tool_bitbucket_accounts.csv",
- e2ehelper.ColumnWithRawData(
- "connection_id",
- "user_name",
- "account_id",
- "account_status",
- "display_name",
- "avatar_url",
- "html_url",
- "uuid",
- "has2_fa_enabled",
- ),
- )
// verify extraction
dataflowTester.FlushTabler(&code.Repo{})
diff --git a/plugins/github/e2e/pr_test.go b/plugins/github/e2e/pr_test.go
index 61b78b35..2c7bd82d 100644
--- a/plugins/github/e2e/pr_test.go
+++ b/plugins/github/e2e/pr_test.go
@@ -56,6 +56,7 @@ func TestPrDataFlow(t *testing.T) {
dataflowTester.FlushTabler(&models.GithubPullRequest{})
dataflowTester.FlushTabler(&models.GithubPrLabel{})
dataflowTester.FlushTabler(&models.GithubAccount{})
+ dataflowTester.FlushTabler(&models.GithubRepoAccount{})
dataflowTester.Subtask(tasks.ExtractApiPullRequestsMeta, taskData)
dataflowTester.VerifyTable(
models.GithubPullRequest{},
diff --git a/plugins/github/e2e/snapshot_tables/_tool_github_accounts_in_pr.csv b/plugins/github/e2e/snapshot_tables/_tool_github_accounts_in_pr.csv
index 5807c3c7..4304d5f1 100644
--- a/plugins/github/e2e/snapshot_tables/_tool_github_accounts_in_pr.csv
+++ b/plugins/github/e2e/snapshot_tables/_tool_github_accounts_in_pr.csv
@@ -1,41 +1,28 @@
connection_id,account_id,repo_github_id,login,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
1,12317,134018330,jdamick,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_pull_requests,282,
1,21979,134018330,appleboy,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_pull_requests,270,
-1,47359,484251804,apache,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake-website""}",_raw_github_api_repositories,9,
1,192964,134018330,egonelbre,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_pull_requests,252,
-1,720086,134018330,huangjunwen,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_issues,27,
1,964542,134018330,sarathsp06,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_pull_requests,256,
-1,1284892,134018330,jjeffcaii,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_issues,39,
1,1290360,134018330,wwjiang,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_pull_requests,258,
1,1940588,134018330,bingoohuang,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_pull_requests,281,
1,2813260,134018330,KevinBaiSg,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_pull_requests,263,
1,2832687,134018330,kaiiak,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_pull_requests,276,
1,3794113,134018330,shanhuhai5739,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_pull_requests,249,
1,3814966,134018330,lilien1010,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_pull_requests,288,
-1,4555057,134018330,huiwq1990,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_issues,13,
-1,5044825,134018330,tenfyzhong,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_issues,30,
1,5227289,134018330,lntotk,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_pull_requests,269,
-1,5244267,134018330,jiashiwen,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_issues,31,
1,5268051,134018330,lord63,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_pull_requests,290,
-1,5668717,134018330,kklinan,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_issues,26,
1,5715152,134018330,RealLiuSha,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_pull_requests,265,
1,7411249,134018330,imxyb,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_pull_requests,275,
1,7496278,134018330,panjf2000,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_pull_requests,261,
1,7931755,134018330,zplzpl,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_pull_requests,254,
-1,8509898,134018330,prprprus,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_issues,35,
1,8518239,134018330,gitter-badger,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_pull_requests,271,
1,8597823,134018330,hongli-my,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_pull_requests,247,
-1,8605102,134018330,pathbox,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_issues,11,
1,8644923,134018330,polar9527,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_pull_requests,264,
1,8923413,134018330,choleraehyq,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_pull_requests,262,
-1,10361713,134018330,Ainiroad,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_issues,32,
-1,11763614,134018330,Moonlight-Zhao,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_issues,16,
1,11977524,134018330,arjunmahishi,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_pull_requests,273,
1,12420699,134018330,shanghai-Jerry,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_pull_requests,267,
1,12890888,134018330,liyonglion,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_pull_requests,250,
1,12907474,134018330,wreulicke,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_pull_requests,268,
-1,13118848,134018330,lovelly,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_issues,12,
-1,13944100,134018330,LinuxForYQH,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_issues,24,
1,15234973,134018330,lucafmarques,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_pull_requests,294,
1,15699766,134018330,TwiN,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_pull_requests,287,
1,16658738,134018330,barryz,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_pull_requests,246,
@@ -43,25 +30,17 @@ connection_id,account_id,repo_github_id,login,_raw_data_params,_raw_data_table,_
1,17495446,134018330,Anteoy,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_pull_requests,259,
1,20608155,134018330,SimePel,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_pull_requests,251,
1,22312935,134018330,lam2003,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_pull_requests,266,
-1,22429695,134018330,codecov[bot],"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3246,
1,22613193,134018330,exfly,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_pull_requests,284,
1,22676124,134018330,liu-song,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_pull_requests,289,
1,24841832,134018330,rikewang,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_pull_requests,253,
1,27898261,134018330,qmdx00,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_pull_requests,291,
-1,29241786,134018330,ylwangs,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3188,
-1,29243953,134018330,naiba,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_issues,34,
-1,29452204,134018330,edcismybrother,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_issues,18,
1,29589055,134018330,Mutated1994,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_pull_requests,274,
1,32893410,134018330,zhangyuanxue,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_pull_requests,278,
-1,32898629,134018330,hawklin2017,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_comments,3238,
1,35493957,134018330,codingfanlt,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_pull_requests,293,
-1,38367404,134018330,liliang8858,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_issues,33,
1,38849208,134018330,king526,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_pull_requests,257,
1,41562937,134018330,yddeng,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_pull_requests,279,
1,42808204,134018330,Nonnnnnnnnn,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_pull_requests,248,
-1,42910462,134018330,golangcibot,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_pull_request_review_comments,11,
1,47921612,134018330,Comolli,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_pull_requests,277,
-1,48135919,134018330,tsatke,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_issues,37,
1,49174849,134018330,thinkgos,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_pull_requests,283,
1,58211133,134018330,icecube092,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_pull_requests,280,
1,64823610,134018330,bright2227,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_github_api_pull_requests,292,