You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by wa...@apache.org on 2022/09/15 10:36:09 UTC
[incubator-devlake] branch main updated: feat(dora): e2e
This is an automated email from the ASF dual-hosted git repository.
warren pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 2d321a2b feat(dora): e2e
2d321a2b is described below
commit 2d321a2b6fc369a95eaaebff870d83093e69d0ae
Author: Yingchu Chen <yi...@merico.dev>
AuthorDate: Thu Sep 15 17:19:06 2022 +0800
feat(dora): e2e
relate to #2853
---
helpers/e2ehelper/data_flow_tester.go | 9 +++++
..._test.go => calculate_change_lead_time_test.go} | 20 ++++++----
...richer_test.go => connect_issue_deploy_test.go} | 21 +++++++----
plugins/dora/e2e/env_enricher_test.go | 3 +-
plugins/dora/e2e/raw_tables/board_issues.csv | 12 ++++++
plugins/dora/e2e/raw_tables/board_repos.csv | 2 +
plugins/dora/e2e/raw_tables/commits.csv | 34 +++++++++++++++++
plugins/dora/e2e/raw_tables/issues.csv | 15 ++++++++
.../raw_tables/lake_cicd_tasks_for_other_jobs.csv | 22 +++++++++++
.../dora/e2e/raw_tables/pull_request_comments.csv | 25 ++++++++++++
.../dora/e2e/raw_tables/pull_request_commits.csv | 34 +++++++++++++++++
plugins/dora/e2e/raw_tables/pull_requests.csv | 10 +++++
plugins/dora/e2e/snapshot_tables/issues.csv | 15 ++++++++
.../dora/e2e/snapshot_tables/lake_cicd_tasks.csv | 44 +++++++++++-----------
plugins/dora/e2e/snapshot_tables/pull_requests.csv | 10 +++++
plugins/dora/tasks/change_lead_time_calculator.go | 18 ++++-----
plugins/dora/tasks/issue_deploy_connector.go | 20 +++++-----
plugins/github/tasks/commit_convertor.go | 2 +-
plugins/github/tasks/pr_commit_extractor.go | 9 ++++-
19 files changed, 265 insertions(+), 60 deletions(-)
diff --git a/helpers/e2ehelper/data_flow_tester.go b/helpers/e2ehelper/data_flow_tester.go
index 3d4e5e47..d847ae63 100644
--- a/helpers/e2ehelper/data_flow_tester.go
+++ b/helpers/e2ehelper/data_flow_tester.go
@@ -249,6 +249,8 @@ func (t *DataFlowTester) CreateSnapshot(dst schema.Tabler, opts TableOptions) {
forScanValues[i] = new(bool)
} else if columnType.ScanType().Name() == `RawBytes` {
forScanValues[i] = new(sql.NullString)
+ } else if columnType.ScanType().Name() == `NullInt64` {
+ forScanValues[i] = new(sql.NullInt64)
} else {
forScanValues[i] = new(string)
}
@@ -282,6 +284,13 @@ func (t *DataFlowTester) CreateSnapshot(dst schema.Tabler, opts TableOptions) {
} else {
values[i] = ``
}
+ case *sql.NullInt64:
+ value := *forScanValues[i].(*sql.NullInt64)
+ if value.Valid {
+ values[i] = strconv.FormatInt(value.Int64, 10)
+ } else {
+ values[i] = ``
+ }
case *string:
values[i] = fmt.Sprint(*forScanValues[i].(*string))
}
diff --git a/plugins/dora/e2e/env_enricher_test.go b/plugins/dora/e2e/calculate_change_lead_time_test.go
similarity index 68%
copy from plugins/dora/e2e/env_enricher_test.go
copy to plugins/dora/e2e/calculate_change_lead_time_test.go
index d0c8da59..e648e297 100644
--- a/plugins/dora/e2e/env_enricher_test.go
+++ b/plugins/dora/e2e/calculate_change_lead_time_test.go
@@ -18,16 +18,17 @@ limitations under the License.
package e2e
import (
+ "github.com/apache/incubator-devlake/models/common"
+ "github.com/apache/incubator-devlake/models/domainlayer/code"
"testing"
"github.com/apache/incubator-devlake/helpers/e2ehelper"
- "github.com/apache/incubator-devlake/models/common"
"github.com/apache/incubator-devlake/models/domainlayer/devops"
"github.com/apache/incubator-devlake/plugins/dora/impl"
"github.com/apache/incubator-devlake/plugins/dora/tasks"
)
-func TestEnrichEnvDataFlow(t *testing.T) {
+func TestCalculateCLTimeDataFlow(t *testing.T) {
var plugin impl.Dora
dataflowTester := e2ehelper.NewDataFlowTester(t, "dora", plugin)
@@ -40,16 +41,19 @@ func TestEnrichEnvDataFlow(t *testing.T) {
},
},
}
-
+ dataflowTester.FlushTabler(&code.PullRequest{})
// import raw data table
dataflowTester.ImportCsvIntoTabler("./raw_tables/lake_cicd_pipeline_repos.csv", &devops.CiCDPipelineRepo{})
- dataflowTester.ImportCsvIntoTabler("./raw_tables/lake_cicd_tasks.csv", &devops.CICDTask{})
+ dataflowTester.ImportCsvIntoTabler("./raw_tables/lake_cicd_tasks_for_other_jobs.csv", &devops.CICDTask{})
+ dataflowTester.ImportCsvIntoTabler("./raw_tables/pull_requests.csv", &code.PullRequest{})
+ dataflowTester.ImportCsvIntoTabler("./raw_tables/pull_request_comments.csv", &code.PullRequestComment{})
+ dataflowTester.ImportCsvIntoTabler("./raw_tables/pull_request_commits.csv", &code.PullRequestCommit{})
+ dataflowTester.ImportCsvIntoTabler("./raw_tables/commits.csv", &code.Commit{})
// verify converter
- dataflowTester.FlushTabler(&devops.CICDTask{})
- dataflowTester.Subtask(tasks.EnrichTaskEnvMeta, taskData)
- dataflowTester.VerifyTableWithOptions(&devops.CICDTask{}, e2ehelper.TableOptions{
- CSVRelPath: "./snapshot_tables/lake_cicd_tasks.csv",
+ dataflowTester.Subtask(tasks.CalculateChangeLeadTimeMeta, taskData)
+ dataflowTester.VerifyTableWithOptions(&code.PullRequest{}, e2ehelper.TableOptions{
+ CSVRelPath: "./snapshot_tables/pull_requests.csv",
IgnoreTypes: []interface{}{common.NoPKModel{}},
})
}
diff --git a/plugins/dora/e2e/env_enricher_test.go b/plugins/dora/e2e/connect_issue_deploy_test.go
similarity index 68%
copy from plugins/dora/e2e/env_enricher_test.go
copy to plugins/dora/e2e/connect_issue_deploy_test.go
index d0c8da59..bfbe9dae 100644
--- a/plugins/dora/e2e/env_enricher_test.go
+++ b/plugins/dora/e2e/connect_issue_deploy_test.go
@@ -18,16 +18,19 @@ limitations under the License.
package e2e
import (
+ "github.com/apache/incubator-devlake/models/common"
+ "github.com/apache/incubator-devlake/models/domainlayer/code"
+ "github.com/apache/incubator-devlake/models/domainlayer/crossdomain"
+ "github.com/apache/incubator-devlake/models/domainlayer/ticket"
"testing"
"github.com/apache/incubator-devlake/helpers/e2ehelper"
- "github.com/apache/incubator-devlake/models/common"
"github.com/apache/incubator-devlake/models/domainlayer/devops"
"github.com/apache/incubator-devlake/plugins/dora/impl"
"github.com/apache/incubator-devlake/plugins/dora/tasks"
)
-func TestEnrichEnvDataFlow(t *testing.T) {
+func TestConnectIssueDeployDataFlow(t *testing.T) {
var plugin impl.Dora
dataflowTester := e2ehelper.NewDataFlowTester(t, "dora", plugin)
@@ -40,16 +43,18 @@ func TestEnrichEnvDataFlow(t *testing.T) {
},
},
}
-
+ dataflowTester.FlushTabler(&code.PullRequest{})
// import raw data table
dataflowTester.ImportCsvIntoTabler("./raw_tables/lake_cicd_pipeline_repos.csv", &devops.CiCDPipelineRepo{})
- dataflowTester.ImportCsvIntoTabler("./raw_tables/lake_cicd_tasks.csv", &devops.CICDTask{})
+ dataflowTester.ImportCsvIntoTabler("./raw_tables/lake_cicd_tasks_for_other_jobs.csv", &devops.CICDTask{})
+ dataflowTester.ImportCsvIntoTabler("./raw_tables/board_issues.csv", &ticket.BoardIssue{})
+ dataflowTester.ImportCsvIntoTabler("./raw_tables/board_repos.csv", &crossdomain.BoardRepo{})
+ dataflowTester.ImportCsvIntoTabler("./raw_tables/issues.csv", &ticket.Issue{})
// verify converter
- dataflowTester.FlushTabler(&devops.CICDTask{})
- dataflowTester.Subtask(tasks.EnrichTaskEnvMeta, taskData)
- dataflowTester.VerifyTableWithOptions(&devops.CICDTask{}, e2ehelper.TableOptions{
- CSVRelPath: "./snapshot_tables/lake_cicd_tasks.csv",
+ dataflowTester.Subtask(tasks.ConnectIssueDeployMeta, taskData)
+ dataflowTester.VerifyTableWithOptions(&ticket.Issue{}, e2ehelper.TableOptions{
+ CSVRelPath: "./snapshot_tables/issues.csv",
IgnoreTypes: []interface{}{common.NoPKModel{}},
})
}
diff --git a/plugins/dora/e2e/env_enricher_test.go b/plugins/dora/e2e/env_enricher_test.go
index d0c8da59..b300bbdd 100644
--- a/plugins/dora/e2e/env_enricher_test.go
+++ b/plugins/dora/e2e/env_enricher_test.go
@@ -41,12 +41,13 @@ func TestEnrichEnvDataFlow(t *testing.T) {
},
}
+ dataflowTester.FlushTabler(&devops.CICDTask{})
+
// import raw data table
dataflowTester.ImportCsvIntoTabler("./raw_tables/lake_cicd_pipeline_repos.csv", &devops.CiCDPipelineRepo{})
dataflowTester.ImportCsvIntoTabler("./raw_tables/lake_cicd_tasks.csv", &devops.CICDTask{})
// verify converter
- dataflowTester.FlushTabler(&devops.CICDTask{})
dataflowTester.Subtask(tasks.EnrichTaskEnvMeta, taskData)
dataflowTester.VerifyTableWithOptions(&devops.CICDTask{}, e2ehelper.TableOptions{
CSVRelPath: "./snapshot_tables/lake_cicd_tasks.csv",
diff --git a/plugins/dora/e2e/raw_tables/board_issues.csv b/plugins/dora/e2e/raw_tables/board_issues.csv
new file mode 100644
index 00000000..7a5d40b1
--- /dev/null
+++ b/plugins/dora/e2e/raw_tables/board_issues.csv
@@ -0,0 +1,12 @@
+board_id,issue_id,created_at,updated_at,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
+github:GithubRepo:1:384111310,github:GithubIssue:1:1367714738,2022-09-15 05:36:36.083,2022-09-15 05:36:36.083,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_issues,3370,""
+github:GithubRepo:1:384111310,github:GithubIssue:1:1370813347,2022-09-15 05:36:36.083,2022-09-15 05:36:36.083,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_issues,3388,""
+github:GithubRepo:1:384111310,github:GithubIssue:1:1370816458,2022-09-15 05:36:36.083,2022-09-15 05:36:36.083,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_issues,3389,""
+github:GithubRepo:1:384111310,github:GithubIssue:1:1370833897,2022-09-15 05:36:36.083,2022-09-15 05:36:36.083,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_issues,3390,""
+github:GithubRepo:1:384111310,github:GithubIssue:1:1371315795,2022-09-15 05:36:36.083,2022-09-15 05:36:36.083,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_issues,3392,""
+github:GithubRepo:1:384111310,github:GithubIssue:1:1371320153,2022-09-15 05:36:36.083,2022-09-15 05:36:36.083,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_issues,3393,""
+github:GithubRepo:1:384111310,github:GithubIssue:1:1371541422,2022-09-15 05:36:36.083,2022-09-15 05:36:36.083,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_issues,3394,""
+github:GithubRepo:1:384111310,github:GithubIssue:1:1371617668,2022-09-15 05:36:36.083,2022-09-15 05:36:36.083,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_issues,3395,""
+github:GithubRepo:1:384111310,github:GithubIssue:1:1372381019,2022-09-15 05:36:36.083,2022-09-15 05:36:36.083,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_issues,3397,""
+github:GithubRepo:1:384111310,github:GithubIssue:1:1372550825,2022-09-15 05:36:36.083,2022-09-15 05:36:36.083,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_issues,3399,""
+github:GithubRepo:1:384111310,github:GithubIssue:1:1372581663,2022-09-15 05:36:36.083,2022-09-15 05:36:36.083,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_issues,3401,""
diff --git a/plugins/dora/e2e/raw_tables/board_repos.csv b/plugins/dora/e2e/raw_tables/board_repos.csv
new file mode 100644
index 00000000..78302224
--- /dev/null
+++ b/plugins/dora/e2e/raw_tables/board_repos.csv
@@ -0,0 +1,2 @@
+board_id,repo_id,created_at,updated_at,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
+github:GithubRepo:1:384111310,github:GithubRepo:1:384111310,2022-09-15 05:36:33.649,2022-09-15 05:36:33.649,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_repositories,14,""
diff --git a/plugins/dora/e2e/raw_tables/commits.csv b/plugins/dora/e2e/raw_tables/commits.csv
new file mode 100644
index 00000000..a11dbf7d
--- /dev/null
+++ b/plugins/dora/e2e/raw_tables/commits.csv
@@ -0,0 +1,34 @@
+created_at,updated_at,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark,sha,additions,deletions,dev_eq,message,author_name,author_email,authored_date,author_id,committer_name,committer_email,committed_date,committer_id
+2022-09-15 06:32:54.669,2022-09-15 06:32:54.669,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_pull_request_commits,26494,"",08d2f2b6de0fa8de4d0e2b55b4b9a2e244214029,0,0,0,"""feat: e2e test for jenkins stages\n\nAdd TestJenkinsStagesDataFlow.\n\nNddtfjiang <zh...@merico.dev>""",Nddtfjiang,jzc001@qq.com,2022-09-09 09:18:48,jzc001@qq.com,Nddtfjiang,jzc001@qq.com,2022-09-09 09:18:48,jzc001@qq.com
+2022-09-15 06:32:54.669,2022-09-15 06:32:54.669,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_pull_request_commits,22153,"",0edd5d0e0d18516d3309fec90553c3ee7380c149,0,0,0,"""build: update all lake-builder tag references to 'latest'""",Keon Amini,keon.amini@merico.dev,2022-09-09 03:00:36,keon.amini@merico.dev,Keon Amini,keon.amini@merico.dev,2022-09-09 03:12:44,keon.amini@merico.dev
+2022-09-15 06:32:55.267,2022-09-15 06:32:55.267,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_pull_request_commits,22100,"",209168a4b12d91a7ad1f3711fe05b39d5bd8e260,0,0,0,"""fix: update pipelineId""",abeizn,zikuan.an@merico.dev,2022-09-09 08:29:21,zikuan.an@merico.dev,abeizn,zikuan.an@merico.dev,2022-09-09 10:32:54,zikuan.an@merico.dev
+2022-09-15 06:32:55.267,2022-09-15 06:32:55.267,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_pull_request_commits,22139,"",2537845559d8db99e9cda6190f32b50ec979c722,0,0,0,"""fix: linting strike 8""",Klesh Wong,zhenmian.huang@merico.dev,2022-09-09 14:28:54,zhenmian.huang@merico.dev,Klesh Wong,zhenmian.huang@merico.dev,2022-09-09 14:28:54,zhenmian.huang@merico.dev
+2022-09-15 06:32:55.735,2022-09-15 06:32:55.735,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_pull_request_commits,22140,"",3aba559962e04949a1731f7d5155c7aac2e900ce,0,0,0,"""fix: linting strike 9""",Klesh Wong,zhenmian.huang@merico.dev,2022-09-09 14:40:36,zhenmian.huang@merico.dev,Klesh Wong,zhenmian.huang@merico.dev,2022-09-09 14:40:36,zhenmian.huang@merico.dev
+2022-09-15 06:32:55.735,2022-09-15 06:32:55.735,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_pull_request_commits,22144,"",3ec3e113ed36a5a8978af295683085054e4dfdd9,0,0,0,"""fix: linting strike 12""",Klesh Wong,zhenmian.huang@merico.dev,2022-09-09 16:25:11,zhenmian.huang@merico.dev,Klesh Wong,zhenmian.huang@merico.dev,2022-09-09 16:25:11,zhenmian.huang@merico.dev
+2022-09-15 06:32:55.735,2022-09-15 06:32:55.735,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_pull_request_commits,22152,"",4b71faf666833c0c7b915a512811e2c5e746d3de,0,0,0,"""build: bump lake-builder version to 0.0.8""",Keon Amini,keon.amini@merico.dev,2022-09-07 20:29:35,keon.amini@merico.dev,Keon Amini,keon.amini@merico.dev,2022-09-09 03:12:44,keon.amini@merico.dev
+2022-09-15 06:32:55.735,2022-09-15 06:32:55.735,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_pull_request_commits,22154,"",4f8cdefc9a9d53af16dd482c61623312eb9e9b5e,0,0,0,"""feat: new plugin for gitea""",tk103331,tk103331@gmail.com,2022-09-07 05:07:29,tk103331@gmail.com,tk,tk103331@gmail.com,2022-09-09 08:08:38,tk103331@gmail.com
+2022-09-15 06:32:56.240,2022-09-15 06:32:56.240,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_pull_request_commits,26356,"",55f445997abbd5918da59d202d28762cd56fbd44,0,0,0,"""feat: sub-task ExtractCustomizedFields for plugin customize""",zhangliang,liang.zhang@merico.dev,2022-09-06 15:47:38,liang.zhang@merico.dev,zhangliang,liang.zhang@merico.dev,2022-09-08 14:39:08,liang.zhang@merico.dev
+2022-09-15 06:32:56.240,2022-09-15 06:32:56.240,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_pull_request_commits,26519,"",56b895f0443730c6d7abfbc51a05ab35abd2971f,0,0,0,"""fix: task failure caused by 404""",zhangliang,liang.zhang@merico.dev,2022-09-09 12:13:06,liang.zhang@merico.dev,zhangliang,liang.zhang@merico.dev,2022-09-09 12:13:06,liang.zhang@merico.dev
+2022-09-15 06:32:56.240,2022-09-15 06:32:56.240,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_pull_request_commits,22136,"",5ad0c09c447c19338f1dfbb65d89a3728962b3b7,0,0,0,"""fix: linting strike 5""",Klesh Wong,zhenmian.huang@merico.dev,2022-09-09 11:49:58,zhenmian.huang@merico.dev,Klesh Wong,zhenmian.huang@merico.dev,2022-09-09 11:49:58,zhenmian.huang@merico.dev
+2022-09-15 06:32:56.240,2022-09-15 06:32:56.240,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_pull_request_commits,22145,"",62535543802631a0d3daf0b0b78c6a7e05e508fb,0,0,0,"""fix: test cases""",Klesh Wong,zhenmian.huang@merico.dev,2022-09-09 16:41:44,zhenmian.huang@merico.dev,Klesh Wong,zhenmian.huang@merico.dev,2022-09-09 16:41:44,zhenmian.huang@merico.dev
+2022-09-15 06:32:56.240,2022-09-15 06:32:56.240,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_pull_request_commits,26360,"",6559c5691ef290d2ff739d15da27e5e95994293d,0,0,0,"""refactor: use join statement reduce query times""",zhangliang,liang.zhang@merico.dev,2022-09-07 06:53:08,liang.zhang@merico.dev,zhangliang,liang.zhang@merico.dev,2022-09-08 14:39:08,liang.zhang@merico.dev
+2022-09-15 06:32:56.240,2022-09-15 06:32:56.240,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_pull_request_commits,26359,"",6a5edde13a90eeeef42e73a87974588443d5f9ca,0,0,0,"""fix: fix e2e test""",zhangliang,liang.zhang@merico.dev,2022-09-07 00:43:31,liang.zhang@merico.dev,zhangliang,liang.zhang@merico.dev,2022-09-08 14:39:08,liang.zhang@merico.dev
+2022-09-15 06:32:56.699,2022-09-15 06:32:56.699,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_pull_request_commits,22135,"",74cc484551329c7f92dc03519398dba62f96f128,0,0,0,"""fix: linting strike 4""",Klesh Wong,zhenmian.huang@merico.dev,2022-09-09 11:26:05,zhenmian.huang@merico.dev,Klesh Wong,zhenmian.huang@merico.dev,2022-09-09 11:26:05,zhenmian.huang@merico.dev
+2022-09-15 06:32:56.699,2022-09-15 06:32:56.699,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_pull_request_commits,26355,"",75ab753225b5b8acf3bc6e40e463b54b6800e7ed,0,0,0,"""feat: add new plugin""",zhangliang,liang.zhang@merico.dev,2022-09-02 02:33:39,liang.zhang@merico.dev,zhangliang,liang.zhang@merico.dev,2022-09-08 14:39:08,liang.zhang@merico.dev
+2022-09-15 06:32:56.699,2022-09-15 06:32:56.699,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_pull_request_commits,22141,"",7e19d411a2b7d95ebaf1c9d1303ac71ca18ea6fb,0,0,0,"""fix: linting strike 10""",Klesh Wong,zhenmian.huang@merico.dev,2022-09-09 16:00:38,zhenmian.huang@merico.dev,Klesh Wong,zhenmian.huang@merico.dev,2022-09-09 16:00:38,zhenmian.huang@merico.dev
+2022-09-15 06:32:56.699,2022-09-15 06:32:56.699,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_pull_request_commits,22132,"",80c93f3ca26ae42567b7c120add343e9b08edf3d,0,0,0,"""fix: icla api unused""",Klesh Wong,zhenmian.huang@merico.dev,2022-09-09 07:51:29,zhenmian.huang@merico.dev,Klesh Wong,zhenmian.huang@merico.dev,2022-09-09 07:51:29,zhenmian.huang@merico.dev
+2022-09-15 06:32:56.699,2022-09-15 06:32:56.699,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_pull_request_commits,26493,"",830edb6b37a4820d479e72532e99854a2883c295,0,0,0,"""fix: add path to stages collect\n\nAdd path to stages collect.\n\nNddtfjiang <zh...@merico.dev>""",Nddtfjiang,jzc001@qq.com,2022-09-09 09:02:15,jzc001@qq.com,Nddtfjiang,jzc001@qq.com,2022-09-09 09:02:15,jzc001@qq.com
+2022-09-15 06:32:57.195,2022-09-15 06:32:57.195,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_pull_request_commits,22134,"",8833329138f6a568a2d28f65f7435b4055623434,0,0,0,"""fix: linting problem iter 3""",Klesh Wong,zhenmian.huang@merico.dev,2022-09-09 09:06:10,zhenmian.huang@merico.dev,Klesh Wong,zhenmian.huang@merico.dev,2022-09-09 09:06:10,zhenmian.huang@merico.dev
+2022-09-15 06:32:57.195,2022-09-15 06:32:57.195,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_pull_request_commits,26525,"",9d53fb594958e65456793caa1bfa8d07a7614291,0,0,0,"""fix(gitlab): update e2e\n\nrelate to #2871""",Yingchu Chen,yingchu.chen@merico.dev,2022-09-09 14:16:26,yingchu.chen@merico.dev,abeizn,zikuan.an@merico.dev,2022-09-09 15:16:06,zikuan.an@merico.dev
+2022-09-15 06:32:57.195,2022-09-15 06:32:57.195,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_pull_request_commits,22099,"",a072ed76b179ddff9ffc2acc0cef70be484d1b65,0,0,0,"""fix: update pipelineId""",abeizn,zikuan.an@merico.dev,2022-09-09 09:26:48,zikuan.an@merico.dev,abeizn,zikuan.an@merico.dev,2022-09-09 09:27:06,zikuan.an@merico.dev
+2022-09-15 06:32:57.640,2022-09-15 06:32:57.640,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_pull_request_commits,22137,"",a15b66f02350ae90a13f6eb11a4f7b051ddf5f5d,0,0,0,"""fix: linting strike 6""",Klesh Wong,zhenmian.huang@merico.dev,2022-09-09 13:39:40,zhenmian.huang@merico.dev,Klesh Wong,zhenmian.huang@merico.dev,2022-09-09 13:39:40,zhenmian.huang@merico.dev
+2022-09-15 06:32:57.640,2022-09-15 06:32:57.640,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_pull_request_commits,26361,"",a57007bfeda7b6a15f2ce7901422ca6c8cdc8ede,0,0,0,"""refactor: switch to new error type""",zhangliang,liang.zhang@merico.dev,2022-09-08 13:40:34,liang.zhang@merico.dev,zhangliang,liang.zhang@merico.dev,2022-09-08 14:39:08,liang.zhang@merico.dev
+2022-09-15 06:32:58.098,2022-09-15 06:32:58.098,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_pull_request_commits,22142,"",bc8364e2bef891ae870a930130cc097d5a17a82a,0,0,0,"""fix: linting strike 11""",Klesh Wong,zhenmian.huang@merico.dev,2022-09-09 16:14:26,zhenmian.huang@merico.dev,Klesh Wong,zhenmian.huang@merico.dev,2022-09-09 16:14:26,zhenmian.huang@merico.dev
+2022-09-15 06:32:58.098,2022-09-15 06:32:58.098,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_pull_request_commits,22098,"",bf9bb07ba5ff7887622abe0455c6f9b884bde683,0,0,0,"""fix: update pipelineId""",abeizn,zikuan.an@merico.dev,2022-09-09 08:29:21,zikuan.an@merico.dev,abeizn,zikuan.an@merico.dev,2022-09-09 08:32:54,zikuan.an@merico.dev
+2022-09-15 06:32:58.098,2022-09-15 06:32:58.098,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_pull_request_commits,22155,"",c3361eb9f43e5f8b6a9f37ff507c313ca3194bf2,0,0,0,"""feat: new plugin for gitea""",tk103331,tk103331@gmail.com,2022-09-07 05:24:07,tk103331@gmail.com,tk,tk103331@gmail.com,2022-09-09 08:08:38,tk103331@gmail.com
+2022-09-15 06:32:58.098,2022-09-15 06:32:58.098,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_pull_request_commits,22133,"",cfa256c80d8ab3e085b45ca66e215bf0e41479cb,0,0,0,"""fix: rest of linting errors""",Klesh Wong,zhenmian.huang@merico.dev,2022-09-09 08:22:20,zhenmian.huang@merico.dev,Klesh Wong,zhenmian.huang@merico.dev,2022-09-09 08:22:20,zhenmian.huang@merico.dev
+2022-09-15 06:32:58.098,2022-09-15 06:32:58.098,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_pull_request_commits,22143,"",d443da3e686a429661193c0b516f9569ed515d36,0,0,0,"""Merge remote-tracking branch 'apache/main' into apache""",Klesh Wong,zhenmian.huang@merico.dev,2022-09-09 16:20:33,zhenmian.huang@merico.dev,Klesh Wong,zhenmian.huang@merico.dev,2022-09-09 16:20:33,zhenmian.huang@merico.dev
+2022-09-15 06:32:58.598,2022-09-15 06:32:58.598,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_pull_request_commits,22156,"",d715995e019be9ad12c5fed99140303afa967e28,0,0,0,"""Merge branch 'feat-plugin-gitea' into gitea-plugin""",Warren Chen,yingchu.chen@merico.dev,2022-09-10 02:35:37,yingchu.chen@merico.dev,GitHub,noreply@github.com,2022-09-10 02:35:37,noreply@github.com
+2022-09-15 06:32:58.598,2022-09-15 06:32:58.598,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_pull_request_commits,26357,"",eb8c38c0ac52131fb893573d457b9a9859323403,0,0,0,"""refactor: new task Options""",zhangliang,liang.zhang@merico.dev,2022-09-07 00:29:30,liang.zhang@merico.dev,zhangliang,liang.zhang@merico.dev,2022-09-08 14:39:08,liang.zhang@merico.dev
+2022-09-15 06:32:58.598,2022-09-15 06:32:58.598,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_pull_request_commits,26358,"",effcab4c6aa6340d8797ac2a8cd1eaeab08fced3,0,0,0,"""fix: update swagger docs""",zhangliang,liang.zhang@merico.dev,2022-09-07 00:33:12,liang.zhang@merico.dev,zhangliang,liang.zhang@merico.dev,2022-09-08 14:39:08,liang.zhang@merico.dev
+2022-09-15 06:32:59.100,2022-09-15 06:32:59.100,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_pull_request_commits,22138,"",f3cdf3814eabbb395c07674352297138a62df00d,0,0,0,"""fix: linting strike 7""",Klesh Wong,zhenmian.huang@merico.dev,2022-09-09 13:49:49,zhenmian.huang@merico.dev,Klesh Wong,zhenmian.huang@merico.dev,2022-09-09 13:49:49,zhenmian.huang@merico.dev
diff --git a/plugins/dora/e2e/raw_tables/issues.csv b/plugins/dora/e2e/raw_tables/issues.csv
new file mode 100644
index 00000000..62635c57
--- /dev/null
+++ b/plugins/dora/e2e/raw_tables/issues.csv
@@ -0,0 +1,15 @@
+id,created_at,updated_at,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark,url,icon_url,issue_key,title,description,epic_key,type,status,original_status,story_point,resolution_date,created_date,updated_date,lead_time_minutes,parent_issue_id,priority,original_estimate_minutes,time_spent_minutes,time_remaining_minutes,creator_id,creator_name,assignee_id,assignee_name,severity,component,deployment_id
+github:GithubIssue:1:1367714738,2022-09-15 05:36:36.048,2022-09-15 05:36:36.048,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_issues,3370,"",https://github.com/apache/incubator-devlake/issues/3036,"",3036,[Bug][GitHub Action] The workflow `check-Apache-license-header` doesn't work,"""### Search before asking\n\n- [X] I had searched in the [issues](https://github.com/apache/incubator-devlake/issues?q=is%3Aissue) and found no similar issues.\n\n [...]
+github:GithubIssue:1:1370813347,2022-09-15 05:36:36.048,2022-09-15 05:36:36.048,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_issues,3388,"",https://github.com/apache/incubator-devlake/issues/3054,"",3054,[Bug][github] collect apache/incubator-devlake fail,"""### Search before asking\n\n- [X] I had searched in the [issues](https://github.com/apache/incubator-devlake/issues?q=is%3Aissue) and found no similar issues.\n\n\n### What happened\n\n`` [...]
+github:GithubIssue:1:1370816458,2022-09-15 05:36:36.048,2022-09-15 05:36:36.048,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_issues,3389,"",https://github.com/apache/incubator-devlake/issues/3055,"",3055,[Bug][config-ui] cannot create my plugin by advance mode in config-ui,"""### Search before asking\n\n- [X] I had searched in the [issues](https://github.com/apache/incubator-devlake/issues?q=is%3Aissue) and found no similar issues.\n\n\n### W [...]
+github:GithubIssue:1:1370833897,2022-09-15 05:36:36.048,2022-09-15 05:36:36.048,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_issues,3390,"",https://github.com/apache/incubator-devlake/issues/3056,"",3056,[Bug][Framework] github/gitlab/gitee issues do not have board_repo record,"""### Search before asking\n\n- [X] I had searched in the [issues](https://github.com/apache/incubator-devlake/issues?q=is%3Aissue) and found no similar issues.\n\n\n [...]
+github:GithubIssue:1:1371315795,2022-09-15 05:36:36.048,2022-09-15 05:36:36.048,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_issues,3392,"",https://github.com/apache/incubator-devlake/issues/3058,"",3058,[Feature][Bitbucket] add Pipeline collector & extractor & convertor,"""### Search before asking\n\n- [X] I had searched in the [issues](https://github.com/apache/incubator-devlake/issues?q=is%3Aissue) and found no similar feature requirement. [...]
+github:GithubIssue:1:1371320153,2022-09-15 05:36:36.048,2022-09-15 05:36:36.048,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_issues,3393,"",https://github.com/apache/incubator-devlake/issues/3059,"",3059,[Bug][customize] e2e test failed in postgres,"""### Search before asking\n\n- [X] I had searched in the [issues](https://github.com/apache/incubator-devlake/issues?q=is%3Aissue) and found no similar issues.\n\n\n### What happened\n\ncached pl [...]
+github:GithubIssue:1:1371541422,2022-09-15 05:36:36.048,2022-09-15 05:36:36.048,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_issues,3394,"",https://github.com/apache/incubator-devlake/issues/3060,"",3060,[Bug][Migrationscripts] use common and domainlayer for migrationscripts table model,"""### Search before asking\r\n\r\n- [X] I had searched in the [issues](https://github.com/apache/incubator-devlake/issues?q=is%3Aissue) and found no similar [...]
+github:GithubIssue:1:1371617668,2022-09-15 05:36:36.048,2022-09-15 05:36:36.048,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_issues,3395,"",https://github.com/apache/incubator-devlake/issues/3061,"",3061,[Feature][api-client] wait more time to prevent long-time collection canceled by network problem,"""### Search before asking\n\n- [X] I had searched in the [issues](https://github.com/apache/incubator-devlake/issues?q=is%3Aissue) and found no [...]
+github:GithubIssue:1:1372381019,2022-09-15 05:36:36.048,2022-09-15 05:36:36.048,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_issues,3397,"",https://github.com/apache/incubator-devlake/issues/3063,"",3063,[Bug][Bitbucket] the other subtasks cannot continue when the number of issues is zero.,"""### Search before asking\r\n\r\n- [X] I had searched in the [issues](https://github.com/apache/incubator-devlake/issues?q=is%3Aissue) and found no simil [...]
+github:GithubIssue:1:1372550825,2022-09-15 05:36:36.048,2022-09-15 05:36:36.048,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_issues,3399,"",https://github.com/apache/incubator-devlake/issues/3065,"",3065,[Bug][tapd] error occurred while adding connection for plugin tapd ,"""### Search before asking\n\n- [X] I had searched in the [issues](https://github.com/apache/incubator-devlake/issues?q=is%3Aissue) and found no similar issues.\n\n\n### Wha [...]
+github:GithubIssue:1:1372581663,2022-09-15 05:36:36.048,2022-09-15 05:36:36.048,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_issues,3401,"",https://github.com/apache/incubator-devlake/issues/3067,"",3067,[Bug][Framework] error getting pipelines,"""### Search before asking\r\n\r\n- [X] I had searched in the [issues](https://github.com/apache/incubator-devlake/issues?q=is%3Aissue) and found no similar issues.\r\n\r\n\r\n### What happened\r\n\r\ [...]
+github:GithubIssue:1:1372644519,2022-09-15 05:36:36.048,2022-09-15 05:36:36.048,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_issues,3402,"",https://github.com/apache/incubator-devlake/issues/3068,"",3068,[Feature][Backend] Establish a standard error logging convention,"""### Search before asking\r\n\r\n- [X] I had searched in the [issues](https://github.com/apache/incubator-devlake/issues?q=is%3Aissue) and found no similar feature requirement [...]
+github:GithubIssue:1:1373079324,2022-09-15 05:36:36.048,2022-09-15 05:36:36.048,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_issues,3406,"",https://github.com/apache/incubator-devlake/issues/3072,"",3072,[Bug][framework] status code 500 returned from the server,"""### Search before asking\n\n- [X] I had searched in the [issues](https://github.com/apache/incubator-devlake/issues?q=is%3Aissue) and found no similar issues.\n\n\n### What happened [...]
+github:GithubIssue:1:1373792477,2022-09-15 05:36:36.048,2022-09-15 05:36:36.048,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_issues,3407,"",https://github.com/apache/incubator-devlake/issues/3073,"",3073,[Bug][github] calculateSince retrun the empty timespace,"""### Search before asking\r\n\r\n- [X] I had searched in the [issues](https://github.com/apache/incubator-devlake/issues?q=is%3Aissue) and found no similar issues.\r\n\r\n\r\n### What [...]
diff --git a/plugins/dora/e2e/raw_tables/lake_cicd_tasks_for_other_jobs.csv b/plugins/dora/e2e/raw_tables/lake_cicd_tasks_for_other_jobs.csv
new file mode 100644
index 00000000..2d68eec8
--- /dev/null
+++ b/plugins/dora/e2e/raw_tables/lake_cicd_tasks_for_other_jobs.csv
@@ -0,0 +1,22 @@
+id,name,pipeline_id,result,status,type,environment,duration_sec,started_date,finished_date
+github:GithubJob:1:8269045794,lint,github:GithubRun:1:3022052551,FAILURE,DONE,,,72,2022-09-09T11:27:13.000+00:00,2022-09-09T11:28:25.000+00:00
+github:GithubJob:1:8269045795,test (ubuntu-latest),github:GithubRun:1:3022052557,SUCCESS,DONE,,,191,2022-09-09T11:27:42.000+00:00,2022-09-09T11:30:53.000+00:00
+github:GithubJob:1:8269045819,commit-msg,github:GithubRun:1:3022052555,SUCCESS,DONE,,,8,2022-09-09T11:26:38.000+00:00,2022-09-09T11:26:46.000+00:00
+github:GithubJob:1:8269045928,e2e-mysql,github:GithubRun:1:3022052558,SUCCESS,DONE,,,200,2022-09-09T11:28:00.000+00:00,2022-09-09T11:31:20.000+00:00
+github:GithubJob:1:8269046116,check Apache license header,github:GithubRun:1:3022052589,SUCCESS,DONE,,,57,2022-09-09T11:28:34.000+00:00,2022-09-09T11:29:31.000+00:00
+github:GithubJob:1:8269415129,lint,github:GithubRun:1:3022190519,FAILURE,DONE,,,84,2022-09-09T11:51:21.000+00:00,2022-09-09T11:52:45.000+00:00
+github:GithubJob:1:8269415130,test (ubuntu-latest),github:GithubRun:1:3022190524,SUCCESS,DONE,,,182,2022-09-09T11:50:31.000+00:00,2022-09-09T11:53:33.000+00:00
+github:GithubJob:1:8269415142,check Apache license header,github:GithubRun:1:3022190525,SUCCESS,DONE,,,63,2022-09-09T11:52:06.000+00:00,2022-09-09T11:53:09.000+00:00
+github:GithubJob:1:8269415165,commit-msg,github:GithubRun:1:3022190521,SUCCESS,DONE,,,10,2022-09-09T11:52:46.000+00:00,2022-09-09T11:52:56.000+00:00
+github:GithubJob:1:8269415315,e2e-mysql,github:GithubRun:1:3022190520,SUCCESS,DONE,,,242,2022-09-09T11:53:45.000+00:00,2022-09-09T11:57:47.000+00:00
+github:GithubJob:1:8269598312,e2e-mysql,github:GithubRun:1:3022255519,SUCCESS,DONE,,,828,2022-09-09T12:05:04.000+00:00,2022-09-09T12:18:52.000+00:00
+github:GithubJob:1:8269598316,test (ubuntu-latest),github:GithubRun:1:3022255511,SUCCESS,DONE,,,1016,2022-09-09T12:05:49.000+00:00,2022-09-09T12:22:45.000+00:00
+github:GithubJob:1:8269598318,lint,github:GithubRun:1:3022255516,FAILURE,DONE,,,62,2022-09-09T12:02:28.000+00:00,2022-09-09T12:03:30.000+00:00
+github:GithubJob:1:8269598325,check Apache license header,github:GithubRun:1:3022255517,SUCCESS,DONE,,,52,2022-09-09T12:03:51.000+00:00,2022-09-09T12:04:43.000+00:00
+github:GithubJob:1:8269598469,commit-msg,github:GithubRun:1:3022255524,FAILURE,DONE,,,41,2022-09-09T12:06:55.000+00:00,2022-09-09T12:07:36.000+00:00
+github:GithubJob:1:8269729324,e2e-mysql,github:GithubRun:1:3022300995,SUCCESS,DONE,,,1007,2022-09-09T12:10:08.000+00:00,2022-09-09T12:26:55.000+00:00
+github:GithubJob:1:8269729327,xx_deploy_xx,github:GithubRun:1:3022301001,FAILURE,DONE,DEPLOY,staging,82,2022-09-09T12:11:22.000+00:00,2022-09-09T12:12:44.000+00:00
+github:GithubJob:1:8269729550,build_deployment,github:GithubRun:1:3022301007,SUCCESS,DONE,DEPLOY,staging,6,2022-09-09T12:12:02.000+00:00,2022-09-09T12:12:08.000+00:00
+github:GithubJob:1:8269729553,build_deploy,github:GithubRun:1:3022300997,SUCCESS,DONE,DEPLOY,staging,57,2022-09-09T12:12:52.000+00:00,2022-09-09T12:13:49.000+00:00
+github:GithubJob:1:8269729821,deployment,github:GithubRun:1:3022301033,SUCCESS,DONE,DEPLOY,staging,991,2022-09-09T12:13:58.000+00:00,2022-09-09T12:30:29.000+00:00
+github:GithubJob:1:8269872730,deploy,github:GithubRun:1:3022348786,SUCCESS,DONE,DEPLOY,staging,1006,2022-09-09T12:21:30.000+00:00,2022-09-09T12:38:16.000+00:00
diff --git a/plugins/dora/e2e/raw_tables/pull_request_comments.csv b/plugins/dora/e2e/raw_tables/pull_request_comments.csv
new file mode 100644
index 00000000..2a0f7553
--- /dev/null
+++ b/plugins/dora/e2e/raw_tables/pull_request_comments.csv
@@ -0,0 +1,25 @@
+id,created_at,updated_at,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark,pull_request_id,body,account_id,created_date,commit_sha,position,type,review_id,status
+github:GithubPrComment:1:964527893,2022-09-15 05:36:44.864,2022-09-15 05:36:44.864,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_pull_request_review_comments,695,"",github:GithubPullRequest:1:1043463302,"""all `error`s should be replaced by `errors.Error` our customized error by @keon94 """,github:GithubAccount:1:61080,2022-09-07 08:13:32,a57007bfeda7b6a15f2ce7901422ca6c8cdc8ede,0,DIFF,github:GithubPrReview:1:1098724785,""
+github:GithubPrComment:1:964664201,2022-09-15 05:36:44.864,2022-09-15 05:36:44.864,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_pull_request_review_comments,696,"",github:GithubPullRequest:1:1043463302,"""Can we delegate the implementation(sql) through the `Dal` interface?""",github:GithubAccount:1:61080,2022-09-07 10:24:10,a57007bfeda7b6a15f2ce7901422ca6c8cdc8ede,0,DIFF,github:GithubPrReview:1:1098724785,""
+github:GithubPrComment:1:964664296,2022-09-15 05:36:44.864,2022-09-15 05:36:44.864,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_pull_request_review_comments,697,"",github:GithubPullRequest:1:1043463302,"""Same as above""",github:GithubAccount:1:61080,2022-09-07 10:24:16,a57007bfeda7b6a15f2ce7901422ca6c8cdc8ede,0,DIFF,github:GithubPrReview:1:1098724785,""
+github:GithubPrReview:1:1098724785,2022-09-15 05:36:41.034,2022-09-15 05:36:41.034,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_pull_request_reviews,1874,"",github:GithubPullRequest:1:1043463302,"",github:GithubAccount:1:61080,2022-09-07 12:08:57,94ddd5147b3f658698ba4762a8a59cfd873b589a,0,REVIEW,"",COMMENTED
+github:GithubPrReview:1:1100879911,2022-09-15 05:36:41.034,2022-09-15 05:36:41.034,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_pull_request_reviews,1875,"",github:GithubPullRequest:1:1043463302,LGTM,github:GithubAccount:1:61080,2022-09-08 14:42:40,a57007bfeda7b6a15f2ce7901422ca6c8cdc8ede,0,REVIEW,"",APPROVED
+github:GithubPrComment:1:1239007576,2022-09-15 05:36:44.864,2022-09-15 05:36:44.864,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_comments,7218,"",github:GithubPullRequest:1:1048233599,"""Amazing work, I will review it later, thanks!""",github:GithubAccount:1:39366025,2022-09-07 07:22:26,"",0,NORMAL,github:GithubPrReview:1:0,""
+github:GithubPrComment:1:1241206605,2022-09-15 05:36:44.864,2022-09-15 05:36:44.864,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_comments,7083,"",github:GithubPullRequest:1:1048233599,"""Let's hold this off until I merge #2907. We will be enforcing errors.Error to be the return type of all functions in Devlake soon.""",github:GithubAccount:1:25063936,2022-09-08 20:35:11,"",0,NORMAL,github:GithubPrReview:1:0,""
+github:GithubPrComment:1:967748714,2022-09-15 05:36:44.864,2022-09-15 05:36:44.864,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_pull_request_review_comments,401,"",github:GithubPullRequest:1:1048233599,"""We should add a func: GetTotalPages""",github:GithubAccount:1:39366025,2022-09-11 03:18:14,d715995e019be9ad12c5fed99140303afa967e28,0,DIFF,github:GithubPrReview:1:1103228459,""
+github:GithubPrReview:1:1103228459,2022-09-15 05:36:41.034,2022-09-15 05:36:41.034,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_pull_request_reviews,2400,"",github:GithubPullRequest:1:1048233599,"",github:GithubAccount:1:39366025,2022-09-11 03:18:14,d715995e019be9ad12c5fed99140303afa967e28,0,REVIEW,"",COMMENTED
+github:GithubPrComment:1:1240012942,2022-09-15 05:36:44.864,2022-09-15 05:36:44.864,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_comments,7235,"",github:GithubPullRequest:1:1049191985,"""The linter errors must be due to the Linter version upgrade.""",github:GithubAccount:1:25063936,2022-09-07 23:21:01,"",0,NORMAL,github:GithubPrReview:1:0,""
+github:GithubPrComment:1:1240216270,2022-09-15 05:36:44.864,2022-09-15 05:36:44.864,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_comments,7243,"",github:GithubPullRequest:1:1049191985,"""@klesh this is ready for review""",github:GithubAccount:1:25063936,2022-09-08 04:50:04,"",0,NORMAL,github:GithubPrReview:1:0,""
+github:GithubPrComment:1:965369774,2022-09-15 05:36:44.864,2022-09-15 05:36:44.864,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_pull_request_review_comments,705,"",github:GithubPullRequest:1:1049191985,"""1.43 won't work with Go 1.18+, so had to upgrade""",github:GithubAccount:1:25063936,2022-09-07 23:07:13,0edd5d0e0d18516d3309fec90553c3ee7380c149,0,DIFF,github:GithubPrReview:1:1099918590,""
+github:GithubPrComment:1:965369998,2022-09-15 05:36:44.864,2022-09-15 05:36:44.864,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_pull_request_review_comments,706,"",github:GithubPullRequest:1:1049191985,"""safer, more robust solution""",github:GithubAccount:1:25063936,2022-09-07 23:07:42,0edd5d0e0d18516d3309fec90553c3ee7380c149,0,DIFF,github:GithubPrReview:1:1099918866,""
+github:GithubPrReview:1:1099918590,2022-09-15 05:36:41.034,2022-09-15 05:36:41.034,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_pull_request_reviews,2119,"",github:GithubPullRequest:1:1049191985,"",github:GithubAccount:1:25063936,2022-09-07 23:07:13,9298d2c1bedd4b6f00a1af33cad962b065e394c9,0,REVIEW,"",COMMENTED
+github:GithubPrReview:1:1099918866,2022-09-15 05:36:41.034,2022-09-15 05:36:41.034,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_pull_request_reviews,2120,"",github:GithubPullRequest:1:1049191985,"",github:GithubAccount:1:25063936,2022-09-07 23:07:42,9298d2c1bedd4b6f00a1af33cad962b065e394c9,0,REVIEW,"",COMMENTED
+github:GithubPrReview:1:1101641680,2022-09-15 05:36:41.034,2022-09-15 05:36:41.034,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_pull_request_reviews,2121,"",github:GithubPullRequest:1:1049191985,LGTM,github:GithubAccount:1:8455907,2022-09-09 03:04:06,87363e95c5430562ada30b060bb95e82dceb3705,0,REVIEW,"",APPROVED
+github:GithubPrComment:1:1241698230,2022-09-15 05:36:44.864,2022-09-15 05:36:44.864,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_comments,7100,"",github:GithubPullRequest:1:1051243958,"""Lol looks like we opened the same PR at almost the same time. Mine is #3033""",github:GithubAccount:1:25063936,2022-09-09 08:56:34,"",0,NORMAL,github:GithubPrReview:1:0,""
+github:GithubPrReview:1:1102643952,2022-09-15 05:36:41.034,2022-09-15 05:36:41.034,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_pull_request_reviews,2099,"",github:GithubPullRequest:1:1051243958,LGTM,github:GithubAccount:1:2908155,2022-09-09 17:09:23,62535543802631a0d3daf0b0b78c6a7e05e508fb,0,REVIEW,"",APPROVED
+github:GithubPrComment:1:966838137,2022-09-15 05:36:44.864,2022-09-15 05:36:44.864,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_pull_request_review_comments,395,"",github:GithubPullRequest:1:1051273681,"""please ignore repoId""",github:GithubAccount:1:39366025,2022-09-09 09:34:01,209168a4b12d91a7ad1f3711fe05b39d5bd8e260,0,DIFF,github:GithubPrReview:1:1102001107,""
+github:GithubPrReview:1:1102001107,2022-09-15 05:36:41.034,2022-09-15 05:36:41.034,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_pull_request_reviews,1018,"",github:GithubPullRequest:1:1051273681,"",github:GithubAccount:1:39366025,2022-09-09 09:34:02,a072ed76b179ddff9ffc2acc0cef70be484d1b65,0,REVIEW,"",COMMENTED
+github:GithubPrReview:1:1102399344,2022-09-15 05:36:41.034,2022-09-15 05:36:41.034,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_pull_request_reviews,1019,"",github:GithubPullRequest:1:1051273681,LGTM,github:GithubAccount:1:39366025,2022-09-09 14:19:05,209168a4b12d91a7ad1f3711fe05b39d5bd8e260,0,REVIEW,"",APPROVED
+github:GithubPrReview:1:1102452792,2022-09-15 05:36:41.034,2022-09-15 05:36:41.034,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_pull_request_reviews,2104,"",github:GithubPullRequest:1:1051340471,LGTM,github:GithubAccount:1:39366025,2022-09-09 14:47:47,08d2f2b6de0fa8de4d0e2b55b4b9a2e244214029,0,REVIEW,"",APPROVED
+github:GithubPrReview:1:1102247212,2022-09-15 05:36:41.034,2022-09-15 05:36:41.034,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_pull_request_reviews,2109,"",github:GithubPullRequest:1:1051524882,LGTM,github:GithubAccount:1:101256042,2022-09-09 12:40:20,56b895f0443730c6d7abfbc51a05ab35abd2971f,0,REVIEW,"",APPROVED
+github:GithubPrReview:1:1102479199,2022-09-15 05:36:41.034,2022-09-15 05:36:41.034,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_pull_request_reviews,2118,"",github:GithubPullRequest:1:1051637383,"",github:GithubAccount:1:2921251,2022-09-09 15:03:13,466e9a0a7c3c5c45a0ca218c98705563f77c4aa4,0,REVIEW,"",APPROVED
diff --git a/plugins/dora/e2e/raw_tables/pull_request_commits.csv b/plugins/dora/e2e/raw_tables/pull_request_commits.csv
new file mode 100644
index 00000000..e2760384
--- /dev/null
+++ b/plugins/dora/e2e/raw_tables/pull_request_commits.csv
@@ -0,0 +1,34 @@
+commit_sha,pull_request_id,created_at,updated_at,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
+08d2f2b6de0fa8de4d0e2b55b4b9a2e244214029,github:GithubPullRequest:1:1051340471,2022-09-15 05:36:38.768,2022-09-15 05:36:38.768,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_pull_request_commits,26494,""
+0edd5d0e0d18516d3309fec90553c3ee7380c149,github:GithubPullRequest:1:1049191985,2022-09-15 05:36:38.768,2022-09-15 05:36:38.768,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_pull_request_commits,22153,""
+209168a4b12d91a7ad1f3711fe05b39d5bd8e260,github:GithubPullRequest:1:1051273681,2022-09-15 05:36:38.768,2022-09-15 05:36:38.768,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_pull_request_commits,22100,""
+2537845559d8db99e9cda6190f32b50ec979c722,github:GithubPullRequest:1:1051243958,2022-09-15 05:36:38.768,2022-09-15 05:36:38.768,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_pull_request_commits,22139,""
+3aba559962e04949a1731f7d5155c7aac2e900ce,github:GithubPullRequest:1:1051243958,2022-09-15 05:36:38.768,2022-09-15 05:36:38.768,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_pull_request_commits,22140,""
+3ec3e113ed36a5a8978af295683085054e4dfdd9,github:GithubPullRequest:1:1051243958,2022-09-15 05:36:38.768,2022-09-15 05:36:38.768,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_pull_request_commits,22144,""
+4b71faf666833c0c7b915a512811e2c5e746d3de,github:GithubPullRequest:1:1049191985,2022-09-15 05:36:38.768,2022-09-15 05:36:38.768,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_pull_request_commits,22152,""
+4f8cdefc9a9d53af16dd482c61623312eb9e9b5e,github:GithubPullRequest:1:1048233599,2022-09-15 05:36:38.768,2022-09-15 05:36:38.768,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_pull_request_commits,22154,""
+55f445997abbd5918da59d202d28762cd56fbd44,github:GithubPullRequest:1:1043463302,2022-09-15 05:36:38.768,2022-09-15 05:36:38.768,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_pull_request_commits,26356,""
+56b895f0443730c6d7abfbc51a05ab35abd2971f,github:GithubPullRequest:1:1051524882,2022-09-15 05:36:38.768,2022-09-15 05:36:38.768,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_pull_request_commits,26519,""
+5ad0c09c447c19338f1dfbb65d89a3728962b3b7,github:GithubPullRequest:1:1051243958,2022-09-15 05:36:38.768,2022-09-15 05:36:38.768,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_pull_request_commits,22136,""
+62535543802631a0d3daf0b0b78c6a7e05e508fb,github:GithubPullRequest:1:1051243958,2022-09-15 05:36:38.768,2022-09-15 05:36:38.768,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_pull_request_commits,22145,""
+6559c5691ef290d2ff739d15da27e5e95994293d,github:GithubPullRequest:1:1043463302,2022-09-15 05:36:38.768,2022-09-15 05:36:38.768,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_pull_request_commits,26360,""
+6a5edde13a90eeeef42e73a87974588443d5f9ca,github:GithubPullRequest:1:1043463302,2022-09-15 05:36:38.768,2022-09-15 05:36:38.768,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_pull_request_commits,26359,""
+74cc484551329c7f92dc03519398dba62f96f128,github:GithubPullRequest:1:1051243958,2022-09-15 05:36:38.768,2022-09-15 05:36:38.768,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_pull_request_commits,22135,""
+75ab753225b5b8acf3bc6e40e463b54b6800e7ed,github:GithubPullRequest:1:1043463302,2022-09-15 05:36:38.768,2022-09-15 05:36:38.768,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_pull_request_commits,26355,""
+7e19d411a2b7d95ebaf1c9d1303ac71ca18ea6fb,github:GithubPullRequest:1:1051243958,2022-09-15 05:36:38.768,2022-09-15 05:36:38.768,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_pull_request_commits,22141,""
+80c93f3ca26ae42567b7c120add343e9b08edf3d,github:GithubPullRequest:1:1051243958,2022-09-15 05:36:38.768,2022-09-15 05:36:38.768,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_pull_request_commits,22132,""
+830edb6b37a4820d479e72532e99854a2883c295,github:GithubPullRequest:1:1051340471,2022-09-15 05:36:38.768,2022-09-15 05:36:38.768,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_pull_request_commits,26493,""
+8833329138f6a568a2d28f65f7435b4055623434,github:GithubPullRequest:1:1051243958,2022-09-15 05:36:38.768,2022-09-15 05:36:38.768,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_pull_request_commits,22134,""
+9d53fb594958e65456793caa1bfa8d07a7614291,github:GithubPullRequest:1:1051637383,2022-09-15 05:36:38.768,2022-09-15 05:36:38.768,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_pull_request_commits,26525,""
+a072ed76b179ddff9ffc2acc0cef70be484d1b65,github:GithubPullRequest:1:1051273681,2022-09-15 05:36:38.768,2022-09-15 05:36:38.768,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_pull_request_commits,22099,""
+a15b66f02350ae90a13f6eb11a4f7b051ddf5f5d,github:GithubPullRequest:1:1051243958,2022-09-15 05:36:38.768,2022-09-15 05:36:38.768,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_pull_request_commits,22137,""
+a57007bfeda7b6a15f2ce7901422ca6c8cdc8ede,github:GithubPullRequest:1:1043463302,2022-09-15 05:36:38.768,2022-09-15 05:36:38.768,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_pull_request_commits,26361,""
+bc8364e2bef891ae870a930130cc097d5a17a82a,github:GithubPullRequest:1:1051243958,2022-09-15 05:36:38.768,2022-09-15 05:36:38.768,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_pull_request_commits,22142,""
+bf9bb07ba5ff7887622abe0455c6f9b884bde683,github:GithubPullRequest:1:1051273681,2022-09-15 05:36:38.768,2022-09-15 05:36:38.768,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_pull_request_commits,22098,""
+c3361eb9f43e5f8b6a9f37ff507c313ca3194bf2,github:GithubPullRequest:1:1048233599,2022-09-15 05:36:38.768,2022-09-15 05:36:38.768,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_pull_request_commits,22155,""
+cfa256c80d8ab3e085b45ca66e215bf0e41479cb,github:GithubPullRequest:1:1051243958,2022-09-15 05:36:38.768,2022-09-15 05:36:38.768,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_pull_request_commits,22133,""
+d443da3e686a429661193c0b516f9569ed515d36,github:GithubPullRequest:1:1051243958,2022-09-15 05:36:38.768,2022-09-15 05:36:38.768,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_pull_request_commits,22143,""
+d715995e019be9ad12c5fed99140303afa967e28,github:GithubPullRequest:1:1048233599,2022-09-15 05:36:38.768,2022-09-15 05:36:38.768,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_pull_request_commits,22156,""
+eb8c38c0ac52131fb893573d457b9a9859323403,github:GithubPullRequest:1:1043463302,2022-09-15 05:36:38.768,2022-09-15 05:36:38.768,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_pull_request_commits,26357,""
+effcab4c6aa6340d8797ac2a8cd1eaeab08fced3,github:GithubPullRequest:1:1043463302,2022-09-15 05:36:38.768,2022-09-15 05:36:38.768,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_pull_request_commits,26358,""
+f3cdf3814eabbb395c07674352297138a62df00d,github:GithubPullRequest:1:1051243958,2022-09-15 05:36:38.768,2022-09-15 05:36:38.768,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_pull_request_commits,22138,""
diff --git a/plugins/dora/e2e/raw_tables/pull_requests.csv b/plugins/dora/e2e/raw_tables/pull_requests.csv
new file mode 100644
index 00000000..b9c1a668
--- /dev/null
+++ b/plugins/dora/e2e/raw_tables/pull_requests.csv
@@ -0,0 +1,10 @@
+id,created_at,updated_at,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark,base_repo_id,head_repo_id,status,title,description,url,author_name,author_id,parent_pr_id,pull_request_key,created_date,merged_date,closed_date,type,component,merge_commit_sha,head_ref,base_ref,base_commit_sha,head_commit_sha,coding_timespan,review_lag,review_timespan,deploy_timespan,change_timespan,orig_coding_timespan,orig_review_lag,orig_review_timespan,orig_deploy_timespan
+github:GithubPullRequest:1:1043463302,2022-09-15 05:36:39.856,2022-09-15 05:36:39.856,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_pull_requests,13176,,github:GithubRepo:1:384111310,github:GithubRepo:1:384111310,closed,feat: implement API for plugin `customize`,"""# Summary\r\n\r\nfulfill the requirement \r\n#2880 ([Feature][customize] implement API for plugin customize) \r\n#2985 ([Feature][customize] new sub-task ExtractCustomizedFields for [...]
+github:GithubPullRequest:1:1048233599,2022-09-15 05:36:39.856,2022-09-15 05:36:39.856,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_pull_requests,13211,,github:GithubRepo:1:384111310,github:GithubRepo:1:384111310,closed,feat: new plugin for gitea,"""# Summary\r\n[Gitea](https://gitea.io/) is an open-source software package for hosting software development version control using Git as well as other collaborative features like bug tracking, code [...]
+github:GithubPullRequest:1:1049191985,2022-09-15 05:36:40.170,2022-09-15 05:36:40.170,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_pull_requests,12680,,github:GithubRepo:1:384111310,github:GithubRepo:1:384111310,closed,[issue-2908]: Bump lake-builder version to 0.0.8,"""### ⚠️ Pre Checklist\r\n\r\n> Please complete _ALL_ items in this checklist, and remove before submitting\r\n\r\n- [x] I have read through the [Contributing Documentation](htt [...]
+github:GithubPullRequest:1:1051243958,2022-09-15 05:36:40.170,2022-09-15 05:36:40.170,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_pull_requests,12691,,github:GithubRepo:1:384111310,github:GithubRepo:1:384111310,closed,fix: golangci-lint error,"""# Summary\r\n\r\ngolint-ci was down for a while, this PR try to fix all missed linting errors during the period.\r\n\r\n### Screenshots\r\n![image](https://user-images.githubusercontent.com/61080/189 [...]
+github:GithubPullRequest:1:1051273681,2022-09-15 05:36:40.170,2022-09-15 05:36:40.170,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_pull_requests,12693,,github:GithubRepo:1:384111310,github:GithubRepo:1:384111310,closed,fix: update pipelineId,"""# Summary\r\n\r\nupdate pipelineId\r\n\r\n### Does this close any open issues?\r\nrelated to #2998 \r\n\r\n### Screenshots\r\n![image](https://user-images.githubusercontent.com/101256042/189307562-88fc [...]
+github:GithubPullRequest:1:1051340471,2022-09-15 05:36:40.170,2022-09-15 05:36:40.170,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_pull_requests,12694,,github:GithubRepo:1:384111310,github:GithubRepo:1:384111310,closed,Jenkins fix,"""# Summary\r\nAdd TestJenkinsStagesDataFlow.\r\nAdd path to stages collect.\r\n\r\n<!--\r\nThanks for submitting a pull request!\r\n\r\nWe appreciate you spending the time to work on these changes.\r\nPlease fill [...]
+github:GithubPullRequest:1:1051524882,2022-09-15 05:36:40.170,2022-09-15 05:36:40.170,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_pull_requests,12696,,github:GithubRepo:1:384111310,github:GithubRepo:1:384111310,closed,fix: task failure caused by 404,"""# Summary\r\nfix #2960 [Bug][gitihub] collect account failed by not found user\r\nIn the case of `err == ErrIgnoreAndContinue`, the `err` should not be wrapped, because on the caller side, the [...]
+github:GithubPullRequest:1:1051574863,2022-09-15 05:36:40.170,2022-09-15 05:36:40.170,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_pull_requests,12697,,github:GithubRepo:1:384111310,github:GithubRepo:1:384111310,closed,fix(jenkins): update e2e,"""# Summary\r\n\r\nUpdate e2e according to recent changes\r\n\r\n### Does this close any open issues?\r\nrelate to #2854\r\n\r\n\r\n### Screenshots\r\nInclude any relevant screenshots here.\r\n\r\n### [...]
+github:GithubPullRequest:1:1051637383,2022-09-15 05:36:40.170,2022-09-15 05:36:40.170,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_pull_requests,12699,,github:GithubRepo:1:384111310,github:GithubRepo:1:384111310,closed,fix(gitlab): update e2e,"""# Summary\r\n\r\nupdate gitlab e2e according to recent changes\r\n\r\n### Does this close any open issues?\r\nrelate to #2871\r\n\r\n\r\n### Screenshots\r\nInclude any relevant screenshots here.\r\n\r [...]
diff --git a/plugins/dora/e2e/snapshot_tables/issues.csv b/plugins/dora/e2e/snapshot_tables/issues.csv
new file mode 100644
index 00000000..ba13b7fd
--- /dev/null
+++ b/plugins/dora/e2e/snapshot_tables/issues.csv
@@ -0,0 +1,15 @@
+id,url,icon_url,issue_key,title,description,epic_key,type,status,original_status,story_point,resolution_date,created_date,updated_date,lead_time_minutes,parent_issue_id,priority,original_estimate_minutes,time_spent_minutes,time_remaining_minutes,creator_id,creator_name,assignee_id,assignee_name,severity,component,deployment_id
+github:GithubIssue:1:1367714738,https://github.com/apache/incubator-devlake/issues/3036,,3036,[Bug][GitHub Action] The workflow `check-Apache-license-header` doesn't work,"""### Search before asking\n\n- [X] I had searched in the [issues](https://github.com/apache/incubator-devlake/issues?q=is%3Aissue) and found no similar issues.\n\n\n### What happened\n\nThe `skywalking-eyes` could not find the config file `.licenserc.yaml `.\r\nthe following files with missing ASF header were not dete [...]
+github:GithubIssue:1:1370813347,https://github.com/apache/incubator-devlake/issues/3054,,3054,[Bug][github] collect apache/incubator-devlake fail,"""### Search before asking\n\n- [X] I had searched in the [issues](https://github.com/apache/incubator-devlake/issues?q=is%3Aissue) and found no similar issues.\n\n\n### What happened\n\n```\r\ntime=\""2022-09-13 11:44:02\"" level=info msg=\"" [pipeline service] [pipeline #17] [task #28] [github] [extractApiIssues] get data from _raw_github_ap [...]
+github:GithubIssue:1:1370816458,https://github.com/apache/incubator-devlake/issues/3055,,3055,[Bug][config-ui] cannot create my plugin by advance mode in config-ui,"""### Search before asking\n\n- [X] I had searched in the [issues](https://github.com/apache/incubator-devlake/issues?q=is%3Aissue) and found no similar issues.\n\n\n### What happened\n\n\r\n\n\n### What you expected [...]
+github:GithubIssue:1:1370833897,https://github.com/apache/incubator-devlake/issues/3056,,3056,[Bug][Framework] github/gitlab/gitee issues do not have board_repo record,"""### Search before asking\n\n- [X] I had searched in the [issues](https://github.com/apache/incubator-devlake/issues?q=is%3Aissue) and found no similar issues.\n\n\n### What happened\n\nThere is no record in board_repos for github/gitlab/gitea/gitee issues\n\n### What you expected to happen\n\nsave record in board_repos [...]
+github:GithubIssue:1:1371315795,https://github.com/apache/incubator-devlake/issues/3058,,3058,[Feature][Bitbucket] add Pipeline collector & extractor & convertor,"""### Search before asking\n\n- [X] I had searched in the [issues](https://github.com/apache/incubator-devlake/issues?q=is%3Aissue) and found no similar feature requirement.\n\n\n### Description\n\n_No response_\n\n### Use case\n\n_No response_\n\n### Related issues\n\n_No response_\n\n### Are you willing to submit a PR?\n\n- [ [...]
+github:GithubIssue:1:1371320153,https://github.com/apache/incubator-devlake/issues/3059,,3059,[Bug][customize] e2e test failed in postgres,"""### Search before asking\n\n- [X] I had searched in the [issues](https://github.com/apache/incubator-devlake/issues?q=is%3Aissue) and found no similar issues.\n\n\n### What happened\n\ncached plan must not change result type in e2e test.\n\n### What you expected to happen\n\nsuccess\n\n### How to reproduce\n\nrun e2e test in postgres check\n\n### A [...]
+github:GithubIssue:1:1371541422,https://github.com/apache/incubator-devlake/issues/3060,,3060,[Bug][Migrationscripts] use common and domainlayer for migrationscripts table model,"""### Search before asking\r\n\r\n- [X] I had searched in the [issues](https://github.com/apache/incubator-devlake/issues?q=is%3Aissue) and found no similar issues.\r\n\r\n\r\n### What happened\r\n\r\n\r [...]
+github:GithubIssue:1:1371617668,https://github.com/apache/incubator-devlake/issues/3061,,3061,[Feature][api-client] wait more time to prevent long-time collection canceled by network problem,"""### Search before asking\n\n- [X] I had searched in the [issues](https://github.com/apache/incubator-devlake/issues?q=is%3Aissue) and found no similar feature requirement.\n\n\n### Description\n\nWait more time to prevent long-time collection cancelled by network problems.\r\nSome users' Jira serv [...]
+github:GithubIssue:1:1372381019,https://github.com/apache/incubator-devlake/issues/3063,,3063,[Bug][Bitbucket] the other subtasks cannot continue when the number of issues is zero.,"""### Search before asking\r\n\r\n- [X] I had searched in the [issues](https://github.com/apache/incubator-devlake/issues?q=is%3Aissue) and found no similar issues.\r\n\r\n\r\n### What happened\r\n\r\nthe other subtasks cannot continue when the number of issues is zero.\r\n<img width=\""1370\"" alt=\""Screen [...]
+github:GithubIssue:1:1372550825,https://github.com/apache/incubator-devlake/issues/3065,,3065,[Bug][tapd] error occurred while adding connection for plugin tapd ,"""### Search before asking\n\n- [X] I had searched in the [issues](https://github.com/apache/incubator-devlake/issues?q=is%3Aissue) and found no similar issues.\n\n\n### What happened\n\nAn error occurred while adding a new connection for plugin `tapd`\r\nfront-end:\r\n![2Owp3eV9t4](https://user-images.githubusercontent.com/845 [...]
+github:GithubIssue:1:1372581663,https://github.com/apache/incubator-devlake/issues/3067,,3067,[Bug][Framework] error getting pipelines,"""### Search before asking\r\n\r\n- [X] I had searched in the [issues](https://github.com/apache/incubator-devlake/issues?q=is%3Aissue) and found no similar issues.\r\n\r\n\r\n### What happened\r\n\r\nWe cannot get pipelines and blueprints on config-ui\r\n[2022-09-14 16:25:30] ERROR HTTP 500 error\r\n caused by: error getting pipelines\r\n [...]
+github:GithubIssue:1:1372644519,https://github.com/apache/incubator-devlake/issues/3068,,3068,[Feature][Backend] Establish a standard error logging convention,"""### Search before asking\r\n\r\n- [X] I had searched in the [issues](https://github.com/apache/incubator-devlake/issues?q=is%3Aissue) and found no similar feature requirement.\r\n\r\n\r\n### Description\r\n\r\nCurrently the codebase is filled with random usages of logger.Error(err, \""msg\""). Calls to this function must be done [...]
+github:GithubIssue:1:1373079324,https://github.com/apache/incubator-devlake/issues/3072,,3072,[Bug][framework] status code 500 returned from the server,"""### Search before asking\n\n- [X] I had searched in the [issues](https://github.com/apache/incubator-devlake/issues?q=is%3Aissue) and found no similar issues.\n\n\n### What happened\n\nstatus code 500 returned from the endpoint `/pipelines`\r\n```\r\n$ curl -i 127.0.0.1:8080/pipelines\r\nHTTP/1.1 500 Internal Server Error\r\nContent-T [...]
+github:GithubIssue:1:1373792477,https://github.com/apache/incubator-devlake/issues/3073,,3073,[Bug][github] calculateSince retrun the empty timespace,"""### Search before asking\r\n\r\n- [X] I had searched in the [issues](https://github.com/apache/incubator-devlake/issues?q=is%3Aissue) and found no similar issues.\r\n\r\n\r\n### What happened\r\n\r\n\r\n\r\n![image](https://user- [...]
diff --git a/plugins/dora/e2e/snapshot_tables/lake_cicd_tasks.csv b/plugins/dora/e2e/snapshot_tables/lake_cicd_tasks.csv
index 33f70df7..2d68eec8 100644
--- a/plugins/dora/e2e/snapshot_tables/lake_cicd_tasks.csv
+++ b/plugins/dora/e2e/snapshot_tables/lake_cicd_tasks.csv
@@ -1,22 +1,22 @@
-id,created_at,updated_at,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark,name,pipeline_id,result,status,type,duration_sec,started_date,finished_date,environment
-github:GithubJob:1:8269045794,2022-09-15 03:20:03.867,2022-09-15 03:20:03.867,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_jobs,2,"",lint,github:GithubRun:1:3022052551,FAILURE,DONE,"",72,2022-09-09 11:27:13,2022-09-09 11:28:25,""
-github:GithubJob:1:8269045795,2022-09-15 03:20:03.867,2022-09-15 03:20:03.867,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_jobs,3,"",test (ubuntu-latest),github:GithubRun:1:3022052557,SUCCESS,DONE,"",191,2022-09-09 11:27:42,2022-09-09 11:30:53,""
-github:GithubJob:1:8269045819,2022-09-15 03:20:03.867,2022-09-15 03:20:03.867,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_jobs,1,"",commit-msg,github:GithubRun:1:3022052555,SUCCESS,DONE,"",8,2022-09-09 11:26:38,2022-09-09 11:26:46,""
-github:GithubJob:1:8269045928,2022-09-15 03:20:03.867,2022-09-15 03:20:03.867,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_jobs,4,"",e2e-mysql,github:GithubRun:1:3022052558,SUCCESS,DONE,"",200,2022-09-09 11:28:00,2022-09-09 11:31:20,""
-github:GithubJob:1:8269046116,2022-09-15 03:20:03.867,2022-09-15 03:20:03.867,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_jobs,6,"",check Apache license header,github:GithubRun:1:3022052589,SUCCESS,DONE,"",57,2022-09-09 11:28:34,2022-09-09 11:29:31,""
-github:GithubJob:1:8269415129,2022-09-15 03:20:03.867,2022-09-15 03:20:03.867,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_jobs,5,"",lint,github:GithubRun:1:3022190519,FAILURE,DONE,"",84,2022-09-09 11:51:21,2022-09-09 11:52:45,""
-github:GithubJob:1:8269415130,2022-09-15 03:20:03.867,2022-09-15 03:20:03.867,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_jobs,9,"",test (ubuntu-latest),github:GithubRun:1:3022190524,SUCCESS,DONE,"",182,2022-09-09 11:50:31,2022-09-09 11:53:33,""
-github:GithubJob:1:8269415142,2022-09-15 03:20:03.867,2022-09-15 03:20:03.867,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_jobs,10,"",check Apache license header,github:GithubRun:1:3022190525,SUCCESS,DONE,"",63,2022-09-09 11:52:06,2022-09-09 11:53:09,""
-github:GithubJob:1:8269415165,2022-09-15 03:20:03.867,2022-09-15 03:20:03.867,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_jobs,8,"",commit-msg,github:GithubRun:1:3022190521,SUCCESS,DONE,"",10,2022-09-09 11:52:46,2022-09-09 11:52:56,""
-github:GithubJob:1:8269415315,2022-09-15 03:20:03.867,2022-09-15 03:20:03.867,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_jobs,7,"",e2e-mysql,github:GithubRun:1:3022190520,SUCCESS,DONE,"",242,2022-09-09 11:53:45,2022-09-09 11:57:47,""
-github:GithubJob:1:8269598312,2022-09-15 03:20:03.867,2022-09-15 03:20:03.867,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_jobs,14,"",e2e-mysql,github:GithubRun:1:3022255519,SUCCESS,DONE,"",828,2022-09-09 12:05:04,2022-09-09 12:18:52,""
-github:GithubJob:1:8269598316,2022-09-15 03:20:03.867,2022-09-15 03:20:03.867,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_jobs,11,"",test (ubuntu-latest),github:GithubRun:1:3022255511,SUCCESS,DONE,"",1016,2022-09-09 12:05:49,2022-09-09 12:22:45,""
-github:GithubJob:1:8269598318,2022-09-15 03:20:03.867,2022-09-15 03:20:03.867,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_jobs,12,"",lint,github:GithubRun:1:3022255516,FAILURE,DONE,"",62,2022-09-09 12:02:28,2022-09-09 12:03:30,""
-github:GithubJob:1:8269598325,2022-09-15 03:20:03.867,2022-09-15 03:20:03.867,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_jobs,13,"",check Apache license header,github:GithubRun:1:3022255517,SUCCESS,DONE,"",52,2022-09-09 12:03:51,2022-09-09 12:04:43,""
-github:GithubJob:1:8269598469,2022-09-15 03:20:03.867,2022-09-15 03:20:03.867,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_jobs,15,"",commit-msg,github:GithubRun:1:3022255524,FAILURE,DONE,"",41,2022-09-09 12:06:55,2022-09-09 12:07:36,""
-github:GithubJob:1:8269729324,2022-09-15 03:20:03.867,2022-09-15 03:20:03.867,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_jobs,16,"",e2e-mysql,github:GithubRun:1:3022300995,SUCCESS,DONE,"",1007,2022-09-09 12:10:08,2022-09-09 12:26:55,""
-github:GithubJob:1:8269729327,2022-09-15 03:20:03.867,2022-09-15 03:38:03.694,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_jobs,18,"",xx_deploy_xx,github:GithubRun:1:3022301001,FAILURE,DONE,DEPLOY,82,2022-09-09 12:11:22,2022-09-09 12:12:44,staging
-github:GithubJob:1:8269729550,2022-09-15 03:20:03.867,2022-09-15 03:38:03.694,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_jobs,19,"",build_deployment,github:GithubRun:1:3022301007,SUCCESS,DONE,DEPLOY,6,2022-09-09 12:12:02,2022-09-09 12:12:08,staging
-github:GithubJob:1:8269729553,2022-09-15 03:20:03.867,2022-09-15 03:38:03.694,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_jobs,17,"",build_deploy,github:GithubRun:1:3022300997,SUCCESS,DONE,DEPLOY,57,2022-09-09 12:12:52,2022-09-09 12:13:49,staging
-github:GithubJob:1:8269729821,2022-09-15 03:20:03.867,2022-09-15 03:38:03.694,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_jobs,20,"",deployment,github:GithubRun:1:3022301033,SUCCESS,DONE,DEPLOY,991,2022-09-09 12:13:58,2022-09-09 12:30:29,staging
-github:GithubJob:1:8269872730,2022-09-15 03:20:03.867,2022-09-15 03:38:03.694,"{""ConnectionId"":1,""Owner"":""apache"",""Repo"":""incubator-devlake""}",_raw_github_api_jobs,21,"",deploy,github:GithubRun:1:3022348786,SUCCESS,DONE,DEPLOY,1006,2022-09-09 12:21:30,2022-09-09 12:38:16,staging
+id,name,pipeline_id,result,status,type,environment,duration_sec,started_date,finished_date
+github:GithubJob:1:8269045794,lint,github:GithubRun:1:3022052551,FAILURE,DONE,,,72,2022-09-09T11:27:13.000+00:00,2022-09-09T11:28:25.000+00:00
+github:GithubJob:1:8269045795,test (ubuntu-latest),github:GithubRun:1:3022052557,SUCCESS,DONE,,,191,2022-09-09T11:27:42.000+00:00,2022-09-09T11:30:53.000+00:00
+github:GithubJob:1:8269045819,commit-msg,github:GithubRun:1:3022052555,SUCCESS,DONE,,,8,2022-09-09T11:26:38.000+00:00,2022-09-09T11:26:46.000+00:00
+github:GithubJob:1:8269045928,e2e-mysql,github:GithubRun:1:3022052558,SUCCESS,DONE,,,200,2022-09-09T11:28:00.000+00:00,2022-09-09T11:31:20.000+00:00
+github:GithubJob:1:8269046116,check Apache license header,github:GithubRun:1:3022052589,SUCCESS,DONE,,,57,2022-09-09T11:28:34.000+00:00,2022-09-09T11:29:31.000+00:00
+github:GithubJob:1:8269415129,lint,github:GithubRun:1:3022190519,FAILURE,DONE,,,84,2022-09-09T11:51:21.000+00:00,2022-09-09T11:52:45.000+00:00
+github:GithubJob:1:8269415130,test (ubuntu-latest),github:GithubRun:1:3022190524,SUCCESS,DONE,,,182,2022-09-09T11:50:31.000+00:00,2022-09-09T11:53:33.000+00:00
+github:GithubJob:1:8269415142,check Apache license header,github:GithubRun:1:3022190525,SUCCESS,DONE,,,63,2022-09-09T11:52:06.000+00:00,2022-09-09T11:53:09.000+00:00
+github:GithubJob:1:8269415165,commit-msg,github:GithubRun:1:3022190521,SUCCESS,DONE,,,10,2022-09-09T11:52:46.000+00:00,2022-09-09T11:52:56.000+00:00
+github:GithubJob:1:8269415315,e2e-mysql,github:GithubRun:1:3022190520,SUCCESS,DONE,,,242,2022-09-09T11:53:45.000+00:00,2022-09-09T11:57:47.000+00:00
+github:GithubJob:1:8269598312,e2e-mysql,github:GithubRun:1:3022255519,SUCCESS,DONE,,,828,2022-09-09T12:05:04.000+00:00,2022-09-09T12:18:52.000+00:00
+github:GithubJob:1:8269598316,test (ubuntu-latest),github:GithubRun:1:3022255511,SUCCESS,DONE,,,1016,2022-09-09T12:05:49.000+00:00,2022-09-09T12:22:45.000+00:00
+github:GithubJob:1:8269598318,lint,github:GithubRun:1:3022255516,FAILURE,DONE,,,62,2022-09-09T12:02:28.000+00:00,2022-09-09T12:03:30.000+00:00
+github:GithubJob:1:8269598325,check Apache license header,github:GithubRun:1:3022255517,SUCCESS,DONE,,,52,2022-09-09T12:03:51.000+00:00,2022-09-09T12:04:43.000+00:00
+github:GithubJob:1:8269598469,commit-msg,github:GithubRun:1:3022255524,FAILURE,DONE,,,41,2022-09-09T12:06:55.000+00:00,2022-09-09T12:07:36.000+00:00
+github:GithubJob:1:8269729324,e2e-mysql,github:GithubRun:1:3022300995,SUCCESS,DONE,,,1007,2022-09-09T12:10:08.000+00:00,2022-09-09T12:26:55.000+00:00
+github:GithubJob:1:8269729327,xx_deploy_xx,github:GithubRun:1:3022301001,FAILURE,DONE,DEPLOY,staging,82,2022-09-09T12:11:22.000+00:00,2022-09-09T12:12:44.000+00:00
+github:GithubJob:1:8269729550,build_deployment,github:GithubRun:1:3022301007,SUCCESS,DONE,DEPLOY,staging,6,2022-09-09T12:12:02.000+00:00,2022-09-09T12:12:08.000+00:00
+github:GithubJob:1:8269729553,build_deploy,github:GithubRun:1:3022300997,SUCCESS,DONE,DEPLOY,staging,57,2022-09-09T12:12:52.000+00:00,2022-09-09T12:13:49.000+00:00
+github:GithubJob:1:8269729821,deployment,github:GithubRun:1:3022301033,SUCCESS,DONE,DEPLOY,staging,991,2022-09-09T12:13:58.000+00:00,2022-09-09T12:30:29.000+00:00
+github:GithubJob:1:8269872730,deploy,github:GithubRun:1:3022348786,SUCCESS,DONE,DEPLOY,staging,1006,2022-09-09T12:21:30.000+00:00,2022-09-09T12:38:16.000+00:00
diff --git a/plugins/dora/e2e/snapshot_tables/pull_requests.csv b/plugins/dora/e2e/snapshot_tables/pull_requests.csv
new file mode 100644
index 00000000..cf050384
--- /dev/null
+++ b/plugins/dora/e2e/snapshot_tables/pull_requests.csv
@@ -0,0 +1,10 @@
+id,base_repo_id,head_repo_id,status,title,description,url,author_name,author_id,parent_pr_id,pull_request_key,created_date,merged_date,closed_date,type,component,merge_commit_sha,head_ref,base_ref,base_commit_sha,head_commit_sha,coding_timespan,review_lag,review_timespan,deploy_timespan,change_timespan,orig_coding_timespan,orig_review_lag,orig_review_timespan,orig_deploy_timespan
+github:GithubPullRequest:1:1043463302,github:GithubRepo:1:384111310,github:GithubRepo:1:384111310,closed,feat: implement API for plugin `customize`,"""# Summary\r\n\r\nfulfill the requirement \r\n#2880 ([Feature][customize] implement API for plugin customize) \r\n#2985 ([Feature][customize] new sub-task ExtractCustomizedFields for plugin customize)\r\n\r\nrelated to #2802 \r\nWe take Jira issue as an example.\r\n\r\n1. find all primary keys of table `issues`, which is `id`\r\n2. retriev [...]
+github:GithubPullRequest:1:1048233599,github:GithubRepo:1:384111310,github:GithubRepo:1:384111310,closed,feat: new plugin for gitea,"""# Summary\r\n[Gitea](https://gitea.io/) is an open-source software package for hosting software development version control using Git as well as other collaborative features like bug tracking, code review, kanban boards, tickets, and wikis.\r\n<!--\r\nThanks for submitting a pull request!\r\n\r\nWe appreciate you spending the time to work on these changes [...]
+github:GithubPullRequest:1:1049191985,github:GithubRepo:1:384111310,github:GithubRepo:1:384111310,closed,[issue-2908]: Bump lake-builder version to 0.0.8,"""### ⚠️ Pre Checklist\r\n\r\n> Please complete _ALL_ items in this checklist, and remove before submitting\r\n\r\n- [x] I have read through the [Contributing Documentation](https://devlake.apache.org/community/).\r\n- [x] I have added relevant tests.\r\n- [x] I have added relevant documentation.\r\n- [ ] I will add labels to the PR, s [...]
+github:GithubPullRequest:1:1051243958,github:GithubRepo:1:384111310,github:GithubRepo:1:384111310,closed,fix: golangci-lint error,"""# Summary\r\n\r\ngolint-ci was down for a while, this PR try to fix all missed linting errors during the period.\r\n\r\n### Screenshots\r\n\r\n\r\n\r\n""",https://github.com/apache/incubator-devlake/pull/3032,klesh,github:GithubAccount:1:61080,,3032,2 [...]
+github:GithubPullRequest:1:1051273681,github:GithubRepo:1:384111310,github:GithubRepo:1:384111310,closed,fix: update pipelineId,"""# Summary\r\n\r\nupdate pipelineId\r\n\r\n### Does this close any open issues?\r\nrelated to #2998 \r\n\r\n### Screenshots\r\n\r\n\r\n### Other Information\r\nAny other information that is important to this PR.\r\n""",https://github.com/apache/incub [...]
+github:GithubPullRequest:1:1051340471,github:GithubRepo:1:384111310,github:GithubRepo:1:384111310,closed,Jenkins fix,"""# Summary\r\nAdd TestJenkinsStagesDataFlow.\r\nAdd path to stages collect.\r\n\r\n<!--\r\nThanks for submitting a pull request!\r\n\r\nWe appreciate you spending the time to work on these changes.\r\nPlease fill out as many sections below as possible.\r\n-->\r\n\r\n### Does this close any open issues?\r\nCloses #3026 \r\n\r\n### Screenshots\r\n![image](https://user-imag [...]
+github:GithubPullRequest:1:1051524882,github:GithubRepo:1:384111310,github:GithubRepo:1:384111310,closed,fix: task failure caused by 404,"""# Summary\r\nfix #2960 [Bug][gitihub] collect account failed by not found user\r\nIn the case of `err == ErrIgnoreAndContinue`, the `err` should not be wrapped, because on the caller side, the expression `err == ErrIgnoreAndContinue` would evaluate as false.\r\n```go\r\nif apiClient.afterResponse != nil {\r\n\t\terr = apiClient.afterResponse(res)\r\ [...]
+github:GithubPullRequest:1:1051574863,github:GithubRepo:1:384111310,github:GithubRepo:1:384111310,closed,fix(jenkins): update e2e,"""# Summary\r\n\r\nUpdate e2e according to recent changes\r\n\r\n### Does this close any open issues?\r\nrelate to #2854\r\n\r\n\r\n### Screenshots\r\nInclude any relevant screenshots here.\r\n\r\n### Other Information\r\nAny other information that is important to this PR.\r\n""",https://github.com/apache/incubator-devlake/pull/3039,warren830,github:GithubAcc [...]
+github:GithubPullRequest:1:1051637383,github:GithubRepo:1:384111310,github:GithubRepo:1:384111310,closed,fix(gitlab): update e2e,"""# Summary\r\n\r\nupdate gitlab e2e according to recent changes\r\n\r\n### Does this close any open issues?\r\nrelate to #2871\r\n\r\n\r\n### Screenshots\r\nInclude any relevant screenshots here.\r\n\r\n### Other Information\r\nAny other information that is important to this PR.\r\n""",https://github.com/apache/incubator-devlake/pull/3041,warren830,github:Git [...]
diff --git a/plugins/dora/tasks/change_lead_time_calculator.go b/plugins/dora/tasks/change_lead_time_calculator.go
index 5fe7b641..cceda1a5 100644
--- a/plugins/dora/tasks/change_lead_time_calculator.go
+++ b/plugins/dora/tasks/change_lead_time_calculator.go
@@ -78,7 +78,7 @@ func CalculateChangeLeadTime(taskCtx core.SubTaskContext) errors.Error {
pr.OrigReviewLag = int64(firstReviewTime.Sub(pr.CreatedDate).Minutes())
pr.OrigReviewTimespan = int64(pr.MergedDate.Sub(*firstReviewTime).Minutes())
}
- deployTime, err := getDeployTime(repoId, pr.MergeCommitSha, *pr.MergedDate, db)
+ deployTime, err := getDeployTime(repoId, data.Options.Environment, *pr.MergedDate, db)
if err != nil {
return nil, err
}
@@ -139,7 +139,7 @@ func getFirstReviewTime(prId string, prCreator string, db dal.Dal) (*time.Time,
dal.Orderby("created_date ASC"),
}
err := db.First(review, commentClauses...)
- if err == gorm.ErrRecordNotFound {
+ if goerror.Is(err, gorm.ErrRecordNotFound) {
return nil, nil
}
if err != nil {
@@ -148,22 +148,20 @@ func getFirstReviewTime(prId string, prCreator string, db dal.Dal) (*time.Time,
return &review.CreatedDate, nil
}
-func getDeployTime(repoId string, mergeSha string, mergeDate time.Time, db dal.Dal) (*time.Time, errors.Error) {
+func getDeployTime(repoId string, environment string, mergeDate time.Time, db dal.Dal) (*time.Time, errors.Error) {
cicdTask := &devops.CICDTask{}
cicdTaskClauses := []dal.Clause{
dal.From(&devops.CICDTask{}),
- dal.Join("left join cicd_pipelines on cicd_pipelines.id = cicd_tasks.pipeline_id"),
- dal.Join("left join cicd_pipeline_repos on cicd_pipelines.id = cicd_pipeline_repos.id"),
- dal.Where(`cicd_pipeline_repos.commit_sha = ?
- and cicd_pipeline_repos.repo = ?
- and cicd_tasks.type = ?
+ dal.Join("left join cicd_pipeline_repos on cicd_tasks.pipeline_id = cicd_pipeline_repos.id"),
+ dal.Where(`cicd_pipeline_repos.repo = ?
+ and cicd_tasks.environment = ?
and cicd_tasks.result = ?
and cicd_tasks.started_date > ?`,
- mergeSha, repoId, "DEPLOY", "SUCCESS", mergeDate),
+ repoId, environment, "SUCCESS", mergeDate),
dal.Orderby("cicd_tasks.started_date ASC"),
}
err := db.First(cicdTask, cicdTaskClauses...)
- if err == gorm.ErrRecordNotFound {
+ if goerror.Is(err, gorm.ErrRecordNotFound) {
return nil, nil
}
if err != nil {
diff --git a/plugins/dora/tasks/issue_deploy_connector.go b/plugins/dora/tasks/issue_deploy_connector.go
index 78e6da3a..1cdbebd8 100644
--- a/plugins/dora/tasks/issue_deploy_connector.go
+++ b/plugins/dora/tasks/issue_deploy_connector.go
@@ -18,12 +18,14 @@ limitations under the License.
package tasks
import (
+ goerror "errors"
"github.com/apache/incubator-devlake/errors"
"github.com/apache/incubator-devlake/models/domainlayer/devops"
"github.com/apache/incubator-devlake/models/domainlayer/ticket"
"github.com/apache/incubator-devlake/plugins/core"
"github.com/apache/incubator-devlake/plugins/core/dal"
"github.com/apache/incubator-devlake/plugins/helper"
+ "gorm.io/gorm"
"reflect"
)
@@ -35,8 +37,6 @@ var ConnectIssueDeployMeta = core.SubTaskMeta{
DomainTypes: []string{core.DOMAIN_TYPE_CICD},
}
-const RAW_ISSUES_TABLE = `dora_issues`
-
func ConnectIssueDeploy(taskCtx core.SubTaskContext) errors.Error {
db := taskCtx.GetDal()
data := taskCtx.GetData().(*DoraTaskData)
@@ -50,7 +50,7 @@ func ConnectIssueDeploy(taskCtx core.SubTaskContext) errors.Error {
dal.Join("left join board_repos on board_repos.board_id = board_issues.board_id"),
dal.Where(
"board_repos.repo_id = ? and issues.type = ?",
- data.Options.RepoId, "Incident",
+ data.Options.RepoId, "INCIDENT",
),
}
cursor, err := db.Cursor(clauses...)
@@ -74,19 +74,21 @@ func ConnectIssueDeploy(taskCtx core.SubTaskContext) errors.Error {
cicdTask := &devops.CICDTask{}
cicdTakClauses := []dal.Clause{
dal.From(cicdTask),
- dal.Join(`left join cicd_pipelines
- on cicd_pipelines.id = cicd_tasks.pipeline_id`),
- dal.Join("left join cicd_pipeline_repos on cicd_pipelines.id = cicd_pipeline_repos.id"),
+ dal.Join("left join cicd_pipeline_repos on cicd_tasks.pipeline_id = cicd_pipeline_repos.id"),
dal.Where(
`cicd_pipeline_repos.repo = ? and cicd_tasks.finished_date < ?
- and cicd_tasks.result = ? and cicd_tasks.type = ?`,
- data.Options.RepoId, issueToBeUpdate.CreatedDate, "SUCCESS", "DEPLOY",
+ and cicd_tasks.result = ? and cicd_tasks.environment = ?`,
+ data.Options.RepoId, issueToBeUpdate.CreatedDate, "SUCCESS", data.Options.Environment,
),
dal.Orderby("cicd_tasks.finished_date DESC"),
}
err = db.First(cicdTask, cicdTakClauses...)
if err != nil {
- return nil, err
+ if goerror.Is(err, gorm.ErrRecordNotFound) {
+ return nil, nil
+ } else {
+ return nil, err
+ }
}
issueToBeUpdate.DeploymentId = cicdTask.Id
diff --git a/plugins/github/tasks/commit_convertor.go b/plugins/github/tasks/commit_convertor.go
index f7229faa..42480dab 100644
--- a/plugins/github/tasks/commit_convertor.go
+++ b/plugins/github/tasks/commit_convertor.go
@@ -33,7 +33,7 @@ import (
var ConvertCommitsMeta = core.SubTaskMeta{
Name: "convertCommits",
EntryPoint: ConvertCommits,
- EnabledByDefault: false,
+ EnabledByDefault: true,
Description: "Convert tool layer table github_commits into domain layer table commits",
DomainTypes: []string{core.DOMAIN_TYPE_CODE},
}
diff --git a/plugins/github/tasks/pr_commit_extractor.go b/plugins/github/tasks/pr_commit_extractor.go
index 84f890b2..94073a95 100644
--- a/plugins/github/tasks/pr_commit_extractor.go
+++ b/plugins/github/tasks/pr_commit_extractor.go
@@ -58,6 +58,7 @@ type PullRequestCommit struct {
func ExtractApiPullRequestCommits(taskCtx core.SubTaskContext) errors.Error {
data := taskCtx.GetData().(*GithubTaskData)
+ repoId := data.Repo.GithubId
extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{
RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
Ctx: taskCtx,
@@ -90,7 +91,13 @@ func ExtractApiPullRequestCommits(taskCtx core.SubTaskContext) errors.Error {
return nil, err
}
// need to extract 2 kinds of entities here
- results := make([]interface{}, 0, 2)
+ results := make([]interface{}, 0, 3)
+ githubRepoCommit := &models.GithubRepoCommit{
+ ConnectionId: data.Options.ConnectionId,
+ RepoId: repoId,
+ CommitSha: apiPullRequestCommit.Sha,
+ }
+ results = append(results, githubRepoCommit)
githubCommit, err := convertPullRequestCommit(apiPullRequestCommit, data.Options.ConnectionId)
if err != nil {