You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by zh...@apache.org on 2023/06/05 06:45:25 UTC
[incubator-devlake] branch main updated: feat: add bug_commit collector,extractor and bug_repo_commit_collector (#5343)
This is an automated email from the ASF dual-hosted git repository.
zhangliang2022 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 11036ff83 feat: add bug_commit collector,extractor and bug_repo_commit_collector (#5343)
11036ff83 is described below
commit 11036ff8379cfa6072005903135581b5a558ef95
Author: abeizn <zi...@merico.dev>
AuthorDate: Mon Jun 5 14:45:21 2023 +0800
feat: add bug_commit collector,extractor and bug_repo_commit_collector (#5343)
* feat: add bug_commit collector,extractor and bug_repo_commit_collector
* feat: add bug_commit collector,extractor and bug_repo_commit_collector
* feat: zentao add bug repo commits extractor
* fix: convertor task
* feat: zentao bug repo commits convertor and add e2e
* fix: zentao e2e
* fix: zentao e2e
* fix: bug_repo_commits_collector
* feat: add zentao story repo commits
* fix: regexp just gets compiled once and check first whether the array is out of bounds
* feat: zentao task repo commits and e2e
* fix: impl and some lint
---
backend/plugins/zentao/e2e/bug_commits_test.go | 76 +++++++++++
.../e2e/raw_tables/_raw_zentao_api_bug_commits.csv | 25 ++++
.../_raw_zentao_api_bug_repo_commits.csv | 2 +
.../raw_tables/_raw_zentao_api_story_commits.csv | 21 +++
.../_raw_zentao_api_story_repo_commits.csv | 2 +
.../raw_tables/_raw_zentao_api_task_commits.csv | 7 +
.../_raw_zentao_api_task_repo_commits.csv | 2 +
.../snapshot_tables/_tool_zentao_bug_commits.csv | 3 +
.../_tool_zentao_bug_repo_commits.csv | 2 +
.../snapshot_tables/_tool_zentao_story_commits.csv | 3 +
.../_tool_zentao_story_repo_commits.csv | 2 +
.../snapshot_tables/_tool_zentao_task_commits.csv | 3 +
.../_tool_zentao_task_repo_commits.csv | 2 +
.../e2e/snapshot_tables/issue_bug_repo_commits.csv | 2 +
.../snapshot_tables/issue_story_repo_commits.csv | 2 +
.../snapshot_tables/issue_task_repo_commits.csv | 2 +
backend/plugins/zentao/e2e/story_commits_test.go | 76 +++++++++++
backend/plugins/zentao/e2e/task_commits_test.go | 76 +++++++++++
backend/plugins/zentao/impl/impl.go | 17 +++
.../plugins/zentao/models/archived/bug_commits.go | 41 +++++-
.../zentao/models/archived/story_commits.go | 62 +++++++++
.../plugins/zentao/models/archived/task_commits.go | 62 +++++++++
backend/plugins/zentao/models/bug_commits.go | 131 ++++++++++++++++--
...mmits.go => 20230605_add_issue_repo_commits.go} | 19 ++-
.../zentao/models/migrationscripts/register.go | 2 +-
backend/plugins/zentao/models/story_commits.go | 151 +++++++++++++++++++++
backend/plugins/zentao/models/task_commits.go | 151 +++++++++++++++++++++
.../plugins/zentao/tasks/bug_commits_collector.go | 28 ++--
.../plugins/zentao/tasks/bug_commits_extractor.go | 113 +++++++++++++++
..._collector.go => bug_repo_commits_collector.go} | 91 +++++--------
.../zentao/tasks/bug_repo_commits_convertor.go | 93 +++++++++++++
.../zentao/tasks/bug_repo_commits_extractor.go | 91 +++++++++++++
backend/plugins/zentao/tasks/shared.go | 35 +++++
...its_collector.go => story_commits_collector.go} | 60 ++++----
.../zentao/tasks/story_commits_extractor.go | 115 ++++++++++++++++
...ollector.go => story_repo_commits_collector.go} | 87 +++++-------
.../zentao/tasks/story_repo_commits_convertor.go | 93 +++++++++++++
.../zentao/tasks/story_repo_commits_extractor.go | 91 +++++++++++++
...mits_collector.go => task_commits_collector.go} | 69 +++++-----
.../plugins/zentao/tasks/task_commits_extractor.go | 115 ++++++++++++++++
...collector.go => task_repo_commits_collector.go} | 91 +++++--------
.../zentao/tasks/task_repo_commits_convertor.go | 93 +++++++++++++
.../zentao/tasks/task_repo_commits_extractor.go | 91 +++++++++++++
43 files changed, 2028 insertions(+), 272 deletions(-)
diff --git a/backend/plugins/zentao/e2e/bug_commits_test.go b/backend/plugins/zentao/e2e/bug_commits_test.go
new file mode 100644
index 000000000..5bd50fdc1
--- /dev/null
+++ b/backend/plugins/zentao/e2e/bug_commits_test.go
@@ -0,0 +1,76 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+ "testing"
+
+ "github.com/apache/incubator-devlake/core/models/common"
+ "github.com/apache/incubator-devlake/core/models/domainlayer/crossdomain"
+ "github.com/apache/incubator-devlake/helpers/e2ehelper"
+ "github.com/apache/incubator-devlake/plugins/zentao/impl"
+ "github.com/apache/incubator-devlake/plugins/zentao/models"
+ "github.com/apache/incubator-devlake/plugins/zentao/tasks"
+)
+
+func TestZentaoBugCommitsDataFlow(t *testing.T) {
+
+ var zentao impl.Zentao
+ dataflowTester := e2ehelper.NewDataFlowTester(t, "zentao", zentao)
+
+ taskData := &tasks.ZentaoTaskData{
+ Options: &tasks.ZentaoOptions{
+ ConnectionId: 1,
+ ProjectId: 0,
+ ProductId: 22,
+ },
+ }
+
+ // import _raw_zentao_api_bug_commits raw data table
+ dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_zentao_api_bug_commits.csv",
+ "_raw_zentao_api_bug_commits")
+
+ // verify bug commit extraction
+ dataflowTester.FlushTabler(&models.ZentaoBugCommit{})
+ dataflowTester.Subtask(tasks.ExtractBugCommitsMeta, taskData)
+ dataflowTester.VerifyTableWithOptions(&models.ZentaoBugCommit{}, e2ehelper.TableOptions{
+ CSVRelPath: "./snapshot_tables/_tool_zentao_bug_commits.csv",
+ IgnoreTypes: []interface{}{common.NoPKModel{}},
+ })
+
+ // import _raw_zentao_api_bug_repo_commits raw data table
+ dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_zentao_api_bug_repo_commits.csv",
+ "_raw_zentao_api_bug_repo_commits")
+
+ // verify bug repo commit extraction
+ dataflowTester.FlushTabler(&models.ZentaoBugRepoCommit{})
+ dataflowTester.Subtask(tasks.ExtractBugRepoCommitsMeta, taskData)
+ dataflowTester.VerifyTableWithOptions(&models.ZentaoBugRepoCommit{}, e2ehelper.TableOptions{
+ CSVRelPath: "./snapshot_tables/_tool_zentao_bug_repo_commits.csv",
+ IgnoreTypes: []interface{}{common.NoPKModel{}},
+ })
+
+ // verify bug repo commit conversion
+ dataflowTester.FlushTabler(&crossdomain.IssueRepoCommit{})
+ dataflowTester.Subtask(tasks.ConvertBugRepoCommitsMeta, taskData)
+ dataflowTester.VerifyTableWithOptions(&crossdomain.IssueRepoCommit{}, e2ehelper.TableOptions{
+ CSVRelPath: "./snapshot_tables/issue_bug_repo_commits.csv",
+ IgnoreTypes: []interface{}{common.NoPKModel{}},
+ })
+
+}
diff --git a/backend/plugins/zentao/e2e/raw_tables/_raw_zentao_api_bug_commits.csv b/backend/plugins/zentao/e2e/raw_tables/_raw_zentao_api_bug_commits.csv
new file mode 100644
index 000000000..01ec64662
--- /dev/null
+++ b/backend/plugins/zentao/e2e/raw_tables/_raw_zentao_api_bug_commits.csv
@@ -0,0 +1,25 @@
+id,params,data,url,input,created_at
+1003,"{""ConnectionId"":1,""ProductId"":22,""ProjectId"":0}","{""id"":15826,""objectType"":""bug"",""objectID"":6060,""product"":"""",""project"":0,""execution"":0,""actor"":""gerile tu"",""action"":""commented"",""date"":""2023-02-20 03:53:48"",""comment"":""\u6240\u6709\u56fe\u8868\u90fd\u662f\u624b\u578b\uff0c\u4e0d\u4f1a\u6839\u636e\u662f\u5426\u914d\u7f6e\u4e86interactions\u800c\u6539\u53d8\u6837\u5f0f\uff0c\u56fe\u8868\u548cinteractions\u662f\u4e24\u6761\u5e76\u884c\u7684\u529f\u80 [...]
+1004,"{""ConnectionId"":1,""ProductId"":22,""ProjectId"":0}","{""id"":13832,""objectType"":""bug"",""objectID"":6067,""product"":"""",""project"":0,""execution"":0,""actor"":""\u674e\u8f89"",""action"":""Opened"",""date"":""2023-05-10 09:11:08"",""comment"":"""",""extra"":null,""read"":""0"",""vision"":""rnd"",""efforted"":0,""history"":[],""desc"":""2023-05-10 09:11:08, \u7531 \u003Cstrong\u003E\u674e\u8f89\u003C\/strong\u003E \u521b\u5efa\u3002\n""}",http://54.158.1.10:30001/api.php/v1 [...]
+1005,"{""ConnectionId"":1,""ProductId"":22,""ProjectId"":0}","{""id"":13833,""objectType"":""bug"",""objectID"":6068,""product"":"""",""project"":0,""execution"":0,""actor"":""57333edf-bbf8-424f-9d17-4956e0391a37"",""action"":""Opened"",""date"":""2023-01-03 09:16:03"",""comment"":"""",""extra"":null,""read"":""0"",""vision"":""rnd"",""efforted"":0,""history"":[],""desc"":""2023-01-03 09:16:03, \u7531 \u003Cstrong\u003E57333edf-bbf8-424f-9d17-4956e0391a37\u003C\/strong\u003E \u521b\u5efa [...]
+1006,"{""ConnectionId"":1,""ProductId"":22,""ProjectId"":0}","{""id"":14117,""objectType"":""bug"",""objectID"":6068,""product"":"""",""project"":0,""execution"":0,""actor"":""0380e3fd-c867-4a94-aff6-e0f3fedea2f5"",""action"":""commented"",""date"":""2023-01-03 09:38:13"",""comment"":""cag_nodes\u4e2d\u7684frequent_hexsha\u5728commits\u8868\u4e2d\u627e\u4e0d\u5230\uff0c\u9700\u8981AE\u540c\u5b66\u5e2e\u5fd9\u770b\u770b\n\n!image-20230103-093806.png|width=1523,height=217!"",""extra"":null [...]
+1007,"{""ConnectionId"":1,""ProductId"":22,""ProjectId"":0}","{""id"":14122,""objectType"":""bug"",""objectID"":6068,""product"":"""",""project"":0,""execution"":0,""actor"":""0380e3fd-c867-4a94-aff6-e0f3fedea2f5"",""action"":""commented"",""date"":""2023-01-03 09:44:19"",""comment"":""{code:sql}select cag_nodes.frequent_hexsha, commits.hexsha from cag_nodes left join commits on cag_nodes.analysis_id = commits.analysis_id and cag_nodes.frequent_hexsha = commits.hexsha where cag_nodes.ana [...]
+1008,"{""ConnectionId"":1,""ProductId"":22,""ProjectId"":0}","{""id"":14131,""objectType"":""bug"",""objectID"":6068,""product"":"""",""project"":0,""execution"":0,""actor"":""0d8c496d-fd2b-486d-a139-1a90d17cc871"",""action"":""commented"",""date"":""2023-01-03 16:25:31"",""comment"":""[~accountid:5fa8b6d142ab3b006eaa6f42] CAG_NODES\u8868\u683c\u4e2d\u7684\u63d0\u4ea4\u4fe1\u606f\u6765\u81ea\u4e8eblame\u5904\u7406\u5206\u6790\uff0ccommits\u8868\u683c\u4e2d\u7684\u63d0\u4ea4\u662f\u904d\u [...]
+1009,"{""ConnectionId"":1,""ProductId"":22,""ProjectId"":0}","{""id"":14132,""objectType"":""bug"",""objectID"":6068,""product"":"""",""project"":0,""execution"":0,""actor"":""0380e3fd-c867-4a94-aff6-e0f3fedea2f5"",""action"":""commented"",""date"":""2023-01-04 00:46:48"",""comment"":""\u770b\u4e86\u4e00\u4e0b\u76ee\u524dvdev.co\u4ee3\u7801\u5e93\u4e2d\u786e\u5b9e\u4e0d\u5b58\u5728\u8fd9\u4e24\u4e2acommit hash\u3002\n\n{quote}\u5728blame\u4e2d\u8fd8\u6709\u5bf9\u5e94\u4fe1\u606f\uff0c\u8 [...]
+1010,"{""ConnectionId"":1,""ProductId"":22,""ProjectId"":0}","{""id"":14188,""objectType"":""bug"",""objectID"":6068,""product"":"""",""project"":0,""execution"":0,""actor"":""0380e3fd-c867-4a94-aff6-e0f3fedea2f5"",""action"":""commented"",""date"":""2023-01-04 02:53:00"",""comment"":""AE\u4fa7\u6392\u67e5\u4fee\u590d\u4e2d"",""extra"":null,""read"":""0"",""vision"":""rnd"",""efforted"":0,""history"":[],""desc"":""2023-01-04 02:53:00, \u7531 \u003Cstrong\u003E0380e3fd-c867-4a94-aff6-e0f3 [...]
+1011,"{""ConnectionId"":1,""ProductId"":22,""ProjectId"":0}","{""id"":14258,""objectType"":""bug"",""objectID"":6068,""product"":"""",""project"":0,""execution"":0,""actor"":""0d8c496d-fd2b-486d-a139-1a90d17cc871"",""action"":""commented"",""date"":""2023-01-05 01:02:40"",""comment"":""AE\u4fa7\u7ed1\u5b9a\u7684MR\uff1a[https:\/\/gitlab.com\/merico-dev\/ae\/meta-analytics\/-\/merge_requests\/2347|https:\/\/gitlab.com\/merico-dev\/ae\/meta-analytics\/-\/merge_requests\/2347]"",""extra"":n [...]
+1012,"{""ConnectionId"":1,""ProductId"":22,""ProjectId"":0}","{""id"":14936,""objectType"":""bug"",""objectID"":6068,""product"":"""",""project"":0,""execution"":0,""actor"":""0380e3fd-c867-4a94-aff6-e0f3fedea2f5"",""action"":""commented"",""date"":""2023-01-13 02:41:34"",""comment"":""\u5c1a\u672a\u5408\u5e76\uff0c\u672c\u8fed\u4ee3\u53d1\u7248\u6709\u98ce\u9669\u3002"",""extra"":null,""read"":""0"",""vision"":""rnd"",""efforted"":0,""history"":[],""desc"":""2023-01-13 02:41:34, \u7531 [...]
+1013,"{""ConnectionId"":1,""ProductId"":22,""ProjectId"":0}","{""id"":14403,""objectType"":""bug"",""objectID"":6068,""product"":"""",""project"":0,""execution"":0,""actor"":""0d8c496d-fd2b-486d-a139-1a90d17cc871"",""action"":""commented"",""date"":""2023-02-03 02:40:36"",""comment"":""2.30.4-stable\u8fdb\u884c\u4e86\u5408\u5e76\u53d1\u5e03\u3002"",""extra"":null,""read"":""0"",""vision"":""rnd"",""efforted"":0,""history"":[],""desc"":""2023-02-03 02:40:36, \u7531 \u003Cstrong\u003E0d8c4 [...]
+1014,"{""ConnectionId"":1,""ProductId"":22,""ProjectId"":0}","{""id"":13842,""objectType"":""bug"",""objectID"":6071,""product"":"""",""project"":0,""execution"":0,""actor"":""\u674e\u8f89"",""action"":""Opened"",""date"":""2023-05-10 09:24:42"",""comment"":"""",""extra"":null,""read"":""0"",""vision"":""rnd"",""efforted"":0,""history"":[],""desc"":""2023-05-10 09:24:42, \u7531 \u003Cstrong\u003E\u674e\u8f89\u003C\/strong\u003E \u521b\u5efa\u3002\n""}",http://54.158.1.10:30001/api.php/v1 [...]
+1015,"{""ConnectionId"":1,""ProductId"":22,""ProjectId"":0}","{""id"":13808,""objectType"":""bug"",""objectID"":6054,""product"":"""",""project"":0,""execution"":0,""actor"":""\u674e\u8f89"",""action"":""Opened"",""date"":""2023-03-07 03:38:20"",""comment"":"""",""extra"":null,""read"":""0"",""vision"":""rnd"",""efforted"":0,""history"":[],""desc"":""2023-03-07 03:38:20, \u7531 \u003Cstrong\u003E\u674e\u8f89\u003C\/strong\u003E \u521b\u5efa\u3002\n""}",http://54.158.1.10:30001/api.php/v1 [...]
+1016,"{""ConnectionId"":1,""ProductId"":22,""ProjectId"":0}","{""id"":16534,""objectType"":""bug"",""objectID"":6054,""product"":"""",""project"":0,""execution"":0,""actor"":""0380e3fd-c867-4a94-aff6-e0f3fedea2f5"",""action"":""commented"",""date"":""2023-03-07 03:41:33"",""comment"":""\u5177\u4f53\u70b9\uff0c\u662f\u54ea\u4e2a\u5b57\u6bb5\u5bf9\u4e0d\u4e0a\u5427\u3002\u3002 \u8fd9\u6837\u770b\u4e0d\u660e\u767d"",""extra"":null,""read"":""0"",""vision"":""rnd"",""efforted"":0,""history"" [...]
+1017,"{""ConnectionId"":1,""ProductId"":22,""ProjectId"":0}","{""id"":16616,""objectType"":""bug"",""objectID"":6054,""product"":"""",""project"":0,""execution"":0,""actor"":""\u674e\u8f89"",""action"":""commented"",""date"":""2023-03-08 01:14:50"",""comment"":""[~accountid:5fa8b6d142ab3b006eaa6f42] \u8fed\u4ee3\u5f00\u53d1\u770b\u677f\uff0c\u51b2\u523a13\uff0c\u4e4b\u524d\u81ea\u52a8\u5316\u9884\u671f\u503c\u662f174\uff0c\u73b0\u5728\u7ed3\u679c\u662f173\uff0c\u7ed3\u679c\u5b58\u5728\u5 [...]
+1018,"{""ConnectionId"":1,""ProductId"":22,""ProjectId"":0}","{""id"":14449,""objectType"":""bug"",""objectID"":6054,""product"":"""",""project"":0,""execution"":0,""actor"":""0380e3fd-c867-4a94-aff6-e0f3fedea2f5"",""action"":""commented"",""date"":""2023-03-08 07:01:01"",""comment"":""\u76ee\u524d\u53ea\u80fd\u770b\u5230\u8fd9\u51e0\u4e2a\u6570\u636e\u53d8\u5316\u4e86\u3002\u4f46\u6ca1\u6709\u53c2\u7167\uff0c\u67e5\u8be2\u96be\u4ee5\u5165\u624b\u3002 \u5efa\u8bae\u5728\u6d4b\u8bd5\u7528 [...]
+1019,"{""ConnectionId"":1,""ProductId"":22,""ProjectId"":0}","{""id"":13846,""objectType"":""bug"",""objectID"":6073,""product"":"""",""project"":0,""execution"":0,""actor"":""\u674e\u8f89"",""action"":""Opened"",""date"":""2023-05-10 09:34:26"",""comment"":"""",""extra"":null,""read"":""0"",""vision"":""rnd"",""efforted"":0,""history"":[],""desc"":""2023-05-10 09:34:26, \u7531 \u003Cstrong\u003E\u674e\u8f89\u003C\/strong\u003E \u521b\u5efa\u3002\n""}",http://54.158.1.10:30001/api.php/v1 [...]
+1020,"{""ConnectionId"":1,""ProductId"":22,""ProjectId"":0}","{""id"":16653,""objectType"":""bug"",""objectID"":6073,""product"":"""",""project"":0,""execution"":0,""actor"":""0380e3fd-c867-4a94-aff6-e0f3fedea2f5"",""action"":""commented"",""date"":""2023-05-10 10:21:00"",""comment"":""\u8fd9\u5e94\u8be5\u662f\u4e00\u4e2amaster\u4e0a\u5c31\u6709\u7684\u95ee\u9898\u3002\u5728master\u4e0a\u5f00\u5206\u652f\u4fee\u590d"",""extra"":null,""read"":""0"",""vision"":""rnd"",""efforted"":0,""hist [...]
+1021,"{""ConnectionId"":1,""ProductId"":22,""ProjectId"":0}","{""id"":13857,""objectType"":""bug"",""objectID"":6082,""product"":"""",""project"":0,""execution"":0,""actor"":""379e86b9-4985-438f-9eed-cd747bb78c54"",""action"":""Opened"",""date"":""2023-05-10 10:02:14"",""comment"":"""",""extra"":null,""read"":""0"",""vision"":""rnd"",""efforted"":0,""history"":[],""desc"":""2023-05-10 10:02:14, \u7531 \u003Cstrong\u003E379e86b9-4985-438f-9eed-cd747bb78c54\u003C\/strong\u003E \u521b\u5efa [...]
+1022,"{""ConnectionId"":1,""ProductId"":22,""ProjectId"":0}","{""id"":13861,""objectType"":""bug"",""objectID"":6084,""product"":"""",""project"":0,""execution"":0,""actor"":""0380e3fd-c867-4a94-aff6-e0f3fedea2f5"",""action"":""Opened"",""date"":""2023-03-02 03:40:18"",""comment"":"""",""extra"":null,""read"":""0"",""vision"":""rnd"",""efforted"":0,""history"":[],""desc"":""2023-03-02 03:40:18, \u7531 \u003Cstrong\u003E0380e3fd-c867-4a94-aff6-e0f3fedea2f5\u003C\/strong\u003E \u521b\u5efa [...]
+1023,"{""ConnectionId"":1,""ProductId"":22,""ProjectId"":0}","{""id"":13862,""objectType"":""bug"",""objectID"":6085,""product"":"""",""project"":0,""execution"":0,""actor"":""lin.hao"",""action"":""Opened"",""date"":""2023-05-09 10:47:13"",""comment"":"""",""extra"":null,""read"":""0"",""vision"":""rnd"",""efforted"":0,""history"":[],""desc"":""2023-05-09 10:47:13, \u7531 \u003Cstrong\u003Elin.hao\u003C\/strong\u003E \u521b\u5efa\u3002\n""}",http://54.158.1.10:30001/api.php/v1/bugs/6085 [...]
+1024,"{""ConnectionId"":1,""ProductId"":22,""ProjectId"":0}","{""id"":37654,""objectType"":""bug"",""objectID"":6094,""product"":"",22,"",""project"":48,""execution"":49,""actor"":""admin"",""action"":""opened"",""date"":""2023-05-30 21:02:00"",""comment"":"""",""extra"":"""",""read"":""1"",""vision"":""rnd"",""efforted"":0,""history"":[],""desc"":""2023-05-30 21:02:00, \u7531 \u003Cstrong\u003Eadmin\u003C\/strong\u003E \u521b\u5efa\u3002\n""}",http://54.158.1.10:30001/api.php/v1/bugs/60 [...]
+1025,"{""ConnectionId"":1,""ProductId"":22,""ProjectId"":0}","{""id"":37655,""objectType"":""bug"",""objectID"":6094,""product"":"",22,"",""project"":0,""execution"":49,""actor"":""Administrator"",""action"":""gitcommited"",""date"":""2023-05-30 21:02:28"",""comment"":""\u7248\u672c: #9ed3c33883\u003Cbr \/\u003EBug#6094\uff0c\u6d4b\u8bd5\u7985\u9053bug"",""extra"":""9ed3c33883"",""read"":""1"",""vision"":""rnd"",""efforted"":0,""history"":[{""id"":87,""action"":37655,""field"":""git"","" [...]
+1026,"{""ConnectionId"":1,""ProductId"":22,""ProjectId"":0}","{""id"":37656,""objectType"":""bug"",""objectID"":6094,""product"":"",22,"",""project"":48,""execution"":49,""actor"":""Administrator"",""action"":""linked2revision"",""date"":""2023-05-30 21:03:07"",""comment"":"""",""extra"":""\u003Ca href='http:\/\/54.158.1.10:30001\/repo-revision-1-0-9ed3c33883a203e0cc90c44be40d105b23d98156.json' data-app='product'\u003E9ed3c33883\u003C\/a\u003E\n"",""read"":""1"",""vision"":""rnd"",""eff [...]
diff --git a/backend/plugins/zentao/e2e/raw_tables/_raw_zentao_api_bug_repo_commits.csv b/backend/plugins/zentao/e2e/raw_tables/_raw_zentao_api_bug_repo_commits.csv
new file mode 100644
index 000000000..5c44aefc0
--- /dev/null
+++ b/backend/plugins/zentao/e2e/raw_tables/_raw_zentao_api_bug_repo_commits.csv
@@ -0,0 +1,2 @@
+id,params,data,url,input,created_at
+1,"{""ConnectionId"":1,""ProductId"":22,""ProjectId"":0}","{""title"":""\u4ee3\u7801-\u67e5\u770b\u4fee\u8ba2"",""log"":{""revision"":""9ed3c33883a203e0cc90c44be40d105b23d98156"",""committer"":""Administrator"",""time"":""2023-05-30 21:02:28"",""comment"":""Bug #<a href='\/bug-view-6094.json' >6094<\/a>\n\uff0c\u6d4b\u8bd5\u7985\u9053bug"",""change"":{""\/test.yaml"":{""action"":""M"",""kind"":""file"",""oldPath"":""\/test.yaml""}},""commit"":""3""},""repo"":{""id"":""1"",""product"":"" [...]
diff --git a/backend/plugins/zentao/e2e/raw_tables/_raw_zentao_api_story_commits.csv b/backend/plugins/zentao/e2e/raw_tables/_raw_zentao_api_story_commits.csv
new file mode 100644
index 000000000..6e95c27d1
--- /dev/null
+++ b/backend/plugins/zentao/e2e/raw_tables/_raw_zentao_api_story_commits.csv
@@ -0,0 +1,21 @@
+id,params,data,url,input,created_at
+1,"{""ConnectionId"":1,""ProductId"":31,""ProjectId"":0}","{""id"":37626,""objectType"":""story"",""objectID"":4564,""product"":"",31,"",""project"":0,""execution"":0,""actor"":""admin"",""action"":""opened"",""date"":""2023-05-18 18:58:51"",""comment"":"""",""extra"":"""",""read"":""0"",""vision"":""rnd"",""efforted"":0,""history"":[],""desc"":""2023-05-18 18:58:51, \u7531 \u003Cstrong\u003Eadmin\u003C\/strong\u003E \u521b\u5efa\u3002\n""}",http://54.158.1.10:30001/api.php/v1/stories/45 [...]
+2,"{""ConnectionId"":1,""ProductId"":31,""ProjectId"":0}","{""id"":37627,""objectType"":""story"",""objectID"":4564,""product"":"",31,"",""project"":0,""execution"":0,""actor"":""admin"",""action"":""submitreview"",""date"":""2023-05-18 18:58:51"",""comment"":"""",""extra"":"""",""read"":""0"",""vision"":""rnd"",""efforted"":0,""history"":[],""desc"":""2023-05-18 18:58:51, \u7531 \u003Cstrong\u003Eadmin\u003C\/strong\u003E \u63d0\u4ea4\u8bc4\u5ba1\u3002\n""}",http://54.158.1.10:30001/api [...]
+3,"{""ConnectionId"":1,""ProductId"":31,""ProjectId"":0}","{""id"":37628,""objectType"":""story"",""objectID"":4564,""product"":"",31,"",""project"":0,""execution"":0,""actor"":""admin"",""action"":""reviewed"",""date"":""2023-05-18 18:59:04"",""comment"":"""",""extra"":""Pass"",""read"":""0"",""vision"":""rnd"",""efforted"":0,""history"":[{""id"":72,""action"":37628,""field"":""reviewedDate"",""old"":""0000-00-00 00:00:00"",""new"":""2023-05-18 18:58:59"",""diff"":"""",""fieldName"":""\ [...]
+4,"{""ConnectionId"":1,""ProductId"":31,""ProjectId"":0}","{""id"":37629,""objectType"":""story"",""objectID"":4564,""product"":"",31,"",""project"":0,""execution"":0,""actor"":""\u7cfb\u7edf"",""action"":""reviewpassed"",""date"":""2023-05-18 18:59:04"",""comment"":"""",""extra"":"""",""read"":""0"",""vision"":""rnd"",""efforted"":0,""history"":[],""desc"":""2023-05-18 18:59:04, \u7531 \u003Cstrong\u003E\u7cfb\u7edf\u003C\/strong\u003E \u5224\u5b9a\uff0c\u7ed3\u679c\u4e3a \u003Cstrong\u [...]
+5,"{""ConnectionId"":1,""ProductId"":31,""ProjectId"":0}","{""id"":37630,""objectType"":""story"",""objectID"":4564,""product"":"",31,"",""project"":0,""execution"":0,""actor"":""admin"",""action"":""assigned"",""date"":""2023-05-18 18:59:12"",""comment"":"""",""extra"":""admin"",""read"":""0"",""vision"":""rnd"",""efforted"":0,""history"":[{""id"":75,""action"":37630,""field"":""assignedTo"",""old"":"""",""new"":""admin"",""diff"":"""",""fieldName"":""\u6307\u6d3e\u7ed9""}],""desc"":""2 [...]
+6,"{""ConnectionId"":1,""ProductId"":31,""ProjectId"":0}","{""id"":37631,""objectType"":""story"",""objectID"":4564,""product"":"",31,"",""project"":65,""execution"":0,""actor"":""admin"",""action"":""linked2project"",""date"":""2023-05-18 19:07:48"",""comment"":"""",""extra"":""\u003Ca href='http:\/\/54.158.1.10:30001\/project-index-65.json' \u003E\u66f2\u7387\u5f15\u64ce\u9879\u76ee-\u770b\u677f\u003C\/a\u003E\n"",""read"":""0"",""vision"":""rnd"",""efforted"":0,""history"":[],""desc" [...]
+7,"{""ConnectionId"":1,""ProductId"":31,""ProjectId"":0}","{""id"":37632,""objectType"":""story"",""objectID"":4564,""product"":"",31,"",""project"":65,""execution"":68,""actor"":""admin"",""action"":""linked2kanban"",""date"":""2023-05-18 19:07:48"",""comment"":"""",""extra"":""\u003Ca href='http:\/\/54.158.1.10:30001\/execution-kanban-68.json' data-app='execution'\u003E\u524d\u7aef\u770b\u677f\u003C\/a\u003E\n"",""read"":""0"",""vision"":""rnd"",""efforted"":0,""history"":[],""desc"":" [...]
+8,"{""ConnectionId"":1,""ProductId"":31,""ProjectId"":0}","{""id"":37633,""objectType"":""story"",""objectID"":4564,""product"":"",31,"",""project"":65,""execution"":68,""actor"":""admin"",""action"":""edited"",""date"":""2023-05-18 19:08:21"",""comment"":"""",""extra"":"""",""read"":""0"",""vision"":""rnd"",""efforted"":0,""history"":[{""id"":76,""action"":37633,""field"":""stage"",""old"":""projected"",""new"":""developing"",""diff"":"""",""fieldName"":""\u6240\u5904\u9636\u6bb5""}],"" [...]
+9,"{""ConnectionId"":1,""ProductId"":31,""ProjectId"":0}","{""id"":37651,""objectType"":""story"",""objectID"":4564,""product"":""31"",""project"":0,""execution"":0,""actor"":""Administrator"",""action"":""gitcommited"",""date"":""2023-05-30 20:58:31"",""comment"":""\u7248\u672c: #659ca32343\u003Cbr \/\u003Estory#4564,\u7985\u9053\u6d4b\u8bd5"",""extra"":""659ca32343"",""read"":""0"",""vision"":""rnd"",""efforted"":0,""history"":[{""id"":86,""action"":37651,""field"":""git"",""old"":"""" [...]
+10,"{""ConnectionId"":1,""ProductId"":31,""ProjectId"":0}","{""id"":37652,""objectType"":""story"",""objectID"":4564,""product"":"",31,"",""project"":65,""execution"":68,""actor"":""Administrator"",""action"":""linked2revision"",""date"":""2023-05-30 20:58:39"",""comment"":"""",""extra"":""\u003Ca href='http:\/\/54.158.1.10:30001\/repo-revision-1-0-659ca323434f22ed01145de9d52eeb0d8288d8bb.json' \u003E659ca32343\u003C\/a\u003E\n"",""read"":""0"",""vision"":""rnd"",""efforted"":0,""histor [...]
+11,"{""ConnectionId"":1,""ProductId"":31,""ProjectId"":0}","{""id"":37626,""objectType"":""story"",""objectID"":4564,""product"":"",31,"",""project"":0,""execution"":0,""actor"":""admin"",""action"":""opened"",""date"":""2023-05-18 18:58:51"",""comment"":"""",""extra"":"""",""read"":""0"",""vision"":""rnd"",""efforted"":0,""history"":[],""desc"":""2023-05-18 18:58:51, \u7531 \u003Cstrong\u003Eadmin\u003C\/strong\u003E \u521b\u5efa\u3002\n""}",http://54.158.1.10:30001/api.php/v1/stories/4 [...]
+12,"{""ConnectionId"":1,""ProductId"":31,""ProjectId"":0}","{""id"":37627,""objectType"":""story"",""objectID"":4564,""product"":"",31,"",""project"":0,""execution"":0,""actor"":""admin"",""action"":""submitreview"",""date"":""2023-05-18 18:58:51"",""comment"":"""",""extra"":"""",""read"":""0"",""vision"":""rnd"",""efforted"":0,""history"":[],""desc"":""2023-05-18 18:58:51, \u7531 \u003Cstrong\u003Eadmin\u003C\/strong\u003E \u63d0\u4ea4\u8bc4\u5ba1\u3002\n""}",http://54.158.1.10:30001/ap [...]
+13,"{""ConnectionId"":1,""ProductId"":31,""ProjectId"":0}","{""id"":37628,""objectType"":""story"",""objectID"":4564,""product"":"",31,"",""project"":0,""execution"":0,""actor"":""admin"",""action"":""reviewed"",""date"":""2023-05-18 18:59:04"",""comment"":"""",""extra"":""Pass"",""read"":""0"",""vision"":""rnd"",""efforted"":0,""history"":[{""id"":72,""action"":37628,""field"":""reviewedDate"",""old"":""0000-00-00 00:00:00"",""new"":""2023-05-18 18:58:59"",""diff"":"""",""fieldName"":"" [...]
+14,"{""ConnectionId"":1,""ProductId"":31,""ProjectId"":0}","{""id"":37629,""objectType"":""story"",""objectID"":4564,""product"":"",31,"",""project"":0,""execution"":0,""actor"":""\u7cfb\u7edf"",""action"":""reviewpassed"",""date"":""2023-05-18 18:59:04"",""comment"":"""",""extra"":"""",""read"":""0"",""vision"":""rnd"",""efforted"":0,""history"":[],""desc"":""2023-05-18 18:59:04, \u7531 \u003Cstrong\u003E\u7cfb\u7edf\u003C\/strong\u003E \u5224\u5b9a\uff0c\u7ed3\u679c\u4e3a \u003Cstrong\ [...]
+15,"{""ConnectionId"":1,""ProductId"":31,""ProjectId"":0}","{""id"":37630,""objectType"":""story"",""objectID"":4564,""product"":"",31,"",""project"":0,""execution"":0,""actor"":""admin"",""action"":""assigned"",""date"":""2023-05-18 18:59:12"",""comment"":"""",""extra"":""admin"",""read"":""0"",""vision"":""rnd"",""efforted"":0,""history"":[{""id"":75,""action"":37630,""field"":""assignedTo"",""old"":"""",""new"":""admin"",""diff"":"""",""fieldName"":""\u6307\u6d3e\u7ed9""}],""desc"":"" [...]
+16,"{""ConnectionId"":1,""ProductId"":31,""ProjectId"":0}","{""id"":37631,""objectType"":""story"",""objectID"":4564,""product"":"",31,"",""project"":65,""execution"":0,""actor"":""admin"",""action"":""linked2project"",""date"":""2023-05-18 19:07:48"",""comment"":"""",""extra"":""\u003Ca href='http:\/\/54.158.1.10:30001\/project-index-65.json' \u003E\u66f2\u7387\u5f15\u64ce\u9879\u76ee-\u770b\u677f\u003C\/a\u003E\n"",""read"":""0"",""vision"":""rnd"",""efforted"":0,""history"":[],""desc [...]
+17,"{""ConnectionId"":1,""ProductId"":31,""ProjectId"":0}","{""id"":37632,""objectType"":""story"",""objectID"":4564,""product"":"",31,"",""project"":65,""execution"":68,""actor"":""admin"",""action"":""linked2kanban"",""date"":""2023-05-18 19:07:48"",""comment"":"""",""extra"":""\u003Ca href='http:\/\/54.158.1.10:30001\/execution-kanban-68.json' data-app='execution'\u003E\u524d\u7aef\u770b\u677f\u003C\/a\u003E\n"",""read"":""0"",""vision"":""rnd"",""efforted"":0,""history"":[],""desc"": [...]
+18,"{""ConnectionId"":1,""ProductId"":31,""ProjectId"":0}","{""id"":37633,""objectType"":""story"",""objectID"":4564,""product"":"",31,"",""project"":65,""execution"":68,""actor"":""admin"",""action"":""edited"",""date"":""2023-05-18 19:08:21"",""comment"":"""",""extra"":"""",""read"":""0"",""vision"":""rnd"",""efforted"":0,""history"":[{""id"":76,""action"":37633,""field"":""stage"",""old"":""projected"",""new"":""developing"",""diff"":"""",""fieldName"":""\u6240\u5904\u9636\u6bb5""}]," [...]
+19,"{""ConnectionId"":1,""ProductId"":31,""ProjectId"":0}","{""id"":37651,""objectType"":""story"",""objectID"":4564,""product"":""31"",""project"":0,""execution"":0,""actor"":""Administrator"",""action"":""gitcommited"",""date"":""2023-05-30 20:58:31"",""comment"":""\u7248\u672c: #659ca32343\u003Cbr \/\u003Estory#4564,\u7985\u9053\u6d4b\u8bd5"",""extra"":""659ca32343"",""read"":""0"",""vision"":""rnd"",""efforted"":0,""history"":[{""id"":86,""action"":37651,""field"":""git"",""old"":""" [...]
+20,"{""ConnectionId"":1,""ProductId"":31,""ProjectId"":0}","{""id"":37652,""objectType"":""story"",""objectID"":4564,""product"":"",31,"",""project"":65,""execution"":68,""actor"":""Administrator"",""action"":""linked2revision"",""date"":""2023-05-30 20:58:39"",""comment"":"""",""extra"":""\u003Ca href='http:\/\/54.158.1.10:30001\/repo-revision-1-0-659ca323434f22ed01145de9d52eeb0d8288d8bb.json' \u003E659ca32343\u003C\/a\u003E\n"",""read"":""0"",""vision"":""rnd"",""efforted"":0,""histor [...]
diff --git a/backend/plugins/zentao/e2e/raw_tables/_raw_zentao_api_story_repo_commits.csv b/backend/plugins/zentao/e2e/raw_tables/_raw_zentao_api_story_repo_commits.csv
new file mode 100644
index 000000000..defeb0f8e
--- /dev/null
+++ b/backend/plugins/zentao/e2e/raw_tables/_raw_zentao_api_story_repo_commits.csv
@@ -0,0 +1,2 @@
+id,params,data,url,input,created_at
+3,"{""ConnectionId"":1,""ProductId"":31,""ProjectId"":0}","{""title"":""\u4ee3\u7801-\u67e5\u770b\u4fee\u8ba2"",""log"":{""revision"":""659ca323434f22ed01145de9d52eeb0d8288d8bb"",""committer"":""Administrator"",""time"":""2023-05-30 20:58:31"",""comment"":""story #<a href='\/story-view-4564.json' >4564<\/a>\n,\u7985\u9053\u6d4b\u8bd5"",""change"":{""\/test.yaml"":{""action"":""M"",""kind"":""file"",""oldPath"":""\/test.yaml""}},""commit"":""2""},""repo"":{""id"":""1"",""product"":""22"" [...]
diff --git a/backend/plugins/zentao/e2e/raw_tables/_raw_zentao_api_task_commits.csv b/backend/plugins/zentao/e2e/raw_tables/_raw_zentao_api_task_commits.csv
new file mode 100644
index 000000000..3199bd471
--- /dev/null
+++ b/backend/plugins/zentao/e2e/raw_tables/_raw_zentao_api_task_commits.csv
@@ -0,0 +1,7 @@
+id,params,data,url,input,created_at
+8554,"{""ConnectionId"":1,""ProductId"":0,""ProjectId"":48}","{""id"":381,""objectType"":""task"",""objectID"":135,""product"":"""",""project"":0,""execution"":0,""actor"":""admin"",""action"":""Opened"",""date"":""2021-01-27 04:17:22"",""comment"":"""",""extra"":null,""read"":""1"",""vision"":""rnd"",""efforted"":0,""history"":[],""desc"":""2021-01-27 04:17:22, \u7531 \u003Cstrong\u003Eadmin\u003C\/strong\u003E \u521b\u5efa\u3002\n""}",http://54.158.1.10:30001/api.php/v1/tasks/135,"{""i [...]
+8555,"{""ConnectionId"":1,""ProductId"":0,""ProjectId"":48}","{""id"":37658,""objectType"":""task"",""objectID"":135,""product"":"",22,"",""project"":0,""execution"":49,""actor"":""Administrator"",""action"":""gitcommited"",""date"":""2023-05-30 21:05:48"",""comment"":""\u7248\u672c: #ad92faa87e\u003Cbr \/\u003Etask#135, \u7985\u9053task\u6d4b\u8bd5"",""extra"":""ad92faa87e"",""read"":""1"",""vision"":""rnd"",""efforted"":0,""history"":[{""id"":88,""action"":37658,""field"":""git"",""old [...]
+8556,"{""ConnectionId"":1,""ProductId"":0,""ProjectId"":48}","{""id"":37659,""objectType"":""task"",""objectID"":135,""product"":"",22,"",""project"":48,""execution"":49,""actor"":""Administrator"",""action"":""linked2revision"",""date"":""2023-05-30 21:05:56"",""comment"":"""",""extra"":""\u003Ca href='http:\/\/54.158.1.10:30001\/repo-revision-1-0-ad92faa87e6eaf121d382f0f775cd6b98c5e65fe.json' \u003Ead92faa87e\u003C\/a\u003E\n"",""read"":""1"",""vision"":""rnd"",""efforted"":0,""histor [...]
+8558,"{""ConnectionId"":1,""ProductId"":0,""ProjectId"":48}","{""id"":381,""objectType"":""task"",""objectID"":135,""product"":"""",""project"":0,""execution"":0,""actor"":""admin"",""action"":""Opened"",""date"":""2021-01-27 04:17:22"",""comment"":"""",""extra"":null,""read"":""1"",""vision"":""rnd"",""efforted"":0,""history"":[],""desc"":""2021-01-27 04:17:22, \u7531 \u003Cstrong\u003Eadmin\u003C\/strong\u003E \u521b\u5efa\u3002\n""}",http://54.158.1.10:30001/api.php/v1/tasks/135,"{""i [...]
+8559,"{""ConnectionId"":1,""ProductId"":0,""ProjectId"":48}","{""id"":37658,""objectType"":""task"",""objectID"":135,""product"":"",22,"",""project"":0,""execution"":49,""actor"":""Administrator"",""action"":""gitcommited"",""date"":""2023-05-30 21:05:48"",""comment"":""\u7248\u672c: #ad92faa87e\u003Cbr \/\u003Etask#135, \u7985\u9053task\u6d4b\u8bd5"",""extra"":""ad92faa87e"",""read"":""1"",""vision"":""rnd"",""efforted"":0,""history"":[{""id"":88,""action"":37658,""field"":""git"",""old [...]
+8560,"{""ConnectionId"":1,""ProductId"":0,""ProjectId"":48}","{""id"":37659,""objectType"":""task"",""objectID"":135,""product"":"",22,"",""project"":48,""execution"":49,""actor"":""Administrator"",""action"":""linked2revision"",""date"":""2023-05-30 21:05:56"",""comment"":"""",""extra"":""\u003Ca href='http:\/\/54.158.1.10:30001\/repo-revision-1-0-ad92faa87e6eaf121d382f0f775cd6b98c5e65fe.json' \u003Ead92faa87e\u003C\/a\u003E\n"",""read"":""1"",""vision"":""rnd"",""efforted"":0,""histor [...]
diff --git a/backend/plugins/zentao/e2e/raw_tables/_raw_zentao_api_task_repo_commits.csv b/backend/plugins/zentao/e2e/raw_tables/_raw_zentao_api_task_repo_commits.csv
new file mode 100644
index 000000000..6843d683e
--- /dev/null
+++ b/backend/plugins/zentao/e2e/raw_tables/_raw_zentao_api_task_repo_commits.csv
@@ -0,0 +1,2 @@
+id,params,data,url,input,created_at
+1,"{""ConnectionId"":1,""ProductId"":0,""ProjectId"":48}","{""title"":""\u4ee3\u7801-\u67e5\u770b\u4fee\u8ba2"",""log"":{""revision"":""ad92faa87e6eaf121d382f0f775cd6b98c5e65fe"",""committer"":""Administrator"",""time"":""2023-05-30 21:05:48"",""comment"":""task #<a href='\/task-view-135.json' >135<\/a>\n, \u7985\u9053task\u6d4b\u8bd5"",""change"":{""\/test.yaml"":{""action"":""M"",""kind"":""file"",""oldPath"":""\/test.yaml""}},""commit"":""4""},""repo"":{""id"":""1"",""product"":""22" [...]
diff --git a/backend/plugins/zentao/e2e/snapshot_tables/_tool_zentao_bug_commits.csv b/backend/plugins/zentao/e2e/snapshot_tables/_tool_zentao_bug_commits.csv
new file mode 100644
index 000000000..57a2f5e55
--- /dev/null
+++ b/backend/plugins/zentao/e2e/snapshot_tables/_tool_zentao_bug_commits.csv
@@ -0,0 +1,3 @@
+connection_id,id,object_type,object_id,product,project,execution,actor,action,date,comment,extra,host,repo_revision,action_read,vision,efforted,action_desc
+1,37656,bug,6094,22,0,49,Administrator,linked2revision,2023-05-30 21:03:07,,http://54.158.1.10:30001/repo-revision-1-0-9ed3c33883a203e0cc90c44be40d105b23d98156.json,54.158.1.10:30001,/repo-revision-1-0-9ed3c33883a203e0cc90c44be40d105b23d98156.json,1,rnd,0,"2023-05-30 21:03:07, 由 <strong>Administrator</strong> 关联到代码提交 <strong><a href='http://54.158.1.10:30001/repo-revision-1-0-9ed3c33883a203e0cc90c44be40d105b23d98156.json' data-app='product'>9ed3c33883</a>
+</strong>."
diff --git a/backend/plugins/zentao/e2e/snapshot_tables/_tool_zentao_bug_repo_commits.csv b/backend/plugins/zentao/e2e/snapshot_tables/_tool_zentao_bug_repo_commits.csv
new file mode 100644
index 000000000..9b0caab4a
--- /dev/null
+++ b/backend/plugins/zentao/e2e/snapshot_tables/_tool_zentao_bug_repo_commits.csv
@@ -0,0 +1,2 @@
+connection_id,issue_id,repo_url,commit_sha,product,project
+1,6094,http://devlake.gitlab.com/root/zentao-test,9ed3c33883a203e0cc90c44be40d105b23d98156,22,0
diff --git a/backend/plugins/zentao/e2e/snapshot_tables/_tool_zentao_story_commits.csv b/backend/plugins/zentao/e2e/snapshot_tables/_tool_zentao_story_commits.csv
new file mode 100644
index 000000000..caa2b1120
--- /dev/null
+++ b/backend/plugins/zentao/e2e/snapshot_tables/_tool_zentao_story_commits.csv
@@ -0,0 +1,3 @@
+connection_id,id,object_type,object_id,product,project,execution,actor,action,date,comment,extra,host,repo_revision,action_read,vision,efforted,action_desc
+1,37652,story,4564,31,0,68,Administrator,linked2revision,2023-05-30 20:58:39,,http://54.158.1.10:30001/repo-revision-1-0-659ca323434f22ed01145de9d52eeb0d8288d8bb.json,54.158.1.10:30001,/repo-revision-1-0-659ca323434f22ed01145de9d52eeb0d8288d8bb.json,0,rnd,0,"2023-05-30 20:58:39, 由 <strong>Administrator</strong> 关联到代码提交 <strong><a href='http://54.158.1.10:30001/repo-revision-1-0-659ca323434f22ed01145de9d52eeb0d8288d8bb.json' >659ca32343</a>
+</strong>"
diff --git a/backend/plugins/zentao/e2e/snapshot_tables/_tool_zentao_story_repo_commits.csv b/backend/plugins/zentao/e2e/snapshot_tables/_tool_zentao_story_repo_commits.csv
new file mode 100644
index 000000000..6954fde26
--- /dev/null
+++ b/backend/plugins/zentao/e2e/snapshot_tables/_tool_zentao_story_repo_commits.csv
@@ -0,0 +1,2 @@
+connection_id,issue_id,repo_url,commit_sha,product,project
+1,4564,http://devlake.gitlab.com/root/zentao-test,659ca323434f22ed01145de9d52eeb0d8288d8bb,31,0
diff --git a/backend/plugins/zentao/e2e/snapshot_tables/_tool_zentao_task_commits.csv b/backend/plugins/zentao/e2e/snapshot_tables/_tool_zentao_task_commits.csv
new file mode 100644
index 000000000..79b8978f7
--- /dev/null
+++ b/backend/plugins/zentao/e2e/snapshot_tables/_tool_zentao_task_commits.csv
@@ -0,0 +1,3 @@
+connection_id,id,object_type,object_id,product,project,execution,actor,action,date,comment,extra,host,repo_revision,action_read,vision,efforted,action_desc
+1,37659,task,135,0,48,49,Administrator,linked2revision,2023-05-30 21:05:56,,http://54.158.1.10:30001/repo-revision-1-0-ad92faa87e6eaf121d382f0f775cd6b98c5e65fe.json,54.158.1.10:30001,/repo-revision-1-0-ad92faa87e6eaf121d382f0f775cd6b98c5e65fe.json,1,rnd,0,"2023-05-30 21:05:56, 由 <strong>Administrator</strong> 关联到代码提交 <strong><a href='http://54.158.1.10:30001/repo-revision-1-0-ad92faa87e6eaf121d382f0f775cd6b98c5e65fe.json' >ad92faa87e</a>
+</strong>。"
diff --git a/backend/plugins/zentao/e2e/snapshot_tables/_tool_zentao_task_repo_commits.csv b/backend/plugins/zentao/e2e/snapshot_tables/_tool_zentao_task_repo_commits.csv
new file mode 100644
index 000000000..b098b25ea
--- /dev/null
+++ b/backend/plugins/zentao/e2e/snapshot_tables/_tool_zentao_task_repo_commits.csv
@@ -0,0 +1,2 @@
+connection_id,issue_id,repo_url,commit_sha,product,project
+1,135,http://devlake.gitlab.com/root/zentao-test,ad92faa87e6eaf121d382f0f775cd6b98c5e65fe,0,48
diff --git a/backend/plugins/zentao/e2e/snapshot_tables/issue_bug_repo_commits.csv b/backend/plugins/zentao/e2e/snapshot_tables/issue_bug_repo_commits.csv
new file mode 100644
index 000000000..959428ba5
--- /dev/null
+++ b/backend/plugins/zentao/e2e/snapshot_tables/issue_bug_repo_commits.csv
@@ -0,0 +1,2 @@
+issue_id,repo_url,commit_sha,host,namespace,repo_name
+zentao:ZentaoBugRepoCommit:1:6094,http://devlake.gitlab.com/root/zentao-test,9ed3c33883a203e0cc90c44be40d105b23d98156,devlake.gitlab.com,root,zentao-test
diff --git a/backend/plugins/zentao/e2e/snapshot_tables/issue_story_repo_commits.csv b/backend/plugins/zentao/e2e/snapshot_tables/issue_story_repo_commits.csv
new file mode 100644
index 000000000..315b330dd
--- /dev/null
+++ b/backend/plugins/zentao/e2e/snapshot_tables/issue_story_repo_commits.csv
@@ -0,0 +1,2 @@
+issue_id,repo_url,commit_sha,host,namespace,repo_name
+zentao:ZentaoStoryRepoCommit:1:4564,http://devlake.gitlab.com/root/zentao-test,659ca323434f22ed01145de9d52eeb0d8288d8bb,devlake.gitlab.com,root,zentao-test
diff --git a/backend/plugins/zentao/e2e/snapshot_tables/issue_task_repo_commits.csv b/backend/plugins/zentao/e2e/snapshot_tables/issue_task_repo_commits.csv
new file mode 100644
index 000000000..0e971b834
--- /dev/null
+++ b/backend/plugins/zentao/e2e/snapshot_tables/issue_task_repo_commits.csv
@@ -0,0 +1,2 @@
+issue_id,repo_url,commit_sha,host,namespace,repo_name
+zentao:ZentaoTaskRepoCommit:1:135,http://devlake.gitlab.com/root/zentao-test,ad92faa87e6eaf121d382f0f775cd6b98c5e65fe,devlake.gitlab.com,root,zentao-test
diff --git a/backend/plugins/zentao/e2e/story_commits_test.go b/backend/plugins/zentao/e2e/story_commits_test.go
new file mode 100644
index 000000000..64fa9bdf5
--- /dev/null
+++ b/backend/plugins/zentao/e2e/story_commits_test.go
@@ -0,0 +1,76 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+ "testing"
+
+ "github.com/apache/incubator-devlake/core/models/common"
+ "github.com/apache/incubator-devlake/core/models/domainlayer/crossdomain"
+ "github.com/apache/incubator-devlake/helpers/e2ehelper"
+ "github.com/apache/incubator-devlake/plugins/zentao/impl"
+ "github.com/apache/incubator-devlake/plugins/zentao/models"
+ "github.com/apache/incubator-devlake/plugins/zentao/tasks"
+)
+
+func TestZentaoStoryCommitsDataFlow(t *testing.T) {
+
+ var zentao impl.Zentao
+ dataflowTester := e2ehelper.NewDataFlowTester(t, "zentao", zentao)
+
+ taskData := &tasks.ZentaoTaskData{
+ Options: &tasks.ZentaoOptions{
+ ConnectionId: 1,
+ ProjectId: 0,
+ ProductId: 31,
+ },
+ }
+
+ // import _raw_zentao_api_story_commits raw data table
+ dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_zentao_api_story_commits.csv",
+ "_raw_zentao_api_story_commits")
+
+ // verify story commit extraction
+ dataflowTester.FlushTabler(&models.ZentaoStoryCommit{})
+ dataflowTester.Subtask(tasks.ExtractStoryCommitsMeta, taskData)
+ dataflowTester.VerifyTableWithOptions(&models.ZentaoStoryCommit{}, e2ehelper.TableOptions{
+ CSVRelPath: "./snapshot_tables/_tool_zentao_story_commits.csv",
+ IgnoreTypes: []interface{}{common.NoPKModel{}},
+ })
+
+ // import _raw_zentao_api_story_repo_commits raw data table
+ dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_zentao_api_story_repo_commits.csv",
+ "_raw_zentao_api_story_repo_commits")
+
+ // verify story repo commit extraction
+ dataflowTester.FlushTabler(&models.ZentaoStoryRepoCommit{})
+ dataflowTester.Subtask(tasks.ExtractStoryRepoCommitsMeta, taskData)
+ dataflowTester.VerifyTableWithOptions(&models.ZentaoStoryRepoCommit{}, e2ehelper.TableOptions{
+ CSVRelPath: "./snapshot_tables/_tool_zentao_story_repo_commits.csv",
+ IgnoreTypes: []interface{}{common.NoPKModel{}},
+ })
+
+ // verify story repo commit conversion
+ dataflowTester.FlushTabler(&crossdomain.IssueRepoCommit{})
+ dataflowTester.Subtask(tasks.ConvertStoryRepoCommitsMeta, taskData)
+ dataflowTester.VerifyTableWithOptions(&crossdomain.IssueRepoCommit{}, e2ehelper.TableOptions{
+ CSVRelPath: "./snapshot_tables/issue_story_repo_commits.csv",
+ IgnoreTypes: []interface{}{common.NoPKModel{}},
+ })
+
+}
diff --git a/backend/plugins/zentao/e2e/task_commits_test.go b/backend/plugins/zentao/e2e/task_commits_test.go
new file mode 100644
index 000000000..478857f5e
--- /dev/null
+++ b/backend/plugins/zentao/e2e/task_commits_test.go
@@ -0,0 +1,76 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+ "testing"
+
+ "github.com/apache/incubator-devlake/core/models/common"
+ "github.com/apache/incubator-devlake/core/models/domainlayer/crossdomain"
+ "github.com/apache/incubator-devlake/helpers/e2ehelper"
+ "github.com/apache/incubator-devlake/plugins/zentao/impl"
+ "github.com/apache/incubator-devlake/plugins/zentao/models"
+ "github.com/apache/incubator-devlake/plugins/zentao/tasks"
+)
+
+func TestZentaoTaskCommitsDataFlow(t *testing.T) {
+
+ var zentao impl.Zentao
+ dataflowTester := e2ehelper.NewDataFlowTester(t, "zentao", zentao)
+
+ taskData := &tasks.ZentaoTaskData{
+ Options: &tasks.ZentaoOptions{
+ ConnectionId: 1,
+ ProjectId: 48,
+ ProductId: 0,
+ },
+ }
+
+ // import _raw_zentao_api_task_commits raw data table
+ dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_zentao_api_task_commits.csv",
+ "_raw_zentao_api_task_commits")
+
+ // verify task commit extraction
+ dataflowTester.FlushTabler(&models.ZentaoTaskCommit{})
+ dataflowTester.Subtask(tasks.ExtractTaskCommitsMeta, taskData)
+ dataflowTester.VerifyTableWithOptions(&models.ZentaoTaskCommit{}, e2ehelper.TableOptions{
+ CSVRelPath: "./snapshot_tables/_tool_zentao_task_commits.csv",
+ IgnoreTypes: []interface{}{common.NoPKModel{}},
+ })
+
+ // import _raw_zentao_api_task_repo_commits raw data table
+ dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_zentao_api_task_repo_commits.csv",
+ "_raw_zentao_api_task_repo_commits")
+
+ // verify task repo commit extraction
+ dataflowTester.FlushTabler(&models.ZentaoTaskRepoCommit{})
+ dataflowTester.Subtask(tasks.ExtractTaskRepoCommitsMeta, taskData)
+ dataflowTester.VerifyTableWithOptions(&models.ZentaoTaskRepoCommit{}, e2ehelper.TableOptions{
+ CSVRelPath: "./snapshot_tables/_tool_zentao_task_repo_commits.csv",
+ IgnoreTypes: []interface{}{common.NoPKModel{}},
+ })
+
+ // verify task repo commit conversion
+ dataflowTester.FlushTabler(&crossdomain.IssueRepoCommit{})
+ dataflowTester.Subtask(tasks.ConvertTaskRepoCommitsMeta, taskData)
+ dataflowTester.VerifyTableWithOptions(&crossdomain.IssueRepoCommit{}, e2ehelper.TableOptions{
+ CSVRelPath: "./snapshot_tables/issue_task_repo_commits.csv",
+ IgnoreTypes: []interface{}{common.NoPKModel{}},
+ })
+
+}
diff --git a/backend/plugins/zentao/impl/impl.go b/backend/plugins/zentao/impl/impl.go
index d10eb47ec..922a0456b 100644
--- a/backend/plugins/zentao/impl/impl.go
+++ b/backend/plugins/zentao/impl/impl.go
@@ -85,7 +85,24 @@ func (p Zentao) SubTaskMetas() []plugin.SubTaskMeta {
tasks.CollectDepartmentMeta,
tasks.ExtractDepartmentMeta,
tasks.ConvertDepartmentMeta,
+
tasks.CollectBugCommitsMeta,
+ tasks.ExtractBugCommitsMeta,
+ tasks.CollectBugRepoCommitsMeta,
+ tasks.ExtractBugRepoCommitsMeta,
+ tasks.ConvertBugRepoCommitsMeta,
+
+ tasks.CollectStoryCommitsMeta,
+ tasks.ExtractStoryCommitsMeta,
+ tasks.CollectStoryRepoCommitsMeta,
+ tasks.ExtractStoryRepoCommitsMeta,
+ tasks.ConvertStoryRepoCommitsMeta,
+
+ tasks.CollectTaskCommitsMeta,
+ tasks.ExtractTaskCommitsMeta,
+ tasks.CollectTaskRepoCommitsMeta,
+ tasks.ExtractTaskRepoCommitsMeta,
+ tasks.ConvertTaskRepoCommitsMeta,
}
}
diff --git a/backend/plugins/zentao/models/archived/bug_commits.go b/backend/plugins/zentao/models/archived/bug_commits.go
index 21a2ad090..7bd02afe7 100644
--- a/backend/plugins/zentao/models/archived/bug_commits.go
+++ b/backend/plugins/zentao/models/archived/bug_commits.go
@@ -21,15 +21,42 @@ import (
"github.com/apache/incubator-devlake/core/models/migrationscripts/archived"
)
-type ZentaoBugCommits struct {
+type ZentaoBugCommit struct {
archived.NoPKModel
- ConnectionId uint64 `gorm:"primaryKey;type:BIGINT NOT NULL"`
- ID int64 `json:"id" gorm:"primaryKey;type:BIGINT NOT NULL;autoIncrement:false"`
- Project int64 `json:"project"`
- Product int64 `json:"product"`
- Actions []string `gorm:"type:json;serializer:json" json:"actions" mapstructure:"actions"`
+ ConnectionId uint64 `gorm:"primaryKey;type:BIGINT NOT NULL"`
+ ID int `json:"id" gorm:"primaryKey;type:BIGINT NOT NULL;autoIncrement:false"`
+ ObjectType string `json:"objectType"`
+ ObjectID int `json:"objectID"`
+ Product int64 `json:"product"`
+ Project int64 `json:"project"`
+ Execution int `json:"execution"`
+ Actor string `json:"actor"`
+ Action string `json:"action"`
+ Date string `json:"date"`
+ Comment string `json:"comment"`
+ Extra string `json:"extra"`
+ Host string `json:"host"` //the host part of extra
+ RepoRevision string `json:"repoRevision"` // the repoRevisionJson part of extra
+ ActionRead string `json:"actionRead"`
+ Vision string `json:"vision"`
+ Efforted int `json:"efforted"`
+ ActionDesc string `json:"cctionDesc"`
}
-func (ZentaoBugCommits) TableName() string {
+func (ZentaoBugCommit) TableName() string {
return "_tool_zentao_bug_commits"
}
+
+type ZentaoBugRepoCommit struct {
+ archived.NoPKModel
+ ConnectionId uint64 `gorm:"primaryKey;type:BIGINT NOT NULL"`
+ Product int64 `json:"product"`
+ Project int64 `json:"project"`
+ IssueId string `gorm:"primaryKey;type:varchar(255)"` // the bug id
+ RepoUrl string `gorm:"primaryKey;type:varchar(255)"`
+ CommitSha string `gorm:"primaryKey;type:varchar(255)"`
+}
+
+func (ZentaoBugRepoCommit) TableName() string {
+ return "_tool_zentao_bug_repo_commits"
+}
diff --git a/backend/plugins/zentao/models/archived/story_commits.go b/backend/plugins/zentao/models/archived/story_commits.go
new file mode 100644
index 000000000..e4dc4d10f
--- /dev/null
+++ b/backend/plugins/zentao/models/archived/story_commits.go
@@ -0,0 +1,62 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package archived
+
+import (
+ "github.com/apache/incubator-devlake/core/models/migrationscripts/archived"
+)
+
+type ZentaoStoryCommit struct {
+ archived.NoPKModel
+ ConnectionId uint64 `gorm:"primaryKey;type:BIGINT NOT NULL"`
+ ID int `json:"id" gorm:"primaryKey;type:BIGINT NOT NULL;autoIncrement:false"`
+ ObjectType string `json:"objectType"`
+ ObjectID int `json:"objectID"`
+ Product int64 `json:"product"`
+ Project int64 `json:"project"`
+ Execution int `json:"execution"`
+ Actor string `json:"actor"`
+ Action string `json:"action"`
+ Date string `json:"date"`
+ Comment string `json:"comment"`
+ Extra string `json:"extra"`
+ Host string `json:"host"` //the host part of extra
+ RepoRevision string `json:"repoRevision"` // the repoRevisionJson part of extra
+ ActionRead string `json:"actionRead"`
+ Vision string `json:"vision"`
+ Efforted int `json:"efforted"`
+ ActionDesc string `json:"cctionDesc"`
+}
+
+func (ZentaoStoryCommit) TableName() string {
+ return "_tool_zentao_story_commits"
+}
+
+type ZentaoStoryRepoCommit struct {
+ archived.NoPKModel
+ ConnectionId uint64 `gorm:"primaryKey;type:BIGINT NOT NULL"`
+ Product int64 `json:"product"`
+ Project int64 `json:"project"`
+ IssueId string `gorm:"primaryKey;type:varchar(255)"` // the story id
+ RepoUrl string `gorm:"primaryKey;type:varchar(255)"`
+ CommitSha string `gorm:"primaryKey;type:varchar(255)"`
+}
+
+func (ZentaoStoryRepoCommit) TableName() string {
+ return "_tool_zentao_story_repo_commits"
+}
diff --git a/backend/plugins/zentao/models/archived/task_commits.go b/backend/plugins/zentao/models/archived/task_commits.go
new file mode 100644
index 000000000..198e0fef7
--- /dev/null
+++ b/backend/plugins/zentao/models/archived/task_commits.go
@@ -0,0 +1,62 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package archived
+
+import (
+ "github.com/apache/incubator-devlake/core/models/migrationscripts/archived"
+)
+
+type ZentaoTaskCommit struct {
+ archived.NoPKModel
+ ConnectionId uint64 `gorm:"primaryKey;type:BIGINT NOT NULL"`
+ ID int `json:"id" gorm:"primaryKey;type:BIGINT NOT NULL;autoIncrement:false"`
+ ObjectType string `json:"objectType"`
+ ObjectID int `json:"objectID"`
+ Product int64 `json:"product"`
+ Project int64 `json:"project"`
+ Execution int `json:"execution"`
+ Actor string `json:"actor"`
+ Action string `json:"action"`
+ Date string `json:"date"`
+ Comment string `json:"comment"`
+ Extra string `json:"extra"`
+ Host string `json:"host"` //the host part of extra
+ RepoRevision string `json:"repoRevision"` // the repoRevisionJson part of extra
+ ActionRead string `json:"actionRead"`
+ Vision string `json:"vision"`
+ Efforted int `json:"efforted"`
+ ActionDesc string `json:"cctionDesc"`
+}
+
+func (ZentaoTaskCommit) TableName() string {
+ return "_tool_zentao_task_commits"
+}
+
+type ZentaoTaskRepoCommit struct {
+ archived.NoPKModel
+ ConnectionId uint64 `gorm:"primaryKey;type:BIGINT NOT NULL"`
+ Product int64 `json:"product"`
+ Project int64 `json:"project"`
+ IssueId string `gorm:"primaryKey;type:varchar(255)"` // the task id
+ RepoUrl string `gorm:"primaryKey;type:varchar(255)"`
+ CommitSha string `gorm:"primaryKey;type:varchar(255)"`
+}
+
+func (ZentaoTaskRepoCommit) TableName() string {
+ return "_tool_zentao_task_repo_commits"
+}
diff --git a/backend/plugins/zentao/models/bug_commits.go b/backend/plugins/zentao/models/bug_commits.go
index 918b8fe06..ce78cfde4 100644
--- a/backend/plugins/zentao/models/bug_commits.go
+++ b/backend/plugins/zentao/models/bug_commits.go
@@ -22,21 +22,130 @@ import (
)
type ZentaoBugCommitsRes struct {
- ID int64 `json:"id"`
- Project int64 `json:"project"`
- Product int64 `json:"product"`
- Actions []string `gorm:"type:json;serializer:json" json:"actions" mapstructure:"actions"`
+ ID int `json:"id" gorm:"primaryKey;type:BIGINT NOT NULL;autoIncrement:false"`
+ ObjectType string `json:"objectType"`
+ ObjectID int `json:"objectID"`
+ Product string `json:"product"`
+ Project int `json:"project"`
+ Execution int `json:"execution"`
+ Actor string `json:"actor"`
+ Action string `json:"action"`
+ Date string `json:"date"`
+ Comment string `json:"comment"`
+ Extra string `json:"extra"`
+ Read string `json:"read"`
+ Vision string `json:"vision"`
+ Efforted int `json:"efforted"`
+ Desc string `json:"desc"`
}
-type ZentaoBugCommits struct {
+type ZentaoBugCommit struct {
common.NoPKModel
- ConnectionId uint64 `gorm:"primaryKey;type:BIGINT NOT NULL"`
- ID int64 `json:"id" gorm:"primaryKey;type:BIGINT NOT NULL;autoIncrement:false"`
- Project int64 `json:"project"`
- Product int64 `json:"product"`
- Actions []string `gorm:"type:json;serializer:json" json:"actions" mapstructure:"actions"`
+ ConnectionId uint64 `gorm:"primaryKey;type:BIGINT NOT NULL"`
+ ID int `json:"id" gorm:"primaryKey;type:BIGINT NOT NULL;autoIncrement:false"`
+ ObjectType string `json:"objectType"`
+ ObjectID int `json:"objectID"`
+ Product int64 `json:"product"`
+ Project int64 `json:"project"`
+ Execution int `json:"execution"`
+ Actor string `json:"actor"`
+ Action string `json:"action"`
+ Date string `json:"date"`
+ Comment string `json:"comment"`
+ Extra string `json:"extra"`
+ Host string `json:"host"` //the host part of extra
+ RepoRevision string `json:"repoRevision"` // the repoRevisionJson part of extra
+ ActionRead string `json:"actionRead"`
+ Vision string `json:"vision"`
+ Efforted int `json:"efforted"`
+ ActionDesc string `json:"cctionDesc"`
}
-func (ZentaoBugCommits) TableName() string {
+func (ZentaoBugCommit) TableName() string {
return "_tool_zentao_bug_commits"
}
+
+type ZentaoBugRepoCommitsRes struct {
+ Title string `json:"title"`
+ Log struct {
+ Revision string `json:"revision"`
+ Committer string `json:"committer"`
+ Time string `json:"time"`
+ Comment string `json:"comment"`
+ Change struct {
+ TestYaml struct {
+ Action string `json:"action"`
+ Kind string `json:"kind"`
+ OldPath string `json:"oldPath"`
+ } `json:"/test.yaml"`
+ } `json:"change"`
+ Commit string `json:"commit"`
+ } `json:"log"`
+ Repo struct {
+ ID string `json:"id"`
+ Product string `json:"product"`
+ Projects string `json:"projects"`
+ Name string `json:"name"`
+ Path string `json:"path"`
+ Prefix string `json:"prefix"`
+ Encoding string `json:"encoding"`
+ Scm string `json:"SCM"`
+ Client string `json:"client"`
+ ServiceHost string `json:"serviceHost"`
+ ServiceProject string `json:"serviceProject"`
+ Commits string `json:"commits"`
+ Account string `json:"account"`
+ Password string `json:"password"`
+ Encrypt string `json:"encrypt"`
+ ACL any `json:"acl"`
+ Synced string `json:"synced"`
+ LastSync string `json:"lastSync"`
+ Desc string `json:"desc"`
+ Extra string `json:"extra"`
+ PreMerge string `json:"preMerge"`
+ Job string `json:"job"`
+ FileServerURL any `json:"fileServerUrl"`
+ FileServerAccount string `json:"fileServerAccount"`
+ FileServerPassword string `json:"fileServerPassword"`
+ Deleted string `json:"deleted"`
+ CodePath string `json:"codePath"`
+ GitService string `json:"gitService"`
+ Project string `json:"project"`
+ } `json:"repo"`
+ Path string `json:"path"`
+ Type string `json:"type"`
+ Changes struct {
+ TestYaml struct {
+ Action string `json:"action"`
+ Kind string `json:"kind"`
+ OldPath string `json:"oldPath"`
+ View string `json:"view"`
+ Diff string `json:"diff"`
+ } `json:"/test.yaml"`
+ } `json:"changes"`
+ RepoID string `json:"repoID"`
+ BranchID bool `json:"branchID"`
+ ObjectID string `json:"objectID"`
+ Revision string `json:"revision"`
+ ParentDir string `json:"parentDir"`
+ OldRevision string `json:"oldRevision"`
+ PreAndNext struct {
+ Pre string `json:"pre"`
+ Next string `json:"next"`
+ } `json:"preAndNext"`
+ Pager any `json:"pager"`
+}
+
+type ZentaoBugRepoCommit struct {
+ common.NoPKModel
+ ConnectionId uint64 `gorm:"primaryKey;type:BIGINT NOT NULL"`
+ Product int64 `json:"product"`
+ Project int64 `json:"project"`
+ IssueId string `gorm:"primaryKey;type:varchar(255)"` // the bug id
+ RepoUrl string `gorm:"primaryKey;type:varchar(255)"`
+ CommitSha string `gorm:"primaryKey;type:varchar(255)"`
+}
+
+func (ZentaoBugRepoCommit) TableName() string {
+ return "_tool_zentao_bug_repo_commits"
+}
diff --git a/backend/plugins/zentao/models/migrationscripts/20230531_add_issue_commits.go b/backend/plugins/zentao/models/migrationscripts/20230605_add_issue_repo_commits.go
similarity index 69%
rename from backend/plugins/zentao/models/migrationscripts/20230531_add_issue_commits.go
rename to backend/plugins/zentao/models/migrationscripts/20230605_add_issue_repo_commits.go
index 75b6cb606..a50cb4aba 100644
--- a/backend/plugins/zentao/models/migrationscripts/20230531_add_issue_commits.go
+++ b/backend/plugins/zentao/models/migrationscripts/20230605_add_issue_repo_commits.go
@@ -24,20 +24,25 @@ import (
"github.com/apache/incubator-devlake/plugins/zentao/models/archived"
)
-type addIssueCommitsTables struct{}
+type addIssueRepoCommitsTables struct{}
-func (*addIssueCommitsTables) Up(basicRes context.BasicRes) errors.Error {
+func (*addIssueRepoCommitsTables) Up(basicRes context.BasicRes) errors.Error {
return migrationhelper.AutoMigrateTables(
basicRes,
- &archived.ZentaoBugCommits{},
+ &archived.ZentaoBugCommit{},
+ &archived.ZentaoBugRepoCommit{},
+ &archived.ZentaoStoryCommit{},
+ &archived.ZentaoStoryRepoCommit{},
+ &archived.ZentaoTaskCommit{},
+ &archived.ZentaoTaskRepoCommit{},
)
}
-func (*addIssueCommitsTables) Version() uint64 {
- return 20230531000001
+func (*addIssueRepoCommitsTables) Version() uint64 {
+ return 20230605000011
}
-func (*addIssueCommitsTables) Name() string {
- return "zentao add issue commits tables"
+func (*addIssueRepoCommitsTables) Name() string {
+ return "zentao add issue repo commits tables"
}
diff --git a/backend/plugins/zentao/models/migrationscripts/register.go b/backend/plugins/zentao/models/migrationscripts/register.go
index df08cb6a2..9ce9af76c 100644
--- a/backend/plugins/zentao/models/migrationscripts/register.go
+++ b/backend/plugins/zentao/models/migrationscripts/register.go
@@ -26,7 +26,7 @@ func All() []plugin.MigrationScript {
return []plugin.MigrationScript{
new(addInitTables),
new(addInitChangelogTables),
- new(addIssueCommitsTables),
new(addScopeConfigTables),
+ new(addIssueRepoCommitsTables),
}
}
diff --git a/backend/plugins/zentao/models/story_commits.go b/backend/plugins/zentao/models/story_commits.go
new file mode 100644
index 000000000..937738149
--- /dev/null
+++ b/backend/plugins/zentao/models/story_commits.go
@@ -0,0 +1,151 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package models
+
+import (
+ "github.com/apache/incubator-devlake/core/models/common"
+)
+
+type ZentaoStoryCommitsRes struct {
+ ID int `json:"id" gorm:"primaryKey;type:BIGINT NOT NULL;autoIncrement:false"`
+ ObjectType string `json:"objectType"`
+ ObjectID int `json:"objectID"`
+ Product string `json:"product"`
+ Project int `json:"project"`
+ Execution int `json:"execution"`
+ Actor string `json:"actor"`
+ Action string `json:"action"`
+ Date string `json:"date"`
+ Comment string `json:"comment"`
+ Extra string `json:"extra"`
+ Read string `json:"read"`
+ Vision string `json:"vision"`
+ Efforted int `json:"efforted"`
+ Desc string `json:"desc"`
+}
+
+type ZentaoStoryCommit struct {
+ common.NoPKModel
+ ConnectionId uint64 `gorm:"primaryKey;type:BIGINT NOT NULL"`
+ ID int `json:"id" gorm:"primaryKey;type:BIGINT NOT NULL;autoIncrement:false"`
+ ObjectType string `json:"objectType"`
+ ObjectID int `json:"objectID"`
+ Product int64 `json:"product"`
+ Project int64 `json:"project"`
+ Execution int `json:"execution"`
+ Actor string `json:"actor"`
+ Action string `json:"action"`
+ Date string `json:"date"`
+ Comment string `json:"comment"`
+ Extra string `json:"extra"`
+ Host string `json:"host"` //the host part of extra
+ RepoRevision string `json:"repoRevision"` // the repoRevisionJson part of extra
+ ActionRead string `json:"actionRead"`
+ Vision string `json:"vision"`
+ Efforted int `json:"efforted"`
+ ActionDesc string `json:"cctionDesc"`
+}
+
+func (ZentaoStoryCommit) TableName() string {
+ return "_tool_zentao_story_commits"
+}
+
+type ZentaoStoryRepoCommitsRes struct {
+ Title string `json:"title"`
+ Log struct {
+ Revision string `json:"revision"`
+ Committer string `json:"committer"`
+ Time string `json:"time"`
+ Comment string `json:"comment"`
+ Change struct {
+ TestYaml struct {
+ Action string `json:"action"`
+ Kind string `json:"kind"`
+ OldPath string `json:"oldPath"`
+ } `json:"/test.yaml"`
+ } `json:"change"`
+ Commit string `json:"commit"`
+ } `json:"log"`
+ Repo struct {
+ ID string `json:"id"`
+ Product string `json:"product"`
+ Projects string `json:"projects"`
+ Name string `json:"name"`
+ Path string `json:"path"`
+ Prefix string `json:"prefix"`
+ Encoding string `json:"encoding"`
+ Scm string `json:"SCM"`
+ Client string `json:"client"`
+ ServiceHost string `json:"serviceHost"`
+ ServiceProject string `json:"serviceProject"`
+ Commits string `json:"commits"`
+ Account string `json:"account"`
+ Password string `json:"password"`
+ Encrypt string `json:"encrypt"`
+ ACL any `json:"acl"`
+ Synced string `json:"synced"`
+ LastSync string `json:"lastSync"`
+ Desc string `json:"desc"`
+ Extra string `json:"extra"`
+ PreMerge string `json:"preMerge"`
+ Job string `json:"job"`
+ FileServerURL any `json:"fileServerUrl"`
+ FileServerAccount string `json:"fileServerAccount"`
+ FileServerPassword string `json:"fileServerPassword"`
+ Deleted string `json:"deleted"`
+ CodePath string `json:"codePath"`
+ GitService string `json:"gitService"`
+ Project string `json:"project"`
+ } `json:"repo"`
+ Path string `json:"path"`
+ Type string `json:"type"`
+ Changes struct {
+ TestYaml struct {
+ Action string `json:"action"`
+ Kind string `json:"kind"`
+ OldPath string `json:"oldPath"`
+ View string `json:"view"`
+ Diff string `json:"diff"`
+ } `json:"/test.yaml"`
+ } `json:"changes"`
+ RepoID string `json:"repoID"`
+ BranchID bool `json:"branchID"`
+ ObjectID string `json:"objectID"`
+ Revision string `json:"revision"`
+ ParentDir string `json:"parentDir"`
+ OldRevision string `json:"oldRevision"`
+ PreAndNext struct {
+ Pre string `json:"pre"`
+ Next string `json:"next"`
+ } `json:"preAndNext"`
+ Pager any `json:"pager"`
+}
+
+type ZentaoStoryRepoCommit struct {
+ common.NoPKModel
+ ConnectionId uint64 `gorm:"primaryKey;type:BIGINT NOT NULL"`
+ Product int64 `json:"product"`
+ Project int64 `json:"project"`
+ IssueId string `gorm:"primaryKey;type:varchar(255)"` // the story id
+ RepoUrl string `gorm:"primaryKey;type:varchar(255)"`
+ CommitSha string `gorm:"primaryKey;type:varchar(255)"`
+}
+
+func (ZentaoStoryRepoCommit) TableName() string {
+ return "_tool_zentao_story_repo_commits"
+}
diff --git a/backend/plugins/zentao/models/task_commits.go b/backend/plugins/zentao/models/task_commits.go
new file mode 100644
index 000000000..f297358b5
--- /dev/null
+++ b/backend/plugins/zentao/models/task_commits.go
@@ -0,0 +1,151 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package models
+
+import (
+ "github.com/apache/incubator-devlake/core/models/common"
+)
+
+type ZentaoTaskCommitsRes struct {
+ ID int `json:"id" gorm:"primaryKey;type:BIGINT NOT NULL;autoIncrement:false"`
+ ObjectType string `json:"objectType"`
+ ObjectID int `json:"objectID"`
+ Product string `json:"product"`
+ Project int `json:"project"`
+ Execution int `json:"execution"`
+ Actor string `json:"actor"`
+ Action string `json:"action"`
+ Date string `json:"date"`
+ Comment string `json:"comment"`
+ Extra string `json:"extra"`
+ Read string `json:"read"`
+ Vision string `json:"vision"`
+ Efforted int `json:"efforted"`
+ Desc string `json:"desc"`
+}
+
+type ZentaoTaskCommit struct {
+ common.NoPKModel
+ ConnectionId uint64 `gorm:"primaryKey;type:BIGINT NOT NULL"`
+ ID int `json:"id" gorm:"primaryKey;type:BIGINT NOT NULL;autoIncrement:false"`
+ ObjectType string `json:"objectType"`
+ ObjectID int `json:"objectID"`
+ Product int64 `json:"product"`
+ Project int64 `json:"project"`
+ Execution int `json:"execution"`
+ Actor string `json:"actor"`
+ Action string `json:"action"`
+ Date string `json:"date"`
+ Comment string `json:"comment"`
+ Extra string `json:"extra"`
+ Host string `json:"host"` //the host part of extra
+ RepoRevision string `json:"repoRevision"` // the repoRevisionJson part of extra
+ ActionRead string `json:"actionRead"`
+ Vision string `json:"vision"`
+ Efforted int `json:"efforted"`
+ ActionDesc string `json:"cctionDesc"`
+}
+
+func (ZentaoTaskCommit) TableName() string {
+ return "_tool_zentao_task_commits"
+}
+
+type ZentaoTaskRepoCommitsRes struct {
+ Title string `json:"title"`
+ Log struct {
+ Revision string `json:"revision"`
+ Committer string `json:"committer"`
+ Time string `json:"time"`
+ Comment string `json:"comment"`
+ Change struct {
+ TestYaml struct {
+ Action string `json:"action"`
+ Kind string `json:"kind"`
+ OldPath string `json:"oldPath"`
+ } `json:"/test.yaml"`
+ } `json:"change"`
+ Commit string `json:"commit"`
+ } `json:"log"`
+ Repo struct {
+ ID string `json:"id"`
+ Product string `json:"product"`
+ Projects string `json:"projects"`
+ Name string `json:"name"`
+ Path string `json:"path"`
+ Prefix string `json:"prefix"`
+ Encoding string `json:"encoding"`
+ Scm string `json:"SCM"`
+ Client string `json:"client"`
+ ServiceHost string `json:"serviceHost"`
+ ServiceProject string `json:"serviceProject"`
+ Commits string `json:"commits"`
+ Account string `json:"account"`
+ Password string `json:"password"`
+ Encrypt string `json:"encrypt"`
+ ACL any `json:"acl"`
+ Synced string `json:"synced"`
+ LastSync string `json:"lastSync"`
+ Desc string `json:"desc"`
+ Extra string `json:"extra"`
+ PreMerge string `json:"preMerge"`
+ Job string `json:"job"`
+ FileServerURL any `json:"fileServerUrl"`
+ FileServerAccount string `json:"fileServerAccount"`
+ FileServerPassword string `json:"fileServerPassword"`
+ Deleted string `json:"deleted"`
+ CodePath string `json:"codePath"`
+ GitService string `json:"gitService"`
+ Project string `json:"project"`
+ } `json:"repo"`
+ Path string `json:"path"`
+ Type string `json:"type"`
+ Changes struct {
+ TestYaml struct {
+ Action string `json:"action"`
+ Kind string `json:"kind"`
+ OldPath string `json:"oldPath"`
+ View string `json:"view"`
+ Diff string `json:"diff"`
+ } `json:"/test.yaml"`
+ } `json:"changes"`
+ RepoID string `json:"repoID"`
+ BranchID bool `json:"branchID"`
+ ObjectID string `json:"objectID"`
+ Revision string `json:"revision"`
+ ParentDir string `json:"parentDir"`
+ OldRevision string `json:"oldRevision"`
+ PreAndNext struct {
+ Pre string `json:"pre"`
+ Next string `json:"next"`
+ } `json:"preAndNext"`
+ Pager any `json:"pager"`
+}
+
+type ZentaoTaskRepoCommit struct {
+ common.NoPKModel
+ ConnectionId uint64 `gorm:"primaryKey;type:BIGINT NOT NULL"`
+ Product int64 `json:"product"`
+ Project int64 `json:"project"`
+ IssueId string `gorm:"primaryKey;type:varchar(255)"` // the task id
+ RepoUrl string `gorm:"primaryKey;type:varchar(255)"`
+ CommitSha string `gorm:"primaryKey;type:varchar(255)"`
+}
+
+func (ZentaoTaskRepoCommit) TableName() string {
+ return "_tool_zentao_task_repo_commits"
+}
diff --git a/backend/plugins/zentao/tasks/bug_commits_collector.go b/backend/plugins/zentao/tasks/bug_commits_collector.go
index 639885f29..de3f32232 100644
--- a/backend/plugins/zentao/tasks/bug_commits_collector.go
+++ b/backend/plugins/zentao/tasks/bug_commits_collector.go
@@ -19,10 +19,7 @@ package tasks
import (
"encoding/json"
- "fmt"
- "io"
"net/http"
- "net/url"
"reflect"
"github.com/apache/incubator-devlake/core/dal"
@@ -54,6 +51,7 @@ func CollectBugCommits(taskCtx plugin.SubTaskContext) errors.Error {
Params: ZentaoApiParams{
ConnectionId: data.Options.ConnectionId,
ProductId: data.Options.ProductId,
+ ProjectId: data.Options.ProjectId,
},
Table: RAW_BUG_COMMITS_TABLE,
}, data.TimeAfter)
@@ -63,7 +61,7 @@ func CollectBugCommits(taskCtx plugin.SubTaskContext) errors.Error {
// load bugs id from db
clauses := []dal.Clause{
- dal.Select("id"),
+ dal.Select("id, last_edited_date"),
dal.From(&models.ZentaoBug{}),
dal.Where(
"product = ? AND connection_id = ?",
@@ -75,7 +73,7 @@ func CollectBugCommits(taskCtx plugin.SubTaskContext) errors.Error {
if incremental {
clauses = append(
clauses,
- dal.Where("updated_at > ?", collectorWithState.LatestState.LatestSuccessStart),
+ dal.Where("last_edited_date is not null and last_edited_date > ?", collectorWithState.LatestState.LatestSuccessStart),
)
}
cursor, err := db.Cursor(clauses...)
@@ -93,6 +91,7 @@ func CollectBugCommits(taskCtx plugin.SubTaskContext) errors.Error {
Params: ZentaoApiParams{
ConnectionId: data.Options.ConnectionId,
ProductId: data.Options.ProductId,
+ ProjectId: data.Options.ProjectId,
},
Table: RAW_BUG_COMMITS_TABLE,
},
@@ -101,21 +100,18 @@ func CollectBugCommits(taskCtx plugin.SubTaskContext) errors.Error {
Input: iterator,
Incremental: incremental,
UrlTemplate: "bugs/{{ .Input.ID }}",
- Query: func(reqData *api.RequestData) (url.Values, errors.Error) {
- query := url.Values{}
- query.Set("page", fmt.Sprintf("%v", reqData.Pager.Page))
- query.Set("per_page", fmt.Sprintf("%v", reqData.Pager.Size))
- return query, nil
- },
- GetTotalPages: GetTotalPagesFromResponse,
ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) {
- body, err := io.ReadAll(res.Body)
+ var data struct {
+ Actions []json.RawMessage `json:"actions"`
+ }
+ err := api.UnmarshalResponse(res, &data)
if err != nil {
- return nil, errors.Convert(err)
+ return nil, err
}
- res.Body.Close()
- return []json.RawMessage{body}, nil
+ return data.Actions, nil
+
},
+ AfterResponse: ignoreHTTPStatus404,
})
if err != nil {
return err
diff --git a/backend/plugins/zentao/tasks/bug_commits_extractor.go b/backend/plugins/zentao/tasks/bug_commits_extractor.go
new file mode 100644
index 000000000..8492810a2
--- /dev/null
+++ b/backend/plugins/zentao/tasks/bug_commits_extractor.go
@@ -0,0 +1,113 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "encoding/json"
+ "net/url"
+ "regexp"
+
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "github.com/apache/incubator-devlake/plugins/zentao/models"
+)
+
+var _ plugin.SubTaskEntryPoint = ExtractBugCommits
+
+var ExtractBugCommitsMeta = plugin.SubTaskMeta{
+ Name: "extractBugCommits",
+ EntryPoint: ExtractBugCommits,
+ EnabledByDefault: true,
+ Description: "extract Zentao bug commits",
+ DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET},
+}
+
+func ExtractBugCommits(taskCtx plugin.SubTaskContext) errors.Error {
+ data := taskCtx.GetData().(*ZentaoTaskData)
+
+ // this Extract only work for product
+ if data.Options.ProductId == 0 {
+ return nil
+ }
+
+ re := regexp.MustCompile(`href='(.*?)'`)
+ extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{
+ RawDataSubTaskArgs: api.RawDataSubTaskArgs{
+ Ctx: taskCtx,
+ Params: ZentaoApiParams{
+ ConnectionId: data.Options.ConnectionId,
+ ProductId: data.Options.ProductId,
+ ProjectId: data.Options.ProjectId,
+ },
+ Table: RAW_BUG_COMMITS_TABLE,
+ },
+ Extract: func(row *api.RawData) ([]interface{}, errors.Error) {
+ res := &models.ZentaoBugCommitsRes{}
+ err := json.Unmarshal(row.Data, res)
+ if err != nil {
+ return nil, errors.Default.WrapRaw(err)
+ }
+ // only linked2revision action is valid
+ if res.Action != "linked2revision" {
+ return nil, nil
+ }
+
+ bugCommits := &models.ZentaoBugCommit{
+ ConnectionId: data.Options.ConnectionId,
+ ID: res.ID,
+ ObjectType: res.ObjectType,
+ ObjectID: res.ObjectID,
+ Product: data.Options.ProductId,
+ Project: data.Options.ProjectId,
+ Execution: res.Execution,
+ Actor: res.Actor,
+ Action: res.Action,
+ Date: res.Date,
+ Comment: res.Comment,
+ ActionRead: res.Read,
+ Vision: res.Vision,
+ Efforted: res.Efforted,
+ ActionDesc: res.Desc,
+ }
+
+ match := re.FindStringSubmatch(res.Extra)
+ if len(match) > 1 {
+ bugCommits.Extra = match[1]
+ } else {
+ return nil, nil
+ }
+ u, err := url.Parse(match[1])
+ if err != nil {
+ return nil, errors.Default.WrapRaw(err)
+ }
+ bugCommits.Host = u.Host
+ bugCommits.RepoRevision = u.Path
+
+ results := make([]interface{}, 0)
+ results = append(results, bugCommits)
+ return results, nil
+ },
+ })
+
+ if err != nil {
+ return err
+ }
+
+ return extractor.Execute()
+}
diff --git a/backend/plugins/zentao/tasks/bug_commits_collector.go b/backend/plugins/zentao/tasks/bug_repo_commits_collector.go
similarity index 53%
copy from backend/plugins/zentao/tasks/bug_commits_collector.go
copy to backend/plugins/zentao/tasks/bug_repo_commits_collector.go
index 639885f29..c9f28ae02 100644
--- a/backend/plugins/zentao/tasks/bug_commits_collector.go
+++ b/backend/plugins/zentao/tasks/bug_repo_commits_collector.go
@@ -19,10 +19,7 @@ package tasks
import (
"encoding/json"
- "fmt"
- "io"
"net/http"
- "net/url"
"reflect"
"github.com/apache/incubator-devlake/core/dal"
@@ -32,98 +29,84 @@ import (
"github.com/apache/incubator-devlake/plugins/zentao/models"
)
-const RAW_BUG_COMMITS_TABLE = "zentao_api_bug_commits"
+const RAW_BUG_REPO_COMMITS_TABLE = "zentao_api_bug_repo_commits"
-var _ plugin.SubTaskEntryPoint = CollectBugCommits
+var _ plugin.SubTaskEntryPoint = CollectBugRepoCommits
-var CollectBugCommitsMeta = plugin.SubTaskMeta{
- Name: "collectBugCommits",
- EntryPoint: CollectBugCommits,
+var CollectBugRepoCommitsMeta = plugin.SubTaskMeta{
+ Name: "collectBugRepoCommits",
+ EntryPoint: CollectBugRepoCommits,
EnabledByDefault: true,
- Description: "Collect Bug Commits data from Zentao api",
+ Description: "Collect Bug Repo Commits data from Zentao api",
DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET},
}
-func CollectBugCommits(taskCtx plugin.SubTaskContext) errors.Error {
+func CollectBugRepoCommits(taskCtx plugin.SubTaskContext) errors.Error {
db := taskCtx.GetDal()
data := taskCtx.GetData().(*ZentaoTaskData)
- // state manager
- collectorWithState, err := api.NewStatefulApiCollector(api.RawDataSubTaskArgs{
- Ctx: taskCtx,
- Params: ZentaoApiParams{
- ConnectionId: data.Options.ConnectionId,
- ProductId: data.Options.ProductId,
- },
- Table: RAW_BUG_COMMITS_TABLE,
- }, data.TimeAfter)
- if err != nil {
- return err
- }
-
// load bugs id from db
clauses := []dal.Clause{
- dal.Select("id"),
- dal.From(&models.ZentaoBug{}),
+ dal.Select("object_id, repo_revision"),
+ dal.From(&models.ZentaoBugCommit{}),
dal.Where(
"product = ? AND connection_id = ?",
data.Options.ProductId, data.Options.ConnectionId,
),
}
- // incremental collection
- incremental := collectorWithState.IsIncremental()
- if incremental {
- clauses = append(
- clauses,
- dal.Where("updated_at > ?", collectorWithState.LatestState.LatestSuccessStart),
- )
- }
+
cursor, err := db.Cursor(clauses...)
if err != nil {
return err
}
- iterator, err := api.NewDalCursorIterator(db, cursor, reflect.TypeOf(SimpleZentaoBug{}))
+
+ iterator, err := api.NewDalCursorIterator(db, cursor, reflect.TypeOf(SimpleZentaoBugCommit{}))
if err != nil {
return err
}
- // collect bug commits
- err = collectorWithState.InitCollector(api.ApiCollectorArgs{
+
+ // collect bug repo commits
+ collector, err := api.NewApiCollector(api.ApiCollectorArgs{
RawDataSubTaskArgs: api.RawDataSubTaskArgs{
Ctx: taskCtx,
Params: ZentaoApiParams{
ConnectionId: data.Options.ConnectionId,
ProductId: data.Options.ProductId,
+ ProjectId: data.Options.ProjectId,
},
- Table: RAW_BUG_COMMITS_TABLE,
+ Table: RAW_BUG_REPO_COMMITS_TABLE,
},
ApiClient: data.ApiClient,
- PageSize: 100,
Input: iterator,
- Incremental: incremental,
- UrlTemplate: "bugs/{{ .Input.ID }}",
- Query: func(reqData *api.RequestData) (url.Values, errors.Error) {
- query := url.Values{}
- query.Set("page", fmt.Sprintf("%v", reqData.Pager.Page))
- query.Set("per_page", fmt.Sprintf("%v", reqData.Pager.Size))
- return query, nil
- },
- GetTotalPages: GetTotalPagesFromResponse,
+ UrlTemplate: "../..{{ .Input.RepoRevision }}",
ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) {
- body, err := io.ReadAll(res.Body)
+ var result RepoRevisionResponse
+ err := api.UnmarshalResponse(res, &result)
if err != nil {
- return nil, errors.Convert(err)
+ return nil, err
}
- res.Body.Close()
- return []json.RawMessage{body}, nil
+ byteData := []byte(result.Data)
+ return []json.RawMessage{byteData}, nil
+
},
+ AfterResponse: ignoreHTTPStatus404,
})
if err != nil {
return err
}
- return collectorWithState.Execute()
+ return collector.Execute()
+}
+
+type SimpleZentaoBugCommit struct {
+ ObjectID int `json:"objectID"`
+ Host string `json:"host"` //the host part of extra
+ RepoRevision string `json:"repoRevision"` // the repoRevisionJson part of extra
+
}
-type SimpleZentaoBug struct {
- ID int64 `json:"id"`
+type RepoRevisionResponse struct {
+ Status string `json:"status"`
+ Data string `json:"data"`
+ MD5 string `json:"md5"`
}
diff --git a/backend/plugins/zentao/tasks/bug_repo_commits_convertor.go b/backend/plugins/zentao/tasks/bug_repo_commits_convertor.go
new file mode 100644
index 000000000..f425f2c76
--- /dev/null
+++ b/backend/plugins/zentao/tasks/bug_repo_commits_convertor.go
@@ -0,0 +1,93 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "reflect"
+
+ "github.com/apache/incubator-devlake/core/dal"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/models/domainlayer/crossdomain"
+ "github.com/apache/incubator-devlake/core/models/domainlayer/didgen"
+ "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "github.com/apache/incubator-devlake/plugins/zentao/models"
+)
+
+var _ plugin.SubTaskEntryPoint = ConvertBugRepoCommits
+
+var ConvertBugRepoCommitsMeta = plugin.SubTaskMeta{
+ Name: "convertBugRepoCommits",
+ EntryPoint: ConvertBugRepoCommits,
+ EnabledByDefault: true,
+ Description: "convert Zentao bug repo commits",
+ DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET},
+}
+
+func ConvertBugRepoCommits(taskCtx plugin.SubTaskContext) errors.Error {
+ data := taskCtx.GetData().(*ZentaoTaskData)
+ db := taskCtx.GetDal()
+
+ cursor, err := db.Cursor(
+ dal.From(&models.ZentaoBugRepoCommit{}),
+ dal.Where(`product = ? and connection_id = ?`, data.Options.ProductId, data.Options.ConnectionId),
+ )
+ if err != nil {
+ return err
+ }
+ defer cursor.Close()
+
+ issueIdGenerator := didgen.NewDomainIdGenerator(&models.ZentaoBugRepoCommit{})
+ convertor, err := api.NewDataConverter(api.DataConverterArgs{
+ InputRowType: reflect.TypeOf(models.ZentaoBugRepoCommit{}),
+ Input: cursor,
+ RawDataSubTaskArgs: api.RawDataSubTaskArgs{
+ Ctx: taskCtx,
+ Params: ZentaoApiParams{
+ ConnectionId: data.Options.ConnectionId,
+ ProductId: data.Options.ProductId,
+ ProjectId: data.Options.ProjectId,
+ },
+ Table: RAW_BUG_REPO_COMMITS_TABLE,
+ },
+ Convert: func(inputRow interface{}) ([]interface{}, errors.Error) {
+ toolEntity := inputRow.(*models.ZentaoBugRepoCommit)
+ domainEntity := &crossdomain.IssueRepoCommit{
+ IssueId: issueIdGenerator.Generate(data.Options.ConnectionId, toolEntity.IssueId),
+ RepoUrl: toolEntity.RepoUrl,
+ CommitSha: toolEntity.CommitSha,
+ }
+ host, namespace, repoName, err := parseRepoUrl(domainEntity.RepoUrl)
+ if err != nil {
+ return nil, errors.Default.WrapRaw(err)
+ }
+ domainEntity.Host = host
+ domainEntity.Namespace = namespace
+ domainEntity.RepoName = repoName
+
+ var results []interface{}
+ results = append(results, domainEntity)
+ return results, nil
+ },
+ })
+ if err != nil {
+ return err
+ }
+
+ return convertor.Execute()
+}
diff --git a/backend/plugins/zentao/tasks/bug_repo_commits_extractor.go b/backend/plugins/zentao/tasks/bug_repo_commits_extractor.go
new file mode 100644
index 000000000..292416bd4
--- /dev/null
+++ b/backend/plugins/zentao/tasks/bug_repo_commits_extractor.go
@@ -0,0 +1,91 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "encoding/json"
+ "regexp"
+
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "github.com/apache/incubator-devlake/plugins/zentao/models"
+)
+
+var _ plugin.SubTaskEntryPoint = ExtractBugRepoCommits
+
+var ExtractBugRepoCommitsMeta = plugin.SubTaskMeta{
+ Name: "extractBugRepoCommits",
+ EntryPoint: ExtractBugRepoCommits,
+ EnabledByDefault: true,
+ Description: "extract Zentao bug repo commits",
+ DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET},
+}
+
+func ExtractBugRepoCommits(taskCtx plugin.SubTaskContext) errors.Error {
+ data := taskCtx.GetData().(*ZentaoTaskData)
+
+ // this Extract only work for product
+ if data.Options.ProductId == 0 {
+ return nil
+ }
+ re := regexp.MustCompile(`(\d+)(?:,\s*(\d+))*`)
+
+ extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{
+ RawDataSubTaskArgs: api.RawDataSubTaskArgs{
+ Ctx: taskCtx,
+ Params: ZentaoApiParams{
+ ConnectionId: data.Options.ConnectionId,
+ ProductId: data.Options.ProductId,
+ ProjectId: data.Options.ProjectId,
+ },
+ Table: RAW_BUG_REPO_COMMITS_TABLE,
+ },
+ Extract: func(row *api.RawData) ([]interface{}, errors.Error) {
+ res := &models.ZentaoBugRepoCommitsRes{}
+ err := json.Unmarshal(row.Data, res)
+ if err != nil {
+ return nil, errors.Default.WrapRaw(err)
+ }
+
+ results := make([]interface{}, 0)
+ match := re.FindStringSubmatch(res.Log.Comment)
+ for i := 1; i < len(match); i++ {
+ if match[i] != "" {
+ bugRepoCommits := &models.ZentaoBugRepoCommit{
+ ConnectionId: data.Options.ConnectionId,
+ Product: data.Options.ProductId,
+ Project: data.Options.ProjectId,
+ RepoUrl: res.Repo.CodePath,
+ CommitSha: res.Revision,
+ IssueId: match[i], // bug id
+ }
+ results = append(results, bugRepoCommits)
+ }
+ }
+
+ return results, nil
+ },
+ })
+
+ if err != nil {
+ return err
+ }
+
+ return extractor.Execute()
+}
diff --git a/backend/plugins/zentao/tasks/shared.go b/backend/plugins/zentao/tasks/shared.go
index d9fcc69c4..3c16b347e 100644
--- a/backend/plugins/zentao/tasks/shared.go
+++ b/backend/plugins/zentao/tasks/shared.go
@@ -18,7 +18,9 @@ limitations under the License.
package tasks
import (
+ "fmt"
"net/http"
+ "net/url"
"strings"
"github.com/apache/incubator-devlake/core/errors"
@@ -142,3 +144,36 @@ func getStdTypeMappings(data *ZentaoTaskData) map[string]string {
}
return stdTypeMappings
}
+
+// parseRepoUrl parses a repository URL and returns the host, namespace, and repository name.
+func parseRepoUrl(repoUrl string) (string, string, string, error) {
+ parsedUrl, err := url.Parse(repoUrl)
+ if err != nil {
+ return "", "", "", err
+ }
+
+ host := parsedUrl.Host
+ host = strings.TrimPrefix(host, "www.")
+ pathParts := strings.Split(parsedUrl.Path, "/")
+ if len(pathParts) < 3 {
+ return "", "", "", fmt.Errorf("invalid RepoUrl: %s", repoUrl)
+ }
+
+ namespace := strings.Join(pathParts[1:len(pathParts)-1], "/")
+ repoName := pathParts[len(pathParts)-1]
+ if repoName == "" {
+ return "", "", "", fmt.Errorf("invalid RepoUrl: %s (empty repository name)", repoUrl)
+ }
+
+ return host, namespace, repoName, nil
+}
+
+func ignoreHTTPStatus404(res *http.Response) errors.Error {
+ if res.StatusCode == http.StatusUnauthorized {
+ return errors.Unauthorized.New("authentication failed, please check your AccessToken")
+ }
+ if res.StatusCode == http.StatusNotFound {
+ return api.ErrIgnoreAndContinue
+ }
+ return nil
+}
diff --git a/backend/plugins/zentao/tasks/bug_commits_collector.go b/backend/plugins/zentao/tasks/story_commits_collector.go
similarity index 68%
copy from backend/plugins/zentao/tasks/bug_commits_collector.go
copy to backend/plugins/zentao/tasks/story_commits_collector.go
index 639885f29..8f3cc5772 100644
--- a/backend/plugins/zentao/tasks/bug_commits_collector.go
+++ b/backend/plugins/zentao/tasks/story_commits_collector.go
@@ -19,10 +19,7 @@ package tasks
import (
"encoding/json"
- "fmt"
- "io"
"net/http"
- "net/url"
"reflect"
"github.com/apache/incubator-devlake/core/dal"
@@ -32,19 +29,19 @@ import (
"github.com/apache/incubator-devlake/plugins/zentao/models"
)
-const RAW_BUG_COMMITS_TABLE = "zentao_api_bug_commits"
+const RAW_STORY_COMMITS_TABLE = "zentao_api_story_commits"
-var _ plugin.SubTaskEntryPoint = CollectBugCommits
+var _ plugin.SubTaskEntryPoint = CollectStoryCommits
-var CollectBugCommitsMeta = plugin.SubTaskMeta{
- Name: "collectBugCommits",
- EntryPoint: CollectBugCommits,
+var CollectStoryCommitsMeta = plugin.SubTaskMeta{
+ Name: "collectStoryCommits",
+ EntryPoint: CollectStoryCommits,
EnabledByDefault: true,
- Description: "Collect Bug Commits data from Zentao api",
+ Description: "Collect Story Commits data from Zentao api",
DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET},
}
-func CollectBugCommits(taskCtx plugin.SubTaskContext) errors.Error {
+func CollectStoryCommits(taskCtx plugin.SubTaskContext) errors.Error {
db := taskCtx.GetDal()
data := taskCtx.GetData().(*ZentaoTaskData)
@@ -54,17 +51,18 @@ func CollectBugCommits(taskCtx plugin.SubTaskContext) errors.Error {
Params: ZentaoApiParams{
ConnectionId: data.Options.ConnectionId,
ProductId: data.Options.ProductId,
+ ProjectId: data.Options.ProjectId,
},
- Table: RAW_BUG_COMMITS_TABLE,
+ Table: RAW_STORY_COMMITS_TABLE,
}, data.TimeAfter)
if err != nil {
return err
}
- // load bugs id from db
+ // load stories id from db
clauses := []dal.Clause{
- dal.Select("id"),
- dal.From(&models.ZentaoBug{}),
+ dal.Select("id, last_edited_date"),
+ dal.From(&models.ZentaoStory{}),
dal.Where(
"product = ? AND connection_id = ?",
data.Options.ProductId, data.Options.ConnectionId,
@@ -75,47 +73,47 @@ func CollectBugCommits(taskCtx plugin.SubTaskContext) errors.Error {
if incremental {
clauses = append(
clauses,
- dal.Where("updated_at > ?", collectorWithState.LatestState.LatestSuccessStart),
+ dal.Where("last_edited_date is not null and last_edited_date > ?", collectorWithState.LatestState.LatestSuccessStart),
)
}
cursor, err := db.Cursor(clauses...)
if err != nil {
return err
}
- iterator, err := api.NewDalCursorIterator(db, cursor, reflect.TypeOf(SimpleZentaoBug{}))
+
+ iterator, err := api.NewDalCursorIterator(db, cursor, reflect.TypeOf(SimpleZentaoStory{}))
if err != nil {
return err
}
- // collect bug commits
+
+ // collect story commits
err = collectorWithState.InitCollector(api.ApiCollectorArgs{
RawDataSubTaskArgs: api.RawDataSubTaskArgs{
Ctx: taskCtx,
Params: ZentaoApiParams{
ConnectionId: data.Options.ConnectionId,
ProductId: data.Options.ProductId,
+ ProjectId: data.Options.ProjectId,
},
- Table: RAW_BUG_COMMITS_TABLE,
+ Table: RAW_STORY_COMMITS_TABLE,
},
ApiClient: data.ApiClient,
PageSize: 100,
Input: iterator,
Incremental: incremental,
- UrlTemplate: "bugs/{{ .Input.ID }}",
- Query: func(reqData *api.RequestData) (url.Values, errors.Error) {
- query := url.Values{}
- query.Set("page", fmt.Sprintf("%v", reqData.Pager.Page))
- query.Set("per_page", fmt.Sprintf("%v", reqData.Pager.Size))
- return query, nil
- },
- GetTotalPages: GetTotalPagesFromResponse,
+ UrlTemplate: "stories/{{ .Input.ID }}",
ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) {
- body, err := io.ReadAll(res.Body)
+ var data struct {
+ Actions []json.RawMessage `json:"actions"`
+ }
+ err := api.UnmarshalResponse(res, &data)
if err != nil {
- return nil, errors.Convert(err)
+ return nil, err
}
- res.Body.Close()
- return []json.RawMessage{body}, nil
+ return data.Actions, nil
+
},
+ AfterResponse: ignoreHTTPStatus404,
})
if err != nil {
return err
@@ -124,6 +122,6 @@ func CollectBugCommits(taskCtx plugin.SubTaskContext) errors.Error {
return collectorWithState.Execute()
}
-type SimpleZentaoBug struct {
+type SimpleZentaoStory struct {
ID int64 `json:"id"`
}
diff --git a/backend/plugins/zentao/tasks/story_commits_extractor.go b/backend/plugins/zentao/tasks/story_commits_extractor.go
new file mode 100644
index 000000000..0e119c3e6
--- /dev/null
+++ b/backend/plugins/zentao/tasks/story_commits_extractor.go
@@ -0,0 +1,115 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "encoding/json"
+ "net/url"
+ "regexp"
+
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "github.com/apache/incubator-devlake/plugins/zentao/models"
+)
+
+var _ plugin.SubTaskEntryPoint = ExtractStoryCommits
+
+var ExtractStoryCommitsMeta = plugin.SubTaskMeta{
+ Name: "extractStoryCommits",
+ EntryPoint: ExtractStoryCommits,
+ EnabledByDefault: true,
+ Description: "extract Zentao story commits",
+ DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET},
+}
+
+func ExtractStoryCommits(taskCtx plugin.SubTaskContext) errors.Error {
+ data := taskCtx.GetData().(*ZentaoTaskData)
+
+ // this Extract only work for product
+ if data.Options.ProductId == 0 {
+ return nil
+ }
+
+ re := regexp.MustCompile(`href='(.*?)'`)
+ extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{
+ RawDataSubTaskArgs: api.RawDataSubTaskArgs{
+ Ctx: taskCtx,
+ Params: ZentaoApiParams{
+ ConnectionId: data.Options.ConnectionId,
+ ProductId: data.Options.ProductId,
+ ProjectId: data.Options.ProjectId,
+ },
+ Table: RAW_STORY_COMMITS_TABLE,
+ },
+ Extract: func(row *api.RawData) ([]interface{}, errors.Error) {
+ res := &models.ZentaoStoryCommitsRes{}
+ err := json.Unmarshal(row.Data, res)
+ if err != nil {
+ return nil, errors.Default.WrapRaw(err)
+ }
+
+ // only linked2revision action is valid
+ if res.Action != "linked2revision" {
+ return nil, nil
+ }
+
+ storyCommits := &models.ZentaoStoryCommit{
+ ConnectionId: data.Options.ConnectionId,
+ ID: res.ID,
+ ObjectType: res.ObjectType,
+ ObjectID: res.ObjectID,
+ Product: data.Options.ProductId,
+ Project: data.Options.ProjectId,
+ Execution: res.Execution,
+ Actor: res.Actor,
+ Action: res.Action,
+ Date: res.Date,
+ Comment: res.Comment,
+ ActionRead: res.Read,
+ Vision: res.Vision,
+ Efforted: res.Efforted,
+ ActionDesc: res.Desc,
+ }
+
+ match := re.FindStringSubmatch(res.Extra)
+ if len(match) > 1 {
+ storyCommits.Extra = match[1]
+ } else {
+ return nil, nil
+ }
+ u, err := url.Parse(match[1])
+ if err != nil {
+ return nil, errors.Default.WrapRaw(err)
+ }
+
+ storyCommits.Host = u.Host
+ storyCommits.RepoRevision = u.Path
+
+ results := make([]interface{}, 0)
+ results = append(results, storyCommits)
+ return results, nil
+ },
+ })
+
+ if err != nil {
+ return err
+ }
+
+ return extractor.Execute()
+}
diff --git a/backend/plugins/zentao/tasks/bug_commits_collector.go b/backend/plugins/zentao/tasks/story_repo_commits_collector.go
similarity index 53%
copy from backend/plugins/zentao/tasks/bug_commits_collector.go
copy to backend/plugins/zentao/tasks/story_repo_commits_collector.go
index 639885f29..ddcdbf47a 100644
--- a/backend/plugins/zentao/tasks/bug_commits_collector.go
+++ b/backend/plugins/zentao/tasks/story_repo_commits_collector.go
@@ -19,10 +19,7 @@ package tasks
import (
"encoding/json"
- "fmt"
- "io"
"net/http"
- "net/url"
"reflect"
"github.com/apache/incubator-devlake/core/dal"
@@ -32,98 +29,78 @@ import (
"github.com/apache/incubator-devlake/plugins/zentao/models"
)
-const RAW_BUG_COMMITS_TABLE = "zentao_api_bug_commits"
+const RAW_STORY_REPO_COMMITS_TABLE = "zentao_api_story_repo_commits"
-var _ plugin.SubTaskEntryPoint = CollectBugCommits
+var _ plugin.SubTaskEntryPoint = CollectStoryRepoCommits
-var CollectBugCommitsMeta = plugin.SubTaskMeta{
- Name: "collectBugCommits",
- EntryPoint: CollectBugCommits,
+var CollectStoryRepoCommitsMeta = plugin.SubTaskMeta{
+ Name: "collectStoryRepoCommits",
+ EntryPoint: CollectStoryRepoCommits,
EnabledByDefault: true,
- Description: "Collect Bug Commits data from Zentao api",
+ Description: "Collect Story Repo Commits data from Zentao api",
DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET},
}
-func CollectBugCommits(taskCtx plugin.SubTaskContext) errors.Error {
+func CollectStoryRepoCommits(taskCtx plugin.SubTaskContext) errors.Error {
db := taskCtx.GetDal()
data := taskCtx.GetData().(*ZentaoTaskData)
- // state manager
- collectorWithState, err := api.NewStatefulApiCollector(api.RawDataSubTaskArgs{
- Ctx: taskCtx,
- Params: ZentaoApiParams{
- ConnectionId: data.Options.ConnectionId,
- ProductId: data.Options.ProductId,
- },
- Table: RAW_BUG_COMMITS_TABLE,
- }, data.TimeAfter)
- if err != nil {
- return err
- }
-
- // load bugs id from db
+ // load stories id from db
clauses := []dal.Clause{
- dal.Select("id"),
- dal.From(&models.ZentaoBug{}),
+ dal.Select("object_id, repo_revision"),
+ dal.From(&models.ZentaoStoryCommit{}),
dal.Where(
"product = ? AND connection_id = ?",
data.Options.ProductId, data.Options.ConnectionId,
),
}
- // incremental collection
- incremental := collectorWithState.IsIncremental()
- if incremental {
- clauses = append(
- clauses,
- dal.Where("updated_at > ?", collectorWithState.LatestState.LatestSuccessStart),
- )
- }
+
cursor, err := db.Cursor(clauses...)
if err != nil {
return err
}
- iterator, err := api.NewDalCursorIterator(db, cursor, reflect.TypeOf(SimpleZentaoBug{}))
+
+ iterator, err := api.NewDalCursorIterator(db, cursor, reflect.TypeOf(SimpleZentaoStoryCommit{}))
if err != nil {
return err
}
- // collect bug commits
- err = collectorWithState.InitCollector(api.ApiCollectorArgs{
+
+ // collect story repo commits
+ collector, err := api.NewApiCollector(api.ApiCollectorArgs{
RawDataSubTaskArgs: api.RawDataSubTaskArgs{
Ctx: taskCtx,
Params: ZentaoApiParams{
ConnectionId: data.Options.ConnectionId,
ProductId: data.Options.ProductId,
+ ProjectId: data.Options.ProjectId,
},
- Table: RAW_BUG_COMMITS_TABLE,
+ Table: RAW_STORY_REPO_COMMITS_TABLE,
},
ApiClient: data.ApiClient,
- PageSize: 100,
Input: iterator,
- Incremental: incremental,
- UrlTemplate: "bugs/{{ .Input.ID }}",
- Query: func(reqData *api.RequestData) (url.Values, errors.Error) {
- query := url.Values{}
- query.Set("page", fmt.Sprintf("%v", reqData.Pager.Page))
- query.Set("per_page", fmt.Sprintf("%v", reqData.Pager.Size))
- return query, nil
- },
- GetTotalPages: GetTotalPagesFromResponse,
+ UrlTemplate: "../..{{ .Input.RepoRevision }}",
ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) {
- body, err := io.ReadAll(res.Body)
+ var result RepoRevisionResponse
+ err := api.UnmarshalResponse(res, &result)
if err != nil {
- return nil, errors.Convert(err)
+ return nil, err
}
- res.Body.Close()
- return []json.RawMessage{body}, nil
+ byteData := []byte(result.Data)
+ return []json.RawMessage{byteData}, nil
+
},
+ AfterResponse: ignoreHTTPStatus404,
})
if err != nil {
return err
}
- return collectorWithState.Execute()
+ return collector.Execute()
}
-type SimpleZentaoBug struct {
- ID int64 `json:"id"`
+type SimpleZentaoStoryCommit struct {
+ ObjectID int `json:"objectID"`
+ Host string `json:"host"` //the host part of extra
+ RepoRevision string `json:"repoRevision"` // the repoRevisionJson part of extra
+
}
diff --git a/backend/plugins/zentao/tasks/story_repo_commits_convertor.go b/backend/plugins/zentao/tasks/story_repo_commits_convertor.go
new file mode 100644
index 000000000..612a61929
--- /dev/null
+++ b/backend/plugins/zentao/tasks/story_repo_commits_convertor.go
@@ -0,0 +1,93 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "reflect"
+
+ "github.com/apache/incubator-devlake/core/dal"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/models/domainlayer/crossdomain"
+ "github.com/apache/incubator-devlake/core/models/domainlayer/didgen"
+ "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "github.com/apache/incubator-devlake/plugins/zentao/models"
+)
+
+var _ plugin.SubTaskEntryPoint = ConvertStoryRepoCommits
+
+var ConvertStoryRepoCommitsMeta = plugin.SubTaskMeta{
+ Name: "convertStoryRepoCommits",
+ EntryPoint: ConvertStoryRepoCommits,
+ EnabledByDefault: true,
+ Description: "convert Zentao story repo commits",
+ DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET},
+}
+
+func ConvertStoryRepoCommits(taskCtx plugin.SubTaskContext) errors.Error {
+ data := taskCtx.GetData().(*ZentaoTaskData)
+ db := taskCtx.GetDal()
+
+ cursor, err := db.Cursor(
+ dal.From(&models.ZentaoStoryRepoCommit{}),
+ dal.Where(`product = ? and connection_id = ?`, data.Options.ProductId, data.Options.ConnectionId),
+ )
+ if err != nil {
+ return err
+ }
+ defer cursor.Close()
+
+ issueIdGenerator := didgen.NewDomainIdGenerator(&models.ZentaoStoryRepoCommit{})
+ convertor, err := api.NewDataConverter(api.DataConverterArgs{
+ InputRowType: reflect.TypeOf(models.ZentaoStoryRepoCommit{}),
+ Input: cursor,
+ RawDataSubTaskArgs: api.RawDataSubTaskArgs{
+ Ctx: taskCtx,
+ Params: ZentaoApiParams{
+ ConnectionId: data.Options.ConnectionId,
+ ProductId: data.Options.ProductId,
+ ProjectId: data.Options.ProjectId,
+ },
+ Table: RAW_STORY_REPO_COMMITS_TABLE,
+ },
+ Convert: func(inputRow interface{}) ([]interface{}, errors.Error) {
+ toolEntity := inputRow.(*models.ZentaoStoryRepoCommit)
+ domainEntity := &crossdomain.IssueRepoCommit{
+ IssueId: issueIdGenerator.Generate(data.Options.ConnectionId, toolEntity.IssueId),
+ RepoUrl: toolEntity.RepoUrl,
+ CommitSha: toolEntity.CommitSha,
+ }
+ host, namespace, repoName, err := parseRepoUrl(domainEntity.RepoUrl)
+ if err != nil {
+ return nil, errors.Default.WrapRaw(err)
+ }
+ domainEntity.Host = host
+ domainEntity.Namespace = namespace
+ domainEntity.RepoName = repoName
+
+ var results []interface{}
+ results = append(results, domainEntity)
+ return results, nil
+ },
+ })
+ if err != nil {
+ return err
+ }
+
+ return convertor.Execute()
+}
diff --git a/backend/plugins/zentao/tasks/story_repo_commits_extractor.go b/backend/plugins/zentao/tasks/story_repo_commits_extractor.go
new file mode 100644
index 000000000..84a6fa294
--- /dev/null
+++ b/backend/plugins/zentao/tasks/story_repo_commits_extractor.go
@@ -0,0 +1,91 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "encoding/json"
+ "regexp"
+
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "github.com/apache/incubator-devlake/plugins/zentao/models"
+)
+
+var _ plugin.SubTaskEntryPoint = ExtractStoryRepoCommits
+
+var ExtractStoryRepoCommitsMeta = plugin.SubTaskMeta{
+ Name: "extractStoryRepoCommits",
+ EntryPoint: ExtractStoryRepoCommits,
+ EnabledByDefault: true,
+ Description: "extract Zentao story repo commits",
+ DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET},
+}
+
+func ExtractStoryRepoCommits(taskCtx plugin.SubTaskContext) errors.Error {
+ data := taskCtx.GetData().(*ZentaoTaskData)
+
+ // this Extract only work for product
+ if data.Options.ProductId == 0 {
+ return nil
+ }
+
+ re := regexp.MustCompile(`(\d+)(?:,\s*(\d+))*`)
+ extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{
+ RawDataSubTaskArgs: api.RawDataSubTaskArgs{
+ Ctx: taskCtx,
+ Params: ZentaoApiParams{
+ ConnectionId: data.Options.ConnectionId,
+ ProductId: data.Options.ProductId,
+ ProjectId: data.Options.ProjectId,
+ },
+ Table: RAW_STORY_REPO_COMMITS_TABLE,
+ },
+ Extract: func(row *api.RawData) ([]interface{}, errors.Error) {
+ res := &models.ZentaoStoryRepoCommitsRes{}
+ err := json.Unmarshal(row.Data, res)
+ if err != nil {
+ return nil, errors.Default.WrapRaw(err)
+ }
+
+ results := make([]interface{}, 0)
+ match := re.FindStringSubmatch(res.Log.Comment)
+ for i := 1; i < len(match); i++ {
+ if match[i] != "" {
+ storyRepoCommits := &models.ZentaoStoryRepoCommit{
+ ConnectionId: data.Options.ConnectionId,
+ Product: data.Options.ProductId,
+ Project: data.Options.ProjectId,
+ RepoUrl: res.Repo.CodePath,
+ CommitSha: res.Revision,
+ IssueId: match[i], // story id
+ }
+ results = append(results, storyRepoCommits)
+ }
+ }
+
+ return results, nil
+ },
+ })
+
+ if err != nil {
+ return err
+ }
+
+ return extractor.Execute()
+}
diff --git a/backend/plugins/zentao/tasks/bug_commits_collector.go b/backend/plugins/zentao/tasks/task_commits_collector.go
similarity index 66%
copy from backend/plugins/zentao/tasks/bug_commits_collector.go
copy to backend/plugins/zentao/tasks/task_commits_collector.go
index 639885f29..928921cb0 100644
--- a/backend/plugins/zentao/tasks/bug_commits_collector.go
+++ b/backend/plugins/zentao/tasks/task_commits_collector.go
@@ -19,10 +19,7 @@ package tasks
import (
"encoding/json"
- "fmt"
- "io"
"net/http"
- "net/url"
"reflect"
"github.com/apache/incubator-devlake/core/dal"
@@ -32,42 +29,48 @@ import (
"github.com/apache/incubator-devlake/plugins/zentao/models"
)
-const RAW_BUG_COMMITS_TABLE = "zentao_api_bug_commits"
+const RAW_TASK_COMMITS_TABLE = "zentao_api_task_commits"
-var _ plugin.SubTaskEntryPoint = CollectBugCommits
+var _ plugin.SubTaskEntryPoint = CollectTaskCommits
-var CollectBugCommitsMeta = plugin.SubTaskMeta{
- Name: "collectBugCommits",
- EntryPoint: CollectBugCommits,
+var CollectTaskCommitsMeta = plugin.SubTaskMeta{
+ Name: "collectTaskCommits",
+ EntryPoint: CollectTaskCommits,
EnabledByDefault: true,
- Description: "Collect Bug Commits data from Zentao api",
+ Description: "Collect Task Commits data from Zentao api",
DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET},
}
-func CollectBugCommits(taskCtx plugin.SubTaskContext) errors.Error {
+func CollectTaskCommits(taskCtx plugin.SubTaskContext) errors.Error {
db := taskCtx.GetDal()
data := taskCtx.GetData().(*ZentaoTaskData)
+ // this collect only work for project
+ if data.Options.ProjectId == 0 {
+ return nil
+ }
+
// state manager
collectorWithState, err := api.NewStatefulApiCollector(api.RawDataSubTaskArgs{
Ctx: taskCtx,
Params: ZentaoApiParams{
ConnectionId: data.Options.ConnectionId,
ProductId: data.Options.ProductId,
+ ProjectId: data.Options.ProjectId,
},
- Table: RAW_BUG_COMMITS_TABLE,
+ Table: RAW_TASK_COMMITS_TABLE,
}, data.TimeAfter)
if err != nil {
return err
}
- // load bugs id from db
+ // load stories id from db
clauses := []dal.Clause{
- dal.Select("id"),
- dal.From(&models.ZentaoBug{}),
+ dal.Select("id, last_edited_date"),
+ dal.From(&models.ZentaoTask{}),
dal.Where(
- "product = ? AND connection_id = ?",
- data.Options.ProductId, data.Options.ConnectionId,
+ "project = ? AND connection_id = ?",
+ data.Options.ProjectId, data.Options.ConnectionId,
),
}
// incremental collection
@@ -75,47 +78,47 @@ func CollectBugCommits(taskCtx plugin.SubTaskContext) errors.Error {
if incremental {
clauses = append(
clauses,
- dal.Where("updated_at > ?", collectorWithState.LatestState.LatestSuccessStart),
+ dal.Where("last_edited_date is not null and last_edited_date > ?", collectorWithState.LatestState.LatestSuccessStart),
)
}
cursor, err := db.Cursor(clauses...)
if err != nil {
return err
}
- iterator, err := api.NewDalCursorIterator(db, cursor, reflect.TypeOf(SimpleZentaoBug{}))
+
+ iterator, err := api.NewDalCursorIterator(db, cursor, reflect.TypeOf(SimpleZentaoTask{}))
if err != nil {
return err
}
- // collect bug commits
+
+ // collect task commits
err = collectorWithState.InitCollector(api.ApiCollectorArgs{
RawDataSubTaskArgs: api.RawDataSubTaskArgs{
Ctx: taskCtx,
Params: ZentaoApiParams{
ConnectionId: data.Options.ConnectionId,
ProductId: data.Options.ProductId,
+ ProjectId: data.Options.ProjectId,
},
- Table: RAW_BUG_COMMITS_TABLE,
+ Table: RAW_TASK_COMMITS_TABLE,
},
ApiClient: data.ApiClient,
PageSize: 100,
Input: iterator,
Incremental: incremental,
- UrlTemplate: "bugs/{{ .Input.ID }}",
- Query: func(reqData *api.RequestData) (url.Values, errors.Error) {
- query := url.Values{}
- query.Set("page", fmt.Sprintf("%v", reqData.Pager.Page))
- query.Set("per_page", fmt.Sprintf("%v", reqData.Pager.Size))
- return query, nil
- },
- GetTotalPages: GetTotalPagesFromResponse,
+ UrlTemplate: "tasks/{{ .Input.ID }}",
ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) {
- body, err := io.ReadAll(res.Body)
+ var data struct {
+ Actions []json.RawMessage `json:"actions"`
+ }
+ err := api.UnmarshalResponse(res, &data)
if err != nil {
- return nil, errors.Convert(err)
+ return nil, err
}
- res.Body.Close()
- return []json.RawMessage{body}, nil
+ return data.Actions, nil
+
},
+ AfterResponse: ignoreHTTPStatus404,
})
if err != nil {
return err
@@ -124,6 +127,6 @@ func CollectBugCommits(taskCtx plugin.SubTaskContext) errors.Error {
return collectorWithState.Execute()
}
-type SimpleZentaoBug struct {
+type SimpleZentaoTask struct {
ID int64 `json:"id"`
}
diff --git a/backend/plugins/zentao/tasks/task_commits_extractor.go b/backend/plugins/zentao/tasks/task_commits_extractor.go
new file mode 100644
index 000000000..1ccef48da
--- /dev/null
+++ b/backend/plugins/zentao/tasks/task_commits_extractor.go
@@ -0,0 +1,115 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "encoding/json"
+ "net/url"
+ "regexp"
+
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "github.com/apache/incubator-devlake/plugins/zentao/models"
+)
+
+var _ plugin.SubTaskEntryPoint = ExtractTaskCommits
+
+var ExtractTaskCommitsMeta = plugin.SubTaskMeta{
+ Name: "extractTaskCommits",
+ EntryPoint: ExtractTaskCommits,
+ EnabledByDefault: true,
+ Description: "extract Zentao task commits",
+ DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET},
+}
+
+func ExtractTaskCommits(taskCtx plugin.SubTaskContext) errors.Error {
+ data := taskCtx.GetData().(*ZentaoTaskData)
+
+ // this Extract only work for project
+ if data.Options.ProjectId == 0 {
+ return nil
+ }
+
+ re := regexp.MustCompile(`href='(.*?)'`)
+ extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{
+ RawDataSubTaskArgs: api.RawDataSubTaskArgs{
+ Ctx: taskCtx,
+ Params: ZentaoApiParams{
+ ConnectionId: data.Options.ConnectionId,
+ ProductId: data.Options.ProductId,
+ ProjectId: data.Options.ProjectId,
+ },
+ Table: RAW_TASK_COMMITS_TABLE,
+ },
+ Extract: func(row *api.RawData) ([]interface{}, errors.Error) {
+ res := &models.ZentaoTaskCommitsRes{}
+ err := json.Unmarshal(row.Data, res)
+ if err != nil {
+ return nil, errors.Default.WrapRaw(err)
+ }
+
+ // only linked2revision action is valid
+ if res.Action != "linked2revision" {
+ return nil, nil
+ }
+
+ taskCommits := &models.ZentaoTaskCommit{
+ ConnectionId: data.Options.ConnectionId,
+ ID: res.ID,
+ ObjectType: res.ObjectType,
+ ObjectID: res.ObjectID,
+ Product: data.Options.ProductId,
+ Project: data.Options.ProjectId,
+ Execution: res.Execution,
+ Actor: res.Actor,
+ Action: res.Action,
+ Date: res.Date,
+ Comment: res.Comment,
+ ActionRead: res.Read,
+ Vision: res.Vision,
+ Efforted: res.Efforted,
+ ActionDesc: res.Desc,
+ }
+
+ match := re.FindStringSubmatch(res.Extra)
+ if len(match) > 1 {
+ taskCommits.Extra = match[1]
+ } else {
+ return nil, nil
+ }
+ u, err := url.Parse(match[1])
+ if err != nil {
+ return nil, errors.Default.WrapRaw(err)
+ }
+
+ taskCommits.Host = u.Host
+ taskCommits.RepoRevision = u.Path
+
+ results := make([]interface{}, 0)
+ results = append(results, taskCommits)
+ return results, nil
+ },
+ })
+
+ if err != nil {
+ return err
+ }
+
+ return extractor.Execute()
+}
diff --git a/backend/plugins/zentao/tasks/bug_commits_collector.go b/backend/plugins/zentao/tasks/task_repo_commits_collector.go
similarity index 50%
copy from backend/plugins/zentao/tasks/bug_commits_collector.go
copy to backend/plugins/zentao/tasks/task_repo_commits_collector.go
index 639885f29..b5fd3f5aa 100644
--- a/backend/plugins/zentao/tasks/bug_commits_collector.go
+++ b/backend/plugins/zentao/tasks/task_repo_commits_collector.go
@@ -19,10 +19,7 @@ package tasks
import (
"encoding/json"
- "fmt"
- "io"
"net/http"
- "net/url"
"reflect"
"github.com/apache/incubator-devlake/core/dal"
@@ -32,98 +29,78 @@ import (
"github.com/apache/incubator-devlake/plugins/zentao/models"
)
-const RAW_BUG_COMMITS_TABLE = "zentao_api_bug_commits"
+const RAW_TASK_REPO_COMMITS_TABLE = "zentao_api_task_repo_commits"
-var _ plugin.SubTaskEntryPoint = CollectBugCommits
+var _ plugin.SubTaskEntryPoint = CollectTaskRepoCommits
-var CollectBugCommitsMeta = plugin.SubTaskMeta{
- Name: "collectBugCommits",
- EntryPoint: CollectBugCommits,
+var CollectTaskRepoCommitsMeta = plugin.SubTaskMeta{
+ Name: "collectTaskRepoCommits",
+ EntryPoint: CollectTaskRepoCommits,
EnabledByDefault: true,
- Description: "Collect Bug Commits data from Zentao api",
+ Description: "Collect Task Repo Commits data from Zentao api",
DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET},
}
-func CollectBugCommits(taskCtx plugin.SubTaskContext) errors.Error {
+func CollectTaskRepoCommits(taskCtx plugin.SubTaskContext) errors.Error {
db := taskCtx.GetDal()
data := taskCtx.GetData().(*ZentaoTaskData)
- // state manager
- collectorWithState, err := api.NewStatefulApiCollector(api.RawDataSubTaskArgs{
- Ctx: taskCtx,
- Params: ZentaoApiParams{
- ConnectionId: data.Options.ConnectionId,
- ProductId: data.Options.ProductId,
- },
- Table: RAW_BUG_COMMITS_TABLE,
- }, data.TimeAfter)
- if err != nil {
- return err
- }
-
- // load bugs id from db
+ // load tasks id from db
clauses := []dal.Clause{
- dal.Select("id"),
- dal.From(&models.ZentaoBug{}),
+ dal.Select("object_id, repo_revision"),
+ dal.From(&models.ZentaoTaskCommit{}),
dal.Where(
- "product = ? AND connection_id = ?",
- data.Options.ProductId, data.Options.ConnectionId,
+ "project = ? AND connection_id = ?",
+ data.Options.ProjectId, data.Options.ConnectionId,
),
}
- // incremental collection
- incremental := collectorWithState.IsIncremental()
- if incremental {
- clauses = append(
- clauses,
- dal.Where("updated_at > ?", collectorWithState.LatestState.LatestSuccessStart),
- )
- }
+
cursor, err := db.Cursor(clauses...)
if err != nil {
return err
}
- iterator, err := api.NewDalCursorIterator(db, cursor, reflect.TypeOf(SimpleZentaoBug{}))
+
+ iterator, err := api.NewDalCursorIterator(db, cursor, reflect.TypeOf(SimpleZentaoTaskCommit{}))
if err != nil {
return err
}
- // collect bug commits
- err = collectorWithState.InitCollector(api.ApiCollectorArgs{
+
+ // collect task repo commits
+ collector, err := api.NewApiCollector(api.ApiCollectorArgs{
RawDataSubTaskArgs: api.RawDataSubTaskArgs{
Ctx: taskCtx,
Params: ZentaoApiParams{
ConnectionId: data.Options.ConnectionId,
ProductId: data.Options.ProductId,
+ ProjectId: data.Options.ProjectId,
},
- Table: RAW_BUG_COMMITS_TABLE,
+ Table: RAW_TASK_REPO_COMMITS_TABLE,
},
ApiClient: data.ApiClient,
- PageSize: 100,
Input: iterator,
- Incremental: incremental,
- UrlTemplate: "bugs/{{ .Input.ID }}",
- Query: func(reqData *api.RequestData) (url.Values, errors.Error) {
- query := url.Values{}
- query.Set("page", fmt.Sprintf("%v", reqData.Pager.Page))
- query.Set("per_page", fmt.Sprintf("%v", reqData.Pager.Size))
- return query, nil
- },
- GetTotalPages: GetTotalPagesFromResponse,
+ UrlTemplate: "../..{{ .Input.RepoRevision }}",
ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) {
- body, err := io.ReadAll(res.Body)
+ var result RepoRevisionResponse
+ err := api.UnmarshalResponse(res, &result)
if err != nil {
- return nil, errors.Convert(err)
+ return nil, err
}
- res.Body.Close()
- return []json.RawMessage{body}, nil
+ byteData := []byte(result.Data)
+ return []json.RawMessage{byteData}, nil
+
},
+ AfterResponse: ignoreHTTPStatus404,
})
if err != nil {
return err
}
- return collectorWithState.Execute()
+ return collector.Execute()
}
-type SimpleZentaoBug struct {
- ID int64 `json:"id"`
+type SimpleZentaoTaskCommit struct {
+ ObjectID int `json:"objectID"`
+ Host string `json:"host"` //the host part of extra
+ RepoRevision string `json:"repoRevision"` // the repoRevisionJson part of extra
+
}
diff --git a/backend/plugins/zentao/tasks/task_repo_commits_convertor.go b/backend/plugins/zentao/tasks/task_repo_commits_convertor.go
new file mode 100644
index 000000000..d2eb7cf18
--- /dev/null
+++ b/backend/plugins/zentao/tasks/task_repo_commits_convertor.go
@@ -0,0 +1,93 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "reflect"
+
+ "github.com/apache/incubator-devlake/core/dal"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/models/domainlayer/crossdomain"
+ "github.com/apache/incubator-devlake/core/models/domainlayer/didgen"
+ "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "github.com/apache/incubator-devlake/plugins/zentao/models"
+)
+
+var _ plugin.SubTaskEntryPoint = ConvertTaskRepoCommits
+
+var ConvertTaskRepoCommitsMeta = plugin.SubTaskMeta{
+ Name: "convertTaskRepoCommits",
+ EntryPoint: ConvertTaskRepoCommits,
+ EnabledByDefault: true,
+ Description: "convert Zentao task repo commits",
+ DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET},
+}
+
+func ConvertTaskRepoCommits(taskCtx plugin.SubTaskContext) errors.Error {
+ data := taskCtx.GetData().(*ZentaoTaskData)
+ db := taskCtx.GetDal()
+
+ cursor, err := db.Cursor(
+ dal.From(&models.ZentaoTaskRepoCommit{}),
+ dal.Where(`project = ? and connection_id = ?`, data.Options.ProjectId, data.Options.ConnectionId),
+ )
+ if err != nil {
+ return err
+ }
+ defer cursor.Close()
+
+ issueIdGenerator := didgen.NewDomainIdGenerator(&models.ZentaoTaskRepoCommit{})
+ convertor, err := api.NewDataConverter(api.DataConverterArgs{
+ InputRowType: reflect.TypeOf(models.ZentaoTaskRepoCommit{}),
+ Input: cursor,
+ RawDataSubTaskArgs: api.RawDataSubTaskArgs{
+ Ctx: taskCtx,
+ Params: ZentaoApiParams{
+ ConnectionId: data.Options.ConnectionId,
+ ProductId: data.Options.ProductId,
+ ProjectId: data.Options.ProjectId,
+ },
+ Table: RAW_TASK_REPO_COMMITS_TABLE,
+ },
+ Convert: func(inputRow interface{}) ([]interface{}, errors.Error) {
+ toolEntity := inputRow.(*models.ZentaoTaskRepoCommit)
+ domainEntity := &crossdomain.IssueRepoCommit{
+ IssueId: issueIdGenerator.Generate(data.Options.ConnectionId, toolEntity.IssueId),
+ RepoUrl: toolEntity.RepoUrl,
+ CommitSha: toolEntity.CommitSha,
+ }
+ host, namespace, repoName, err := parseRepoUrl(domainEntity.RepoUrl)
+ if err != nil {
+ return nil, errors.Default.WrapRaw(err)
+ }
+ domainEntity.Host = host
+ domainEntity.Namespace = namespace
+ domainEntity.RepoName = repoName
+
+ var results []interface{}
+ results = append(results, domainEntity)
+ return results, nil
+ },
+ })
+ if err != nil {
+ return err
+ }
+
+ return convertor.Execute()
+}
diff --git a/backend/plugins/zentao/tasks/task_repo_commits_extractor.go b/backend/plugins/zentao/tasks/task_repo_commits_extractor.go
new file mode 100644
index 000000000..072872331
--- /dev/null
+++ b/backend/plugins/zentao/tasks/task_repo_commits_extractor.go
@@ -0,0 +1,91 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "encoding/json"
+ "regexp"
+
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "github.com/apache/incubator-devlake/plugins/zentao/models"
+)
+
+var _ plugin.SubTaskEntryPoint = ExtractTaskRepoCommits
+
+var ExtractTaskRepoCommitsMeta = plugin.SubTaskMeta{
+ Name: "extractTaskRepoCommits",
+ EntryPoint: ExtractTaskRepoCommits,
+ EnabledByDefault: true,
+ Description: "extract Zentao task repo commits",
+ DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET},
+}
+
+func ExtractTaskRepoCommits(taskCtx plugin.SubTaskContext) errors.Error {
+ data := taskCtx.GetData().(*ZentaoTaskData)
+
+ // this Extract only work for project
+ if data.Options.ProjectId == 0 {
+ return nil
+ }
+
+ re := regexp.MustCompile(`(\d+)(?:,\s*(\d+))*`)
+ extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{
+ RawDataSubTaskArgs: api.RawDataSubTaskArgs{
+ Ctx: taskCtx,
+ Params: ZentaoApiParams{
+ ConnectionId: data.Options.ConnectionId,
+ ProductId: data.Options.ProductId,
+ ProjectId: data.Options.ProjectId,
+ },
+ Table: RAW_TASK_REPO_COMMITS_TABLE,
+ },
+ Extract: func(row *api.RawData) ([]interface{}, errors.Error) {
+ res := &models.ZentaoTaskRepoCommitsRes{}
+ err := json.Unmarshal(row.Data, res)
+ if err != nil {
+ return nil, errors.Default.WrapRaw(err)
+ }
+
+ results := make([]interface{}, 0)
+ match := re.FindStringSubmatch(res.Log.Comment)
+ for i := 1; i < len(match); i++ {
+ if match[i] != "" {
+ taskRepoCommits := &models.ZentaoTaskRepoCommit{
+ ConnectionId: data.Options.ConnectionId,
+ Product: data.Options.ProductId,
+ Project: data.Options.ProjectId,
+ RepoUrl: res.Repo.CodePath,
+ CommitSha: res.Revision,
+ IssueId: match[i], // task id
+ }
+ results = append(results, taskRepoCommits)
+ }
+ }
+
+ return results, nil
+ },
+ })
+
+ if err != nil {
+ return err
+ }
+
+ return extractor.Execute()
+}