You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by ab...@apache.org on 2022/06/22 06:29:15 UTC
[incubator-devlake] branch main updated: refactor(tapd): add e2e for iters
This is an automated email from the ASF dual-hosted git repository.
abeizn pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 52712568 refactor(tapd): add e2e for iters
52712568 is described below
commit 5271256899d0ca047b113d1ee78624bedc85a2de
Author: Yingchu Chen <yi...@merico.dev>
AuthorDate: Wed Jun 22 14:23:32 2022 +0800
refactor(tapd): add e2e for iters
---
plugins/tapd/e2e/iterations_test.go | 128 +++++++++++++++++++++
.../e2e/snapshot_tables/_tool_tapd_iterations.csv | 11 ++
.../_tool_tapd_workspace_iterations.csv | 11 ++
plugins/tapd/e2e/snapshot_tables/board_sprints.csv | 11 ++
plugins/tapd/e2e/snapshot_tables/boards.csv | 1 +
plugins/tapd/e2e/snapshot_tables/sprints.csv | 11 ++
plugins/tapd/e2e/sub_workspace_test.go | 19 +++
plugins/tapd/tasks/iteration_converter.go | 5 +-
8 files changed, 196 insertions(+), 1 deletion(-)
diff --git a/plugins/tapd/e2e/iterations_test.go b/plugins/tapd/e2e/iterations_test.go
new file mode 100644
index 00000000..81db376c
--- /dev/null
+++ b/plugins/tapd/e2e/iterations_test.go
@@ -0,0 +1,128 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+ "fmt"
+ "github.com/apache/incubator-devlake/models/domainlayer/ticket"
+ "testing"
+
+ "github.com/apache/incubator-devlake/helpers/e2ehelper"
+ "github.com/apache/incubator-devlake/plugins/tapd/impl"
+ "github.com/apache/incubator-devlake/plugins/tapd/models"
+ "github.com/apache/incubator-devlake/plugins/tapd/tasks"
+)
+
+func TestTapdIterationDataFlow(t *testing.T) {
+
+ var tapd impl.Tapd
+ dataflowTester := e2ehelper.NewDataFlowTester(t, "tapd", tapd)
+
+ taskData := &tasks.TapdTaskData{
+ Options: &tasks.TapdOptions{
+ ConnectionId: 1,
+ CompanyId: 99,
+ WorkspaceId: 991,
+ },
+ }
+ // import raw data table
+ dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_tapd_api_iterations.csv",
+ "_raw_tapd_api_iterations")
+
+ // verify extraction
+ dataflowTester.FlushTabler(&models.TapdIteration{})
+ dataflowTester.FlushTabler(&models.TapdWorkspaceIteration{})
+ dataflowTester.Subtask(tasks.ExtractIterationMeta, taskData)
+ dataflowTester.VerifyTable(
+ models.TapdIteration{},
+ fmt.Sprintf("./snapshot_tables/%s.csv", models.TapdIteration{}.TableName()),
+ []string{"connection_id", "id"},
+ []string{
+ "name",
+ "workspace_id",
+ "startdate",
+ "enddate",
+ "status",
+ "release_id",
+ "description",
+ "creator",
+ "created",
+ "modified",
+ "completed",
+ "releaseowner",
+ "launchdate",
+ "notice",
+ "releasename",
+ "_raw_data_params",
+ "_raw_data_table",
+ "_raw_data_id",
+ "_raw_data_remark",
+ },
+ )
+ dataflowTester.VerifyTable(
+ models.TapdWorkspaceIteration{},
+ fmt.Sprintf("./snapshot_tables/%s.csv", models.TapdWorkspaceIteration{}.TableName()),
+ []string{
+ "connection_id",
+ "workspace_id",
+ "iteration_id",
+ },
+ []string{
+ "_raw_data_params",
+ "_raw_data_table",
+ "_raw_data_id",
+ "_raw_data_remark",
+ },
+ )
+ dataflowTester.FlushTabler(&ticket.Sprint{})
+ dataflowTester.FlushTabler(&ticket.BoardSprint{})
+ dataflowTester.Subtask(tasks.ConvertIterationMeta, taskData)
+ dataflowTester.VerifyTable(
+ ticket.Sprint{},
+ fmt.Sprintf("./snapshot_tables/%s.csv", ticket.Sprint{}.TableName()),
+ []string{"id"},
+ []string{
+ "_raw_data_params",
+ "_raw_data_table",
+ "_raw_data_id",
+ "_raw_data_remark",
+ "name",
+ "url",
+ "status",
+ "started_date",
+ "ended_date",
+ "completed_date",
+ "original_board_id",
+ },
+ )
+ dataflowTester.VerifyTable(
+ ticket.BoardSprint{},
+ fmt.Sprintf("./snapshot_tables/%s.csv", ticket.BoardSprint{}.TableName()),
+ []string{
+ "board_id",
+ "sprint_id",
+ },
+ []string{
+ "_raw_data_params",
+ "_raw_data_table",
+ "_raw_data_id",
+ "_raw_data_remark",
+ },
+ )
+
+}
diff --git a/plugins/tapd/e2e/snapshot_tables/_tool_tapd_iterations.csv b/plugins/tapd/e2e/snapshot_tables/_tool_tapd_iterations.csv
new file mode 100644
index 00000000..e33e1746
--- /dev/null
+++ b/plugins/tapd/e2e/snapshot_tables/_tool_tapd_iterations.csv
@@ -0,0 +1,11 @@
+connection_id,id,name,workspace_id,startdate,enddate,status,release_id,description,creator,created,modified,completed,releaseowner,launchdate,notice,releasename,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
+1,11991001000077,test-11test-11test-11test-11test-11test-11,991,2020-02-09T16:00:00.000+00:00,2020-02-20T16:00:00.000+00:00,done,0,test-11test-11test-11test-11test-11test-11test-11test-11,TAPD,2020-02-13T02:16:21.000+00:00,2021-06-21T02:44:22.000+00:00,2021-06-21T02:44:22.000+00:00,,,,,"{""ConnectionId"":1,""CompanyId"":99,""WorkspaceId"":991}",_raw_tapd_api_iterations,1,
+1,11991001000078,test-11test-11test-11test-11test-11test-11,991,2020-02-16T16:00:00.000+00:00,2020-02-27T16:00:00.000+00:00,done,0,test-11test-11test-11test-11test-11test-11test-11test-11,TAPD,2020-02-13T02:16:21.000+00:00,2021-06-21T02:44:22.000+00:00,2021-06-21T02:44:22.000+00:00,,,,,"{""ConnectionId"":1,""CompanyId"":99,""WorkspaceId"":991}",_raw_tapd_api_iterations,2,
+1,11991001000099,test-11test-11test-11test-11test-11test-11test-11,991,2020-02-24T16:00:00.000+00:00,2020-03-23T16:00:00.000+00:00,done,0,,test-11test-11,2020-02-25T06:45:45.000+00:00,2021-06-21T02:44:22.000+00:00,2021-06-21T02:44:22.000+00:00,,,,,"{""ConnectionId"":1,""CompanyId"":99,""WorkspaceId"":991}",_raw_tapd_api_iterations,3,
+1,11991001000205,test-11test-11test-11test-11test-11test-11test-11test-11test-11test-11test-11,991,2020-04-28T16:00:00.000+00:00,2020-05-13T16:00:00.000+00:00,done,0,,suhaonan,2020-04-29T02:32:41.000+00:00,2021-06-21T02:44:22.000+00:00,2021-06-21T02:44:22.000+00:00,,,,,"{""ConnectionId"":1,""CompanyId"":99,""WorkspaceId"":991}",_raw_tapd_api_iterations,4,
+1,11991001000353,test-11test-11test-11test-11test-11test-11test-11test-11test-11test-11test-11test-11test-11test-11,991,2020-06-17T16:00:00.000+00:00,2020-07-30T16:00:00.000+00:00,done,0,,test-11test-11,2020-06-18T09:57:55.000+00:00,2021-06-21T02:44:22.000+00:00,2021-06-21T02:44:22.000+00:00,,,,,"{""ConnectionId"":1,""CompanyId"":99,""WorkspaceId"":991}",_raw_tapd_api_iterations,5,
+1,11991001000379,test-11test-11test-11test-11test-11,991,2020-07-19T16:00:00.000+00:00,2020-07-30T16:00:00.000+00:00,done,0,,test-11test-11test-11,2020-07-20T10:25:32.000+00:00,2021-06-21T02:44:22.000+00:00,2021-06-21T02:44:22.000+00:00,,,,,"{""ConnectionId"":1,""CompanyId"":99,""WorkspaceId"":991}",_raw_tapd_api_iterations,6,
+1,11991001000381,PCtest-11test-11test-11test-11test-11test-11test-11test-11,991,2020-07-20T16:00:00.000+00:00,2020-07-29T16:00:00.000+00:00,done,0,,test-11test-11test-11,2020-07-21T06:56:13.000+00:00,2021-06-21T02:44:22.000+00:00,2021-06-21T02:44:22.000+00:00,,,,,"{""ConnectionId"":1,""CompanyId"":99,""WorkspaceId"":991}",_raw_tapd_api_iterations,7,
+1,11991001000395,test-11test-11test-11test-11,991,2020-08-06T16:00:00.000+00:00,2020-08-30T16:00:00.000+00:00,done,0,,test-11test-11test-11,2020-08-07T08:45:02.000+00:00,2021-06-21T02:44:22.000+00:00,2021-06-21T02:44:22.000+00:00,,,,,"{""ConnectionId"":1,""CompanyId"":99,""WorkspaceId"":991}",_raw_tapd_api_iterations,8,
+1,11991001000404,test-11test-11test-11test-11test-11test-11test-11test-11test-11test-11test-11test-11test-11test-11test-11test-11test-11,991,2020-08-20T16:00:00.000+00:00,2020-08-25T16:00:00.000+00:00,done,0,<div> 1. test-11test-11test-11test-11test-11test-11test-11test-11test-11test-11test-11test-11test-11test-11test-11test-11test-11test-11test-11test-11test-11test-11test-11test-11 test-11test-11 test-11test-11test-11test-11test-11test-11test-11test-11test-11test-11test-11test-11te [...]
+1,11991001000406,test-11test-11test-11test-11test-11test-11test-11test-11test-11,991,2020-08-16T16:00:00.000+00:00,2020-08-30T16:00:00.000+00:00,done,0,,test-11test-11,2020-08-24T07:28:32.000+00:00,2021-06-21T02:44:22.000+00:00,2021-06-21T02:44:22.000+00:00,,,,,"{""ConnectionId"":1,""CompanyId"":99,""WorkspaceId"":991}",_raw_tapd_api_iterations,10,
diff --git a/plugins/tapd/e2e/snapshot_tables/_tool_tapd_workspace_iterations.csv b/plugins/tapd/e2e/snapshot_tables/_tool_tapd_workspace_iterations.csv
new file mode 100644
index 00000000..89c0e4ac
--- /dev/null
+++ b/plugins/tapd/e2e/snapshot_tables/_tool_tapd_workspace_iterations.csv
@@ -0,0 +1,11 @@
+connection_id,workspace_id,iteration_id,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
+1,991,11991001000077,"{""ConnectionId"":1,""CompanyId"":99,""WorkspaceId"":991}",_raw_tapd_api_iterations,1,
+1,991,11991001000078,"{""ConnectionId"":1,""CompanyId"":99,""WorkspaceId"":991}",_raw_tapd_api_iterations,2,
+1,991,11991001000099,"{""ConnectionId"":1,""CompanyId"":99,""WorkspaceId"":991}",_raw_tapd_api_iterations,3,
+1,991,11991001000205,"{""ConnectionId"":1,""CompanyId"":99,""WorkspaceId"":991}",_raw_tapd_api_iterations,4,
+1,991,11991001000353,"{""ConnectionId"":1,""CompanyId"":99,""WorkspaceId"":991}",_raw_tapd_api_iterations,5,
+1,991,11991001000379,"{""ConnectionId"":1,""CompanyId"":99,""WorkspaceId"":991}",_raw_tapd_api_iterations,6,
+1,991,11991001000381,"{""ConnectionId"":1,""CompanyId"":99,""WorkspaceId"":991}",_raw_tapd_api_iterations,7,
+1,991,11991001000395,"{""ConnectionId"":1,""CompanyId"":99,""WorkspaceId"":991}",_raw_tapd_api_iterations,8,
+1,991,11991001000404,"{""ConnectionId"":1,""CompanyId"":99,""WorkspaceId"":991}",_raw_tapd_api_iterations,9,
+1,991,11991001000406,"{""ConnectionId"":1,""CompanyId"":99,""WorkspaceId"":991}",_raw_tapd_api_iterations,10,
diff --git a/plugins/tapd/e2e/snapshot_tables/board_sprints.csv b/plugins/tapd/e2e/snapshot_tables/board_sprints.csv
new file mode 100644
index 00000000..9c7d295b
--- /dev/null
+++ b/plugins/tapd/e2e/snapshot_tables/board_sprints.csv
@@ -0,0 +1,11 @@
+board_id,sprint_id,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
+tapd:TapdWorkspace:1:991,tapd:TapdIteration:1:11991001000077,"{""ConnectionId"":1,""CompanyId"":99,""WorkspaceId"":991}",_raw_tapd_api_iterations,1,
+tapd:TapdWorkspace:1:991,tapd:TapdIteration:1:11991001000078,"{""ConnectionId"":1,""CompanyId"":99,""WorkspaceId"":991}",_raw_tapd_api_iterations,2,
+tapd:TapdWorkspace:1:991,tapd:TapdIteration:1:11991001000099,"{""ConnectionId"":1,""CompanyId"":99,""WorkspaceId"":991}",_raw_tapd_api_iterations,3,
+tapd:TapdWorkspace:1:991,tapd:TapdIteration:1:11991001000205,"{""ConnectionId"":1,""CompanyId"":99,""WorkspaceId"":991}",_raw_tapd_api_iterations,4,
+tapd:TapdWorkspace:1:991,tapd:TapdIteration:1:11991001000353,"{""ConnectionId"":1,""CompanyId"":99,""WorkspaceId"":991}",_raw_tapd_api_iterations,5,
+tapd:TapdWorkspace:1:991,tapd:TapdIteration:1:11991001000379,"{""ConnectionId"":1,""CompanyId"":99,""WorkspaceId"":991}",_raw_tapd_api_iterations,6,
+tapd:TapdWorkspace:1:991,tapd:TapdIteration:1:11991001000381,"{""ConnectionId"":1,""CompanyId"":99,""WorkspaceId"":991}",_raw_tapd_api_iterations,7,
+tapd:TapdWorkspace:1:991,tapd:TapdIteration:1:11991001000395,"{""ConnectionId"":1,""CompanyId"":99,""WorkspaceId"":991}",_raw_tapd_api_iterations,8,
+tapd:TapdWorkspace:1:991,tapd:TapdIteration:1:11991001000404,"{""ConnectionId"":1,""CompanyId"":99,""WorkspaceId"":991}",_raw_tapd_api_iterations,9,
+tapd:TapdWorkspace:1:991,tapd:TapdIteration:1:11991001000406,"{""ConnectionId"":1,""CompanyId"":99,""WorkspaceId"":991}",_raw_tapd_api_iterations,10,
diff --git a/plugins/tapd/e2e/snapshot_tables/boards.csv b/plugins/tapd/e2e/snapshot_tables/boards.csv
new file mode 100644
index 00000000..1e80682c
--- /dev/null
+++ b/plugins/tapd/e2e/snapshot_tables/boards.csv
@@ -0,0 +1 @@
+id,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark,name,description,url,created_date
diff --git a/plugins/tapd/e2e/snapshot_tables/sprints.csv b/plugins/tapd/e2e/snapshot_tables/sprints.csv
new file mode 100644
index 00000000..d333c533
--- /dev/null
+++ b/plugins/tapd/e2e/snapshot_tables/sprints.csv
@@ -0,0 +1,11 @@
+id,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark,name,url,status,started_date,ended_date,completed_date,original_board_id
+tapd:TapdIteration:1:11991001000077,"{""ConnectionId"":1,""CompanyId"":99,""WorkspaceId"":991}",_raw_tapd_api_iterations,1,,test-11test-11test-11test-11test-11test-11,https://www.tapd.cn/991/prong/iterations/view/11991001000077,DONE,2020-02-09T16:00:00.000+00:00,2020-02-20T16:00:00.000+00:00,2021-06-21T02:44:22.000+00:00,tapd:TapdWorkspace:1:991
+tapd:TapdIteration:1:11991001000078,"{""ConnectionId"":1,""CompanyId"":99,""WorkspaceId"":991}",_raw_tapd_api_iterations,2,,test-11test-11test-11test-11test-11test-11,https://www.tapd.cn/991/prong/iterations/view/11991001000078,DONE,2020-02-16T16:00:00.000+00:00,2020-02-27T16:00:00.000+00:00,2021-06-21T02:44:22.000+00:00,tapd:TapdWorkspace:1:991
+tapd:TapdIteration:1:11991001000099,"{""ConnectionId"":1,""CompanyId"":99,""WorkspaceId"":991}",_raw_tapd_api_iterations,3,,test-11test-11test-11test-11test-11test-11test-11,https://www.tapd.cn/991/prong/iterations/view/11991001000099,DONE,2020-02-24T16:00:00.000+00:00,2020-03-23T16:00:00.000+00:00,2021-06-21T02:44:22.000+00:00,tapd:TapdWorkspace:1:991
+tapd:TapdIteration:1:11991001000205,"{""ConnectionId"":1,""CompanyId"":99,""WorkspaceId"":991}",_raw_tapd_api_iterations,4,,test-11test-11test-11test-11test-11test-11test-11test-11test-11test-11test-11,https://www.tapd.cn/991/prong/iterations/view/11991001000205,DONE,2020-04-28T16:00:00.000+00:00,2020-05-13T16:00:00.000+00:00,2021-06-21T02:44:22.000+00:00,tapd:TapdWorkspace:1:991
+tapd:TapdIteration:1:11991001000353,"{""ConnectionId"":1,""CompanyId"":99,""WorkspaceId"":991}",_raw_tapd_api_iterations,5,,test-11test-11test-11test-11test-11test-11test-11test-11test-11test-11test-11test-11test-11test-11,https://www.tapd.cn/991/prong/iterations/view/11991001000353,DONE,2020-06-17T16:00:00.000+00:00,2020-07-30T16:00:00.000+00:00,2021-06-21T02:44:22.000+00:00,tapd:TapdWorkspace:1:991
+tapd:TapdIteration:1:11991001000379,"{""ConnectionId"":1,""CompanyId"":99,""WorkspaceId"":991}",_raw_tapd_api_iterations,6,,test-11test-11test-11test-11test-11,https://www.tapd.cn/991/prong/iterations/view/11991001000379,DONE,2020-07-19T16:00:00.000+00:00,2020-07-30T16:00:00.000+00:00,2021-06-21T02:44:22.000+00:00,tapd:TapdWorkspace:1:991
+tapd:TapdIteration:1:11991001000381,"{""ConnectionId"":1,""CompanyId"":99,""WorkspaceId"":991}",_raw_tapd_api_iterations,7,,PCtest-11test-11test-11test-11test-11test-11test-11test-11,https://www.tapd.cn/991/prong/iterations/view/11991001000381,DONE,2020-07-20T16:00:00.000+00:00,2020-07-29T16:00:00.000+00:00,2021-06-21T02:44:22.000+00:00,tapd:TapdWorkspace:1:991
+tapd:TapdIteration:1:11991001000395,"{""ConnectionId"":1,""CompanyId"":99,""WorkspaceId"":991}",_raw_tapd_api_iterations,8,,test-11test-11test-11test-11,https://www.tapd.cn/991/prong/iterations/view/11991001000395,DONE,2020-08-06T16:00:00.000+00:00,2020-08-30T16:00:00.000+00:00,2021-06-21T02:44:22.000+00:00,tapd:TapdWorkspace:1:991
+tapd:TapdIteration:1:11991001000404,"{""ConnectionId"":1,""CompanyId"":99,""WorkspaceId"":991}",_raw_tapd_api_iterations,9,,test-11test-11test-11test-11test-11test-11test-11test-11test-11test-11test-11test-11test-11test-11test-11test-11test-11,https://www.tapd.cn/991/prong/iterations/view/11991001000404,DONE,2020-08-20T16:00:00.000+00:00,2020-08-25T16:00:00.000+00:00,2021-06-21T02:44:22.000+00:00,tapd:TapdWorkspace:1:991
+tapd:TapdIteration:1:11991001000406,"{""ConnectionId"":1,""CompanyId"":99,""WorkspaceId"":991}",_raw_tapd_api_iterations,10,,test-11test-11test-11test-11test-11test-11test-11test-11test-11,https://www.tapd.cn/991/prong/iterations/view/11991001000406,DONE,2020-08-16T16:00:00.000+00:00,2020-08-30T16:00:00.000+00:00,2021-06-21T02:44:22.000+00:00,tapd:TapdWorkspace:1:991
diff --git a/plugins/tapd/e2e/sub_workspace_test.go b/plugins/tapd/e2e/sub_workspace_test.go
index a331c272..fd5dd2af 100644
--- a/plugins/tapd/e2e/sub_workspace_test.go
+++ b/plugins/tapd/e2e/sub_workspace_test.go
@@ -19,6 +19,7 @@ package e2e
import (
"fmt"
+ "github.com/apache/incubator-devlake/models/domainlayer/ticket"
"testing"
"github.com/apache/incubator-devlake/helpers/e2ehelper"
@@ -68,4 +69,22 @@ func TestTapdWorkspaceDataFlow(t *testing.T) {
},
)
+ dataflowTester.FlushTabler(&ticket.Board{})
+ dataflowTester.Subtask(tasks.ConvertSubWorkspaceMeta, taskData)
+ dataflowTester.VerifyTable(
+ ticket.Board{},
+ fmt.Sprintf("./snapshot_tables/%s.csv", ticket.Board{}.TableName()),
+ []string{"id"},
+ []string{
+ "_raw_data_params",
+ "_raw_data_table",
+ "_raw_data_id",
+ "_raw_data_remark",
+ "name",
+ "description",
+ "url",
+ "created_date",
+ },
+ )
+
}
diff --git a/plugins/tapd/tasks/iteration_converter.go b/plugins/tapd/tasks/iteration_converter.go
index 99be3df4..d4141d16 100644
--- a/plugins/tapd/tasks/iteration_converter.go
+++ b/plugins/tapd/tasks/iteration_converter.go
@@ -48,6 +48,9 @@ func ConvertIteration(taskCtx core.SubTaskContext) error {
return err
}
defer cursor.Close()
+
+ workspaceIdGen := didgen.NewDomainIdGenerator(&models.TapdWorkspace{})
+
converter, err := helper.NewDataConverter(helper.DataConverterArgs{
RawDataSubTaskArgs: *rawDataSubTaskArgs,
InputRowType: reflect.TypeOf(models.TapdIteration{}),
@@ -61,7 +64,7 @@ func ConvertIteration(taskCtx core.SubTaskContext) error {
Name: iter.Name,
StartedDate: (*time.Time)(iter.Startdate),
EndedDate: (*time.Time)(iter.Enddate),
- OriginalBoardID: WorkspaceIdGen.Generate(iter.ConnectionId, iter.WorkspaceId),
+ OriginalBoardID: workspaceIdGen.Generate(iter.ConnectionId, iter.WorkspaceId),
CompletedDate: (*time.Time)(iter.Completed),
}
results := make([]interface{}, 0)