You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by ab...@apache.org on 2022/06/21 15:04:43 UTC
[incubator-devlake] branch main updated: refactor(tapd): add cadd e2e -- workspace
This is an automated email from the ASF dual-hosted git repository.
abeizn pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new dab6c059 refactor(tapd): add cadd e2e -- workspace
dab6c059 is described below
commit dab6c0592a1e1daf2b174c1c19a5ca750f0581f6
Author: Yingchu Chen <yi...@merico.dev>
AuthorDate: Tue Jun 21 22:51:39 2022 +0800
refactor(tapd): add cadd e2e -- workspace
---
.../_tool_tapd_workspaces_ws_test.csv | 11 ++
plugins/tapd/e2e/workspace_test.go | 72 +++++++
plugins/tapd/models/story.go | 208 ++++++++++-----------
plugins/tapd/tasks/company_extractor.go | 4 +
plugins/tapd/tasks/workspace_extractor.go | 9 +-
5 files changed, 195 insertions(+), 109 deletions(-)
diff --git a/plugins/tapd/e2e/snapshot_tables/_tool_tapd_workspaces_ws_test.csv b/plugins/tapd/e2e/snapshot_tables/_tool_tapd_workspaces_ws_test.csv
new file mode 100644
index 00000000..06f034db
--- /dev/null
+++ b/plugins/tapd/e2e/snapshot_tables/_tool_tapd_workspaces_ws_test.csv
@@ -0,0 +1,11 @@
+connection_id,id,name,pretty_name,category,status,description,begin_date,end_date,external_on,parent_id,creator,created,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
+1,90,abc0,90,,closed,财道社区委托账户,,,,40907,huadd (huadd@abeizn.com),2020-01-03T02:47:15.000+00:00,"{""ConnectionId"":1,""CompanyId"":99,""WorkspaceId"":991}",_raw_tapd_api_sub_workspaces,10,
+1,91,abc2,91,,normal,财经APP,,,,40908,feiwang (feiwang@abeizn.com),2019-12-12T07:45:04.000+00:00,"{""ConnectionId"":1,""CompanyId"":99,""WorkspaceId"":991}",_raw_tapd_api_sub_workspaces,2,
+1,92,abc3,92,,normal,财道文字直播室,,,,40908,huadd (huadd@abeizn.com),2019-12-13T08:58:09.000+00:00,"{""ConnectionId"":1,""CompanyId"":99,""WorkspaceId"":991}",_raw_tapd_api_sub_workspaces,3,
+1,93,abc4,93,,normal,证券事业部财道社区促销项目,,,,40908,huadd (huadd@abeizn.com),2019-12-23T03:08:30.000+00:00,"{""ConnectionId"":1,""CompanyId"":99,""WorkspaceId"":991}",_raw_tapd_api_sub_workspaces,4,
+1,94,abc5,94,,normal,财道App,,,,40908,chenzj (chenzj@abeizn.com),2019-12-26T10:18:28.000+00:00,"{""ConnectionId"":1,""CompanyId"":99,""WorkspaceId"":991}",_raw_tapd_api_sub_workspaces,5,
+1,95,abc6,95,,closed,,,,,99,chenzj (chenzj@abeizn.com),2019-12-27T01:20:23.000+00:00,"{""ConnectionId"":1,""CompanyId"":99,""WorkspaceId"":991}",_raw_tapd_api_sub_workspaces,6,
+1,96,abc7,96,,normal,财道社区量化精选项目,,,,40908,huadd (huadd@abeizn.com),2020-01-03T00:59:08.000+00:00,"{""ConnectionId"":1,""CompanyId"":99,""WorkspaceId"":991}",_raw_tapd_api_sub_workspaces,7,
+1,97,abc8,97,,normal,包括财道社区频道页、老师工作室、语音课、用户中心等内容,,,,40908,huadd (huadd@abeizn.com),2020-01-03T01:47:24.000+00:00,"{""ConnectionId"":1,""CompanyId"":99,""WorkspaceId"":991}",_raw_tapd_api_sub_workspaces,8,
+1,98,abc9,98,,normal,用户登录、注册、鉴权等,,,,40907,huadd (huadd@abeizn.com),2020-01-03T01:52:45.000+00:00,"{""ConnectionId"":1,""CompanyId"":99,""WorkspaceId"":991}",_raw_tapd_api_sub_workspaces,9,
+1,99,abc1,99,,normal,,,,,1,马畅 (machang@abeizn.com),2019-11-29T07:55:53.000+00:00,"{""ConnectionId"":1,""CompanyId"":99,""WorkspaceId"":991}",_raw_tapd_api_sub_workspaces,1,
diff --git a/plugins/tapd/e2e/workspace_test.go b/plugins/tapd/e2e/workspace_test.go
new file mode 100644
index 00000000..65d1149a
--- /dev/null
+++ b/plugins/tapd/e2e/workspace_test.go
@@ -0,0 +1,72 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+ "fmt"
+ "testing"
+
+ "github.com/apache/incubator-devlake/helpers/e2ehelper"
+ "github.com/apache/incubator-devlake/plugins/tapd/impl"
+ "github.com/apache/incubator-devlake/plugins/tapd/models"
+ "github.com/apache/incubator-devlake/plugins/tapd/tasks"
+)
+
+func TestTapdWorkspaceDataFlow(t *testing.T) {
+
+ var tapd impl.Tapd
+ dataflowTester := e2ehelper.NewDataFlowTester(t, "tapd", tapd)
+
+ taskData := &tasks.TapdTaskData{
+ Options: &tasks.TapdOptions{
+ ConnectionId: 1,
+ CompanyId: 99,
+ WorkspaceId: 991,
+ },
+ }
+ // import raw data table
+ dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_tapd_api_companies.csv",
+ "_raw_tapd_api_sub_workspaces")
+
+ // verify extraction
+ dataflowTester.FlushTabler(&models.TapdWorkspace{})
+ dataflowTester.Subtask(tasks.ExtractWorkspaceMeta, taskData)
+ dataflowTester.VerifyTable(
+ models.TapdWorkspace{},
+ fmt.Sprintf("./snapshot_tables/%s_ws_test.csv", models.TapdWorkspace{}.TableName()),
+ []string{"connection_id", "id"},
+ []string{
+ "name",
+ "pretty_name",
+ "category",
+ "status",
+ "description",
+ "begin_date",
+ "end_date",
+ "external_on",
+ "parent_id",
+ "creator",
+ "created",
+ "_raw_data_params",
+ "_raw_data_table",
+ "_raw_data_id",
+ "_raw_data_remark",
+ },
+ )
+
+}
diff --git a/plugins/tapd/models/story.go b/plugins/tapd/models/story.go
index 16264ace..15c8bdec 100644
--- a/plugins/tapd/models/story.go
+++ b/plugins/tapd/models/story.go
@@ -23,110 +23,110 @@ import (
)
type TapdStory struct {
- ConnectionId uint64 `gorm:"primaryKey"`
- Id uint64 `gorm:"primaryKey;type:BIGINT" json:"id,string"`
- WorkitemTypeId uint64 `json:"workitem_type_id,string"`
- Name string `gorm:"type:varchar(255)" json:"name"`
- Description string `json:"description"`
- WorkspaceId uint64 `json:"workspace_id,string"`
- Creator string `gorm:"type:varchar(255)"`
- Created *helper.CSTTime `json:"created"`
- Modified *helper.CSTTime `json:"modified" gorm:"index"`
- Status string `json:"status" gorm:"type:varchar(255)"`
- Owner string `json:"owner" gorm:"type:varchar(255)"`
- Cc string `json:"cc" gorm:"type:varchar(255)"`
- Begin *helper.CSTTime `json:"begin"`
- Due *helper.CSTTime `json:"due"`
- Size int16 `json:"size,string"`
- Priority string `gorm:"type:varchar(255)" json:"priority"`
- Developer string `gorm:"type:varchar(255)" json:"developer"`
- IterationId uint64 `json:"iteration_id,string"`
- TestFocus string `json:"test_focus" gorm:"type:varchar(255)"`
- Type string `json:"type" gorm:"type:varchar(20)"`
- Source string `json:"source" gorm:"type:varchar(255)"`
- Module string `json:"module" gorm:"type:varchar(255)"`
- Version string `json:"version" gorm:"type:varchar(255)"`
- Completed *helper.CSTTime `json:"completed"`
- CategoryId int64 `json:"category_id,string"`
- Path string `gorm:"type:varchar(255)" json:"path"`
- ParentId uint64 `json:"parent_id,string"`
- ChildrenId string `gorm:"type:text" json:"children_id"`
- AncestorId uint64 `json:"ancestor_id,string"`
- BusinessValue string `gorm:"type:varchar(255)" json:"business_value"`
- Effort float32 `json:"effort,string"`
- EffortCompleted float32 `json:"effort_completed,string"`
- Exceed float32 `json:"exceed,string"`
- Remain float32 `json:"remain,string"`
- ReleaseId uint64 `json:"release_id,string"`
- Confidential string `gorm:"type:varchar(255)" json:"confidential"`
- TemplatedId uint64 `json:"templated_id,string"`
- CreatedFrom string `gorm:"type:varchar(255)" json:"created_from"`
- Feature string `gorm:"type:varchar(255)" json:"feature"`
- StdStatus string `gorm:"type:varchar(20)"`
- StdType string `gorm:"type:varchar(20)"`
- Url string `gorm:"type:varchar(255)"`
-
- AttachmentCount int16 `json:"attachment_count,string"`
- HasAttachment string `json:"has_attachment" gorm:"type:varchar(255)"`
- BugId uint64 `json:"bug_id,string"`
- Follower string `json:"follower" gorm:"type:varchar(255)"`
- SyncType string `json:"sync_type" gorm:"type:text"`
- PredecessorCount int16 `json:"predecessor_count,string"`
- IsArchived string `json:"is_archived" gorm:"type:varchar(255)"`
- Modifier string `json:"modifier" gorm:"type:varchar(255)"`
- ProgressManual string `json:"progress_manual" gorm:"type:varchar(255)"`
- SuccessorCount int16 `json:"successor_count,string"`
- Label string `json:"label" gorm:"type:varchar(255)"`
- CustomFieldOne string `json:"custom_field_one" gorm:"type:text"`
- CustomFieldTwo string `json:"custom_field_two" gorm:"type:text"`
- CustomFieldThree string `json:"custom_field_three" gorm:"type:text"`
- CustomFieldFour string `json:"custom_field_four" gorm:"type:text"`
- CustomFieldFive string `json:"custom_field_five" gorm:"type:text"`
- CustomField6 string `json:"custom_field_6" gorm:"type:text"`
- CustomField7 string `json:"custom_field_7" gorm:"type:text"`
- CustomField8 string `json:"custom_field_8" gorm:"type:text"`
- CustomField9 string `json:"custom_field_9" gorm:"type:text"`
- CustomField10 string `json:"custom_field_10" gorm:"type:text"`
- CustomField11 string `json:"custom_field_11" gorm:"type:text"`
- CustomField12 string `json:"custom_field_12" gorm:"type:text"`
- CustomField13 string `json:"custom_field_13" gorm:"type:text"`
- CustomField14 string `json:"custom_field_14" gorm:"type:text"`
- CustomField15 string `json:"custom_field_15" gorm:"type:text"`
- CustomField16 string `json:"custom_field_16" gorm:"type:text"`
- CustomField17 string `json:"custom_field_17" gorm:"type:text"`
- CustomField18 string `json:"custom_field_18" gorm:"type:text"`
- CustomField19 string `json:"custom_field_19" gorm:"type:text"`
- CustomField20 string `json:"custom_field_20" gorm:"type:text"`
- CustomField21 string `json:"custom_field_21" gorm:"type:text"`
- CustomField22 string `json:"custom_field_22" gorm:"type:text"`
- CustomField23 string `json:"custom_field_23" gorm:"type:text"`
- CustomField24 string `json:"custom_field_24" gorm:"type:text"`
- CustomField25 string `json:"custom_field_25" gorm:"type:text"`
- CustomField26 string `json:"custom_field_26" gorm:"type:text"`
- CustomField27 string `json:"custom_field_27" gorm:"type:text"`
- CustomField28 string `json:"custom_field_28" gorm:"type:text"`
- CustomField29 string `json:"custom_field_29" gorm:"type:text"`
- CustomField30 string `json:"custom_field_30" gorm:"type:text"`
- CustomField31 string `json:"custom_field_31" gorm:"type:text"`
- CustomField32 string `json:"custom_field_32" gorm:"type:text"`
- CustomField33 string `json:"custom_field_33" gorm:"type:text"`
- CustomField34 string `json:"custom_field_34" gorm:"type:text"`
- CustomField35 string `json:"custom_field_35" gorm:"type:text"`
- CustomField36 string `json:"custom_field_36" gorm:"type:text"`
- CustomField37 string `json:"custom_field_37" gorm:"type:text"`
- CustomField38 string `json:"custom_field_38" gorm:"type:text"`
- CustomField39 string `json:"custom_field_39" gorm:"type:text"`
- CustomField40 string `json:"custom_field_40" gorm:"type:text"`
- CustomField41 string `json:"custom_field_41" gorm:"type:text"`
- CustomField42 string `json:"custom_field_42" gorm:"type:text"`
- CustomField43 string `json:"custom_field_43" gorm:"type:text"`
- CustomField44 string `json:"custom_field_44" gorm:"type:text"`
- CustomField45 string `json:"custom_field_45" gorm:"type:text"`
- CustomField46 string `json:"custom_field_46" gorm:"type:text"`
- CustomField47 string `json:"custom_field_47" gorm:"type:text"`
- CustomField48 string `json:"custom_field_48" gorm:"type:text"`
- CustomField49 string `json:"custom_field_49" gorm:"type:text"`
- CustomField50 string `json:"custom_field_50" gorm:"type:text"`
+ ConnectionId uint64 `gorm:"primaryKey"`
+ Id uint64 `gorm:"primaryKey;type:BIGINT" json:"id,string"`
+ WorkitemTypeId uint64 `json:"workitem_type_id,string"`
+ Name string `gorm:"type:varchar(255)" json:"name"`
+ Description string `json:"description"`
+ WorkspaceId uint64 `json:"workspace_id,string"`
+ Creator string `gorm:"type:varchar(255)"`
+ Created *helper.CSTTime `json:"created"`
+ Modified *helper.CSTTime `json:"modified" gorm:"index"`
+ Status string `json:"status" gorm:"type:varchar(255)"`
+ Owner string `json:"owner" gorm:"type:varchar(255)"`
+ Cc string `json:"cc" gorm:"type:varchar(255)"`
+ Begin *helper.CSTTime `json:"begin"`
+ Due *helper.CSTTime `json:"due"`
+ Size int16 `json:"size,string"`
+ Priority string `gorm:"type:varchar(255)" json:"priority"`
+ Developer string `gorm:"type:varchar(255)" json:"developer"`
+ IterationId uint64 `json:"iteration_id,string"`
+ TestFocus string `json:"test_focus" gorm:"type:varchar(255)"`
+ Type string `json:"type" gorm:"type:varchar(20)"`
+ Source string `json:"source" gorm:"type:varchar(255)"`
+ Module string `json:"module" gorm:"type:varchar(255)"`
+ Version string `json:"version" gorm:"type:varchar(255)"`
+ Completed *helper.CSTTime `json:"completed"`
+ CategoryId int64 `json:"category_id,string"`
+ Path string `gorm:"type:varchar(255)" json:"path"`
+ ParentId uint64 `json:"parent_id,string"`
+ ChildrenId string `gorm:"type:text" json:"children_id"`
+ AncestorId uint64 `json:"ancestor_id,string"`
+ BusinessValue string `gorm:"type:varchar(255)" json:"business_value"`
+ Effort float32 `json:"effort,string"`
+ EffortCompleted float32 `json:"effort_completed,string"`
+ Exceed float32 `json:"exceed,string"`
+ Remain float32 `json:"remain,string"`
+ ReleaseId uint64 `json:"release_id,string"`
+ Confidential string `gorm:"type:varchar(255)" json:"confidential"`
+ TemplatedId uint64 `json:"templated_id,string"`
+ CreatedFrom string `gorm:"type:varchar(255)" json:"created_from"`
+ Feature string `gorm:"type:varchar(255)" json:"feature"`
+ StdStatus string `gorm:"type:varchar(20)"`
+ StdType string `gorm:"type:varchar(20)"`
+ Url string `gorm:"type:varchar(255)"`
+
+ AttachmentCount int16 `json:"attachment_count,string"`
+ HasAttachment string `json:"has_attachment" gorm:"type:varchar(255)"`
+ BugId uint64 `json:"bug_id,string"`
+ Follower string `json:"follower" gorm:"type:varchar(255)"`
+ SyncType string `json:"sync_type" gorm:"type:text"`
+ PredecessorCount int16 `json:"predecessor_count,string"`
+ IsArchived string `json:"is_archived" gorm:"type:varchar(255)"`
+ Modifier string `json:"modifier" gorm:"type:varchar(255)"`
+ ProgressManual string `json:"progress_manual" gorm:"type:varchar(255)"`
+ SuccessorCount int16 `json:"successor_count,string"`
+ Label string `json:"label" gorm:"type:varchar(255)"`
+ CustomFieldOne string `json:"custom_field_one" gorm:"type:text"`
+ CustomFieldTwo string `json:"custom_field_two" gorm:"type:text"`
+ CustomFieldThree string `json:"custom_field_three" gorm:"type:text"`
+ CustomFieldFour string `json:"custom_field_four" gorm:"type:text"`
+ CustomFieldFive string `json:"custom_field_five" gorm:"type:text"`
+ CustomField6 string `json:"custom_field_6" gorm:"type:text"`
+ CustomField7 string `json:"custom_field_7" gorm:"type:text"`
+ CustomField8 string `json:"custom_field_8" gorm:"type:text"`
+ CustomField9 string `json:"custom_field_9" gorm:"type:text"`
+ CustomField10 string `json:"custom_field_10" gorm:"type:text"`
+ CustomField11 string `json:"custom_field_11" gorm:"type:text"`
+ CustomField12 string `json:"custom_field_12" gorm:"type:text"`
+ CustomField13 string `json:"custom_field_13" gorm:"type:text"`
+ CustomField14 string `json:"custom_field_14" gorm:"type:text"`
+ CustomField15 string `json:"custom_field_15" gorm:"type:text"`
+ CustomField16 string `json:"custom_field_16" gorm:"type:text"`
+ CustomField17 string `json:"custom_field_17" gorm:"type:text"`
+ CustomField18 string `json:"custom_field_18" gorm:"type:text"`
+ CustomField19 string `json:"custom_field_19" gorm:"type:text"`
+ CustomField20 string `json:"custom_field_20" gorm:"type:text"`
+ CustomField21 string `json:"custom_field_21" gorm:"type:text"`
+ CustomField22 string `json:"custom_field_22" gorm:"type:text"`
+ CustomField23 string `json:"custom_field_23" gorm:"type:text"`
+ CustomField24 string `json:"custom_field_24" gorm:"type:text"`
+ CustomField25 string `json:"custom_field_25" gorm:"type:text"`
+ CustomField26 string `json:"custom_field_26" gorm:"type:text"`
+ CustomField27 string `json:"custom_field_27" gorm:"type:text"`
+ CustomField28 string `json:"custom_field_28" gorm:"type:text"`
+ CustomField29 string `json:"custom_field_29" gorm:"type:text"`
+ CustomField30 string `json:"custom_field_30" gorm:"type:text"`
+ CustomField31 string `json:"custom_field_31" gorm:"type:text"`
+ CustomField32 string `json:"custom_field_32" gorm:"type:text"`
+ CustomField33 string `json:"custom_field_33" gorm:"type:text"`
+ CustomField34 string `json:"custom_field_34" gorm:"type:text"`
+ CustomField35 string `json:"custom_field_35" gorm:"type:text"`
+ CustomField36 string `json:"custom_field_36" gorm:"type:text"`
+ CustomField37 string `json:"custom_field_37" gorm:"type:text"`
+ CustomField38 string `json:"custom_field_38" gorm:"type:text"`
+ CustomField39 string `json:"custom_field_39" gorm:"type:text"`
+ CustomField40 string `json:"custom_field_40" gorm:"type:text"`
+ CustomField41 string `json:"custom_field_41" gorm:"type:text"`
+ CustomField42 string `json:"custom_field_42" gorm:"type:text"`
+ CustomField43 string `json:"custom_field_43" gorm:"type:text"`
+ CustomField44 string `json:"custom_field_44" gorm:"type:text"`
+ CustomField45 string `json:"custom_field_45" gorm:"type:text"`
+ CustomField46 string `json:"custom_field_46" gorm:"type:text"`
+ CustomField47 string `json:"custom_field_47" gorm:"type:text"`
+ CustomField48 string `json:"custom_field_48" gorm:"type:text"`
+ CustomField49 string `json:"custom_field_49" gorm:"type:text"`
+ CustomField50 string `json:"custom_field_50" gorm:"type:text"`
common.NoPKModel
}
diff --git a/plugins/tapd/tasks/company_extractor.go b/plugins/tapd/tasks/company_extractor.go
index f508dc9c..5b748983 100644
--- a/plugins/tapd/tasks/company_extractor.go
+++ b/plugins/tapd/tasks/company_extractor.go
@@ -19,6 +19,7 @@ package tasks
import (
"encoding/json"
+ "github.com/apache/incubator-devlake/plugins/tapd/models"
"github.com/apache/incubator-devlake/plugins/core"
"github.com/apache/incubator-devlake/plugins/helper"
@@ -38,6 +39,9 @@ func ExtractCompanies(taskCtx core.SubTaskContext) error {
extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{
RawDataSubTaskArgs: *rawDataSubTaskArgs,
Extract: func(row *helper.RawData) ([]interface{}, error) {
+ var workspaceRes struct {
+ Workspace models.TapdWorkspace
+ }
err := json.Unmarshal(row.Data, &workspaceRes)
if err != nil {
return nil, err
diff --git a/plugins/tapd/tasks/workspace_extractor.go b/plugins/tapd/tasks/workspace_extractor.go
index eebef7ae..37e91180 100644
--- a/plugins/tapd/tasks/workspace_extractor.go
+++ b/plugins/tapd/tasks/workspace_extractor.go
@@ -34,15 +34,14 @@ var ExtractWorkspaceMeta = core.SubTaskMeta{
Description: "Extract raw workspace data into tool layer table _tool_tapd_workspaces",
}
-var workspaceRes struct {
- Workspace models.TapdWorkspace
-}
-
func ExtractWorkspaces(taskCtx core.SubTaskContext) error {
rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_WORKSPACE_TABLE)
extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{
RawDataSubTaskArgs: *rawDataSubTaskArgs,
Extract: func(row *helper.RawData) ([]interface{}, error) {
+ var workspaceRes struct {
+ Workspace models.TapdWorkspace
+ }
err := json.Unmarshal(row.Data, &workspaceRes)
if err != nil {
return nil, err
@@ -50,7 +49,7 @@ func ExtractWorkspaces(taskCtx core.SubTaskContext) error {
ws := workspaceRes.Workspace
- ws.ConnectionId = data.Connection.ID
+ ws.ConnectionId = data.Options.ConnectionId
return []interface{}{
&ws,
}, nil