You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by wa...@apache.org on 2022/06/22 04:04:41 UTC
[incubator-devlake] branch main updated: feat: e2e on tapd user
This is an automated email from the ASF dual-hosted git repository.
warren pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new a52ef879 feat: e2e on tapd user
a52ef879 is described below
commit a52ef8791d118e9b80f537f52c8f972d716081dc
Author: abeizn <zi...@merico.dev>
AuthorDate: Wed Jun 22 11:53:54 2022 +0800
feat: e2e on tapd user
---
models/domainlayer/user/user.go | 4 ++
.../tapd/e2e/raw_tables/_raw_tapd_api_users.csv | 12 ++++
.../tapd/e2e/snapshot_tables/_tool_tapd_users.csv | 12 ++++
plugins/tapd/e2e/snapshot_tables/users.csv | 12 ++++
plugins/tapd/e2e/user_test.go | 82 ++++++++++++++++++++++
plugins/tapd/impl/impl.go | 1 -
plugins/tapd/tasks/user_converter.go | 4 +-
7 files changed, 125 insertions(+), 2 deletions(-)
diff --git a/models/domainlayer/user/user.go b/models/domainlayer/user/user.go
index b10fb4b6..9eccd340 100644
--- a/models/domainlayer/user/user.go
+++ b/models/domainlayer/user/user.go
@@ -28,3 +28,7 @@ type User struct {
AvatarUrl string `gorm:"type:varchar(255)"`
Timezone string `gorm:"type:varchar(255)"`
}
+
+func (User) TableName() string {
+ return "users"
+}
diff --git a/plugins/tapd/e2e/raw_tables/_raw_tapd_api_users.csv b/plugins/tapd/e2e/raw_tables/_raw_tapd_api_users.csv
new file mode 100644
index 00000000..d6329d70
--- /dev/null
+++ b/plugins/tapd/e2e/raw_tables/_raw_tapd_api_users.csv
@@ -0,0 +1,12 @@
+"id","params","data","url","input","created_at"
+"1","{""ConnectionId"":1,""CompanyId"":99,""WorkspaceId"":991}","{""UserWorkspace"":{""user"":""abc"",""role_id"":[""1000000000000000089""],""name"":""abc""}}","https://api.tapd.cn/workspaces/users?workspace_id=991","null","2022-6-21 12:13:07.951"
+"2","{""ConnectionId"":1,""CompanyId"":99,""WorkspaceId"":991}","{""UserWorkspace"":{""user"":""abc1"",""role_id"":[""1000000000000000002""],""name"":""abc1""}}","https://api.tapd.cn/workspaces/users?workspace_id=991","null","2022-6-21 12:13:07.951"
+"3","{""ConnectionId"":1,""CompanyId"":99,""WorkspaceId"":991}","{""UserWorkspace"":{""user"":""abc2"",""role_id"":[""1000000000000000089""],""name"":""abc2""}}","https://api.tapd.cn/workspaces/users?workspace_id=991","null","2022-6-21 12:13:07.951"
+"4","{""ConnectionId"":1,""CompanyId"":99,""WorkspaceId"":991}","{""UserWorkspace"":{""user"":""abc3"",""role_id"":[""11991001000059""],""name"":""abc3""}}","https://api.tapd.cn/workspaces/users?workspace_id=991","null","2022-6-21 12:13:07.951"
+"5","{""ConnectionId"":1,""CompanyId"":99,""WorkspaceId"":991}","{""UserWorkspace"":{""user"":""abc4"",""role_id"":[""1000000000000000089"",""1000000000000000002""],""name"":""abc4""}}","https://api.tapd.cn/workspaces/users?workspace_id=991","null","2022-6-21 12:13:07.951"
+"6","{""ConnectionId"":1,""CompanyId"":99,""WorkspaceId"":991}","{""UserWorkspace"":{""user"":""abc5"",""role_id"":[""1000000000000000089""],""name"":""abc5""}}","https://api.tapd.cn/workspaces/users?workspace_id=991","null","2022-6-21 12:13:07.951"
+"7","{""ConnectionId"":1,""CompanyId"":99,""WorkspaceId"":991}","{""UserWorkspace"":{""user"":""abc6"",""role_id"":[""1000000000000000002""],""name"":""abc6""}}","https://api.tapd.cn/workspaces/users?workspace_id=991","null","2022-6-21 12:13:07.951"
+"8","{""ConnectionId"":1,""CompanyId"":99,""WorkspaceId"":991}","{""UserWorkspace"":{""user"":""abc7"",""role_id"":[""1000000000000000089""],""name"":""abc7""}}","https://api.tapd.cn/workspaces/users?workspace_id=991","null","2022-6-21 12:13:07.951"
+"9","{""ConnectionId"":1,""CompanyId"":99,""WorkspaceId"":991}","{""UserWorkspace"":{""user"":""abc8"",""role_id"":[""1000000000000000089""],""name"":""abc8""}}","https://api.tapd.cn/workspaces/users?workspace_id=991","null","2022-6-21 12:13:07.951"
+"10","{""ConnectionId"":1,""CompanyId"":99,""WorkspaceId"":991}","{""UserWorkspace"":{""user"":""abc9"",""role_id"":[""11991001000014""],""name"":""abc9""}}","https://api.tapd.cn/workspaces/users?workspace_id=991","null","2022-6-21 12:13:07.951"
+"11","{""ConnectionId"":1,""CompanyId"":99,""WorkspaceId"":991}","{""UserWorkspace"":{""user"":""abc10"",""role_id"":[""1000000000000000089""],""name"":""abc10""}}","https://api.tapd.cn/workspaces/users?workspace_id=991","null","2022-6-21 12:13:07.951"
diff --git a/plugins/tapd/e2e/snapshot_tables/_tool_tapd_users.csv b/plugins/tapd/e2e/snapshot_tables/_tool_tapd_users.csv
new file mode 100644
index 00000000..f53a03c8
--- /dev/null
+++ b/plugins/tapd/e2e/snapshot_tables/_tool_tapd_users.csv
@@ -0,0 +1,12 @@
+connection_id,workspace_id,user,name,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
+1,991,abc,abc,"{""ConnectionId"":1,""CompanyId"":99,""WorkspaceId"":991}",_raw_tapd_api_users,1,
+1,991,abc1,abc1,"{""ConnectionId"":1,""CompanyId"":99,""WorkspaceId"":991}",_raw_tapd_api_users,2,
+1,991,abc10,abc10,"{""ConnectionId"":1,""CompanyId"":99,""WorkspaceId"":991}",_raw_tapd_api_users,11,
+1,991,abc2,abc2,"{""ConnectionId"":1,""CompanyId"":99,""WorkspaceId"":991}",_raw_tapd_api_users,3,
+1,991,abc3,abc3,"{""ConnectionId"":1,""CompanyId"":99,""WorkspaceId"":991}",_raw_tapd_api_users,4,
+1,991,abc4,abc4,"{""ConnectionId"":1,""CompanyId"":99,""WorkspaceId"":991}",_raw_tapd_api_users,5,
+1,991,abc5,abc5,"{""ConnectionId"":1,""CompanyId"":99,""WorkspaceId"":991}",_raw_tapd_api_users,6,
+1,991,abc6,abc6,"{""ConnectionId"":1,""CompanyId"":99,""WorkspaceId"":991}",_raw_tapd_api_users,7,
+1,991,abc7,abc7,"{""ConnectionId"":1,""CompanyId"":99,""WorkspaceId"":991}",_raw_tapd_api_users,8,
+1,991,abc8,abc8,"{""ConnectionId"":1,""CompanyId"":99,""WorkspaceId"":991}",_raw_tapd_api_users,9,
+1,991,abc9,abc9,"{""ConnectionId"":1,""CompanyId"":99,""WorkspaceId"":991}",_raw_tapd_api_users,10,
diff --git a/plugins/tapd/e2e/snapshot_tables/users.csv b/plugins/tapd/e2e/snapshot_tables/users.csv
new file mode 100644
index 00000000..3820448a
--- /dev/null
+++ b/plugins/tapd/e2e/snapshot_tables/users.csv
@@ -0,0 +1,12 @@
+id,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark,name,email,avatar_url,timezone
+tapd:TapdUser:1:991:abc,"{""ConnectionId"":1,""CompanyId"":99,""WorkspaceId"":991}",_raw_tapd_api_users,1,,abc,,,
+tapd:TapdUser:1:991:abc1,"{""ConnectionId"":1,""CompanyId"":99,""WorkspaceId"":991}",_raw_tapd_api_users,2,,abc1,,,
+tapd:TapdUser:1:991:abc10,"{""ConnectionId"":1,""CompanyId"":99,""WorkspaceId"":991}",_raw_tapd_api_users,11,,abc10,,,
+tapd:TapdUser:1:991:abc2,"{""ConnectionId"":1,""CompanyId"":99,""WorkspaceId"":991}",_raw_tapd_api_users,3,,abc2,,,
+tapd:TapdUser:1:991:abc3,"{""ConnectionId"":1,""CompanyId"":99,""WorkspaceId"":991}",_raw_tapd_api_users,4,,abc3,,,
+tapd:TapdUser:1:991:abc4,"{""ConnectionId"":1,""CompanyId"":99,""WorkspaceId"":991}",_raw_tapd_api_users,5,,abc4,,,
+tapd:TapdUser:1:991:abc5,"{""ConnectionId"":1,""CompanyId"":99,""WorkspaceId"":991}",_raw_tapd_api_users,6,,abc5,,,
+tapd:TapdUser:1:991:abc6,"{""ConnectionId"":1,""CompanyId"":99,""WorkspaceId"":991}",_raw_tapd_api_users,7,,abc6,,,
+tapd:TapdUser:1:991:abc7,"{""ConnectionId"":1,""CompanyId"":99,""WorkspaceId"":991}",_raw_tapd_api_users,8,,abc7,,,
+tapd:TapdUser:1:991:abc8,"{""ConnectionId"":1,""CompanyId"":99,""WorkspaceId"":991}",_raw_tapd_api_users,9,,abc8,,,
+tapd:TapdUser:1:991:abc9,"{""ConnectionId"":1,""CompanyId"":99,""WorkspaceId"":991}",_raw_tapd_api_users,10,,abc9,,,
diff --git a/plugins/tapd/e2e/user_test.go b/plugins/tapd/e2e/user_test.go
new file mode 100644
index 00000000..9beb86c8
--- /dev/null
+++ b/plugins/tapd/e2e/user_test.go
@@ -0,0 +1,82 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+ "fmt"
+ "testing"
+
+ "github.com/apache/incubator-devlake/helpers/e2ehelper"
+ "github.com/apache/incubator-devlake/models/domainlayer/user"
+ "github.com/apache/incubator-devlake/plugins/tapd/impl"
+ "github.com/apache/incubator-devlake/plugins/tapd/models"
+ "github.com/apache/incubator-devlake/plugins/tapd/tasks"
+)
+
+func TestTapdUserDataFlow(t *testing.T) {
+
+ var tapd impl.Tapd
+ dataflowTester := e2ehelper.NewDataFlowTester(t, "tapd", tapd)
+
+ taskData := &tasks.TapdTaskData{
+ Options: &tasks.TapdOptions{
+ ConnectionId: 1,
+ CompanyId: 99,
+ WorkspaceId: 991,
+ },
+ }
+ // import raw data table
+ dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_tapd_api_users.csv",
+ "_raw_tapd_api_users")
+
+ // verify extraction
+ dataflowTester.FlushTabler(&models.TapdUser{})
+ dataflowTester.Subtask(tasks.ExtractUserMeta, taskData)
+ dataflowTester.VerifyTable(
+ models.TapdUser{},
+ fmt.Sprintf("./snapshot_tables/%s.csv", models.TapdUser{}.TableName()),
+ []string{"connection_id", "workspace_id", "user"},
+ []string{
+ "name",
+ "_raw_data_params",
+ "_raw_data_table",
+ "_raw_data_id",
+ "_raw_data_remark",
+ },
+ )
+
+ // verify conversion{
+ dataflowTester.FlushTabler(&user.User{})
+ dataflowTester.Subtask(tasks.ConvertUserMeta, taskData)
+ dataflowTester.VerifyTable(
+ user.User{},
+ fmt.Sprintf("./snapshot_tables/%s.csv", user.User{}.TableName()),
+ []string{"id"},
+ []string{
+ "_raw_data_params",
+ "_raw_data_table",
+ "_raw_data_id",
+ "_raw_data_remark",
+ "name",
+ "email",
+ "avatar_url",
+ "timezone",
+ },
+ )
+
+}
diff --git a/plugins/tapd/impl/impl.go b/plugins/tapd/impl/impl.go
index c102121f..404f0cbd 100644
--- a/plugins/tapd/impl/impl.go
+++ b/plugins/tapd/impl/impl.go
@@ -156,7 +156,6 @@ func (plugin Tapd) PrepareTaskData(taskCtx core.TaskContext, options map[string]
if !since.IsZero() {
taskData.Since = &since
}
- tasks.UserIdGen = didgen.NewDomainIdGenerator(&models.TapdUser{})
tasks.WorkspaceIdGen = didgen.NewDomainIdGenerator(&models.TapdWorkspace{})
tasks.IssueIdGen = didgen.NewDomainIdGenerator(&models.TapdIssue{})
tasks.IterIdGen = didgen.NewDomainIdGenerator(&models.TapdIteration{})
diff --git a/plugins/tapd/tasks/user_converter.go b/plugins/tapd/tasks/user_converter.go
index 1a39b58c..d309313f 100644
--- a/plugins/tapd/tasks/user_converter.go
+++ b/plugins/tapd/tasks/user_converter.go
@@ -21,6 +21,7 @@ import (
"reflect"
"github.com/apache/incubator-devlake/models/domainlayer"
+ "github.com/apache/incubator-devlake/models/domainlayer/didgen"
"github.com/apache/incubator-devlake/models/domainlayer/user"
"github.com/apache/incubator-devlake/plugins/core"
"github.com/apache/incubator-devlake/plugins/core/dal"
@@ -43,6 +44,7 @@ func ConvertUser(taskCtx core.SubTaskContext) error {
return err
}
defer cursor.Close()
+ userIdGen := didgen.NewDomainIdGenerator(&models.TapdUser{})
converter, err := helper.NewDataConverter(helper.DataConverterArgs{
RawDataSubTaskArgs: *rawDataSubTaskArgs,
InputRowType: reflect.TypeOf(models.TapdUser{}),
@@ -51,7 +53,7 @@ func ConvertUser(taskCtx core.SubTaskContext) error {
userTool := inputRow.(*models.TapdUser)
issue := &user.User{
DomainEntity: domainlayer.DomainEntity{
- Id: UserIdGen.Generate(userTool.ConnectionId, userTool.WorkspaceId, userTool.User),
+ Id: userIdGen.Generate(data.Options.ConnectionId, userTool.WorkspaceId, userTool.User),
},
Name: userTool.Name,
}