You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by kl...@apache.org on 2022/09/06 09:16:54 UTC
[incubator-devlake] branch main updated: Azure e2e (#2955)
This is an automated email from the ASF dual-hosted git repository.
klesh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 510f4ff1 Azure e2e (#2955)
510f4ff1 is described below
commit 510f4ff111014f77828eb4f38e8d7652863501d1
Author: mappjzc <zh...@merico.dev>
AuthorDate: Tue Sep 6 17:16:49 2022 +0800
Azure e2e (#2955)
* feat: add e2e test for azure
Add TestAzureRepoDataFlow
Add TestAzureBuildDefinitionDataFlow
Nddtfjiang <zh...@merico.dev>
* style: change tool table name to be const
use const string but not fmt.
Nddtfjiang <zh...@merico.dev>
---
plugins/azure/e2e/build_definition_test.go | 70 ++++++++++++++++++++++
.../_raw_azure_api_build_definitions.csv | 13 ++++
.../e2e/raw_tables/_raw_azure_api_repositories.csv | 2 +
plugins/azure/e2e/repo_test.go | 69 +++++++++++++++++++++
.../_tool_azure_build_definitions.csv | 13 ++++
.../e2e/snapshot_tables/_tool_azure_repos.csv | 2 +
6 files changed, 169 insertions(+)
diff --git a/plugins/azure/e2e/build_definition_test.go b/plugins/azure/e2e/build_definition_test.go
new file mode 100644
index 00000000..8bee0194
--- /dev/null
+++ b/plugins/azure/e2e/build_definition_test.go
@@ -0,0 +1,70 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+ "testing"
+
+ "github.com/apache/incubator-devlake/helpers/e2ehelper"
+ "github.com/apache/incubator-devlake/plugins/azure/impl"
+ "github.com/apache/incubator-devlake/plugins/azure/models"
+ "github.com/apache/incubator-devlake/plugins/azure/tasks"
+)
+
+func TestAzureBuildDefinitionDataFlow(t *testing.T) {
+ var github impl.Azure
+ dataflowTester := e2ehelper.NewDataFlowTester(t, "gitlab", github)
+
+ taskData := &tasks.AzureTaskData{
+ Options: &tasks.AzureOptions{
+ ConnectionId: 1,
+ Project: "test",
+ },
+ }
+
+ // import raw data table
+ // SELECT * FROM _raw_azure_api_build_definitions INTO OUTFILE "/tmp/_raw_azure_api_build_definitions.csv" FIELDS TERMINATED BY ',' OPTIONALLY ENCLOSED BY '"' LINES TERMINATED BY '\r\n';
+ dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_azure_api_build_definitions.csv", "_raw_azure_api_build_definitions")
+
+ // verify extraction
+ dataflowTester.FlushTabler(&models.AzureBuildDefinition{})
+ dataflowTester.Subtask(tasks.ExtractApiBuildDefinitionMeta, taskData)
+ dataflowTester.VerifyTable(
+ models.AzureBuildDefinition{},
+ "./snapshot_tables/_tool_azure_build_definitions.csv",
+ []string{
+ "connection_id",
+ "project_id",
+ "azure_id",
+ "author_id",
+ "queue_id",
+ "url",
+ "name",
+ "path",
+ "type",
+ "queue_status",
+ "revision",
+ "azure_created_date",
+
+ "_raw_data_params",
+ "_raw_data_table",
+ "_raw_data_id",
+ "_raw_data_remark",
+ },
+ )
+}
diff --git a/plugins/azure/e2e/raw_tables/_raw_azure_api_build_definitions.csv b/plugins/azure/e2e/raw_tables/_raw_azure_api_build_definitions.csv
new file mode 100644
index 00000000..4007e82e
--- /dev/null
+++ b/plugins/azure/e2e/raw_tables/_raw_azure_api_build_definitions.csv
@@ -0,0 +1,13 @@
+id,params,data,url,input,created_at
+4,"{""ConnectionId"":1,""Project"":""test""}","{""_links"":{""self"":{""href"":""https://dev.azure.com/mericojzc/30473eea-ca3f-4f40-a711-9cfa2e75e4b0/_apis/build/Definitions/1?revision=1""},""web"":{""href"":""https://dev.azure.com/mericojzc/30473eea-ca3f-4f40-a711-9cfa2e75e4b0/_build/definition?definitionId=1""},""editor"":{""href"":""https://dev.azure.com/mericojzc/30473eea-ca3f-4f40-a711-9cfa2e75e4b0/_build/designer?id=1&_a=edit-build-definition""},""badge"":{""href"":""https://dev.az [...]
+5,"{""ConnectionId"":1,""Project"":""test""}","{""_links"":{""self"":{""href"":""https://dev.azure.com/mericojzc/30473eea-ca3f-4f40-a711-9cfa2e75e4b0/_apis/build/Definitions/2?revision=1""},""web"":{""href"":""https://dev.azure.com/mericojzc/30473eea-ca3f-4f40-a711-9cfa2e75e4b0/_build/definition?definitionId=2""},""editor"":{""href"":""https://dev.azure.com/mericojzc/30473eea-ca3f-4f40-a711-9cfa2e75e4b0/_build/designer?id=2&_a=edit-build-definition""},""badge"":{""href"":""https://dev.az [...]
+6,"{""ConnectionId"":1,""Project"":""test""}","{""_links"":{""self"":{""href"":""https://dev.azure.com/mericojzc/30473eea-ca3f-4f40-a711-9cfa2e75e4b0/_apis/build/Definitions/2?revision=1""},""web"":{""href"":""https://dev.azure.com/mericojzc/30473eea-ca3f-4f40-a711-9cfa2e75e4b0/_build/definition?definitionId=3""},""editor"":{""href"":""https://dev.azure.com/mericojzc/30473eea-ca3f-4f40-a711-9cfa2e75e4b0/_build/designer?id=3&_a=edit-build-definition""},""badge"":{""href"":""https://dev.az [...]
+7,"{""ConnectionId"":1,""Project"":""test""}","{""_links"":{""self"":{""href"":""https://dev.azure.com/mericojzc/30473eea-ca3f-4f40-a711-9cfa2e75e4b0/_apis/build/Definitions/2?revision=1""},""web"":{""href"":""https://dev.azure.com/mericojzc/30473eea-ca3f-4f40-a711-9cfa2e75e4b0/_build/definition?definitionId=4""},""editor"":{""href"":""https://dev.azure.com/mericojzc/30473eea-ca3f-4f40-a711-9cfa2e75e4b0/_build/designer?id=4&_a=edit-build-definition""},""badge"":{""href"":""https://dev.az [...]
+8,"{""ConnectionId"":1,""Project"":""test""}","{""_links"":{""self"":{""href"":""https://dev.azure.com/mericojzc/30473eea-ca3f-4f40-a711-9cfa2e75e4b0/_apis/build/Definitions/2?revision=1""},""web"":{""href"":""https://dev.azure.com/mericojzc/30473eea-ca3f-4f40-a711-9cfa2e75e4b0/_build/definition?definitionId=5""},""editor"":{""href"":""https://dev.azure.com/mericojzc/30473eea-ca3f-4f40-a711-9cfa2e75e4b0/_build/designer?id=5&_a=edit-build-definition""},""badge"":{""href"":""https://dev.az [...]
+9,"{""ConnectionId"":1,""Project"":""test""}","{""_links"":{""self"":{""href"":""https://dev.azure.com/mericojzc/30473eea-ca3f-4f40-a711-9cfa2e75e4b0/_apis/build/Definitions/2?revision=1""},""web"":{""href"":""https://dev.azure.com/mericojzc/30473eea-ca3f-4f40-a711-9cfa2e75e4b0/_build/definition?definitionId=6""},""editor"":{""href"":""https://dev.azure.com/mericojzc/30473eea-ca3f-4f40-a711-9cfa2e75e4b0/_build/designer?id=6&_a=edit-build-definition""},""badge"":{""href"":""https://dev.az [...]
+10,"{""ConnectionId"":1,""Project"":""test""}","{""_links"":{""self"":{""href"":""https://dev.azure.com/mericojzc/30473eea-ca3f-4f40-a711-9cfa2e75e4b0/_apis/build/Definitions/2?revision=1""},""web"":{""href"":""https://dev.azure.com/mericojzc/30473eea-ca3f-4f40-a711-9cfa2e75e4b0/_build/definition?definitionId=7""},""editor"":{""href"":""https://dev.azure.com/mericojzc/30473eea-ca3f-4f40-a711-9cfa2e75e4b0/_build/designer?id=7&_a=edit-build-definition""},""badge"":{""href"":""https://dev.a [...]
+11,"{""ConnectionId"":1,""Project"":""test""}","{""_links"":{""self"":{""href"":""https://dev.azure.com/mericojzc/30473eea-ca3f-4f40-a711-9cfa2e75e4b0/_apis/build/Definitions/2?revision=1""},""web"":{""href"":""https://dev.azure.com/mericojzc/30473eea-ca3f-4f40-a711-9cfa2e75e4b0/_build/definition?definitionId=8""},""editor"":{""href"":""https://dev.azure.com/mericojzc/30473eea-ca3f-4f40-a711-9cfa2e75e4b0/_build/designer?id=8&_a=edit-build-definition""},""badge"":{""href"":""https://dev.a [...]
+12,"{""ConnectionId"":1,""Project"":""test""}","{""_links"":{""self"":{""href"":""https://dev.azure.com/mericojzc/30473eea-ca3f-4f40-a711-9cfa2e75e4b0/_apis/build/Definitions/2?revision=1""},""web"":{""href"":""https://dev.azure.com/mericojzc/30473eea-ca3f-4f40-a711-9cfa2e75e4b0/_build/definition?definitionId=9""},""editor"":{""href"":""https://dev.azure.com/mericojzc/30473eea-ca3f-4f40-a711-9cfa2e75e4b0/_build/designer?id=9&_a=edit-build-definition""},""badge"":{""href"":""https://dev.a [...]
+13,"{""ConnectionId"":1,""Project"":""test""}","{""_links"":{""self"":{""href"":""https://dev.azure.com/mericojzc/30473eea-ca3f-4f40-a711-9cfa2e75e4b0/_apis/build/Definitions/2?revision=1""},""web"":{""href"":""https://dev.azure.com/mericojzc/30473eea-ca3f-4f40-a711-9cfa2e75e4b0/_build/definition?definitionId=10""},""editor"":{""href"":""https://dev.azure.com/mericojzc/30473eea-ca3f-4f40-a711-9cfa2e75e4b0/_build/designer?id=10&_a=edit-build-definition""},""badge"":{""href"":""https://dev [...]
+14,"{""ConnectionId"":1,""Project"":""test""}","{""_links"":{""self"":{""href"":""https://dev.azure.com/mericojzc/30473eea-ca3f-4f40-a711-9cfa2e75e4b0/_apis/build/Definitions/2?revision=1""},""web"":{""href"":""https://dev.azure.com/mericojzc/30473eea-ca3f-4f40-a711-9cfa2e75e4b0/_build/definition?definitionId=11""},""editor"":{""href"":""https://dev.azure.com/mericojzc/30473eea-ca3f-4f40-a711-9cfa2e75e4b0/_build/designer?id=11&_a=edit-build-definition""},""badge"":{""href"":""https://dev [...]
+15,"{""ConnectionId"":1,""Project"":""test""}","{""_links"":{""self"":{""href"":""https://dev.azure.com/mericojzc/30473eea-ca3f-4f40-a711-9cfa2e75e4b0/_apis/build/Definitions/2?revision=1""},""web"":{""href"":""https://dev.azure.com/mericojzc/30473eea-ca3f-4f40-a711-9cfa2e75e4b0/_build/definition?definitionId=12""},""editor"":{""href"":""https://dev.azure.com/mericojzc/30473eea-ca3f-4f40-a711-9cfa2e75e4b0/_build/designer?id=12&_a=edit-build-definition""},""badge"":{""href"":""https://dev [...]
diff --git a/plugins/azure/e2e/raw_tables/_raw_azure_api_repositories.csv b/plugins/azure/e2e/raw_tables/_raw_azure_api_repositories.csv
new file mode 100644
index 00000000..d9bb7df7
--- /dev/null
+++ b/plugins/azure/e2e/raw_tables/_raw_azure_api_repositories.csv
@@ -0,0 +1,2 @@
+id,params,data,url,input,created_at
+5,"{""ConnectionId"":1,""Project"":""test""}","{""id"":""5dc348ab-98a9-4c49-95da-b70b24a62932"",""name"":""test"",""url"":""https://dev.azure.com/mericojzc/30473eea-ca3f-4f40-a711-9cfa2e75e4b0/_apis/git/repositories/5dc348ab-98a9-4c49-95da-b70b24a62932"",""project"":{""id"":""30473eea-ca3f-4f40-a711-9cfa2e75e4b0"",""name"":""test"",""url"":""https://dev.azure.com/mericojzc/_apis/projects/30473eea-ca3f-4f40-a711-9cfa2e75e4b0"",""state"":""wellFormed"",""revision"":11,""visibility"":""priv [...]
\ No newline at end of file
diff --git a/plugins/azure/e2e/repo_test.go b/plugins/azure/e2e/repo_test.go
new file mode 100644
index 00000000..d6703581
--- /dev/null
+++ b/plugins/azure/e2e/repo_test.go
@@ -0,0 +1,69 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+ "testing"
+
+ "github.com/apache/incubator-devlake/helpers/e2ehelper"
+ "github.com/apache/incubator-devlake/plugins/azure/impl"
+ "github.com/apache/incubator-devlake/plugins/azure/models"
+ "github.com/apache/incubator-devlake/plugins/azure/tasks"
+)
+
+func TestAzureRepoDataFlow(t *testing.T) {
+ var github impl.Azure
+ dataflowTester := e2ehelper.NewDataFlowTester(t, "gitlab", github)
+
+ taskData := &tasks.AzureTaskData{
+ Options: &tasks.AzureOptions{
+ ConnectionId: 1,
+ Project: "test",
+ },
+ }
+
+ // import raw data table
+ // SELECT * FROM _raw_azure_api_repositories INTO OUTFILE "/tmp/_raw_azure_api_repositories.csv" FIELDS TERMINATED BY ',' OPTIONALLY ENCLOSED BY '"' LINES TERMINATED BY '\r\n';
+ dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_azure_api_repositories.csv", "_raw_azure_api_repositories")
+
+ // verify extraction
+ dataflowTester.FlushTabler(&models.AzureRepo{})
+ dataflowTester.Subtask(tasks.ExtractApiRepoMeta, taskData)
+ dataflowTester.VerifyTable(
+ models.AzureRepo{},
+ "./snapshot_tables/_tool_azure_repos.csv",
+ []string{
+ "connection_id",
+ "azure_id",
+ "name",
+ "url",
+ "project_id",
+ "default_branch",
+ "size",
+ "remote_url",
+ "ssh_url",
+ "web_url",
+ "is_disabled",
+
+ "_raw_data_params",
+ "_raw_data_table",
+ "_raw_data_id",
+ "_raw_data_remark",
+ },
+ )
+}
diff --git a/plugins/azure/e2e/snapshot_tables/_tool_azure_build_definitions.csv b/plugins/azure/e2e/snapshot_tables/_tool_azure_build_definitions.csv
new file mode 100644
index 00000000..d8519019
--- /dev/null
+++ b/plugins/azure/e2e/snapshot_tables/_tool_azure_build_definitions.csv
@@ -0,0 +1,13 @@
+connection_id,project_id,azure_id,author_id,queue_id,url,name,path,type,queue_status,revision,azure_created_date,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
+1,30473eea-ca3f-4f40-a711-9cfa2e75e4b0,1,2b39fa56-bd8f-6805-94e2-16e535dfb387,9,https://dev.azure.com/mericojzc/30473eea-ca3f-4f40-a711-9cfa2e75e4b0/_apis/build/Definitions/1?revision=1,test,\\,build,enabled,1,2022-08-25T16:12:50.260+00:00,"{""ConnectionId"":1,""Project"":""test""}",_raw_azure_api_build_definitions,4,
+1,30473eea-ca3f-4f40-a711-9cfa2e75e4b0,2,2b39fa56-bd8f-6805-94e2-16e535dfb387,9,https://dev.azure.com/mericojzc/30473eea-ca3f-4f40-a711-9cfa2e75e4b0/_apis/build/Definitions/2?revision=1,test (1),\\,build,enabled,1,2022-09-02T14:56:10.680+00:00,"{""ConnectionId"":1,""Project"":""test""}",_raw_azure_api_build_definitions,5,
+1,30473eea-ca3f-4f40-a711-9cfa2e75e4b0,3,2b39fa56-bd8f-6805-94e2-16e535dfb387,9,https://dev.azure.com/mericojzc/30473eea-ca3f-4f40-a711-9cfa2e75e4b0/_apis/build/Definitions/2?revision=1,test (2),\\,build,enabled,1,2022-09-02T14:56:10.680+00:00,"{""ConnectionId"":1,""Project"":""test""}",_raw_azure_api_build_definitions,6,
+1,30473eea-ca3f-4f40-a711-9cfa2e75e4b0,4,2b39fa56-bd8f-6805-94e2-16e535dfb387,9,https://dev.azure.com/mericojzc/30473eea-ca3f-4f40-a711-9cfa2e75e4b0/_apis/build/Definitions/2?revision=1,test (3),\\,build,enabled,1,2022-09-02T14:56:10.680+00:00,"{""ConnectionId"":1,""Project"":""test""}",_raw_azure_api_build_definitions,7,
+1,30473eea-ca3f-4f40-a711-9cfa2e75e4b0,5,2b39fa56-bd8f-6805-94e2-16e535dfb387,9,https://dev.azure.com/mericojzc/30473eea-ca3f-4f40-a711-9cfa2e75e4b0/_apis/build/Definitions/2?revision=1,test (4),\\,build,enabled,1,2022-09-02T14:56:10.680+00:00,"{""ConnectionId"":1,""Project"":""test""}",_raw_azure_api_build_definitions,8,
+1,30473eea-ca3f-4f40-a711-9cfa2e75e4b0,6,2b39fa56-bd8f-6805-94e2-16e535dfb387,9,https://dev.azure.com/mericojzc/30473eea-ca3f-4f40-a711-9cfa2e75e4b0/_apis/build/Definitions/2?revision=1,test (5),\\,build,enabled,1,2022-09-02T14:56:10.680+00:00,"{""ConnectionId"":1,""Project"":""test""}",_raw_azure_api_build_definitions,9,
+1,30473eea-ca3f-4f40-a711-9cfa2e75e4b0,7,2b39fa56-bd8f-6805-94e2-16e535dfb387,9,https://dev.azure.com/mericojzc/30473eea-ca3f-4f40-a711-9cfa2e75e4b0/_apis/build/Definitions/2?revision=1,test (6),\\,build,enabled,1,2022-09-02T14:56:10.680+00:00,"{""ConnectionId"":1,""Project"":""test""}",_raw_azure_api_build_definitions,10,
+1,30473eea-ca3f-4f40-a711-9cfa2e75e4b0,8,2b39fa56-bd8f-6805-94e2-16e535dfb387,9,https://dev.azure.com/mericojzc/30473eea-ca3f-4f40-a711-9cfa2e75e4b0/_apis/build/Definitions/2?revision=1,test (7),\\,build,enabled,1,2022-09-02T14:56:10.680+00:00,"{""ConnectionId"":1,""Project"":""test""}",_raw_azure_api_build_definitions,11,
+1,30473eea-ca3f-4f40-a711-9cfa2e75e4b0,9,2b39fa56-bd8f-6805-94e2-16e535dfb387,9,https://dev.azure.com/mericojzc/30473eea-ca3f-4f40-a711-9cfa2e75e4b0/_apis/build/Definitions/2?revision=1,test (8),\\,build,enabled,1,2022-09-02T14:56:10.680+00:00,"{""ConnectionId"":1,""Project"":""test""}",_raw_azure_api_build_definitions,12,
+1,30473eea-ca3f-4f40-a711-9cfa2e75e4b0,10,2b39fa56-bd8f-6805-94e2-16e535dfb387,9,https://dev.azure.com/mericojzc/30473eea-ca3f-4f40-a711-9cfa2e75e4b0/_apis/build/Definitions/2?revision=1,test (9),\\,build,enabled,1,2022-09-02T14:56:10.680+00:00,"{""ConnectionId"":1,""Project"":""test""}",_raw_azure_api_build_definitions,13,
+1,30473eea-ca3f-4f40-a711-9cfa2e75e4b0,11,2b39fa56-bd8f-6805-94e2-16e535dfb387,9,https://dev.azure.com/mericojzc/30473eea-ca3f-4f40-a711-9cfa2e75e4b0/_apis/build/Definitions/2?revision=1,test (10),\\,build,enabled,1,2022-09-02T14:56:10.680+00:00,"{""ConnectionId"":1,""Project"":""test""}",_raw_azure_api_build_definitions,14,
+1,30473eea-ca3f-4f40-a711-9cfa2e75e4b0,12,2b39fa56-bd8f-6805-94e2-16e535dfb387,9,https://dev.azure.com/mericojzc/30473eea-ca3f-4f40-a711-9cfa2e75e4b0/_apis/build/Definitions/2?revision=1,test (11),\\,build,enabled,1,2022-09-02T14:56:10.680+00:00,"{""ConnectionId"":1,""Project"":""test""}",_raw_azure_api_build_definitions,15,
diff --git a/plugins/azure/e2e/snapshot_tables/_tool_azure_repos.csv b/plugins/azure/e2e/snapshot_tables/_tool_azure_repos.csv
new file mode 100644
index 00000000..22e50dac
--- /dev/null
+++ b/plugins/azure/e2e/snapshot_tables/_tool_azure_repos.csv
@@ -0,0 +1,2 @@
+connection_id,azure_id,name,url,project_id,default_branch,size,remote_url,ssh_url,web_url,is_disabled,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
+1,5dc348ab-98a9-4c49-95da-b70b24a62932,test,https://dev.azure.com/mericojzc/30473eea-ca3f-4f40-a711-9cfa2e75e4b0/_apis/git/repositories/5dc348ab-98a9-4c49-95da-b70b24a62932,30473eea-ca3f-4f40-a711-9cfa2e75e4b0,refs/heads/asf-site,38627917,https://mericojzc@dev.azure.com/mericojzc/test/_git/test,git@ssh.dev.azure.com:v3/mericojzc/test/test,https://dev.azure.com/mericojzc/test/_git/test,0,"{""ConnectionId"":1,""Project"":""test""}",_raw_azure_api_repositories,5,