[incubator-devlake] branch main updated: feat: sonarqube filemetrics e2e (#4384)

[wa...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

warren pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git


The following commit(s) were added to refs/heads/main by this push:
     new 3fc6885c6 feat: sonarqube filemetrics e2e (#4384)
3fc6885c6 is described below

commit 3fc6885c6e64b436b22ae6fcd5ce4e6fe5a943ef
Author: abeizn <zi...@merico.dev>
AuthorDate: Fri Feb 10 16:42:46 2023 +0800

    feat: sonarqube filemetrics e2e (#4384)
---
 backend/plugins/sonarqube/e2e/filemetrics_test.go  | 70 ++++++++++++++++++++++
 .../e2e/raw_tables/_raw_sonarqube_filemetrics.csv  | 11 ++++
 .../_tool_sonarqube_filemetrics.csv                |  9 +++
 .../sonarqube/e2e/snapshot_tables/filemetrics.csv  |  7 +++
 4 files changed, 97 insertions(+)

diff --git a/backend/plugins/sonarqube/e2e/filemetrics_test.go b/backend/plugins/sonarqube/e2e/filemetrics_test.go
new file mode 100644
index 000000000..09975764a
--- /dev/null
+++ b/backend/plugins/sonarqube/e2e/filemetrics_test.go
@@ -0,0 +1,70 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+	"testing"
+
+	"github.com/apache/incubator-devlake/core/models/common"
+	"github.com/apache/incubator-devlake/core/models/domainlayer/securitytesting"
+	"github.com/apache/incubator-devlake/helpers/e2ehelper"
+	"github.com/apache/incubator-devlake/plugins/sonarqube/impl"
+	"github.com/apache/incubator-devlake/plugins/sonarqube/models"
+	"github.com/apache/incubator-devlake/plugins/sonarqube/tasks"
+)
+
+func TestSonarqubeFileMetricsDataFlow(t *testing.T) {
+
+	var sonarqube impl.Sonarqube
+	dataflowTester := e2ehelper.NewDataFlowTester(t, "sonarqube", sonarqube)
+
+	// import raw data table
+	dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_sonarqube_filemetrics.csv",
+		"_raw_sonarqube_filemetrics")
+
+	// Standard data
+	taskData := &tasks.SonarqubeTaskData{
+		Options: &tasks.SonarqubeOptions{
+			ConnectionId: 1,
+			ProjectKey:   "02c1047b-f87c-4c35-a6b5-76c6b607d37f",
+		},
+	}
+	// Interfered data
+	taskData2 := &tasks.SonarqubeTaskData{
+		Options: &tasks.SonarqubeOptions{
+			ConnectionId: 2,
+			ProjectKey:   "e2c6d5e9-a321-4e8c-b322-03d9599ef962",
+		},
+	}
+
+	// verify extraction
+	dataflowTester.FlushTabler(&models.SonarqubeFileMetrics{})
+	dataflowTester.Subtask(tasks.ExtractFilemetricsMeta, taskData)
+
+	dataflowTester.Subtask(tasks.ExtractFilemetricsMeta, taskData2)
+	dataflowTester.VerifyTableWithOptions(&models.SonarqubeFileMetrics{}, e2ehelper.TableOptions{
+		CSVRelPath:  "./snapshot_tables/_tool_sonarqube_filemetrics.csv",
+		IgnoreTypes: []interface{}{common.NoPKModel{}},
+	})
+
+	// verify convertor
+	dataflowTester.FlushTabler(&securitytesting.StFileMetrics{})
+	dataflowTester.Subtask(tasks.ConvertFileMetricsMeta, taskData)
+	dataflowTester.VerifyTableWithOptions(&securitytesting.StFileMetrics{}, e2ehelper.TableOptions{
+		CSVRelPath:  "./snapshot_tables/filemetrics.csv",
+		IgnoreTypes: []interface{}{common.NoPKModel{}},
+	})
+}
diff --git a/backend/plugins/sonarqube/e2e/raw_tables/_raw_sonarqube_filemetrics.csv b/backend/plugins/sonarqube/e2e/raw_tables/_raw_sonarqube_filemetrics.csv
new file mode 100644
index 000000000..b2940880e
--- /dev/null
+++ b/backend/plugins/sonarqube/e2e/raw_tables/_raw_sonarqube_filemetrics.csv
@@ -0,0 +1,11 @@
+"id","params","data","url","input","created_at"
+1,"{""connectionId"":1,""ProjectKey"":""02c1047b-f87c-4c35-a6b5-76c6b607d37f""}","{""key"":""02c1047b-f87c-4c35-a6b5-76c6b607d37f:db/migrate/20170206220334_add_start_date_to_sponsors_and_paid.rb"",""name"":""20170206220334_add_start_date_to_sponsors_and_paid.rb"",""qualifier"":""FIL"",""path"":""db/migrate/20170206220334_add_start_date_to_sponsors_and_paid.rb"",""language"":""ruby"",""measures"":[{""metric"":""sqale_rating"",""value"":""1.0"",""bestValue"":true},{""metric"":""reliability [...]
+2,"{""connectionId"":1,""ProjectKey"":""02c1047b-f87c-4c35-a6b5-76c6b607d37f""}","{""key"":""02c1047b-f87c-4c35-a6b5-76c6b607d37f:db/migrate/20170208152018_acts_as_follower_migration.rb"",""name"":""20170208152018_acts_as_follower_migration.rb"",""qualifier"":""FIL"",""path"":""db/migrate/20170208152018_acts_as_follower_migration.rb"",""language"":""ruby"",""measures"":[{""metric"":""sqale_rating"",""value"":""1.0"",""bestValue"":true},{""metric"":""reliability_rating"",""value"":""1.0"" [...]
+3,"{""connectionId"":1,""ProjectKey"":""02c1047b-f87c-4c35-a6b5-76c6b607d37f""}","{""key"":""02c1047b-f87c-4c35-a6b5-76c6b607d37f:db/migrate/20170209164016_add_email_follower_notifications.rb"",""name"":""20170209164016_add_email_follower_notifications.rb"",""qualifier"":""FIL"",""path"":""db/migrate/20170209164016_add_email_follower_notifications.rb"",""language"":""ruby"",""measures"":[{""metric"":""sqale_rating"",""value"":""1.0"",""bestValue"":true},{""metric"":""reliability_rating"" [...]
+4,"{""connectionId"":1,""ProjectKey"":""02c1047b-f87c-4c35-a6b5-76c6b607d37f""}","{""key"":""02c1047b-f87c-4c35-a6b5-76c6b607d37f:db/migrate/20170213183337_add_sign_up_information_to_users.rb"",""name"":""20170213183337_add_sign_up_information_to_users.rb"",""qualifier"":""FIL"",""path"":""db/migrate/20170213183337_add_sign_up_information_to_users.rb"",""language"":""ruby"",""measures"":[{""metric"":""sqale_rating"",""value"":""1.0"",""bestValue"":true},{""metric"":""reliability_rating"" [...]
+5,"{""connectionId"":1,""ProjectKey"":""02c1047b-f87c-4c35-a6b5-76c6b607d37f""}","{""key"":""02c1047b-f87c-4c35-a6b5-76c6b607d37f:db/migrate/20170216145500_add_location_fields_etc_to_users.rb"",""name"":""20170216145500_add_location_fields_etc_to_users.rb"",""qualifier"":""FIL"",""path"":""db/migrate/20170216145500_add_location_fields_etc_to_users.rb"",""language"":""ruby"",""measures"":[{""metric"":""sqale_rating"",""value"":""1.0"",""bestValue"":true},{""metric"":""reliability_rating"" [...]
+6,"{""connectionId"":1,""ProjectKey"":""02c1047b-f87c-4c35-a6b5-76c6b607d37f""}","{""key"":""02c1047b-f87c-4c35-a6b5-76c6b607d37f:db/migrate/20170228174838_add_identity_data_to_users.rb"",""name"":""20170228174838_add_identity_data_to_users.rb"",""qualifier"":""FIL"",""path"":""db/migrate/20170228174838_add_identity_data_to_users.rb"",""language"":""ruby"",""measures"":[{""metric"":""sqale_rating"",""value"":""1.0"",""bestValue"":true},{""metric"":""reliability_rating"",""value"":""1.0"" [...]
+7,"{""connectionId"":2,""ProjectKey"":""e2c6d5e9-a321-4e8c-b322-03d9599ef962""}","{""key"":""e2c6d5e9-a321-4e8c-b322-03d9599ef962:db/migrate/20170302152930_add_attributes_to_tags.rb"",""name"":""20170302152930_add_attributes_to_tags.rb"",""qualifier"":""FIL"",""path"":""db/migrate/20170302152930_add_attributes_to_tags.rb"",""language"":""ruby"",""measures"":[{""metric"":""sqale_rating"",""value"":""1.0"",""bestValue"":true},{""metric"":""reliability_rating"",""value"":""1.0"",""bestValue [...]
+8,"{""connectionId"":2,""ProjectKey"":""e2c6d5e9-a321-4e8c-b322-03d9599ef962""}","{""key"":""e2c6d5e9-a321-4e8c-b322-03d9599ef962:db/migrate/20170303171502_add_social_image_to_tags.rb"",""name"":""20170303171502_add_social_image_to_tags.rb"",""qualifier"":""FIL"",""path"":""db/migrate/20170303171502_add_social_image_to_tags.rb"",""language"":""ruby"",""measures"":[{""metric"":""sqale_rating"",""value"":""1.0"",""bestValue"":true},{""metric"":""reliability_rating"",""value"":""1.0"",""bes [...]
+9,"{""connectionId"":1,""ProjectKey"":""xxxx""}","{""key"":""xxxx:db/migrate/20170303180353_create_features.rb"",""name"":""20170303180353_create_features.rb"",""qualifier"":""FIL"",""path"":""db/migrate/20170303180353_create_features.rb"",""language"":""ruby"",""measures"":[{""metric"":""sqale_rating"",""value"":""1.0"",""bestValue"":true},{""metric"":""reliability_rating"",""value"":""1.0"",""bestValue"":true},{""metric"":""security_rating"",""value"":""1.0"",""bestValue"":true},{""met [...]
+10,"{""connectionId"":1,""ProjectKey"":""e2c6d5e9-a321-4e8c-b322-03d9599ef962""}","{""key"":""e2c6d5e9-a321-4e8c-b322-03d9599ef962:db/migrate/20170309162937_add_markdown_character_count_to_comments.rb"",""name"":""20170309162937_add_markdown_character_count_to_comments.rb"",""qualifier"":""FIL"",""path"":""db/migrate/20170309162937_add_markdown_character_count_to_comments.rb"",""language"":""ruby"",""measures"":[{""metric"":""sqale_rating"",""value"":""1.0"",""bestValue"":true},{""metric [...]
diff --git a/backend/plugins/sonarqube/e2e/snapshot_tables/_tool_sonarqube_filemetrics.csv b/backend/plugins/sonarqube/e2e/snapshot_tables/_tool_sonarqube_filemetrics.csv
new file mode 100644
index 000000000..7efe88a5f
--- /dev/null
+++ b/backend/plugins/sonarqube/e2e/snapshot_tables/_tool_sonarqube_filemetrics.csv
@@ -0,0 +1,9 @@
+connection_id,file_metrics_key,project_key,file_name,file_path,file_language,code_smells,sqale_index,sqale_rating,bugs,reliability_rating,vulnerabilities,security_rating,security_hotspots,security_hotspots_reviewed,security_review_rating,ncloc,coverage,lines_to_cover,duplicated_lines_density,duplicated_blocks
+1,02c1047b-f87c-4c35-a6b5-76c6b607d37f:db/migrate/20170206220334_add_start_date_to_sponsors_and_paid.rb,02c1047b-f87c-4c35-a6b5-76c6b607d37f,20170206220334_add_start_date_to_sponsors_and_paid.rb,db/migrate/20170206220334_add_start_date_to_sponsors_and_paid.rb,ruby,0,0,1.0,0,A,0,A,0,0,A,6,0,2,0,0
+1,02c1047b-f87c-4c35-a6b5-76c6b607d37f:db/migrate/20170208152018_acts_as_follower_migration.rb,02c1047b-f87c-4c35-a6b5-76c6b607d37f,20170208152018_acts_as_follower_migration.rb,db/migrate/20170208152018_acts_as_follower_migration.rb,ruby,0,0,1.0,0,A,0,A,0,0,A,15,0,8,0,0
+1,02c1047b-f87c-4c35-a6b5-76c6b607d37f:db/migrate/20170209164016_add_email_follower_notifications.rb,02c1047b-f87c-4c35-a6b5-76c6b607d37f,20170209164016_add_email_follower_notifications.rb,db/migrate/20170209164016_add_email_follower_notifications.rb,ruby,0,0,1.0,0,A,0,A,0,0,A,5,0,1,0,0
+1,02c1047b-f87c-4c35-a6b5-76c6b607d37f:db/migrate/20170213183337_add_sign_up_information_to_users.rb,02c1047b-f87c-4c35-a6b5-76c6b607d37f,20170213183337_add_sign_up_information_to_users.rb,db/migrate/20170213183337_add_sign_up_information_to_users.rb,ruby,0,0,1.0,0,A,0,A,0,0,A,7,0,3,0,0
+1,02c1047b-f87c-4c35-a6b5-76c6b607d37f:db/migrate/20170216145500_add_location_fields_etc_to_users.rb,02c1047b-f87c-4c35-a6b5-76c6b607d37f,20170216145500_add_location_fields_etc_to_users.rb,db/migrate/20170216145500_add_location_fields_etc_to_users.rb,ruby,0,0,1.0,0,A,0,A,0,0,A,20,0,16,0,0
+1,02c1047b-f87c-4c35-a6b5-76c6b607d37f:db/migrate/20170228174838_add_identity_data_to_users.rb,02c1047b-f87c-4c35-a6b5-76c6b607d37f,20170228174838_add_identity_data_to_users.rb,db/migrate/20170228174838_add_identity_data_to_users.rb,ruby,0,0,1.0,0,A,0,A,0,0,A,9,0,5,0,0
+2,e2c6d5e9-a321-4e8c-b322-03d9599ef962:db/migrate/20170302152930_add_attributes_to_tags.rb,e2c6d5e9-a321-4e8c-b322-03d9599ef962,20170302152930_add_attributes_to_tags.rb,db/migrate/20170302152930_add_attributes_to_tags.rb,ruby,0,0,1.0,0,A,0,A,0,0,A,16,0,12,0,0
+2,e2c6d5e9-a321-4e8c-b322-03d9599ef962:db/migrate/20170303171502_add_social_image_to_tags.rb,e2c6d5e9-a321-4e8c-b322-03d9599ef962,20170303171502_add_social_image_to_tags.rb,db/migrate/20170303171502_add_social_image_to_tags.rb,ruby,0,0,1.0,0,A,0,A,0,0,A,5,0,1,0,0
diff --git a/backend/plugins/sonarqube/e2e/snapshot_tables/filemetrics.csv b/backend/plugins/sonarqube/e2e/snapshot_tables/filemetrics.csv
new file mode 100644
index 000000000..b2390e3ab
--- /dev/null
+++ b/backend/plugins/sonarqube/e2e/snapshot_tables/filemetrics.csv
@@ -0,0 +1,7 @@
+id,project_key,file_name,file_path,file_language,code_smells,sqale_index,sqale_rating,bugs,reliability_rating,vulnerabilities,security_rating,security_hotspots,security_hotspots_reviewed,security_review_rating,ncloc,coverage,lines_to_cover,duplicated_lines_density,duplicated_blocks
+sonarqube:SonarqubeFileMetrics:1:02c1047b-f87c-4c35-a6b5-76c6b607d37f:db/migrate/20170206220334_add_start_date_to_sponsors_and_paid.rb,sonarqube:SonarqubeProject:1:02c1047b-f87c-4c35-a6b5-76c6b607d37f,20170206220334_add_start_date_to_sponsors_and_paid.rb,db/migrate/20170206220334_add_start_date_to_sponsors_and_paid.rb,ruby,0,0,1.0,0,A,0,A,0,0,A,6,0,2,0,0
+sonarqube:SonarqubeFileMetrics:1:02c1047b-f87c-4c35-a6b5-76c6b607d37f:db/migrate/20170208152018_acts_as_follower_migration.rb,sonarqube:SonarqubeProject:1:02c1047b-f87c-4c35-a6b5-76c6b607d37f,20170208152018_acts_as_follower_migration.rb,db/migrate/20170208152018_acts_as_follower_migration.rb,ruby,0,0,1.0,0,A,0,A,0,0,A,15,0,8,0,0
+sonarqube:SonarqubeFileMetrics:1:02c1047b-f87c-4c35-a6b5-76c6b607d37f:db/migrate/20170209164016_add_email_follower_notifications.rb,sonarqube:SonarqubeProject:1:02c1047b-f87c-4c35-a6b5-76c6b607d37f,20170209164016_add_email_follower_notifications.rb,db/migrate/20170209164016_add_email_follower_notifications.rb,ruby,0,0,1.0,0,A,0,A,0,0,A,5,0,1,0,0
+sonarqube:SonarqubeFileMetrics:1:02c1047b-f87c-4c35-a6b5-76c6b607d37f:db/migrate/20170213183337_add_sign_up_information_to_users.rb,sonarqube:SonarqubeProject:1:02c1047b-f87c-4c35-a6b5-76c6b607d37f,20170213183337_add_sign_up_information_to_users.rb,db/migrate/20170213183337_add_sign_up_information_to_users.rb,ruby,0,0,1.0,0,A,0,A,0,0,A,7,0,3,0,0
+sonarqube:SonarqubeFileMetrics:1:02c1047b-f87c-4c35-a6b5-76c6b607d37f:db/migrate/20170216145500_add_location_fields_etc_to_users.rb,sonarqube:SonarqubeProject:1:02c1047b-f87c-4c35-a6b5-76c6b607d37f,20170216145500_add_location_fields_etc_to_users.rb,db/migrate/20170216145500_add_location_fields_etc_to_users.rb,ruby,0,0,1.0,0,A,0,A,0,0,A,20,0,16,0,0
+sonarqube:SonarqubeFileMetrics:1:02c1047b-f87c-4c35-a6b5-76c6b607d37f:db/migrate/20170228174838_add_identity_data_to_users.rb,sonarqube:SonarqubeProject:1:02c1047b-f87c-4c35-a6b5-76c6b607d37f,20170228174838_add_identity_data_to_users.rb,db/migrate/20170228174838_add_identity_data_to_users.rb,ruby,0,0,1.0,0,A,0,A,0,0,A,9,0,5,0,0