You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by ab...@apache.org on 2023/02/10 06:13:16 UTC
[incubator-devlake] branch main updated: test(sonarqube): add hotspots e2e (#4379)
This is an automated email from the ASF dual-hosted git repository.
abeizn pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 0fde141d8 test(sonarqube): add hotspots e2e (#4379)
0fde141d8 is described below
commit 0fde141d8e9caa255584d5d025b7f831fed1ce09
Author: Warren Chen <yi...@merico.dev>
AuthorDate: Fri Feb 10 14:13:11 2023 +0800
test(sonarqube): add hotspots e2e (#4379)
---
backend/plugins/sonarqube/e2e/hotspot_test.go | 70 ++++++++++++++++++++++
.../e2e/raw_tables/_raw_sonarqube_hotspots.csv | 6 ++
.../snapshot_tables/_tool_sonarqube_hotspots.csv | 6 ++
.../e2e/snapshot_tables/issue_hotspots.csv | 4 ++
.../plugins/sonarqube/tasks/hotspots_extractor.go | 35 ++++++++++-
5 files changed, 118 insertions(+), 3 deletions(-)
diff --git a/backend/plugins/sonarqube/e2e/hotspot_test.go b/backend/plugins/sonarqube/e2e/hotspot_test.go
new file mode 100644
index 000000000..5b4339334
--- /dev/null
+++ b/backend/plugins/sonarqube/e2e/hotspot_test.go
@@ -0,0 +1,70 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+ "testing"
+
+ "github.com/apache/incubator-devlake/core/models/common"
+ "github.com/apache/incubator-devlake/core/models/domainlayer/securitytesting"
+ "github.com/apache/incubator-devlake/helpers/e2ehelper"
+ "github.com/apache/incubator-devlake/plugins/sonarqube/impl"
+ "github.com/apache/incubator-devlake/plugins/sonarqube/models"
+ "github.com/apache/incubator-devlake/plugins/sonarqube/tasks"
+)
+
+func TestSonarqubeHotspotDataFlow(t *testing.T) {
+
+ var sonarqube impl.Sonarqube
+ dataflowTester := e2ehelper.NewDataFlowTester(t, "sonarqube", sonarqube)
+
+ // import raw data table
+ dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_sonarqube_hotspots.csv",
+ "_raw_sonarqube_hotspots")
+
+ // Standard data
+ taskData := &tasks.SonarqubeTaskData{
+ Options: &tasks.SonarqubeOptions{
+ ConnectionId: 1,
+ ProjectKey: "f5a50c63-2e8f-4107-9014-853f6f467757",
+ },
+ }
+ // Interfered data
+ taskData2 := &tasks.SonarqubeTaskData{
+ Options: &tasks.SonarqubeOptions{
+ ConnectionId: 2,
+ ProjectKey: "testWarrenEtcd",
+ },
+ }
+
+ // verify extraction
+ dataflowTester.FlushTabler(&models.SonarqubeHotspot{})
+ dataflowTester.Subtask(tasks.ExtractHotspotsMeta, taskData)
+
+ dataflowTester.Subtask(tasks.ExtractHotspotsMeta, taskData2)
+ dataflowTester.VerifyTableWithOptions(&models.SonarqubeHotspot{}, e2ehelper.TableOptions{
+ CSVRelPath: "./snapshot_tables/_tool_sonarqube_hotspots.csv",
+ IgnoreTypes: []interface{}{common.NoPKModel{}},
+ })
+
+ // verify convertor
+ dataflowTester.FlushTabler(&securitytesting.StIssue{})
+ dataflowTester.Subtask(tasks.ConvertHotspotsMeta, taskData)
+ dataflowTester.VerifyTableWithOptions(&securitytesting.StIssue{}, e2ehelper.TableOptions{
+ CSVRelPath: "./snapshot_tables/issue_hotspots.csv",
+ IgnoreTypes: []interface{}{common.NoPKModel{}},
+ })
+}
diff --git a/backend/plugins/sonarqube/e2e/raw_tables/_raw_sonarqube_hotspots.csv b/backend/plugins/sonarqube/e2e/raw_tables/_raw_sonarqube_hotspots.csv
new file mode 100644
index 000000000..12278fdf2
--- /dev/null
+++ b/backend/plugins/sonarqube/e2e/raw_tables/_raw_sonarqube_hotspots.csv
@@ -0,0 +1,6 @@
+"id","params","data","url","input","created_at"
+5517,"{""connectionId"":1,""ProjectKey"":""f5a50c63-2e8f-4107-9014-853f6f467757""}","{""key"":""AYUwBamj46XwcL-YZOPh"",""component"":""f5a50c63-2e8f-4107-9014-853f6f467757:core/src/main/java/com/airbnb/aerosolve/core/function/MultiDimensionSpline.java"",""project"":""f5a50c63-2e8f-4107-9014-853f6f467757"",""securityCategory"":""weak-cryptography"",""vulnerabilityProbability"":""MEDIUM"",""status"":""REVIEWED"",""resolution"":""SAFE"",""line"":295,""message"":""Make sure that using this p [...]
+5518,"{""connectionId"":1,""ProjectKey"":""f5a50c63-2e8f-4107-9014-853f6f467757""}","{""key"":""AYUwBajH46XwcL-YZONY"",""component"":""f5a50c63-2e8f-4107-9014-853f6f467757:core/src/main/java/com/airbnb/aerosolve/core/models/MlpModel.java"",""project"":""f5a50c63-2e8f-4107-9014-853f6f467757"",""securityCategory"":""weak-cryptography"",""vulnerabilityProbability"":""MEDIUM"",""status"":""TO_REVIEW"",""line"":143,""message"":""Make sure that using this pseudorandom number generator is safe [...]
+5519,"{""connectionId"":1,""ProjectKey"":""f5a50c63-2e8f-4107-9014-853f6f467757""}","{""key"":""AYUwBajH46XwcL-YZONZ"",""component"":""f5a50c63-2e8f-4107-9014-853f6f467757:core/src/main/java/com/airbnb/aerosolve/core/models/MlpModel.java"",""project"":""f5a50c63-2e8f-4107-9014-853f6f467757"",""securityCategory"":""weak-cryptography"",""vulnerabilityProbability"":""MEDIUM"",""status"":""TO_REVIEW"",""line"":172,""message"":""Make sure that using this pseudorandom number generator is safe [...]
+5727,"{""connectionId"":2,""ProjectKey"":""testWarrenEtcd""}","{""key"":""AYYrCCi2CVdTZLqEeJeQ"",""component"":""testWarrenEtcd:tests/common/user_test.go"",""project"":""testWarrenEtcd"",""securityCategory"":""auth"",""vulnerabilityProbability"":""HIGH"",""status"":""TO_REVIEW"",""line"":110,""message"":""\""password\"" detected here, make sure this is not a hard-coded credential."",""author"":"""",""creationDate"":""2023-02-07T08:39:18+0000"",""updateDate"":""2023-02-07T08:39:18+0000""} [...]
+5728,"{""connectionId"":2,""ProjectKey"":""testWarrenEtcd""}","{""key"":""AYYrCCi2CVdTZLqEeJeS"",""component"":""testWarrenEtcd:tests/common/user_test.go"",""project"":""testWarrenEtcd"",""securityCategory"":""auth"",""vulnerabilityProbability"":""HIGH"",""status"":""TO_REVIEW"",""line"":148,""message"":""\""password\"" detected here, make sure this is not a hard-coded credential."",""author"":"""",""creationDate"":""2023-02-07T08:39:18+0000"",""updateDate"":""2023-02-07T08:39:18+0000""} [...]
diff --git a/backend/plugins/sonarqube/e2e/snapshot_tables/_tool_sonarqube_hotspots.csv b/backend/plugins/sonarqube/e2e/snapshot_tables/_tool_sonarqube_hotspots.csv
new file mode 100644
index 000000000..b700d465c
--- /dev/null
+++ b/backend/plugins/sonarqube/e2e/snapshot_tables/_tool_sonarqube_hotspots.csv
@@ -0,0 +1,6 @@
+connection_id,hotspot_key,rule_key,component,project_key,line,status,message,author,assignee,security_category,vulnerability_probability,creation_date,update_date
+1,AYUwBajH46XwcL-YZONY,,f5a50c63-2e8f-4107-9014-853f6f467757:core/src/main/java/com/airbnb/aerosolve/core/models/MlpModel.java,f5a50c63-2e8f-4107-9014-853f6f467757,143,TO_REVIEW,Make sure that using this pseudorandom number generator is safe here.,peng.ye@airbnb.com,,weak-cryptography,MEDIUM,2016-02-26T22:41:48.000+00:00,2022-12-20T14:50:30.000+00:00
+1,AYUwBajH46XwcL-YZONZ,,f5a50c63-2e8f-4107-9014-853f6f467757:core/src/main/java/com/airbnb/aerosolve/core/models/MlpModel.java,f5a50c63-2e8f-4107-9014-853f6f467757,172,TO_REVIEW,Make sure that using this pseudorandom number generator is safe here.,peng.ye@airbnb.com,,weak-cryptography,MEDIUM,2016-02-26T22:41:48.000+00:00,2022-12-20T14:50:30.000+00:00
+1,AYUwBamj46XwcL-YZOPh,,f5a50c63-2e8f-4107-9014-853f6f467757:core/src/main/java/com/airbnb/aerosolve/core/function/MultiDimensionSpline.java,f5a50c63-2e8f-4107-9014-853f6f467757,295,REVIEWED,Make sure that using this pseudorandom number generator is safe here.,julian.qian@airbnb.com,,weak-cryptography,MEDIUM,2016-04-25T19:02:16.000+00:00,2022-12-21T08:30:43.000+00:00
+2,AYYrCCi2CVdTZLqEeJeQ,,testWarrenEtcd:tests/common/user_test.go,testWarrenEtcd,110,TO_REVIEW,"""password"" detected here, make sure this is not a hard-coded credential.",,,auth,HIGH,2023-02-07T08:39:18.000+00:00,2023-02-07T08:39:18.000+00:00
+2,AYYrCCi2CVdTZLqEeJeS,,testWarrenEtcd:tests/common/user_test.go,testWarrenEtcd,148,TO_REVIEW,"""password"" detected here, make sure this is not a hard-coded credential.",,,auth,HIGH,2023-02-07T08:39:18.000+00:00,2023-02-07T08:39:18.000+00:00
diff --git a/backend/plugins/sonarqube/e2e/snapshot_tables/issue_hotspots.csv b/backend/plugins/sonarqube/e2e/snapshot_tables/issue_hotspots.csv
new file mode 100644
index 000000000..5f7a7de85
--- /dev/null
+++ b/backend/plugins/sonarqube/e2e/snapshot_tables/issue_hotspots.csv
@@ -0,0 +1,4 @@
+id,rule,severity,component,project_key,line,status,message,debt,effort,commit_author_email,assignee,hash,tags,type,scope,start_line,end_line,start_offset,end_offset,vulnerability_probability,security_category,creation_date,update_date
+sonarqube:SonarqubeHotspot:1:AYUwBajH46XwcL-YZONY,,weak-cryptography,f5a50c63-2e8f-4107-9014-853f6f467757:core/src/main/java/com/airbnb/aerosolve/core/models/MlpModel.java,sonarqube:SonarqubeProject:1:f5a50c63-2e8f-4107-9014-853f6f467757,143,TO_REVIEW,Make sure that using this pseudorandom number generator is safe here.,,,peng.ye@airbnb.com,,,,HOTSPOTS,,143,0,0,0,MEDIUM,,2016-02-26T22:41:48.000+00:00,2022-12-20T14:50:30.000+00:00
+sonarqube:SonarqubeHotspot:1:AYUwBajH46XwcL-YZONZ,,weak-cryptography,f5a50c63-2e8f-4107-9014-853f6f467757:core/src/main/java/com/airbnb/aerosolve/core/models/MlpModel.java,sonarqube:SonarqubeProject:1:f5a50c63-2e8f-4107-9014-853f6f467757,172,TO_REVIEW,Make sure that using this pseudorandom number generator is safe here.,,,peng.ye@airbnb.com,,,,HOTSPOTS,,172,0,0,0,MEDIUM,,2016-02-26T22:41:48.000+00:00,2022-12-20T14:50:30.000+00:00
+sonarqube:SonarqubeHotspot:1:AYUwBamj46XwcL-YZOPh,,weak-cryptography,f5a50c63-2e8f-4107-9014-853f6f467757:core/src/main/java/com/airbnb/aerosolve/core/function/MultiDimensionSpline.java,sonarqube:SonarqubeProject:1:f5a50c63-2e8f-4107-9014-853f6f467757,295,REVIEWED,Make sure that using this pseudorandom number generator is safe here.,,,julian.qian@airbnb.com,,,,HOTSPOTS,,295,0,0,0,MEDIUM,,2016-04-25T19:02:16.000+00:00,2022-12-21T08:30:43.000+00:00
diff --git a/backend/plugins/sonarqube/tasks/hotspots_extractor.go b/backend/plugins/sonarqube/tasks/hotspots_extractor.go
index b6daae80b..d231fa5a5 100644
--- a/backend/plugins/sonarqube/tasks/hotspots_extractor.go
+++ b/backend/plugins/sonarqube/tasks/hotspots_extractor.go
@@ -33,12 +33,41 @@ func ExtractHotspots(taskCtx plugin.SubTaskContext) errors.Error {
extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{
RawDataSubTaskArgs: *rawDataSubTaskArgs,
Extract: func(resData *helper.RawData) ([]interface{}, errors.Error) {
- body := &models.SonarqubeHotspot{}
- err := errors.Convert(json.Unmarshal(resData.Data, body))
+ var res struct {
+ Key string `json:"key" gorm:"primaryKey"`
+ RuleKey string `json:"ruleKey"`
+ Component string `json:"component" gorm:"index"`
+ ProjectKey string `json:"project" gorm:"index"`
+ Line int `json:"line"`
+ Status string `json:"status"`
+ Message string `json:"message"`
+ Author string `json:"author"`
+ Assignee string `json:"assignee"`
+ SecurityCategory string `json:"securityCategory"`
+ VulnerabilityProbability string `json:"vulnerabilityProbability"`
+ CreationDate *helper.Iso8601Time `json:"creationDate"`
+ UpdateDate *helper.Iso8601Time `json:"updateDate"`
+ }
+ err := errors.Convert(json.Unmarshal(resData.Data, &res))
if err != nil {
return nil, err
}
- body.ConnectionId = data.Options.ConnectionId
+ body := &models.SonarqubeHotspot{
+ ConnectionId: data.Options.ConnectionId,
+ HotspotKey: res.Key,
+ RuleKey: res.RuleKey,
+ Component: res.Component,
+ ProjectKey: res.ProjectKey,
+ Line: res.Line,
+ Status: res.Status,
+ Message: res.Message,
+ Author: res.Author,
+ Assignee: res.Assignee,
+ SecurityCategory: res.SecurityCategory,
+ VulnerabilityProbability: res.VulnerabilityProbability,
+ CreationDate: res.CreationDate,
+ UpdateDate: res.UpdateDate,
+ }
return []interface{}{body}, nil
},
})