You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by ab...@apache.org on 2023/02/10 03:52:35 UTC
[incubator-devlake] branch main updated: fix:(sonarqube) fix issues from second review (#4368)
This is an automated email from the ASF dual-hosted git repository.
abeizn pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new e7a42ad83 fix:(sonarqube) fix issues from second review (#4368)
e7a42ad83 is described below
commit e7a42ad831523eeb3d0d9d2b957ee7fc3b641ace
Author: Warren Chen <yi...@merico.dev>
AuthorDate: Fri Feb 10 11:52:30 2023 +0800
fix:(sonarqube) fix issues from second review (#4368)
* fix(sonarqube): add issue key to encode new id
* fix(sonarqube): delete batch id
* fix(sonarqube): add check
---
.../domainlayer/securitytesting/st_file_metrics.go | 1 -
.../domainlayer/securitytesting/st_issues.go | 3 +-
.../migrationscripts/archived/st_file_metrics.go | 1 -
.../models/migrationscripts/archived/st_issues.go | 3 +-
.../migrationscripts/archived/sonarqube_account.go | 1 -
.../archived/sonarqube_file_metrics.go | 1 -
.../migrationscripts/archived/sonarqube_hotspot.go | 1 -
.../migrationscripts/archived/sonarqube_issue.go | 1 -
.../plugins/sonarqube/models/sonarqube_account.go | 1 -
.../sonarqube/models/sonarqube_file_metrics.go | 1 -
.../plugins/sonarqube/models/sonarqube_hotspot.go | 1 -
.../plugins/sonarqube/models/sonarqube_issue.go | 1 -
.../sonarqube/tasks/filemetrics_convertor.go | 11 ++++---
.../sonarqube/tasks/filemetrics_extractor.go | 1 -
.../plugins/sonarqube/tasks/hotspots_convertor.go | 1 -
.../plugins/sonarqube/tasks/hotspots_extractor.go | 1 -
.../plugins/sonarqube/tasks/issues_collector.go | 22 ++++++++++----
.../plugins/sonarqube/tasks/issues_convertor.go | 1 -
.../plugins/sonarqube/tasks/issues_extractor.go | 3 +-
.../plugins/sonarqube/tasks/projects_extractor.go | 34 +++++++++++++++++++---
backend/plugins/sonarqube/tasks/task_data.go | 1 +
21 files changed, 55 insertions(+), 36 deletions(-)
diff --git a/backend/core/models/domainlayer/securitytesting/st_file_metrics.go b/backend/core/models/domainlayer/securitytesting/st_file_metrics.go
index 7f4ed32ab..c22952a02 100644
--- a/backend/core/models/domainlayer/securitytesting/st_file_metrics.go
+++ b/backend/core/models/domainlayer/securitytesting/st_file_metrics.go
@@ -24,7 +24,6 @@ import (
type StFileMetrics struct {
domainlayer.DomainEntity
ProjectKey string `gorm:"index;type:varchar(255)"` //domain project key
- BatchID string `json:"batch_id"`
FileName string `json:"file_name"`
FilePath string `json:"file_path"`
FileLanguage string `json:"file_language"`
diff --git a/backend/core/models/domainlayer/securitytesting/st_issues.go b/backend/core/models/domainlayer/securitytesting/st_issues.go
index 6a9fc1fd5..4963f8dbb 100644
--- a/backend/core/models/domainlayer/securitytesting/st_issues.go
+++ b/backend/core/models/domainlayer/securitytesting/st_issues.go
@@ -27,8 +27,7 @@ type StIssue struct {
Rule string `json:"rule" gorm:"type:varchar(255)"`
Severity string `json:"severity" gorm:"type:varchar(255)"`
Component string `json:"component" gorm:"type:varchar(255)"`
- ProjectKey string `gorm:"index;type:varchar(255)"` //domain project key
- BatchId string `json:"batchId" gorm:"type:varchar(100)"` // from collection time
+ ProjectKey string `gorm:"index;type:varchar(255)"` //domain project key
Line int `json:"line"`
Status string `json:"status" gorm:"type:varchar(255)"`
Message string `json:"message"`
diff --git a/backend/core/models/migrationscripts/archived/st_file_metrics.go b/backend/core/models/migrationscripts/archived/st_file_metrics.go
index 307c93b69..5a5e11162 100644
--- a/backend/core/models/migrationscripts/archived/st_file_metrics.go
+++ b/backend/core/models/migrationscripts/archived/st_file_metrics.go
@@ -20,7 +20,6 @@ package archived
type StFileMetrics struct {
DomainEntity
ProjectKey string `gorm:"index;type:varchar(255)"` //domain project key
- BatchID string `json:"batch_id"`
FileName string `json:"file_name"`
FilePath string `json:"file_path"`
FileLanguage string `json:"file_language"`
diff --git a/backend/core/models/migrationscripts/archived/st_issues.go b/backend/core/models/migrationscripts/archived/st_issues.go
index 55f7493a3..72c0748d8 100644
--- a/backend/core/models/migrationscripts/archived/st_issues.go
+++ b/backend/core/models/migrationscripts/archived/st_issues.go
@@ -26,8 +26,7 @@ type StIssue struct {
Rule string `json:"rule" gorm:"type:varchar(255)"`
Severity string `json:"severity" gorm:"type:varchar(255)"`
Component string `json:"component" gorm:"type:varchar(255)"`
- ProjectKey string `gorm:"index;type:varchar(255)"` //domain project key
- BatchId string `json:"batchId" gorm:"type:varchar(100)"` // from collection time
+ ProjectKey string `gorm:"index;type:varchar(255)"` //domain project key
Line int `json:"line"`
Status string `json:"status" gorm:"type:varchar(255)"`
Message string `json:"message"`
diff --git a/backend/plugins/sonarqube/models/migrationscripts/archived/sonarqube_account.go b/backend/plugins/sonarqube/models/migrationscripts/archived/sonarqube_account.go
index 046644fbe..418c0a64c 100644
--- a/backend/plugins/sonarqube/models/migrationscripts/archived/sonarqube_account.go
+++ b/backend/plugins/sonarqube/models/migrationscripts/archived/sonarqube_account.go
@@ -24,7 +24,6 @@ import (
type SonarqubeAccount struct {
archived.NoPKModel
ConnectionId uint64 `gorm:"primaryKey"`
- BatchId string `json:"batchId" gorm:"type:varchar(100)"` // from collection time
Login string `json:"login" gorm:"primaryKey"`
Name string `json:"name"`
Email string `json:"email"`
diff --git a/backend/plugins/sonarqube/models/migrationscripts/archived/sonarqube_file_metrics.go b/backend/plugins/sonarqube/models/migrationscripts/archived/sonarqube_file_metrics.go
index ab779ef11..95a2fa746 100644
--- a/backend/plugins/sonarqube/models/migrationscripts/archived/sonarqube_file_metrics.go
+++ b/backend/plugins/sonarqube/models/migrationscripts/archived/sonarqube_file_metrics.go
@@ -25,7 +25,6 @@ type SonarqubeFileMetrics struct {
ConnectionId uint64 `gorm:"primaryKey"`
FileMetricsKey string `gorm:"primaryKey"`
ProjectKey string `gorm:"index"`
- BatchID string
FileName string
FilePath string
FileLanguage string
diff --git a/backend/plugins/sonarqube/models/migrationscripts/archived/sonarqube_hotspot.go b/backend/plugins/sonarqube/models/migrationscripts/archived/sonarqube_hotspot.go
index 8ac27b491..ab107f8f6 100644
--- a/backend/plugins/sonarqube/models/migrationscripts/archived/sonarqube_hotspot.go
+++ b/backend/plugins/sonarqube/models/migrationscripts/archived/sonarqube_hotspot.go
@@ -28,7 +28,6 @@ type SonarqubeHotspot struct {
RuleKey string
Component string `gorm:"index"`
ProjectKey string `gorm:"index"`
- BatchId string `gorm:"type:varchar(100)"` // from collection time
Line int
Status string
Message string
diff --git a/backend/plugins/sonarqube/models/migrationscripts/archived/sonarqube_issue.go b/backend/plugins/sonarqube/models/migrationscripts/archived/sonarqube_issue.go
index f4fc781b0..66a470c49 100644
--- a/backend/plugins/sonarqube/models/migrationscripts/archived/sonarqube_issue.go
+++ b/backend/plugins/sonarqube/models/migrationscripts/archived/sonarqube_issue.go
@@ -29,7 +29,6 @@ type SonarqubeIssue struct {
Severity string `gorm:"type:varchar(255)"`
Component string `gorm:"type:varchar(255)"`
ProjectKey string `gorm:"index;type:varchar(255)"` //domain project key
- BatchId string `gorm:"type:varchar(100)"` // from collection time
Line int
Status string
Message string
diff --git a/backend/plugins/sonarqube/models/sonarqube_account.go b/backend/plugins/sonarqube/models/sonarqube_account.go
index e728fd2e2..233ffb538 100644
--- a/backend/plugins/sonarqube/models/sonarqube_account.go
+++ b/backend/plugins/sonarqube/models/sonarqube_account.go
@@ -24,7 +24,6 @@ import (
type SonarqubeAccount struct {
common.NoPKModel
ConnectionId uint64 `gorm:"primaryKey"`
- BatchId string `json:"batchId" gorm:"type:varchar(100)"` // from collection time
Login string `json:"login" gorm:"primaryKey"`
Name string `json:"name"`
Email string `json:"email"`
diff --git a/backend/plugins/sonarqube/models/sonarqube_file_metrics.go b/backend/plugins/sonarqube/models/sonarqube_file_metrics.go
index 4050f4916..da647170f 100644
--- a/backend/plugins/sonarqube/models/sonarqube_file_metrics.go
+++ b/backend/plugins/sonarqube/models/sonarqube_file_metrics.go
@@ -25,7 +25,6 @@ type SonarqubeFileMetrics struct {
ConnectionId uint64 `gorm:"primaryKey"`
FileMetricsKey string `gorm:"primaryKey"`
ProjectKey string `gorm:"index"`
- BatchID string
FileName string
FilePath string
FileLanguage string
diff --git a/backend/plugins/sonarqube/models/sonarqube_hotspot.go b/backend/plugins/sonarqube/models/sonarqube_hotspot.go
index 8b032051c..7bb669374 100644
--- a/backend/plugins/sonarqube/models/sonarqube_hotspot.go
+++ b/backend/plugins/sonarqube/models/sonarqube_hotspot.go
@@ -28,7 +28,6 @@ type SonarqubeHotspot struct {
RuleKey string
Component string `gorm:"index"`
ProjectKey string `gorm:"index"`
- BatchId string `gorm:"type:varchar(100)"` // from collection time
Line int
Status string
Message string
diff --git a/backend/plugins/sonarqube/models/sonarqube_issue.go b/backend/plugins/sonarqube/models/sonarqube_issue.go
index 8d39561a4..bbe327e22 100644
--- a/backend/plugins/sonarqube/models/sonarqube_issue.go
+++ b/backend/plugins/sonarqube/models/sonarqube_issue.go
@@ -29,7 +29,6 @@ type SonarqubeIssue struct {
Severity string `gorm:"type:varchar(255)"`
Component string `gorm:"type:varchar(255)"`
ProjectKey string `gorm:"index;type:varchar(255)"` //domain project key
- BatchId string `gorm:"type:varchar(100)"` // from collection time
Line int
Status string
Message string
diff --git a/backend/plugins/sonarqube/tasks/filemetrics_convertor.go b/backend/plugins/sonarqube/tasks/filemetrics_convertor.go
index 3e8e6c593..b17af0822 100644
--- a/backend/plugins/sonarqube/tasks/filemetrics_convertor.go
+++ b/backend/plugins/sonarqube/tasks/filemetrics_convertor.go
@@ -56,12 +56,11 @@ func ConvertFileMetrics(taskCtx plugin.SubTaskContext) errors.Error {
Convert: func(inputRow interface{}) ([]interface{}, errors.Error) {
sonarqubeFileMetric := inputRow.(*sonarqubeModels.SonarqubeFileMetrics)
domainFileMetric := &securitytesting.StFileMetrics{
- DomainEntity: domainlayer.DomainEntity{Id: issueIdGen.Generate(data.Options.ConnectionId, sonarqubeFileMetric.FileMetricsKey)},
- FileName: sonarqubeFileMetric.FileName,
- FilePath: sonarqubeFileMetric.FilePath,
- FileLanguage: sonarqubeFileMetric.FileLanguage,
- ProjectKey: projectIdGen.Generate(data.Options.ConnectionId, sonarqubeFileMetric.ProjectKey),
- //BatchID: sonarqubeFileMetric.BatchID,
+ DomainEntity: domainlayer.DomainEntity{Id: issueIdGen.Generate(data.Options.ConnectionId, sonarqubeFileMetric.FileMetricsKey)},
+ FileName: sonarqubeFileMetric.FileName,
+ FilePath: sonarqubeFileMetric.FilePath,
+ FileLanguage: sonarqubeFileMetric.FileLanguage,
+ ProjectKey: projectIdGen.Generate(data.Options.ConnectionId, sonarqubeFileMetric.ProjectKey),
CodeSmells: sonarqubeFileMetric.CodeSmells,
SqaleIndex: sonarqubeFileMetric.SqaleIndex,
SqaleRating: sonarqubeFileMetric.SqaleRating,
diff --git a/backend/plugins/sonarqube/tasks/filemetrics_extractor.go b/backend/plugins/sonarqube/tasks/filemetrics_extractor.go
index 16d39a194..163e5ee9b 100644
--- a/backend/plugins/sonarqube/tasks/filemetrics_extractor.go
+++ b/backend/plugins/sonarqube/tasks/filemetrics_extractor.go
@@ -47,7 +47,6 @@ func ExtractFilemetrics(taskCtx plugin.SubTaskContext) errors.Error {
FilePath: body.Path,
FileLanguage: body.Language,
ProjectKey: data.Options.ProjectKey,
- //BatchID: "",
}
alphabetMap := map[string]string{
"1.0": "A",
diff --git a/backend/plugins/sonarqube/tasks/hotspots_convertor.go b/backend/plugins/sonarqube/tasks/hotspots_convertor.go
index 20d9ded8d..a2ce2577d 100644
--- a/backend/plugins/sonarqube/tasks/hotspots_convertor.go
+++ b/backend/plugins/sonarqube/tasks/hotspots_convertor.go
@@ -57,7 +57,6 @@ func ConvertHotspots(taskCtx plugin.SubTaskContext) errors.Error {
sonarqubeHotspot := inputRow.(*sonarqubeModels.SonarqubeHotspot)
domainHotspot := &securitytesting.StIssue{
DomainEntity: domainlayer.DomainEntity{Id: issueIdGen.Generate(data.Options.ConnectionId, sonarqubeHotspot.HotspotKey)},
- BatchId: sonarqubeHotspot.BatchId,
Component: sonarqubeHotspot.Component,
ProjectKey: projectIdGen.Generate(data.Options.ConnectionId, sonarqubeHotspot.ProjectKey),
Line: sonarqubeHotspot.Line,
diff --git a/backend/plugins/sonarqube/tasks/hotspots_extractor.go b/backend/plugins/sonarqube/tasks/hotspots_extractor.go
index 0c394cbc2..b6daae80b 100644
--- a/backend/plugins/sonarqube/tasks/hotspots_extractor.go
+++ b/backend/plugins/sonarqube/tasks/hotspots_extractor.go
@@ -39,7 +39,6 @@ func ExtractHotspots(taskCtx plugin.SubTaskContext) errors.Error {
return nil, err
}
body.ConnectionId = data.Options.ConnectionId
- //body.BatchId = ""
return []interface{}{body}, nil
},
})
diff --git a/backend/plugins/sonarqube/tasks/issues_collector.go b/backend/plugins/sonarqube/tasks/issues_collector.go
index 57a785542..c8ca5b3cd 100644
--- a/backend/plugins/sonarqube/tasks/issues_collector.go
+++ b/backend/plugins/sonarqube/tasks/issues_collector.go
@@ -20,12 +20,11 @@ package tasks
import (
"encoding/json"
"fmt"
- "net/http"
- "net/url"
-
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "net/http"
+ "net/url"
)
const RAW_ISSUES_TABLE = "sonarqube_issues"
@@ -41,7 +40,6 @@ func CollectIssues(taskCtx plugin.SubTaskContext) (err errors.Error) {
if err != nil {
return err
}
-
err = collectorWithState.InitCollector(helper.ApiCollectorArgs{
ApiClient: data.ApiClient,
PageSize: 100,
@@ -58,17 +56,29 @@ func CollectIssues(taskCtx plugin.SubTaskContext) (err errors.Error) {
var resData struct {
Data []json.RawMessage `json:"issues"`
}
- err := helper.UnmarshalResponse(res, &resData)
+ var issue struct {
+ UpdateDate *helper.Iso8601Time `json:"updateDate"`
+ }
+ err = helper.UnmarshalResponse(res, &resData)
if err != nil {
return nil, err
}
+ for _, v := range resData.Data {
+ err = errors.Convert(json.Unmarshal(v, &issue))
+ if err != nil {
+ return nil, err
+ }
+ if issue.UpdateDate.ToTime().After(*data.LastAnalysisDate) {
+ return nil, errors.Default.New(fmt.Sprintf(`Your data is affected by the latest analysis\n
+ Please recollect this project: %s`, data.Options.ProjectKey))
+ }
+ }
return resData.Data, nil
},
})
if err != nil {
return err
}
-
return collectorWithState.Execute()
}
diff --git a/backend/plugins/sonarqube/tasks/issues_convertor.go b/backend/plugins/sonarqube/tasks/issues_convertor.go
index ed0861341..20658ec99 100644
--- a/backend/plugins/sonarqube/tasks/issues_convertor.go
+++ b/backend/plugins/sonarqube/tasks/issues_convertor.go
@@ -58,7 +58,6 @@ func ConvertIssues(taskCtx plugin.SubTaskContext) errors.Error {
sonarqubeIssue := inputRow.(*sonarqubeModels.SonarqubeIssue)
domainIssue := &securitytesting.StIssue{
DomainEntity: domainlayer.DomainEntity{Id: issueIdGen.Generate(data.Options.ConnectionId, sonarqubeIssue.IssueKey)},
- BatchId: sonarqubeIssue.BatchId,
Rule: sonarqubeIssue.Rule,
Severity: sonarqubeIssue.Severity,
Component: sonarqubeIssue.Component,
diff --git a/backend/plugins/sonarqube/tasks/issues_extractor.go b/backend/plugins/sonarqube/tasks/issues_extractor.go
index 382cb118f..978efa852 100644
--- a/backend/plugins/sonarqube/tasks/issues_extractor.go
+++ b/backend/plugins/sonarqube/tasks/issues_extractor.go
@@ -49,7 +49,6 @@ func ExtractIssues(taskCtx plugin.SubTaskContext) errors.Error {
sonarqubeIssue := &models.SonarqubeIssue{
ConnectionId: data.Options.ConnectionId,
IssueKey: body.Key,
- // BatchId string `json:"batchId" gorm:"primaryKey"`
Rule: body.Rule,
Severity: body.Severity,
Component: body.Component,
@@ -155,6 +154,6 @@ type Location struct {
}
func generateId(entity *models.SonarqubeIssueCodeBlock) {
- hashCodeBlock.Write([]byte(fmt.Sprintf("%s-%d-%d-%d-%d-%s", entity.Component, entity.StartLine, entity.EndLine, entity.StartOffset, entity.EndOffset, entity.Msg)))
+ hashCodeBlock.Write([]byte(fmt.Sprintf("%s-%s-%d-%d-%d-%d-%s", entity.IssueKey, entity.Component, entity.StartLine, entity.EndLine, entity.StartOffset, entity.EndOffset, entity.Msg)))
entity.Id = hex.EncodeToString(hashCodeBlock.Sum(nil))
}
diff --git a/backend/plugins/sonarqube/tasks/projects_extractor.go b/backend/plugins/sonarqube/tasks/projects_extractor.go
index 3fec3ffbe..fe4d47afc 100644
--- a/backend/plugins/sonarqube/tasks/projects_extractor.go
+++ b/backend/plugins/sonarqube/tasks/projects_extractor.go
@@ -28,13 +28,39 @@ import (
var _ plugin.SubTaskEntryPoint = ExtractProjects
func ExtractProjects(taskCtx plugin.SubTaskContext) errors.Error {
- rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_PROJECTS_TABLE)
+ // As we need to assign data.LastAnalysisDate, we can not use CreateRawDataSubTaskArgs
+ data := taskCtx.GetData().(*SonarqubeTaskData)
+ var params = SonarqubeApiParams{
+ ConnectionId: data.Options.ConnectionId,
+ ProjectKey: data.Options.ProjectKey,
+ }
+ rawDataSubTaskArgs := &helper.RawDataSubTaskArgs{
+ Ctx: taskCtx,
+ Params: params,
+ Table: RAW_PROJECTS_TABLE,
+ }
extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{
RawDataSubTaskArgs: *rawDataSubTaskArgs,
Extract: func(resData *helper.RawData) ([]interface{}, errors.Error) {
- body := &models.SonarqubeProject{}
- err := errors.Convert(json.Unmarshal(resData.Data, body))
- body.ConnectionId = data.Options.ConnectionId
+ var res struct {
+ ProjectKey string `json:"key"`
+ Name string `json:"name"`
+ Qualifier string `json:"qualifier"`
+ Visibility string `json:"visibility"`
+ LastAnalysisDate *helper.Iso8601Time `json:"lastAnalysisDate"`
+ Revision string `json:"revision"`
+ }
+ err := errors.Convert(json.Unmarshal(resData.Data, &res))
+ body := &models.SonarqubeProject{
+ ConnectionId: data.Options.ConnectionId,
+ ProjectKey: res.ProjectKey,
+ Name: res.Name,
+ Qualifier: res.Qualifier,
+ Visibility: res.Visibility,
+ LastAnalysisDate: res.LastAnalysisDate,
+ Revision: res.Revision,
+ }
+ data.LastAnalysisDate = body.LastAnalysisDate.ToNullableTime()
if err != nil {
return nil, err
}
diff --git a/backend/plugins/sonarqube/tasks/task_data.go b/backend/plugins/sonarqube/tasks/task_data.go
index 9b1b70f5e..7e191885d 100644
--- a/backend/plugins/sonarqube/tasks/task_data.go
+++ b/backend/plugins/sonarqube/tasks/task_data.go
@@ -41,6 +41,7 @@ type SonarqubeOptions struct {
type SonarqubeTaskData struct {
Options *SonarqubeOptions
ApiClient *api.ApiAsyncClient
+ LastAnalysisDate *time.Time
CreatedDateAfter *time.Time
}