You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by ab...@apache.org on 2023/02/07 01:50:07 UTC
[incubator-devlake] branch main updated: feat(sonarqube): extract code blocks (#4336)
This is an automated email from the ASF dual-hosted git repository.
abeizn pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 3c62ee2c6 feat(sonarqube): extract code blocks (#4336)
3c62ee2c6 is described below
commit 3c62ee2c6f3168c74a1c32c0cd2f3d0c3288f6a7
Author: Warren Chen <yi...@merico.dev>
AuthorDate: Tue Feb 7 09:50:02 2023 +0800
feat(sonarqube): extract code blocks (#4336)
---
.../migrationscripts/20230111_add_init_tables.go | 3 +-
.../migrationscripts/archived/sonarqube_hotspot.go | 6 +--
.../migrationscripts/archived/sonarqube_issue.go | 4 +-
...be_project.go => sonarqube_issue_code_block.go} | 27 ++++++------
.../migrationscripts/archived/sonarqube_project.go | 2 +-
.../plugins/sonarqube/models/sonarqube_hotspot.go | 2 +-
.../plugins/sonarqube/models/sonarqube_issue.go | 2 +-
...be_project.go => sonarqube_issue_code_block.go} | 29 +++++++------
.../plugins/sonarqube/tasks/hotspots_extractor.go | 1 +
.../plugins/sonarqube/tasks/issues_extractor.go | 50 ++++++++++++++++++++--
10 files changed, 86 insertions(+), 40 deletions(-)
diff --git a/backend/plugins/sonarqube/models/migrationscripts/20230111_add_init_tables.go b/backend/plugins/sonarqube/models/migrationscripts/20230111_add_init_tables.go
index 62a792433..90c19cc7b 100644
--- a/backend/plugins/sonarqube/models/migrationscripts/20230111_add_init_tables.go
+++ b/backend/plugins/sonarqube/models/migrationscripts/20230111_add_init_tables.go
@@ -33,11 +33,12 @@ func (*addInitTables) Up(basicRes context.BasicRes) errors.Error {
&archived.SonarqubeProject{},
&archived.SonarqubeHotspot{},
&archived.SonarqubeIssue{},
+ &archived.SonarqubeIssueCodeBlock{},
)
}
func (*addInitTables) Version() uint64 {
- return 20230206000011
+ return 20230206200015
}
func (*addInitTables) Name() string {
diff --git a/backend/plugins/sonarqube/models/migrationscripts/archived/sonarqube_hotspot.go b/backend/plugins/sonarqube/models/migrationscripts/archived/sonarqube_hotspot.go
index 6217826ee..a94aeac94 100644
--- a/backend/plugins/sonarqube/models/migrationscripts/archived/sonarqube_hotspot.go
+++ b/backend/plugins/sonarqube/models/migrationscripts/archived/sonarqube_hotspot.go
@@ -23,11 +23,10 @@ import (
)
type SonarqubeHotspot struct {
- archived.NoPKModel
ConnectionId uint64 `gorm:"primaryKey"`
Key string `json:"key" gorm:"primaryKey"`
- BatchId string `json:"batchId" gorm:"primaryKey"` // from collection time
- Component string `json:"component"`
+ BatchId string `json:"batchId" gorm:"type:varchar(100)"` // from collection time
+ Component string `json:"component" gorm:"index"`
Project string `json:"project" gorm:"index"`
SecurityCategory string `json:"securityCategory"`
VulnerabilityProbability string `json:"vulnerabilityProbability"`
@@ -39,6 +38,7 @@ type SonarqubeHotspot struct {
CreationDate *api.Iso8601Time `json:"creationDate"`
UpdateDate *api.Iso8601Time `json:"updateDate"`
RuleKey string `json:"ruleKey"`
+ archived.NoPKModel
}
func (SonarqubeHotspot) TableName() string {
diff --git a/backend/plugins/sonarqube/models/migrationscripts/archived/sonarqube_issue.go b/backend/plugins/sonarqube/models/migrationscripts/archived/sonarqube_issue.go
index 21b2a1942..0beef00aa 100644
--- a/backend/plugins/sonarqube/models/migrationscripts/archived/sonarqube_issue.go
+++ b/backend/plugins/sonarqube/models/migrationscripts/archived/sonarqube_issue.go
@@ -23,10 +23,9 @@ import (
)
type SonarqubeIssue struct {
- archived.NoPKModel
ConnectionId uint64 `gorm:"primaryKey"`
Key string `json:"key" gorm:"primaryKey"`
- BatchId string `json:"batchId" gorm:"primaryKey"`
+ BatchId string `json:"batchId" gorm:"type:varchar(100)"` // from collection time
Rule string `json:"rule"`
Severity string `json:"severity"`
Component string `json:"component"`
@@ -47,6 +46,7 @@ type SonarqubeIssue struct {
EndOffset int `json:"endOffset"`
CreationDate *api.Iso8601Time `json:"creationDate"`
UpdateDate *api.Iso8601Time `json:"updateDate"`
+ archived.NoPKModel
}
func (SonarqubeIssue) TableName() string {
diff --git a/backend/plugins/sonarqube/models/migrationscripts/archived/sonarqube_project.go b/backend/plugins/sonarqube/models/migrationscripts/archived/sonarqube_issue_code_block.go
similarity index 54%
copy from backend/plugins/sonarqube/models/migrationscripts/archived/sonarqube_project.go
copy to backend/plugins/sonarqube/models/migrationscripts/archived/sonarqube_issue_code_block.go
index ef1754dcb..cab175c35 100644
--- a/backend/plugins/sonarqube/models/migrationscripts/archived/sonarqube_project.go
+++ b/backend/plugins/sonarqube/models/migrationscripts/archived/sonarqube_issue_code_block.go
@@ -17,22 +17,21 @@ limitations under the License.
package archived
-import (
- "github.com/apache/incubator-devlake/core/models/migrationscripts/archived"
- "time"
-)
+import "github.com/apache/incubator-devlake/core/models/migrationscripts/archived"
-type SonarqubeProject struct {
+type SonarqubeIssueCodeBlock struct {
+ ConnectionId uint64 `gorm:"primaryKey"`
+ Id string `gorm:"primaryKey"`
+ IssueKey string `json:"key" gorm:"index"`
+ Component string `json:"component" gorm:"index"`
+ Msg string `json:"msg" `
+ StartLine int `json:"startLine" `
+ EndLine int `json:"endLine" `
+ StartOffset int `json:"startOffset" `
+ EndOffset int `json:"endOffset" `
archived.NoPKModel
- ConnectionId uint64 `gorm:"primaryKey"`
- Key string `json:"key" gorm:"type:varchar(64);primaryKey"`
- Name string `json:"name" gorm:"type:varchar(255)"`
- Qualifier string `json:"qualifier" gorm:"type:varchar(255)"`
- Visibility string `json:"visibility" gorm:"type:varchar(64)"`
- LastAnalysisDate *time.Time `json:"lastAnalysisDate"`
- Revision string `json:"revision" gorm:"type:varchar(128)"`
}
-func (SonarqubeProject) TableName() string {
- return "_tool_sonarqube_projects"
+func (SonarqubeIssueCodeBlock) TableName() string {
+ return "_tool_sonarqube_issue_code_blocks"
}
diff --git a/backend/plugins/sonarqube/models/migrationscripts/archived/sonarqube_project.go b/backend/plugins/sonarqube/models/migrationscripts/archived/sonarqube_project.go
index ef1754dcb..5d1a2e466 100644
--- a/backend/plugins/sonarqube/models/migrationscripts/archived/sonarqube_project.go
+++ b/backend/plugins/sonarqube/models/migrationscripts/archived/sonarqube_project.go
@@ -23,7 +23,6 @@ import (
)
type SonarqubeProject struct {
- archived.NoPKModel
ConnectionId uint64 `gorm:"primaryKey"`
Key string `json:"key" gorm:"type:varchar(64);primaryKey"`
Name string `json:"name" gorm:"type:varchar(255)"`
@@ -31,6 +30,7 @@ type SonarqubeProject struct {
Visibility string `json:"visibility" gorm:"type:varchar(64)"`
LastAnalysisDate *time.Time `json:"lastAnalysisDate"`
Revision string `json:"revision" gorm:"type:varchar(128)"`
+ archived.NoPKModel
}
func (SonarqubeProject) TableName() string {
diff --git a/backend/plugins/sonarqube/models/sonarqube_hotspot.go b/backend/plugins/sonarqube/models/sonarqube_hotspot.go
index 03bea2877..ed99ce892 100644
--- a/backend/plugins/sonarqube/models/sonarqube_hotspot.go
+++ b/backend/plugins/sonarqube/models/sonarqube_hotspot.go
@@ -26,7 +26,7 @@ type SonarqubeHotspot struct {
common.NoPKModel
ConnectionId uint64 `gorm:"primaryKey"`
Key string `json:"key" gorm:"primaryKey"`
- BatchId string `json:"batchId" gorm:"primaryKey"` // from collection time
+ BatchId string `json:"batchId" gorm:"type:varchar(100)"` // from collection time
Component string `json:"component"`
Project string `json:"project" gorm:"index"` // projects.key
SecurityCategory string `json:"securityCategory"`
diff --git a/backend/plugins/sonarqube/models/sonarqube_issue.go b/backend/plugins/sonarqube/models/sonarqube_issue.go
index c33058ba0..67027625b 100644
--- a/backend/plugins/sonarqube/models/sonarqube_issue.go
+++ b/backend/plugins/sonarqube/models/sonarqube_issue.go
@@ -26,7 +26,7 @@ type SonarqubeIssue struct {
common.NoPKModel
ConnectionId uint64 `gorm:"primaryKey"`
Key string `json:"key" gorm:"primaryKey"`
- BatchId string `json:"batchId" gorm:"primaryKey"`
+ BatchId string `json:"batchId" gorm:"type:varchar(100)"` // from collection time
Rule string `json:"rule"`
Severity string `json:"severity"`
Component string `json:"component"`
diff --git a/backend/plugins/sonarqube/models/migrationscripts/archived/sonarqube_project.go b/backend/plugins/sonarqube/models/sonarqube_issue_code_block.go
similarity index 52%
copy from backend/plugins/sonarqube/models/migrationscripts/archived/sonarqube_project.go
copy to backend/plugins/sonarqube/models/sonarqube_issue_code_block.go
index ef1754dcb..d16939d3e 100644
--- a/backend/plugins/sonarqube/models/migrationscripts/archived/sonarqube_project.go
+++ b/backend/plugins/sonarqube/models/sonarqube_issue_code_block.go
@@ -15,24 +15,25 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
-package archived
+package models
import (
- "github.com/apache/incubator-devlake/core/models/migrationscripts/archived"
- "time"
+ "github.com/apache/incubator-devlake/core/models/common"
)
-type SonarqubeProject struct {
- archived.NoPKModel
- ConnectionId uint64 `gorm:"primaryKey"`
- Key string `json:"key" gorm:"type:varchar(64);primaryKey"`
- Name string `json:"name" gorm:"type:varchar(255)"`
- Qualifier string `json:"qualifier" gorm:"type:varchar(255)"`
- Visibility string `json:"visibility" gorm:"type:varchar(64)"`
- LastAnalysisDate *time.Time `json:"lastAnalysisDate"`
- Revision string `json:"revision" gorm:"type:varchar(128)"`
+type SonarqubeIssueCodeBlock struct {
+ common.NoPKModel
+ ConnectionId uint64 `gorm:"primaryKey"`
+ Id string `gorm:"primaryKey"`
+ IssueKey string `json:"key" gorm:"index"`
+ Component string `json:"component" gorm:"index"`
+ Msg string `json:"msg" `
+ StartLine int `json:"startLine" `
+ EndLine int `json:"endLine" `
+ StartOffset int `json:"startOffset" `
+ EndOffset int `json:"endOffset" `
}
-func (SonarqubeProject) TableName() string {
- return "_tool_sonarqube_projects"
+func (SonarqubeIssueCodeBlock) TableName() string {
+ return "_tool_sonarqube_issue_code_blocks"
}
diff --git a/backend/plugins/sonarqube/tasks/hotspots_extractor.go b/backend/plugins/sonarqube/tasks/hotspots_extractor.go
index 21dd96353..22d4ea440 100644
--- a/backend/plugins/sonarqube/tasks/hotspots_extractor.go
+++ b/backend/plugins/sonarqube/tasks/hotspots_extractor.go
@@ -36,6 +36,7 @@ func ExtractHotspots(taskCtx plugin.SubTaskContext) errors.Error {
body := &models.SonarqubeHotspot{}
err := errors.Convert(json.Unmarshal(resData.Data, body))
body.ConnectionId = data.Options.ConnectionId
+ //body.BatchId = ""
if err != nil {
return nil, err
}
diff --git a/backend/plugins/sonarqube/tasks/issues_extractor.go b/backend/plugins/sonarqube/tasks/issues_extractor.go
index 78265e658..97e941797 100644
--- a/backend/plugins/sonarqube/tasks/issues_extractor.go
+++ b/backend/plugins/sonarqube/tasks/issues_extractor.go
@@ -18,7 +18,11 @@ limitations under the License.
package tasks
import (
+ "crypto/sha256"
+ "encoding/hex"
"encoding/json"
+ "fmt"
+ "hash"
"strings"
"github.com/apache/incubator-devlake/core/errors"
@@ -28,10 +32,11 @@ import (
)
var _ plugin.SubTaskEntryPoint = ExtractIssues
+var hashCodeBlock hash.Hash
func ExtractIssues(taskCtx plugin.SubTaskContext) errors.Error {
rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_ISSUES_TABLE)
-
+ hashCodeBlock = sha256.New()
extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{
RawDataSubTaskArgs: *rawDataSubTaskArgs,
Extract: func(resData *helper.RawData) ([]interface{}, errors.Error) {
@@ -69,7 +74,26 @@ func ExtractIssues(taskCtx plugin.SubTaskContext) errors.Error {
sonarqubeIssue.Tags = strings.Join(body.Tags, ",")
}
- return []interface{}{sonarqubeIssue}, nil
+ results := make([]interface{}, 0)
+ results = append(results, sonarqubeIssue)
+ for _, v := range body.Flows {
+ for _, location := range v.Locations {
+ codeBlock := &models.SonarqubeIssueCodeBlock{
+ ConnectionId: data.Options.ConnectionId,
+ IssueKey: sonarqubeIssue.Key,
+ Component: location.Component,
+ Msg: location.Msg,
+ StartLine: location.TextRange.StartLine,
+ EndLine: location.TextRange.EndLine,
+ StartOffset: location.TextRange.StartOffset,
+ EndOffset: location.TextRange.EndOffset,
+ }
+ generateId(codeBlock)
+ results = append(results, codeBlock)
+ }
+ }
+
+ return results, nil
},
})
if err != nil {
@@ -100,7 +124,7 @@ type IssuesResponse struct {
StartOffset int `json:"startOffset"`
EndOffset int `json:"endOffset"`
} `json:"textRange"`
- Flows []interface{} `json:"flows"`
+ Flows []flow `json:"flows"`
Status string `json:"status"`
Message string `json:"message"`
Effort string `json:"effort"`
@@ -113,3 +137,23 @@ type IssuesResponse struct {
Scope string `json:"scope"`
QuickFixAvailable bool `json:"quickFixAvailable"`
}
+
+type flow struct {
+ Locations []Location `json:"locations"`
+}
+type TextRange struct {
+ StartLine int `json:"startLine"`
+ EndLine int `json:"endLine"`
+ StartOffset int `json:"startOffset"`
+ EndOffset int `json:"endOffset"`
+}
+type Location struct {
+ Component string `json:"component"`
+ TextRange TextRange `json:"textRange"`
+ Msg string `json:"msg"`
+}
+
+func generateId(entity *models.SonarqubeIssueCodeBlock) {
+ hashCodeBlock.Write([]byte(fmt.Sprintf("%s-%d-%d-%d-%d-%s", entity.Component, entity.StartLine, entity.EndLine, entity.StartOffset, entity.EndOffset, entity.Msg)))
+ entity.Id = hex.EncodeToString(hashCodeBlock.Sum(nil))
+}