You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by ab...@apache.org on 2023/11/29 00:56:59 UTC
(incubator-devlake) branch release-v0.20 updated: fix: add collect sonarqube mode by file (#6507) (#6532)
This is an automated email from the ASF dual-hosted git repository.
abeizn pushed a commit to branch release-v0.20
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/release-v0.20 by this push:
new 970b8e378 fix: add collect sonarqube mode by file (#6507) (#6532)
970b8e378 is described below
commit 970b8e378b1ed8d1a418f009aa6c319d6717658f
Author: github-actions[bot] <41...@users.noreply.github.com>
AuthorDate: Wed Nov 29 08:56:54 2023 +0800
fix: add collect sonarqube mode by file (#6507) (#6532)
* fix: add collect sonarqube mode by file
* fix: update some collector logic
* fix: some params
* fix: update some params
* fix: still need to push issues when the count exceed MAXISSUECOUNT
* fix: update log format and variable name
---------
Co-authored-by: abeizn <zi...@merico.dev>
Co-authored-by: Lynwee <li...@merico.dev>
---
backend/plugins/sonarqube/impl/impl.go | 8 +-
.../plugins/sonarqube/tasks/issues_collector.go | 109 ++++++++++++++++++---
2 files changed, 99 insertions(+), 18 deletions(-)
diff --git a/backend/plugins/sonarqube/impl/impl.go b/backend/plugins/sonarqube/impl/impl.go
index 3bf4894ae..65540112d 100644
--- a/backend/plugins/sonarqube/impl/impl.go
+++ b/backend/plugins/sonarqube/impl/impl.go
@@ -89,14 +89,14 @@ func (p Sonarqube) GetTablesInfo() []dal.Tabler {
func (p Sonarqube) SubTaskMetas() []plugin.SubTaskMeta {
return []plugin.SubTaskMeta{
- tasks.CollectIssuesMeta,
- tasks.ExtractIssuesMeta,
- tasks.CollectHotspotsMeta,
- tasks.ExtractHotspotsMeta,
tasks.CollectAdditionalFilemetricsMeta,
tasks.ExtractAdditionalFileMetricsMeta,
tasks.CollectFilemetricsMeta,
tasks.ExtractFilemetricsMeta,
+ tasks.CollectIssuesMeta,
+ tasks.ExtractIssuesMeta,
+ tasks.CollectHotspotsMeta,
+ tasks.ExtractHotspotsMeta,
tasks.CollectAccountsMeta,
tasks.ExtractAccountsMeta,
tasks.ConvertProjectsMeta,
diff --git a/backend/plugins/sonarqube/tasks/issues_collector.go b/backend/plugins/sonarqube/tasks/issues_collector.go
index e9942b362..76e2e1f64 100644
--- a/backend/plugins/sonarqube/tasks/issues_collector.go
+++ b/backend/plugins/sonarqube/tasks/issues_collector.go
@@ -32,6 +32,10 @@ import (
const RAW_ISSUES_TABLE = "sonarqube_api_issues"
+const MAXPAGES = 100
+const MAXISSUECOUNT = 10000
+const MININTERVAL = 10
+
var _ plugin.SubTaskEntryPoint = CollectIssues
type SonarqubeIssueIteratorNode struct {
@@ -40,6 +44,7 @@ type SonarqubeIssueIteratorNode struct {
Type string
CreatedAfter *time.Time
CreatedBefore *time.Time
+ FilePath string
}
func CollectIssues(taskCtx plugin.SubTaskContext) (err errors.Error) {
@@ -60,6 +65,7 @@ func CollectIssues(taskCtx plugin.SubTaskContext) (err errors.Error) {
Type: typ,
CreatedAfter: nil,
CreatedBefore: nil,
+ FilePath: "",
},
)
}
@@ -74,14 +80,20 @@ func CollectIssues(taskCtx plugin.SubTaskContext) (err errors.Error) {
Input: iterator,
Query: func(reqData *helper.RequestData) (url.Values, errors.Error) {
query := url.Values{}
- query.Set("componentKeys", fmt.Sprintf("%v", data.Options.ProjectKey))
input, ok := reqData.Input.(*SonarqubeIssueIteratorNode)
if !ok {
return nil, errors.Default.New(fmt.Sprintf("Input to SonarqubeIssueIteratorNode failed:%+v", reqData.Input))
}
- query.Set("severities", reqData.Input.(*SonarqubeIssueIteratorNode).Severity)
- query.Set("statuses", reqData.Input.(*SonarqubeIssueIteratorNode).Status)
- query.Set("types", reqData.Input.(*SonarqubeIssueIteratorNode).Type)
+ if input.FilePath != "" {
+ query.Del("facets")
+ query.Set("componentKeys", fmt.Sprintf("%v:%v", data.Options.ProjectKey, input.FilePath))
+ } else {
+ query.Set("componentKeys", fmt.Sprintf("%v", data.Options.ProjectKey))
+ query.Set("facets", "directories")
+ }
+ query.Set("severities", input.Severity)
+ query.Set("statuses", input.Status)
+ query.Set("types", input.Type)
if input.CreatedAfter != nil {
query.Set("createdAfter", GetFormatTime(input.CreatedAfter))
}
@@ -90,11 +102,12 @@ func CollectIssues(taskCtx plugin.SubTaskContext) (err errors.Error) {
}
query.Set("p", fmt.Sprintf("%v", reqData.Pager.Page))
query.Set("ps", fmt.Sprintf("%v", reqData.Pager.Size))
+
query.Encode()
return query, nil
},
GetTotalPages: func(res *http.Response, args *helper.ApiCollectorArgs) (int, errors.Error) {
- body := &SonarqubePagination{}
+ body := &SonarqubePageInfo{}
err := helper.UnmarshalResponse(res, body)
if err != nil {
return 0, err
@@ -106,9 +119,11 @@ func CollectIssues(taskCtx plugin.SubTaskContext) (err errors.Error) {
}
query := res.Request.URL.Query()
-
// if get more than 10000 data, that need split it
- if pages > 100 {
+ if pages > MAXPAGES {
+ severity := query.Get("severities")
+ status := query.Get("statuses")
+ typ := query.Get("types")
var createdAfterUnix int64
var createdBeforeUnix int64
@@ -133,17 +148,62 @@ func CollectIssues(taskCtx plugin.SubTaskContext) (err errors.Error) {
createdBeforeUnix = createdBefore.Unix()
}
- // can not split it any more
- if createdBeforeUnix-createdAfterUnix <= 10 {
- return 100, nil
+ // can not split it by time
+ if createdBeforeUnix-createdAfterUnix <= MININTERVAL {
+ // split it by dir/fil
+ for _, facet := range body.Facets {
+ for _, value := range facet.Values {
+ if value.Count <= MAXISSUECOUNT {
+ iterator.Push(&SonarqubeIssueIteratorNode{
+ Severity: severity,
+ Status: status,
+ Type: typ,
+ CreatedAfter: createdAfter,
+ CreatedBefore: createdBefore,
+ FilePath: value.Val,
+ })
+ logger.Info("split by dir, and it's issue count:[%d] and file path:[%s]", value.Count, value.Val)
+ } else {
+ // split it by dir when it's issue count > 10000
+ resWithPath, err := data.ApiClient.Get("issues/search", url.Values{
+ "componentKeys": {fmt.Sprintf("%v", data.Options.ProjectKey)},
+ "directories": {value.Val},
+ "facets": {"files"},
+ "ps": {"1"},
+ }, nil)
+ if err != nil {
+ return 0, err
+ }
+ bodyWithPath := &SonarqubePageInfo{}
+ err = helper.UnmarshalResponse(resWithPath, bodyWithPath)
+ if err != nil {
+ return 0, err
+ }
+ if len(bodyWithPath.Facets) != 1 {
+ return 0, errors.Default.New(fmt.Sprintf("the facets count [%d] is not 1", len(bodyWithPath.Facets)))
+ }
+ for _, value2 := range bodyWithPath.Facets[0].Values {
+ if value2.Count > MAXISSUECOUNT {
+ logger.Warn(fmt.Errorf("the issue count [%d] exceeds the maximum page size", value2.Count), "")
+ }
+ iterator.Push(&SonarqubeIssueIteratorNode{
+ Severity: severity,
+ Status: status,
+ Type: typ,
+ CreatedAfter: createdAfter,
+ CreatedBefore: createdBefore,
+ FilePath: value2.Val,
+ })
+ logger.Info("split by fil, and it's issue count:[%d] and file path:[%s]", value2.Count, value2.Val)
+ }
+ }
+ }
+ }
+ return 0, nil
}
// split it
MidTime := time.Unix((createdAfterUnix+createdBeforeUnix)/2+1, 0)
-
- severity := query.Get("severities")
- status := query.Get("statuses")
- typ := query.Get("types")
// left part
iterator.Push(&SonarqubeIssueIteratorNode{
Severity: severity,
@@ -239,3 +299,24 @@ func getTimeFromFormatTime(formatTime string) (*time.Time, errors.Error) {
return &t, nil
}
+
+type SonarqubePageInfo struct {
+ Total int `json:"total"`
+ P int `json:"p"`
+ Ps int `json:"ps"`
+ Paging struct {
+ PageIndex int `json:"pageIndex"`
+ PageSize int `json:"pageSize"`
+ Total int `json:"total"`
+ } `json:"paging"`
+ EffortTotal int `json:"effortTotal"`
+ Issues []any `json:"issues"`
+ Components []any `json:"components"`
+ Facets []struct {
+ Property string `json:"property"`
+ Values []struct {
+ Val string `json:"val"`
+ Count int `json:"count"`
+ } `json:"values"`
+ } `json:"facets"`
+}