You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by wa...@apache.org on 2023/02/06 09:45:45 UTC
[incubator-devlake] branch main updated: feat: sonarqube issue collector and extractor (#4332)
This is an automated email from the ASF dual-hosted git repository.
warren pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 2d71172fb feat: sonarqube issue collector and extractor (#4332)
2d71172fb is described below
commit 2d71172fb29c2b5b8b288ed24b1166e749ae5f11
Author: abeizn <zi...@merico.dev>
AuthorDate: Mon Feb 6 17:45:40 2023 +0800
feat: sonarqube issue collector and extractor (#4332)
* feat: sonarqube issue collector and extractor
* feat: sonarqube add data.option.ProjectKey and remove cursor
---
backend/plugins/sonarqube/impl/impl.go | 5 +-
.../migrationscripts/20230111_add_init_tables.go | 3 +-
.../migrationscripts/archived/sonarqube_issue.go | 54 ++++++++++
.../plugins/sonarqube/models/sonarqube_issue.go | 54 ++++++++++
.../plugins/sonarqube/tasks/issues_collector.go | 80 ++++++++++++++
.../plugins/sonarqube/tasks/issues_extractor.go | 115 +++++++++++++++++++++
6 files changed, 309 insertions(+), 2 deletions(-)
diff --git a/backend/plugins/sonarqube/impl/impl.go b/backend/plugins/sonarqube/impl/impl.go
index 833a8c41f..aaaff7872 100644
--- a/backend/plugins/sonarqube/impl/impl.go
+++ b/backend/plugins/sonarqube/impl/impl.go
@@ -19,11 +19,12 @@ package impl
import (
"fmt"
+ "time"
+
"github.com/apache/incubator-devlake/core/context"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
- "time"
"github.com/apache/incubator-devlake/plugins/sonarqube/api"
"github.com/apache/incubator-devlake/plugins/sonarqube/models"
@@ -54,6 +55,8 @@ func (p Sonarqube) SubTaskMetas() []plugin.SubTaskMeta {
return []plugin.SubTaskMeta{
tasks.CollectProjectsMeta,
tasks.ExtractProjectsMeta,
+ tasks.CollectIssuesMeta,
+ tasks.ExtractIssuesMeta,
tasks.CollectHotspotsMeta,
tasks.ExtractHotspotsMeta,
}
diff --git a/backend/plugins/sonarqube/models/migrationscripts/20230111_add_init_tables.go b/backend/plugins/sonarqube/models/migrationscripts/20230111_add_init_tables.go
index 13078555c..62a792433 100644
--- a/backend/plugins/sonarqube/models/migrationscripts/20230111_add_init_tables.go
+++ b/backend/plugins/sonarqube/models/migrationscripts/20230111_add_init_tables.go
@@ -32,11 +32,12 @@ func (*addInitTables) Up(basicRes context.BasicRes) errors.Error {
&archived.SonarqubeConnection{},
&archived.SonarqubeProject{},
&archived.SonarqubeHotspot{},
+ &archived.SonarqubeIssue{},
)
}
func (*addInitTables) Version() uint64 {
- return 20230203000011
+ return 20230206000011
}
func (*addInitTables) Name() string {
diff --git a/backend/plugins/sonarqube/models/migrationscripts/archived/sonarqube_issue.go b/backend/plugins/sonarqube/models/migrationscripts/archived/sonarqube_issue.go
new file mode 100644
index 000000000..21b2a1942
--- /dev/null
+++ b/backend/plugins/sonarqube/models/migrationscripts/archived/sonarqube_issue.go
@@ -0,0 +1,54 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package archived
+
+import (
+ "github.com/apache/incubator-devlake/core/models/migrationscripts/archived"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+)
+
+type SonarqubeIssue struct {
+ archived.NoPKModel
+ ConnectionId uint64 `gorm:"primaryKey"`
+ Key string `json:"key" gorm:"primaryKey"`
+ BatchId string `json:"batchId" gorm:"primaryKey"`
+ Rule string `json:"rule"`
+ Severity string `json:"severity"`
+ Component string `json:"component"`
+ Project string `json:"project"`
+ Line int `json:"line"`
+ Status string `json:"status"`
+ Message string `json:"message"`
+ Debt string `json:"debt"`
+ Effort string `json:"effort"`
+ Author string `json:"author"`
+ Hash string `json:"hash"`
+ Tags string `json:"tags"`
+ Type string `json:"type"`
+ Scope string `json:"scope"`
+ StartLine int `json:"startLine"`
+ EndLine int `json:"endLine"`
+ StartOffset int `json:"startOffset"`
+ EndOffset int `json:"endOffset"`
+ CreationDate *api.Iso8601Time `json:"creationDate"`
+ UpdateDate *api.Iso8601Time `json:"updateDate"`
+}
+
+func (SonarqubeIssue) TableName() string {
+ return "_tool_sonarqube_issues"
+}
diff --git a/backend/plugins/sonarqube/models/sonarqube_issue.go b/backend/plugins/sonarqube/models/sonarqube_issue.go
new file mode 100644
index 000000000..c33058ba0
--- /dev/null
+++ b/backend/plugins/sonarqube/models/sonarqube_issue.go
@@ -0,0 +1,54 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package models
+
+import (
+ "github.com/apache/incubator-devlake/core/models/common"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+)
+
+type SonarqubeIssue struct {
+ common.NoPKModel
+ ConnectionId uint64 `gorm:"primaryKey"`
+ Key string `json:"key" gorm:"primaryKey"`
+ BatchId string `json:"batchId" gorm:"primaryKey"`
+ Rule string `json:"rule"`
+ Severity string `json:"severity"`
+ Component string `json:"component"`
+ Project string `json:"project"`
+ Line int `json:"line"`
+ Status string `json:"status"`
+ Message string `json:"message"`
+ Debt string `json:"debt"`
+ Effort string `json:"effort"`
+ Author string `json:"author"`
+ Hash string `json:"hash"`
+ Tags string `json:"tags"`
+ Type string `json:"type"`
+ Scope string `json:"scope"`
+ StartLine int `json:"startLine"`
+ EndLine int `json:"endLine"`
+ StartOffset int `json:"startOffset"`
+ EndOffset int `json:"endOffset"`
+ CreationDate *api.Iso8601Time `json:"creationDate"`
+ UpdateDate *api.Iso8601Time `json:"updateDate"`
+}
+
+func (SonarqubeIssue) TableName() string {
+ return "_tool_sonarqube_issues"
+}
diff --git a/backend/plugins/sonarqube/tasks/issues_collector.go b/backend/plugins/sonarqube/tasks/issues_collector.go
new file mode 100644
index 000000000..bd1b85755
--- /dev/null
+++ b/backend/plugins/sonarqube/tasks/issues_collector.go
@@ -0,0 +1,80 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "encoding/json"
+ "fmt"
+ "net/http"
+ "net/url"
+
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
+ helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+)
+
+const RAW_ISSUES_TABLE = "sonarqube_issues"
+
+var _ plugin.SubTaskEntryPoint = CollectIssues
+
+func CollectIssues(taskCtx plugin.SubTaskContext) (err errors.Error) {
+ rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_ISSUES_TABLE)
+ logger := taskCtx.GetLogger()
+ logger.Info("collect issues")
+
+ collectorWithState, err := helper.NewApiCollectorWithState(*rawDataSubTaskArgs, data.CreatedDateAfter)
+ if err != nil {
+ return err
+ }
+
+ err = collectorWithState.InitCollector(helper.ApiCollectorArgs{
+ ApiClient: data.ApiClient,
+ PageSize: 100,
+ UrlTemplate: "issues/search",
+ Query: func(reqData *helper.RequestData) (url.Values, errors.Error) {
+ query := url.Values{}
+ query.Set("componentKeys", fmt.Sprintf("%v", data.Options.ProjectKey))
+ query.Set("p", fmt.Sprintf("%v", reqData.Pager.Page))
+ query.Set("ps", fmt.Sprintf("%v", reqData.Pager.Size))
+ return query, nil
+ },
+ GetTotalPages: GetTotalPagesFromResponse,
+ ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) {
+ var resData struct {
+ Data []json.RawMessage `json:"issues"`
+ }
+ err := helper.UnmarshalResponse(res, &resData)
+ if err != nil {
+ return nil, err
+ }
+ return resData.Data, nil
+ },
+ })
+ if err != nil {
+ return err
+ }
+
+ return collectorWithState.Execute()
+}
+
+var CollectIssuesMeta = plugin.SubTaskMeta{
+ Name: "CollectIssues",
+ EntryPoint: CollectIssues,
+ EnabledByDefault: true,
+ Description: "Collect issues data from Sonarqube api",
+}
diff --git a/backend/plugins/sonarqube/tasks/issues_extractor.go b/backend/plugins/sonarqube/tasks/issues_extractor.go
new file mode 100644
index 000000000..78265e658
--- /dev/null
+++ b/backend/plugins/sonarqube/tasks/issues_extractor.go
@@ -0,0 +1,115 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "encoding/json"
+ "strings"
+
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
+ helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "github.com/apache/incubator-devlake/plugins/sonarqube/models"
+)
+
+var _ plugin.SubTaskEntryPoint = ExtractIssues
+
+func ExtractIssues(taskCtx plugin.SubTaskContext) errors.Error {
+ rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_ISSUES_TABLE)
+
+ extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{
+ RawDataSubTaskArgs: *rawDataSubTaskArgs,
+ Extract: func(resData *helper.RawData) ([]interface{}, errors.Error) {
+ body := &IssuesResponse{}
+ err := errors.Convert(json.Unmarshal(resData.Data, body))
+ if err != nil {
+ return nil, err
+ }
+
+ sonarqubeIssue := &models.SonarqubeIssue{
+ ConnectionId: data.Options.ConnectionId,
+ Key: body.Key,
+ // BatchId string `json:"batchId" gorm:"primaryKey"`
+ Rule: body.Rule,
+ Severity: body.Severity,
+ Component: body.Component,
+ Project: body.Project,
+ Line: body.Line,
+ Status: body.Status,
+ Message: body.Message,
+ Debt: body.Debt,
+ Effort: body.Effort,
+ Author: body.Author,
+ Hash: body.Hash,
+ Type: body.Type,
+ Scope: body.Scope,
+ StartLine: body.TextRange.StartLine,
+ EndLine: body.TextRange.EndLine,
+ StartOffset: body.TextRange.StartOffset,
+ EndOffset: body.TextRange.EndOffset,
+ CreationDate: body.CreationDate,
+ UpdateDate: body.UpdateDate,
+ }
+ if len(body.Tags) > 0 {
+ sonarqubeIssue.Tags = strings.Join(body.Tags, ",")
+ }
+
+ return []interface{}{sonarqubeIssue}, nil
+ },
+ })
+ if err != nil {
+ return err
+ }
+
+ return extractor.Execute()
+}
+
+var ExtractIssuesMeta = plugin.SubTaskMeta{
+ Name: "ExtractIssues",
+ EntryPoint: ExtractIssues,
+ EnabledByDefault: true,
+ Description: "Extract raw data into tool layer table sonarqube_issues",
+}
+
+type IssuesResponse struct {
+ Key string `json:"key"`
+ Rule string `json:"rule"`
+ Severity string `json:"severity"`
+ Component string `json:"component"`
+ Project string `json:"project"`
+ Line int `json:"line"`
+ Hash string `json:"hash"`
+ TextRange struct {
+ StartLine int `json:"startLine"`
+ EndLine int `json:"endLine"`
+ StartOffset int `json:"startOffset"`
+ EndOffset int `json:"endOffset"`
+ } `json:"textRange"`
+ Flows []interface{} `json:"flows"`
+ Status string `json:"status"`
+ Message string `json:"message"`
+ Effort string `json:"effort"`
+ Debt string `json:"debt"`
+ Author string `json:"author"`
+ Tags []string `json:"tags"`
+ CreationDate *helper.Iso8601Time `json:"creationDate"`
+ UpdateDate *helper.Iso8601Time `json:"updateDate"`
+ Type string `json:"type"`
+ Scope string `json:"scope"`
+ QuickFixAvailable bool `json:"quickFixAvailable"`
+}