You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by ab...@apache.org on 2023/02/21 12:59:07 UTC
[incubator-devlake] branch main updated: feat(sonarqube): add bp v100 support (#4469)
This is an automated email from the ASF dual-hosted git repository.
abeizn pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 5b7e0da90 feat(sonarqube): add bp v100 support (#4469)
5b7e0da90 is described below
commit 5b7e0da900936bfcb1907d4a30dc9cc58e8ce8ae
Author: Warren Chen <yi...@merico.dev>
AuthorDate: Tue Feb 21 20:59:02 2023 +0800
feat(sonarqube): add bp v100 support (#4469)
---
backend/plugins/sonarqube/api/blueprint_v200.go | 30 +++++++
backend/plugins/sonarqube/e2e/project_test.go | 25 +-----
.../e2e/raw_tables/_raw_sonarqube_projects.csv | 12 ---
.../_tool_sonarqube_projects.csv | 0
backend/plugins/sonarqube/impl/impl.go | 26 ++++++-
.../sonarqube/tasks/filemetrics_convertor.go | 2 +
.../plugins/sonarqube/tasks/projects_collector.go | 82 -------------------
.../plugins/sonarqube/tasks/projects_extractor.go | 91 ----------------------
backend/plugins/sonarqube/tasks/shared.go | 23 ++++++
9 files changed, 83 insertions(+), 208 deletions(-)
diff --git a/backend/plugins/sonarqube/api/blueprint_v200.go b/backend/plugins/sonarqube/api/blueprint_v200.go
index d2920f8bc..611cec815 100644
--- a/backend/plugins/sonarqube/api/blueprint_v200.go
+++ b/backend/plugins/sonarqube/api/blueprint_v200.go
@@ -27,7 +27,11 @@ import (
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/core/utils"
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ aha "github.com/apache/incubator-devlake/helpers/pluginhelper/api/apihelperabstract"
"github.com/apache/incubator-devlake/plugins/sonarqube/models"
+ "github.com/apache/incubator-devlake/plugins/sonarqube/tasks"
+ "net/http"
+ "net/url"
"time"
)
@@ -104,3 +108,29 @@ func makeScopesV200(bpScopes []*plugin.BlueprintScopeV200, connectionId uint64)
}
return scopes, nil
}
+
+func GetApiProject(
+ projectKey string,
+ apiClient aha.ApiClientAbstract,
+) (*tasks.SonarqubeApiProject, errors.Error) {
+ var resData struct {
+ Data []tasks.SonarqubeApiProject `json:"components"`
+ }
+ query := url.Values{}
+ query.Set("q", projectKey)
+ res, err := apiClient.Get("projects/search", query, nil)
+ if err != nil {
+ return nil, err
+ }
+ if res.StatusCode != http.StatusOK {
+ return nil, errors.HttpStatus(res.StatusCode).New(fmt.Sprintf("unexpected status code when requesting project detail from %s", res.Request.URL.String()))
+ }
+ err = helper.UnmarshalResponse(res, &resData)
+ if err != nil {
+ return nil, err
+ }
+ if len(resData.Data) > 0 {
+ return &resData.Data[0], nil
+ }
+ return nil, errors.BadInput.New(fmt.Sprintf("Cannot find project: %s", projectKey))
+}
diff --git a/backend/plugins/sonarqube/e2e/project_test.go b/backend/plugins/sonarqube/e2e/project_test.go
index aeea1adec..930b9a835 100644
--- a/backend/plugins/sonarqube/e2e/project_test.go
+++ b/backend/plugins/sonarqube/e2e/project_test.go
@@ -33,36 +33,19 @@ func TestSonarqubeProjectDataFlow(t *testing.T) {
var sonarqube impl.Sonarqube
dataflowTester := e2ehelper.NewDataFlowTester(t, "sonarqube", sonarqube)
- taskData := &tasks.SonarqubeTaskData{
- Options: &tasks.SonarqubeOptions{
- ConnectionId: 1,
- ProjectKey: "fa2cf9cd-c448-4fc3-99a5-1c893f15d84c",
- },
- }
-
// import raw data table
- dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_sonarqube_projects.csv",
- "_raw_sonarqube_projects")
+ dataflowTester.ImportCsvIntoTabler("./raw_tables/_tool_sonarqube_projects.csv",
+ &models.SonarqubeProject{})
- // verify extraction
- dataflowTester.FlushTabler(&models.SonarqubeProject{})
- dataflowTester.Subtask(tasks.ExtractProjectsMeta, taskData)
-
- taskData2 := &tasks.SonarqubeTaskData{
+ taskData := &tasks.SonarqubeTaskData{
Options: &tasks.SonarqubeOptions{
ConnectionId: 2,
ProjectKey: "e2c6d5e9-a321-4e8c-b322-03d9599ef962",
},
}
- dataflowTester.Subtask(tasks.ExtractProjectsMeta, taskData2)
- dataflowTester.VerifyTableWithOptions(&models.SonarqubeProject{}, e2ehelper.TableOptions{
- CSVRelPath: "./snapshot_tables/_tool_sonarqube_projects.csv",
- IgnoreTypes: []interface{}{common.NoPKModel{}},
- })
-
dataflowTester.FlushTabler(&codequality.CqProject{})
- dataflowTester.Subtask(tasks.ConvertProjectsMeta, taskData2)
+ dataflowTester.Subtask(tasks.ConvertProjectsMeta, taskData)
dataflowTester.VerifyTableWithOptions(&codequality.CqProject{}, e2ehelper.TableOptions{
CSVRelPath: "./snapshot_tables/projects.csv",
IgnoreTypes: []interface{}{common.NoPKModel{}},
diff --git a/backend/plugins/sonarqube/e2e/raw_tables/_raw_sonarqube_projects.csv b/backend/plugins/sonarqube/e2e/raw_tables/_raw_sonarqube_projects.csv
deleted file mode 100644
index a4da44afd..000000000
--- a/backend/plugins/sonarqube/e2e/raw_tables/_raw_sonarqube_projects.csv
+++ /dev/null
@@ -1,12 +0,0 @@
-"id","params","data","url","input","created_at"
-642,"{""connectionId"":1,""ProjectKey"":""fa2cf9cd-c448-4fc3-99a5-1c893f15d84c""}","{""key"":""fa2cf9cd-c448-4fc3-99a5-1c893f15d84c"",""name"":""activeadmin"",""qualifier"":""TRK"",""visibility"":""public"",""lastAnalysisDate"":""2022-12-23T13:52:39+0000"",""revision"":""51689e48ec3faa51127b8462b2219e5e0c09ad8a""}","http://35.89.154.221:30008/api/projects/search?p=1&ps=100","null","2023-02-10 02:46:14.044"
-643,"{""connectionId"":1,""ProjectKey"":""""}","{""key"":""a67b8166-5768-46a5-8830-6d86d797419a"",""name"":""actix-web"",""qualifier"":""TRK"",""visibility"":""public"",""lastAnalysisDate"":""2023-01-04T22:36:46+0000"",""revision"":""08c2cdf6416287d63387584213ea3dd41b007ef6""}","http://35.89.154.221:30008/api/projects/search?p=1&ps=100","null","2023-02-10 02:46:14.044"
-644,"{""connectionId"":2,""ProjectKey"":""""}","{""key"":""a7c268ed-525b-438d-8729-18e83229e66c"",""name"":""Aerial"",""qualifier"":""TRK"",""visibility"":""public"",""lastAnalysisDate"":""2022-12-22T18:26:29+0000"",""revision"":""405bc2494af895f8b9ccbf12f3d0c9530a154011""}","http://35.89.154.221:30008/api/projects/search?p=1&ps=100","null","2023-02-10 02:46:14.044"
-645,"{""connectionId"":2,""ProjectKey"":""""}","{""key"":""f5a50c63-2e8f-4107-9014-853f6f467757"",""name"":""aerosolve"",""qualifier"":""TRK"",""visibility"":""public"",""lastAnalysisDate"":""2023-01-02T06:58:08+0000"",""revision"":""442e76ebb7cbc7f60b04fbfac30dbf862aaffc67""}","http://35.89.154.221:30008/api/projects/search?p=1&ps=100","null","2023-02-10 02:46:14.044"
-646,"{""connectionId"":2,""ProjectKey"":""""}","{""key"":""f960efd5-84eb-44c9-888c-996457ed76d7"",""name"":""AFNetworking"",""qualifier"":""TRK"",""visibility"":""public"",""lastAnalysisDate"":""2022-12-22T12:28:19+0000"",""revision"":""4eaec5b586ddd897ebeda896e332a62a9fdab818""}","http://35.89.154.221:30008/api/projects/search?p=1&ps=100","null","2023-02-10 02:46:14.044"
-647,"{""connectionId"":2,""ProjectKey"":""""}","{""key"":""8b075027-0a3f-44ff-89e1-751d173b4f9d"",""name"":""akka"",""qualifier"":""TRK"",""visibility"":""public"",""lastAnalysisDate"":""2022-12-20T16:27:23+0000"",""revision"":""b7b79a2da1231520fe66532b7af53179d1714952""}","http://35.89.154.221:30008/api/projects/search?p=1&ps=100","null","2023-02-10 02:46:14.044"
-648,"{""connectionId"":3,""ProjectKey"":""""}","{""key"":""eb4e9de1-85f1-4568-9352-94a264ba5bef"",""name"":""akka"",""qualifier"":""TRK"",""visibility"":""public"",""lastAnalysisDate"":""2023-01-01T03:54:58+0000"",""revision"":""50372b37c5cf9ef467c4637f4e526a5fe95fc9c1""}","http://35.89.154.221:30008/api/projects/search?p=1&ps=100","null","2023-02-10 02:46:14.044"
-649,"{""connectionId"":3,""ProjectKey"":""""}","{""key"":""4f3b6f44-dbcd-4507-9cc8-c3e460f1c69e"",""name"":""Alamofire"",""qualifier"":""TRK"",""visibility"":""public"",""lastAnalysisDate"":""2023-01-05T01:14:55+0000"",""revision"":""ce7f662709a152e840b130a158b9afef3402e146""}","http://35.89.154.221:30008/api/projects/search?p=1&ps=100","null","2023-02-10 02:46:14.044"
-650,"{""connectionId"":3,""ProjectKey"":""f1f70ac0-e607-42b8-8fda-46b28f7d3430""}","{""key"":""f1f70ac0-e607-42b8-8fda-46b28f7d3430"",""name"":""algorithms"",""qualifier"":""TRK"",""visibility"":""public"",""lastAnalysisDate"":""2022-12-26T10:57:53+0000"",""revision"":""52b9408c98eaa8458083b27410c0c364406d6e24""}","http://35.89.154.221:30008/api/projects/search?p=1&ps=100","null","2023-02-10 02:46:14.044"
-651,"{""connectionId"":2,""ProjectKey"":""e2c6d5e9-a321-4e8c-b322-03d9599ef962""}","{""key"":""e2c6d5e9-a321-4e8c-b322-03d9599ef962"",""name"":""Android-Universal-Image-Loader"",""qualifier"":""TRK"",""visibility"":""public"",""lastAnalysisDate"":""2022-12-24T18:42:09+0000"",""revision"":""458df4da2e23ba9ad76c79241a948cdfcccf72ae""}","http://35.89.154.221:30008/api/projects/search?p=1&ps=100","null","2023-02-10 02:46:14.044"
-652,"{""connectionId"":1,""ProjectKey"":""""}","{""key"":""0892ae2f-6a07-4613-a1dc-bf29158afc52"",""name"":""AndroidUtilCode"",""qualifier"":""TRK"",""visibility"":""public"",""lastAnalysisDate"":""2023-01-05T01:25:34+0000"",""revision"":""51c8124045ec5b7c6d342b94a4020ed2f7fda496""}","http://35.89.154.221:30008/api/projects/search?p=1&ps=100","null","2023-02-10 02:46:14.044"
diff --git a/backend/plugins/sonarqube/e2e/snapshot_tables/_tool_sonarqube_projects.csv b/backend/plugins/sonarqube/e2e/raw_tables/_tool_sonarqube_projects.csv
similarity index 100%
rename from backend/plugins/sonarqube/e2e/snapshot_tables/_tool_sonarqube_projects.csv
rename to backend/plugins/sonarqube/e2e/raw_tables/_tool_sonarqube_projects.csv
diff --git a/backend/plugins/sonarqube/impl/impl.go b/backend/plugins/sonarqube/impl/impl.go
index 653fcb420..e4655274d 100644
--- a/backend/plugins/sonarqube/impl/impl.go
+++ b/backend/plugins/sonarqube/impl/impl.go
@@ -82,8 +82,6 @@ func (p Sonarqube) GetTablesInfo() []dal.Tabler {
func (p Sonarqube) SubTaskMetas() []plugin.SubTaskMeta {
return []plugin.SubTaskMeta{
- tasks.CollectProjectsMeta,
- tasks.ExtractProjectsMeta,
tasks.CollectIssuesMeta,
tasks.ExtractIssuesMeta,
tasks.CollectHotspotsMeta,
@@ -125,6 +123,30 @@ func (p Sonarqube) PrepareTaskData(taskCtx plugin.TaskContext, options map[strin
Options: op,
ApiClient: apiClient,
}
+ if op.ProjectKey != "" {
+ var scope *models.SonarqubeProject
+ var apiProject *tasks.SonarqubeApiProject
+ // support v100 & advance mode
+ // If we still cannot find the record in db, we have to request from remote server and save it to db
+ db := taskCtx.GetDal()
+ err = db.First(&scope, dal.Where("connection_id = ? AND project_key = ?", op.ConnectionId, op.ProjectKey))
+ if err != nil && db.IsErrorNotFound(err) {
+ apiProject, err = api.GetApiProject(op.ProjectKey, apiClient)
+ if err != nil {
+ return nil, err
+ }
+ logger.Debug(fmt.Sprintf("Current project: %s", apiProject.ProjectKey))
+ scope = tasks.ConvertProject(apiProject)
+ scope.ConnectionId = op.ConnectionId
+ err = taskCtx.GetDal().CreateIfNotExist(&scope)
+ if err != nil {
+ return nil, err
+ }
+ }
+ if err != nil {
+ return nil, errors.Default.Wrap(err, fmt.Sprintf("fail to find project: %s", op.ProjectKey))
+ }
+ }
var createdDateAfter time.Time
if op.CreatedDateAfter != "" {
createdDateAfter, err = errors.Convert01(time.Parse(time.RFC3339, op.CreatedDateAfter))
diff --git a/backend/plugins/sonarqube/tasks/filemetrics_convertor.go b/backend/plugins/sonarqube/tasks/filemetrics_convertor.go
index 4440d3208..2cc7d911d 100644
--- a/backend/plugins/sonarqube/tasks/filemetrics_convertor.go
+++ b/backend/plugins/sonarqube/tasks/filemetrics_convertor.go
@@ -29,6 +29,8 @@ import (
"reflect"
)
+const RAW_PROJECTS_TABLE = "sonarqube_projects"
+
var ConvertFileMetricsMeta = plugin.SubTaskMeta{
Name: "convertFileMetrics",
EntryPoint: ConvertFileMetrics,
diff --git a/backend/plugins/sonarqube/tasks/projects_collector.go b/backend/plugins/sonarqube/tasks/projects_collector.go
deleted file mode 100644
index ba57b81bf..000000000
--- a/backend/plugins/sonarqube/tasks/projects_collector.go
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package tasks
-
-import (
- "encoding/json"
- "fmt"
- "github.com/apache/incubator-devlake/core/errors"
- "github.com/apache/incubator-devlake/core/plugin"
- helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
- "net/http"
- "net/url"
-)
-
-const RAW_PROJECTS_TABLE = "sonarqube_projects"
-
-var _ plugin.SubTaskEntryPoint = CollectProjects
-
-func CollectProjects(taskCtx plugin.SubTaskContext) errors.Error {
- rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_PROJECTS_TABLE)
- logger := taskCtx.GetLogger()
- logger.Info("collect projects")
-
- collectorWithState, err := helper.NewApiCollectorWithState(*rawDataSubTaskArgs, data.CreatedDateAfter)
- if err != nil {
- return err
- }
- err = collectorWithState.InitCollector(helper.ApiCollectorArgs{
- ApiClient: data.ApiClient,
- PageSize: 100,
- UrlTemplate: "projects/search",
- Query: func(reqData *helper.RequestData) (url.Values, errors.Error) {
- query := url.Values{}
- if data.CreatedDateAfter != nil {
- query.Set("analyzedBefore",
- data.CreatedDateAfter.Format("2006-01-02"))
- }
- if data.Options.ProjectKey != "" {
- query.Set("q", data.Options.ProjectKey)
- }
- query.Set("p", fmt.Sprintf("%v", reqData.Pager.Page))
- query.Set("ps", fmt.Sprintf("%v", reqData.Pager.Size))
- return query, nil
- },
- GetTotalPages: GetTotalPagesFromResponse,
- ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) {
- var resData struct {
- Data []json.RawMessage `json:"components"`
- }
- err = helper.UnmarshalResponse(res, &resData)
- return resData.Data, err
- },
- })
- if err != nil {
- return err
- }
-
- return collectorWithState.Execute()
-}
-
-var CollectProjectsMeta = plugin.SubTaskMeta{
- Name: "CollectProjects",
- EntryPoint: CollectProjects,
- EnabledByDefault: true,
- Description: "Collect Projects data from Sonarqube api",
- DomainTypes: []string{plugin.DOMAIN_TYPE_CODE_QUALITY},
-}
diff --git a/backend/plugins/sonarqube/tasks/projects_extractor.go b/backend/plugins/sonarqube/tasks/projects_extractor.go
deleted file mode 100644
index aaf28f13d..000000000
--- a/backend/plugins/sonarqube/tasks/projects_extractor.go
+++ /dev/null
@@ -1,91 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package tasks
-
-import (
- "encoding/json"
- "github.com/apache/incubator-devlake/core/errors"
- "github.com/apache/incubator-devlake/core/plugin"
- helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
- "github.com/apache/incubator-devlake/plugins/sonarqube/models"
-)
-
-var _ plugin.SubTaskEntryPoint = ExtractProjects
-
-func ExtractProjects(taskCtx plugin.SubTaskContext) errors.Error {
- // As we need to assign data.LastAnalysisDate, we can not use CreateRawDataSubTaskArgs
- data := taskCtx.GetData().(*SonarqubeTaskData)
- var params = SonarqubeApiParams{
- ConnectionId: data.Options.ConnectionId,
- ProjectKey: data.Options.ProjectKey,
- }
- rawDataSubTaskArgs := &helper.RawDataSubTaskArgs{
- Ctx: taskCtx,
- Params: params,
- Table: RAW_PROJECTS_TABLE,
- }
- extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{
- RawDataSubTaskArgs: *rawDataSubTaskArgs,
- Extract: func(resData *helper.RawData) ([]interface{}, errors.Error) {
- res := SonarqubeApiProject{}
- err := errors.Convert(json.Unmarshal(resData.Data, &res))
- if err != nil {
- return nil, err
- }
- body := ConvertProject(&res)
- body.ConnectionId = data.Options.ConnectionId
- data.LastAnalysisDate = body.LastAnalysisDate.ToNullableTime()
- return []interface{}{body}, nil
- },
- })
- if err != nil {
- return err
- }
-
- return extractor.Execute()
-}
-
-var ExtractProjectsMeta = plugin.SubTaskMeta{
- Name: "ExtractProjects",
- EntryPoint: ExtractProjects,
- EnabledByDefault: true,
- Description: "Extract raw data into tool layer table sonarqube_projects",
- DomainTypes: []string{plugin.DOMAIN_TYPE_CODE_QUALITY},
-}
-
-type SonarqubeApiProject struct {
- ProjectKey string `json:"key"`
- Name string `json:"name"`
- Qualifier string `json:"qualifier"`
- Visibility string `json:"visibility"`
- LastAnalysisDate *helper.Iso8601Time `json:"lastAnalysisDate"`
- Revision string `json:"revision"`
-}
-
-// Convert the API response to our DB model instance
-func ConvertProject(sonarqubeApiProject *SonarqubeApiProject) *models.SonarqubeProject {
- sonarqubeProject := &models.SonarqubeProject{
- ProjectKey: sonarqubeApiProject.ProjectKey,
- Name: sonarqubeApiProject.Name,
- Qualifier: sonarqubeApiProject.Qualifier,
- Visibility: sonarqubeApiProject.Visibility,
- LastAnalysisDate: sonarqubeApiProject.LastAnalysisDate,
- Revision: sonarqubeApiProject.Revision,
- }
- return sonarqubeProject
-}
diff --git a/backend/plugins/sonarqube/tasks/shared.go b/backend/plugins/sonarqube/tasks/shared.go
index f0c4746cf..024cffcb7 100644
--- a/backend/plugins/sonarqube/tasks/shared.go
+++ b/backend/plugins/sonarqube/tasks/shared.go
@@ -21,6 +21,7 @@ import (
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "github.com/apache/incubator-devlake/plugins/sonarqube/models"
"net/http"
)
@@ -62,3 +63,25 @@ type Paging struct {
PageSize int `json:"pageSize"`
Total int `json:"total"`
}
+
+type SonarqubeApiProject struct {
+ ProjectKey string `json:"key"`
+ Name string `json:"name"`
+ Qualifier string `json:"qualifier"`
+ Visibility string `json:"visibility"`
+ LastAnalysisDate *api.Iso8601Time `json:"lastAnalysisDate"`
+ Revision string `json:"revision"`
+}
+
+// Convert the API response to our DB model instance
+func ConvertProject(sonarqubeApiProject *SonarqubeApiProject) *models.SonarqubeProject {
+ sonarqubeProject := &models.SonarqubeProject{
+ ProjectKey: sonarqubeApiProject.ProjectKey,
+ Name: sonarqubeApiProject.Name,
+ Qualifier: sonarqubeApiProject.Qualifier,
+ Visibility: sonarqubeApiProject.Visibility,
+ LastAnalysisDate: sonarqubeApiProject.LastAnalysisDate,
+ Revision: sonarqubeApiProject.Revision,
+ }
+ return sonarqubeProject
+}