You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by wa...@apache.org on 2023/02/07 09:35:21 UTC
[incubator-devlake] branch main updated: feat: sonarqube user collector and extractor (#4343)
This is an automated email from the ASF dual-hosted git repository.
warren pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new f82cf1b0a feat: sonarqube user collector and extractor (#4343)
f82cf1b0a is described below
commit f82cf1b0a0e5679e6e74f93fce85dab6618276af
Author: abeizn <zi...@merico.dev>
AuthorDate: Tue Feb 7 17:35:16 2023 +0800
feat: sonarqube user collector and extractor (#4343)
* feat: sonarqube user collector and extractor
* feat: sonarqube account collector and extractor
---
backend/plugins/sonarqube/impl/impl.go | 2 +
.../migrationscripts/20230111_add_init_tables.go | 3 +-
.../sonarqube_account.go} | 36 ++++-------
...111_add_init_tables.go => sonarqube_account.go} | 36 ++++-------
.../plugins/sonarqube/tasks/accounts_collector.go | 75 ++++++++++++++++++++++
.../plugins/sonarqube/tasks/accounts_extractor.go | 57 ++++++++++++++++
6 files changed, 162 insertions(+), 47 deletions(-)
diff --git a/backend/plugins/sonarqube/impl/impl.go b/backend/plugins/sonarqube/impl/impl.go
index aba0c7785..d26867f3b 100644
--- a/backend/plugins/sonarqube/impl/impl.go
+++ b/backend/plugins/sonarqube/impl/impl.go
@@ -61,6 +61,8 @@ func (p Sonarqube) SubTaskMetas() []plugin.SubTaskMeta {
tasks.ExtractHotspotsMeta,
tasks.CollectFilemetricsMeta,
tasks.ExtractFilemetricsMeta,
+ tasks.CollectAccountsMeta,
+ tasks.ExtractAccountsMeta,
}
}
diff --git a/backend/plugins/sonarqube/models/migrationscripts/20230111_add_init_tables.go b/backend/plugins/sonarqube/models/migrationscripts/20230111_add_init_tables.go
index 1aaf21f04..b917af525 100644
--- a/backend/plugins/sonarqube/models/migrationscripts/20230111_add_init_tables.go
+++ b/backend/plugins/sonarqube/models/migrationscripts/20230111_add_init_tables.go
@@ -35,11 +35,12 @@ func (*addInitTables) Up(basicRes context.BasicRes) errors.Error {
&archived.SonarqubeIssue{},
&archived.SonarqubeFileMetrics{},
&archived.SonarqubeIssueCodeBlock{},
+ &archived.SonarqubeAccount{},
)
}
func (*addInitTables) Version() uint64 {
- return 20230206200021
+ return 20230207220015
}
func (*addInitTables) Name() string {
diff --git a/backend/plugins/sonarqube/models/migrationscripts/20230111_add_init_tables.go b/backend/plugins/sonarqube/models/migrationscripts/archived/sonarqube_account.go
similarity index 50%
copy from backend/plugins/sonarqube/models/migrationscripts/20230111_add_init_tables.go
copy to backend/plugins/sonarqube/models/migrationscripts/archived/sonarqube_account.go
index 1aaf21f04..046644fbe 100644
--- a/backend/plugins/sonarqube/models/migrationscripts/20230111_add_init_tables.go
+++ b/backend/plugins/sonarqube/models/migrationscripts/archived/sonarqube_account.go
@@ -15,33 +15,23 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
-package migrationscripts
+package archived
import (
- "github.com/apache/incubator-devlake/core/context"
- "github.com/apache/incubator-devlake/core/errors"
- "github.com/apache/incubator-devlake/helpers/migrationhelper"
- "github.com/apache/incubator-devlake/plugins/sonarqube/models/migrationscripts/archived"
+ "github.com/apache/incubator-devlake/core/models/migrationscripts/archived"
)
-type addInitTables struct{}
-
-func (*addInitTables) Up(basicRes context.BasicRes) errors.Error {
- return migrationhelper.AutoMigrateTables(
- basicRes,
- &archived.SonarqubeConnection{},
- &archived.SonarqubeProject{},
- &archived.SonarqubeHotspot{},
- &archived.SonarqubeIssue{},
- &archived.SonarqubeFileMetrics{},
- &archived.SonarqubeIssueCodeBlock{},
- )
-}
-
-func (*addInitTables) Version() uint64 {
- return 20230206200021
+type SonarqubeAccount struct {
+ archived.NoPKModel
+ ConnectionId uint64 `gorm:"primaryKey"`
+ BatchId string `json:"batchId" gorm:"type:varchar(100)"` // from collection time
+ Login string `json:"login" gorm:"primaryKey"`
+ Name string `json:"name"`
+ Email string `json:"email"`
+ Active bool `json:"active"`
+ Local bool `json:"local"`
}
-func (*addInitTables) Name() string {
- return "sonarqube init schemas"
+func (SonarqubeAccount) TableName() string {
+ return "_tool_sonarqube_accounts"
}
diff --git a/backend/plugins/sonarqube/models/migrationscripts/20230111_add_init_tables.go b/backend/plugins/sonarqube/models/sonarqube_account.go
similarity index 50%
copy from backend/plugins/sonarqube/models/migrationscripts/20230111_add_init_tables.go
copy to backend/plugins/sonarqube/models/sonarqube_account.go
index 1aaf21f04..e728fd2e2 100644
--- a/backend/plugins/sonarqube/models/migrationscripts/20230111_add_init_tables.go
+++ b/backend/plugins/sonarqube/models/sonarqube_account.go
@@ -15,33 +15,23 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
-package migrationscripts
+package models
import (
- "github.com/apache/incubator-devlake/core/context"
- "github.com/apache/incubator-devlake/core/errors"
- "github.com/apache/incubator-devlake/helpers/migrationhelper"
- "github.com/apache/incubator-devlake/plugins/sonarqube/models/migrationscripts/archived"
+ "github.com/apache/incubator-devlake/core/models/common"
)
-type addInitTables struct{}
-
-func (*addInitTables) Up(basicRes context.BasicRes) errors.Error {
- return migrationhelper.AutoMigrateTables(
- basicRes,
- &archived.SonarqubeConnection{},
- &archived.SonarqubeProject{},
- &archived.SonarqubeHotspot{},
- &archived.SonarqubeIssue{},
- &archived.SonarqubeFileMetrics{},
- &archived.SonarqubeIssueCodeBlock{},
- )
-}
-
-func (*addInitTables) Version() uint64 {
- return 20230206200021
+type SonarqubeAccount struct {
+ common.NoPKModel
+ ConnectionId uint64 `gorm:"primaryKey"`
+ BatchId string `json:"batchId" gorm:"type:varchar(100)"` // from collection time
+ Login string `json:"login" gorm:"primaryKey"`
+ Name string `json:"name"`
+ Email string `json:"email"`
+ Active bool `json:"active"`
+ Local bool `json:"local"`
}
-func (*addInitTables) Name() string {
- return "sonarqube init schemas"
+func (SonarqubeAccount) TableName() string {
+ return "_tool_sonarqube_accounts"
}
diff --git a/backend/plugins/sonarqube/tasks/accounts_collector.go b/backend/plugins/sonarqube/tasks/accounts_collector.go
new file mode 100644
index 000000000..24aeff1a4
--- /dev/null
+++ b/backend/plugins/sonarqube/tasks/accounts_collector.go
@@ -0,0 +1,75 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "encoding/json"
+ "fmt"
+ "net/http"
+ "net/url"
+
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
+ helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+)
+
+const RAW_ACCOUNTS_TABLE = "sonarqube_accounts"
+
+var _ plugin.SubTaskEntryPoint = CollectAccounts
+
+func CollectAccounts(taskCtx plugin.SubTaskContext) errors.Error {
+ rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_ACCOUNTS_TABLE)
+ logger := taskCtx.GetLogger()
+ logger.Info("collect accounts")
+
+ collectorWithState, err := helper.NewApiCollectorWithState(*rawDataSubTaskArgs, data.CreatedDateAfter)
+ if err != nil {
+ return err
+ }
+ err = collectorWithState.InitCollector(helper.ApiCollectorArgs{
+ ApiClient: data.ApiClient,
+ PageSize: 100,
+ UrlTemplate: "users/search",
+ Query: func(reqData *helper.RequestData) (url.Values, errors.Error) {
+ query := url.Values{}
+ query.Set("p", fmt.Sprintf("%v", reqData.Pager.Page))
+ query.Set("ps", fmt.Sprintf("%v", reqData.Pager.Size))
+ return query, nil
+ },
+ GetTotalPages: GetTotalPagesFromResponse,
+ ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) {
+ var resData struct {
+ Data []json.RawMessage `json:"users"`
+ }
+ err = helper.UnmarshalResponse(res, &resData)
+ return resData.Data, err
+ },
+ })
+ if err != nil {
+ return err
+ }
+
+ return collectorWithState.Execute()
+}
+
+var CollectAccountsMeta = plugin.SubTaskMeta{
+ Name: "CollectAccounts",
+ EntryPoint: CollectAccounts,
+ EnabledByDefault: true,
+ Description: "Collect Accounts data from Sonarqube user api",
+}
diff --git a/backend/plugins/sonarqube/tasks/accounts_extractor.go b/backend/plugins/sonarqube/tasks/accounts_extractor.go
new file mode 100644
index 000000000..bee692751
--- /dev/null
+++ b/backend/plugins/sonarqube/tasks/accounts_extractor.go
@@ -0,0 +1,57 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "encoding/json"
+
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
+ helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "github.com/apache/incubator-devlake/plugins/sonarqube/models"
+)
+
+var _ plugin.SubTaskEntryPoint = ExtractAccounts
+
+func ExtractAccounts(taskCtx plugin.SubTaskContext) errors.Error {
+ rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_ACCOUNTS_TABLE)
+ extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{
+ RawDataSubTaskArgs: *rawDataSubTaskArgs,
+ Extract: func(resData *helper.RawData) ([]interface{}, errors.Error) {
+ body := &models.SonarqubeAccount{}
+ err := errors.Convert(json.Unmarshal(resData.Data, body))
+ body.ConnectionId = data.Options.ConnectionId
+ if err != nil {
+ return nil, err
+ }
+ return []interface{}{body}, nil
+ },
+ })
+ if err != nil {
+ return err
+ }
+
+ return extractor.Execute()
+}
+
+var ExtractAccountsMeta = plugin.SubTaskMeta{
+ Name: "ExtractAccounts",
+ EntryPoint: ExtractAccounts,
+ EnabledByDefault: true,
+ Description: "Extract raw data into tool layer table sonarqube_accounts",
+}