You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by ab...@apache.org on 2023/02/08 07:19:51 UTC
[incubator-devlake] branch main updated: feat(sonarqube): add project convertor (#4355)
This is an automated email from the ASF dual-hosted git repository.
abeizn pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new c554d1294 feat(sonarqube): add project convertor (#4355)
c554d1294 is described below
commit c554d129463285e54fcbd8142e2c5d0b567e0011
Author: Warren Chen <yi...@merico.dev>
AuthorDate: Wed Feb 8 15:19:46 2023 +0800
feat(sonarqube): add project convertor (#4355)
---
.../domainlayer/securitytesting/st_projects.go | 48 ++++++++++++++
.../20230208_add_security_testing.go | 42 ++++++++++++
.../migrationscripts/archived/st_projects.go | 36 ++++++++++
backend/core/models/migrationscripts/register.go | 1 +
backend/core/plugin/plugin_task.go | 12 ++--
backend/plugins/dora/tasks/task_data.go | 4 --
backend/plugins/sonarqube/impl/impl.go | 1 +
backend/plugins/sonarqube/sonarqube.go | 3 +-
.../plugins/sonarqube/tasks/project_convertor.go | 77 ++++++++++++++++++++++
9 files changed, 213 insertions(+), 11 deletions(-)
diff --git a/backend/core/models/domainlayer/securitytesting/st_projects.go b/backend/core/models/domainlayer/securitytesting/st_projects.go
new file mode 100644
index 000000000..ce676570f
--- /dev/null
+++ b/backend/core/models/domainlayer/securitytesting/st_projects.go
@@ -0,0 +1,48 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package securitytesting
+
+import (
+ "github.com/apache/incubator-devlake/core/models/domainlayer"
+ "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+)
+
+var _ plugin.Scope = (*StProject)(nil)
+
+type StProject struct {
+ domainlayer.DomainEntity
+ Key string `json:"key" gorm:"type:varchar(64);primaryKey"`
+ Name string `json:"name" gorm:"type:varchar(255)"`
+ Qualifier string `json:"qualifier" gorm:"type:varchar(255)"`
+ Visibility string `json:"visibility" gorm:"type:varchar(64)"`
+ LastAnalysisDate *api.Iso8601Time `json:"lastAnalysisDate"`
+ Revision string `json:"revision" gorm:"type:varchar(128)"`
+}
+
+func (StProject) TableName() string {
+ return "st_projects"
+}
+
+func (r *StProject) ScopeId() string {
+ return r.Id
+}
+
+func (r *StProject) ScopeName() string {
+ return r.Name
+}
diff --git a/backend/core/models/migrationscripts/20230208_add_security_testing.go b/backend/core/models/migrationscripts/20230208_add_security_testing.go
new file mode 100644
index 000000000..e9e2b9db4
--- /dev/null
+++ b/backend/core/models/migrationscripts/20230208_add_security_testing.go
@@ -0,0 +1,42 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+ "github.com/apache/incubator-devlake/core/context"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/models/migrationscripts/archived"
+ "github.com/apache/incubator-devlake/helpers/migrationhelper"
+)
+
+type addSecurityTesting struct{}
+
+func (u *addSecurityTesting) Up(basicRes context.BasicRes) errors.Error {
+ return migrationhelper.AutoMigrateTables(
+ basicRes,
+ &archived.StProject{},
+ )
+}
+
+func (*addSecurityTesting) Version() uint64 {
+ return 20230208000001
+}
+
+func (*addSecurityTesting) Name() string {
+ return "add SecurityTesting domain"
+}
diff --git a/backend/core/models/migrationscripts/archived/st_projects.go b/backend/core/models/migrationscripts/archived/st_projects.go
new file mode 100644
index 000000000..b9716b491
--- /dev/null
+++ b/backend/core/models/migrationscripts/archived/st_projects.go
@@ -0,0 +1,36 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package archived
+
+import (
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+)
+
+type StProject struct {
+ DomainEntity
+ Key string `json:"key" gorm:"type:varchar(64);primaryKey"`
+ Name string `json:"name" gorm:"type:varchar(255)"`
+ Qualifier string `json:"qualifier" gorm:"type:varchar(255)"`
+ Visibility string `json:"visibility" gorm:"type:varchar(64)"`
+ LastAnalysisDate *api.Iso8601Time `json:"lastAnalysisDate"`
+ Revision string `json:"revision" gorm:"type:varchar(128)"`
+}
+
+func (StProject) TableName() string {
+ return "st_projects"
+}
diff --git a/backend/core/models/migrationscripts/register.go b/backend/core/models/migrationscripts/register.go
index 9cbcb6aa3..3f77dd82f 100644
--- a/backend/core/models/migrationscripts/register.go
+++ b/backend/core/models/migrationscripts/register.go
@@ -69,5 +69,6 @@ func All() []plugin.MigrationScript {
new(encryptTask221221),
new(renameProjectMetrics),
new(addOriginalTypeToIssue221230),
+ new(addSecurityTesting),
}
}
diff --git a/backend/core/plugin/plugin_task.go b/backend/core/plugin/plugin_task.go
index d04db8075..ccf197e36 100644
--- a/backend/core/plugin/plugin_task.go
+++ b/backend/core/plugin/plugin_task.go
@@ -72,11 +72,12 @@ type SubTask interface {
// SubTaskEntryPoint All subtasks from plugins should comply to this prototype, so they could be orchestrated by framework
type SubTaskEntryPoint func(c SubTaskContext) errors.Error
-const DOMAIN_TYPE_CODE = "CODE" //nolint
-const DOMAIN_TYPE_TICKET = "TICKET" //nolint
-const DOMAIN_TYPE_CODE_REVIEW = "CODEREVIEW" //nolint
-const DOMAIN_TYPE_CROSS = "CROSS" //nolint
-const DOMAIN_TYPE_CICD = "CICD" //nolint
+const DOMAIN_TYPE_CODE = "CODE" //nolint
+const DOMAIN_TYPE_TICKET = "TICKET" //nolint
+const DOMAIN_TYPE_CODE_REVIEW = "CODEREVIEW" //nolint
+const DOMAIN_TYPE_CROSS = "CROSS" //nolint
+const DOMAIN_TYPE_CICD = "CICD" //nolint
+const DOMAIN_TYPE_SECURITY_TESTING = "SECURITYTESTING" //nolint
var DOMAIN_TYPES = []string{
DOMAIN_TYPE_CODE,
@@ -84,6 +85,7 @@ var DOMAIN_TYPES = []string{
DOMAIN_TYPE_CODE_REVIEW,
DOMAIN_TYPE_CROSS,
DOMAIN_TYPE_CICD,
+ DOMAIN_TYPE_SECURITY_TESTING,
} //nolint
// SubTaskMeta Metadata of a subtask
diff --git a/backend/plugins/dora/tasks/task_data.go b/backend/plugins/dora/tasks/task_data.go
index d9bea87ed..96b7ca44b 100644
--- a/backend/plugins/dora/tasks/task_data.go
+++ b/backend/plugins/dora/tasks/task_data.go
@@ -35,10 +35,6 @@ type TransformationRules struct {
type DoraOptions struct {
Tasks []string `json:"tasks,omitempty"`
Since string
- RepoId string `json:"repoId"`
- CicdScopeId string `json:"cicdScopeId"`
- BoardId string `json:"boardId"`
- Prefix string `json:"prefix"`
ProjectName string `json:"projectName"`
TransformationRules `mapstructure:"transformationRules" json:"transformationRules"`
}
diff --git a/backend/plugins/sonarqube/impl/impl.go b/backend/plugins/sonarqube/impl/impl.go
index d26867f3b..90e91f503 100644
--- a/backend/plugins/sonarqube/impl/impl.go
+++ b/backend/plugins/sonarqube/impl/impl.go
@@ -63,6 +63,7 @@ func (p Sonarqube) SubTaskMetas() []plugin.SubTaskMeta {
tasks.ExtractFilemetricsMeta,
tasks.CollectAccountsMeta,
tasks.ExtractAccountsMeta,
+ tasks.ConvertProjectsMeta,
}
}
diff --git a/backend/plugins/sonarqube/sonarqube.go b/backend/plugins/sonarqube/sonarqube.go
index 9fd2222ed..64a683db0 100644
--- a/backend/plugins/sonarqube/sonarqube.go
+++ b/backend/plugins/sonarqube/sonarqube.go
@@ -28,10 +28,9 @@ var PluginEntry impl.Sonarqube //nolint
// standalone mode for debugging
func main() {
cmd := &cobra.Command{Use: "sonarqube"}
-
connectionId := cmd.Flags().Uint64P("connectionId", "c", 0, "sonarqube connection id")
projectKey := cmd.Flags().StringP("projectKey", "p", "", "sonarqube projectKey")
- createdDateAfter := cmd.Flags().StringP("createdDateAfter", "a", "", "collect data that are created after specified time, ie 2006-05-06T07:08:09Z")
+ createdDateAfter := cmd.Flags().StringP("createdDateAfter", "a", "", "collect data that are created after specified time, ie 2006-05-06T07:08:09+08:00")
_ = cmd.MarkFlagRequired("connectionId")
//_ = cmd.MarkFlagRequired("projectKey")
diff --git a/backend/plugins/sonarqube/tasks/project_convertor.go b/backend/plugins/sonarqube/tasks/project_convertor.go
new file mode 100644
index 000000000..aabe498cb
--- /dev/null
+++ b/backend/plugins/sonarqube/tasks/project_convertor.go
@@ -0,0 +1,77 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "github.com/apache/incubator-devlake/core/dal"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/models/domainlayer"
+ "github.com/apache/incubator-devlake/core/models/domainlayer/didgen"
+ "github.com/apache/incubator-devlake/core/models/domainlayer/securitytesting"
+ "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ sonarqubeModels "github.com/apache/incubator-devlake/plugins/sonarqube/models"
+ "reflect"
+)
+
+var ConvertProjectsMeta = plugin.SubTaskMeta{
+ Name: "convertProjects",
+ EntryPoint: ConvertProjects,
+ EnabledByDefault: true,
+ Description: "Convert tool layer table sonarqube_projects into domain layer table projects",
+ DomainTypes: []string{plugin.DOMAIN_TYPE_SECURITY_TESTING},
+}
+
+func ConvertProjects(taskCtx plugin.SubTaskContext) errors.Error {
+ db := taskCtx.GetDal()
+ rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_PROJECTS_TABLE)
+ cursor, err := db.Cursor(dal.From(sonarqubeModels.SonarqubeProject{}),
+ dal.Where("connection_id = ? and `key` = ?", data.Options.ConnectionId, data.Options.ProjectKey))
+ if err != nil {
+ return err
+ }
+ defer cursor.Close()
+
+ accountIdGen := didgen.NewDomainIdGenerator(&sonarqubeModels.SonarqubeProject{})
+ converter, err := api.NewDataConverter(api.DataConverterArgs{
+ InputRowType: reflect.TypeOf(sonarqubeModels.SonarqubeProject{}),
+ Input: cursor,
+ RawDataSubTaskArgs: *rawDataSubTaskArgs,
+ Convert: func(inputRow interface{}) ([]interface{}, errors.Error) {
+ sonarqubeProject := inputRow.(*sonarqubeModels.SonarqubeProject)
+ domainProject := &securitytesting.StProject{
+ DomainEntity: domainlayer.DomainEntity{Id: accountIdGen.Generate(data.Options.ConnectionId, sonarqubeProject.Key)},
+ Key: sonarqubeProject.Key,
+ Name: sonarqubeProject.Name,
+ Qualifier: sonarqubeProject.Qualifier,
+ Visibility: sonarqubeProject.Visibility,
+ LastAnalysisDate: sonarqubeProject.LastAnalysisDate,
+ Revision: sonarqubeProject.Revision,
+ }
+ return []interface{}{
+ domainProject,
+ }, nil
+ },
+ })
+
+ if err != nil {
+ return err
+ }
+
+ return converter.Execute()
+}