You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by ma...@apache.org on 2023/02/17 10:24:37 UTC
[incubator-devlake] branch main updated: feat(bamboo): add project (#4444)
This is an automated email from the ASF dual-hosted git repository.
mappjzc pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new c23303d14 feat(bamboo): add project (#4444)
c23303d14 is described below
commit c23303d1481a43730d1a7cc301926b3fa561fabe
Author: Warren Chen <yi...@merico.dev>
AuthorDate: Fri Feb 17 18:24:31 2023 +0800
feat(bamboo): add project (#4444)
---
backend/helpers/e2ehelper/data_flow_tester.go | 2 +-
.../api/apihelperabstract/connection_abstract.go | 2 +-
.../helpers/pluginhelper/api/connection_auths.go | 2 +-
backend/plugins/bamboo/bamboo.go | 4 +-
backend/plugins/bamboo/impl/impl.go | 7 +-
.../migrationscripts/20230216_add_init_tables.go | 5 +-
.../project.go} | 31 ++---
.../20230216_add_init_tables.go => project.go} | 37 +++---
backend/plugins/bamboo/tasks/project_collector.go | 79 ++++++++++++
backend/plugins/bamboo/tasks/project_extractor.go | 140 +++++++++++++++++++++
backend/plugins/bamboo/tasks/projects_convertor.go | 75 +++++++++++
backend/plugins/bamboo/tasks/shared.go | 64 ++++++++++
backend/plugins/bamboo/tasks/task_data.go | 9 +-
.../bitbucket/tasks/deployment_extractor.go | 2 +-
backend/plugins/sonarqube/api/scope.go | 2 +-
.../plugins/sonarqube/tasks/projects_extractor.go | 6 +-
16 files changed, 411 insertions(+), 56 deletions(-)
diff --git a/backend/helpers/e2ehelper/data_flow_tester.go b/backend/helpers/e2ehelper/data_flow_tester.go
index 7727c3273..380a8e1b2 100644
--- a/backend/helpers/e2ehelper/data_flow_tester.go
+++ b/backend/helpers/e2ehelper/data_flow_tester.go
@@ -368,7 +368,7 @@ func (t *DataFlowTester) VerifyTableWithRawData(dst schema.Tabler, csvRelPath st
}
// VerifyTable reads rows from csv file and compare with records from database one by one. You must specify the
-// Primary Key Fields with `pkFields` so DataFlowTester could select the exact record from database, as well as which
+// Primary ProjectKey Fields with `pkFields` so DataFlowTester could select the exact record from database, as well as which
// fields to compare with by specifying `targetFields` parameter. Leaving `targetFields` empty/nil will compare all fields.
func (t *DataFlowTester) VerifyTable(dst schema.Tabler, csvRelPath string, targetFields []string) {
t.VerifyTableWithOptions(dst, TableOptions{
diff --git a/backend/helpers/pluginhelper/api/apihelperabstract/connection_abstract.go b/backend/helpers/pluginhelper/api/apihelperabstract/connection_abstract.go
index 39feaad0d..baa59fe84 100644
--- a/backend/helpers/pluginhelper/api/apihelperabstract/connection_abstract.go
+++ b/backend/helpers/pluginhelper/api/apihelperabstract/connection_abstract.go
@@ -77,7 +77,7 @@ type AccessTokenAuthenticator interface {
GetAccessTokenAuthenticator() ApiAuthenticator
}
-// AppKeyAuthenticator represents the API Key and Secret authentication mechanism
+// AppKeyAuthenticator represents the API ProjectKey and Secret authentication mechanism
type AppKeyAuthenticator interface {
GetAppKeyAuthenticator() ApiAuthenticator
}
diff --git a/backend/helpers/pluginhelper/api/connection_auths.go b/backend/helpers/pluginhelper/api/connection_auths.go
index 50c0973bb..5ff321caa 100644
--- a/backend/helpers/pluginhelper/api/connection_auths.go
+++ b/backend/helpers/pluginhelper/api/connection_auths.go
@@ -69,7 +69,7 @@ func (at *AccessToken) GetAccessTokenAuthenticator() apihelperabstract.ApiAuthen
return at
}
-// AppKey implements the API Key and Secret authentication mechanism
+// AppKey implements the API ProjectKey and Secret authentication mechanism
type AppKey struct {
AppId string `mapstructure:"appId" validate:"required" json:"appId"`
SecretKey string `mapstructure:"secretKey" validate:"required" json:"secretKey" gorm:"serializer:encdec"`
diff --git a/backend/plugins/bamboo/bamboo.go b/backend/plugins/bamboo/bamboo.go
index 076323e69..bab746a2b 100644
--- a/backend/plugins/bamboo/bamboo.go
+++ b/backend/plugins/bamboo/bamboo.go
@@ -30,12 +30,12 @@ var PluginEntry impl.Bamboo //nolint
func main() {
bambooCmd := &cobra.Command{Use: "bamboo"}
connectionId := bambooCmd.Flags().Uint64P("Connection-id", "c", 0, "bamboo connection id")
- projectId := bambooCmd.Flags().IntP("project-id", "p", 0, "bamboo project id")
+ projectKey := bambooCmd.Flags().StringP("project-key", "p", "", "bamboo project key")
_ = bambooCmd.MarkFlagRequired("project-id")
bambooCmd.Run = func(cmd *cobra.Command, args []string) {
runner.DirectRun(cmd, args, PluginEntry, map[string]interface{}{
"connectionId": *connectionId,
- "projectId": *projectId,
+ "projectKey": *projectKey,
})
}
runner.RunCmd(bambooCmd)
diff --git a/backend/plugins/bamboo/impl/impl.go b/backend/plugins/bamboo/impl/impl.go
index a7c922d88..f8347c0b7 100644
--- a/backend/plugins/bamboo/impl/impl.go
+++ b/backend/plugins/bamboo/impl/impl.go
@@ -71,6 +71,7 @@ func (p Bamboo) MakeDataSourcePipelinePlanV200(connectionId uint64, scopes []*pl
func (p Bamboo) GetTablesInfo() []dal.Tabler {
return []dal.Tabler{
&models.BambooConnection{},
+ &models.BambooProject{},
}
}
@@ -80,7 +81,11 @@ func (p Bamboo) Description() string {
func (p Bamboo) SubTaskMetas() []plugin.SubTaskMeta {
// TODO add your sub task here
- return []plugin.SubTaskMeta{}
+ return []plugin.SubTaskMeta{
+ tasks.CollectProjectMeta,
+ tasks.ExtractProjectMeta,
+ tasks.ConvertProjectsMeta,
+ }
}
func (p Bamboo) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]interface{}) (interface{}, errors.Error) {
diff --git a/backend/plugins/bamboo/models/migrationscripts/20230216_add_init_tables.go b/backend/plugins/bamboo/models/migrationscripts/20230216_add_init_tables.go
index 0a25a788d..1b208b546 100644
--- a/backend/plugins/bamboo/models/migrationscripts/20230216_add_init_tables.go
+++ b/backend/plugins/bamboo/models/migrationscripts/20230216_add_init_tables.go
@@ -29,12 +29,13 @@ type addInitTables struct{}
func (u *addInitTables) Up(baseRes context.BasicRes) errors.Error {
return migrationhelper.AutoMigrateTables(
baseRes,
- archived.BambooConnection{},
+ &archived.BambooConnection{},
+ &archived.BambooProject{},
)
}
func (*addInitTables) Version() uint64 {
- return 20230216205025
+ return 20230216205028
}
func (*addInitTables) Name() string {
diff --git a/backend/plugins/bamboo/models/migrationscripts/20230216_add_init_tables.go b/backend/plugins/bamboo/models/migrationscripts/archived/project.go
similarity index 56%
copy from backend/plugins/bamboo/models/migrationscripts/20230216_add_init_tables.go
copy to backend/plugins/bamboo/models/migrationscripts/archived/project.go
index 0a25a788d..0ac9d3b8d 100644
--- a/backend/plugins/bamboo/models/migrationscripts/20230216_add_init_tables.go
+++ b/backend/plugins/bamboo/models/migrationscripts/archived/project.go
@@ -15,28 +15,23 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
-package migrationscripts
+package archived
import (
- "github.com/apache/incubator-devlake/core/context"
- "github.com/apache/incubator-devlake/core/errors"
- "github.com/apache/incubator-devlake/helpers/migrationhelper"
- "github.com/apache/incubator-devlake/plugins/bamboo/models/migrationscripts/archived"
+ "github.com/apache/incubator-devlake/core/models/migrationscripts/archived"
)
-type addInitTables struct{}
-
-func (u *addInitTables) Up(baseRes context.BasicRes) errors.Error {
- return migrationhelper.AutoMigrateTables(
- baseRes,
- archived.BambooConnection{},
- )
-}
-
-func (*addInitTables) Version() uint64 {
- return 20230216205025
+type BambooProject struct {
+ ConnectionId uint64 `gorm:"primaryKey"`
+ ProjectKey string `gorm:"primaryKey;type:varchar(100)"`
+ Expand string `json:"expand"`
+ Name string `gorm:"index;type:varchar(100)"`
+ Description string `json:"description"`
+ Href string `json:"link"`
+ Rel string `gorm:"type:varchar(100)"`
+ archived.NoPKModel
}
-func (*addInitTables) Name() string {
- return "bamboo init schemas"
+func (BambooProject) TableName() string {
+ return "_tool_bamboo_projects"
}
diff --git a/backend/plugins/bamboo/models/migrationscripts/20230216_add_init_tables.go b/backend/plugins/bamboo/models/project.go
similarity index 55%
copy from backend/plugins/bamboo/models/migrationscripts/20230216_add_init_tables.go
copy to backend/plugins/bamboo/models/project.go
index 0a25a788d..7da7ba561 100644
--- a/backend/plugins/bamboo/models/migrationscripts/20230216_add_init_tables.go
+++ b/backend/plugins/bamboo/models/project.go
@@ -15,28 +15,21 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
-package migrationscripts
-
-import (
- "github.com/apache/incubator-devlake/core/context"
- "github.com/apache/incubator-devlake/core/errors"
- "github.com/apache/incubator-devlake/helpers/migrationhelper"
- "github.com/apache/incubator-devlake/plugins/bamboo/models/migrationscripts/archived"
-)
-
-type addInitTables struct{}
-
-func (u *addInitTables) Up(baseRes context.BasicRes) errors.Error {
- return migrationhelper.AutoMigrateTables(
- baseRes,
- archived.BambooConnection{},
- )
-}
-
-func (*addInitTables) Version() uint64 {
- return 20230216205025
+package models
+
+import "github.com/apache/incubator-devlake/core/models/common"
+
+type BambooProject struct {
+ ConnectionId uint64 `gorm:"primaryKey"`
+ ProjectKey string `gorm:"primaryKey;type:varchar(100)"`
+ Expand string `json:"expand"`
+ Name string `gorm:"index;type:varchar(100)"`
+ Description string `json:"description"`
+ Href string `json:"link"`
+ Rel string `gorm:"type:varchar(100)"`
+ common.NoPKModel
}
-func (*addInitTables) Name() string {
- return "bamboo init schemas"
+func (BambooProject) TableName() string {
+ return "_tool_bamboo_projects"
}
diff --git a/backend/plugins/bamboo/tasks/project_collector.go b/backend/plugins/bamboo/tasks/project_collector.go
new file mode 100644
index 000000000..108a142bf
--- /dev/null
+++ b/backend/plugins/bamboo/tasks/project_collector.go
@@ -0,0 +1,79 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "encoding/json"
+ "fmt"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
+ helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "io"
+ "net/http"
+ "net/url"
+)
+
+const RAW_PROJECT_TABLE = "bamboo_project"
+
+var _ plugin.SubTaskEntryPoint = CollectProject
+
+func CollectProject(taskCtx plugin.SubTaskContext) errors.Error {
+ rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_PROJECT_TABLE)
+
+ collectorWithState, err := helper.NewApiCollectorWithState(*rawDataSubTaskArgs, nil)
+ if err != nil {
+ return err
+ }
+ incremental := collectorWithState.IsIncremental()
+
+ err = collectorWithState.InitCollector(helper.ApiCollectorArgs{
+ Incremental: incremental,
+ ApiClient: data.ApiClient,
+ // TODO write which api would you want request
+ UrlTemplate: "project/{{ .Params.ProjectKey }}.json",
+ Query: func(reqData *helper.RequestData) (url.Values, errors.Error) {
+ query := url.Values{}
+ query.Set("showEmpty", fmt.Sprintf("%v", true))
+ return query, nil
+ },
+ GetTotalPages: func(res *http.Response, args *helper.ApiCollectorArgs) (int, errors.Error) {
+ return 1, nil
+ },
+
+ ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) {
+ body, err := io.ReadAll(res.Body)
+ if err != nil {
+ return nil, errors.Convert(err)
+ }
+ res.Body.Close()
+ return []json.RawMessage{body}, nil
+ },
+ })
+ if err != nil {
+ return err
+ }
+ return collectorWithState.Execute()
+}
+
+var CollectProjectMeta = plugin.SubTaskMeta{
+ Name: "CollectProject",
+ EntryPoint: CollectProject,
+ EnabledByDefault: true,
+ Description: "Collect Project data from Bamboo api",
+ DomainTypes: []string{plugin.DOMAIN_TYPE_CICD},
+}
diff --git a/backend/plugins/bamboo/tasks/project_extractor.go b/backend/plugins/bamboo/tasks/project_extractor.go
new file mode 100644
index 000000000..7263ea14e
--- /dev/null
+++ b/backend/plugins/bamboo/tasks/project_extractor.go
@@ -0,0 +1,140 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "encoding/json"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
+ helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "github.com/apache/incubator-devlake/plugins/bamboo/models"
+)
+
+var _ plugin.SubTaskEntryPoint = ExtractProject
+
+func ExtractProject(taskCtx plugin.SubTaskContext) errors.Error {
+ rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_PROJECT_TABLE)
+
+ extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{
+ RawDataSubTaskArgs: *rawDataSubTaskArgs,
+
+ Extract: func(resData *helper.RawData) ([]interface{}, errors.Error) {
+ res := &ApiProject{}
+ err := errors.Convert(json.Unmarshal(resData.Data, res))
+ if err != nil {
+ return nil, err
+ }
+ body := ConvertProject(res)
+ body.ConnectionId = data.Options.ConnectionId
+ return []interface{}{body}, nil
+ },
+ })
+ if err != nil {
+ return err
+ }
+
+ return extractor.Execute()
+}
+
+var ExtractProjectMeta = plugin.SubTaskMeta{
+ Name: "ExtractProject",
+ EntryPoint: ExtractProject,
+ EnabledByDefault: true,
+ Description: "Extract raw data into tool layer table bamboo_project",
+ DomainTypes: []string{plugin.DOMAIN_TYPE_CICD},
+}
+
+// Convert the API response to our DB model instance
+func ConvertProject(bambooApiProject *ApiProject) *models.BambooProject {
+ bambooProject := &models.BambooProject{
+ Expand: bambooApiProject.Expand,
+ ProjectKey: bambooApiProject.Key,
+ Name: bambooApiProject.Name,
+ Description: bambooApiProject.Description,
+ Href: bambooApiProject.Link.Href,
+ Rel: bambooApiProject.Link.Rel,
+ }
+ return bambooProject
+}
+
+type ApiProject struct {
+ Expand string `json:"expand"`
+ Key string `json:"key"`
+ Name string `json:"name"`
+ Description string `json:"description"`
+ Link ApiLink `json:"link"`
+ //Plans []ApiPlan `json:"plans"`
+}
+type ApiLink struct {
+ Href string `json:"href"`
+ Rel string `json:"rel"`
+}
+
+//type ApiProject struct {
+// ProjectKey string `json:"key"`
+// Name string `json:"name"`
+// Description string `json:"description"`
+// Link Link `json:"link"`
+//}
+//type ApiActions struct {
+// Size int `json:"size"`
+// StartIndex int `json:"start-index"`
+// MaxResult int `json:"max-result"`
+//}
+//type ApiStages struct {
+// Size int `json:"size"`
+// StartIndex int `json:"start-index"`
+// MaxResult int `json:"max-result"`
+//}
+//type ApiBranches struct {
+// Size int `json:"size"`
+// StartIndex int `json:"start-index"`
+// MaxResult int `json:"max-result"`
+//}
+//type ApiPlanKey struct {
+// ProjectKey string `json:"key"`
+//}
+//type ApiPlan struct {
+// Expand string `json:"expand"`
+// ProjectKey string `json:"projectKey"`
+// ProjectName string `json:"projectName"`
+// Description string `json:"description"`
+// ShortName string `json:"shortName"`
+// BuildName string `json:"buildName"`
+// ShortKey string `json:"shortKey"`
+// Type string `json:"type"`
+// Enabled bool `json:"enabled"`
+// ApiLink `json:"link"`
+// IsFavourite bool `json:"isFavourite"`
+// IsActive bool `json:"isActive"`
+// IsBuilding bool `json:"isBuilding"`
+// AverageBuildTimeInSeconds int `json:"averageBuildTimeInSeconds"`
+// Actions Actions `json:"actions"`
+// Stages Stages `json:"stages"`
+// Branches Branches `json:"branches"`
+// ProjectKey string `json:"key"`
+// Name string `json:"name"`
+// PlanKey PlanKey `json:"planKey"`
+//}
+//type ApiPlans struct {
+// Size int `json:"size"`
+// Expand string `json:"expand"`
+// StartIndex int `json:"start-index"`
+// MaxResult int `json:"max-result"`
+// Plan []ApiPlan `json:"plan"`
+//}
diff --git a/backend/plugins/bamboo/tasks/projects_convertor.go b/backend/plugins/bamboo/tasks/projects_convertor.go
new file mode 100644
index 000000000..3de31efda
--- /dev/null
+++ b/backend/plugins/bamboo/tasks/projects_convertor.go
@@ -0,0 +1,75 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "github.com/apache/incubator-devlake/core/models/domainlayer/devops"
+ "reflect"
+
+ "github.com/apache/incubator-devlake/core/dal"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/models/domainlayer"
+ "github.com/apache/incubator-devlake/core/models/domainlayer/didgen"
+ "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ bambooModels "github.com/apache/incubator-devlake/plugins/bamboo/models"
+)
+
+var ConvertProjectsMeta = plugin.SubTaskMeta{
+ Name: "convertProjects",
+ EntryPoint: ConvertProjects,
+ EnabledByDefault: true,
+ Description: "Convert tool layer table bamboo_projects into domain layer table projects",
+ DomainTypes: []string{plugin.DOMAIN_TYPE_CICD},
+}
+
+func ConvertProjects(taskCtx plugin.SubTaskContext) errors.Error {
+ db := taskCtx.GetDal()
+ rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_PROJECT_TABLE)
+ cursor, err := db.Cursor(dal.From(bambooModels.BambooProject{}),
+ dal.Where("connection_id = ? and project_key = ?", data.Options.ConnectionId, data.Options.ProjectKey))
+ if err != nil {
+ return err
+ }
+ defer cursor.Close()
+
+ projectIdGen := didgen.NewDomainIdGenerator(&bambooModels.BambooProject{})
+ converter, err := api.NewDataConverter(api.DataConverterArgs{
+ InputRowType: reflect.TypeOf(bambooModels.BambooProject{}),
+ Input: cursor,
+ RawDataSubTaskArgs: *rawDataSubTaskArgs,
+ Convert: func(inputRow interface{}) ([]interface{}, errors.Error) {
+ bambooProject := inputRow.(*bambooModels.BambooProject)
+ domainProject := &devops.CicdScope{
+ DomainEntity: domainlayer.DomainEntity{Id: projectIdGen.Generate(data.Options.ConnectionId, bambooProject.ProjectKey)},
+ Name: bambooProject.Name,
+ Description: bambooProject.Description,
+ Url: bambooProject.Href,
+ }
+ return []interface{}{
+ domainProject,
+ }, nil
+ },
+ })
+
+ if err != nil {
+ return err
+ }
+
+ return converter.Execute()
+}
diff --git a/backend/plugins/bamboo/tasks/shared.go b/backend/plugins/bamboo/tasks/shared.go
new file mode 100644
index 000000000..97f96278a
--- /dev/null
+++ b/backend/plugins/bamboo/tasks/shared.go
@@ -0,0 +1,64 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "net/http"
+)
+
+func CreateRawDataSubTaskArgs(taskCtx plugin.SubTaskContext, rawTable string) (*api.RawDataSubTaskArgs, *BambooTaskData) {
+ data := taskCtx.GetData().(*BambooTaskData)
+ filteredData := *data
+ filteredData.Options = &BambooOptions{}
+ *filteredData.Options = *data.Options
+ var params = BambooApiParams{
+ ConnectionId: data.Options.ConnectionId,
+ ProjectKey: data.Options.ProjectKey,
+ }
+ rawDataSubTaskArgs := &api.RawDataSubTaskArgs{
+ Ctx: taskCtx,
+ Params: params,
+ Table: rawTable,
+ }
+ return rawDataSubTaskArgs, &filteredData
+}
+
+func GetTotalPagesFromResponse(res *http.Response, args *api.ApiCollectorArgs) (int, errors.Error) {
+ body := &BambooPagination{}
+ err := api.UnmarshalResponse(res, body)
+ if err != nil {
+ return 0, err
+ }
+ pages := body.Paging.Total / args.PageSize
+ if body.Paging.Total%args.PageSize > 0 {
+ pages++
+ }
+ return pages, nil
+}
+
+type BambooPagination struct {
+ Paging Paging `json:"paging"`
+}
+type Paging struct {
+ PageIndex int `json:"pageIndex"`
+ PageSize int `json:"pageSize"`
+ Total int `json:"total"`
+}
diff --git a/backend/plugins/bamboo/tasks/task_data.go b/backend/plugins/bamboo/tasks/task_data.go
index ea5c62bce..48528e9e9 100644
--- a/backend/plugins/bamboo/tasks/task_data.go
+++ b/backend/plugins/bamboo/tasks/task_data.go
@@ -23,6 +23,8 @@ import (
)
type BambooApiParams struct {
+ ConnectionId uint64 `json:"connectionId"`
+ ProjectKey string
}
type BambooOptions struct {
@@ -30,9 +32,10 @@ type BambooOptions struct {
// options means some custom params required by plugin running.
// Such As How many rows do your want
// You can use it in sub tasks and you need pass it in main.go and pipelines.
- ConnectionId uint64 `json:"connectionId"`
- Tasks []string `json:"tasks,omitempty"`
- Since string
+ ConnectionId uint64 `json:"connectionId"`
+ ProjectKey string `json:"projectKey"`
+ CreatedDateAfter string `json:"createdDateAfter" mapstructure:"createdDateAfter,omitempty"`
+ Tasks []string `json:"tasks,omitempty"`
}
type BambooTaskData struct {
diff --git a/backend/plugins/bitbucket/tasks/deployment_extractor.go b/backend/plugins/bitbucket/tasks/deployment_extractor.go
index 1b5fa12ea..f3215a990 100644
--- a/backend/plugins/bitbucket/tasks/deployment_extractor.go
+++ b/backend/plugins/bitbucket/tasks/deployment_extractor.go
@@ -29,7 +29,7 @@ import (
type bitbucketApiDeploymentsResponse struct {
Type string `json:"type"`
UUID string `json:"uuid"`
- //Key string `json:"key"`
+ //ProjectKey string `json:"key"`
//Step struct {
// UUID string `json:"uuid"`
//} `json:"step"`
diff --git a/backend/plugins/sonarqube/api/scope.go b/backend/plugins/sonarqube/api/scope.go
index 063020133..57818c9d9 100644
--- a/backend/plugins/sonarqube/api/scope.go
+++ b/backend/plugins/sonarqube/api/scope.go
@@ -80,7 +80,7 @@ func PutScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors
// @Tags plugins/sonarqube
// @Accept application/json
// @Param connectionId path int false "connection ID"
-// @Param projectKey path string false "project Key"
+// @Param projectKey path string false "project ProjectKey"
// @Param scope body models.SonarqubeProject true "json"
// @Success 200 {object} models.SonarqubeProject
// @Failure 400 {object} shared.ApiBody "Bad Request"
diff --git a/backend/plugins/sonarqube/tasks/projects_extractor.go b/backend/plugins/sonarqube/tasks/projects_extractor.go
index 46d92bef0..aaf28f13d 100644
--- a/backend/plugins/sonarqube/tasks/projects_extractor.go
+++ b/backend/plugins/sonarqube/tasks/projects_extractor.go
@@ -44,12 +44,12 @@ func ExtractProjects(taskCtx plugin.SubTaskContext) errors.Error {
Extract: func(resData *helper.RawData) ([]interface{}, errors.Error) {
res := SonarqubeApiProject{}
err := errors.Convert(json.Unmarshal(resData.Data, &res))
- body := ConvertProject(&res)
- body.ConnectionId = data.Options.ConnectionId
- data.LastAnalysisDate = body.LastAnalysisDate.ToNullableTime()
if err != nil {
return nil, err
}
+ body := ConvertProject(&res)
+ body.ConnectionId = data.Options.ConnectionId
+ data.LastAnalysisDate = body.LastAnalysisDate.ToNullableTime()
return []interface{}{body}, nil
},
})