You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by ma...@apache.org on 2023/02/20 08:34:20 UTC
[incubator-devlake] branch main updated: feat(bamboo): add job collector and extractor (#4454)
This is an automated email from the ASF dual-hosted git repository.
mappjzc pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new c58dd2ce8 feat(bamboo): add job collector and extractor (#4454)
c58dd2ce8 is described below
commit c58dd2ce8ff11c59d7d9d61da00c26c1b1f0da68
Author: Warren Chen <yi...@merico.dev>
AuthorDate: Mon Feb 20 16:34:15 2023 +0800
feat(bamboo): add job collector and extractor (#4454)
---
backend/plugins/bamboo/impl/impl.go | 2 +
backend/plugins/bamboo/models/job.go | 72 ++++++++++++++
.../migrationscripts/20230216_add_init_tables.go | 3 +-
.../job.go} | 38 ++++---
backend/plugins/bamboo/models/plan.go | 42 ++++----
backend/plugins/bamboo/tasks/job_collector.go | 109 +++++++++++++++++++++
.../tasks/{plan_extractor.go => job_extractor.go} | 26 +++--
backend/plugins/bamboo/tasks/plan_extractor.go | 3 +-
8 files changed, 242 insertions(+), 53 deletions(-)
diff --git a/backend/plugins/bamboo/impl/impl.go b/backend/plugins/bamboo/impl/impl.go
index 734f6471e..b32ba7443 100644
--- a/backend/plugins/bamboo/impl/impl.go
+++ b/backend/plugins/bamboo/impl/impl.go
@@ -83,6 +83,8 @@ func (p Bamboo) SubTaskMetas() []plugin.SubTaskMeta {
return []plugin.SubTaskMeta{
tasks.CollectPlanMeta,
tasks.ExtractPlanMeta,
+ tasks.CollectJobMeta,
+ tasks.ExtractJobMeta,
tasks.ConvertProjectsMeta,
}
}
diff --git a/backend/plugins/bamboo/models/job.go b/backend/plugins/bamboo/models/job.go
new file mode 100644
index 000000000..6ee00dc08
--- /dev/null
+++ b/backend/plugins/bamboo/models/job.go
@@ -0,0 +1,72 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package models
+
+import (
+ "github.com/apache/incubator-devlake/core/models/common"
+)
+
+type BambooJob struct {
+ ConnectionId uint64 `gorm:"primaryKey"`
+ JobKey string `gorm:"primaryKey"`
+ Id string
+ Name string `json:"name"`
+ PlanKey string `json:"planKey"`
+ PlanName string `json:"planName"`
+ ProjectKey string `gorm:"index"`
+ ProjectName string `json:"projectName"`
+ Description string `json:"description"`
+ BranchName string `json:"branchName"`
+ StageName string `json:"stageName"`
+ Type string `json:"type"`
+ common.NoPKModel
+}
+
+func (BambooJob) Convert(apiRes *ApiBambooJob) *BambooJob {
+ b := &BambooJob{
+ JobKey: apiRes.SearchEntity.Key,
+ Id: apiRes.Id,
+ Name: apiRes.SearchEntity.JobName,
+ PlanName: apiRes.SearchEntity.PlanName,
+ ProjectName: apiRes.SearchEntity.ProjectName,
+ Description: apiRes.SearchEntity.Description,
+ BranchName: apiRes.SearchEntity.BranchName,
+ Type: apiRes.SearchEntity.Type,
+ }
+ return b
+}
+
+func (BambooJob) TableName() string {
+ return "_tool_bamboo_jobs"
+}
+
+type ApiBambooJob struct {
+ Id string `json:"id"`
+ Type string `json:"type"`
+ SearchEntity struct {
+ Id string `json:"id"`
+ Key string `json:"key"`
+ ProjectName string `json:"projectName"`
+ PlanName string `json:"planName"`
+ BranchName string `json:"branchName"`
+ StageName string `json:"stageName"`
+ JobName string `json:"jobName"`
+ Description string `json:"description"`
+ Type string `json:"type"`
+ } `json:"searchEntity"`
+}
diff --git a/backend/plugins/bamboo/models/migrationscripts/20230216_add_init_tables.go b/backend/plugins/bamboo/models/migrationscripts/20230216_add_init_tables.go
index 61c335e79..b12a01564 100644
--- a/backend/plugins/bamboo/models/migrationscripts/20230216_add_init_tables.go
+++ b/backend/plugins/bamboo/models/migrationscripts/20230216_add_init_tables.go
@@ -32,11 +32,12 @@ func (u *addInitTables) Up(baseRes context.BasicRes) errors.Error {
&archived.BambooConnection{},
&archived.BambooProject{},
&archived.BambooPlan{},
+ &archived.BambooJob{},
)
}
func (*addInitTables) Version() uint64 {
- return 20230216205031
+ return 20230216205032
}
func (*addInitTables) Name() string {
diff --git a/backend/plugins/bamboo/models/migrationscripts/20230216_add_init_tables.go b/backend/plugins/bamboo/models/migrationscripts/archived/job.go
similarity index 54%
copy from backend/plugins/bamboo/models/migrationscripts/20230216_add_init_tables.go
copy to backend/plugins/bamboo/models/migrationscripts/archived/job.go
index 61c335e79..82a2d4229 100644
--- a/backend/plugins/bamboo/models/migrationscripts/20230216_add_init_tables.go
+++ b/backend/plugins/bamboo/models/migrationscripts/archived/job.go
@@ -15,30 +15,28 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
-package migrationscripts
+package archived
import (
- "github.com/apache/incubator-devlake/core/context"
- "github.com/apache/incubator-devlake/core/errors"
- "github.com/apache/incubator-devlake/helpers/migrationhelper"
- "github.com/apache/incubator-devlake/plugins/bamboo/models/migrationscripts/archived"
+ "github.com/apache/incubator-devlake/core/models/migrationscripts/archived"
)
-type addInitTables struct{}
-
-func (u *addInitTables) Up(baseRes context.BasicRes) errors.Error {
- return migrationhelper.AutoMigrateTables(
- baseRes,
- &archived.BambooConnection{},
- &archived.BambooProject{},
- &archived.BambooPlan{},
- )
-}
-
-func (*addInitTables) Version() uint64 {
- return 20230216205031
+type BambooJob struct {
+ ConnectionId uint64 `gorm:"primaryKey"`
+ JobKey string `gorm:"primaryKey"`
+ Id string
+ Name string `json:"name"`
+ PlanKey string `json:"planKey"`
+ PlanName string `json:"planName"`
+ ProjectKey string `gorm:"index"`
+ ProjectName string `json:"projectName"`
+ Description string `json:"description"`
+ BranchName string `json:"branchName"`
+ StageName string `json:"stageName"`
+ Type string `json:"type"`
+ archived.NoPKModel
}
-func (*addInitTables) Name() string {
- return "bamboo init schemas"
+func (BambooJob) TableName() string {
+ return "_tool_bamboo_jobs"
}
diff --git a/backend/plugins/bamboo/models/plan.go b/backend/plugins/bamboo/models/plan.go
index fa1ecce92..4720f8842 100644
--- a/backend/plugins/bamboo/models/plan.go
+++ b/backend/plugins/bamboo/models/plan.go
@@ -43,28 +43,30 @@ type BambooPlan struct {
common.NoPKModel
}
-func (b *BambooPlan) Convert(apiProject *ApiBambooPlan) {
- b.PlanKey = apiProject.Key
- b.Name = apiProject.Name
- b.Expand = apiProject.Expand
- b.ProjectKey = apiProject.ProjectKey
- b.ProjectName = apiProject.ProjectName
- b.Description = apiProject.Description
- b.ShortName = apiProject.ShortName
- b.BuildName = apiProject.BuildName
- b.ShortKey = apiProject.ShortKey
- b.Type = apiProject.Type
- b.Enabled = apiProject.Enabled
- b.Href = apiProject.Href
- b.Rel = apiProject.Rel
- b.IsFavourite = apiProject.IsFavourite
- b.IsActive = apiProject.IsActive
- b.IsBuilding = apiProject.IsBuilding
- b.AverageBuildTimeInSeconds = apiProject.AverageBuildTimeInSeconds
-
+func (BambooPlan) Convert(apiProject *ApiBambooPlan) *BambooPlan {
+ b := &BambooPlan{
+ PlanKey: apiProject.Key,
+ Name: apiProject.Name,
+ Expand: apiProject.Expand,
+ ProjectKey: apiProject.ProjectKey,
+ ProjectName: apiProject.ProjectName,
+ Description: apiProject.Description,
+ ShortName: apiProject.ShortName,
+ BuildName: apiProject.BuildName,
+ ShortKey: apiProject.ShortKey,
+ Type: apiProject.Type,
+ Enabled: apiProject.Enabled,
+ Href: apiProject.Href,
+ Rel: apiProject.Rel,
+ IsFavourite: apiProject.IsFavourite,
+ IsActive: apiProject.IsActive,
+ IsBuilding: apiProject.IsBuilding,
+ AverageBuildTimeInSeconds: apiProject.AverageBuildTimeInSeconds,
+ }
+ return b
}
-func (b *BambooPlan) TableName() string {
+func (BambooPlan) TableName() string {
return "_tool_bamboo_plans"
}
diff --git a/backend/plugins/bamboo/tasks/job_collector.go b/backend/plugins/bamboo/tasks/job_collector.go
new file mode 100644
index 000000000..ebecb7254
--- /dev/null
+++ b/backend/plugins/bamboo/tasks/job_collector.go
@@ -0,0 +1,109 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "encoding/json"
+ "fmt"
+ "github.com/apache/incubator-devlake/core/dal"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
+ helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "github.com/apache/incubator-devlake/plugins/bamboo/models"
+ "net/http"
+ "net/url"
+ "reflect"
+)
+
+const RAW_JOB_TABLE = "bamboo_job"
+
+var _ plugin.SubTaskEntryPoint = CollectJob
+
+type SimplePlan struct {
+ PlanKey string
+}
+
+func CollectJob(taskCtx plugin.SubTaskContext) errors.Error {
+ rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_JOB_TABLE)
+ db := taskCtx.GetDal()
+ collectorWithState, err := helper.NewApiCollectorWithState(*rawDataSubTaskArgs, nil)
+ if err != nil {
+ return err
+ }
+ clauses := []dal.Clause{
+ dal.Select("plan_key"),
+ dal.From(models.BambooPlan{}.TableName()),
+ dal.Where("project_key = ? and connection_id=?", data.Options.ProjectKey, data.Options.ConnectionId),
+ }
+ cursor, err := db.Cursor(
+ clauses...,
+ )
+ if err != nil {
+ return err
+ }
+ iterator, err := helper.NewDalCursorIterator(db, cursor, reflect.TypeOf(SimplePlan{}))
+ if err != nil {
+ return err
+ }
+
+ err = collectorWithState.InitCollector(helper.ApiCollectorArgs{
+ ApiClient: data.ApiClient,
+ PageSize: 100,
+ Input: iterator,
+ UrlTemplate: "search/jobs/{{ .Input.PlanKey }}.json",
+ Query: func(reqData *helper.RequestData) (url.Values, errors.Error) {
+ query := url.Values{}
+ query.Set("showEmpty", fmt.Sprintf("%v", true))
+ query.Set("expand", "jobs.job")
+ query.Set("max-result", fmt.Sprintf("%v", reqData.Pager.Size))
+ query.Set("start-index", fmt.Sprintf("%v", (reqData.Pager.Page-1)*reqData.Pager.Size))
+ return query, nil
+ },
+ GetTotalPages: func(res *http.Response, args *helper.ApiCollectorArgs) (int, errors.Error) {
+ body := models.ApiBambooSizeData{}
+ err = helper.UnmarshalResponse(res, &body)
+ if err != nil {
+ return 0, err
+ }
+ return GetTotalPagesFromSizeInfo(&body, args)
+ },
+
+ ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) {
+ var results struct {
+ SearchResults []json.RawMessage `json:"searchResults"`
+ }
+ err = helper.UnmarshalResponse(res, &results)
+ if err != nil {
+ return nil, err
+ }
+ return results.SearchResults, nil
+ },
+ })
+ if err != nil {
+ return err
+ }
+ return collectorWithState.Execute()
+}
+
+var CollectJobMeta = plugin.SubTaskMeta{
+ Name: "CollectJob",
+ EntryPoint: CollectJob,
+ EnabledByDefault: true,
+ Description: "Collect Job data from Bamboo api",
+ DomainTypes: []string{plugin.DOMAIN_TYPE_CICD},
+}
diff --git a/backend/plugins/bamboo/tasks/plan_extractor.go b/backend/plugins/bamboo/tasks/job_extractor.go
similarity index 76%
copy from backend/plugins/bamboo/tasks/plan_extractor.go
copy to backend/plugins/bamboo/tasks/job_extractor.go
index 275060701..74515fcbf 100644
--- a/backend/plugins/bamboo/tasks/plan_extractor.go
+++ b/backend/plugins/bamboo/tasks/job_extractor.go
@@ -25,23 +25,29 @@ import (
"github.com/apache/incubator-devlake/plugins/bamboo/models"
)
-var _ plugin.SubTaskEntryPoint = ExtractPlan
+var _ plugin.SubTaskEntryPoint = ExtractJob
-func ExtractPlan(taskCtx plugin.SubTaskContext) errors.Error {
- rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_PLAN_TABLE)
+func ExtractJob(taskCtx plugin.SubTaskContext) errors.Error {
+ rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_JOB_TABLE)
extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{
RawDataSubTaskArgs: *rawDataSubTaskArgs,
Extract: func(resData *helper.RawData) ([]interface{}, errors.Error) {
- body := &models.BambooPlan{}
- res := &models.ApiBambooPlan{}
+ res := &models.ApiBambooJob{}
err := errors.Convert(json.Unmarshal(resData.Data, res))
if err != nil {
return nil, err
}
- body.Convert(res)
+ plan := &SimplePlan{}
+ err = errors.Convert(json.Unmarshal(resData.Input, plan))
+ if err != nil {
+ return nil, err
+ }
+ body := models.BambooJob{}.Convert(res)
body.ConnectionId = data.Options.ConnectionId
+ body.ProjectKey = data.Options.ProjectKey
+ body.PlanKey = plan.PlanKey
return []interface{}{body}, nil
},
})
@@ -52,10 +58,10 @@ func ExtractPlan(taskCtx plugin.SubTaskContext) errors.Error {
return extractor.Execute()
}
-var ExtractPlanMeta = plugin.SubTaskMeta{
- Name: "ExtractPlan",
- EntryPoint: ExtractPlan,
+var ExtractJobMeta = plugin.SubTaskMeta{
+ Name: "ExtractJob",
+ EntryPoint: ExtractJob,
EnabledByDefault: true,
- Description: "Extract raw data into tool layer table bamboo_plan",
+ Description: "Extract raw data into tool layer table bamboo_job",
DomainTypes: []string{plugin.DOMAIN_TYPE_CICD},
}
diff --git a/backend/plugins/bamboo/tasks/plan_extractor.go b/backend/plugins/bamboo/tasks/plan_extractor.go
index 275060701..5174b7b5e 100644
--- a/backend/plugins/bamboo/tasks/plan_extractor.go
+++ b/backend/plugins/bamboo/tasks/plan_extractor.go
@@ -34,13 +34,12 @@ func ExtractPlan(taskCtx plugin.SubTaskContext) errors.Error {
RawDataSubTaskArgs: *rawDataSubTaskArgs,
Extract: func(resData *helper.RawData) ([]interface{}, errors.Error) {
- body := &models.BambooPlan{}
res := &models.ApiBambooPlan{}
err := errors.Convert(json.Unmarshal(resData.Data, res))
if err != nil {
return nil, err
}
- body.Convert(res)
+ body := models.BambooPlan{}.Convert(res)
body.ConnectionId = data.Options.ConnectionId
return []interface{}{body}, nil
},