You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by wa...@apache.org on 2023/02/20 14:11:18 UTC
[incubator-devlake] branch main updated: feat(bamboo): add plan and job build (#4460)
This is an automated email from the ASF dual-hosted git repository.
warren pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new c1f7d3623 feat(bamboo): add plan and job build (#4460)
c1f7d3623 is described below
commit c1f7d3623e365c76e35b4be75759effbf019b8d0
Author: Warren Chen <yi...@merico.dev>
AuthorDate: Mon Feb 20 22:11:13 2023 +0800
feat(bamboo): add plan and job build (#4460)
---
backend/plugins/bamboo/impl/impl.go | 10 +-
backend/plugins/bamboo/models/job_build.go | 115 +++++++++++++++
.../migrationscripts/20230216_add_init_tables.go | 13 +-
.../models/migrationscripts/archived/job_build.go | 70 +++++++++
.../models/migrationscripts/archived/plan_build.go | 68 +++++++++
.../plan_build_commit.go} | 32 ++--
backend/plugins/bamboo/models/plan_build.go | 163 +++++++++++++++++++++
...216_add_init_tables.go => plan_build_commit.go} | 32 ++--
.../plugins/bamboo/tasks/job_build_collector.go | 115 +++++++++++++++
.../plugins/bamboo/tasks/job_build_extractor.go | 81 ++++++++++
.../plugins/bamboo/tasks/plan_build_collector.go | 109 ++++++++++++++
.../plugins/bamboo/tasks/plan_build_extractor.go | 83 +++++++++++
12 files changed, 847 insertions(+), 44 deletions(-)
diff --git a/backend/plugins/bamboo/impl/impl.go b/backend/plugins/bamboo/impl/impl.go
index b32ba7443..d8fe3b244 100644
--- a/backend/plugins/bamboo/impl/impl.go
+++ b/backend/plugins/bamboo/impl/impl.go
@@ -19,7 +19,6 @@ package impl
import (
"fmt"
-
"github.com/apache/incubator-devlake/core/context"
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
@@ -71,6 +70,11 @@ func (p Bamboo) GetTablesInfo() []dal.Tabler {
return []dal.Tabler{
&models.BambooConnection{},
&models.BambooProject{},
+ &models.BambooPlan{},
+ &models.BambooJob{},
+ &models.BambooPlanBuild{},
+ &models.BambooPlanBuildVcsRevision{},
+ &models.BambooJobBuild{},
}
}
@@ -85,6 +89,10 @@ func (p Bamboo) SubTaskMetas() []plugin.SubTaskMeta {
tasks.ExtractPlanMeta,
tasks.CollectJobMeta,
tasks.ExtractJobMeta,
+ tasks.CollectPlanBuildMeta,
+ tasks.ExtractPlanBuildMeta,
+ tasks.CollectJobBuildMeta,
+ tasks.ExtractJobBuildMeta,
tasks.ConvertProjectsMeta,
}
}
diff --git a/backend/plugins/bamboo/models/job_build.go b/backend/plugins/bamboo/models/job_build.go
new file mode 100644
index 000000000..df4081042
--- /dev/null
+++ b/backend/plugins/bamboo/models/job_build.go
@@ -0,0 +1,115 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package models
+
+import (
+ "github.com/apache/incubator-devlake/core/models/common"
+ "time"
+)
+
+type BambooJobBuild struct {
+ ConnectionId uint64 `gorm:"primaryKey"`
+ JobBuildKey string `gorm:"primaryKey"`
+ Expand string `json:"expand"`
+ Number int `json:"number"`
+ BuildNumber int `json:"buildNumber"`
+ JobName string `gorm:"index"`
+ JobKey string `gorm:"index"`
+ PlanName string `gorm:"index"`
+ PlanKey string `gorm:"index"`
+ ProjectName string `gorm:"index"`
+ ProjectKey string `gorm:"index"`
+ BuildResultKey string `json:"buildResultKey"`
+ LifeCycleState string `json:"lifeCycleState"`
+ BuildStartedTime time.Time `json:"buildStartedTime"`
+ PrettyBuildStartedTime string `json:"prettyBuildStartedTime"`
+ BuildCompletedTime time.Time `json:"buildCompletedTime"`
+ BuildCompletedDate time.Time `json:"buildCompletedDate"`
+ PrettyBuildCompletedTime string `json:"prettyBuildCompletedTime"`
+ BuildDurationInSeconds int `json:"buildDurationInSeconds"`
+ BuildDuration int `json:"buildDuration"`
+ BuildDurationDescription string `json:"buildDurationDescription"`
+ BuildRelativeTime string `json:"buildRelativeTime"`
+ VcsRevisionKey string `json:"vcsRevisionKey"`
+ BuildTestSummary string `json:"buildTestSummary"`
+ SuccessfulTestCount int `json:"successfulTestCount"`
+ FailedTestCount int `json:"failedTestCount"`
+ QuarantinedTestCount int `json:"quarantinedTestCount"`
+ SkippedTestCount int `json:"skippedTestCount"`
+ Continuable bool `json:"continuable"`
+ OnceOff bool `json:"onceOff"`
+ Restartable bool `json:"restartable"`
+ NotRunYet bool `json:"notRunYet"`
+ Finished bool `json:"finished"`
+ Successful bool `json:"successful"`
+ BuildReason string `json:"buildReason"`
+ ReasonSummary string `json:"reasonSummary"`
+ State string `json:"state"`
+ BuildState string `json:"buildState"`
+ JobResultKey string
+ common.NoPKModel
+}
+
+func (BambooJobBuild) TableName() string {
+ return "_tool_bamboo_job_builds"
+}
+
+func (BambooJobBuild) Convert(apiRes *ApiBambooPlanBuild) *BambooJobBuild {
+ b := &BambooJobBuild{
+ ConnectionId: 0,
+ JobBuildKey: apiRes.Key,
+ Expand: apiRes.Expand,
+ Number: apiRes.Number,
+ BuildNumber: apiRes.BuildNumber,
+ JobName: apiRes.PlanName,
+ JobKey: "",
+ PlanName: "",
+ PlanKey: "",
+ ProjectName: apiRes.ProjectName,
+ ProjectKey: "",
+ BuildResultKey: apiRes.BuildResultKey,
+ LifeCycleState: apiRes.LifeCycleState,
+ BuildStartedTime: apiRes.BuildStartedTime,
+ PrettyBuildStartedTime: apiRes.PrettyBuildStartedTime,
+ BuildCompletedTime: apiRes.BuildCompletedTime,
+ BuildCompletedDate: apiRes.BuildCompletedDate,
+ PrettyBuildCompletedTime: apiRes.PrettyBuildCompletedTime,
+ BuildDurationInSeconds: apiRes.BuildDurationInSeconds,
+ BuildDuration: apiRes.BuildDuration,
+ BuildDurationDescription: apiRes.BuildDurationDescription,
+ BuildRelativeTime: apiRes.BuildRelativeTime,
+ VcsRevisionKey: apiRes.VcsRevisionKey,
+ BuildTestSummary: apiRes.BuildTestSummary,
+ SuccessfulTestCount: apiRes.SuccessfulTestCount,
+ FailedTestCount: apiRes.FailedTestCount,
+ QuarantinedTestCount: apiRes.QuarantinedTestCount,
+ SkippedTestCount: apiRes.SkippedTestCount,
+ Continuable: apiRes.Continuable,
+ OnceOff: apiRes.OnceOff,
+ Restartable: apiRes.Restartable,
+ NotRunYet: apiRes.NotRunYet,
+ Finished: apiRes.Finished,
+ Successful: apiRes.Successful,
+ BuildReason: apiRes.BuildReason,
+ ReasonSummary: apiRes.ReasonSummary,
+ State: apiRes.State,
+ BuildState: apiRes.BuildState,
+ JobResultKey: apiRes.PlanResultKey.Key,
+ }
+ return b
+}
diff --git a/backend/plugins/bamboo/models/migrationscripts/20230216_add_init_tables.go b/backend/plugins/bamboo/models/migrationscripts/20230216_add_init_tables.go
index b12a01564..c78946a2e 100644
--- a/backend/plugins/bamboo/models/migrationscripts/20230216_add_init_tables.go
+++ b/backend/plugins/bamboo/models/migrationscripts/20230216_add_init_tables.go
@@ -27,17 +27,28 @@ import (
type addInitTables struct{}
func (u *addInitTables) Up(baseRes context.BasicRes) errors.Error {
+ // will be deleted after finish bamboo
+ _ = baseRes.GetDal().DropTables(
+ &archived.BambooPlan{},
+ &archived.BambooJob{},
+ &archived.BambooPlanBuild{},
+ &archived.BambooPlanBuildVcsRevision{},
+ &archived.BambooJobBuild{},
+ )
return migrationhelper.AutoMigrateTables(
baseRes,
&archived.BambooConnection{},
&archived.BambooProject{},
&archived.BambooPlan{},
&archived.BambooJob{},
+ &archived.BambooPlanBuild{},
+ &archived.BambooPlanBuildVcsRevision{},
+ &archived.BambooJobBuild{},
)
}
func (*addInitTables) Version() uint64 {
- return 20230216205032
+ return 20230216205037
}
func (*addInitTables) Name() string {
diff --git a/backend/plugins/bamboo/models/migrationscripts/archived/job_build.go b/backend/plugins/bamboo/models/migrationscripts/archived/job_build.go
new file mode 100644
index 000000000..9991e62d7
--- /dev/null
+++ b/backend/plugins/bamboo/models/migrationscripts/archived/job_build.go
@@ -0,0 +1,70 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package archived
+
+import (
+ "github.com/apache/incubator-devlake/core/models/migrationscripts/archived"
+ "time"
+)
+
+type BambooJobBuild struct {
+ ConnectionId uint64 `gorm:"primaryKey"`
+ JobBuildKey string `gorm:"primaryKey"`
+ Expand string `json:"expand"`
+ Number int `json:"number"`
+ BuildNumber int `json:"buildNumber"`
+ JobName string `gorm:"index"`
+ JobKey string `gorm:"index"`
+ PlanName string `gorm:"index"`
+ PlanKey string `gorm:"index"`
+ ProjectName string `gorm:"index"`
+ ProjectKey string `gorm:"index"`
+ BuildResultKey string `json:"buildResultKey"`
+ LifeCycleState string `json:"lifeCycleState"`
+ BuildStartedTime time.Time `json:"buildStartedTime"`
+ PrettyBuildStartedTime string `json:"prettyBuildStartedTime"`
+ BuildCompletedTime time.Time `json:"buildCompletedTime"`
+ BuildCompletedDate time.Time `json:"buildCompletedDate"`
+ PrettyBuildCompletedTime string `json:"prettyBuildCompletedTime"`
+ BuildDurationInSeconds int `json:"buildDurationInSeconds"`
+ BuildDuration int `json:"buildDuration"`
+ BuildDurationDescription string `json:"buildDurationDescription"`
+ BuildRelativeTime string `json:"buildRelativeTime"`
+ VcsRevisionKey string `json:"vcsRevisionKey"`
+ BuildTestSummary string `json:"buildTestSummary"`
+ SuccessfulTestCount int `json:"successfulTestCount"`
+ FailedTestCount int `json:"failedTestCount"`
+ QuarantinedTestCount int `json:"quarantinedTestCount"`
+ SkippedTestCount int `json:"skippedTestCount"`
+ Continuable bool `json:"continuable"`
+ OnceOff bool `json:"onceOff"`
+ Restartable bool `json:"restartable"`
+ NotRunYet bool `json:"notRunYet"`
+ Finished bool `json:"finished"`
+ Successful bool `json:"successful"`
+ BuildReason string `json:"buildReason"`
+ ReasonSummary string `json:"reasonSummary"`
+ State string `json:"state"`
+ BuildState string `json:"buildState"`
+ JobResultKey string
+ archived.NoPKModel
+}
+
+func (BambooJobBuild) TableName() string {
+ return "_tool_bamboo_job_builds"
+}
diff --git a/backend/plugins/bamboo/models/migrationscripts/archived/plan_build.go b/backend/plugins/bamboo/models/migrationscripts/archived/plan_build.go
new file mode 100644
index 000000000..e9391ced5
--- /dev/null
+++ b/backend/plugins/bamboo/models/migrationscripts/archived/plan_build.go
@@ -0,0 +1,68 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package archived
+
+import (
+ "github.com/apache/incubator-devlake/core/models/migrationscripts/archived"
+ "time"
+)
+
+type BambooPlanBuild struct {
+ ConnectionId uint64 `gorm:"primaryKey"`
+ PlanBuildKey string `gorm:"primaryKey"`
+ Expand string `json:"expand"`
+ Number int `json:"number"`
+ BuildNumber int `json:"buildNumber"`
+ PlanName string `json:"planName"`
+ PlanKey string
+ ProjectName string `json:"projectName"`
+ ProjectKey string
+ BuildResultKey string `json:"buildResultKey"`
+ LifeCycleState string `json:"lifeCycleState"`
+ BuildStartedTime time.Time `json:"buildStartedTime"`
+ PrettyBuildStartedTime string `json:"prettyBuildStartedTime"`
+ BuildCompletedTime time.Time `json:"buildCompletedTime"`
+ BuildCompletedDate time.Time `json:"buildCompletedDate"`
+ PrettyBuildCompletedTime string `json:"prettyBuildCompletedTime"`
+ BuildDurationInSeconds int `json:"buildDurationInSeconds"`
+ BuildDuration int `json:"buildDuration"`
+ BuildDurationDescription string `json:"buildDurationDescription"`
+ BuildRelativeTime string `json:"buildRelativeTime"`
+ VcsRevisionKey string `json:"vcsRevisionKey"`
+ BuildTestSummary string `json:"buildTestSummary"`
+ SuccessfulTestCount int `json:"successfulTestCount"`
+ FailedTestCount int `json:"failedTestCount"`
+ QuarantinedTestCount int `json:"quarantinedTestCount"`
+ SkippedTestCount int `json:"skippedTestCount"`
+ Continuable bool `json:"continuable"`
+ OnceOff bool `json:"onceOff"`
+ Restartable bool `json:"restartable"`
+ NotRunYet bool `json:"notRunYet"`
+ Finished bool `json:"finished"`
+ Successful bool `json:"successful"`
+ BuildReason string `json:"buildReason"`
+ ReasonSummary string `json:"reasonSummary"`
+ State string `json:"state"`
+ BuildState string `json:"buildState"`
+ PlanResultKey string
+ archived.NoPKModel
+}
+
+func (BambooPlanBuild) TableName() string {
+ return "_tool_bamboo_plan_builds"
+}
diff --git a/backend/plugins/bamboo/models/migrationscripts/20230216_add_init_tables.go b/backend/plugins/bamboo/models/migrationscripts/archived/plan_build_commit.go
similarity index 53%
copy from backend/plugins/bamboo/models/migrationscripts/20230216_add_init_tables.go
copy to backend/plugins/bamboo/models/migrationscripts/archived/plan_build_commit.go
index b12a01564..75cbdd019 100644
--- a/backend/plugins/bamboo/models/migrationscripts/20230216_add_init_tables.go
+++ b/backend/plugins/bamboo/models/migrationscripts/archived/plan_build_commit.go
@@ -15,31 +15,21 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
-package migrationscripts
+package archived
import (
- "github.com/apache/incubator-devlake/core/context"
- "github.com/apache/incubator-devlake/core/errors"
- "github.com/apache/incubator-devlake/helpers/migrationhelper"
- "github.com/apache/incubator-devlake/plugins/bamboo/models/migrationscripts/archived"
+ "github.com/apache/incubator-devlake/core/models/migrationscripts/archived"
)
-type addInitTables struct{}
-
-func (u *addInitTables) Up(baseRes context.BasicRes) errors.Error {
- return migrationhelper.AutoMigrateTables(
- baseRes,
- &archived.BambooConnection{},
- &archived.BambooProject{},
- &archived.BambooPlan{},
- &archived.BambooJob{},
- )
-}
-
-func (*addInitTables) Version() uint64 {
- return 20230216205032
+type BambooPlanBuildVcsRevision struct {
+ ConnectionId uint64 `gorm:"primaryKey"`
+ PlanBuildKey string `gorm:"primaryKey"`
+ RepositoryId int
+ RepositoryName string `json:"repositoryName"`
+ VcsRevisionKey string `gorm:"primaryKey"`
+ archived.NoPKModel
}
-func (*addInitTables) Name() string {
- return "bamboo init schemas"
+func (BambooPlanBuildVcsRevision) TableName() string {
+ return "_tool_bamboo_plan_build_commits"
}
diff --git a/backend/plugins/bamboo/models/plan_build.go b/backend/plugins/bamboo/models/plan_build.go
new file mode 100644
index 000000000..07588e1a5
--- /dev/null
+++ b/backend/plugins/bamboo/models/plan_build.go
@@ -0,0 +1,163 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package models
+
+import (
+ "github.com/apache/incubator-devlake/core/models/common"
+ "time"
+)
+
+type BambooPlanBuild struct {
+ ConnectionId uint64 `gorm:"primaryKey"`
+ PlanBuildKey string `gorm:"primaryKey"`
+ Expand string `json:"expand"`
+ Number int `json:"number"`
+ BuildNumber int `json:"buildNumber"`
+ PlanName string `json:"planName"`
+ PlanKey string
+ ProjectName string `json:"projectName"`
+ ProjectKey string
+ BuildResultKey string `json:"buildResultKey"`
+ LifeCycleState string `json:"lifeCycleState"`
+ BuildStartedTime time.Time `json:"buildStartedTime"`
+ PrettyBuildStartedTime string `json:"prettyBuildStartedTime"`
+ BuildCompletedTime time.Time `json:"buildCompletedTime"`
+ BuildCompletedDate time.Time `json:"buildCompletedDate"`
+ PrettyBuildCompletedTime string `json:"prettyBuildCompletedTime"`
+ BuildDurationInSeconds int `json:"buildDurationInSeconds"`
+ BuildDuration int `json:"buildDuration"`
+ BuildDurationDescription string `json:"buildDurationDescription"`
+ BuildRelativeTime string `json:"buildRelativeTime"`
+ VcsRevisionKey string `json:"vcsRevisionKey"`
+ BuildTestSummary string `json:"buildTestSummary"`
+ SuccessfulTestCount int `json:"successfulTestCount"`
+ FailedTestCount int `json:"failedTestCount"`
+ QuarantinedTestCount int `json:"quarantinedTestCount"`
+ SkippedTestCount int `json:"skippedTestCount"`
+ Continuable bool `json:"continuable"`
+ OnceOff bool `json:"onceOff"`
+ Restartable bool `json:"restartable"`
+ NotRunYet bool `json:"notRunYet"`
+ Finished bool `json:"finished"`
+ Successful bool `json:"successful"`
+ BuildReason string `json:"buildReason"`
+ ReasonSummary string `json:"reasonSummary"`
+ State string `json:"state"`
+ BuildState string `json:"buildState"`
+ PlanResultKey string
+ common.NoPKModel
+}
+
+func (BambooPlanBuild) TableName() string {
+ return "_tool_bamboo_plan_builds"
+}
+
+func (BambooPlanBuild) Convert(apiRes *ApiBambooPlanBuild) *BambooPlanBuild {
+ b := &BambooPlanBuild{
+ PlanBuildKey: apiRes.Key,
+ Expand: apiRes.Expand,
+ Number: apiRes.Number,
+ BuildNumber: apiRes.BuildNumber,
+ PlanName: apiRes.PlanName,
+ ProjectName: apiRes.ProjectName,
+ BuildResultKey: apiRes.BuildResultKey,
+ LifeCycleState: apiRes.LifeCycleState,
+ BuildStartedTime: apiRes.BuildStartedTime,
+ PrettyBuildStartedTime: apiRes.PrettyBuildStartedTime,
+ BuildCompletedTime: apiRes.BuildCompletedTime,
+ BuildCompletedDate: apiRes.BuildCompletedDate,
+ PrettyBuildCompletedTime: apiRes.PrettyBuildCompletedTime,
+ BuildDurationInSeconds: apiRes.BuildDurationInSeconds,
+ BuildDuration: apiRes.BuildDuration,
+ BuildDurationDescription: apiRes.BuildDurationDescription,
+ BuildRelativeTime: apiRes.BuildRelativeTime,
+ VcsRevisionKey: apiRes.VcsRevisionKey,
+ BuildTestSummary: apiRes.BuildTestSummary,
+ SuccessfulTestCount: apiRes.SuccessfulTestCount,
+ FailedTestCount: apiRes.FailedTestCount,
+ QuarantinedTestCount: apiRes.QuarantinedTestCount,
+ SkippedTestCount: apiRes.SkippedTestCount,
+ Continuable: apiRes.Continuable,
+ OnceOff: apiRes.OnceOff,
+ Restartable: apiRes.Restartable,
+ NotRunYet: apiRes.NotRunYet,
+ Finished: apiRes.Finished,
+ Successful: apiRes.Successful,
+ BuildReason: apiRes.BuildReason,
+ ReasonSummary: apiRes.ReasonSummary,
+ State: apiRes.State,
+ BuildState: apiRes.BuildState,
+ PlanResultKey: apiRes.PlanResultKey.Key,
+ }
+ return b
+}
+
+type ApiBambooPlanBuild struct {
+ Expand string `json:"expand"`
+ PlanName string `json:"planName"`
+ ProjectName string `json:"projectName"`
+ BuildResultKey string `json:"buildResultKey"`
+ LifeCycleState string `json:"lifeCycleState"`
+ Id int `json:"id"`
+ BuildStartedTime time.Time `json:"buildStartedTime"`
+ PrettyBuildStartedTime string `json:"prettyBuildStartedTime"`
+ BuildCompletedTime time.Time `json:"buildCompletedTime"`
+ BuildCompletedDate time.Time `json:"buildCompletedDate"`
+ PrettyBuildCompletedTime string `json:"prettyBuildCompletedTime"`
+ BuildDurationInSeconds int `json:"buildDurationInSeconds"`
+ BuildDuration int `json:"buildDuration"`
+ BuildDurationDescription string `json:"buildDurationDescription"`
+ BuildRelativeTime string `json:"buildRelativeTime"`
+ VcsRevisionKey string `json:"vcsRevisionKey"`
+ VcsRevisions struct {
+ Size int `json:"size"`
+ Expand string `json:"expand"`
+ VcsRevision []struct {
+ RepositoryId int `json:"repositoryId"`
+ RepositoryName string `json:"repositoryName"`
+ VcsRevisionKey string `json:"vcsRevisionKey"`
+ } `json:"vcsRevision"`
+ StartIndex int `json:"start-index"`
+ MaxResult int `json:"max-result"`
+ } `json:"vcsRevisions"`
+ BuildTestSummary string `json:"buildTestSummary"`
+ SuccessfulTestCount int `json:"successfulTestCount"`
+ FailedTestCount int `json:"failedTestCount"`
+ QuarantinedTestCount int `json:"quarantinedTestCount"`
+ SkippedTestCount int `json:"skippedTestCount"`
+ Continuable bool `json:"continuable"`
+ OnceOff bool `json:"onceOff"`
+ Restartable bool `json:"restartable"`
+ NotRunYet bool `json:"notRunYet"`
+ Finished bool `json:"finished"`
+ Successful bool `json:"successful"`
+ BuildReason string `json:"buildReason"`
+ ReasonSummary string `json:"reasonSummary"`
+ Key string `json:"key"`
+ PlanResultKey struct {
+ Key string `json:"key"`
+ EntityKey struct {
+ Key string `json:"key"`
+ } `json:"entityKey"`
+ ResultNumber int `json:"resultNumber"`
+ } `json:"planResultKey"`
+ State string `json:"state"`
+ BuildState string `json:"buildState"`
+ Number int `json:"number"`
+ BuildNumber int `json:"buildNumber"`
+}
diff --git a/backend/plugins/bamboo/models/migrationscripts/20230216_add_init_tables.go b/backend/plugins/bamboo/models/plan_build_commit.go
similarity index 53%
copy from backend/plugins/bamboo/models/migrationscripts/20230216_add_init_tables.go
copy to backend/plugins/bamboo/models/plan_build_commit.go
index b12a01564..3129a94da 100644
--- a/backend/plugins/bamboo/models/migrationscripts/20230216_add_init_tables.go
+++ b/backend/plugins/bamboo/models/plan_build_commit.go
@@ -15,31 +15,21 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
-package migrationscripts
+package models
import (
- "github.com/apache/incubator-devlake/core/context"
- "github.com/apache/incubator-devlake/core/errors"
- "github.com/apache/incubator-devlake/helpers/migrationhelper"
- "github.com/apache/incubator-devlake/plugins/bamboo/models/migrationscripts/archived"
+ "github.com/apache/incubator-devlake/core/models/common"
)
-type addInitTables struct{}
-
-func (u *addInitTables) Up(baseRes context.BasicRes) errors.Error {
- return migrationhelper.AutoMigrateTables(
- baseRes,
- &archived.BambooConnection{},
- &archived.BambooProject{},
- &archived.BambooPlan{},
- &archived.BambooJob{},
- )
-}
-
-func (*addInitTables) Version() uint64 {
- return 20230216205032
+type BambooPlanBuildVcsRevision struct {
+ ConnectionId uint64 `gorm:"primaryKey"`
+ PlanBuildKey string `gorm:"primaryKey"`
+ RepositoryId int
+ RepositoryName string `json:"repositoryName"`
+ VcsRevisionKey string `gorm:"primaryKey"`
+ common.NoPKModel
}
-func (*addInitTables) Name() string {
- return "bamboo init schemas"
+func (BambooPlanBuildVcsRevision) TableName() string {
+ return "_tool_bamboo_plan_build_commits"
}
diff --git a/backend/plugins/bamboo/tasks/job_build_collector.go b/backend/plugins/bamboo/tasks/job_build_collector.go
new file mode 100644
index 000000000..81674986d
--- /dev/null
+++ b/backend/plugins/bamboo/tasks/job_build_collector.go
@@ -0,0 +1,115 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "encoding/json"
+ "fmt"
+ "github.com/apache/incubator-devlake/core/dal"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
+ helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "github.com/apache/incubator-devlake/plugins/bamboo/models"
+ "net/http"
+ "net/url"
+ "reflect"
+)
+
+const RAW_JOB_BUILD_TABLE = "bamboo_job_build"
+
+var _ plugin.SubTaskEntryPoint = CollectJobBuild
+
+type SimpleJob struct {
+ JobKey string
+ PlanName string
+ PlanKey string
+}
+
+func CollectJobBuild(taskCtx plugin.SubTaskContext) errors.Error {
+ rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_JOB_BUILD_TABLE)
+ db := taskCtx.GetDal()
+ collectorWithState, err := helper.NewApiCollectorWithState(*rawDataSubTaskArgs, nil)
+ if err != nil {
+ return err
+ }
+ clauses := []dal.Clause{
+ dal.Select("job_key, plan_name, plan_key"),
+ dal.From(models.BambooJob{}.TableName()),
+ dal.Where("project_key = ? and connection_id=?", data.Options.ProjectKey, data.Options.ConnectionId),
+ }
+ cursor, err := db.Cursor(
+ clauses...,
+ )
+ if err != nil {
+ return err
+ }
+ iterator, err := helper.NewDalCursorIterator(db, cursor, reflect.TypeOf(SimpleJob{}))
+ if err != nil {
+ return err
+ }
+
+ err = collectorWithState.InitCollector(helper.ApiCollectorArgs{
+ ApiClient: data.ApiClient,
+ PageSize: 100,
+ Input: iterator,
+ UrlTemplate: "result/{{ .Input.JobKey }}.json",
+ Query: func(reqData *helper.RequestData) (url.Values, errors.Error) {
+ query := url.Values{}
+ query.Set("showEmpty", fmt.Sprintf("%v", true))
+ query.Set("expand", "results.result.vcsRevisions")
+ query.Set("max-result", fmt.Sprintf("%v", reqData.Pager.Size))
+ query.Set("start-index", fmt.Sprintf("%v", (reqData.Pager.Page-1)*reqData.Pager.Size))
+ return query, nil
+ },
+ GetTotalPages: func(res *http.Response, args *helper.ApiCollectorArgs) (int, errors.Error) {
+ var body struct {
+ SizeInfo models.ApiBambooSizeData `json:"results"`
+ }
+ err = helper.UnmarshalResponse(res, &body)
+ if err != nil {
+ return 0, err
+ }
+ return GetTotalPagesFromSizeInfo(&body.SizeInfo, args)
+ },
+
+ ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) {
+ var resData struct {
+ Results struct {
+ Result []json.RawMessage `json:"result"`
+ } `json:"results"`
+ }
+ err = helper.UnmarshalResponse(res, &resData)
+ if err != nil {
+ return nil, err
+ }
+ return resData.Results.Result, nil
+ },
+ })
+ if err != nil {
+ return err
+ }
+ return collectorWithState.Execute()
+}
+
+var CollectJobBuildMeta = plugin.SubTaskMeta{
+ Name: "CollectJobBuild",
+ EntryPoint: CollectJobBuild,
+ EnabledByDefault: true,
+ Description: "Collect JobBuild data from Bamboo api",
+ DomainTypes: []string{plugin.DOMAIN_TYPE_CICD},
+}
diff --git a/backend/plugins/bamboo/tasks/job_build_extractor.go b/backend/plugins/bamboo/tasks/job_build_extractor.go
new file mode 100644
index 000000000..b7ff68e21
--- /dev/null
+++ b/backend/plugins/bamboo/tasks/job_build_extractor.go
@@ -0,0 +1,81 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "encoding/json"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
+ helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "github.com/apache/incubator-devlake/plugins/bamboo/models"
+)
+
+var _ plugin.SubTaskEntryPoint = ExtractJobBuild
+
+func ExtractJobBuild(taskCtx plugin.SubTaskContext) errors.Error {
+ rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_JOB_BUILD_TABLE)
+
+ extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{
+ RawDataSubTaskArgs: *rawDataSubTaskArgs,
+
+ Extract: func(resData *helper.RawData) ([]interface{}, errors.Error) {
+ res := &models.ApiBambooPlanBuild{}
+ err := errors.Convert(json.Unmarshal(resData.Data, res))
+ if err != nil {
+ return nil, err
+ }
+ plan := &SimpleJob{}
+ err = errors.Convert(json.Unmarshal(resData.Input, plan))
+ if err != nil {
+ return nil, err
+ }
+ body := models.BambooJobBuild{}.Convert(res)
+ body.ConnectionId = data.Options.ConnectionId
+ body.ProjectKey = data.Options.ProjectKey
+ body.JobKey = plan.JobKey
+ body.PlanKey = plan.PlanKey
+ body.PlanName = plan.PlanName
+
+ results := make([]interface{}, 0)
+ results = append(results, body)
+ for _, v := range res.VcsRevisions.VcsRevision {
+ results = append(results, &models.BambooPlanBuildVcsRevision{
+ ConnectionId: data.Options.ConnectionId,
+ PlanBuildKey: body.PlanKey,
+ RepositoryId: v.RepositoryId,
+ RepositoryName: v.RepositoryName,
+ VcsRevisionKey: v.VcsRevisionKey,
+ })
+ }
+ return results, nil
+ },
+ })
+ if err != nil {
+ return err
+ }
+
+ return extractor.Execute()
+}
+
+var ExtractJobBuildMeta = plugin.SubTaskMeta{
+ Name: "ExtractJobBuild",
+ EntryPoint: ExtractJobBuild,
+ EnabledByDefault: true,
+ Description: "Extract raw data into tool layer table bamboo_plan_builds",
+ DomainTypes: []string{plugin.DOMAIN_TYPE_CICD},
+}
diff --git a/backend/plugins/bamboo/tasks/plan_build_collector.go b/backend/plugins/bamboo/tasks/plan_build_collector.go
new file mode 100644
index 000000000..a7f2edf90
--- /dev/null
+++ b/backend/plugins/bamboo/tasks/plan_build_collector.go
@@ -0,0 +1,109 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "encoding/json"
+ "fmt"
+ "github.com/apache/incubator-devlake/core/dal"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
+ helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "github.com/apache/incubator-devlake/plugins/bamboo/models"
+ "net/http"
+ "net/url"
+ "reflect"
+)
+
+const RAW_PLAN_BUILD_TABLE = "bamboo_plan_build"
+
+var _ plugin.SubTaskEntryPoint = CollectPlanBuild
+
+func CollectPlanBuild(taskCtx plugin.SubTaskContext) errors.Error {
+ rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_PLAN_BUILD_TABLE)
+ db := taskCtx.GetDal()
+ collectorWithState, err := helper.NewApiCollectorWithState(*rawDataSubTaskArgs, nil)
+ if err != nil {
+ return err
+ }
+ clauses := []dal.Clause{
+ dal.Select("plan_key"),
+ dal.From(models.BambooPlan{}.TableName()),
+ dal.Where("project_key = ? and connection_id=?", data.Options.ProjectKey, data.Options.ConnectionId),
+ }
+ cursor, err := db.Cursor(
+ clauses...,
+ )
+ if err != nil {
+ return err
+ }
+ iterator, err := helper.NewDalCursorIterator(db, cursor, reflect.TypeOf(SimplePlan{}))
+ if err != nil {
+ return err
+ }
+
+ err = collectorWithState.InitCollector(helper.ApiCollectorArgs{
+ ApiClient: data.ApiClient,
+ PageSize: 100,
+ Input: iterator,
+ UrlTemplate: "result/{{ .Input.PlanKey }}.json",
+ Query: func(reqData *helper.RequestData) (url.Values, errors.Error) {
+ query := url.Values{}
+ query.Set("showEmpty", fmt.Sprintf("%v", true))
+ query.Set("expand", "results.result.vcsRevisions")
+ query.Set("max-result", fmt.Sprintf("%v", reqData.Pager.Size))
+ query.Set("start-index", fmt.Sprintf("%v", (reqData.Pager.Page-1)*reqData.Pager.Size))
+ return query, nil
+ },
+ GetTotalPages: func(res *http.Response, args *helper.ApiCollectorArgs) (int, errors.Error) {
+ var body struct {
+ SizeInfo models.ApiBambooSizeData `json:"results"`
+ }
+ err = helper.UnmarshalResponse(res, &body)
+ if err != nil {
+ return 0, err
+ }
+ return GetTotalPagesFromSizeInfo(&body.SizeInfo, args)
+ },
+
+ ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) {
+ var resData struct {
+ Results struct {
+ Result []json.RawMessage `json:"result"`
+ } `json:"results"`
+ }
+ err = helper.UnmarshalResponse(res, &resData)
+ if err != nil {
+ return nil, err
+ }
+ return resData.Results.Result, nil
+ },
+ })
+ if err != nil {
+ return err
+ }
+ return collectorWithState.Execute()
+}
+
+var CollectPlanBuildMeta = plugin.SubTaskMeta{
+ Name: "CollectPlanBuild",
+ EntryPoint: CollectPlanBuild,
+ EnabledByDefault: true,
+ Description: "Collect PlanBuild data from Bamboo api",
+ DomainTypes: []string{plugin.DOMAIN_TYPE_CICD},
+}
diff --git a/backend/plugins/bamboo/tasks/plan_build_extractor.go b/backend/plugins/bamboo/tasks/plan_build_extractor.go
new file mode 100644
index 000000000..6c0b8d3ae
--- /dev/null
+++ b/backend/plugins/bamboo/tasks/plan_build_extractor.go
@@ -0,0 +1,83 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "encoding/json"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
+ helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "github.com/apache/incubator-devlake/plugins/bamboo/models"
+)
+
+var _ plugin.SubTaskEntryPoint = ExtractPlanBuild
+
+func ExtractPlanBuild(taskCtx plugin.SubTaskContext) errors.Error {
+ rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_PLAN_BUILD_TABLE)
+
+ extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{
+ RawDataSubTaskArgs: *rawDataSubTaskArgs,
+
+ Extract: func(resData *helper.RawData) ([]interface{}, errors.Error) {
+ res := &models.ApiBambooPlanBuild{}
+ err := errors.Convert(json.Unmarshal(resData.Data, res))
+ if err != nil {
+ return nil, err
+ }
+ plan := &SimplePlan{}
+ err = errors.Convert(json.Unmarshal(resData.Input, plan))
+ if err != nil {
+ return nil, err
+ }
+ body := models.BambooPlanBuild{}.Convert(res)
+ body.ConnectionId = data.Options.ConnectionId
+ body.ProjectKey = data.Options.ProjectKey
+ body.PlanKey = plan.PlanKey
+
+ results := make([]interface{}, 0)
+ results = append(results, body)
+ // As job build can get more accuracy repo info,
+ // we can collect BambooPlanBuildVcsRevision in job_biuld_extractor
+ /*
+ for _, v := range res.VcsRevisions.VcsRevision {
+ results = append(results, &models.BambooPlanBuildVcsRevision{
+ ConnectionId: data.Options.ConnectionId,
+ PlanBuildKey: body.PlanBuildKey,
+ RepositoryId: v.RepositoryId,
+ RepositoryName: v.RepositoryName,
+ VcsRevisionKey: v.VcsRevisionKey,
+ })
+ }
+ */
+ return results, nil
+ },
+ })
+ if err != nil {
+ return err
+ }
+
+ return extractor.Execute()
+}
+
+var ExtractPlanBuildMeta = plugin.SubTaskMeta{
+ Name: "ExtractPlanBuild",
+ EntryPoint: ExtractPlanBuild,
+ EnabledByDefault: true,
+ Description: "Extract raw data into tool layer table bamboo_plan_builds",
+ DomainTypes: []string{plugin.DOMAIN_TYPE_CICD},
+}