You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by ab...@apache.org on 2022/08/03 08:19:08 UTC
[incubator-devlake] branch main updated (e00e728c -> beede034)
This is an automated email from the ASF dual-hosted git repository.
abeizn pushed a change to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
from e00e728c fix(gitlab): rollback some fields
new e3941b6b feat(jenkins): collect more fields
new beede034 feat(jenkins): modify builds
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../e2e/snapshot_tables/_tool_jenkins_builds.csv | 16 +--
plugins/jenkins/e2e/snapshot_tables/builds.csv | 16 +--
plugins/jenkins/impl/impl.go | 2 +
plugins/jenkins/models/build.go | 4 +
plugins/jenkins/models/build_repo.go | 16 +++
plugins/jenkins/models/job_dag.go | 14 +++
.../20220729_modify_all_entities.go | 113 +++++++++++++++++++++
.../jenkins/models/migrationscripts/register.go | 1 +
plugins/jenkins/models/response.go | 53 ++++++++--
.../issue_type.go => jenkins/models/stage.go} | 26 ++---
plugins/jenkins/tasks/build_collector.go | 2 +-
plugins/jenkins/tasks/build_extractor.go | 38 ++++++-
plugins/jenkins/tasks/job_collector.go | 2 +-
plugins/jenkins/tasks/job_extractor.go | 12 ++-
.../{build_collector.go => stage_collector.go} | 41 ++++----
.../tasks/{job_extractor.go => stage_extractor.go} | 40 ++++----
16 files changed, 309 insertions(+), 87 deletions(-)
create mode 100644 plugins/jenkins/models/build_repo.go
create mode 100644 plugins/jenkins/models/job_dag.go
create mode 100644 plugins/jenkins/models/migrationscripts/20220729_modify_all_entities.go
copy plugins/{jira/models/issue_type.go => jenkins/models/stage.go} (55%)
copy plugins/jenkins/tasks/{build_collector.go => stage_collector.go} (68%)
copy plugins/jenkins/tasks/{job_extractor.go => stage_extractor.go} (69%)
[incubator-devlake] 01/02: feat(jenkins): collect more fields
Posted by ab...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
abeizn pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit e3941b6b4eb89cc1492c058c5e7c55d6c61001df
Author: Yingchu Chen <yi...@merico.dev>
AuthorDate: Fri Jul 29 21:52:47 2022 +0800
feat(jenkins): collect more fields
add builds_triggered builds and up_down_jobs
closes #2637
---
.../e2e/snapshot_tables/_tool_jenkins_builds.csv | 16 +--
plugins/jenkins/e2e/snapshot_tables/builds.csv | 16 +--
plugins/jenkins/impl/impl.go | 2 +
plugins/jenkins/models/build.go | 1 +
plugins/jenkins/models/build_build.go | 14 +++
plugins/jenkins/models/build_commit_repo_url.go | 15 +++
plugins/jenkins/models/job.go | 13 ++-
.../20220729_modify_all_entities.go | 125 +++++++++++++++++++++
.../jenkins/models/migrationscripts/register.go | 1 +
plugins/jenkins/models/response.go | 44 ++++++--
plugins/jenkins/models/{build.go => stage.go} | 30 ++---
plugins/jenkins/models/up_down_job.go | 14 +++
plugins/jenkins/tasks/build_collector.go | 2 +-
plugins/jenkins/tasks/build_extractor.go | 37 +++++-
plugins/jenkins/tasks/job_collector.go | 2 +-
plugins/jenkins/tasks/job_extractor.go | 23 ++--
.../{build_collector.go => stage_collector.go} | 41 ++++---
.../tasks/{job_extractor.go => stage_extractor.go} | 40 +++----
18 files changed, 331 insertions(+), 105 deletions(-)
diff --git a/plugins/jenkins/e2e/snapshot_tables/_tool_jenkins_builds.csv b/plugins/jenkins/e2e/snapshot_tables/_tool_jenkins_builds.csv
index b61c1434..e6133952 100644
--- a/plugins/jenkins/e2e/snapshot_tables/_tool_jenkins_builds.csv
+++ b/plugins/jenkins/e2e/snapshot_tables/_tool_jenkins_builds.csv
@@ -1,9 +1,9 @@
connection_id,job_name,number,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark,duration,display_name,estimated_duration,result,timestamp,start_time,commit_sha
-1,这是一个改了名字的测试任务,1,"{""ConnectionId"":1}",_raw_jenkins_api_builds,82,,57,#1,10,SUCCESS,1650017153775,2022-04-15T10:05:53.000+00:00,
-1,这是一个改了名字的测试任务,2,"{""ConnectionId"":1}",_raw_jenkins_api_builds,81,,4,#2,10,SUCCESS,1650017177939,2022-04-15T10:06:17.000+00:00,
-1,这是一个改了名字的测试任务,3,"{""ConnectionId"":1}",_raw_jenkins_api_builds,80,,3,#3,10,SUCCESS,1650017186253,2022-04-15T10:06:26.000+00:00,
-1,这是一个改了名字的测试任务,4,"{""ConnectionId"":1}",_raw_jenkins_api_builds,79,,6,#4,10,SUCCESS,1650022556910,2022-04-15T11:35:56.000+00:00,
-1,这是一个改了名字的测试任务,5,"{""ConnectionId"":1}",_raw_jenkins_api_builds,78,,6,#5,10,SUCCESS,1650022558491,2022-04-15T11:35:58.000+00:00,
-1,这是一个改了名字的测试任务,6,"{""ConnectionId"":1}",_raw_jenkins_api_builds,77,,10,#6,10,SUCCESS,1650022560954,2022-04-15T11:36:00.000+00:00,
-1,这是一个改了名字的测试任务,7,"{""ConnectionId"":1}",_raw_jenkins_api_builds,76,,8,#7,10,SUCCESS,1650023883294,2022-04-15T11:58:03.000+00:00,
-1,这是一个改了名字的测试任务,8,"{""ConnectionId"":1}",_raw_jenkins_api_builds,75,,11,#8,10,SUCCESS,1650023894336,2022-04-15T11:58:14.000+00:00,
+1,这是一个改了名字的测试任务,1,"{""ConnectionId"":1}",_raw_jenkins_api_builds,82,,57,,10,SUCCESS,1650017153775,2022-04-15T10:05:53.000+00:00,
+1,这是一个改了名字的测试任务,2,"{""ConnectionId"":1}",_raw_jenkins_api_builds,81,,4,,10,SUCCESS,1650017177939,2022-04-15T10:06:17.000+00:00,
+1,这是一个改了名字的测试任务,3,"{""ConnectionId"":1}",_raw_jenkins_api_builds,80,,3,,10,SUCCESS,1650017186253,2022-04-15T10:06:26.000+00:00,
+1,这是一个改了名字的测试任务,4,"{""ConnectionId"":1}",_raw_jenkins_api_builds,79,,6,,10,SUCCESS,1650022556910,2022-04-15T11:35:56.000+00:00,
+1,这是一个改了名字的测试任务,5,"{""ConnectionId"":1}",_raw_jenkins_api_builds,78,,6,,10,SUCCESS,1650022558491,2022-04-15T11:35:58.000+00:00,
+1,这是一个改了名字的测试任务,6,"{""ConnectionId"":1}",_raw_jenkins_api_builds,77,,10,,10,SUCCESS,1650022560954,2022-04-15T11:36:00.000+00:00,
+1,这是一个改了名字的测试任务,7,"{""ConnectionId"":1}",_raw_jenkins_api_builds,76,,8,,10,SUCCESS,1650023883294,2022-04-15T11:58:03.000+00:00,
+1,这是一个改了名字的测试任务,8,"{""ConnectionId"":1}",_raw_jenkins_api_builds,75,,11,,10,SUCCESS,1650023894336,2022-04-15T11:58:14.000+00:00,
diff --git a/plugins/jenkins/e2e/snapshot_tables/builds.csv b/plugins/jenkins/e2e/snapshot_tables/builds.csv
index 38476b9b..a2b50a07 100644
--- a/plugins/jenkins/e2e/snapshot_tables/builds.csv
+++ b/plugins/jenkins/e2e/snapshot_tables/builds.csv
@@ -1,9 +1,9 @@
id,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark,job_id,name,commit_sha,duration_sec,status,started_date
-jenkins:JenkinsBuild:1:这是一个改了名字的测试任务:1,"{""ConnectionId"":1}",_raw_jenkins_api_builds,82,,jenkins:JenkinsJob:1:这是一个改了名字的测试任务,#1,,0,SUCCESS,2022-04-15T10:05:53.000+00:00
-jenkins:JenkinsBuild:1:这是一个改了名字的测试任务:2,"{""ConnectionId"":1}",_raw_jenkins_api_builds,81,,jenkins:JenkinsJob:1:这是一个改了名字的测试任务,#2,,0,SUCCESS,2022-04-15T10:06:17.000+00:00
-jenkins:JenkinsBuild:1:这是一个改了名字的测试任务:3,"{""ConnectionId"":1}",_raw_jenkins_api_builds,80,,jenkins:JenkinsJob:1:这是一个改了名字的测试任务,#3,,0,SUCCESS,2022-04-15T10:06:26.000+00:00
-jenkins:JenkinsBuild:1:这是一个改了名字的测试任务:4,"{""ConnectionId"":1}",_raw_jenkins_api_builds,79,,jenkins:JenkinsJob:1:这是一个改了名字的测试任务,#4,,0,SUCCESS,2022-04-15T11:35:56.000+00:00
-jenkins:JenkinsBuild:1:这是一个改了名字的测试任务:5,"{""ConnectionId"":1}",_raw_jenkins_api_builds,78,,jenkins:JenkinsJob:1:这是一个改了名字的测试任务,#5,,0,SUCCESS,2022-04-15T11:35:58.000+00:00
-jenkins:JenkinsBuild:1:这是一个改了名字的测试任务:6,"{""ConnectionId"":1}",_raw_jenkins_api_builds,77,,jenkins:JenkinsJob:1:这是一个改了名字的测试任务,#6,,0,SUCCESS,2022-04-15T11:36:00.000+00:00
-jenkins:JenkinsBuild:1:这是一个改了名字的测试任务:7,"{""ConnectionId"":1}",_raw_jenkins_api_builds,76,,jenkins:JenkinsJob:1:这是一个改了名字的测试任务,#7,,0,SUCCESS,2022-04-15T11:58:03.000+00:00
-jenkins:JenkinsBuild:1:这是一个改了名字的测试任务:8,"{""ConnectionId"":1}",_raw_jenkins_api_builds,75,,jenkins:JenkinsJob:1:这是一个改了名字的测试任务,#8,,0,SUCCESS,2022-04-15T11:58:14.000+00:00
+jenkins:JenkinsBuild:1:这是一个改了名字的测试任务:1,"{""ConnectionId"":1}",_raw_jenkins_api_builds,82,,jenkins:JenkinsJob:1:这是一个改了名字的测试任务,,,0,SUCCESS,2022-04-15T10:05:53.000+00:00
+jenkins:JenkinsBuild:1:这是一个改了名字的测试任务:2,"{""ConnectionId"":1}",_raw_jenkins_api_builds,81,,jenkins:JenkinsJob:1:这是一个改了名字的测试任务,,,0,SUCCESS,2022-04-15T10:06:17.000+00:00
+jenkins:JenkinsBuild:1:这是一个改了名字的测试任务:3,"{""ConnectionId"":1}",_raw_jenkins_api_builds,80,,jenkins:JenkinsJob:1:这是一个改了名字的测试任务,,,0,SUCCESS,2022-04-15T10:06:26.000+00:00
+jenkins:JenkinsBuild:1:这是一个改了名字的测试任务:4,"{""ConnectionId"":1}",_raw_jenkins_api_builds,79,,jenkins:JenkinsJob:1:这是一个改了名字的测试任务,,,0,SUCCESS,2022-04-15T11:35:56.000+00:00
+jenkins:JenkinsBuild:1:这是一个改了名字的测试任务:5,"{""ConnectionId"":1}",_raw_jenkins_api_builds,78,,jenkins:JenkinsJob:1:这是一个改了名字的测试任务,,,0,SUCCESS,2022-04-15T11:35:58.000+00:00
+jenkins:JenkinsBuild:1:这是一个改了名字的测试任务:6,"{""ConnectionId"":1}",_raw_jenkins_api_builds,77,,jenkins:JenkinsJob:1:这是一个改了名字的测试任务,,,0,SUCCESS,2022-04-15T11:36:00.000+00:00
+jenkins:JenkinsBuild:1:这是一个改了名字的测试任务:7,"{""ConnectionId"":1}",_raw_jenkins_api_builds,76,,jenkins:JenkinsJob:1:这是一个改了名字的测试任务,,,0,SUCCESS,2022-04-15T11:58:03.000+00:00
+jenkins:JenkinsBuild:1:这是一个改了名字的测试任务:8,"{""ConnectionId"":1}",_raw_jenkins_api_builds,75,,jenkins:JenkinsJob:1:这是一个改了名字的测试任务,,,0,SUCCESS,2022-04-15T11:58:14.000+00:00
diff --git a/plugins/jenkins/impl/impl.go b/plugins/jenkins/impl/impl.go
index d9d6f92d..167a7355 100644
--- a/plugins/jenkins/impl/impl.go
+++ b/plugins/jenkins/impl/impl.go
@@ -55,6 +55,8 @@ func (plugin Jenkins) SubTaskMetas() []core.SubTaskMeta {
tasks.ExtractApiJobsMeta,
tasks.CollectApiBuildsMeta,
tasks.ExtractApiBuildsMeta,
+ tasks.CollectApiStagesMeta,
+ tasks.ExtractApiStagesMeta,
tasks.ConvertJobsMeta,
tasks.ConvertBuildsMeta,
}
diff --git a/plugins/jenkins/models/build.go b/plugins/jenkins/models/build.go
index 98d4b974..6506d9ef 100644
--- a/plugins/jenkins/models/build.go
+++ b/plugins/jenkins/models/build.go
@@ -38,6 +38,7 @@ type JenkinsBuild struct {
Timestamp int64 // start time
StartTime time.Time // convered by timestamp
CommitSha string `gorm:"type:varchar(255)"`
+ Type string `gorm:"index;type:varchar(255)"`
}
func (JenkinsBuild) TableName() string {
diff --git a/plugins/jenkins/models/build_build.go b/plugins/jenkins/models/build_build.go
new file mode 100644
index 00000000..be6e2500
--- /dev/null
+++ b/plugins/jenkins/models/build_build.go
@@ -0,0 +1,14 @@
+package models
+
+import "github.com/apache/incubator-devlake/models/common"
+
+type JenkinsBuildTriggeredBuilds struct {
+ ConnectionId uint64 `gorm:"primaryKey"`
+ BuildName string `gorm:"primaryKey;type:varchar(255)"`
+ TriggeredBuildName string `gorm:"primaryKey;type:varchar(255)"`
+ common.NoPKModel
+}
+
+func (JenkinsBuildTriggeredBuilds) TableName() string {
+ return "_tool_jenkins_build_triggered_builds"
+}
diff --git a/plugins/jenkins/models/build_commit_repo_url.go b/plugins/jenkins/models/build_commit_repo_url.go
new file mode 100644
index 00000000..de81ba38
--- /dev/null
+++ b/plugins/jenkins/models/build_commit_repo_url.go
@@ -0,0 +1,15 @@
+package models
+
+import "github.com/apache/incubator-devlake/models/common"
+
+type JenkinsBuildCommitRepoUrl struct {
+ ConnectionId uint64 `gorm:"primaryKey"`
+ BuildName string `gorm:"primaryKey;type:varchar(255)"`
+ CommitSha string `gorm:"primaryKey;type:varchar(255)"`
+ RemoteUrl string `gorm:"primaryKey;type:varchar(255)"`
+ common.NoPKModel
+}
+
+func (JenkinsBuildCommitRepoUrl) TableName() string {
+ return "_tool_jenkins_build_commit_repo_urls"
+}
diff --git a/plugins/jenkins/models/job.go b/plugins/jenkins/models/job.go
index 43687ce2..f525343b 100644
--- a/plugins/jenkins/models/job.go
+++ b/plugins/jenkins/models/job.go
@@ -25,12 +25,13 @@ import (
// JenkinsJobProps current used jenkins job props
type JenkinsJobProps struct {
// collected fields
- ConnectionId uint64 `gorm:"primaryKey"`
- Name string `gorm:"primaryKey;type:varchar(255)"`
- Path string `gorm:"primaryKey;type:varchar(511)"`
- Class string `gorm:"type:varchar(255)"`
- Color string `gorm:"type:varchar(255)"`
- Base string `gorm:"type:varchar(255)"`
+ ConnectionId uint64 `gorm:"primaryKey"`
+ Name string `gorm:"primaryKey;type:varchar(255)"`
+ Path string `gorm:"primaryKey;type:varchar(511)"`
+ Class string `gorm:"type:varchar(255)"`
+ Color string `gorm:"type:varchar(255)"`
+ Base string `gorm:"type:varchar(255)"`
+ HasUpstreamProjects bool
}
// JenkinsJob db entity for jenkins job
diff --git a/plugins/jenkins/models/migrationscripts/20220729_modify_all_entities.go b/plugins/jenkins/models/migrationscripts/20220729_modify_all_entities.go
new file mode 100644
index 00000000..43d8b4e7
--- /dev/null
+++ b/plugins/jenkins/models/migrationscripts/20220729_modify_all_entities.go
@@ -0,0 +1,125 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+ "context"
+ "github.com/apache/incubator-devlake/models/common"
+ "github.com/apache/incubator-devlake/models/migrationscripts/archived"
+ "gorm.io/gorm"
+)
+
+type modifyAllEntities struct{}
+
+// JenkinsBuild db entity for jenkins build
+type JenkinsBuild0729 struct {
+ Type string `gorm:"index;type:varchar(255)" `
+}
+
+func (JenkinsBuild0729) TableName() string {
+ return "_tool_jenkins_builds"
+}
+
+type JenkinsJob0729 struct {
+ HasUpstreamProjects bool
+}
+
+func (JenkinsJob0729) TableName() string {
+ return "_tool_jenkins_jobs"
+}
+
+type JenkinsUpDownJob0729 struct {
+ ConnetionId uint64 `gorm:"primaryKey"`
+ UpstreamJob string `gorm:"primaryKey;type:varchar(255)"`
+ DownstreamJob string `gorm:"primaryKey;type:varchar(255)"`
+ archived.NoPKModel
+}
+
+func (JenkinsUpDownJob0729) TableName() string {
+ return "_tool_jenkins_up_down_jobs"
+}
+
+type JenkinsBuildCommitRepoUrl0729 struct {
+ ConnectionId uint64 `gorm:"primaryKey"`
+ BuildName string `gorm:"primaryKey;type:varchar(255)"`
+ CommitSha string `gorm:"primaryKey;type:varchar(255)"`
+ RemoteUrl string `gorm:"primaryKey;type:varchar(255)"`
+ common.NoPKModel
+}
+
+func (JenkinsBuildCommitRepoUrl0729) TableName() string {
+ return "_tool_jenkins_build_commit_repo_urls"
+}
+
+type JenkinsBuildTriggeredBuilds0729 struct {
+ ConnectionId uint64 `gorm:"primaryKey"`
+ BuildName string `gorm:"primaryKey;type:varchar(255)"`
+ TriggeredBuildName string `gorm:"primaryKey;type:varchar(255)"`
+ archived.NoPKModel
+}
+
+func (JenkinsBuildTriggeredBuilds0729) TableName() string {
+ return "_tool_jenkins_build_triggered_builds"
+}
+
+type JenkinsStage0729 struct {
+ archived.NoPKModel
+ ConnectionId uint64 `gorm:"primaryKey"`
+ ID string `json:"id" gorm:"primaryKey;type:varchar(255)"`
+ Name string `json:"name" gorm:"type:varchar(255)"`
+ ExecNode string `json:"execNode" gorm:"type:varchar(255)"`
+ Status string `json:"status" gorm:"type:varchar(255)"`
+ StartTimeMillis int64 `json:"startTimeMillis"`
+ DurationMillis int `json:"durationMillis"`
+ PauseDurationMillis int `json:"pauseDurationMillis"`
+ BuildName string `gorm:"primaryKey;type:varchar(255)"`
+}
+
+func (JenkinsStage0729) TableName() string {
+ return "_tool_jenkins_stages"
+}
+
+func (*modifyAllEntities) Up(ctx context.Context, db *gorm.DB) error {
+ err := db.Migrator().AddColumn(JenkinsBuild0729{}, "type")
+ if err != nil {
+ return err
+ }
+ err = db.Migrator().AddColumn(JenkinsJob0729{}, "has_upstream_projects")
+ if err != nil {
+ return err
+ }
+ err = db.Migrator().AutoMigrate(
+ JenkinsUpDownJob0729{},
+ JenkinsBuildCommitRepoUrl0729{},
+ JenkinsBuildTriggeredBuilds0729{},
+ JenkinsStage0729{},
+ )
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func (*modifyAllEntities) Version() uint64 {
+ return 20220729231237
+}
+
+func (*modifyAllEntities) Name() string {
+ return "Jenkins modify build and job"
+}
diff --git a/plugins/jenkins/models/migrationscripts/register.go b/plugins/jenkins/models/migrationscripts/register.go
index c1365f7d..63eb2733 100644
--- a/plugins/jenkins/models/migrationscripts/register.go
+++ b/plugins/jenkins/models/migrationscripts/register.go
@@ -25,5 +25,6 @@ import (
func All() []migration.Script {
return []migration.Script{
new(addInitTables),
+ new(modifyAllEntities),
}
}
diff --git a/plugins/jenkins/models/response.go b/plugins/jenkins/models/response.go
index ace21939..3171bf27 100644
--- a/plugins/jenkins/models/response.go
+++ b/plugins/jenkins/models/response.go
@@ -37,12 +37,19 @@ type ApiResponse struct {
NodeDescription string `json:"nodeDescription"`
}
type Job struct {
- URL string `json:"url"`
- Name string `json:"name"`
- Color string `json:"color"`
+ URL string `json:"url"`
+ Name string `json:"name"`
+ Color string `json:"color"`
+ Class string `json:"_class"`
+ Jobs *[]Job `json:"jobs"`
+ UpstreamProjects []Project `json:"upstreamProjects"`
+}
+
+type Project struct {
Class string `json:"_class"`
- Jobs *[]Job `json:"jobs"`
+ Name string `json:"name"`
}
+
type Views struct {
URL string `json:"url"`
Name string `json:"name"`
@@ -66,10 +73,10 @@ type ApiBuildResponse struct {
Class string `json:"_class"`
Number int64 `json:"number"`
Result string `json:"result"`
- Actions []Actions `json:"actions"`
+ Actions []Action `json:"actions"`
Duration float64 `json:"duration"`
Timestamp int64 `json:"timestamp"`
- DisplayName string `json:"displayName"`
+ DisplayName string `json:"fullDisplayName"`
EstimatedDuration float64 `json:"estimatedDuration"`
ChangeSet ChangeSet `json:"changeSet"`
}
@@ -77,10 +84,12 @@ type LastBuiltRevision struct {
SHA1 string `json:"SHA1"`
}
-type Actions struct {
- Class string `json:"_class,omitempty"`
- LastBuiltRevision LastBuiltRevision `json:"lastBuiltRevision,omitempty"`
- MercurialRevisionNumber string `json:"mercurialRevisionNumber"`
+type Action struct {
+ Class string `json:"_class,omitempty"`
+ LastBuiltRevision LastBuiltRevision `json:"lastBuiltRevision,omitempty"`
+ MercurialRevisionNumber string `json:"mercurialRevisionNumber"`
+ RemoteUrls []string `json:"remoteUrls"`
+ TriggeredBuilds []ApiBuildResponse `json:"triggeredBuilds"`
}
type ChangeSet struct {
Class string `json:"_class"`
@@ -92,3 +101,18 @@ type Revision struct {
Module string
Revision int
}
+
+type Stage struct {
+ Links struct {
+ Self struct {
+ Href string `json:"href"`
+ } `json:"self"`
+ } `json:"_links"`
+ ID string `json:"id"`
+ Name string `json:"name"`
+ ExecNode string `json:"execNode"`
+ Status string `json:"status"`
+ StartTimeMillis int64 `json:"startTimeMillis"`
+ DurationMillis int `json:"durationMillis"`
+ PauseDurationMillis int `json:"pauseDurationMillis"`
+}
diff --git a/plugins/jenkins/models/build.go b/plugins/jenkins/models/stage.go
similarity index 55%
copy from plugins/jenkins/models/build.go
copy to plugins/jenkins/models/stage.go
index 98d4b974..a8654d90 100644
--- a/plugins/jenkins/models/build.go
+++ b/plugins/jenkins/models/stage.go
@@ -18,28 +18,22 @@ limitations under the License.
package models
import (
- "time"
-
"github.com/apache/incubator-devlake/models/common"
)
-// JenkinsBuild db entity for jenkins build
-type JenkinsBuild struct {
+type JenkinsStage struct {
common.NoPKModel
-
- // collected fields
- ConnectionId uint64 `gorm:"primaryKey"`
- JobName string `gorm:"primaryKey;type:varchar(255)"`
- Duration float64 // build time
- DisplayName string `gorm:"type:varchar(255)"` // "#7"
- EstimatedDuration float64 // EstimatedDuration
- Number int64 `gorm:"primaryKey"`
- Result string // Result
- Timestamp int64 // start time
- StartTime time.Time // convered by timestamp
- CommitSha string `gorm:"type:varchar(255)"`
+ ConnectionId uint64 `gorm:"primaryKey"`
+ ID string `json:"id" gorm:"primaryKey;type:varchar(255)"`
+ Name string `json:"name" gorm:"type:varchar(255)"`
+ ExecNode string `json:"execNode" gorm:"type:varchar(255)"`
+ Status string `json:"status" gorm:"type:varchar(255)"`
+ StartTimeMillis int64 `json:"startTimeMillis"`
+ DurationMillis int `json:"durationMillis"`
+ PauseDurationMillis int `json:"pauseDurationMillis"`
+ BuildName string `gorm:"primaryKey;type:varchar(255)"`
}
-func (JenkinsBuild) TableName() string {
- return "_tool_jenkins_builds"
+func (JenkinsStage) TableName() string {
+ return "_tool_jenkins_stages"
}
diff --git a/plugins/jenkins/models/up_down_job.go b/plugins/jenkins/models/up_down_job.go
new file mode 100644
index 00000000..fcca32f2
--- /dev/null
+++ b/plugins/jenkins/models/up_down_job.go
@@ -0,0 +1,14 @@
+package models
+
+import "github.com/apache/incubator-devlake/models/common"
+
+type JenkinsUpDownJob struct {
+ ConnetionId uint64 `gorm:"primaryKey"`
+ UpstreamJob string `gorm:"primaryKey;type:varchar(255)"`
+ DownstreamJob string `gorm:"primaryKey;type:varchar(255)"`
+ common.NoPKModel
+}
+
+func (JenkinsUpDownJob) TableName() string {
+ return "_tool_jenkins_up_down_jobs"
+}
diff --git a/plugins/jenkins/tasks/build_collector.go b/plugins/jenkins/tasks/build_collector.go
index dc601421..86877c24 100644
--- a/plugins/jenkins/tasks/build_collector.go
+++ b/plugins/jenkins/tasks/build_collector.go
@@ -82,7 +82,7 @@ func CollectApiBuilds(taskCtx core.SubTaskContext) error {
Query: func(reqData *helper.RequestData) (url.Values, error) {
query := url.Values{}
treeValue := fmt.Sprintf(
- "allBuilds[number,timestamp,duration,estimatedDuration,displayName,result,actions[lastBuiltRevision[SHA1],mercurialRevisionNumber],changeSet[kind,revisions[revision]]]{%d,%d}",
+ "allBuilds[number,timestamp,duration,estimatedDuration,fullDisplayName,result,actions[lastBuiltRevision[SHA1],remoteUrls,mercurialRevisionNumber,triggeredBuilds[fullDisplayName,number,url,result,status,duration]],changeSet[kind,revisions[revision]]]{%d,%d}",
reqData.Pager.Skip, reqData.Pager.Skip+reqData.Pager.Size)
query.Set("tree", treeValue)
return query, nil
diff --git a/plugins/jenkins/tasks/build_extractor.go b/plugins/jenkins/tasks/build_extractor.go
index bfc8e2ed..66a818d4 100644
--- a/plugins/jenkins/tasks/build_extractor.go
+++ b/plugins/jenkins/tasks/build_extractor.go
@@ -20,6 +20,7 @@ package tasks
import (
"encoding/json"
"strconv"
+ "strings"
"time"
"github.com/apache/incubator-devlake/plugins/core"
@@ -66,10 +67,12 @@ func ExtractApiBuilds(taskCtx core.SubTaskContext) error {
return nil, err
}
- results := make([]interface{}, 0, 1)
-
+ results := make([]interface{}, 0)
+ strList := strings.Split(body.Class, ".")
+ class := strList[len(strList)-1]
build := &models.JenkinsBuild{
ConnectionId: data.Options.ConnectionId,
+ Type: class,
JobName: input.Name,
Duration: body.Duration,
DisplayName: body.DisplayName,
@@ -82,11 +85,37 @@ func ExtractApiBuilds(taskCtx core.SubTaskContext) error {
vcs := body.ChangeSet.Kind
if vcs == "git" || vcs == "hg" {
for _, a := range body.Actions {
+ sha := ""
if a.LastBuiltRevision.SHA1 != "" {
- build.CommitSha = a.LastBuiltRevision.SHA1
+ sha = a.LastBuiltRevision.SHA1
}
if a.MercurialRevisionNumber != "" {
- build.CommitSha = a.MercurialRevisionNumber
+ sha = a.MercurialRevisionNumber
+ }
+ build.CommitSha = sha
+ for _, url := range a.RemoteUrls {
+ if url != "" {
+ buildCommitRemoteUrl := models.JenkinsBuildCommitRepoUrl{
+ ConnectionId: data.Options.ConnectionId,
+ BuildName: build.DisplayName,
+ CommitSha: sha,
+ RemoteUrl: url,
+ }
+ results = append(results, &buildCommitRemoteUrl)
+ }
+ }
+ if a.TriggeredBuilds != nil && len(a.TriggeredBuilds) > 0 {
+ for _, b := range a.TriggeredBuilds {
+ if b.DisplayName == "" {
+ continue
+ }
+ buildTrigger := models.JenkinsBuildTriggeredBuilds{
+ ConnectionId: data.Options.ConnectionId,
+ BuildName: build.DisplayName,
+ TriggeredBuildName: b.DisplayName,
+ }
+ results = append(results, &buildTrigger)
+ }
}
}
} else if vcs == "svn" {
diff --git a/plugins/jenkins/tasks/job_collector.go b/plugins/jenkins/tasks/job_collector.go
index 59abde31..dd5b7c2e 100644
--- a/plugins/jenkins/tasks/job_collector.go
+++ b/plugins/jenkins/tasks/job_collector.go
@@ -69,7 +69,7 @@ func CollectApiJobs(taskCtx core.SubTaskContext) error {
Query: func(reqData *helper.RequestData) (url.Values, error) {
query := url.Values{}
treeValue := fmt.Sprintf(
- "jobs[name,class,url,color,base,jobs]{%d,%d}",
+ "jobs[name,class,url,color,base,jobs,upstreamProjects[name]]{%d,%d}",
reqData.Pager.Skip, reqData.Pager.Skip+reqData.Pager.Size)
query.Set("tree", treeValue)
return query, nil
diff --git a/plugins/jenkins/tasks/job_extractor.go b/plugins/jenkins/tasks/job_extractor.go
index 1955c8fb..f78742be 100644
--- a/plugins/jenkins/tasks/job_extractor.go
+++ b/plugins/jenkins/tasks/job_extractor.go
@@ -19,7 +19,6 @@ package tasks
import (
"encoding/json"
-
"github.com/apache/incubator-devlake/plugins/core"
"github.com/apache/incubator-devlake/plugins/helper"
"github.com/apache/incubator-devlake/plugins/jenkins/models"
@@ -65,17 +64,27 @@ func ExtractApiJobs(taskCtx core.SubTaskContext) error {
return nil, err
}
- results := make([]interface{}, 0, 1)
+ results := make([]interface{}, 0, 1+len(body.UpstreamProjects))
job := &models.JenkinsJob{
JenkinsJobProps: models.JenkinsJobProps{
- ConnectionId: data.Options.ConnectionId,
- Name: body.Name,
- Path: input.Path,
- Class: body.Class,
- Color: body.Color,
+ ConnectionId: data.Options.ConnectionId,
+ Name: body.Name,
+ Path: input.Path,
+ Class: body.Class,
+ Color: body.Color,
+ HasUpstreamProjects: len(body.UpstreamProjects) > 0,
},
}
+ for _, upstreamProject := range body.UpstreamProjects {
+ upDownJob := models.JenkinsUpDownJob{
+ ConnetionId: data.Options.ConnectionId,
+ UpstreamJob: upstreamProject.Name,
+ DownstreamJob: job.Name,
+ }
+ results = append(results, &upDownJob)
+ }
+
results = append(results, job)
return results, nil
diff --git a/plugins/jenkins/tasks/build_collector.go b/plugins/jenkins/tasks/stage_collector.go
similarity index 68%
copy from plugins/jenkins/tasks/build_collector.go
copy to plugins/jenkins/tasks/stage_collector.go
index dc601421..19f6a83f 100644
--- a/plugins/jenkins/tasks/build_collector.go
+++ b/plugins/jenkins/tasks/stage_collector.go
@@ -19,7 +19,6 @@ package tasks
import (
"encoding/json"
- "fmt"
"net/http"
"net/url"
"reflect"
@@ -29,28 +28,30 @@ import (
"github.com/apache/incubator-devlake/plugins/helper"
)
-const RAW_BUILD_TABLE = "jenkins_api_builds"
+const RAW_STAGE_TABLE = "jenkins_api_stages"
-var CollectApiBuildsMeta = core.SubTaskMeta{
- Name: "collectApiBuilds",
- EntryPoint: CollectApiBuilds,
+var CollectApiStagesMeta = core.SubTaskMeta{
+ Name: "collectApiStages",
+ EntryPoint: CollectApiStages,
EnabledByDefault: true,
- Description: "Collect builds data from jenkins api",
+ Description: "Collect stages data from jenkins api",
DomainTypes: []string{core.DOMAIN_TYPE_CICD},
}
-type SimpleJob struct {
- Name string
- Path string
+type SimpleBuild struct {
+ JobName string
+ Number string
+ DisplayName string
}
-func CollectApiBuilds(taskCtx core.SubTaskContext) error {
+func CollectApiStages(taskCtx core.SubTaskContext) error {
db := taskCtx.GetDal()
data := taskCtx.GetData().(*JenkinsTaskData)
clauses := []dal.Clause{
- dal.Select("tjj.name,tjj.path"),
- dal.From("_tool_jenkins_jobs tjj"),
- dal.Where(`tjj.connection_id = ?`, data.Options.ConnectionId),
+ dal.Select("tjb.job_name,tjb.number, tjb.display_name"),
+ dal.From("_tool_jenkins_builds tjb"),
+ dal.Where(`tjb.connection_id = ? and tjb.type = ?`,
+ data.Options.ConnectionId, "WorkflowRun"),
}
cursor, err := db.Cursor(clauses...)
@@ -59,7 +60,7 @@ func CollectApiBuilds(taskCtx core.SubTaskContext) error {
}
defer cursor.Close()
- iterator, err := helper.NewDalCursorIterator(db, cursor, reflect.TypeOf(SimpleJob{}))
+ iterator, err := helper.NewDalCursorIterator(db, cursor, reflect.TypeOf(SimpleBuild{}))
if err != nil {
return err
}
@@ -70,32 +71,28 @@ func CollectApiBuilds(taskCtx core.SubTaskContext) error {
ConnectionId: data.Options.ConnectionId,
},
Ctx: taskCtx,
- Table: RAW_BUILD_TABLE,
+ Table: RAW_STAGE_TABLE,
},
ApiClient: data.ApiClient,
PageSize: 100,
Input: iterator,
- UrlTemplate: "{{ .Input.Path }}job/{{ .Input.Name }}/api/json",
+ UrlTemplate: "job/{{ .Input.JobName }}/{{ .Input.Number }}/wfapi/describe",
/*
(Optional) Return query string for request, or you can plug them into UrlTemplate directly
*/
Query: func(reqData *helper.RequestData) (url.Values, error) {
query := url.Values{}
- treeValue := fmt.Sprintf(
- "allBuilds[number,timestamp,duration,estimatedDuration,displayName,result,actions[lastBuiltRevision[SHA1],mercurialRevisionNumber],changeSet[kind,revisions[revision]]]{%d,%d}",
- reqData.Pager.Skip, reqData.Pager.Skip+reqData.Pager.Size)
- query.Set("tree", treeValue)
return query, nil
},
ResponseParser: func(res *http.Response) ([]json.RawMessage, error) {
var data struct {
- Builds []json.RawMessage `json:"allBuilds"`
+ Stages []json.RawMessage `json:"stages"`
}
err := helper.UnmarshalResponse(res, &data)
if err != nil {
return nil, err
}
- return data.Builds, nil
+ return data.Stages, nil
},
})
diff --git a/plugins/jenkins/tasks/job_extractor.go b/plugins/jenkins/tasks/stage_extractor.go
similarity index 69%
copy from plugins/jenkins/tasks/job_extractor.go
copy to plugins/jenkins/tasks/stage_extractor.go
index 1955c8fb..cbe0634f 100644
--- a/plugins/jenkins/tasks/job_extractor.go
+++ b/plugins/jenkins/tasks/stage_extractor.go
@@ -19,7 +19,6 @@ package tasks
import (
"encoding/json"
-
"github.com/apache/incubator-devlake/plugins/core"
"github.com/apache/incubator-devlake/plugins/helper"
"github.com/apache/incubator-devlake/plugins/jenkins/models"
@@ -27,15 +26,15 @@ import (
// this struct should be moved to `gitub_api_common.go`
-var ExtractApiJobsMeta = core.SubTaskMeta{
- Name: "extractApiJobs",
- EntryPoint: ExtractApiJobs,
+var ExtractApiStagesMeta = core.SubTaskMeta{
+ Name: "extractApiStages",
+ EntryPoint: ExtractApiStages,
EnabledByDefault: true,
- Description: "Extract raw jobs data into tool layer table jenkins_jobs",
+ Description: "Extract raw stages data into tool layer table jenkins_stages",
DomainTypes: []string{core.DOMAIN_TYPE_CICD},
}
-func ExtractApiJobs(taskCtx core.SubTaskContext) error {
+func ExtractApiStages(taskCtx core.SubTaskContext) error {
data := taskCtx.GetData().(*JenkinsTaskData)
extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{
RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
@@ -50,34 +49,35 @@ func ExtractApiJobs(taskCtx core.SubTaskContext) error {
/*
Table store raw data
*/
- Table: RAW_JOB_TABLE,
+ Table: RAW_STAGE_TABLE,
},
Extract: func(row *helper.RawData) ([]interface{}, error) {
- body := &models.Job{}
+ body := &models.Stage{}
err := json.Unmarshal(row.Data, body)
if err != nil {
return nil, err
}
-
- input := &models.FolderInput{}
+ input := &SimpleBuild{}
err = json.Unmarshal(row.Input, input)
if err != nil {
return nil, err
}
- results := make([]interface{}, 0, 1)
+ results := make([]interface{}, 0)
- job := &models.JenkinsJob{
- JenkinsJobProps: models.JenkinsJobProps{
- ConnectionId: data.Options.ConnectionId,
- Name: body.Name,
- Path: input.Path,
- Class: body.Class,
- Color: body.Color,
- },
+ stage := &models.JenkinsStage{
+ ConnectionId: data.Options.ConnectionId,
+ ID: body.ID,
+ Name: body.Name,
+ ExecNode: body.ExecNode,
+ Status: body.Status,
+ StartTimeMillis: body.StartTimeMillis,
+ DurationMillis: body.DurationMillis,
+ PauseDurationMillis: body.PauseDurationMillis,
+ BuildName: input.DisplayName,
}
- results = append(results, job)
+ results = append(results, stage)
return results, nil
},
})
[incubator-devlake] 02/02: feat(jenkins): modify builds
Posted by ab...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
abeizn pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit beede0343119cdd7ecf614959fb78be2a3888291
Author: Yingchu Chen <yi...@merico.dev>
AuthorDate: Mon Aug 1 21:35:44 2022 +0800
feat(jenkins): modify builds
---
plugins/jenkins/models/build.go | 3 +
plugins/jenkins/models/build_build.go | 14 -----
.../{build_commit_repo_url.go => build_repo.go} | 9 +--
plugins/jenkins/models/job.go | 13 ++---
.../jenkins/models/{up_down_job.go => job_dag.go} | 6 +-
.../20220729_modify_all_entities.go | 66 +++++++++-------------
plugins/jenkins/models/response.go | 25 ++++++--
plugins/jenkins/models/stage.go | 1 +
plugins/jenkins/tasks/build_collector.go | 2 +-
plugins/jenkins/tasks/build_extractor.go | 27 ++++-----
plugins/jenkins/tasks/job_extractor.go | 13 ++---
11 files changed, 85 insertions(+), 94 deletions(-)
diff --git a/plugins/jenkins/models/build.go b/plugins/jenkins/models/build.go
index 6506d9ef..f3af1a10 100644
--- a/plugins/jenkins/models/build.go
+++ b/plugins/jenkins/models/build.go
@@ -39,6 +39,9 @@ type JenkinsBuild struct {
StartTime time.Time // convered by timestamp
CommitSha string `gorm:"type:varchar(255)"`
Type string `gorm:"index;type:varchar(255)"`
+ Class string `gorm:"index;type:varchar(255)" `
+ TriggeredBy string `gorm:"type:varchar(255)"`
+ Building bool
}
func (JenkinsBuild) TableName() string {
diff --git a/plugins/jenkins/models/build_build.go b/plugins/jenkins/models/build_build.go
deleted file mode 100644
index be6e2500..00000000
--- a/plugins/jenkins/models/build_build.go
+++ /dev/null
@@ -1,14 +0,0 @@
-package models
-
-import "github.com/apache/incubator-devlake/models/common"
-
-type JenkinsBuildTriggeredBuilds struct {
- ConnectionId uint64 `gorm:"primaryKey"`
- BuildName string `gorm:"primaryKey;type:varchar(255)"`
- TriggeredBuildName string `gorm:"primaryKey;type:varchar(255)"`
- common.NoPKModel
-}
-
-func (JenkinsBuildTriggeredBuilds) TableName() string {
- return "_tool_jenkins_build_triggered_builds"
-}
diff --git a/plugins/jenkins/models/build_commit_repo_url.go b/plugins/jenkins/models/build_repo.go
similarity index 54%
rename from plugins/jenkins/models/build_commit_repo_url.go
rename to plugins/jenkins/models/build_repo.go
index de81ba38..7f26c54a 100644
--- a/plugins/jenkins/models/build_commit_repo_url.go
+++ b/plugins/jenkins/models/build_repo.go
@@ -2,14 +2,15 @@ package models
import "github.com/apache/incubator-devlake/models/common"
-type JenkinsBuildCommitRepoUrl struct {
+type JenkinsBuildRepo struct {
ConnectionId uint64 `gorm:"primaryKey"`
BuildName string `gorm:"primaryKey;type:varchar(255)"`
CommitSha string `gorm:"primaryKey;type:varchar(255)"`
- RemoteUrl string `gorm:"primaryKey;type:varchar(255)"`
+ Branch string `gorm:"type:varchar(255)"`
+ RepoUrl string `gorm:"primaryKey;type:varchar(255)"`
common.NoPKModel
}
-func (JenkinsBuildCommitRepoUrl) TableName() string {
- return "_tool_jenkins_build_commit_repo_urls"
+func (JenkinsBuildRepo) TableName() string {
+ return "_tool_jenkins_build_repos"
}
diff --git a/plugins/jenkins/models/job.go b/plugins/jenkins/models/job.go
index f525343b..43687ce2 100644
--- a/plugins/jenkins/models/job.go
+++ b/plugins/jenkins/models/job.go
@@ -25,13 +25,12 @@ import (
// JenkinsJobProps current used jenkins job props
type JenkinsJobProps struct {
// collected fields
- ConnectionId uint64 `gorm:"primaryKey"`
- Name string `gorm:"primaryKey;type:varchar(255)"`
- Path string `gorm:"primaryKey;type:varchar(511)"`
- Class string `gorm:"type:varchar(255)"`
- Color string `gorm:"type:varchar(255)"`
- Base string `gorm:"type:varchar(255)"`
- HasUpstreamProjects bool
+ ConnectionId uint64 `gorm:"primaryKey"`
+ Name string `gorm:"primaryKey;type:varchar(255)"`
+ Path string `gorm:"primaryKey;type:varchar(511)"`
+ Class string `gorm:"type:varchar(255)"`
+ Color string `gorm:"type:varchar(255)"`
+ Base string `gorm:"type:varchar(255)"`
}
// JenkinsJob db entity for jenkins job
diff --git a/plugins/jenkins/models/up_down_job.go b/plugins/jenkins/models/job_dag.go
similarity index 69%
rename from plugins/jenkins/models/up_down_job.go
rename to plugins/jenkins/models/job_dag.go
index fcca32f2..dcbe7253 100644
--- a/plugins/jenkins/models/up_down_job.go
+++ b/plugins/jenkins/models/job_dag.go
@@ -2,13 +2,13 @@ package models
import "github.com/apache/incubator-devlake/models/common"
-type JenkinsUpDownJob struct {
+type JenkinsJobDag struct {
ConnetionId uint64 `gorm:"primaryKey"`
UpstreamJob string `gorm:"primaryKey;type:varchar(255)"`
DownstreamJob string `gorm:"primaryKey;type:varchar(255)"`
common.NoPKModel
}
-func (JenkinsUpDownJob) TableName() string {
- return "_tool_jenkins_up_down_jobs"
+func (JenkinsJobDag) TableName() string {
+ return "_tool_jenkins_job_dags"
}
diff --git a/plugins/jenkins/models/migrationscripts/20220729_modify_all_entities.go b/plugins/jenkins/models/migrationscripts/20220729_modify_all_entities.go
index 43d8b4e7..33a00e19 100644
--- a/plugins/jenkins/models/migrationscripts/20220729_modify_all_entities.go
+++ b/plugins/jenkins/models/migrationscripts/20220729_modify_all_entities.go
@@ -19,62 +19,46 @@ package migrationscripts
import (
"context"
- "github.com/apache/incubator-devlake/models/common"
"github.com/apache/incubator-devlake/models/migrationscripts/archived"
"gorm.io/gorm"
)
type modifyAllEntities struct{}
-// JenkinsBuild db entity for jenkins build
-type JenkinsBuild0729 struct {
- Type string `gorm:"index;type:varchar(255)" `
-}
-
-func (JenkinsBuild0729) TableName() string {
- return "_tool_jenkins_builds"
-}
-
-type JenkinsJob0729 struct {
- HasUpstreamProjects bool
-}
-
-func (JenkinsJob0729) TableName() string {
- return "_tool_jenkins_jobs"
-}
-
-type JenkinsUpDownJob0729 struct {
+type JenkinsJobDag0729 struct {
ConnetionId uint64 `gorm:"primaryKey"`
UpstreamJob string `gorm:"primaryKey;type:varchar(255)"`
DownstreamJob string `gorm:"primaryKey;type:varchar(255)"`
archived.NoPKModel
}
-func (JenkinsUpDownJob0729) TableName() string {
- return "_tool_jenkins_up_down_jobs"
+func (JenkinsJobDag0729) TableName() string {
+ return "_tool_jenkins_job_dags"
}
-type JenkinsBuildCommitRepoUrl0729 struct {
- ConnectionId uint64 `gorm:"primaryKey"`
- BuildName string `gorm:"primaryKey;type:varchar(255)"`
- CommitSha string `gorm:"primaryKey;type:varchar(255)"`
- RemoteUrl string `gorm:"primaryKey;type:varchar(255)"`
- common.NoPKModel
+// JenkinsBuild db entity for jenkins build
+type JenkinsBuild0729 struct {
+ TriggeredBy string `gorm:"type:varchar(255)"`
+ Type string `gorm:"index;type:varchar(255)" `
+ Class string `gorm:"index;type:varchar(255)" `
+ Building bool
}
-func (JenkinsBuildCommitRepoUrl0729) TableName() string {
- return "_tool_jenkins_build_commit_repo_urls"
+func (JenkinsBuild0729) TableName() string {
+ return "_tool_jenkins_builds"
}
-type JenkinsBuildTriggeredBuilds0729 struct {
- ConnectionId uint64 `gorm:"primaryKey"`
- BuildName string `gorm:"primaryKey;type:varchar(255)"`
- TriggeredBuildName string `gorm:"primaryKey;type:varchar(255)"`
+type JenkinsBuildRepo0729 struct {
+ ConnectionId uint64 `gorm:"primaryKey"`
+ BuildName string `gorm:"primaryKey;type:varchar(255)"`
+ CommitSha string `gorm:"primaryKey;type:varchar(255)"`
+ Branch string `gorm:"type:varchar(255)"`
+ RepoUrl string `gorm:"primaryKey;type:varchar(255)"`
archived.NoPKModel
}
-func (JenkinsBuildTriggeredBuilds0729) TableName() string {
- return "_tool_jenkins_build_triggered_builds"
+func (JenkinsBuildRepo0729) TableName() string {
+ return "_tool_jenkins_build_repos"
}
type JenkinsStage0729 struct {
@@ -88,6 +72,7 @@ type JenkinsStage0729 struct {
DurationMillis int `json:"durationMillis"`
PauseDurationMillis int `json:"pauseDurationMillis"`
BuildName string `gorm:"primaryKey;type:varchar(255)"`
+ Type string `gorm:"index;type:varchar(255)"`
}
func (JenkinsStage0729) TableName() string {
@@ -99,14 +84,17 @@ func (*modifyAllEntities) Up(ctx context.Context, db *gorm.DB) error {
if err != nil {
return err
}
- err = db.Migrator().AddColumn(JenkinsJob0729{}, "has_upstream_projects")
+ err = db.Migrator().AddColumn(JenkinsBuild0729{}, "triggered_by")
+ if err != nil {
+ return err
+ }
+ err = db.Migrator().AddColumn(JenkinsBuild0729{}, "building")
if err != nil {
return err
}
err = db.Migrator().AutoMigrate(
- JenkinsUpDownJob0729{},
- JenkinsBuildCommitRepoUrl0729{},
- JenkinsBuildTriggeredBuilds0729{},
+ JenkinsJobDag0729{},
+ JenkinsBuildRepo0729{},
JenkinsStage0729{},
)
if err != nil {
diff --git a/plugins/jenkins/models/response.go b/plugins/jenkins/models/response.go
index 3171bf27..1727c197 100644
--- a/plugins/jenkins/models/response.go
+++ b/plugins/jenkins/models/response.go
@@ -81,15 +81,16 @@ type ApiBuildResponse struct {
ChangeSet ChangeSet `json:"changeSet"`
}
type LastBuiltRevision struct {
- SHA1 string `json:"SHA1"`
+ SHA1 string `json:"SHA1"`
+ Branches []Branch `json:"branch"`
}
type Action struct {
- Class string `json:"_class,omitempty"`
- LastBuiltRevision LastBuiltRevision `json:"lastBuiltRevision,omitempty"`
- MercurialRevisionNumber string `json:"mercurialRevisionNumber"`
- RemoteUrls []string `json:"remoteUrls"`
- TriggeredBuilds []ApiBuildResponse `json:"triggeredBuilds"`
+ Class string `json:"_class,omitempty"`
+ LastBuiltRevision LastBuiltRevision `json:"lastBuiltRevision,omitempty"`
+ MercurialRevisionNumber string `json:"mercurialRevisionNumber"`
+ RemoteUrls []string `json:"remoteUrls"`
+ Causes []Cause `json:"causes"`
}
type ChangeSet struct {
Class string `json:"_class"`
@@ -97,6 +98,10 @@ type ChangeSet struct {
Revisions []Revision `json:"revision"`
}
+type Branch struct {
+ Name string `json:"name"`
+}
+
type Revision struct {
Module string
Revision int
@@ -116,3 +121,11 @@ type Stage struct {
DurationMillis int `json:"durationMillis"`
PauseDurationMillis int `json:"pauseDurationMillis"`
}
+
+type Cause struct {
+ Class string `json:"_class"`
+ ShortDescription string `json:"shortDescription"`
+ UpstreamBuild int `json:"upstreamBuild"`
+ UpstreamProject string `json:"upstreamProject"`
+ UpstreamURL string `json:"upstreamUrl"`
+}
diff --git a/plugins/jenkins/models/stage.go b/plugins/jenkins/models/stage.go
index a8654d90..108b1952 100644
--- a/plugins/jenkins/models/stage.go
+++ b/plugins/jenkins/models/stage.go
@@ -31,6 +31,7 @@ type JenkinsStage struct {
StartTimeMillis int64 `json:"startTimeMillis"`
DurationMillis int `json:"durationMillis"`
PauseDurationMillis int `json:"pauseDurationMillis"`
+ Type string `gorm:"index;type:varchar(255)"`
BuildName string `gorm:"primaryKey;type:varchar(255)"`
}
diff --git a/plugins/jenkins/tasks/build_collector.go b/plugins/jenkins/tasks/build_collector.go
index 86877c24..7e7d3667 100644
--- a/plugins/jenkins/tasks/build_collector.go
+++ b/plugins/jenkins/tasks/build_collector.go
@@ -82,7 +82,7 @@ func CollectApiBuilds(taskCtx core.SubTaskContext) error {
Query: func(reqData *helper.RequestData) (url.Values, error) {
query := url.Values{}
treeValue := fmt.Sprintf(
- "allBuilds[number,timestamp,duration,estimatedDuration,fullDisplayName,result,actions[lastBuiltRevision[SHA1],remoteUrls,mercurialRevisionNumber,triggeredBuilds[fullDisplayName,number,url,result,status,duration]],changeSet[kind,revisions[revision]]]{%d,%d}",
+ "allBuilds[number,timestamp,duration,estimatedDuration,fullDisplayName,result,actions[lastBuiltRevision[SHA1,branch[name]],remoteUrls,mercurialRevisionNumber,causes[*]],changeSet[kind,revisions[revision]]]{%d,%d}",
reqData.Pager.Skip, reqData.Pager.Skip+reqData.Pager.Size)
query.Set("tree", treeValue)
return query, nil
diff --git a/plugins/jenkins/tasks/build_extractor.go b/plugins/jenkins/tasks/build_extractor.go
index 66a818d4..cbe48f71 100644
--- a/plugins/jenkins/tasks/build_extractor.go
+++ b/plugins/jenkins/tasks/build_extractor.go
@@ -19,6 +19,7 @@ package tasks
import (
"encoding/json"
+ "fmt"
"strconv"
"strings"
"time"
@@ -72,7 +73,6 @@ func ExtractApiBuilds(taskCtx core.SubTaskContext) error {
class := strList[len(strList)-1]
build := &models.JenkinsBuild{
ConnectionId: data.Options.ConnectionId,
- Type: class,
JobName: input.Name,
Duration: body.Duration,
DisplayName: body.DisplayName,
@@ -80,12 +80,14 @@ func ExtractApiBuilds(taskCtx core.SubTaskContext) error {
Number: body.Number,
Result: body.Result,
Timestamp: body.Timestamp,
+ Class: class,
StartTime: time.Unix(body.Timestamp/1000, 0),
}
vcs := body.ChangeSet.Kind
if vcs == "git" || vcs == "hg" {
for _, a := range body.Actions {
sha := ""
+ branch := ""
if a.LastBuiltRevision.SHA1 != "" {
sha = a.LastBuiltRevision.SHA1
}
@@ -93,28 +95,27 @@ func ExtractApiBuilds(taskCtx core.SubTaskContext) error {
sha = a.MercurialRevisionNumber
}
build.CommitSha = sha
+ if len(a.LastBuiltRevision.Branches) > 0 {
+ branch = a.LastBuiltRevision.Branches[0].Name
+ }
for _, url := range a.RemoteUrls {
if url != "" {
- buildCommitRemoteUrl := models.JenkinsBuildCommitRepoUrl{
+ buildCommitRemoteUrl := models.JenkinsBuildRepo{
ConnectionId: data.Options.ConnectionId,
BuildName: build.DisplayName,
CommitSha: sha,
- RemoteUrl: url,
+ RepoUrl: url,
+ Branch: branch,
}
results = append(results, &buildCommitRemoteUrl)
}
}
- if a.TriggeredBuilds != nil && len(a.TriggeredBuilds) > 0 {
- for _, b := range a.TriggeredBuilds {
- if b.DisplayName == "" {
- continue
- }
- buildTrigger := models.JenkinsBuildTriggeredBuilds{
- ConnectionId: data.Options.ConnectionId,
- BuildName: build.DisplayName,
- TriggeredBuildName: b.DisplayName,
+ if len(a.Causes) > 0 {
+ for _, cause := range a.Causes {
+ if cause.UpstreamProject != "" {
+ triggeredByBuild := fmt.Sprintf("%s #%d", cause.UpstreamProject, cause.UpstreamBuild)
+ build.TriggeredBy = triggeredByBuild
}
- results = append(results, &buildTrigger)
}
}
}
diff --git a/plugins/jenkins/tasks/job_extractor.go b/plugins/jenkins/tasks/job_extractor.go
index f78742be..3b715960 100644
--- a/plugins/jenkins/tasks/job_extractor.go
+++ b/plugins/jenkins/tasks/job_extractor.go
@@ -68,16 +68,15 @@ func ExtractApiJobs(taskCtx core.SubTaskContext) error {
job := &models.JenkinsJob{
JenkinsJobProps: models.JenkinsJobProps{
- ConnectionId: data.Options.ConnectionId,
- Name: body.Name,
- Path: input.Path,
- Class: body.Class,
- Color: body.Color,
- HasUpstreamProjects: len(body.UpstreamProjects) > 0,
+ ConnectionId: data.Options.ConnectionId,
+ Name: body.Name,
+ Path: input.Path,
+ Class: body.Class,
+ Color: body.Color,
},
}
for _, upstreamProject := range body.UpstreamProjects {
- upDownJob := models.JenkinsUpDownJob{
+ upDownJob := models.JenkinsJobDag{
ConnetionId: data.Options.ConnectionId,
UpstreamJob: upstreamProject.Name,
DownstreamJob: job.Name,