You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by ab...@apache.org on 2022/08/05 03:20:45 UTC
[incubator-devlake] branch main updated: feat: github cicd enricher from run to pipeline (#2685)
This is an automated email from the ASF dual-hosted git repository.
abeizn pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 637204af feat: github cicd enricher from run to pipeline (#2685)
637204af is described below
commit 637204af7eccb9d9462675a77c2d843f0041d91a
Author: abeizn <10...@users.noreply.github.com>
AuthorDate: Fri Aug 5 11:20:42 2022 +0800
feat: github cicd enricher from run to pipeline (#2685)
* feat: github add cicd feature
* feat: github add cicd feature and update fields
* feat: github cicd enricher from run to pipeline
* feat: github add cicd feature
* feat: github add cicd feature and update fields
* feat: github cicd enricher from run to pipeline
* feat: github cicd enricher from run to pipeline
* feat: github cicd enricher from run to pipeline
* feat: github cicd enricher from run to pipeline
---
plugins/github/impl/impl.go | 1 +
plugins/github/models/job.go | 2 +
...ub_runs_table.go => 20220728_add_runs_table.go} | 3 +-
...ub_jobs_table.go => 20220729_add_jobs_table.go} | 2 +
.../20220802_add_pipeline_table.go | 65 +++++++++++++++
plugins/github/models/migrationscripts/register.go | 1 +
plugins/github/models/pipeline.go | 42 ++++++++++
plugins/github/models/run.go | 3 +-
plugins/github/tasks/cicd_job_extractor.go | 4 +
plugins/github/tasks/cicd_run_enricher.go | 95 ++++++++++++++++++++++
plugins/github/tasks/cicd_run_extractor.go | 4 +-
11 files changed, 219 insertions(+), 3 deletions(-)
diff --git a/plugins/github/impl/impl.go b/plugins/github/impl/impl.go
index d57345f2..bdb99223 100644
--- a/plugins/github/impl/impl.go
+++ b/plugins/github/impl/impl.go
@@ -82,6 +82,7 @@ func (plugin Github) SubTaskMetas() []core.SubTaskMeta {
tasks.ExtractRunsMeta,
tasks.CollectJobsMeta,
tasks.ExtractJobsMeta,
+ tasks.EnrichPipelinesMeta,
tasks.EnrichPullRequestIssuesMeta,
tasks.ConvertRepoMeta,
tasks.ConvertIssuesMeta,
diff --git a/plugins/github/models/job.go b/plugins/github/models/job.go
index 15acaa6b..31a95430 100644
--- a/plugins/github/models/job.go
+++ b/plugins/github/models/job.go
@@ -27,6 +27,7 @@ import (
type GithubJob struct {
common.NoPKModel
ConnectionId uint64 `gorm:"primaryKey"`
+ RepoId int `gorm:"primaryKey"`
ID int `json:"id" gorm:"primaryKey;autoIncrement:false"`
RunID int `json:"run_id"`
RunURL string `json:"run_url" gorm:"type:varchar(255)"`
@@ -45,6 +46,7 @@ type GithubJob struct {
RunnerID int `json:"runner_id"`
RunnerName string `json:"runner_name" gorm:"type:varchar(255)"`
RunnerGroupID int `json:"runner_group_id"`
+ Type string `json:"type" gorm:"type:varchar(255)"`
}
func (GithubJob) TableName() string {
diff --git a/plugins/github/models/migrationscripts/20220728_add_github_runs_table.go b/plugins/github/models/migrationscripts/20220728_add_runs_table.go
similarity index 96%
rename from plugins/github/models/migrationscripts/20220728_add_github_runs_table.go
rename to plugins/github/models/migrationscripts/20220728_add_runs_table.go
index bc5b6307..aee05c57 100644
--- a/plugins/github/models/migrationscripts/20220728_add_github_runs_table.go
+++ b/plugins/github/models/migrationscripts/20220728_add_runs_table.go
@@ -29,7 +29,7 @@ import (
type GithubRun20220728 struct {
archived.NoPKModel
ConnectionId uint64 `gorm:"primaryKey"`
- GithubId int `gorm:"primaryKey"`
+ RepoId int `gorm:"primaryKey"`
ID int64 `json:"id" gorm:"primaryKey;autoIncrement:false"`
Name string `json:"name" gorm:"type:varchar(255)"`
NodeID string `json:"node_id" gorm:"type:varchar(255)"`
@@ -56,6 +56,7 @@ type GithubRun20220728 struct {
CancelURL string `json:"cancel_url" gorm:"type:varchar(255)"`
RerunURL string `json:"rerun_url" gorm:"type:varchar(255)"`
WorkflowURL string `json:"workflow_url" gorm:"type:varchar(255)"`
+ Type string `json:"type" gorm:"type:varchar(255)"`
}
func (GithubRun20220728) TableName() string {
diff --git a/plugins/github/models/migrationscripts/20220729_add_github_jobs_table.go b/plugins/github/models/migrationscripts/20220729_add_jobs_table.go
similarity index 95%
rename from plugins/github/models/migrationscripts/20220729_add_github_jobs_table.go
rename to plugins/github/models/migrationscripts/20220729_add_jobs_table.go
index 981addd4..9631dceb 100644
--- a/plugins/github/models/migrationscripts/20220729_add_github_jobs_table.go
+++ b/plugins/github/models/migrationscripts/20220729_add_jobs_table.go
@@ -30,6 +30,7 @@ import (
type GithubJob20220729 struct {
archived.NoPKModel
ConnectionId uint64 `gorm:"primaryKey"`
+ RepoId int `gorm:"primaryKey"`
ID int `json:"id" gorm:"primaryKey;autoIncrement:false"`
RunID int `json:"run_id"`
RunURL string `json:"run_url" gorm:"type:varchar(255)"`
@@ -48,6 +49,7 @@ type GithubJob20220729 struct {
RunnerID int `json:"runner_id"`
RunnerName string `json:"runner_name" gorm:"type:varchar(255)"`
RunnerGroupID int `json:"runner_group_id"`
+ Type string `json:"type" gorm:"type:varchar(255)"`
}
func (GithubJob20220729) TableName() string {
diff --git a/plugins/github/models/migrationscripts/20220802_add_pipeline_table.go b/plugins/github/models/migrationscripts/20220802_add_pipeline_table.go
new file mode 100644
index 00000000..84e1c87a
--- /dev/null
+++ b/plugins/github/models/migrationscripts/20220802_add_pipeline_table.go
@@ -0,0 +1,65 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+ "context"
+ "fmt"
+ "time"
+
+ "github.com/apache/incubator-devlake/models/migrationscripts/archived"
+ "gorm.io/gorm"
+)
+
+type GithubPipeline20220803 struct {
+ archived.NoPKModel
+ ConnectionId uint64 `gorm:"primaryKey"`
+ RepoId int `gorm:"primaryKey"`
+ Branch string `json:"branch" gorm:"primaryKey;type:varchar(255)"`
+ Commit string `json:"commit" gorm:"primaryKey;type:varchar(255)"`
+ StartedDate *time.Time `json:"started_time"`
+ FinishedDate *time.Time `json:"finished_time"`
+ Duration float64 `json:"duration"`
+ Status string `json:"status" gorm:"type:varchar(255)"`
+ Result string `json:"results" gorm:"type:varchar(255)"`
+ Type string `json:"type" gorm:"type:varchar(255)"`
+}
+
+func (GithubPipeline20220803) TableName() string {
+ return "_tool_github_pipelines"
+}
+
+type addGithubPipelineTable struct{}
+
+func (u *addGithubPipelineTable) Up(ctx context.Context, db *gorm.DB) error {
+ // create table
+ err := db.Migrator().CreateTable(GithubPipeline20220803{})
+ if err != nil {
+ return fmt.Errorf("create table _tool_github_pipelines error")
+ }
+ return nil
+
+}
+
+func (*addGithubPipelineTable) Version() uint64 {
+ return 20220803000001
+}
+
+func (*addGithubPipelineTable) Name() string {
+ return "Github add github_pipelines table"
+}
diff --git a/plugins/github/models/migrationscripts/register.go b/plugins/github/models/migrationscripts/register.go
index 4f899e78..439d77d4 100644
--- a/plugins/github/models/migrationscripts/register.go
+++ b/plugins/github/models/migrationscripts/register.go
@@ -27,5 +27,6 @@ func All() []migration.Script {
new(addInitTables),
new(addGithubRunsTable),
new(addGithubJobsTable),
+ new(addGithubPipelineTable),
}
}
diff --git a/plugins/github/models/pipeline.go b/plugins/github/models/pipeline.go
new file mode 100644
index 00000000..0b18e6b5
--- /dev/null
+++ b/plugins/github/models/pipeline.go
@@ -0,0 +1,42 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package models
+
+import (
+ "time"
+
+ "github.com/apache/incubator-devlake/models/common"
+)
+
+type GithubPipeline struct {
+ common.NoPKModel
+ ConnectionId uint64 `gorm:"primaryKey"`
+ RepoId int `gorm:"primaryKey"`
+ Branch string `json:"branch" gorm:"primaryKey;type:varchar(255)"`
+ Commit string `json:"commit" gorm:"primaryKey;type:varchar(255)"`
+ StartedDate *time.Time `json:"started_time"`
+ FinishedDate *time.Time `json:"finished_time"`
+ Duration float64 `json:"duration"`
+ Status string `json:"status" gorm:"type:varchar(255)"`
+ Result string `json:"results" gorm:"type:varchar(255)"`
+ Type string `json:"type" gorm:"type:varchar(255)"`
+}
+
+func (GithubPipeline) TableName() string {
+ return "_tool_github_pipelines"
+}
diff --git a/plugins/github/models/run.go b/plugins/github/models/run.go
index d749f3a6..bd2ec06c 100644
--- a/plugins/github/models/run.go
+++ b/plugins/github/models/run.go
@@ -26,7 +26,7 @@ import (
type GithubRun struct {
common.NoPKModel
ConnectionId uint64 `gorm:"primaryKey"`
- GithubId int `gorm:"primaryKey"`
+ RepoId int `gorm:"primaryKey"`
ID int64 `json:"id" gorm:"primaryKey;autoIncrement:false"`
Name string `json:"name" gorm:"type:varchar(255)"`
NodeID string `json:"node_id" gorm:"type:varchar(255)"`
@@ -53,6 +53,7 @@ type GithubRun struct {
CancelURL string `json:"cancel_url" gorm:"type:varchar(255)"`
RerunURL string `json:"rerun_url" gorm:"type:varchar(255)"`
WorkflowURL string `json:"workflow_url" gorm:"type:varchar(255)"`
+ Type string `json:"type" gorm:"type:varchar(255)"`
}
func (GithubRun) TableName() string {
diff --git a/plugins/github/tasks/cicd_job_extractor.go b/plugins/github/tasks/cicd_job_extractor.go
index 9cbf58db..b8e8fcae 100644
--- a/plugins/github/tasks/cicd_job_extractor.go
+++ b/plugins/github/tasks/cicd_job_extractor.go
@@ -35,6 +35,7 @@ var ExtractJobsMeta = core.SubTaskMeta{
func ExtractJobs(taskCtx core.SubTaskContext) error {
data := taskCtx.GetData().(*GithubTaskData)
+ repoId := data.Repo.GithubId
extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{
RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
@@ -56,6 +57,7 @@ func ExtractJobs(taskCtx core.SubTaskContext) error {
results := make([]interface{}, 0, 1)
githubJobResult := &models.GithubJob{
ConnectionId: data.Options.ConnectionId,
+ RepoId: repoId,
ID: githubJob.ID,
RunID: githubJob.RunID,
RunURL: githubJob.RunURL,
@@ -74,6 +76,8 @@ func ExtractJobs(taskCtx core.SubTaskContext) error {
RunnerID: githubJob.RunID,
RunnerName: githubJob.RunnerName,
RunnerGroupID: githubJob.RunnerGroupID,
+ // TODO
+ //Type: "CI/CD",
}
results = append(results, githubJobResult)
return results, nil
diff --git a/plugins/github/tasks/cicd_run_enricher.go b/plugins/github/tasks/cicd_run_enricher.go
new file mode 100644
index 00000000..ee90e7d6
--- /dev/null
+++ b/plugins/github/tasks/cicd_run_enricher.go
@@ -0,0 +1,95 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/core/dal"
+ githubModels "github.com/apache/incubator-devlake/plugins/github/models"
+)
+
+var EnrichPipelinesMeta = core.SubTaskMeta{
+ Name: "enrichPipelines",
+ EntryPoint: EnrichPipelines,
+ EnabledByDefault: true,
+ Description: "Create tool layer table github_pipelines from github_runs",
+ DomainTypes: []string{core.DOMAIN_TYPE_CICD},
+}
+
+func EnrichPipelines(taskCtx core.SubTaskContext) (err error) {
+ db := taskCtx.GetDal()
+ data := taskCtx.GetData().(*GithubTaskData)
+ repoId := data.Repo.GithubId
+
+ cursor, err := db.Cursor(
+ dal.Select("head_sha, head_branch, status, conclusion, github_created_at, github_updated_at, run_attempt, run_started_at"),
+ dal.From(&githubModels.GithubRun{}),
+ dal.Orderby("head_sha, github_created_at"),
+ )
+ if err != nil {
+ return err
+ }
+ defer cursor.Close()
+
+ for cursor.Next() {
+ entity := &githubModels.GithubPipeline{}
+ var item githubModels.GithubRun
+ err = db.Fetch(cursor, &item)
+ if err != nil {
+ return err
+ }
+
+ if item.HeadSha != entity.Commit {
+ entity.ConnectionId = data.Options.ConnectionId
+ entity.RepoId = repoId
+ entity.Commit = item.HeadSha
+ entity.Branch = item.HeadBranch
+ entity.StartedDate = item.GithubCreatedAt
+ entity.FinishedDate = item.GithubUpdatedAt
+ entity.Status = item.Status
+ if entity.Status == "completed" {
+ entity.Duration = float64(item.GithubUpdatedAt.Sub(*item.GithubCreatedAt).Seconds())
+ }
+ entity.Result = item.Conclusion
+ // TODO
+ entity.Type = "CI/CD"
+ } else {
+ if item.GithubCreatedAt.Before(*entity.StartedDate) {
+ entity.StartedDate = item.GithubCreatedAt
+ }
+ if item.GithubUpdatedAt.After(*entity.FinishedDate) && item.Status == "completed" {
+ entity.FinishedDate = item.GithubCreatedAt
+ entity.Duration = float64(item.GithubUpdatedAt.Sub(*item.GithubCreatedAt).Seconds())
+ }
+ if item.Status != "completed" {
+ entity.Status = item.Status
+ }
+ if item.Conclusion != "success" {
+ entity.Result = item.Conclusion
+ }
+
+ }
+ err := db.CreateOrUpdate(entity)
+ if err != nil {
+ return err
+ }
+ }
+
+ return err
+
+}
diff --git a/plugins/github/tasks/cicd_run_extractor.go b/plugins/github/tasks/cicd_run_extractor.go
index 87d07b72..703bc652 100644
--- a/plugins/github/tasks/cicd_run_extractor.go
+++ b/plugins/github/tasks/cicd_run_extractor.go
@@ -57,8 +57,8 @@ func ExtractRuns(taskCtx core.SubTaskContext) error {
results := make([]interface{}, 0, 1)
githubRunResult := &models.GithubRun{
ConnectionId: data.Options.ConnectionId,
- GithubId: repoId,
ID: githubRun.ID,
+ RepoId: repoId,
Name: githubRun.Name,
NodeID: githubRun.NodeID,
HeadBranch: githubRun.HeadBranch,
@@ -84,6 +84,8 @@ func ExtractRuns(taskCtx core.SubTaskContext) error {
CancelURL: githubRun.CancelURL,
RerunURL: githubRun.RerunURL,
WorkflowURL: githubRun.WorkflowURL,
+ // TODO
+ //Type: "CI/CD",
}
results = append(results, githubRunResult)
return results, nil