You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by kl...@apache.org on 2022/09/19 06:49:16 UTC
[incubator-devlake] branch main updated: feat: finish pipeline and issue webhook (#3052)
This is an automated email from the ASF dual-hosted git repository.
klesh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 5be63862 feat: finish pipeline and issue webhook (#3052)
5be63862 is described below
commit 5be63862bf2fd15bb6af2ed6e77a2a47793a54bf
Author: likyh <l...@likyh.com>
AuthorDate: Mon Sep 19 14:49:11 2022 +0800
feat: finish pipeline and issue webhook (#3052)
* feat: finish pipeline and issue webhook
* feat: add webhooks cicd_task and pipeline_finish
* test: add unittest for type and result on webhook pipeline
* feat: use new type; add environment for task
* fix: try to fix for lint
* fix: change a name
Co-authored-by: linyh <ya...@meri.co>
---
plugins/gitlab/tasks/pipeline_project_convertor.go | 4 +-
plugins/webhook/api/cicd_pipeline.go | 182 +++++++++++++++++++--
plugins/webhook/api/cicd_pipeline_test.go | 67 ++++++++
plugins/webhook/api/issue.go | 97 ++++++++++-
plugins/webhook/impl/impl.go | 7 +-
5 files changed, 338 insertions(+), 19 deletions(-)
diff --git a/plugins/gitlab/tasks/pipeline_project_convertor.go b/plugins/gitlab/tasks/pipeline_project_convertor.go
index eada51e5..11da9bcc 100644
--- a/plugins/gitlab/tasks/pipeline_project_convertor.go
+++ b/plugins/gitlab/tasks/pipeline_project_convertor.go
@@ -62,7 +62,7 @@ func ConvertPipelineProjects(taskCtx core.SubTaskContext) errors.Error {
Convert: func(inputRow interface{}) ([]interface{}, errors.Error) {
gitlabPipelineProject := inputRow.(*gitlabModels.GitlabPipelineProject)
- domainPipelineRepo := &devops.CiCDPipelineCommit{
+ domainPipelineCommit := &devops.CiCDPipelineCommit{
PipelineId: pipelineIdGen.Generate(data.Options.ConnectionId, gitlabPipelineProject.PipelineId),
CommitSha: gitlabPipelineProject.Sha,
Branch: gitlabPipelineProject.Ref,
@@ -71,7 +71,7 @@ func ConvertPipelineProjects(taskCtx core.SubTaskContext) errors.Error {
}
return []interface{}{
- domainPipelineRepo,
+ domainPipelineCommit,
}, nil
},
})
diff --git a/plugins/webhook/api/cicd_pipeline.go b/plugins/webhook/api/cicd_pipeline.go
index c15cf63c..064ca983 100644
--- a/plugins/webhook/api/cicd_pipeline.go
+++ b/plugins/webhook/api/cicd_pipeline.go
@@ -18,41 +18,199 @@ limitations under the License.
package api
import (
+ "fmt"
"github.com/apache/incubator-devlake/errors"
+ "github.com/apache/incubator-devlake/models/domainlayer"
+ "github.com/apache/incubator-devlake/models/domainlayer/devops"
+ "github.com/apache/incubator-devlake/models/domainlayer/ticket"
"github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/core/dal"
+ "github.com/apache/incubator-devlake/plugins/helper"
"github.com/apache/incubator-devlake/plugins/webhook/models"
+ "github.com/go-playground/validator/v10"
"net/http"
"time"
)
-type WebhookPipelineRequest struct {
- Id string `validate:"required"`
- Result string `validate:"oneof=SUCCESS FAILURE ABORT"`
+type WebhookTaskRequest struct {
+ // PipelineName can be filled by any string unique in one pipeline
+ PipelineName string `mapstructure:"pipeline_name" validate:"required"`
+
+ Name string `validate:"required"` // Name should be unique in one pipeline
+ Result string `validate:"oneof=SUCCESS FAILURE ABORT IN_PROGRESS"`
Status string `validate:"oneof=IN_PROGRESS DONE"`
- Type string `validate:"oneof=CI CD CI/CD"`
- CreatedDate time.Time `mapstructure:"created_date" validate:"required"`
+ Type string `validate:"oneof=TEST LINT BUILD DEPLOYMENT"`
+ Environment string `validate:"oneof=PRODUCTION STAGING TESTING"`
+ StartedDate time.Time `mapstructure:"created_date" validate:"required"`
FinishedDate *time.Time `mapstructure:"finished_date"`
- Repo string `validate:"required"`
+ RepoId string `mapstructure:"repo_id" validate:"required"` // RepoId should be unique string
Branch string
CommitSha string `mapstructure:"commit_sha"`
}
-// PostCicdPipeline
+// PostCicdTask
// @Summary create pipeline by webhook
-// @Description Create pipeline by webhook, example: {"id":"A123123","result":"one of SUCCESS/FAILURE/ABORT","status":"one of IN_PROGRESS/DONE","type":"CI/CD","created_date":"2020-01-01T12:00:00+00:00","finished_date":"2020-01-01T12:59:59+00:00","repo":"devlake","branch":"main","commit_sha":"015e3d3b480e417aede5a1293bd61de9b0fd051d"}
+// @Description Create pipeline by webhook.<br/>
+// @Description example1: {"pipeline_name":"A123","name":"unit-test","result":"IN_PROGRESS","status":"IN_PROGRESS","type":"TEST","environment":"PRODUCTION","created_date":"2020-01-01T12:00:00+00:00","finished_date":"2020-01-01T12:59:59+00:00","repo_id":"devlake","branch":"main","commit_sha":"015e3d3b480e417aede5a1293bd61de9b0fd051d"}<br/>
+// @Description example2: {"pipeline_name":"A123","name":"unit-test","result":"SUCCESS","status":"DONE","type":"DEPLOYMENT","environment":"PRODUCTION","created_date":"2020-01-01T12:00:00+00:00","finished_date":"2020-01-01T12:59:59+00:00","repo_id":"devlake","branch":"main","commit_sha":"015e3d3b480e417aede5a1293bd61de9b0fd051d"}<br/>
+// @Description When request webhook first time for each pipeline, it will be created.
+// @Description So we suggest request before task start and after pipeline finish.
+// @Description Remember fill all data to request after pipeline finish.
// @Tags plugins/webhook
-// @Param body body WebhookPipelineRequest true "json body"
+// @Param body body WebhookTaskRequest true "json body"
// @Success 200
// @Failure 400 {string} errcode.Error "Bad Request"
+// @Failure 403 {string} errcode.Error "Forbidden"
// @Failure 500 {string} errcode.Error "Internal Error"
-// @Router /plugins/webhook/:connectionId/cicd_pipelines [POST]
-func PostCicdPipeline(input *core.ApiResourceInput) (*core.ApiResourceOutput, errors.Error) {
+// @Router /plugins/webhook/:connectionId/cicd_tasks [POST]
+func PostCicdTask(input *core.ApiResourceInput) (*core.ApiResourceOutput, errors.Error) {
connection := &models.WebhookConnection{}
err := connectionHelper.First(connection, input.Params)
if err != nil {
return nil, err
}
- // TODO save pipeline
+ // get request
+ request := &WebhookTaskRequest{}
+ err = helper.DecodeMapStruct(input.Body, request)
+ if err != nil {
+ return &core.ApiResourceOutput{Body: err.Error(), Status: http.StatusBadRequest}, nil
+ }
+ // validate
+ vld = validator.New()
+ err = errors.Convert(vld.Struct(request))
+ if err != nil {
+ return nil, errors.BadInput.Wrap(vld.Struct(request), `input json error`)
+ }
+
+ db := basicRes.GetDal()
+ pipelineId := fmt.Sprintf("%s:%d:%s", "webhook", connection.ID, request.PipelineName)
+ domainCicdTask := &devops.CICDTask{
+ DomainEntity: domainlayer.DomainEntity{
+ Id: fmt.Sprintf("%s:%d:%s:%s", "webhook", connection.ID, request.PipelineName, request.Name),
+ },
+ PipelineId: pipelineId,
+ Name: request.Name,
+ Result: request.Result,
+ Status: request.Status,
+ Type: request.Type,
+ Environment: request.Environment,
+ StartedDate: request.StartedDate,
+ FinishedDate: request.FinishedDate,
+ }
+ if domainCicdTask.FinishedDate != nil {
+ domainCicdTask.DurationSec = uint64(domainCicdTask.FinishedDate.Sub(domainCicdTask.StartedDate).Seconds())
+ }
+
+ domainPipeline := &devops.CICDPipeline{}
+ err = db.First(domainPipeline, dal.Where("id = ?", pipelineId))
+ if err != nil {
+ domainPipeline = &devops.CICDPipeline{
+ DomainEntity: domainlayer.DomainEntity{
+ Id: pipelineId,
+ },
+ Name: request.PipelineName,
+ Result: ``,
+ Status: `IN_PROGRESS`,
+ Type: ``,
+ CreatedDate: request.StartedDate,
+ FinishedDate: nil,
+ }
+ } else if domainPipeline.Status == `DONE` {
+ return nil, errors.Forbidden.New(`can not receive this task because pipeline has already been done.`)
+ }
+
+ domainPipelineCommit := &devops.CiCDPipelineCommit{
+ PipelineId: pipelineId,
+ CommitSha: request.CommitSha,
+ Branch: request.Branch,
+ RepoId: request.RepoId,
+ }
+
+ // save
+ err = db.CreateOrUpdate(domainCicdTask)
+ if err != nil {
+ return nil, err
+ }
+ err = db.CreateOrUpdate(domainPipeline)
+ if err != nil {
+ return nil, err
+ }
+ err = db.CreateOrUpdate(domainPipelineCommit)
+ if err != nil {
+ return nil, err
+ }
+
return &core.ApiResourceOutput{Body: nil, Status: http.StatusOK}, nil
}
+
+// PostPipelineFinish
+// @Summary set pipeline's status to DONE
+// @Description set pipeline's status to DONE and cal duration
+// @Tags plugins/webhook
+// @Success 200
+// @Failure 400 {string} errcode.Error "Bad Request"
+// @Failure 500 {string} errcode.Error "Internal Error"
+// @Router /plugins/webhook/:connectionId/cicd_pipeline/:pipelineName/finish [POST]
+func PostPipelineFinish(input *core.ApiResourceInput) (*core.ApiResourceOutput, errors.Error) {
+ connection := &models.WebhookConnection{}
+ err := connectionHelper.First(connection, input.Params)
+ if err != nil {
+ return nil, err
+ }
+
+ db := basicRes.GetDal()
+ pipelineId := fmt.Sprintf("%s:%d:%s", "webhook", connection.ID, input.Params[`pipelineName`])
+ println(pipelineId)
+ domainPipeline := &devops.CICDPipeline{}
+ err = db.First(domainPipeline, dal.Where("id = ?", pipelineId))
+ if err != nil {
+ return nil, errors.NotFound.Wrap(err, `pipeline not found`)
+ }
+
+ domainTasks := []devops.CICDTask{}
+ err = db.All(&domainTasks, dal.Where("pipeline_id = ?", pipelineId))
+ if err != nil {
+ return nil, errors.NotFound.Wrap(err, `tasks not found`)
+ }
+ pipelineType, result := getTypeAndResultFromTasks(domainTasks)
+ domainPipeline.Type = pipelineType
+ domainPipeline.Result = result
+ domainPipeline.Status = ticket.DONE
+ now := time.Now()
+ domainPipeline.FinishedDate = &now
+ domainPipeline.DurationSec = uint64(domainPipeline.FinishedDate.Sub(domainPipeline.CreatedDate).Seconds())
+
+ // save
+ err = db.Update(domainPipeline)
+ if err != nil {
+ return nil, err
+ }
+
+ return &core.ApiResourceOutput{Body: nil, Status: http.StatusOK}, nil
+}
+
+// getTypeAndResultFromTasks will extract pipeline type and result from tasks
+// type = tasks' type if all tasks have the same type, or empty string
+// result = ABORT if any tasks' type is ABORT,
+// or result = FAILURE if any tasks' type is ABORT and others are SUCCESS
+// or result = SUCCESS if all tasks' type is SUCCESS
+func getTypeAndResultFromTasks(domainTasks []devops.CICDTask) (pipelineType, result string) {
+ result = `SUCCESS`
+ if len(domainTasks) > 0 {
+ pipelineType = domainTasks[0].Type
+ }
+ for _, domainTask := range domainTasks {
+ if domainTask.Type != pipelineType {
+ pipelineType = ``
+ }
+ if domainTask.Result == `ABORT` {
+ result = `ABORT`
+ } else if domainTask.Result == `FAILURE` {
+ if result == `SUCCESS` {
+ result = `FAILURE`
+ }
+ }
+ }
+ return
+}
diff --git a/plugins/webhook/api/cicd_pipeline_test.go b/plugins/webhook/api/cicd_pipeline_test.go
new file mode 100644
index 00000000..f1862c33
--- /dev/null
+++ b/plugins/webhook/api/cicd_pipeline_test.go
@@ -0,0 +1,67 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package api
+
+import (
+ "github.com/apache/incubator-devlake/models/domainlayer/devops"
+ "github.com/stretchr/testify/assert"
+ "testing"
+)
+
+func TestGetTypeAndResultFromTasks(t *testing.T) {
+ testDomainTask := devops.CICDTask{Type: devops.TEST}
+ buildDomainTask := devops.CICDTask{Type: devops.BUILD}
+ deploymentDomainTask := devops.CICDTask{Type: devops.DEPLOYMENT}
+
+ abortDomainTask := devops.CICDTask{Result: `ABORT`}
+ failureDomainTask := devops.CICDTask{Result: `FAILURE`}
+ successDomainTask := devops.CICDTask{Result: `SUCCESS`}
+
+ pipelineType, _ := getTypeAndResultFromTasks([]devops.CICDTask{testDomainTask})
+ assert.Equal(t, devops.TEST, pipelineType)
+
+ pipelineType, _ = getTypeAndResultFromTasks([]devops.CICDTask{buildDomainTask})
+ assert.Equal(t, devops.BUILD, pipelineType)
+
+ pipelineType, _ = getTypeAndResultFromTasks([]devops.CICDTask{deploymentDomainTask, deploymentDomainTask, deploymentDomainTask})
+ assert.Equal(t, devops.DEPLOYMENT, pipelineType)
+
+ pipelineType, _ = getTypeAndResultFromTasks([]devops.CICDTask{buildDomainTask, deploymentDomainTask, buildDomainTask})
+ assert.Equal(t, ``, pipelineType)
+
+ _, result := getTypeAndResultFromTasks([]devops.CICDTask{abortDomainTask, failureDomainTask, successDomainTask})
+ assert.Equal(t, `ABORT`, result)
+
+ _, result = getTypeAndResultFromTasks([]devops.CICDTask{failureDomainTask, successDomainTask})
+ assert.Equal(t, `FAILURE`, result)
+
+ _, result = getTypeAndResultFromTasks([]devops.CICDTask{successDomainTask})
+ assert.Equal(t, `SUCCESS`, result)
+
+ _, result = getTypeAndResultFromTasks([]devops.CICDTask{successDomainTask, successDomainTask})
+ assert.Equal(t, `SUCCESS`, result)
+
+ _, result = getTypeAndResultFromTasks([]devops.CICDTask{successDomainTask, successDomainTask, failureDomainTask})
+ assert.Equal(t, `FAILURE`, result)
+
+ _, result = getTypeAndResultFromTasks([]devops.CICDTask{successDomainTask, successDomainTask, abortDomainTask})
+ assert.Equal(t, `ABORT`, result)
+
+ _, result = getTypeAndResultFromTasks([]devops.CICDTask{failureDomainTask, failureDomainTask, abortDomainTask})
+ assert.Equal(t, `ABORT`, result)
+}
diff --git a/plugins/webhook/api/issue.go b/plugins/webhook/api/issue.go
index 93cfea37..06a3fc4b 100644
--- a/plugins/webhook/api/issue.go
+++ b/plugins/webhook/api/issue.go
@@ -18,9 +18,15 @@ limitations under the License.
package api
import (
+ "fmt"
"github.com/apache/incubator-devlake/errors"
+ "github.com/apache/incubator-devlake/models/domainlayer"
+ "github.com/apache/incubator-devlake/models/domainlayer/ticket"
"github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/core/dal"
+ "github.com/apache/incubator-devlake/plugins/helper"
"github.com/apache/incubator-devlake/plugins/webhook/models"
+ "github.com/go-playground/validator/v10"
"net/http"
"time"
)
@@ -59,7 +65,7 @@ type WebhookIssueRequest struct {
// @Summary receive a record as defined and save it
// @Description receive a record as follow and save it, example: {"board_key":"DLK","url":"","issue_key":"DLK-1234","title":"a feature from DLK","description":"","epic_key":"","type":"BUG","status":"TODO","original_status":"created","story_point":0,"resolution_date":null,"created_date":"2020-01-01T12:00:00+00:00","updated_date":null,"lead_time_minutes":0,"parent_issue_key":"DLK-1200","priority":"","original_estimate_minutes":0,"time_spent_minutes":0,"time_remaining_minutes":0,"creator_id [...]
// @Tags plugins/webhook
-// @Param body body WebhookPipelineRequest true "json body"
+// @Param body body WebhookIssueRequest true "json body"
// @Success 200 {string} noResponse ""
// @Failure 400 {string} errcode.Error "Bad Request"
// @Failure 500 {string} errcode.Error "Internal Error"
@@ -70,7 +76,78 @@ func PostIssue(input *core.ApiResourceInput) (*core.ApiResourceOutput, errors.Er
if err != nil {
return nil, err
}
- // TODO save issue
+ // get request
+ request := &WebhookIssueRequest{}
+ err = helper.DecodeMapStruct(input.Body, request)
+ if err != nil {
+ return &core.ApiResourceOutput{Body: err.Error(), Status: http.StatusBadRequest}, nil
+ }
+ // validate
+ vld = validator.New()
+ err = errors.Convert(vld.Struct(request))
+ if err != nil {
+ return &core.ApiResourceOutput{Body: err.Error(), Status: http.StatusBadRequest}, nil
+ }
+
+ db := basicRes.GetDal()
+ domainIssue := &ticket.Issue{
+ DomainEntity: domainlayer.DomainEntity{
+ Id: fmt.Sprintf("%s:%d:%s:%s", "webhook", connection.ID, request.BoardKey, request.IssueKey),
+ },
+ Url: request.Url,
+ IssueKey: request.IssueKey,
+ Title: request.Title,
+ Description: request.Description,
+ EpicKey: request.EpicKey,
+ Type: request.Type,
+ Status: request.Status,
+ OriginalStatus: request.OriginalStatus,
+ StoryPoint: request.StoryPoint,
+ ResolutionDate: request.ResolutionDate,
+ CreatedDate: request.CreatedDate,
+ UpdatedDate: request.UpdatedDate,
+ LeadTimeMinutes: request.LeadTimeMinutes,
+ Priority: request.Priority,
+ OriginalEstimateMinutes: request.OriginalEstimateMinutes,
+ TimeSpentMinutes: request.TimeSpentMinutes,
+ TimeRemainingMinutes: request.TimeRemainingMinutes,
+ CreatorName: request.CreatorName,
+ AssigneeName: request.AssigneeName,
+ Severity: request.Severity,
+ Component: request.Component,
+ }
+ if request.CreatorId != "" {
+ domainIssue.CreatorId = fmt.Sprintf("%s:%d:%s", "webhook", connection.ID, request.CreatorId)
+ }
+ if request.AssigneeId != "" {
+ domainIssue.AssigneeId = fmt.Sprintf("%s:%d:%s", "webhook", connection.ID, request.AssigneeId)
+ }
+ if request.ParentIssueKey != "" {
+ domainIssue.ParentIssueId = fmt.Sprintf("%s:%d:%s:%s", "webhook", connection.ID, request.BoardKey, request.ParentIssueKey)
+ }
+ domainBoard := &ticket.Board{
+ DomainEntity: domainlayer.DomainEntity{
+ Id: fmt.Sprintf("%s:%d:%s", "webhook", connection.ID, request.BoardKey),
+ },
+ }
+ boardIssue := &ticket.BoardIssue{
+ BoardId: domainBoard.Id,
+ IssueId: domainIssue.Id,
+ }
+ // save
+ err = db.CreateOrUpdate(domainIssue)
+ if err != nil {
+ return nil, err
+ }
+ err = db.CreateOrUpdate(domainBoard)
+ if err != nil {
+ return nil, err
+ }
+ err = db.CreateOrUpdate(boardIssue)
+ if err != nil {
+ return nil, err
+ }
+
return &core.ApiResourceOutput{Body: nil, Status: http.StatusOK}, nil
}
@@ -88,6 +165,20 @@ func CloseIssue(input *core.ApiResourceInput) (*core.ApiResourceOutput, errors.E
if err != nil {
return nil, err
}
- // TODO close issue
+
+ db := basicRes.GetDal()
+ domainIssue := &ticket.Issue{}
+ err = db.First(domainIssue, dal.Where("id = ?", fmt.Sprintf("%s:%d:%s:%s", "webhook", connection.ID, input.Params[`boardKey`], input.Params[`issueId`])))
+ if err != nil {
+ return nil, errors.NotFound.Wrap(err, `issue not found`)
+ }
+ domainIssue.Status = ticket.DONE
+ domainIssue.OriginalStatus = ``
+
+ // save
+ err = db.Update(domainIssue)
+ if err != nil {
+ return nil, err
+ }
return &core.ApiResourceOutput{Body: nil, Status: http.StatusOK}, nil
}
diff --git a/plugins/webhook/impl/impl.go b/plugins/webhook/impl/impl.go
index 670423e3..b8001434 100644
--- a/plugins/webhook/impl/impl.go
+++ b/plugins/webhook/impl/impl.go
@@ -63,8 +63,11 @@ func (plugin Webhook) ApiResources() map[string]map[string]core.ApiResourceHandl
"PATCH": api.PatchConnection,
"DELETE": api.DeleteConnection,
},
- ":connectionId/cicd_pipelines": {
- "POST": api.PostCicdPipeline,
+ ":connectionId/cicd_tasks": {
+ "POST": api.PostCicdTask,
+ },
+ ":connectionId/cicd_pipeline/:pipelineName/finish": {
+ "POST": api.PostPipelineFinish,
},
":connectionId/issues": {
"POST": api.PostIssue,