You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by kl...@apache.org on 2022/06/06 13:45:51 UTC
[incubator-devlake] branch main updated: refactor: rename tasks field from pipeline and move it to higher level (#2061)
This is an automated email from the ASF dual-hosted git repository.
klesh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 68d6725b refactor: rename tasks field from pipeline and move it to higher level (#2061)
68d6725b is described below
commit 68d6725be09dcb3e0becfbf6a4655439a9a969b4
Author: abeizn <10...@users.noreply.github.com>
AuthorDate: Mon Jun 6 21:45:47 2022 +0800
refactor: rename tasks field from pipeline and move it to higher level (#2061)
* refactor: rename tasks field from pipeline and move it to higher level
* feat(tapd): add story category
closes #2010
* fix: jira apiv2models json value (#2060)
* refactor: rename tasks field from pipeline and move it to higher level
* fix: plugins README
* refactor: rename tasks field from pipeline and move it to higher level
* refactor: rename tasks field from pipeline and move it to higher level
* refactor: rename tasks field from pipeline and move it to higher level
Co-authored-by: Yingchu Chen <yi...@merico.dev>
---
models/migrationscripts/register.go | 3 +-
models/migrationscripts/updateSchemas20220601.go | 68 +++++++++++++++++
models/pipeline.go | 23 +++---
models/task.go | 5 +-
plugins/ae/ae.go | 2 +-
plugins/dbt/dbt.go | 2 +-
plugins/feishu/README.md | 38 +++++++++-
plugins/feishu/feishu.go | 2 +-
plugins/gitextractor/README-zh-CN.md | 45 -----------
plugins/gitextractor/README.md | 58 ++++++++++++++-
plugins/github/README.md | 35 +++++++++
plugins/github/github.go | 2 +-
plugins/gitlab/README.md | 50 ++++++++++++-
plugins/gitlab/gitlab.go | 32 ++++++--
plugins/jenkins/README.md | 50 ++++++++++++-
plugins/jenkins/jenkins.go | 2 +-
plugins/jira/README.md | 66 +++++++++++++++-
plugins/jira/jira.go | 2 +-
plugins/refdiff/README.md | 95 ++++++++++++++++++++++--
plugins/refdiff/refdiff.go | 2 +-
plugins/tapd/tapd.go | 2 +-
runner/directrun.go | 9 ++-
runner/run_task.go | 38 ++++++----
services/task.go | 6 ++
24 files changed, 524 insertions(+), 113 deletions(-)
diff --git a/models/migrationscripts/register.go b/models/migrationscripts/register.go
index e014cc4e..ebbfcbd7 100644
--- a/models/migrationscripts/register.go
+++ b/models/migrationscripts/register.go
@@ -25,8 +25,7 @@ func RegisterAll() {
new(initSchemas),
new(updateSchemas20220505), new(updateSchemas20220507), new(updateSchemas20220510),
new(updateSchemas20220513), new(updateSchemas20220524), new(updateSchemas20220526),
- new(updateSchemas20220527),
- new(updateSchemas20220528),
+ new(updateSchemas20220527), new(updateSchemas20220528), new(updateSchemas20220601),
new(updateSchemas20220602),
}, "Framework")
}
diff --git a/models/migrationscripts/updateSchemas20220601.go b/models/migrationscripts/updateSchemas20220601.go
new file mode 100644
index 00000000..55a77d80
--- /dev/null
+++ b/models/migrationscripts/updateSchemas20220601.go
@@ -0,0 +1,68 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+ "context"
+ "time"
+
+ "github.com/apache/incubator-devlake/models/common"
+ "gorm.io/datatypes"
+ "gorm.io/gorm"
+)
+
+type Task20220601 struct {
+ common.Model
+ Plugin string `json:"plugin" gorm:"index"`
+ Subtasks datatypes.JSON `json:"subtasks"`
+ Options datatypes.JSON `json:"options"`
+ Status string `json:"status"`
+ Message string `json:"message"`
+ Progress float32 `json:"progress"`
+ FailedSubTask string `json:"failedSubTask"`
+ PipelineId uint64 `json:"pipelineId" gorm:"index"`
+ PipelineRow int `json:"pipelineRow"`
+ PipelineCol int `json:"pipelineCol"`
+ BeganAt *time.Time `json:"beganAt"`
+ FinishedAt *time.Time `json:"finishedAt" gorm:"index"`
+ SpentSeconds int `json:"spentSeconds"`
+}
+
+func (Task20220601) TableName() string {
+ return "_devlake_tasks"
+}
+
+type updateSchemas20220601 struct{}
+
+func (*updateSchemas20220601) Up(ctx context.Context, db *gorm.DB) error {
+
+ err := db.Migrator().AddColumn(Task20220601{}, "subtasks")
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func (*updateSchemas20220601) Version() uint64 {
+ return 20220601000005
+}
+
+func (*updateSchemas20220601) Name() string {
+ return "add column `subtasks` at _devlake_tasks"
+}
diff --git a/models/pipeline.go b/models/pipeline.go
index da1656b6..6a271ac7 100644
--- a/models/pipeline.go
+++ b/models/pipeline.go
@@ -27,18 +27,17 @@ import (
type Pipeline struct {
common.Model
- Name string `json:"name" gorm:"index"`
- BlueprintId uint64 `json:"blueprintId"`
- Tasks datatypes.JSON `json:"tasks"`
- TotalTasks int `json:"totalTasks"`
- // Deprecated
- FinishedTasks int `json:"finishedTasks"`
- BeganAt *time.Time `json:"beganAt"`
- FinishedAt *time.Time `json:"finishedAt" gorm:"index"`
- Status string `json:"status"`
- Message string `json:"message"`
- SpentSeconds int `json:"spentSeconds"`
- Stage int `json:"stage"`
+ Name string `json:"name" gorm:"index"`
+ BlueprintId uint64 `json:"blueprintId"`
+ Tasks datatypes.JSON `json:"tasks"`
+ TotalTasks int `json:"totalTasks"`
+ FinishedTasks int `json:"finishedTasks"`
+ BeganAt *time.Time `json:"beganAt"`
+ FinishedAt *time.Time `json:"finishedAt" gorm:"index"`
+ Status string `json:"status"`
+ Message string `json:"message"`
+ SpentSeconds int `json:"spentSeconds"`
+ Stage int `json:"stage"`
}
// We use a 2D array because the request body must be an array of a set of tasks
diff --git a/models/task.go b/models/task.go
index 0d77a607..d4e32420 100644
--- a/models/task.go
+++ b/models/task.go
@@ -43,6 +43,7 @@ type TaskProgressDetail struct {
type Task struct {
common.Model
Plugin string `json:"plugin" gorm:"index"`
+ Subtasks datatypes.JSON `json:"subtasks"`
Options datatypes.JSON `json:"options"`
Status string `json:"status"`
Message string `json:"message"`
@@ -60,8 +61,8 @@ type Task struct {
type NewTask struct {
// Plugin name
- Plugin string `json:"plugin" binding:"required"`
- // Options for the plugin task to be triggered
+ Plugin string `json:"plugin" binding:"required"`
+ Subtasks []string `json:"subtasks"`
Options map[string]interface{} `json:"options"`
PipelineId uint64 `json:"-"`
PipelineRow int `json:"-"`
diff --git a/plugins/ae/ae.go b/plugins/ae/ae.go
index e2bad02f..f55262c5 100644
--- a/plugins/ae/ae.go
+++ b/plugins/ae/ae.go
@@ -108,7 +108,7 @@ func main() {
projectId := aeCmd.Flags().IntP("project-id", "p", 0, "ae project id")
_ = aeCmd.MarkFlagRequired("project-id")
aeCmd.Run = func(cmd *cobra.Command, args []string) {
- runner.DirectRun(cmd, args, PluginEntry, map[string]interface{}{
+ runner.DirectRun(cmd, args, PluginEntry, []string{}, map[string]interface{}{
"projectId": *projectId,
})
}
diff --git a/plugins/dbt/dbt.go b/plugins/dbt/dbt.go
index c3ce3d3e..b0a4be40 100644
--- a/plugins/dbt/dbt.go
+++ b/plugins/dbt/dbt.go
@@ -97,7 +97,7 @@ func main() {
for k, v := range projectVars {
projectVarsConvert[k] = v
}
- runner.DirectRun(cmd, args, PluginEntry, map[string]interface{}{
+ runner.DirectRun(cmd, args, PluginEntry, []string{}, map[string]interface{}{
"projectPath": *projectPath,
"projectName": *projectName,
"projectTarget": *projectTarget,
diff --git a/plugins/feishu/README.md b/plugins/feishu/README.md
index fbeb234b..d5349c60 100644
--- a/plugins/feishu/README.md
+++ b/plugins/feishu/README.md
@@ -32,7 +32,7 @@ FEISHU_APPSCRECT=app_secret
In order to collect data, you have to compose a JSON looks like following one, and send it by selecting `Advanced Mode` on `Create Pipeline Run` page:
numOfDaysToCollect： The number of days you want to collect
rateLimitPerSecond: The number of requests to send(Maximum is 8)
-
+1. Configure-UI Mode
```json
[
[
@@ -47,6 +47,23 @@ rateLimitPerSecond: The number of requests to send(Maximum is 8)
]
```
+and if you want to perform certain subtasks.
+```
+[
+ [
+ {
+ "plugin": "feishu",
+ "subtasks": ["collectXXX", "extractXXX", "convertXXX"],
+ "options": {
+ "numOfDaysToCollect" : 80,
+ "rateLimitPerSecond" : 5
+ }
+ }
+ ]
+]
+```
+
+2. Curl Mode:
You can also trigger data collection by making a POST request to `/pipelines`.
```
curl --location --request POST 'localhost:8080/pipelines' \
@@ -63,4 +80,23 @@ curl --location --request POST 'localhost:8080/pipelines' \
}]]
}
'
+```
+
+and if you want to perform certain subtasks.
+```
+curl --location --request POST 'localhost:8080/pipelines' \
+--header 'Content-Type: application/json' \
+--data-raw '
+{
+ "name": "feishu 20211126",
+ "tasks": [[{
+ "plugin": "feishu",
+ "subtasks": ["collectXXX", "extractXXX", "convertXXX"],
+ "options": {
+ "numOfDaysToCollect" : 80,
+ "rateLimitPerSecond" : 5
+ }
+ }]]
+}
+'
```
\ No newline at end of file
diff --git a/plugins/feishu/feishu.go b/plugins/feishu/feishu.go
index b5a5828f..3b1635d1 100644
--- a/plugins/feishu/feishu.go
+++ b/plugins/feishu/feishu.go
@@ -90,7 +90,7 @@ func main() {
numOfDaysToCollect := feishuCmd.Flags().IntP("numOfDaysToCollect", "n", 8, "feishu collect days")
_ = feishuCmd.MarkFlagRequired("numOfDaysToCollect")
feishuCmd.Run = func(cmd *cobra.Command, args []string) {
- runner.DirectRun(cmd, args, PluginEntry, map[string]interface{}{
+ runner.DirectRun(cmd, args, PluginEntry, []string{}, map[string]interface{}{
"numOfDaysToCollect": *numOfDaysToCollect,
})
}
diff --git a/plugins/gitextractor/README-zh-CN.md b/plugins/gitextractor/README-zh-CN.md
deleted file mode 100644
index 20a83b4a..00000000
--- a/plugins/gitextractor/README-zh-CN.md
+++ /dev/null
@@ -1,45 +0,0 @@
-# Git Repo Extractor插件
-
-## Summary
-
-该插件可以从远端或本地git仓库提取commit和reference信息，并保存到数据库或csv文件。
-
-## 示例
-
-```
-curl --location --request POST 'localhost:8080/pipelines' \
---header 'Content-Type: application/json' \
---data-raw '
-{
- "name": "git repo extractor",
- "tasks": [
- [
- {
- "Plugin": "gitextractor",
- "Options": {
- "url": "https://github.com/apache/incubator-devlake.git",
- "repoId": "github:GithubRepos:384111310"
- }
- }
- ]
- ]
-}
-'
-```
-- `url`: git仓库的位置，如果是远端仓库应当以`http`或`https`开头， 如果是本地仓库则应当以`/`开头
-- `repoId`: `repos`表的`id`字段.
-- `proxy`: 可选, 只支持`http`代理，例如：`http://your-proxy-server.com:1080`.
-- `user`: 可选, 通过HTTP/HTTPS协议克隆私有代码库时使用
-- `password`: 可选, 通过HTTP/HTTPS协议克隆私有代码库时使用
-- `privateKey`: 可选, 通过SSH协议克隆代码库时使用, 值为经过base64编码的`PEM`文件
-- `passphrase`: 可选, 私钥的密码
-
-## 独立运行本插件
-
-本插件可以作为独立于DevLake服务的命令行工具使用:
-
-```
-go run plugins/gitextractor/main.go -url https://github.com/apache/incubator-devlake.git -id github:GithubRepo:384111310 -db "merico:merico@tcp(127.0.0.1:3306)/lake?charset=utf8mb4&parseTime=True"
-```
-
-如果想了解命令行工具的更多选项，比如如何输出收集结果到csv文件，请直接阅读`plugins/gitextractor/main.go`。
\ No newline at end of file
diff --git a/plugins/gitextractor/README.md b/plugins/gitextractor/README.md
index 5866a899..9c18c416 100644
--- a/plugins/gitextractor/README.md
+++ b/plugins/gitextractor/README.md
@@ -10,7 +10,60 @@ This plugin extract commits and references from a remote or local git repository
3. Use the [RefDiff](../refdiff) plugin to calculate version diff, which will be stored in `refs_commits_diffs` table.
## Sample Request
+1. Configure-UI Mode
+In order to collect data, you have to compose a JSON looks like following one, and send it by selecting `Advanced Mode` on `Create Pipeline Run` page:
+```
+[
+ [
+ {
+ "Plugin": "gitextractor",
+ "Options": {
+ "url": "https://github.com/apache/incubator-devlake.git",
+ "repoId": "github:GithubRepo:384111310"
+ }
+ }
+ ]
+]
+```
+and if you want to perform certain subtasks.
+```
+[
+ [
+ {
+ "plugin": "gitextractor",
+ "subtasks": ["collectXXX", "extractXXX", "convertXXX"],
+ "options": {
+ "url": "https://github.com/apache/incubator-devlake.git",
+ "repoId": "github:GithubRepo:384111310"
+ }
+ }
+ ]
+]
+```
+2. Curl Mode:
+You can also trigger data collection by making a POST request to `/pipelines`.
+```
+curl --location --request POST 'localhost:8080/pipelines' \
+--header 'Content-Type: application/json' \
+--data-raw '
+{
+ "name": "git repo extractor",
+ "tasks": [
+ [
+ {
+ "plugin": "gitextractor",
+ "options": {
+ "url": "https://github.com/apache/incubator-devlake.git",
+ "repoId": "github:GithubRepo:384111310"
+ }
+ }
+ ]
+ ]
+}
+'
+```
+and if you want to perform certain subtasks.
```
curl --location --request POST 'localhost:8080/pipelines' \
--header 'Content-Type: application/json' \
@@ -20,8 +73,9 @@ curl --location --request POST 'localhost:8080/pipelines' \
"tasks": [
[
{
- "Plugin": "gitextractor",
- "Options": {
+ "plugin": "gitextractor",
+ "subtasks": ["collectXXX", "extractXXX", "convertXXX"],
+ "options": {
"url": "https://github.com/apache/incubator-devlake.git",
"repoId": "github:GithubRepo:384111310"
}
diff --git a/plugins/github/README.md b/plugins/github/README.md
index 0ada072d..0e8660f7 100644
--- a/plugins/github/README.md
+++ b/plugins/github/README.md
@@ -70,6 +70,7 @@ Define regex pattern in .env
## Sample Request
In order to collect data, you have to compose a JSON looks like following one, and send it by selecting `Advanced Mode` on `Create Pipeline Run` page:
+1. Configure-UI Mode
```json
[
[
@@ -83,7 +84,23 @@ In order to collect data, you have to compose a JSON looks like following one, a
]
]
```
+and if you want to perform certain subtasks.
+```json
+[
+ [
+ {
+ "plugin": "github",
+ "subtasks": ["collectXXX", "extractXXX", "convertXXX"],
+ "options": {
+ "repo": "lake",
+ "owner": "merico-dev"
+ }
+ }
+ ]
+]
+```
+2. Curl Mode:
You can also trigger data collection by making a POST request to `/pipelines`.
```
curl --location --request POST 'localhost:8080/pipelines' \
@@ -101,3 +118,21 @@ curl --location --request POST 'localhost:8080/pipelines' \
}
'
```
+and if you want to perform certain subtasks.
+```
+curl --location --request POST 'localhost:8080/pipelines' \
+--header 'Content-Type: application/json' \
+--data-raw '
+{
+ "name": "github 20211126",
+ "tasks": [[{
+ "plugin": "github",
+ "subtasks": ["collectXXX", "extractXXX", "convertXXX"],
+ "options": {
+ "repo": "lake",
+ "owner": "merico-dev"
+ }
+ }]]
+}
+'
+```
diff --git a/plugins/github/github.go b/plugins/github/github.go
index 765a4d28..e0806c6f 100644
--- a/plugins/github/github.go
+++ b/plugins/github/github.go
@@ -144,7 +144,7 @@ func main() {
_ = githubCmd.MarkFlagRequired("repo")
githubCmd.Run = func(cmd *cobra.Command, args []string) {
- runner.DirectRun(cmd, args, PluginEntry, map[string]interface{}{
+ runner.DirectRun(cmd, args, PluginEntry, []string{}, map[string]interface{}{
"owner": *owner,
"repo": *repo,
})
diff --git a/plugins/gitlab/README.md b/plugins/gitlab/README.md
index 0e99ec2b..01053513 100644
--- a/plugins/gitlab/README.md
+++ b/plugins/gitlab/README.md
@@ -54,9 +54,37 @@ There are no additional settings for the GitLab Datasource Provider at this time
NOTE: `GitLab Project ID` Mappings feature has been deprecated.
## Gathering Data with Gitlab
+In order to collect data, you have to compose a JSON looks like following one, and send it by selecting `Advanced Mode` on `Create Pipeline Run` page:
+1. Configure-UI Mode
+```json
+[
+ [
+ {
+ "plugin": "gitlab",
+ "options": {
+ "projectId": <Your gitlab project id>
+ }
+ }
+ ]
+]
+```
+and if you want to perform certain subtasks.
+```json
+[
+ [
+ {
+ "plugin": "gitlab",
+ "subtasks": ["collectXXX", "extractXXX", "convertXXX"],
+ "options": {
+ "projectId": <Your gitlab project id>
+ }
+ }
+ ]
+]
+```
-To collect data, you can make a POST request to `/pipelines`
-
+2. Curl Mode:
+You can also trigger data collection by making a POST request to `/pipelines`.
```
curl --location --request POST 'localhost:8080/pipelines' \
--header 'Content-Type: application/json' \
@@ -72,6 +100,24 @@ curl --location --request POST 'localhost:8080/pipelines' \
}
'
```
+and if you want to perform certain subtasks.`
+```
+curl --location --request POST 'localhost:8080/pipelines' \
+--header 'Content-Type: application/json' \
+--data-raw '
+{
+ "name": "gitlab 20211126",
+ "tasks": [[{
+ "plugin": "gitlab",
+ "subtasks": ["collectXXX", "extractXXX", "convertXXX"],
+ "options": {
+ "projectId": <Your gitlab project id>
+ }
+ }]]
+}
+'
+```
+
## Finding Project Id
diff --git a/plugins/gitlab/gitlab.go b/plugins/gitlab/gitlab.go
index 88ce9f10..7d2be2e5 100644
--- a/plugins/gitlab/gitlab.go
+++ b/plugins/gitlab/gitlab.go
@@ -18,7 +18,10 @@ limitations under the License.
package main // must be main for plugin entry point
import (
+ "github.com/apache/incubator-devlake/config"
+ "github.com/apache/incubator-devlake/logger"
"github.com/apache/incubator-devlake/plugins/gitlab/impl"
+ "github.com/apache/incubator-devlake/plugins/tapd/models"
"github.com/apache/incubator-devlake/runner"
"github.com/spf13/cobra"
)
@@ -29,13 +32,28 @@ var PluginEntry impl.Gitlab //nolint
// standalone mode for debugging
func main() {
gitlabCmd := &cobra.Command{Use: "gitlab"}
- projectId := gitlabCmd.Flags().IntP("project-id", "p", 0, "gitlab project id")
-
- _ = gitlabCmd.MarkFlagRequired("project-id")
- gitlabCmd.Run = func(cmd *cobra.Command, args []string) {
- runner.DirectRun(cmd, args, PluginEntry, map[string]interface{}{
- "projectId": *projectId,
- })
+ gitlabCmd.Run = func(c *cobra.Command, args []string) {
+ cfg := config.GetConfig()
+ log := logger.Global.Nested(gitlabCmd.Use)
+ db, err := runner.NewGormDb(cfg, log)
+ if err != nil {
+ panic(err)
+ }
+ wsList := make([]*models.TapdWorkspace, 0)
+ err = db.Find(&wsList, "parent_id = ?", 59169984).Error
+ projectList := []uint64{63281714,
+ 34276182,
+ 46319043,
+ 50328292,
+ 63984859,
+ 55805854,
+ 38496185,
+ }
+ for _, v := range projectList {
+ runner.DirectRun(gitlabCmd, args, PluginEntry, []string{}, map[string]interface{}{
+ "projectId": v,
+ })
+ }
}
runner.RunCmd(gitlabCmd)
diff --git a/plugins/jenkins/README.md b/plugins/jenkins/README.md
index dd62e06b..812500bb 100644
--- a/plugins/jenkins/README.md
+++ b/plugins/jenkins/README.md
@@ -44,10 +44,8 @@ The connection aspect of the configuration screen requires the following key fie
Click Save Connection to update connection settings.
## Collect Data From Jenkins
-
-In order to collect data from Jenkins, you have to compose a JSON looks like following one, and send it via `Triggers` page on `config-ui`:
-
-
+In order to collect data, you have to compose a JSON looks like following one, and send it by selecting `Advanced Mode` on `Create Pipeline Run` page:
+1. Configure-UI Mode
```json
[
[
@@ -58,6 +56,50 @@ In order to collect data from Jenkins, you have to compose a JSON looks like fol
]
]
```
+and if you want to perform certain subtasks.
+```json
+[
+ [
+ {
+ "plugin": "jenkins",
+ "subtasks": ["collectXXX", "extractXXX", "convertXXX"],
+ "options": {}
+ }
+ ]
+]
+```
+
+2. Curl Mode:
+You can also trigger data collection by making a POST request to `/pipelines`.
+```
+curl --location --request POST 'localhost:8080/pipelines' \
+--header 'Content-Type: application/json' \
+--data-raw '
+{
+ "name": "jenkins 20211126",
+ "tasks": [[{
+ "plugin": "jenkins",
+ "options": {}
+ }]]
+}
+'
+```
+and if you want to perform certain subtasks.
+```
+curl --location --request POST 'localhost:8080/pipelines' \
+--header 'Content-Type: application/json' \
+--data-raw '
+{
+ "name": "jenkins 20211126",
+ "tasks": [[{
+ "plugin": "jenkins",
+ "subtasks": ["collectXXX", "extractXXX", "convertXXX"],
+ "options": {}
+ }]]
+}
+'
+```
+
## Relationship between job and build
diff --git a/plugins/jenkins/jenkins.go b/plugins/jenkins/jenkins.go
index 4edc89df..b1e2d6a8 100644
--- a/plugins/jenkins/jenkins.go
+++ b/plugins/jenkins/jenkins.go
@@ -102,7 +102,7 @@ var PluginEntry Jenkins //nolint
func main() {
jenkinsCmd := &cobra.Command{Use: "jenkins"}
jenkinsCmd.Run = func(cmd *cobra.Command, args []string) {
- runner.DirectRun(cmd, args, PluginEntry, map[string]interface{}{})
+ runner.DirectRun(cmd, args, PluginEntry, []string{}, map[string]interface{}{})
}
runner.RunCmd(jenkinsCmd)
}
diff --git a/plugins/jira/README.md b/plugins/jira/README.md
index e9901c9a..6b133d63 100644
--- a/plugins/jira/README.md
+++ b/plugins/jira/README.md
@@ -67,14 +67,75 @@ Please follow this guide: [How to find Jira the custom field ID in Jira? · meri
## Collect Data From JIRA
-In order to collect data from JIRA, you have to compose a JSON looks like following one, and send it via `Triggers` page on `config-ui`:
+In order to collect data, you have to compose a JSON looks like following one, and send it by selecting `Advanced Mode` on `Create Pipeline Run` page:
<font color=“red”>Warning: Data collection only supports single-task execution, and the results of concurrent multi-task execution may not meet expectations.</font>
+1. Configure-UI Mode:
+```json
+[
+ [
+ {
+ "plugin": "jira",
+ "options": {
+ "connectionId": 1,
+ "boardId": 8,
+ "since": "2006-01-02T15:04:05Z"
+ }
+ }
+ ]
+]
```
+and if you want to perform certain subtasks.
+```json
[
[
{
"plugin": "jira",
+ "subtasks": ["collectXXX", "extractXXX", "convertXXX"],
+ "options": {
+ "connectionId": 1,
+ "boardId": 8,
+ "since": "2006-01-02T15:04:05Z"
+ }
+ }
+ ]
+]
+```
+
+2. Curl Mode:
+```
+curl --location --request POST 'localhost:8080/pipelines' \
+--header 'Content-Type: application/json' \
+--data-raw '
+{
+ "name": "jenkins 20211126",
+ "tasks": [
+ [
+ {
+ "plugin": "jira",
+ "options": {
+ "connectionId": 1,
+ "boardId": 8,
+ "since": "2006-01-02T15:04:05Z"
+ }
+ }
+ ]
+]
+}
+'
+```
+and if you want to perform certain subtasks.
+```
+curl --location --request POST 'localhost:8080/pipelines' \
+--header 'Content-Type: application/json' \
+--data-raw '
+{
+ "name": "jenkins 20211126",
+ "tasks": [
+ [
+ {
+ "plugin": "jira",
+ "subtasks": ["collectXXX", "extractXXX", "convertXXX"],
"options": {
"connectionId": 1,
"boardId": 8,
@@ -83,8 +144,11 @@ In order to collect data from JIRA, you have to compose a JSON looks like follow
}
]
]
+}
+'
```
+
- `connectionId`: The `ID` field from **JIRA Integration** page.
- `boardId`: JIRA board id, see [Find Board Id](#find-board-id) for detail.
- `since`: optional, download data since specified date/time only.
diff --git a/plugins/jira/jira.go b/plugins/jira/jira.go
index 9f758190..2b865ea6 100644
--- a/plugins/jira/jira.go
+++ b/plugins/jira/jira.go
@@ -217,7 +217,7 @@ func main() {
_ = cmd.MarkFlagRequired("connection")
_ = cmd.MarkFlagRequired("board")
cmd.Run = func(c *cobra.Command, args []string) {
- runner.DirectRun(c, args, PluginEntry, map[string]interface{}{
+ runner.DirectRun(c, args, PluginEntry, []string{}, map[string]interface{}{
"connectionId": *connectionId,
"boardId": *boardId,
})
diff --git a/plugins/refdiff/README.md b/plugins/refdiff/README.md
index e0e9dca6..01f649e3 100644
--- a/plugins/refdiff/README.md
+++ b/plugins/refdiff/README.md
@@ -37,31 +37,110 @@ github:GithubRepo:384111310:refs/tags/v0.6.1 TAG
2. If you want to run calculateIssuesDiff, please configure GITHUB_PR_BODY_CLOSE_PATTERN in .env, you can check the example in .env.example(we have a default value, please make sure your pattern is disclosed by single quotes '')
3. If you want to run calculatePrCherryPick, please configure GITHUB_PR_TITLE_PATTERN in .env, you can check the example in .env.example(we have a default value, please make sure your pattern is disclosed by single quotes '')
4. And then, trigger a pipeline like following, you can also define sub tasks, calculateRefDiff will calculate commits between two ref, and creatRefBugStats will create a table to show bug list between two ref:
+
+In order to collect data, you have to compose a JSON looks like following one, and send it by selecting `Advanced Mode` on `Create Pipeline Run` page:
+1. Configure-UI Mode:
+```json
+[
+ [
+ {
+ "plugin": "refdiff",
+ "options": {
+ "repoId": "github:GithubRepo:384111310",
+ "pairs": [
+ {
+ "newRef": "refs/tags/v0.6.0",
+ "oldRef": "refs/tags/0.5.0"
+ },
+ {
+ "newRef": "refs/tags/0.5.0",
+ "oldRef": "refs/tags/0.4.0"
+ }
+ ]
+ }
+ }
+ ]
+]
+```
+and if you want to perform certain subtasks.
+```json
+[
+ [
+ {
+ "plugin": "refdiff",
+ "subtasks": [
+ "calculateCommitsDiff",
+ "calculateIssuesDiff",
+ "calculatePrCherryPick"
+ ],
+ "options": {
+ "repoId": "github:GithubRepo:384111310",
+ "pairs": [
+ {
+ "newRef": "refs/tags/v0.6.0",
+ "oldRef": "refs/tags/0.5.0"
+ },
+ {
+ "newRef": "refs/tags/0.5.0",
+ "oldRef": "refs/tags/0.4.0"
+ }
+ ]
+ }
+ }
+ ]
+]
+```
+
+2. Curl Mode:
+```
+curl --location --request POST 'localhost:8080/pipelines' \
+--header 'Content-Type: application/json' \
+--data-raw '
+{
+ "name": "test-refdiff",
+ "tasks": [
+ [
+ {
+ "plugin": "refdiff",
+ "options": {
+ "repoId": "github:GithubRepo:384111310",
+ "pairs": [
+ { "newRef": "refs/tags/v0.6.0", "oldRef": "refs/tags/0.5.0" },
+ { "newRef": "refs/tags/0.5.0", "oldRef": "refs/tags/0.4.0" }
+ ]
+ }
+ }
+ ]
+ ]
+}'
+```
+and if you want to perform certain subtasks.
```
-curl -v -XPOST http://localhost:8080/pipelines --data @- <<'JSON'
+curl --location --request POST 'localhost:8080/pipelines' \
+--header 'Content-Type: application/json' \
+--data-raw '
{
"name": "test-refdiff",
"tasks": [
[
{
"plugin": "refdiff",
+ "subtasks": [
+ "calculateCommitsDiff",
+ "calculateIssuesDiff",
+ "calculatePrCherryPick"
+ ],
"options": {
"repoId": "github:GithubRepo:384111310",
"pairs": [
{ "newRef": "refs/tags/v0.6.0", "oldRef": "refs/tags/0.5.0" },
{ "newRef": "refs/tags/0.5.0", "oldRef": "refs/tags/0.4.0" }
- ],
- "tasks": [
- "calculateCommitsDiff",
- "calculateIssuesDiff",
- "calculatePrCherryPick",
]
}
}
]
]
-}
-JSON
+}'
```
## Development
diff --git a/plugins/refdiff/refdiff.go b/plugins/refdiff/refdiff.go
index 881feab1..b2a2ef44 100644
--- a/plugins/refdiff/refdiff.go
+++ b/plugins/refdiff/refdiff.go
@@ -88,7 +88,7 @@ func main() {
_ = refdiffCmd.MarkFlagRequired("old-ref")
refdiffCmd.Run = func(cmd *cobra.Command, args []string) {
- runner.DirectRun(cmd, args, PluginEntry, map[string]interface{}{
+ runner.DirectRun(cmd, args, PluginEntry, []string{}, map[string]interface{}{
"repoId": repoId,
"pairs": []map[string]string{
{
diff --git a/plugins/tapd/tapd.go b/plugins/tapd/tapd.go
index e20194be..e7212fa1 100644
--- a/plugins/tapd/tapd.go
+++ b/plugins/tapd/tapd.go
@@ -222,7 +222,7 @@ func main() {
err = db.Find(&wsList, "parent_id = ?", 59169984).Error
for _, v := range wsList {
*workspaceId = v.ID
- runner.DirectRun(c, args, PluginEntry, map[string]interface{}{
+ runner.DirectRun(c, args, PluginEntry, []string{}, map[string]interface{}{
"connectionId": *connectionId,
"workspaceId": *workspaceId,
"companyId": *companyId,
diff --git a/runner/directrun.go b/runner/directrun.go
index 5e5851e0..6ab5b83d 100644
--- a/runner/directrun.go
+++ b/runner/directrun.go
@@ -30,7 +30,7 @@ import (
)
func RunCmd(cmd *cobra.Command) {
- cmd.Flags().StringSliceP("tasks", "t", nil, "specify what tasks to run, --tasks=collectIssues,extractIssues")
+ cmd.Flags().StringSliceP("subtasks", "t", nil, "specify what tasks to run, --subtasks=collectIssues,extractIssues")
err := cmd.Execute()
if err != nil {
panic(err)
@@ -42,12 +42,12 @@ func RunCmd(cmd *cobra.Command) {
// args: command line arguments
// pluginTask: specific built-in plugin, for example: feishu, jira...
// options: plugin config
-func DirectRun(cmd *cobra.Command, args []string, pluginTask core.PluginTask, options map[string]interface{}) {
- tasks, err := cmd.Flags().GetStringSlice("tasks")
+func DirectRun(cmd *cobra.Command, args []string, pluginTask core.PluginTask, subtasks []string, options map[string]interface{}) {
+ tasks, err := cmd.Flags().GetStringSlice("subtasks")
if err != nil {
panic(err)
}
- options["tasks"] = tasks
+ subtasks = tasks
cfg := config.GetConfig()
log := logger.Global.Nested(cmd.Use)
db, err := NewGormDb(cfg, log)
@@ -102,6 +102,7 @@ func DirectRun(cmd *cobra.Command, args []string, pluginTask core.PluginTask, op
db,
ctx,
cmd.Use,
+ subtasks,
options,
pluginTask,
nil,
diff --git a/runner/run_task.go b/runner/run_task.go
index e021b0a0..568d47b6 100644
--- a/runner/run_task.go
+++ b/runner/run_task.go
@@ -101,6 +101,11 @@ func RunTask(
if err != nil {
return err
}
+ var subtasks []string
+ err = json.Unmarshal(task.Subtasks, &subtasks)
+ if err != nil {
+ return err
+ }
err = RunPluginTask(
config.GetConfig(),
@@ -108,6 +113,7 @@ func RunTask(
db,
ctx,
task.Plugin,
+ subtasks,
options,
progress,
)
@@ -120,6 +126,7 @@ func RunPluginTask(
db *gorm.DB,
ctx context.Context,
name string,
+ subtasks []string,
options map[string]interface{},
progress chan core.RunningProgress,
) error {
@@ -138,6 +145,7 @@ func RunPluginTask(
db,
ctx,
name,
+ subtasks,
options,
pluginTask,
progress,
@@ -150,6 +158,7 @@ func RunPluginSubTasks(
db *gorm.DB,
ctx context.Context,
name string,
+ subtasks []string,
options map[string]interface{},
pluginTask core.PluginTask,
progress chan core.RunningProgress,
@@ -158,36 +167,35 @@ func RunPluginSubTasks(
// find out all possible subtasks this plugin can offer
subtaskMetas := pluginTask.SubTaskMetas()
- subtasks := make(map[string]bool)
+ subtasksFlag := make(map[string]bool)
for _, subtaskMeta := range subtaskMetas {
- subtasks[subtaskMeta.Name] = subtaskMeta.EnabledByDefault
+ subtasksFlag[subtaskMeta.Name] = subtaskMeta.EnabledByDefault
}
- /* subtasks example
- subtasks := map[string]bool{
+ /* subtasksFlag example
+ subtasksFlag := map[string]bool{
"collectProject": true,
"convertCommits": true,
...
}
*/
- // if user specified what subtasks to run, obey
- // TODO: move tasks field to outer level, and rename it to subtasks
- if _, ok := options["tasks"]; ok {
+ // user specifies what subtasks to run
+ if len(subtasks) != 0 {
// decode user specified subtasks
var specifiedTasks []string
- err := mapstructure.Decode(options["tasks"], &specifiedTasks)
+ err := mapstructure.Decode(subtasks, &specifiedTasks)
if err != nil {
return err
}
if len(specifiedTasks) > 0 {
// first, disable all subtasks
- for task := range subtasks {
- subtasks[task] = false
+ for task := range subtasksFlag {
+ subtasksFlag[task] = false
}
// second, check specified subtasks is valid and enable them if so
for _, task := range specifiedTasks {
- if _, ok := subtasks[task]; ok {
- subtasks[task] = true
+ if _, ok := subtasksFlag[task]; ok {
+ subtasksFlag[task] = true
} else {
return fmt.Errorf("subtask %s does not exist", task)
}
@@ -197,18 +205,18 @@ func RunPluginSubTasks(
// make sure `Required` subtasks are always enabled
for _, subtaskMeta := range subtaskMetas {
if subtaskMeta.Required {
- subtasks[subtaskMeta.Name] = true
+ subtasksFlag[subtaskMeta.Name] = true
}
}
// calculate total step(number of task to run)
steps := 0
- for _, enabled := range subtasks {
+ for _, enabled := range subtasksFlag {
if enabled {
steps++
}
}
- taskCtx := helper.NewDefaultTaskContext(cfg, logger, db, ctx, name, subtasks, progress)
+ taskCtx := helper.NewDefaultTaskContext(cfg, logger, db, ctx, name, subtasksFlag, progress)
taskData, err := pluginTask.PrepareTaskData(taskCtx, options)
if err != nil {
return err
diff --git a/services/task.go b/services/task.go
index 176b99e1..a73fbae6 100644
--- a/services/task.go
+++ b/services/task.go
@@ -131,8 +131,14 @@ func CreateTask(newTask *models.NewTask) (*models.Task, error) {
if err != nil {
return nil, err
}
+ s, err := json.Marshal(newTask.Subtasks)
+ if err != nil {
+ return nil, err
+ }
+
task := models.Task{
Plugin: newTask.Plugin,
+ Subtasks: s,
Options: b,
Status: models.TASK_CREATED,
Message: "",