You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by ab...@apache.org on 2024/03/29 09:44:35 UTC
(incubator-devlake) 12/40: feat: add subtask logs details api (#7205)
This is an automated email from the ASF dual-hosted git repository.
abeizn pushed a commit to branch release-v1.0
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit 2fdc61fd7190a7c0491a097f16e245a47a968ac0
Author: abeizn <zi...@merico.dev>
AuthorDate: Tue Mar 26 11:29:31 2024 +0800
feat: add subtask logs details api (#7205)
* fix: add subtask logs details api and update some subtasks name
* fix: gitextractor name
* fix: subtask migration add fix panic
* feat: add completed rate
* fix: rename some plugins subtasks name
* feat: add total status field
* fix: update task status logic
* feat: get lastest subtask plugins
---
.../20240322_add_field_to_subtasks.go | 52 ++++++++
backend/core/models/migrationscripts/register.go | 1 +
backend/core/models/task.go | 72 ++++++++--
backend/core/plugin/plugin_task.go | 12 +-
backend/core/runner/run_task.go | 67 +++++++++-
backend/helpers/pluginhelper/api/api_collector.go | 2 +-
.../plugins/bitbucket/tasks/account_convertor.go | 2 +-
.../plugins/bitbucket/tasks/commit_collector.go | 2 +-
.../plugins/bitbucket/tasks/commit_convertor.go | 5 +-
.../plugins/bitbucket/tasks/commit_extractor.go | 5 +-
.../bitbucket/tasks/deployment_collector.go | 2 +-
.../bitbucket/tasks/deployment_convertor.go | 9 +-
.../bitbucket/tasks/deployment_extractor.go | 5 +-
backend/plugins/bitbucket/tasks/issue_collector.go | 2 +-
.../bitbucket/tasks/issue_comment_collector.go | 2 +-
.../bitbucket/tasks/issue_comment_convertor.go | 2 +-
.../bitbucket/tasks/issue_comment_extractor.go | 2 +-
backend/plugins/bitbucket/tasks/issue_convertor.go | 2 +-
backend/plugins/bitbucket/tasks/issue_extractor.go | 2 +-
.../plugins/bitbucket/tasks/pipeline_collector.go | 2 +-
.../plugins/bitbucket/tasks/pipeline_convertor.go | 2 +-
.../plugins/bitbucket/tasks/pipeline_extractor.go | 3 +-
.../bitbucket/tasks/pipeline_steps_collector.go | 2 +-
.../bitbucket/tasks/pipeline_steps_convertor.go | 2 +-
.../bitbucket/tasks/pipeline_steps_extractor.go | 2 +-
backend/plugins/bitbucket/tasks/pr_collector.go | 2 +-
.../bitbucket/tasks/pr_comment_collector.go | 2 +-
.../bitbucket/tasks/pr_comment_convertor.go | 2 +-
.../bitbucket/tasks/pr_comment_extractor.go | 2 +-
.../plugins/bitbucket/tasks/pr_commit_collector.go | 2 +-
.../plugins/bitbucket/tasks/pr_commit_convertor.go | 2 +-
.../plugins/bitbucket/tasks/pr_commit_extractor.go | 2 +-
backend/plugins/bitbucket/tasks/pr_convertor.go | 2 +-
backend/plugins/bitbucket/tasks/pr_extractor.go | 2 +-
backend/plugins/bitbucket/tasks/repo_convertor.go | 2 +-
backend/plugins/gitextractor/tasks/clone.go | 5 +-
.../gitextractor/tasks/git_repo_collector.go | 8 +-
backend/plugins/github/tasks/account_collector.go | 2 +-
backend/plugins/github/tasks/account_convertor.go | 2 +-
backend/plugins/github/tasks/account_extractor.go | 2 +-
.../plugins/github/tasks/account_org_collector.go | 2 +-
.../plugins/github/tasks/account_org_extractor.go | 2 +-
backend/plugins/github/tasks/cicd_job_collector.go | 2 +-
backend/plugins/github/tasks/cicd_job_convertor.go | 2 +-
backend/plugins/github/tasks/cicd_job_extractor.go | 2 +-
backend/plugins/github/tasks/cicd_run_collector.go | 2 +-
backend/plugins/github/tasks/cicd_run_convertor.go | 7 +-
backend/plugins/github/tasks/cicd_run_extractor.go | 2 +-
backend/plugins/github/tasks/comment_collector.go | 2 +-
backend/plugins/github/tasks/comment_extractor.go | 2 +-
backend/plugins/github/tasks/commit_collector.go | 2 +-
backend/plugins/github/tasks/commit_convertor.go | 2 +-
backend/plugins/github/tasks/commit_extractor.go | 2 +-
.../plugins/github/tasks/commit_stats_collector.go | 2 +-
.../plugins/github/tasks/commit_stats_extractor.go | 2 +-
.../plugins/github/tasks/deployment_convertor.go | 2 +-
backend/plugins/github/tasks/event_collector.go | 2 +-
backend/plugins/github/tasks/event_extractor.go | 2 +-
.../github/tasks/issue_assignee_convertor.go | 2 +-
backend/plugins/github/tasks/issue_collector.go | 2 +-
.../github/tasks/issue_comment_convertor.go | 2 +-
backend/plugins/github/tasks/issue_convertor.go | 2 +-
backend/plugins/github/tasks/issue_extractor.go | 2 +-
.../plugins/github/tasks/issue_label_convertor.go | 2 +-
.../plugins/github/tasks/milestone_collector.go | 2 +-
.../plugins/github/tasks/milestone_converter.go | 2 +-
.../plugins/github/tasks/milestone_extractor.go | 2 +-
backend/plugins/github/tasks/pr_collector.go | 2 +-
.../plugins/github/tasks/pr_comment_convertor.go | 2 +-
.../plugins/github/tasks/pr_commit_collector.go | 2 +-
.../plugins/github/tasks/pr_commit_convertor.go | 2 +-
.../plugins/github/tasks/pr_commit_extractor.go | 2 +-
backend/plugins/github/tasks/pr_convertor.go | 2 +-
backend/plugins/github/tasks/pr_extractor.go | 2 +-
backend/plugins/github/tasks/pr_issue_convertor.go | 2 +-
backend/plugins/github/tasks/pr_issue_enricher.go | 2 +-
backend/plugins/github/tasks/pr_label_convertor.go | 2 +-
.../plugins/github/tasks/pr_review_collector.go | 2 +-
.../github/tasks/pr_review_comment_collector.go | 2 +-
.../github/tasks/pr_review_comment_extractor.go | 2 +-
.../plugins/github/tasks/pr_review_convertor.go | 2 +-
.../plugins/github/tasks/pr_review_extractor.go | 2 +-
backend/plugins/github/tasks/repo_convertor.go | 2 +-
.../github_graphql/tasks/account_collector.go | 2 +-
.../github_graphql/tasks/account_extractor.go | 2 +-
.../github_graphql/tasks/deployment_collector.go | 2 +-
.../github_graphql/tasks/deployment_extractor.go | 2 +-
.../github_graphql/tasks/issue_collector.go | 2 +-
.../github_graphql/tasks/issue_extractor.go | 2 +-
.../plugins/github_graphql/tasks/job_collector.go | 2 +-
.../plugins/github_graphql/tasks/job_extractor.go | 2 +-
.../plugins/github_graphql/tasks/pr_collector.go | 2 +-
.../plugins/github_graphql/tasks/pr_extractor.go | 2 +-
backend/plugins/gitlab/tasks/account_collector.go | 2 +-
backend/plugins/gitlab/tasks/account_convertor.go | 2 +-
backend/plugins/gitlab/tasks/account_extractor.go | 2 +-
backend/plugins/gitlab/tasks/commit_collector.go | 2 +-
backend/plugins/gitlab/tasks/commit_convertor.go | 5 +-
backend/plugins/gitlab/tasks/commit_extractor.go | 3 +-
.../plugins/gitlab/tasks/deployment_collector.go | 2 +-
.../plugins/gitlab/tasks/deployment_convertor.go | 7 +-
.../plugins/gitlab/tasks/deployment_extractor.go | 2 +-
.../gitlab/tasks/issue_assignee_convertor.go | 2 +-
backend/plugins/gitlab/tasks/issue_collector.go | 2 +-
backend/plugins/gitlab/tasks/issue_convertor.go | 2 +-
backend/plugins/gitlab/tasks/issue_extractor.go | 2 +-
.../plugins/gitlab/tasks/issue_label_convertor.go | 5 +-
backend/plugins/gitlab/tasks/job_collector.go | 2 +-
backend/plugins/gitlab/tasks/job_convertor.go | 7 +-
backend/plugins/gitlab/tasks/job_extractor.go | 2 +-
backend/plugins/gitlab/tasks/mr_collector.go | 2 +-
.../plugins/gitlab/tasks/mr_comment_convertor.go | 2 +-
.../plugins/gitlab/tasks/mr_commit_collector.go | 2 +-
.../plugins/gitlab/tasks/mr_commit_convertor.go | 5 +-
.../plugins/gitlab/tasks/mr_commit_extractor.go | 2 +-
backend/plugins/gitlab/tasks/mr_convertor.go | 2 +-
.../plugins/gitlab/tasks/mr_detail_collector.go | 2 +-
.../plugins/gitlab/tasks/mr_detail_extractor.go | 2 +-
backend/plugins/gitlab/tasks/mr_enricher.go | 7 +-
backend/plugins/gitlab/tasks/mr_extractor.go | 2 +-
backend/plugins/gitlab/tasks/mr_label_convertor.go | 5 +-
backend/plugins/gitlab/tasks/mr_note_collector.go | 2 +-
backend/plugins/gitlab/tasks/mr_note_extractor.go | 2 +-
backend/plugins/gitlab/tasks/pipeline_collector.go | 2 +-
.../gitlab/tasks/pipeline_commit_convertor.go | 2 +-
.../gitlab/tasks/pipeline_detail_collector.go | 2 +-
.../gitlab/tasks/pipeline_detail_convertor.go | 2 +-
.../gitlab/tasks/pipeline_detail_extractor.go | 2 +-
backend/plugins/gitlab/tasks/pipeline_extractor.go | 2 +-
backend/plugins/gitlab/tasks/project_convertor.go | 2 +-
backend/plugins/gitlab/tasks/tag_collector.go | 2 +-
backend/plugins/gitlab/tasks/tag_extractor.go | 3 +-
.../plugins/gitlab/tasks/trigger_job_collector.go | 2 +-
.../plugins/gitlab/tasks/trigger_job_extractor.go | 5 +-
backend/python/pydevlake/pydevlake/subtasks.py | 3 +-
backend/server/api/router.go | 1 +
backend/server/api/task/task.go | 24 ++++
backend/server/services/task.go | 145 +++++++++++++++++++++
138 files changed, 526 insertions(+), 172 deletions(-)
diff --git a/backend/core/models/migrationscripts/20240322_add_field_to_subtasks.go b/backend/core/models/migrationscripts/20240322_add_field_to_subtasks.go
new file mode 100644
index 000000000..0dfb7b6fb
--- /dev/null
+++ b/backend/core/models/migrationscripts/20240322_add_field_to_subtasks.go
@@ -0,0 +1,52 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+ "github.com/apache/incubator-devlake/core/context"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
+)
+
+var _ plugin.MigrationScript = (*addSubtaskField)(nil)
+
+type subtask20240322 struct {
+ FinishedRecords int `json:"finishedRecords"`
+ Sequence int `json:"sequence"`
+ IsCollector bool `json:"isCollector"`
+ IsFailed bool `json:"isFailed"`
+ Message string `json:"message"`
+}
+
+func (subtask20240322) TableName() string {
+ return "_devlake_subtasks"
+}
+
+type addSubtaskField struct{}
+
+func (*addSubtaskField) Up(basicRes context.BasicRes) errors.Error {
+ return basicRes.GetDal().AutoMigrate(subtask20240322{})
+}
+
+func (*addSubtaskField) Version() uint64 {
+ return 20240322111247
+}
+
+func (*addSubtaskField) Name() string {
+ return "add some fields to _devlake_subtasks table"
+}
diff --git a/backend/core/models/migrationscripts/register.go b/backend/core/models/migrationscripts/register.go
index 26b3ec9bb..f4641d03b 100644
--- a/backend/core/models/migrationscripts/register.go
+++ b/backend/core/models/migrationscripts/register.go
@@ -110,5 +110,6 @@ func All() []plugin.MigrationScript {
new(addOriginalEnvironmentToCicdDeploymentsAndCicdDeploymentCommits),
new(addSubtabknameToDeployment),
new(addStore),
+ new(addSubtaskField),
}
}
diff --git a/backend/core/models/task.go b/backend/core/models/task.go
index 6fba0f760..ba6f3f4ca 100644
--- a/backend/core/models/task.go
+++ b/backend/core/models/task.go
@@ -36,12 +36,14 @@ const (
var PendingTaskStatus = []string{TASK_CREATED, TASK_RERUN, TASK_RUNNING}
type TaskProgressDetail struct {
- TotalSubTasks int `json:"totalSubTasks"`
- FinishedSubTasks int `json:"finishedSubTasks"`
- TotalRecords int `json:"totalRecords"`
- FinishedRecords int `json:"finishedRecords"`
- SubTaskName string `json:"subTaskName"`
- SubTaskNumber int `json:"subTaskNumber"`
+ TotalSubTasks int `json:"totalSubTasks"`
+ FinishedSubTasks int `json:"finishedSubTasks"`
+ TotalRecords int `json:"totalRecords"`
+ FinishedRecords int `json:"finishedRecords"`
+ SubTaskName string `json:"subTaskName"`
+ SubTaskNumber int `json:"subTaskNumber"`
+ CollectSubtaskNumber int `json:"collectSubtaskNumber"`
+ OtherSubtaskNumber int `json:"otherSubtaskNumber"`
}
type NewTask struct {
@@ -79,14 +81,60 @@ func (Task) TableName() string {
type Subtask struct {
common.Model
- TaskID uint64 `json:"task_id" gorm:"index"`
- Name string `json:"name" gorm:"index"`
- Number int `json:"number"`
- BeganAt *time.Time `json:"beganAt"`
- FinishedAt *time.Time `json:"finishedAt" gorm:"index"`
- SpentSeconds int64 `json:"spentSeconds"`
+ TaskID uint64 `json:"task_id" gorm:"index"`
+ Name string `json:"name" gorm:"index"`
+ Number int `json:"number"`
+ BeganAt *time.Time `json:"beganAt"`
+ FinishedAt *time.Time `json:"finishedAt" gorm:"index"`
+ SpentSeconds int64 `json:"spentSeconds"`
+ FinishedRecords int `json:"finishedRecords"`
+ Sequence int `json:"sequence"`
+ IsCollector bool `json:"isCollector"`
+ IsFailed bool `json:"isFailed"`
+ Message string `json:"message"`
}
func (Subtask) TableName() string {
return "_devlake_subtasks"
}
+
+type SubtaskDetails struct {
+ ID uint64 `json:"id"`
+ CreatedAt time.Time `json:"created_at"`
+ UpdatedAt time.Time `json:"updated_at"`
+ TaskID uint64 `json:"task_id"`
+ Name string `json:"name"`
+ Number int `json:"number"`
+ BeganAt *time.Time `json:"began_at"`
+ FinishedAt *time.Time `json:"finished_at"`
+ SpentSeconds int64 `json:"spent_seconds"`
+ FinishedRecords int `json:"finished_records"`
+ Sequence int `json:"sequence"`
+ IsCollector bool `json:"is_collector"`
+ IsFailed bool `json:"is_failed"`
+ Message string `json:"message"`
+}
+
+type SubtasksInfo struct {
+ ID uint64 `json:"id"`
+ PipelineID uint64 `json:"pipelineId"`
+ CreatedAt time.Time `json:"createdAt"`
+ UpdatedAt time.Time `json:"updatedAt"`
+ BeganAt *time.Time `json:"beganAt"`
+ FinishedAt *time.Time `json:"finishedAt"`
+ Plugin string `json:"plugin"`
+ Options any `json:"options"`
+ Status string `json:"status"`
+ FailedSubTask string `json:"failedSubTask"`
+ Message string `json:"message"`
+ ErrorName string `json:"errorName"`
+ SpentSeconds int `json:"spentSeconds"`
+ SubtaskDetails []*SubtaskDetails `json:"subtaskDetails"`
+}
+
+type SubTasksOuput struct {
+ SubtasksInfo []SubtasksInfo `json:"subtasks"`
+ CompletionRate float64 `json:"completionRate"`
+ Status string `json:"status"`
+ Count int64 `json:"count"`
+}
diff --git a/backend/core/plugin/plugin_task.go b/backend/core/plugin/plugin_task.go
index 3bb44b5bb..357fa2dc4 100644
--- a/backend/core/plugin/plugin_task.go
+++ b/backend/core/plugin/plugin_task.go
@@ -36,11 +36,13 @@ const (
)
type RunningProgress struct {
- Type ProgressType
- Current int
- Total int
- SubTaskName string
- SubTaskNumber int
+ Type ProgressType
+ Current int
+ Total int
+ SubTaskName string
+ SubTaskNumber int
+ CollectSubtaskNumber int
+ OtherSubtaskNumber int
} // nolint
// ExecContext This interface define all resources that needed for task/subtask execution
diff --git a/backend/core/runner/run_task.go b/backend/core/runner/run_task.go
index 6ee04eccb..0f7428d66 100644
--- a/backend/core/runner/run_task.go
+++ b/backend/core/runner/run_task.go
@@ -246,6 +246,44 @@ func RunPluginSubTasks(
taskCtx.SetSyncPolicy(syncPolicy)
taskCtx.SetData(taskData)
+ // record subtasks sequence to DB
+ collectSubtaskNumber := 0
+ otherSubtaskNumber := 0
+ isCollector := false
+ subtask := []models.Subtask{}
+ for _, subtaskMeta := range subtaskMetas {
+ subtaskCtx, err := taskCtx.SubTaskContext(subtaskMeta.Name)
+ if err != nil {
+ // sth went wrong
+ return errors.Default.Wrap(err, fmt.Sprintf("error getting context subtask %s", subtaskMeta.Name))
+ }
+ if subtaskCtx == nil {
+ // subtask was disabled
+ continue
+ }
+ if strings.Contains(strings.ToLower(subtaskMeta.Name), "collect") || strings.Contains(strings.ToLower(subtaskMeta.Name), "clone git repo") {
+ collectSubtaskNumber++
+ isCollector = true
+ } else {
+ otherSubtaskNumber++
+ isCollector = false
+ }
+ s := models.Subtask{
+ Name: subtaskCtx.GetName(),
+ TaskID: task.ID,
+ IsCollector: isCollector,
+ }
+ if isCollector {
+ s.Sequence = collectSubtaskNumber
+ } else {
+ s.Sequence = otherSubtaskNumber
+ }
+ subtask = append(subtask, s)
+ }
+ if err := basicRes.GetDal().CreateOrUpdate(subtask); err != nil {
+ basicRes.GetLogger().Error(err, "error writing subtask list to DB")
+ }
+
// execute subtasks in order
taskCtx.SetProgress(0, steps)
subtaskNumber := 0
@@ -259,7 +297,6 @@ func RunPluginSubTasks(
// subtask was disabled
continue
}
-
// run subtask
logger.Info("executing subtask %s", subtaskMeta.Name)
subtaskNumber++
@@ -274,6 +311,13 @@ func RunPluginSubTasks(
if err != nil {
err = errors.SubtaskErr.Wrap(err, fmt.Sprintf("subtask %s ended unexpectedly", subtaskMeta.Name), errors.WithData(&subtaskMeta))
logger.Error(err, "")
+ where := dal.Where("task_id = ? and name = ?", task.ID, subtaskCtx.GetName())
+ if err := basicRes.GetDal().UpdateColumns(subtask, []dal.DalSet{
+ {ColumnName: "is_failed", Value: 1},
+ {ColumnName: "message", Value: err.Error()},
+ }, where); err != nil {
+ basicRes.GetLogger().Error(err, "error writing subtask %v status to DB", subtaskCtx.GetName())
+ }
return err
}
taskCtx.IncProgress(1)
@@ -286,6 +330,7 @@ func RunPluginSubTasks(
func UpdateProgressDetail(basicRes context.BasicRes, taskId uint64, progressDetail *models.TaskProgressDetail, p *plugin.RunningProgress) {
task := &models.Task{}
task.ID = taskId
+ subtask := &models.Subtask{}
switch p.Type {
case plugin.TaskSetProgress:
progressDetail.TotalSubTasks = p.Total
@@ -307,6 +352,14 @@ func UpdateProgressDetail(basicRes context.BasicRes, taskId uint64, progressDeta
progressDetail.SubTaskName = p.SubTaskName
progressDetail.SubTaskNumber = p.SubTaskNumber
}
+ // update subtask progress
+ where := dal.Where("task_id = ? and name = ?", taskId, progressDetail.SubTaskName)
+ err := basicRes.GetDal().UpdateColumns(subtask, []dal.DalSet{
+ {ColumnName: "finished_records", Value: progressDetail.FinishedRecords},
+ }, where)
+ if err != nil {
+ basicRes.GetLogger().Error(err, "failed to update _devlake_subtasks progress")
+ }
}
func runSubtask(
@@ -323,17 +376,27 @@ func runSubtask(
Number: subtaskNumber,
BeganAt: &beginAt,
}
+ recordSubtask(basicRes, subtask)
+ // defer to record subtask status
defer func() {
finishedAt := time.Now()
subtask.FinishedAt = &finishedAt
subtask.SpentSeconds = finishedAt.Unix() - beginAt.Unix()
+
recordSubtask(basicRes, subtask)
}()
return entryPoint(ctx)
}
func recordSubtask(basicRes context.BasicRes, subtask *models.Subtask) {
- if err := basicRes.GetDal().Create(subtask); err != nil {
+ where := dal.Where("task_id = ? and name = ?", subtask.TaskID, subtask.Name)
+ if err := basicRes.GetDal().UpdateColumns(subtask, []dal.DalSet{
+ {ColumnName: "began_at", Value: subtask.BeganAt},
+ {ColumnName: "finished_at", Value: subtask.FinishedAt},
+ {ColumnName: "spent_seconds", Value: subtask.SpentSeconds},
+ {ColumnName: "finished_records", Value: subtask.FinishedRecords},
+ {ColumnName: "number", Value: subtask.Number},
+ }, where); err != nil {
basicRes.GetLogger().Error(err, "error writing subtask %d status to DB: %v", subtask.ID)
}
}
diff --git a/backend/helpers/pluginhelper/api/api_collector.go b/backend/helpers/pluginhelper/api/api_collector.go
index 990610b20..f79d09148 100644
--- a/backend/helpers/pluginhelper/api/api_collector.go
+++ b/backend/helpers/pluginhelper/api/api_collector.go
@@ -434,7 +434,7 @@ func (collector *ApiCollector) fetchAsync(reqData *RequestData, handler func(int
logger := collector.args.Ctx.GetLogger()
logger.Debug("fetchAsync <<< enqueueing for %s %v", apiUrl, apiQuery)
responseHandler := func(res *http.Response) errors.Error {
- defer logger.Debug("fetchAsync >>> done for %s %v %v", apiUrl, apiQuery, collector.args.RequestBody)
+ defer logger.Debug("fetchAsync >>> done for %s %v", apiUrl, apiQuery)
logger := collector.args.Ctx.GetLogger()
// read body to buffer
body, err := io.ReadAll(res.Body)
diff --git a/backend/plugins/bitbucket/tasks/account_convertor.go b/backend/plugins/bitbucket/tasks/account_convertor.go
index cd076b8c7..19f013f0e 100644
--- a/backend/plugins/bitbucket/tasks/account_convertor.go
+++ b/backend/plugins/bitbucket/tasks/account_convertor.go
@@ -33,7 +33,7 @@ import (
const RAW_ACCOUNT_TABLE = "bitbucket_api_accounts"
var ConvertAccountsMeta = plugin.SubTaskMeta{
- Name: "convertAccounts",
+ Name: "Convert Users",
EntryPoint: ConvertAccounts,
EnabledByDefault: true,
Required: false,
diff --git a/backend/plugins/bitbucket/tasks/commit_collector.go b/backend/plugins/bitbucket/tasks/commit_collector.go
index d7fd4f0ec..83931db38 100644
--- a/backend/plugins/bitbucket/tasks/commit_collector.go
+++ b/backend/plugins/bitbucket/tasks/commit_collector.go
@@ -26,7 +26,7 @@ import (
const RAW_COMMIT_TABLE = "bitbucket_api_commits"
var CollectApiCommitsMeta = plugin.SubTaskMeta{
- Name: "collectApiCommits",
+ Name: "Collect Commits",
EntryPoint: CollectApiCommits,
EnabledByDefault: false,
Required: false,
diff --git a/backend/plugins/bitbucket/tasks/commit_convertor.go b/backend/plugins/bitbucket/tasks/commit_convertor.go
index 98e57a0ee..16b2af11d 100644
--- a/backend/plugins/bitbucket/tasks/commit_convertor.go
+++ b/backend/plugins/bitbucket/tasks/commit_convertor.go
@@ -18,6 +18,8 @@ limitations under the License.
package tasks
import (
+ "reflect"
+
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models/domainlayer/code"
@@ -25,11 +27,10 @@ import (
plugin "github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/bitbucket/models"
- "reflect"
)
var ConvertCommitsMeta = plugin.SubTaskMeta{
- Name: "convertCommits",
+ Name: "Convert Commits",
EntryPoint: ConvertCommits,
EnabledByDefault: false,
Required: false,
diff --git a/backend/plugins/bitbucket/tasks/commit_extractor.go b/backend/plugins/bitbucket/tasks/commit_extractor.go
index 00d2d48fd..0020be9d5 100644
--- a/backend/plugins/bitbucket/tasks/commit_extractor.go
+++ b/backend/plugins/bitbucket/tasks/commit_extractor.go
@@ -19,15 +19,16 @@ package tasks
import (
"encoding/json"
+ "time"
+
"github.com/apache/incubator-devlake/core/errors"
plugin "github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/bitbucket/models"
- "time"
)
var ExtractApiCommitsMeta = plugin.SubTaskMeta{
- Name: "extractApiCommits",
+ Name: "Extract Commits",
EntryPoint: ExtractApiCommits,
EnabledByDefault: false,
Required: false,
diff --git a/backend/plugins/bitbucket/tasks/deployment_collector.go b/backend/plugins/bitbucket/tasks/deployment_collector.go
index bf7f4f4ca..f48526d64 100644
--- a/backend/plugins/bitbucket/tasks/deployment_collector.go
+++ b/backend/plugins/bitbucket/tasks/deployment_collector.go
@@ -26,7 +26,7 @@ import (
const RAW_DEPLOYMENT_TABLE = "bitbucket_api_deployments"
var CollectApiDeploymentsMeta = plugin.SubTaskMeta{
- Name: "collectApiDeployments",
+ Name: "Collect Deployments",
EntryPoint: CollectApiDeployments,
EnabledByDefault: true,
Description: "Collect deployment data from bitbucket api",
diff --git a/backend/plugins/bitbucket/tasks/deployment_convertor.go b/backend/plugins/bitbucket/tasks/deployment_convertor.go
index 6c026f705..ca6ea9a4a 100644
--- a/backend/plugins/bitbucket/tasks/deployment_convertor.go
+++ b/backend/plugins/bitbucket/tasks/deployment_convertor.go
@@ -18,6 +18,10 @@ limitations under the License.
package tasks
import (
+ "reflect"
+ "strings"
+ "time"
+
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models/domainlayer"
@@ -26,13 +30,10 @@ import (
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/bitbucket/models"
- "reflect"
- "strings"
- "time"
)
var ConvertiDeploymentMeta = plugin.SubTaskMeta{
- Name: "convertDeployments",
+ Name: "Convert Deployments",
EntryPoint: ConvertDeployments,
EnabledByDefault: true,
Description: "Convert tool layer table bitbucket_deployment into domain layer tables",
diff --git a/backend/plugins/bitbucket/tasks/deployment_extractor.go b/backend/plugins/bitbucket/tasks/deployment_extractor.go
index a93877378..f1705db5a 100644
--- a/backend/plugins/bitbucket/tasks/deployment_extractor.go
+++ b/backend/plugins/bitbucket/tasks/deployment_extractor.go
@@ -19,11 +19,12 @@ package tasks
import (
"encoding/json"
+ "time"
+
"github.com/apache/incubator-devlake/core/errors"
plugin "github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/bitbucket/models"
- "time"
)
type bitbucketApiDeploymentsResponse struct {
@@ -76,7 +77,7 @@ type bitbucketApiDeploymentsResponse struct {
}
var ExtractApiDeploymentsMeta = plugin.SubTaskMeta{
- Name: "extractApiDeployments",
+ Name: "Extract Deployments",
EntryPoint: ExtractApiDeployments,
EnabledByDefault: true,
Description: "Extract raw deployments data into tool layer table BitbucketDeployment",
diff --git a/backend/plugins/bitbucket/tasks/issue_collector.go b/backend/plugins/bitbucket/tasks/issue_collector.go
index 3edec1bea..19bf67ef0 100644
--- a/backend/plugins/bitbucket/tasks/issue_collector.go
+++ b/backend/plugins/bitbucket/tasks/issue_collector.go
@@ -26,7 +26,7 @@ import (
const RAW_ISSUE_TABLE = "bitbucket_api_issues"
var CollectApiIssuesMeta = plugin.SubTaskMeta{
- Name: "collectApiIssues",
+ Name: "Collect Issues",
EntryPoint: CollectApiIssues,
EnabledByDefault: true,
Description: "Collect issues data from Bitbucket api",
diff --git a/backend/plugins/bitbucket/tasks/issue_comment_collector.go b/backend/plugins/bitbucket/tasks/issue_comment_collector.go
index cdc9c2e3d..5b1ad5a9b 100644
--- a/backend/plugins/bitbucket/tasks/issue_comment_collector.go
+++ b/backend/plugins/bitbucket/tasks/issue_comment_collector.go
@@ -26,7 +26,7 @@ import (
const RAW_ISSUE_COMMENTS_TABLE = "bitbucket_api_issue_comments"
var CollectApiIssueCommentsMeta = plugin.SubTaskMeta{
- Name: "collectApiIssueComments",
+ Name: "Collect Issue Comments",
EntryPoint: CollectApiIssueComments,
EnabledByDefault: true,
Required: false,
diff --git a/backend/plugins/bitbucket/tasks/issue_comment_convertor.go b/backend/plugins/bitbucket/tasks/issue_comment_convertor.go
index dcae35d10..4787dcae4 100644
--- a/backend/plugins/bitbucket/tasks/issue_comment_convertor.go
+++ b/backend/plugins/bitbucket/tasks/issue_comment_convertor.go
@@ -31,7 +31,7 @@ import (
)
var ConvertIssueCommentsMeta = plugin.SubTaskMeta{
- Name: "convertIssueComments",
+ Name: "Convert Issue Comments",
EntryPoint: ConvertIssueComments,
EnabledByDefault: true,
Description: "ConvertIssueComments data from Bitbucket api",
diff --git a/backend/plugins/bitbucket/tasks/issue_comment_extractor.go b/backend/plugins/bitbucket/tasks/issue_comment_extractor.go
index 3adc0c749..86eaf7620 100644
--- a/backend/plugins/bitbucket/tasks/issue_comment_extractor.go
+++ b/backend/plugins/bitbucket/tasks/issue_comment_extractor.go
@@ -46,7 +46,7 @@ type BitbucketIssueCommentsResponse struct {
}
var ExtractApiIssueCommentsMeta = plugin.SubTaskMeta{
- Name: "extractApiIssueComments",
+ Name: "Extract Issue Comments",
EntryPoint: ExtractApiIssueComments,
EnabledByDefault: true,
Required: false,
diff --git a/backend/plugins/bitbucket/tasks/issue_convertor.go b/backend/plugins/bitbucket/tasks/issue_convertor.go
index 3c3455fbb..c5fb4f801 100644
--- a/backend/plugins/bitbucket/tasks/issue_convertor.go
+++ b/backend/plugins/bitbucket/tasks/issue_convertor.go
@@ -32,7 +32,7 @@ import (
)
var ConvertIssuesMeta = plugin.SubTaskMeta{
- Name: "convertIssues",
+ Name: "Convert Issues",
EntryPoint: ConvertIssues,
EnabledByDefault: true,
Description: "Convert tool layer table bitbucket_issues into domain layer table issues",
diff --git a/backend/plugins/bitbucket/tasks/issue_extractor.go b/backend/plugins/bitbucket/tasks/issue_extractor.go
index ee0f2c80c..ede3ad730 100644
--- a/backend/plugins/bitbucket/tasks/issue_extractor.go
+++ b/backend/plugins/bitbucket/tasks/issue_extractor.go
@@ -54,7 +54,7 @@ type IssuesResponse struct {
}
var ExtractApiIssuesMeta = plugin.SubTaskMeta{
- Name: "extractApiIssues",
+ Name: "Extract Issues",
EntryPoint: ExtractApiIssues,
EnabledByDefault: true,
Description: "Extract raw Issues data into tool layer table bitbucket_issues",
diff --git a/backend/plugins/bitbucket/tasks/pipeline_collector.go b/backend/plugins/bitbucket/tasks/pipeline_collector.go
index 8998add3c..c19652e65 100644
--- a/backend/plugins/bitbucket/tasks/pipeline_collector.go
+++ b/backend/plugins/bitbucket/tasks/pipeline_collector.go
@@ -26,7 +26,7 @@ import (
const RAW_PIPELINE_TABLE = "bitbucket_api_pipelines"
var CollectApiPipelinesMeta = plugin.SubTaskMeta{
- Name: "collectApiPipelines",
+ Name: "Collect Pipelines",
EntryPoint: CollectApiPipelines,
EnabledByDefault: true,
Description: "Collect pipeline data from bitbucket api",
diff --git a/backend/plugins/bitbucket/tasks/pipeline_convertor.go b/backend/plugins/bitbucket/tasks/pipeline_convertor.go
index 442ae3086..9a8d80bdf 100644
--- a/backend/plugins/bitbucket/tasks/pipeline_convertor.go
+++ b/backend/plugins/bitbucket/tasks/pipeline_convertor.go
@@ -33,7 +33,7 @@ import (
)
var ConvertPipelineMeta = plugin.SubTaskMeta{
- Name: "convertPipelines",
+ Name: "Convert Pipelines",
EntryPoint: ConvertPipelines,
EnabledByDefault: true,
Description: "Convert tool layer table bitbucket_pipeline into domain layer table pipeline",
diff --git a/backend/plugins/bitbucket/tasks/pipeline_extractor.go b/backend/plugins/bitbucket/tasks/pipeline_extractor.go
index 7cd4425f5..dd7e68880 100644
--- a/backend/plugins/bitbucket/tasks/pipeline_extractor.go
+++ b/backend/plugins/bitbucket/tasks/pipeline_extractor.go
@@ -19,6 +19,7 @@ package tasks
import (
"encoding/json"
+
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models/common"
"github.com/apache/incubator-devlake/core/models/domainlayer/devops"
@@ -70,7 +71,7 @@ type BitbucketApiPipeline struct {
}
var ExtractApiPipelinesMeta = plugin.SubTaskMeta{
- Name: "extractApiPipelines",
+ Name: "Extract Pipelines",
EntryPoint: ExtractApiPipelines,
EnabledByDefault: true,
Description: "Extract raw pipelines data into tool layer table BitbucketPipeline",
diff --git a/backend/plugins/bitbucket/tasks/pipeline_steps_collector.go b/backend/plugins/bitbucket/tasks/pipeline_steps_collector.go
index 8c9d70aa7..9fd99e178 100644
--- a/backend/plugins/bitbucket/tasks/pipeline_steps_collector.go
+++ b/backend/plugins/bitbucket/tasks/pipeline_steps_collector.go
@@ -28,7 +28,7 @@ const RAW_PIPELINE_STEPS_TABLE = "bitbucket_api_pipeline_steps"
var _ plugin.SubTaskEntryPoint = CollectPipelineSteps
var CollectPipelineStepsMeta = plugin.SubTaskMeta{
- Name: "CollectPipelineSteps",
+ Name: "Collect Pipeline Steps",
EntryPoint: CollectPipelineSteps,
EnabledByDefault: true,
Description: "Collect PipelineSteps data from Bitbucket api",
diff --git a/backend/plugins/bitbucket/tasks/pipeline_steps_convertor.go b/backend/plugins/bitbucket/tasks/pipeline_steps_convertor.go
index 2276129fa..0a72041ba 100644
--- a/backend/plugins/bitbucket/tasks/pipeline_steps_convertor.go
+++ b/backend/plugins/bitbucket/tasks/pipeline_steps_convertor.go
@@ -32,7 +32,7 @@ import (
)
var ConvertPipelineStepMeta = plugin.SubTaskMeta{
- Name: "convertPipelineSteps",
+ Name: "Convert Pipeline Steps",
EntryPoint: ConvertPipelineSteps,
EnabledByDefault: true,
Description: "Convert tool layer table bitbucket_pipeline into domain layer table pipeline",
diff --git a/backend/plugins/bitbucket/tasks/pipeline_steps_extractor.go b/backend/plugins/bitbucket/tasks/pipeline_steps_extractor.go
index 5ad13535f..82bd93bd4 100644
--- a/backend/plugins/bitbucket/tasks/pipeline_steps_extractor.go
+++ b/backend/plugins/bitbucket/tasks/pipeline_steps_extractor.go
@@ -31,7 +31,7 @@ import (
var _ plugin.SubTaskEntryPoint = ExtractPipelineSteps
var ExtractPipelineStepsMeta = plugin.SubTaskMeta{
- Name: "ExtractPipelineSteps",
+ Name: "Extract Pipeline Steps",
EntryPoint: ExtractPipelineSteps,
EnabledByDefault: true,
Description: "Extract raw data into tool layer table bitbucket_pipeline_steps",
diff --git a/backend/plugins/bitbucket/tasks/pr_collector.go b/backend/plugins/bitbucket/tasks/pr_collector.go
index 7aa9641e9..9ab2b4eac 100644
--- a/backend/plugins/bitbucket/tasks/pr_collector.go
+++ b/backend/plugins/bitbucket/tasks/pr_collector.go
@@ -28,7 +28,7 @@ const RAW_PULL_REQUEST_TABLE = "bitbucket_api_pull_requests"
// this struct should be moved to `bitbucket_api_common.go`
var CollectApiPullRequestsMeta = plugin.SubTaskMeta{
- Name: "collectApiPullRequests",
+ Name: "Collect Pull Requests",
EntryPoint: CollectApiPullRequests,
EnabledByDefault: true,
Required: false,
diff --git a/backend/plugins/bitbucket/tasks/pr_comment_collector.go b/backend/plugins/bitbucket/tasks/pr_comment_collector.go
index ee7ccd228..30e856cd1 100644
--- a/backend/plugins/bitbucket/tasks/pr_comment_collector.go
+++ b/backend/plugins/bitbucket/tasks/pr_comment_collector.go
@@ -26,7 +26,7 @@ import (
const RAW_PULL_REQUEST_COMMENTS_TABLE = "bitbucket_api_pull_request_comments"
var CollectApiPrCommentsMeta = plugin.SubTaskMeta{
- Name: "collectApiPullRequestsComments",
+ Name: "Collect PR Comments",
EntryPoint: CollectApiPullRequestsComments,
EnabledByDefault: true,
Required: false,
diff --git a/backend/plugins/bitbucket/tasks/pr_comment_convertor.go b/backend/plugins/bitbucket/tasks/pr_comment_convertor.go
index 4c67ba83b..d841dd790 100644
--- a/backend/plugins/bitbucket/tasks/pr_comment_convertor.go
+++ b/backend/plugins/bitbucket/tasks/pr_comment_convertor.go
@@ -31,7 +31,7 @@ import (
)
var ConvertPrCommentsMeta = plugin.SubTaskMeta{
- Name: "convertPullRequestComments",
+ Name: "Convert PR Comments",
EntryPoint: ConvertPullRequestComments,
EnabledByDefault: true,
Description: "ConvertPullRequestComments data from Bitbucket api",
diff --git a/backend/plugins/bitbucket/tasks/pr_comment_extractor.go b/backend/plugins/bitbucket/tasks/pr_comment_extractor.go
index 9b1784c72..f685897bc 100644
--- a/backend/plugins/bitbucket/tasks/pr_comment_extractor.go
+++ b/backend/plugins/bitbucket/tasks/pr_comment_extractor.go
@@ -28,7 +28,7 @@ import (
)
var ExtractApiPrCommentsMeta = plugin.SubTaskMeta{
- Name: "extractApiPullRequestsComments",
+ Name: "Extract PR Comments",
EntryPoint: ExtractApiPullRequestsComments,
EnabledByDefault: true,
Required: false,
diff --git a/backend/plugins/bitbucket/tasks/pr_commit_collector.go b/backend/plugins/bitbucket/tasks/pr_commit_collector.go
index 8ed4a5a11..5260ee630 100644
--- a/backend/plugins/bitbucket/tasks/pr_commit_collector.go
+++ b/backend/plugins/bitbucket/tasks/pr_commit_collector.go
@@ -29,7 +29,7 @@ import (
const RAW_PULL_REQUEST_COMMITS_TABLE = "bitbucket_api_pull_request_commits"
var CollectApiPrCommitsMeta = plugin.SubTaskMeta{
- Name: "collectApiPullRequestCommits",
+ Name: "Collect PR Commits",
EntryPoint: CollectApiPullRequestCommits,
EnabledByDefault: true,
Description: "Collect PullRequestCommits data from Bitbucket api",
diff --git a/backend/plugins/bitbucket/tasks/pr_commit_convertor.go b/backend/plugins/bitbucket/tasks/pr_commit_convertor.go
index dd7d28aad..04abeb14e 100644
--- a/backend/plugins/bitbucket/tasks/pr_commit_convertor.go
+++ b/backend/plugins/bitbucket/tasks/pr_commit_convertor.go
@@ -31,7 +31,7 @@ import (
)
var ConvertPrCommitsMeta = plugin.SubTaskMeta{
- Name: "convertPullRequestCommits",
+ Name: "Convert PR Commits",
EntryPoint: ConvertPullRequestCommits,
EnabledByDefault: true,
Description: "Convert tool layer table bitbucket_pull_request_commits into domain layer table pull_request_commits",
diff --git a/backend/plugins/bitbucket/tasks/pr_commit_extractor.go b/backend/plugins/bitbucket/tasks/pr_commit_extractor.go
index c8ca476a3..7862ca505 100644
--- a/backend/plugins/bitbucket/tasks/pr_commit_extractor.go
+++ b/backend/plugins/bitbucket/tasks/pr_commit_extractor.go
@@ -30,7 +30,7 @@ import (
)
var ExtractApiPrCommitsMeta = plugin.SubTaskMeta{
- Name: "extractApiPullRequestCommits",
+ Name: "Extract PR Commits",
EntryPoint: ExtractApiPullRequestCommits,
EnabledByDefault: true,
Description: "Extract raw PullRequestCommits data into tool layer table bitbucket_commits",
diff --git a/backend/plugins/bitbucket/tasks/pr_convertor.go b/backend/plugins/bitbucket/tasks/pr_convertor.go
index b2fc68d56..ed12d0e12 100644
--- a/backend/plugins/bitbucket/tasks/pr_convertor.go
+++ b/backend/plugins/bitbucket/tasks/pr_convertor.go
@@ -31,7 +31,7 @@ import (
)
var ConvertPullRequestsMeta = plugin.SubTaskMeta{
- Name: "convertPullRequests",
+ Name: "Convert Pull Requests",
EntryPoint: ConvertPullRequests,
EnabledByDefault: true,
Required: false,
diff --git a/backend/plugins/bitbucket/tasks/pr_extractor.go b/backend/plugins/bitbucket/tasks/pr_extractor.go
index d88a59d4f..855104c80 100644
--- a/backend/plugins/bitbucket/tasks/pr_extractor.go
+++ b/backend/plugins/bitbucket/tasks/pr_extractor.go
@@ -29,7 +29,7 @@ import (
)
var ExtractApiPullRequestsMeta = plugin.SubTaskMeta{
- Name: "extractApiPullRequests",
+ Name: "Extract Pull Requests",
EntryPoint: ExtractApiPullRequests,
EnabledByDefault: true,
Required: false,
diff --git a/backend/plugins/bitbucket/tasks/repo_convertor.go b/backend/plugins/bitbucket/tasks/repo_convertor.go
index 4961462c3..798d6b252 100644
--- a/backend/plugins/bitbucket/tasks/repo_convertor.go
+++ b/backend/plugins/bitbucket/tasks/repo_convertor.go
@@ -40,7 +40,7 @@ import (
const RAW_REPOSITORIES_TABLE = "bitbucket_api_repositories"
var ConvertRepoMeta = plugin.SubTaskMeta{
- Name: "convertRepo",
+ Name: "Convert Repos",
EntryPoint: ConvertRepo,
EnabledByDefault: true,
Description: "Convert tool layer table bitbucket_repos into domain layer table repos and boards",
diff --git a/backend/plugins/gitextractor/tasks/clone.go b/backend/plugins/gitextractor/tasks/clone.go
index 0b88e2ff6..c27921a63 100644
--- a/backend/plugins/gitextractor/tasks/clone.go
+++ b/backend/plugins/gitextractor/tasks/clone.go
@@ -19,19 +19,20 @@ package tasks
import (
"fmt"
+ "strings"
+
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/log"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/plugins/gitextractor/models"
"github.com/apache/incubator-devlake/plugins/gitextractor/parser"
"github.com/apache/incubator-devlake/plugins/gitextractor/store"
- "strings"
)
const useGoGitInGitExtractor = "USE_GO_GIT_IN_GIT_EXTRACTOR"
var CloneGitRepoMeta = plugin.SubTaskMeta{
- Name: "cloneGitRepo",
+ Name: "Clone Git Repo",
EntryPoint: CloneGitRepo,
EnabledByDefault: true,
Required: true,
diff --git a/backend/plugins/gitextractor/tasks/git_repo_collector.go b/backend/plugins/gitextractor/tasks/git_repo_collector.go
index 5af4fe98a..00abf28cf 100644
--- a/backend/plugins/gitextractor/tasks/git_repo_collector.go
+++ b/backend/plugins/gitextractor/tasks/git_repo_collector.go
@@ -116,7 +116,7 @@ func getGitRepo(subTaskCtx plugin.SubTaskContext) parser.RepoCollector {
}
var CollectGitCommitMeta = plugin.SubTaskMeta{
- Name: "collectGitCommits",
+ Name: "Collect Commits",
EntryPoint: CollectGitCommits,
EnabledByDefault: true,
Description: "collect git commits into Domain Layer Tables",
@@ -125,7 +125,7 @@ var CollectGitCommitMeta = plugin.SubTaskMeta{
}
var CollectGitBranchMeta = plugin.SubTaskMeta{
- Name: "collectGitBranches",
+ Name: "Collect Branches",
EntryPoint: CollectGitBranches,
EnabledByDefault: true,
Description: "collect git branch into Domain Layer Tables",
@@ -134,7 +134,7 @@ var CollectGitBranchMeta = plugin.SubTaskMeta{
}
var CollectGitTagMeta = plugin.SubTaskMeta{
- Name: "collectGitTags",
+ Name: "Collect Tags",
EntryPoint: CollectGitTags,
EnabledByDefault: true,
Description: "collect git tag into Domain Layer Tables",
@@ -143,7 +143,7 @@ var CollectGitTagMeta = plugin.SubTaskMeta{
}
var CollectGitDiffLineMeta = plugin.SubTaskMeta{
- Name: "collectDiffLine",
+ Name: "Collect DiffLine",
EntryPoint: CollectGitDiffLines,
EnabledByDefault: false,
Description: "collect git commit diff line into Domain Layer Tables",
diff --git a/backend/plugins/github/tasks/account_collector.go b/backend/plugins/github/tasks/account_collector.go
index 931a7ae7c..3f2edf2e5 100644
--- a/backend/plugins/github/tasks/account_collector.go
+++ b/backend/plugins/github/tasks/account_collector.go
@@ -41,7 +41,7 @@ type SimpleAccount struct {
}
var CollectAccountsMeta = plugin.SubTaskMeta{
- Name: "collectAccounts",
+ Name: "Collect Users",
EntryPoint: CollectAccounts,
EnabledByDefault: true,
Description: "Collect accounts data from Github api, does not support either timeFilter or diffSync.",
diff --git a/backend/plugins/github/tasks/account_convertor.go b/backend/plugins/github/tasks/account_convertor.go
index c93c80407..dfd9a929e 100644
--- a/backend/plugins/github/tasks/account_convertor.go
+++ b/backend/plugins/github/tasks/account_convertor.go
@@ -37,7 +37,7 @@ func init() {
}
var ConvertAccountsMeta = plugin.SubTaskMeta{
- Name: "convertAccounts",
+ Name: "Convert Users",
EntryPoint: ConvertAccounts,
EnabledByDefault: true,
Description: "Convert tool layer table github_accounts into domain layer table accounts",
diff --git a/backend/plugins/github/tasks/account_extractor.go b/backend/plugins/github/tasks/account_extractor.go
index 7fa2c0f13..88e4e8efb 100644
--- a/backend/plugins/github/tasks/account_extractor.go
+++ b/backend/plugins/github/tasks/account_extractor.go
@@ -32,7 +32,7 @@ func init() {
}
var ExtractAccountsMeta = plugin.SubTaskMeta{
- Name: "extractAccounts",
+ Name: "Extract Users",
EntryPoint: ExtractAccounts,
EnabledByDefault: true,
Description: "Extract raw account data into tool layer table github_accounts",
diff --git a/backend/plugins/github/tasks/account_org_collector.go b/backend/plugins/github/tasks/account_org_collector.go
index 590474a52..5f9cf5a29 100644
--- a/backend/plugins/github/tasks/account_org_collector.go
+++ b/backend/plugins/github/tasks/account_org_collector.go
@@ -42,7 +42,7 @@ type SimpleAccountWithId struct {
}
var CollectAccountOrgMeta = plugin.SubTaskMeta{
- Name: "collectAccountOrg",
+ Name: "Collect User Org",
EntryPoint: CollectAccountOrg,
EnabledByDefault: true,
Description: "Collect accounts org data from Github api, does not support either timeFilter or diffSync.",
diff --git a/backend/plugins/github/tasks/account_org_extractor.go b/backend/plugins/github/tasks/account_org_extractor.go
index 60d93ce7f..8aa69b664 100644
--- a/backend/plugins/github/tasks/account_org_extractor.go
+++ b/backend/plugins/github/tasks/account_org_extractor.go
@@ -31,7 +31,7 @@ func init() {
}
var ExtractAccountOrgMeta = plugin.SubTaskMeta{
- Name: "extractAccountOrg",
+ Name: "Extract User Org",
EntryPoint: ExtractAccountOrg,
EnabledByDefault: true,
Description: "Extract raw account org data into tool layer table github_account_orgs",
diff --git a/backend/plugins/github/tasks/cicd_job_collector.go b/backend/plugins/github/tasks/cicd_job_collector.go
index 1c9fe63e6..dda4b90c0 100644
--- a/backend/plugins/github/tasks/cicd_job_collector.go
+++ b/backend/plugins/github/tasks/cicd_job_collector.go
@@ -38,7 +38,7 @@ func init() {
const RAW_JOB_TABLE = "github_api_jobs"
var CollectJobsMeta = plugin.SubTaskMeta{
- Name: "collectJobs",
+ Name: "Collect Job Runs",
EntryPoint: CollectJobs,
EnabledByDefault: true,
Description: "Collect Jobs data from Github action api, supports both timeFilter and diffSync.",
diff --git a/backend/plugins/github/tasks/cicd_job_convertor.go b/backend/plugins/github/tasks/cicd_job_convertor.go
index f9134c460..e5e734d4e 100644
--- a/backend/plugins/github/tasks/cicd_job_convertor.go
+++ b/backend/plugins/github/tasks/cicd_job_convertor.go
@@ -36,7 +36,7 @@ func init() {
}
var ConvertJobsMeta = plugin.SubTaskMeta{
- Name: "convertJobs",
+ Name: "Convert Jobs",
EntryPoint: ConvertJobs,
EnabledByDefault: true,
Description: "Convert tool layer table github_jobs into domain layer table cicd_tasks",
diff --git a/backend/plugins/github/tasks/cicd_job_extractor.go b/backend/plugins/github/tasks/cicd_job_extractor.go
index 9193b0342..5701476bc 100644
--- a/backend/plugins/github/tasks/cicd_job_extractor.go
+++ b/backend/plugins/github/tasks/cicd_job_extractor.go
@@ -33,7 +33,7 @@ func init() {
}
var ExtractJobsMeta = plugin.SubTaskMeta{
- Name: "extractJobs",
+ Name: "Extract Jobs",
EntryPoint: ExtractJobs,
EnabledByDefault: true,
Description: "Extract raw run data into tool layer table github_jobs",
diff --git a/backend/plugins/github/tasks/cicd_run_collector.go b/backend/plugins/github/tasks/cicd_run_collector.go
index 3cd1bd5d5..85ef6108b 100644
--- a/backend/plugins/github/tasks/cicd_run_collector.go
+++ b/backend/plugins/github/tasks/cicd_run_collector.go
@@ -47,7 +47,7 @@ type SimpleGithubApiJob struct {
}
var CollectRunsMeta = plugin.SubTaskMeta{
- Name: "collectRuns",
+ Name: "Collect Workflow Runs",
EntryPoint: CollectRuns,
EnabledByDefault: true,
Description: "Collect Runs data from Github action api, supports both timeFilter and diffSync.",
diff --git a/backend/plugins/github/tasks/cicd_run_convertor.go b/backend/plugins/github/tasks/cicd_run_convertor.go
index b28432369..505d85fce 100644
--- a/backend/plugins/github/tasks/cicd_run_convertor.go
+++ b/backend/plugins/github/tasks/cicd_run_convertor.go
@@ -18,6 +18,9 @@ limitations under the License.
package tasks
import (
+ "reflect"
+ "time"
+
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models/domainlayer"
@@ -26,8 +29,6 @@ import (
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/github/models"
- "reflect"
- "time"
)
func init() {
@@ -35,7 +36,7 @@ func init() {
}
var ConvertRunsMeta = plugin.SubTaskMeta{
- Name: "convertRuns",
+ Name: "Convert Workflow Runs",
EntryPoint: ConvertRuns,
EnabledByDefault: true,
Description: "Convert tool layer table github_runs into domain layer table cicd_pipeline",
diff --git a/backend/plugins/github/tasks/cicd_run_extractor.go b/backend/plugins/github/tasks/cicd_run_extractor.go
index bc052299a..1e6a3f41a 100644
--- a/backend/plugins/github/tasks/cicd_run_extractor.go
+++ b/backend/plugins/github/tasks/cicd_run_extractor.go
@@ -32,7 +32,7 @@ func init() {
}
var ExtractRunsMeta = plugin.SubTaskMeta{
- Name: "extractRuns",
+ Name: "Extract Workflow Runs",
EntryPoint: ExtractRuns,
EnabledByDefault: true,
Description: "Extract raw run data into tool layer table github_runs",
diff --git a/backend/plugins/github/tasks/comment_collector.go b/backend/plugins/github/tasks/comment_collector.go
index 73dfad834..e06f023f4 100644
--- a/backend/plugins/github/tasks/comment_collector.go
+++ b/backend/plugins/github/tasks/comment_collector.go
@@ -35,7 +35,7 @@ func init() {
const RAW_COMMENTS_TABLE = "github_api_comments"
var CollectApiCommentsMeta = plugin.SubTaskMeta{
- Name: "collectApiComments",
+ Name: "Collect Comments",
EntryPoint: CollectApiComments,
EnabledByDefault: true,
Description: "Collect comments data from Github api, supports both timeFilter and diffSync.",
diff --git a/backend/plugins/github/tasks/comment_extractor.go b/backend/plugins/github/tasks/comment_extractor.go
index 4846c1d40..386babe79 100644
--- a/backend/plugins/github/tasks/comment_extractor.go
+++ b/backend/plugins/github/tasks/comment_extractor.go
@@ -34,7 +34,7 @@ func init() {
}
var ExtractApiCommentsMeta = plugin.SubTaskMeta{
- Name: "extractApiComments",
+ Name: "Extract Comments",
EntryPoint: ExtractApiComments,
EnabledByDefault: true,
Description: "Extract raw comment data into tool layer table github_pull_request_comments" +
diff --git a/backend/plugins/github/tasks/commit_collector.go b/backend/plugins/github/tasks/commit_collector.go
index 5a9be44db..b735a508d 100644
--- a/backend/plugins/github/tasks/commit_collector.go
+++ b/backend/plugins/github/tasks/commit_collector.go
@@ -35,7 +35,7 @@ func init() {
const RAW_COMMIT_TABLE = "github_api_commits"
var CollectApiCommitsMeta = plugin.SubTaskMeta{
- Name: "collectApiCommits",
+ Name: "Collect Commits",
EntryPoint: CollectApiCommits,
EnabledByDefault: false,
Description: "Collect commits data from Github api, supports both timeFilter and diffSync.",
diff --git a/backend/plugins/github/tasks/commit_convertor.go b/backend/plugins/github/tasks/commit_convertor.go
index e8dcfa5e9..4dc3fa43c 100644
--- a/backend/plugins/github/tasks/commit_convertor.go
+++ b/backend/plugins/github/tasks/commit_convertor.go
@@ -34,7 +34,7 @@ func init() {
}
var ConvertCommitsMeta = plugin.SubTaskMeta{
- Name: "convertCommits",
+ Name: "Convert Commits",
EntryPoint: ConvertCommits,
EnabledByDefault: false,
Description: "Convert tool layer table github_commits into domain layer table commits",
diff --git a/backend/plugins/github/tasks/commit_extractor.go b/backend/plugins/github/tasks/commit_extractor.go
index a81c2b5f9..a0ed8d11d 100644
--- a/backend/plugins/github/tasks/commit_extractor.go
+++ b/backend/plugins/github/tasks/commit_extractor.go
@@ -32,7 +32,7 @@ func init() {
}
var ExtractApiCommitsMeta = plugin.SubTaskMeta{
- Name: "extractApiCommits",
+ Name: "Extract Commits",
EntryPoint: ExtractApiCommits,
EnabledByDefault: false,
Description: "Extract raw commit data into tool layer table github_commits",
diff --git a/backend/plugins/github/tasks/commit_stats_collector.go b/backend/plugins/github/tasks/commit_stats_collector.go
index 3aa71635f..0514a3096 100644
--- a/backend/plugins/github/tasks/commit_stats_collector.go
+++ b/backend/plugins/github/tasks/commit_stats_collector.go
@@ -39,7 +39,7 @@ func init() {
const RAW_COMMIT_STATS_TABLE = "github_api_commit_stats"
var CollectApiCommitStatsMeta = plugin.SubTaskMeta{
- Name: "collectApiCommitStats",
+ Name: "Collect Commit Stats",
EntryPoint: CollectApiCommitStats,
EnabledByDefault: false,
Description: "Collect commitStats data from Github api, does not support either timeFilter or diffSync.",
diff --git a/backend/plugins/github/tasks/commit_stats_extractor.go b/backend/plugins/github/tasks/commit_stats_extractor.go
index 696305262..5f99b1ed5 100644
--- a/backend/plugins/github/tasks/commit_stats_extractor.go
+++ b/backend/plugins/github/tasks/commit_stats_extractor.go
@@ -33,7 +33,7 @@ func init() {
}
var ExtractApiCommitStatsMeta = plugin.SubTaskMeta{
- Name: "extractApiCommitStats",
+ Name: "Extract Commit Stats",
EntryPoint: ExtractApiCommitStats,
EnabledByDefault: false,
Description: "Extract raw commit stats data into tool layer table github_commit_stats",
diff --git a/backend/plugins/github/tasks/deployment_convertor.go b/backend/plugins/github/tasks/deployment_convertor.go
index 69f17390d..0d9f89dc6 100644
--- a/backend/plugins/github/tasks/deployment_convertor.go
+++ b/backend/plugins/github/tasks/deployment_convertor.go
@@ -39,7 +39,7 @@ const (
)
var ConvertDeploymentsMeta = plugin.SubTaskMeta{
- Name: "ConvertDeployments",
+ Name: "Convert Deployments",
EntryPoint: ConvertDeployment,
EnabledByDefault: true,
Description: "Convert tool layer table github_deployments into domain layer table deployment",
diff --git a/backend/plugins/github/tasks/event_collector.go b/backend/plugins/github/tasks/event_collector.go
index 491cc021c..ea192ee71 100644
--- a/backend/plugins/github/tasks/event_collector.go
+++ b/backend/plugins/github/tasks/event_collector.go
@@ -46,7 +46,7 @@ type SimpleGithubApiEvents struct {
}
var CollectApiEventsMeta = plugin.SubTaskMeta{
- Name: "collectApiEvents",
+ Name: "Collect Events",
EntryPoint: CollectApiEvents,
EnabledByDefault: true,
Description: "Collect Events data from Github api, supports both timeFilter and diffSync.",
diff --git a/backend/plugins/github/tasks/event_extractor.go b/backend/plugins/github/tasks/event_extractor.go
index 9e09c9920..133527d16 100644
--- a/backend/plugins/github/tasks/event_extractor.go
+++ b/backend/plugins/github/tasks/event_extractor.go
@@ -32,7 +32,7 @@ func init() {
}
var ExtractApiEventsMeta = plugin.SubTaskMeta{
- Name: "extractApiEvents",
+ Name: "Extract Events",
EntryPoint: ExtractApiEvents,
EnabledByDefault: true,
Description: "Extract raw Events data into tool layer table github_issue_events",
diff --git a/backend/plugins/github/tasks/issue_assignee_convertor.go b/backend/plugins/github/tasks/issue_assignee_convertor.go
index 24c3c0f16..5f8c883b8 100644
--- a/backend/plugins/github/tasks/issue_assignee_convertor.go
+++ b/backend/plugins/github/tasks/issue_assignee_convertor.go
@@ -34,7 +34,7 @@ func init() {
}
var ConvertIssueAssigneeMeta = plugin.SubTaskMeta{
- Name: "convertIssueAssignee",
+ Name: "Convert Issue Assignees",
EntryPoint: ConvertIssueAssignee,
EnabledByDefault: true,
Description: "Convert tool layer table _tool_github_issue_assignees into domain layer table issue_assignees",
diff --git a/backend/plugins/github/tasks/issue_collector.go b/backend/plugins/github/tasks/issue_collector.go
index 67d2ce506..4e9b25d21 100644
--- a/backend/plugins/github/tasks/issue_collector.go
+++ b/backend/plugins/github/tasks/issue_collector.go
@@ -35,7 +35,7 @@ func init() {
}
var CollectApiIssuesMeta = plugin.SubTaskMeta{
- Name: "collectApiIssues",
+ Name: "Collect Issues",
EntryPoint: CollectApiIssues,
EnabledByDefault: true,
Description: "Collect issues data from Github api, supports both timeFilter and diffSync.",
diff --git a/backend/plugins/github/tasks/issue_comment_convertor.go b/backend/plugins/github/tasks/issue_comment_convertor.go
index 860a5fc14..07ddeec65 100644
--- a/backend/plugins/github/tasks/issue_comment_convertor.go
+++ b/backend/plugins/github/tasks/issue_comment_convertor.go
@@ -35,7 +35,7 @@ func init() {
}
var ConvertIssueCommentsMeta = plugin.SubTaskMeta{
- Name: "convertIssueComments",
+ Name: "Convert Issue Comments",
EntryPoint: ConvertIssueComments,
EnabledByDefault: true,
Description: "ConvertIssueComments data from Github api",
diff --git a/backend/plugins/github/tasks/issue_convertor.go b/backend/plugins/github/tasks/issue_convertor.go
index a1c5f9188..e50f6f22a 100644
--- a/backend/plugins/github/tasks/issue_convertor.go
+++ b/backend/plugins/github/tasks/issue_convertor.go
@@ -37,7 +37,7 @@ func init() {
}
var ConvertIssuesMeta = plugin.SubTaskMeta{
- Name: "convertIssues",
+ Name: "Convert Issues",
EntryPoint: ConvertIssues,
EnabledByDefault: true,
Description: "Convert tool layer table github_issues into domain layer table issues",
diff --git a/backend/plugins/github/tasks/issue_extractor.go b/backend/plugins/github/tasks/issue_extractor.go
index a61c33541..101b1e5b9 100644
--- a/backend/plugins/github/tasks/issue_extractor.go
+++ b/backend/plugins/github/tasks/issue_extractor.go
@@ -35,7 +35,7 @@ func init() {
}
var ExtractApiIssuesMeta = plugin.SubTaskMeta{
- Name: "extractApiIssues",
+ Name: "Extract Issues",
EntryPoint: ExtractApiIssues,
EnabledByDefault: true,
Description: "Extract raw Issues data into tool layer table github_issues",
diff --git a/backend/plugins/github/tasks/issue_label_convertor.go b/backend/plugins/github/tasks/issue_label_convertor.go
index 8643d48cf..bff0c0143 100644
--- a/backend/plugins/github/tasks/issue_label_convertor.go
+++ b/backend/plugins/github/tasks/issue_label_convertor.go
@@ -34,7 +34,7 @@ func init() {
}
var ConvertIssueLabelsMeta = plugin.SubTaskMeta{
- Name: "convertIssueLabels",
+ Name: "Convert Issue Labels",
EntryPoint: ConvertIssueLabels,
EnabledByDefault: true,
Description: "Convert tool layer table github_issue_labels into domain layer table issue_labels",
diff --git a/backend/plugins/github/tasks/milestone_collector.go b/backend/plugins/github/tasks/milestone_collector.go
index 03d4ff586..9e60431ce 100644
--- a/backend/plugins/github/tasks/milestone_collector.go
+++ b/backend/plugins/github/tasks/milestone_collector.go
@@ -35,7 +35,7 @@ func init() {
const RAW_MILESTONE_TABLE = "github_milestones"
var CollectMilestonesMeta = plugin.SubTaskMeta{
- Name: "collectApiMilestones",
+ Name: "Collect Milestones",
EntryPoint: CollectApiMilestones,
EnabledByDefault: true,
Description: "Collect milestone data from Github api, does not support either timeFilter or diffSync.",
diff --git a/backend/plugins/github/tasks/milestone_converter.go b/backend/plugins/github/tasks/milestone_converter.go
index 6d872023d..5261c7674 100644
--- a/backend/plugins/github/tasks/milestone_converter.go
+++ b/backend/plugins/github/tasks/milestone_converter.go
@@ -36,7 +36,7 @@ func init() {
}
var ConvertMilestonesMeta = plugin.SubTaskMeta{
- Name: "convertMilestones",
+ Name: "Cnvert Milestones",
EntryPoint: ConvertMilestones,
EnabledByDefault: true,
Description: "Convert tool layer table github_milestones into domain layer table milestones",
diff --git a/backend/plugins/github/tasks/milestone_extractor.go b/backend/plugins/github/tasks/milestone_extractor.go
index 83e10b5cd..7828b2c26 100644
--- a/backend/plugins/github/tasks/milestone_extractor.go
+++ b/backend/plugins/github/tasks/milestone_extractor.go
@@ -32,7 +32,7 @@ func init() {
}
var ExtractMilestonesMeta = plugin.SubTaskMeta{
- Name: "extractMilestones",
+ Name: "Extract Milestones",
EntryPoint: ExtractMilestones,
EnabledByDefault: true,
Description: "Extract raw milestone data into tool layer table github_milestones",
diff --git a/backend/plugins/github/tasks/pr_collector.go b/backend/plugins/github/tasks/pr_collector.go
index a83cd3085..cc6d31e3a 100644
--- a/backend/plugins/github/tasks/pr_collector.go
+++ b/backend/plugins/github/tasks/pr_collector.go
@@ -40,7 +40,7 @@ func init() {
const RAW_PULL_REQUEST_TABLE = "github_api_pull_requests"
var CollectApiPullRequestsMeta = plugin.SubTaskMeta{
- Name: "collectApiPullRequests",
+ Name: "Collect Pull Requests",
EntryPoint: CollectApiPullRequests,
EnabledByDefault: true,
Description: "Collect PullRequests data from Github api, supports both timeFilter and diffSync.",
diff --git a/backend/plugins/github/tasks/pr_comment_convertor.go b/backend/plugins/github/tasks/pr_comment_convertor.go
index 7df730c92..43e0e73ba 100644
--- a/backend/plugins/github/tasks/pr_comment_convertor.go
+++ b/backend/plugins/github/tasks/pr_comment_convertor.go
@@ -35,7 +35,7 @@ func init() {
}
var ConvertPullRequestCommentsMeta = plugin.SubTaskMeta{
- Name: "convertPullRequestComments",
+ Name: "Convert PR Comments",
EntryPoint: ConvertPullRequestComments,
EnabledByDefault: true,
Description: "ConvertPullRequestComments data from Github api",
diff --git a/backend/plugins/github/tasks/pr_commit_collector.go b/backend/plugins/github/tasks/pr_commit_collector.go
index 25c5402ac..487e6ac14 100644
--- a/backend/plugins/github/tasks/pr_commit_collector.go
+++ b/backend/plugins/github/tasks/pr_commit_collector.go
@@ -43,7 +43,7 @@ const RAW_PR_COMMIT_TABLE = "github_api_pull_request_commits"
// this struct should be moved to `gitub_api_common.go`
var CollectApiPullRequestCommitsMeta = plugin.SubTaskMeta{
- Name: "collectApiPullRequestCommits",
+ Name: "Collect PR Commits",
EntryPoint: CollectApiPullRequestCommits,
EnabledByDefault: true,
Description: "Collect PullRequestCommits data from Github api, supports both timeFilter and diffSync.",
diff --git a/backend/plugins/github/tasks/pr_commit_convertor.go b/backend/plugins/github/tasks/pr_commit_convertor.go
index 5b6ab7042..b9cbbbbf5 100644
--- a/backend/plugins/github/tasks/pr_commit_convertor.go
+++ b/backend/plugins/github/tasks/pr_commit_convertor.go
@@ -34,7 +34,7 @@ func init() {
}
var ConvertPullRequestCommitsMeta = plugin.SubTaskMeta{
- Name: "convertPullRequestCommits",
+ Name: "Convert PR Commits",
EntryPoint: ConvertPullRequestCommits,
EnabledByDefault: true,
Description: "Convert tool layer table github_pull_request_commits into domain layer table pull_request_commits",
diff --git a/backend/plugins/github/tasks/pr_commit_extractor.go b/backend/plugins/github/tasks/pr_commit_extractor.go
index d357e7a94..569e39aba 100644
--- a/backend/plugins/github/tasks/pr_commit_extractor.go
+++ b/backend/plugins/github/tasks/pr_commit_extractor.go
@@ -34,7 +34,7 @@ func init() {
}
var ExtractApiPullRequestCommitsMeta = plugin.SubTaskMeta{
- Name: "extractApiPullRequestCommits",
+ Name: "Extract PR Commits",
EntryPoint: ExtractApiPullRequestCommits,
EnabledByDefault: true,
Description: "Extract raw PullRequestCommits data into tool layer table github_commits",
diff --git a/backend/plugins/github/tasks/pr_convertor.go b/backend/plugins/github/tasks/pr_convertor.go
index 672484de0..6f7250127 100644
--- a/backend/plugins/github/tasks/pr_convertor.go
+++ b/backend/plugins/github/tasks/pr_convertor.go
@@ -35,7 +35,7 @@ func init() {
}
var ConvertPullRequestsMeta = plugin.SubTaskMeta{
- Name: "convertPullRequests",
+ Name: "Convert Pull Requests",
EntryPoint: ConvertPullRequests,
EnabledByDefault: true,
Description: "ConvertPullRequests data from Github api",
diff --git a/backend/plugins/github/tasks/pr_extractor.go b/backend/plugins/github/tasks/pr_extractor.go
index 17fec4a45..6551dedbb 100644
--- a/backend/plugins/github/tasks/pr_extractor.go
+++ b/backend/plugins/github/tasks/pr_extractor.go
@@ -33,7 +33,7 @@ func init() {
}
var ExtractApiPullRequestsMeta = plugin.SubTaskMeta{
- Name: "extractApiPullRequests",
+ Name: "Extract Pull Requests",
EntryPoint: ExtractApiPullRequests,
EnabledByDefault: true,
Description: "Extract raw PullRequests data into tool layer table github_pull_requests",
diff --git a/backend/plugins/github/tasks/pr_issue_convertor.go b/backend/plugins/github/tasks/pr_issue_convertor.go
index 1c5906847..a0cafb1ff 100644
--- a/backend/plugins/github/tasks/pr_issue_convertor.go
+++ b/backend/plugins/github/tasks/pr_issue_convertor.go
@@ -34,7 +34,7 @@ func init() {
}
var ConvertPullRequestIssuesMeta = plugin.SubTaskMeta{
- Name: "convertPullRequestIssues",
+ Name: "Convert PR Issues",
EntryPoint: ConvertPullRequestIssues,
EnabledByDefault: true,
Description: "Convert tool layer table github_pull_request_issues into domain layer table pull_request_issues",
diff --git a/backend/plugins/github/tasks/pr_issue_enricher.go b/backend/plugins/github/tasks/pr_issue_enricher.go
index f2b8c13fe..bf4d2837d 100644
--- a/backend/plugins/github/tasks/pr_issue_enricher.go
+++ b/backend/plugins/github/tasks/pr_issue_enricher.go
@@ -35,7 +35,7 @@ func init() {
}
var EnrichPullRequestIssuesMeta = plugin.SubTaskMeta{
- Name: "enrichPullRequestIssues",
+ Name: "Enrich PR Issues",
EntryPoint: EnrichPullRequestIssues,
EnabledByDefault: true,
Description: "Create tool layer table github_pull_request_issues from github_pull_requests",
diff --git a/backend/plugins/github/tasks/pr_label_convertor.go b/backend/plugins/github/tasks/pr_label_convertor.go
index 9053dcd98..3351218a7 100644
--- a/backend/plugins/github/tasks/pr_label_convertor.go
+++ b/backend/plugins/github/tasks/pr_label_convertor.go
@@ -34,7 +34,7 @@ func init() {
}
var ConvertPullRequestLabelsMeta = plugin.SubTaskMeta{
- Name: "convertPullRequestLabels",
+ Name: "Convert PR Labels",
EntryPoint: ConvertPullRequestLabels,
EnabledByDefault: true,
Description: "Convert tool layer table github_pull_request_labels into domain layer table pull_request_labels",
diff --git a/backend/plugins/github/tasks/pr_review_collector.go b/backend/plugins/github/tasks/pr_review_collector.go
index a0f6576ef..60430c583 100644
--- a/backend/plugins/github/tasks/pr_review_collector.go
+++ b/backend/plugins/github/tasks/pr_review_collector.go
@@ -40,7 +40,7 @@ const RAW_PR_REVIEW_TABLE = "github_api_pull_request_reviews"
// this struct should be moved to `gitub_api_common.go`
var CollectApiPullRequestReviewsMeta = plugin.SubTaskMeta{
- Name: "collectApiPullRequestReviews",
+ Name: "Collect PR Reviews",
EntryPoint: CollectApiPullRequestReviews,
EnabledByDefault: true,
Description: "Collect PullRequestReviews data from Github api, supports both timeFilter and diffSync.",
diff --git a/backend/plugins/github/tasks/pr_review_comment_collector.go b/backend/plugins/github/tasks/pr_review_comment_collector.go
index e05ae2812..d53371444 100644
--- a/backend/plugins/github/tasks/pr_review_comment_collector.go
+++ b/backend/plugins/github/tasks/pr_review_comment_collector.go
@@ -37,7 +37,7 @@ const RAW_PR_REVIEW_COMMENTS_TABLE = "github_api_pull_request_review_comments"
// this struct should be moved to `github_api_common.go`
var CollectApiPrReviewCommentsMeta = plugin.SubTaskMeta{
- Name: "collectApiPrReviewCommentsMeta",
+ Name: "Collect PR Review Comments",
EntryPoint: CollectPrReviewComments,
EnabledByDefault: true,
Description: "Collect pr review comments data from Github api, supports both timeFilter and diffSync.",
diff --git a/backend/plugins/github/tasks/pr_review_comment_extractor.go b/backend/plugins/github/tasks/pr_review_comment_extractor.go
index 8daeb8920..7d93f2344 100644
--- a/backend/plugins/github/tasks/pr_review_comment_extractor.go
+++ b/backend/plugins/github/tasks/pr_review_comment_extractor.go
@@ -36,7 +36,7 @@ func init() {
}
var ExtractApiPrReviewCommentsMeta = plugin.SubTaskMeta{
- Name: "extractApiPrReviewComments",
+ Name: "Extract PR Review Comments",
EntryPoint: ExtractApiPrReviewComments,
EnabledByDefault: true,
Description: "Extract raw comment data into tool layer table github_pull_request_comments" +
diff --git a/backend/plugins/github/tasks/pr_review_convertor.go b/backend/plugins/github/tasks/pr_review_convertor.go
index 280f204a6..4ff2f3531 100644
--- a/backend/plugins/github/tasks/pr_review_convertor.go
+++ b/backend/plugins/github/tasks/pr_review_convertor.go
@@ -35,7 +35,7 @@ func init() {
}
var ConvertPullRequestReviewsMeta = plugin.SubTaskMeta{
- Name: "convertPullRequestReviews",
+ Name: "Convert PR Reviews",
EntryPoint: ConvertPullRequestReviews,
EnabledByDefault: true,
Description: "ConvertPullRequestReviews data from Github api",
diff --git a/backend/plugins/github/tasks/pr_review_extractor.go b/backend/plugins/github/tasks/pr_review_extractor.go
index 05d11f92d..c14a179e8 100644
--- a/backend/plugins/github/tasks/pr_review_extractor.go
+++ b/backend/plugins/github/tasks/pr_review_extractor.go
@@ -33,7 +33,7 @@ func init() {
}
var ExtractApiPullRequestReviewsMeta = plugin.SubTaskMeta{
- Name: "extractApiPullRequestReviews",
+ Name: "Extract PR Reviews",
EntryPoint: ExtractApiPullRequestReviews,
EnabledByDefault: true,
Description: "Extract raw PullRequestReviewers data into tool layer table github_reviewers",
diff --git a/backend/plugins/github/tasks/repo_convertor.go b/backend/plugins/github/tasks/repo_convertor.go
index 1866d2b7b..b9c711ad9 100644
--- a/backend/plugins/github/tasks/repo_convertor.go
+++ b/backend/plugins/github/tasks/repo_convertor.go
@@ -54,7 +54,7 @@ type GithubApiRepo struct {
}
var ConvertRepoMeta = plugin.SubTaskMeta{
- Name: "convertRepo",
+ Name: "Convert Repos",
EntryPoint: ConvertRepo,
EnabledByDefault: true,
Description: "Convert tool layer table github_repos into domain layer table repos and boards",
diff --git a/backend/plugins/github_graphql/tasks/account_collector.go b/backend/plugins/github_graphql/tasks/account_collector.go
index 3aed8c6a9..ad16068c4 100644
--- a/backend/plugins/github_graphql/tasks/account_collector.go
+++ b/backend/plugins/github_graphql/tasks/account_collector.go
@@ -58,7 +58,7 @@ type GraphqlQueryAccount struct {
}
var CollectAccountMeta = plugin.SubTaskMeta{
- Name: "CollectAccount",
+ Name: "Collect Users",
EntryPoint: CollectAccount,
EnabledByDefault: true,
Description: "Collect Account data from GithubGraphql api, does not support either timeFilter or diffSync.",
diff --git a/backend/plugins/github_graphql/tasks/account_extractor.go b/backend/plugins/github_graphql/tasks/account_extractor.go
index a4fff15ba..deb5877a9 100644
--- a/backend/plugins/github_graphql/tasks/account_extractor.go
+++ b/backend/plugins/github_graphql/tasks/account_extractor.go
@@ -30,7 +30,7 @@ import (
var _ plugin.SubTaskEntryPoint = ExtractAccounts
var ExtractAccountsMeta = plugin.SubTaskMeta{
- Name: "extractAccounts",
+ Name: "Extract Users",
EntryPoint: ExtractAccounts,
EnabledByDefault: true,
Description: "extract raw account data into tool layer table github_accounts",
diff --git a/backend/plugins/github_graphql/tasks/deployment_collector.go b/backend/plugins/github_graphql/tasks/deployment_collector.go
index 6de272f4f..437088629 100644
--- a/backend/plugins/github_graphql/tasks/deployment_collector.go
+++ b/backend/plugins/github_graphql/tasks/deployment_collector.go
@@ -35,7 +35,7 @@ const (
)
var CollectDeploymentsMeta = plugin.SubTaskMeta{
- Name: "CollectDeployments",
+ Name: "Collect Deployments",
EntryPoint: CollectDeployments,
EnabledByDefault: true,
Description: "collect github deployments to raw and tool layer from GithubGraphql api",
diff --git a/backend/plugins/github_graphql/tasks/deployment_extractor.go b/backend/plugins/github_graphql/tasks/deployment_extractor.go
index 408e70ca4..b2150effd 100644
--- a/backend/plugins/github_graphql/tasks/deployment_extractor.go
+++ b/backend/plugins/github_graphql/tasks/deployment_extractor.go
@@ -31,7 +31,7 @@ import (
var _ plugin.SubTaskEntryPoint = ExtractDeployments
var ExtractDeploymentsMeta = plugin.SubTaskMeta{
- Name: "extractDeployments",
+ Name: "Extract Deployments",
EntryPoint: ExtractDeployments,
EnabledByDefault: true,
Description: "extract raw deployment data into tool layer table github_graphql_deployment",
diff --git a/backend/plugins/github_graphql/tasks/issue_collector.go b/backend/plugins/github_graphql/tasks/issue_collector.go
index 5bd27267f..313b7cbac 100644
--- a/backend/plugins/github_graphql/tasks/issue_collector.go
+++ b/backend/plugins/github_graphql/tasks/issue_collector.go
@@ -71,7 +71,7 @@ type GraphqlQueryIssue struct {
}
var CollectIssuesMeta = plugin.SubTaskMeta{
- Name: "CollectIssues",
+ Name: "Collect Issues",
EntryPoint: CollectIssues,
EnabledByDefault: true,
Description: "Collect Issue data from GithubGraphql api, supports both timeFilter and diffSync.",
diff --git a/backend/plugins/github_graphql/tasks/issue_extractor.go b/backend/plugins/github_graphql/tasks/issue_extractor.go
index 9bc318589..b5652587e 100644
--- a/backend/plugins/github_graphql/tasks/issue_extractor.go
+++ b/backend/plugins/github_graphql/tasks/issue_extractor.go
@@ -33,7 +33,7 @@ import (
var _ plugin.SubTaskEntryPoint = ExtractAccounts
var ExtractIssuesMeta = plugin.SubTaskMeta{
- Name: "extractIssues",
+ Name: "Extract Issues",
EntryPoint: ExtractIssues,
EnabledByDefault: true,
Description: "Extract raw Issues data into tool layer table github_issues",
diff --git a/backend/plugins/github_graphql/tasks/job_collector.go b/backend/plugins/github_graphql/tasks/job_collector.go
index 650087de3..48ec17045 100644
--- a/backend/plugins/github_graphql/tasks/job_collector.go
+++ b/backend/plugins/github_graphql/tasks/job_collector.go
@@ -87,7 +87,7 @@ type SimpleWorkflowRun struct {
}
var CollectJobsMeta = plugin.SubTaskMeta{
- Name: "CollectJobs",
+ Name: "Collect Job Runs",
EntryPoint: CollectJobs,
EnabledByDefault: true,
Description: "Collect Jobs(CheckRun) data from GithubGraphql api, supports both timeFilter and diffSync.",
diff --git a/backend/plugins/github_graphql/tasks/job_extractor.go b/backend/plugins/github_graphql/tasks/job_extractor.go
index f5ce0dcec..4e38cc727 100644
--- a/backend/plugins/github_graphql/tasks/job_extractor.go
+++ b/backend/plugins/github_graphql/tasks/job_extractor.go
@@ -32,7 +32,7 @@ import (
var _ plugin.SubTaskEntryPoint = ExtractAccounts
var ExtractJobsMeta = plugin.SubTaskMeta{
- Name: "extractJobs",
+ Name: "Extract Jobs",
EntryPoint: ExtractJobs,
EnabledByDefault: true,
Description: "Extract raw run data into tool layer table github_jobs",
diff --git a/backend/plugins/github_graphql/tasks/pr_collector.go b/backend/plugins/github_graphql/tasks/pr_collector.go
index 66e007a8b..2bc887305 100644
--- a/backend/plugins/github_graphql/tasks/pr_collector.go
+++ b/backend/plugins/github_graphql/tasks/pr_collector.go
@@ -118,7 +118,7 @@ type GraphqlQueryCommit struct {
}
var CollectPrsMeta = plugin.SubTaskMeta{
- Name: "CollectPrs",
+ Name: "Collect Pull Requests",
EntryPoint: CollectPrs,
EnabledByDefault: true,
Description: "Collect Pr data from GithubGraphql api, supports both timeFilter and diffSync.",
diff --git a/backend/plugins/github_graphql/tasks/pr_extractor.go b/backend/plugins/github_graphql/tasks/pr_extractor.go
index fd90d1f99..c82975fc8 100644
--- a/backend/plugins/github_graphql/tasks/pr_extractor.go
+++ b/backend/plugins/github_graphql/tasks/pr_extractor.go
@@ -31,7 +31,7 @@ import (
var _ plugin.SubTaskEntryPoint = ExtractPrs
var ExtractPrsMeta = plugin.SubTaskMeta{
- Name: "extractPrs",
+ Name: "Extract Pull Requests",
EntryPoint: ExtractPrs,
EnabledByDefault: true,
Description: "Extract raw PullRequests data into tool layer table github_pull_requests",
diff --git a/backend/plugins/gitlab/tasks/account_collector.go b/backend/plugins/gitlab/tasks/account_collector.go
index 4d234056c..52ea1a72d 100644
--- a/backend/plugins/gitlab/tasks/account_collector.go
+++ b/backend/plugins/gitlab/tasks/account_collector.go
@@ -38,7 +38,7 @@ func init() {
const RAW_USER_TABLE = "gitlab_api_users"
var CollectAccountsMeta = plugin.SubTaskMeta{
- Name: "collectAccounts",
+ Name: "Collect Users",
EntryPoint: CollectAccounts,
EnabledByDefault: true,
Description: "collect gitlab users, does not support either timeFilter or diffSync.",
diff --git a/backend/plugins/gitlab/tasks/account_convertor.go b/backend/plugins/gitlab/tasks/account_convertor.go
index 9db00334c..c6bac24a1 100644
--- a/backend/plugins/gitlab/tasks/account_convertor.go
+++ b/backend/plugins/gitlab/tasks/account_convertor.go
@@ -35,7 +35,7 @@ func init() {
}
var ConvertAccountsMeta = plugin.SubTaskMeta{
- Name: "convertAccounts",
+ Name: "Convert Users",
EntryPoint: ConvertAccounts,
EnabledByDefault: true,
Description: "Convert tool layer table gitlab_users into domain layer table accounts",
diff --git a/backend/plugins/gitlab/tasks/account_extractor.go b/backend/plugins/gitlab/tasks/account_extractor.go
index 1ab39dcb7..d2d24b48c 100644
--- a/backend/plugins/gitlab/tasks/account_extractor.go
+++ b/backend/plugins/gitlab/tasks/account_extractor.go
@@ -32,7 +32,7 @@ func init() {
}
var ExtractAccountsMeta = plugin.SubTaskMeta{
- Name: "extractAccounts",
+ Name: "Extract Users",
EntryPoint: ExtractAccounts,
EnabledByDefault: true,
Description: "Extract raw workspace data into tool layer table _tool_gitlab_accounts",
diff --git a/backend/plugins/gitlab/tasks/commit_collector.go b/backend/plugins/gitlab/tasks/commit_collector.go
index 3649d0240..4bb2e960f 100644
--- a/backend/plugins/gitlab/tasks/commit_collector.go
+++ b/backend/plugins/gitlab/tasks/commit_collector.go
@@ -31,7 +31,7 @@ func init() {
const RAW_COMMIT_TABLE = "gitlab_api_commit"
var CollectApiCommitsMeta = plugin.SubTaskMeta{
- Name: "collectApiCommits",
+ Name: "Collect Commits",
EntryPoint: CollectApiCommits,
EnabledByDefault: false,
Description: "Collect commit data from gitlab api, does not support either timeFilter or diffSync.",
diff --git a/backend/plugins/gitlab/tasks/commit_convertor.go b/backend/plugins/gitlab/tasks/commit_convertor.go
index c285a785e..c0fb77a79 100644
--- a/backend/plugins/gitlab/tasks/commit_convertor.go
+++ b/backend/plugins/gitlab/tasks/commit_convertor.go
@@ -18,6 +18,8 @@ limitations under the License.
package tasks
import (
+ "reflect"
+
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models/domainlayer/code"
@@ -25,7 +27,6 @@ import (
"github.com/apache/incubator-devlake/core/plugin"
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/gitlab/models"
- "reflect"
)
func init() {
@@ -33,7 +34,7 @@ func init() {
}
var ConvertCommitsMeta = plugin.SubTaskMeta{
- Name: "convertApiCommits",
+ Name: "Convert Commits",
EntryPoint: ConvertApiCommits,
EnabledByDefault: false,
Description: "Update domain layer commit according to GitlabCommit",
diff --git a/backend/plugins/gitlab/tasks/commit_extractor.go b/backend/plugins/gitlab/tasks/commit_extractor.go
index 7cbacf997..664f8eae3 100644
--- a/backend/plugins/gitlab/tasks/commit_extractor.go
+++ b/backend/plugins/gitlab/tasks/commit_extractor.go
@@ -19,6 +19,7 @@ package tasks
import (
"encoding/json"
+
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
@@ -30,7 +31,7 @@ func init() {
}
var ExtractApiCommitsMeta = plugin.SubTaskMeta{
- Name: "extractApiCommits",
+ Name: "Extract Commits",
EntryPoint: ExtractApiCommits,
EnabledByDefault: false,
Description: "Extract raw commit data into tool layer table GitlabCommit,GitlabAccount and GitlabProjectCommit",
diff --git a/backend/plugins/gitlab/tasks/deployment_collector.go b/backend/plugins/gitlab/tasks/deployment_collector.go
index 616f44df3..6c4c2a74a 100644
--- a/backend/plugins/gitlab/tasks/deployment_collector.go
+++ b/backend/plugins/gitlab/tasks/deployment_collector.go
@@ -39,7 +39,7 @@ func init() {
}
var CollectDeploymentMeta = plugin.SubTaskMeta{
- Name: "CollectDeployment",
+ Name: "Collect Deployments",
EntryPoint: CollectDeployment,
EnabledByDefault: true,
Description: "Collect gitlab deployment from api into raw layer table",
diff --git a/backend/plugins/gitlab/tasks/deployment_convertor.go b/backend/plugins/gitlab/tasks/deployment_convertor.go
index b27b84d55..4db399b95 100644
--- a/backend/plugins/gitlab/tasks/deployment_convertor.go
+++ b/backend/plugins/gitlab/tasks/deployment_convertor.go
@@ -19,6 +19,9 @@ package tasks
import (
"fmt"
+ "reflect"
+ "time"
+
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models/domainlayer"
@@ -28,8 +31,6 @@ import (
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/gitlab/models"
"github.com/spf13/cast"
- "reflect"
- "time"
)
var _ plugin.SubTaskEntryPoint = ConvertDeployment
@@ -39,7 +40,7 @@ func init() {
}
var ConvertDeploymentMeta = plugin.SubTaskMeta{
- Name: "ConvertDeployment",
+ Name: "Convert Deployments",
EntryPoint: ConvertDeployment,
EnabledByDefault: true,
Description: "Convert gitlab deployment from tool layer to domain layer",
diff --git a/backend/plugins/gitlab/tasks/deployment_extractor.go b/backend/plugins/gitlab/tasks/deployment_extractor.go
index a84796cbf..0b439aceb 100644
--- a/backend/plugins/gitlab/tasks/deployment_extractor.go
+++ b/backend/plugins/gitlab/tasks/deployment_extractor.go
@@ -35,7 +35,7 @@ func init() {
}
var ExtractDeploymentMeta = plugin.SubTaskMeta{
- Name: "ExtractDeployment",
+ Name: "Extract Deployments",
EntryPoint: ExtractDeployment,
EnabledByDefault: true,
Description: "Extract gitlab deployment from raw layer to tool layer",
diff --git a/backend/plugins/gitlab/tasks/issue_assignee_convertor.go b/backend/plugins/gitlab/tasks/issue_assignee_convertor.go
index 9e7366e8b..be40d9675 100644
--- a/backend/plugins/gitlab/tasks/issue_assignee_convertor.go
+++ b/backend/plugins/gitlab/tasks/issue_assignee_convertor.go
@@ -30,7 +30,7 @@ import (
)
var ConvertIssueAssigneeMeta = plugin.SubTaskMeta{
- Name: "convertIssueAssignee",
+ Name: "convert Issue Assignees",
EntryPoint: ConvertIssueAssignee,
EnabledByDefault: true,
Description: "Convert tool layer table _tool_gitlab_issue_assignees into domain layer table issue_assignees",
diff --git a/backend/plugins/gitlab/tasks/issue_collector.go b/backend/plugins/gitlab/tasks/issue_collector.go
index 81c78be37..92c3fbc21 100644
--- a/backend/plugins/gitlab/tasks/issue_collector.go
+++ b/backend/plugins/gitlab/tasks/issue_collector.go
@@ -36,7 +36,7 @@ func init() {
const RAW_ISSUE_TABLE = "gitlab_api_issues"
var CollectApiIssuesMeta = plugin.SubTaskMeta{
- Name: "collectApiIssues",
+ Name: "Collect Issues",
EntryPoint: CollectApiIssues,
EnabledByDefault: true,
Description: "Collect issues data from Gitlab api, supports both timeFilter and diffSync.",
diff --git a/backend/plugins/gitlab/tasks/issue_convertor.go b/backend/plugins/gitlab/tasks/issue_convertor.go
index 8a09bd7d2..9be3c1d18 100644
--- a/backend/plugins/gitlab/tasks/issue_convertor.go
+++ b/backend/plugins/gitlab/tasks/issue_convertor.go
@@ -37,7 +37,7 @@ func init() {
}
var ConvertIssuesMeta = plugin.SubTaskMeta{
- Name: "convertIssues",
+ Name: "Convert Issues",
EntryPoint: ConvertIssues,
EnabledByDefault: true,
Description: "Convert tool layer table gitlab_issues into domain layer table issues",
diff --git a/backend/plugins/gitlab/tasks/issue_extractor.go b/backend/plugins/gitlab/tasks/issue_extractor.go
index 9e76e7444..c834935b3 100644
--- a/backend/plugins/gitlab/tasks/issue_extractor.go
+++ b/backend/plugins/gitlab/tasks/issue_extractor.go
@@ -33,7 +33,7 @@ func init() {
}
var ExtractApiIssuesMeta = plugin.SubTaskMeta{
- Name: "extractApiIssues",
+ Name: "Extract Issues",
EntryPoint: ExtractApiIssues,
EnabledByDefault: true,
Description: "Extract raw Issues data into tool layer table gitlab_issues",
diff --git a/backend/plugins/gitlab/tasks/issue_label_convertor.go b/backend/plugins/gitlab/tasks/issue_label_convertor.go
index c79ce1590..c5e8968dd 100644
--- a/backend/plugins/gitlab/tasks/issue_label_convertor.go
+++ b/backend/plugins/gitlab/tasks/issue_label_convertor.go
@@ -18,6 +18,8 @@ limitations under the License.
package tasks
import (
+ "reflect"
+
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models/domainlayer/didgen"
@@ -25,7 +27,6 @@ import (
"github.com/apache/incubator-devlake/core/plugin"
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/gitlab/models"
- "reflect"
)
func init() {
@@ -33,7 +34,7 @@ func init() {
}
var ConvertIssueLabelsMeta = plugin.SubTaskMeta{
- Name: "convertIssueLabels",
+ Name: "Convert Issue Labels",
EntryPoint: ConvertIssueLabels,
EnabledByDefault: true,
Description: "Convert tool layer table gitlab_issue_labels into domain layer table issue_labels",
diff --git a/backend/plugins/gitlab/tasks/job_collector.go b/backend/plugins/gitlab/tasks/job_collector.go
index 755f8800b..04dda2998 100644
--- a/backend/plugins/gitlab/tasks/job_collector.go
+++ b/backend/plugins/gitlab/tasks/job_collector.go
@@ -42,7 +42,7 @@ type SimpleGitlabApiJob struct {
}
var CollectApiJobsMeta = plugin.SubTaskMeta{
- Name: "collectApiJobs",
+ Name: "Collect Job Runs",
EntryPoint: CollectApiJobs,
EnabledByDefault: true,
Description: "Collect job data from gitlab api, supports both timeFilter and diffSync.",
diff --git a/backend/plugins/gitlab/tasks/job_convertor.go b/backend/plugins/gitlab/tasks/job_convertor.go
index dfb70abab..58bd8f712 100644
--- a/backend/plugins/gitlab/tasks/job_convertor.go
+++ b/backend/plugins/gitlab/tasks/job_convertor.go
@@ -18,6 +18,9 @@ limitations under the License.
package tasks
import (
+ "reflect"
+ "time"
+
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models/domainlayer"
@@ -26,8 +29,6 @@ import (
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
gitlabModels "github.com/apache/incubator-devlake/plugins/gitlab/models"
- "reflect"
- "time"
)
func init() {
@@ -35,7 +36,7 @@ func init() {
}
var ConvertJobMeta = plugin.SubTaskMeta{
- Name: "convertJobs",
+ Name: "Convert Job Runs",
EntryPoint: ConvertJobs,
EnabledByDefault: true,
Description: "Convert tool layer table gitlab_job into domain layer table job",
diff --git a/backend/plugins/gitlab/tasks/job_extractor.go b/backend/plugins/gitlab/tasks/job_extractor.go
index 7eb989770..ff5031ce6 100644
--- a/backend/plugins/gitlab/tasks/job_extractor.go
+++ b/backend/plugins/gitlab/tasks/job_extractor.go
@@ -51,7 +51,7 @@ type ApiJob struct {
}
var ExtractApiJobsMeta = plugin.SubTaskMeta{
- Name: "extractApiJobs",
+ Name: "Extract Job Runs",
EntryPoint: ExtractApiJobs,
EnabledByDefault: true,
Description: "Extract raw GitlabJob data into tool layer table GitlabPipeline",
diff --git a/backend/plugins/gitlab/tasks/mr_collector.go b/backend/plugins/gitlab/tasks/mr_collector.go
index ce0f21b3c..029b19043 100644
--- a/backend/plugins/gitlab/tasks/mr_collector.go
+++ b/backend/plugins/gitlab/tasks/mr_collector.go
@@ -33,7 +33,7 @@ func init() {
}
var CollectApiMergeRequestsMeta = plugin.SubTaskMeta{
- Name: "collectApiMergeRequests",
+ Name: "Collect Merge Requests",
EntryPoint: CollectApiMergeRequests,
EnabledByDefault: true,
Description: "Collect merge requests data from gitlab api, supports both timeFilter and diffSync.",
diff --git a/backend/plugins/gitlab/tasks/mr_comment_convertor.go b/backend/plugins/gitlab/tasks/mr_comment_convertor.go
index c309792cf..3b6d1ccee 100644
--- a/backend/plugins/gitlab/tasks/mr_comment_convertor.go
+++ b/backend/plugins/gitlab/tasks/mr_comment_convertor.go
@@ -35,7 +35,7 @@ func init() {
}
var ConvertMrCommentMeta = plugin.SubTaskMeta{
- Name: "convertMergeRequestComment",
+ Name: "Convert MR Comments",
EntryPoint: ConvertMergeRequestComment,
EnabledByDefault: true,
Description: "Add domain layer Comment according to GitlabMrComment",
diff --git a/backend/plugins/gitlab/tasks/mr_commit_collector.go b/backend/plugins/gitlab/tasks/mr_commit_collector.go
index 2fbbdd48a..f99211218 100644
--- a/backend/plugins/gitlab/tasks/mr_commit_collector.go
+++ b/backend/plugins/gitlab/tasks/mr_commit_collector.go
@@ -30,7 +30,7 @@ func init() {
const RAW_MERGE_REQUEST_COMMITS_TABLE = "gitlab_api_merge_request_commits"
var CollectApiMrCommitsMeta = plugin.SubTaskMeta{
- Name: "collectApiMergeRequestsCommits",
+ Name: "Collect MR Commits",
EntryPoint: CollectApiMergeRequestsCommits,
EnabledByDefault: true,
Description: "Collect merge requests commits data from gitlab api, supports timeFilter but not diffSync.",
diff --git a/backend/plugins/gitlab/tasks/mr_commit_convertor.go b/backend/plugins/gitlab/tasks/mr_commit_convertor.go
index 751e84a96..74ae0d926 100644
--- a/backend/plugins/gitlab/tasks/mr_commit_convertor.go
+++ b/backend/plugins/gitlab/tasks/mr_commit_convertor.go
@@ -18,6 +18,8 @@ limitations under the License.
package tasks
import (
+ "reflect"
+
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models/domainlayer/code"
@@ -25,7 +27,6 @@ import (
"github.com/apache/incubator-devlake/core/plugin"
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/gitlab/models"
- "reflect"
)
func init() {
@@ -33,7 +34,7 @@ func init() {
}
var ConvertApiMrCommitsMeta = plugin.SubTaskMeta{
- Name: "convertApiMergeRequestsCommits",
+ Name: "Convert MR Commits",
EntryPoint: ConvertApiMergeRequestsCommits,
EnabledByDefault: true,
Description: "Add domain layer PullRequestCommit according to GitlabMrCommit",
diff --git a/backend/plugins/gitlab/tasks/mr_commit_extractor.go b/backend/plugins/gitlab/tasks/mr_commit_extractor.go
index aa71305fb..5c73c8665 100644
--- a/backend/plugins/gitlab/tasks/mr_commit_extractor.go
+++ b/backend/plugins/gitlab/tasks/mr_commit_extractor.go
@@ -32,7 +32,7 @@ func init() {
}
var ExtractApiMrCommitsMeta = plugin.SubTaskMeta{
- Name: "extractApiMergeRequestsCommits",
+ Name: "Extract MR Commits",
EntryPoint: ExtractApiMergeRequestsCommits,
EnabledByDefault: true,
Description: "Extract raw merge requests commit data into tool layer table GitlabMrCommit and GitlabCommit",
diff --git a/backend/plugins/gitlab/tasks/mr_convertor.go b/backend/plugins/gitlab/tasks/mr_convertor.go
index 53f5ea436..478276c71 100644
--- a/backend/plugins/gitlab/tasks/mr_convertor.go
+++ b/backend/plugins/gitlab/tasks/mr_convertor.go
@@ -35,7 +35,7 @@ func init() {
}
var ConvertApiMergeRequestsMeta = plugin.SubTaskMeta{
- Name: "convertApiMergeRequests",
+ Name: "Convert Merge Requests",
EntryPoint: ConvertApiMergeRequests,
EnabledByDefault: true,
Description: "Add domain layer PullRequest according to GitlabMergeRequest",
diff --git a/backend/plugins/gitlab/tasks/mr_detail_collector.go b/backend/plugins/gitlab/tasks/mr_detail_collector.go
index f60871a24..eb4f8f3a6 100644
--- a/backend/plugins/gitlab/tasks/mr_detail_collector.go
+++ b/backend/plugins/gitlab/tasks/mr_detail_collector.go
@@ -34,7 +34,7 @@ func init() {
const RAW_MERGE_REQUEST_DETAIL_TABLE = "gitlab_api_merge_request_details"
var CollectApiMergeRequestDetailsMeta = plugin.SubTaskMeta{
- Name: "collectApiMergeRequestDetails",
+ Name: "Collect MR Details",
EntryPoint: CollectApiMergeRequestDetails,
EnabledByDefault: true,
Description: "Collect merge request Details data from gitlab api, supports timeFilter but not diffSync.",
diff --git a/backend/plugins/gitlab/tasks/mr_detail_extractor.go b/backend/plugins/gitlab/tasks/mr_detail_extractor.go
index 9453a59fc..38ce2901c 100644
--- a/backend/plugins/gitlab/tasks/mr_detail_extractor.go
+++ b/backend/plugins/gitlab/tasks/mr_detail_extractor.go
@@ -32,7 +32,7 @@ func init() {
}
var ExtractApiMergeRequestDetailsMeta = plugin.SubTaskMeta{
- Name: "extractApiMergeRequestDetails",
+ Name: "Extract MR Details",
EntryPoint: ExtractApiMergeRequestDetails,
EnabledByDefault: true,
Description: "Extract raw merge request Details data into tool layer table GitlabMergeRequest and GitlabReviewer",
diff --git a/backend/plugins/gitlab/tasks/mr_enricher.go b/backend/plugins/gitlab/tasks/mr_enricher.go
index 256dffb7e..327765208 100644
--- a/backend/plugins/gitlab/tasks/mr_enricher.go
+++ b/backend/plugins/gitlab/tasks/mr_enricher.go
@@ -18,13 +18,14 @@ limitations under the License.
package tasks
import (
+ "reflect"
+ "time"
+
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/gitlab/models"
- "reflect"
- "time"
)
func init() {
@@ -32,7 +33,7 @@ func init() {
}
var EnrichMergeRequestsMeta = plugin.SubTaskMeta{
- Name: "enrichMrs",
+ Name: "Enrich Merge Requests",
EntryPoint: EnrichMergeRequests,
EnabledByDefault: true,
Description: "Enrich merge requests data from GitlabCommit, GitlabMrNote and GitlabMergeRequest",
diff --git a/backend/plugins/gitlab/tasks/mr_extractor.go b/backend/plugins/gitlab/tasks/mr_extractor.go
index 92bfccfa3..a45bc924e 100644
--- a/backend/plugins/gitlab/tasks/mr_extractor.go
+++ b/backend/plugins/gitlab/tasks/mr_extractor.go
@@ -77,7 +77,7 @@ type Reviewer struct {
}
var ExtractApiMergeRequestsMeta = plugin.SubTaskMeta{
- Name: "extractApiMergeRequests",
+ Name: "Extract Merge Requests",
EntryPoint: ExtractApiMergeRequests,
EnabledByDefault: true,
Description: "Extract raw merge requests data into tool layer table GitlabMergeRequest and GitlabReviewer",
diff --git a/backend/plugins/gitlab/tasks/mr_label_convertor.go b/backend/plugins/gitlab/tasks/mr_label_convertor.go
index 603332438..934085e81 100644
--- a/backend/plugins/gitlab/tasks/mr_label_convertor.go
+++ b/backend/plugins/gitlab/tasks/mr_label_convertor.go
@@ -18,6 +18,8 @@ limitations under the License.
package tasks
import (
+ "reflect"
+
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models/domainlayer/code"
@@ -25,7 +27,6 @@ import (
"github.com/apache/incubator-devlake/core/plugin"
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/gitlab/models"
- "reflect"
)
func init() {
@@ -33,7 +34,7 @@ func init() {
}
var ConvertMrLabelsMeta = plugin.SubTaskMeta{
- Name: "convertMrLabels",
+ Name: "Convert MR Labels",
EntryPoint: ConvertMrLabels,
EnabledByDefault: true,
Description: "Convert tool layer table gitlab_mr_labels into domain layer table pull_request_labels",
diff --git a/backend/plugins/gitlab/tasks/mr_note_collector.go b/backend/plugins/gitlab/tasks/mr_note_collector.go
index 282135982..469648f71 100644
--- a/backend/plugins/gitlab/tasks/mr_note_collector.go
+++ b/backend/plugins/gitlab/tasks/mr_note_collector.go
@@ -30,7 +30,7 @@ func init() {
const RAW_MERGE_REQUEST_NOTES_TABLE = "gitlab_api_merge_request_notes"
var CollectApiMrNotesMeta = plugin.SubTaskMeta{
- Name: "collectApiMergeRequestsNotes",
+ Name: "Collect MR Notes",
EntryPoint: CollectApiMergeRequestsNotes,
EnabledByDefault: true,
Description: "Collect merge requests notes data from gitlab api, supports timeFilter but not diffSync.",
diff --git a/backend/plugins/gitlab/tasks/mr_note_extractor.go b/backend/plugins/gitlab/tasks/mr_note_extractor.go
index 3b6786c52..a27d1141c 100644
--- a/backend/plugins/gitlab/tasks/mr_note_extractor.go
+++ b/backend/plugins/gitlab/tasks/mr_note_extractor.go
@@ -49,7 +49,7 @@ type MergeRequestNote struct {
}
var ExtractApiMrNotesMeta = plugin.SubTaskMeta{
- Name: "extractApiMergeRequestsNotes",
+ Name: "Extract MR Notes",
EntryPoint: ExtractApiMergeRequestsNotes,
EnabledByDefault: true,
Description: "Extract raw merge requests notes data into tool layer table GitlabMrNote",
diff --git a/backend/plugins/gitlab/tasks/pipeline_collector.go b/backend/plugins/gitlab/tasks/pipeline_collector.go
index 06a14c925..ceb911d68 100644
--- a/backend/plugins/gitlab/tasks/pipeline_collector.go
+++ b/backend/plugins/gitlab/tasks/pipeline_collector.go
@@ -34,7 +34,7 @@ func init() {
const RAW_PIPELINE_TABLE = "gitlab_api_pipeline"
var CollectApiPipelinesMeta = plugin.SubTaskMeta{
- Name: "collectApiPipelines",
+ Name: "Collect Pipelines",
EntryPoint: CollectApiPipelines,
EnabledByDefault: true,
Description: "Collect pipeline data from gitlab api, supports both timeFilter and diffSync.",
diff --git a/backend/plugins/gitlab/tasks/pipeline_commit_convertor.go b/backend/plugins/gitlab/tasks/pipeline_commit_convertor.go
index 9950d39c4..a15928c4d 100644
--- a/backend/plugins/gitlab/tasks/pipeline_commit_convertor.go
+++ b/backend/plugins/gitlab/tasks/pipeline_commit_convertor.go
@@ -34,7 +34,7 @@ func init() {
}
var ConvertPipelineCommitMeta = plugin.SubTaskMeta{
- Name: "convertPipelineCommits",
+ Name: "Convert Pipeline Commits",
EntryPoint: ConvertPipelineCommits,
EnabledByDefault: true,
Description: "Convert tool layer table gitlab_pipeline_project into domain layer table pipeline",
diff --git a/backend/plugins/gitlab/tasks/pipeline_detail_collector.go b/backend/plugins/gitlab/tasks/pipeline_detail_collector.go
index ebafdc2c0..227d19e93 100644
--- a/backend/plugins/gitlab/tasks/pipeline_detail_collector.go
+++ b/backend/plugins/gitlab/tasks/pipeline_detail_collector.go
@@ -35,7 +35,7 @@ func init() {
const RAW_PIPELINE_DETAILS_TABLE = "gitlab_api_pipeline_details"
var CollectApiPipelineDetailsMeta = plugin.SubTaskMeta{
- Name: "collectApiPipelineDetails",
+ Name: "Collect Pipeline Details",
EntryPoint: CollectApiPipelineDetails,
EnabledByDefault: true,
Description: "Collect pipeline details data from gitlab api, supports both timeFilter and diffSync.",
diff --git a/backend/plugins/gitlab/tasks/pipeline_detail_convertor.go b/backend/plugins/gitlab/tasks/pipeline_detail_convertor.go
index 36f9bcea6..3818a9a69 100644
--- a/backend/plugins/gitlab/tasks/pipeline_detail_convertor.go
+++ b/backend/plugins/gitlab/tasks/pipeline_detail_convertor.go
@@ -37,7 +37,7 @@ func init() {
}
var ConvertDetailPipelineMeta = plugin.SubTaskMeta{
- Name: "convertDetailPipelines",
+ Name: "Convert Detail Pipelines",
EntryPoint: ConvertDetailPipelines,
EnabledByDefault: true,
Description: "Convert tool layer table gitlab_detail_pipeline into domain layer table pipeline",
diff --git a/backend/plugins/gitlab/tasks/pipeline_detail_extractor.go b/backend/plugins/gitlab/tasks/pipeline_detail_extractor.go
index 52c12c04a..83dab29c7 100644
--- a/backend/plugins/gitlab/tasks/pipeline_detail_extractor.go
+++ b/backend/plugins/gitlab/tasks/pipeline_detail_extractor.go
@@ -33,7 +33,7 @@ func init() {
}
var ExtractApiPipelineDetailsMeta = plugin.SubTaskMeta{
- Name: "extractApiPipelineDetails",
+ Name: "Extract Pipeline Details",
EntryPoint: ExtractApiPipelineDetails,
EnabledByDefault: true,
Description: "Extract raw pipeline details data into tool layer table GitlabPipeline",
diff --git a/backend/plugins/gitlab/tasks/pipeline_extractor.go b/backend/plugins/gitlab/tasks/pipeline_extractor.go
index 3b392bf09..738109ac4 100644
--- a/backend/plugins/gitlab/tasks/pipeline_extractor.go
+++ b/backend/plugins/gitlab/tasks/pipeline_extractor.go
@@ -61,7 +61,7 @@ type ApiPipeline struct {
}
var ExtractApiPipelinesMeta = plugin.SubTaskMeta{
- Name: "extractApiPipelines",
+ Name: "Extract Pipelines",
EntryPoint: ExtractApiPipelines,
EnabledByDefault: true,
Description: "Extract raw pipelines data into tool layer table GitlabPipeline",
diff --git a/backend/plugins/gitlab/tasks/project_convertor.go b/backend/plugins/gitlab/tasks/project_convertor.go
index 614355f30..920f1a990 100644
--- a/backend/plugins/gitlab/tasks/project_convertor.go
+++ b/backend/plugins/gitlab/tasks/project_convertor.go
@@ -59,7 +59,7 @@ type GitlabApiProject struct {
}
var ConvertProjectMeta = plugin.SubTaskMeta{
- Name: "convertApiProject",
+ Name: "Convert Projects",
EntryPoint: ConvertApiProjects,
EnabledByDefault: true,
Description: "Add domain layer Repo according to GitlabProject",
diff --git a/backend/plugins/gitlab/tasks/tag_collector.go b/backend/plugins/gitlab/tasks/tag_collector.go
index af5e518ae..92800c5c9 100644
--- a/backend/plugins/gitlab/tasks/tag_collector.go
+++ b/backend/plugins/gitlab/tasks/tag_collector.go
@@ -30,7 +30,7 @@ func init() {
const RAW_TAG_TABLE = "gitlab_api_tag"
var CollectTagMeta = plugin.SubTaskMeta{
- Name: "collectApiTag",
+ Name: "Collect Tags",
EntryPoint: CollectApiTag,
EnabledByDefault: false,
Description: "Collect tag data from gitlab api, does not support either timeFilter or diffSync.",
diff --git a/backend/plugins/gitlab/tasks/tag_extractor.go b/backend/plugins/gitlab/tasks/tag_extractor.go
index 9a94de337..8aba401f3 100644
--- a/backend/plugins/gitlab/tasks/tag_extractor.go
+++ b/backend/plugins/gitlab/tasks/tag_extractor.go
@@ -19,6 +19,7 @@ package tasks
import (
"encoding/json"
+
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
@@ -41,7 +42,7 @@ type GitlabApiTag struct {
}
var ExtractTagMeta = plugin.SubTaskMeta{
- Name: "extractApiTag",
+ Name: "Extract Tags",
EntryPoint: ExtractApiTag,
EnabledByDefault: false,
Description: "Extract raw tag data into tool layer table GitlabTag",
diff --git a/backend/plugins/gitlab/tasks/trigger_job_collector.go b/backend/plugins/gitlab/tasks/trigger_job_collector.go
index 50e4d04e1..755eeb67a 100644
--- a/backend/plugins/gitlab/tasks/trigger_job_collector.go
+++ b/backend/plugins/gitlab/tasks/trigger_job_collector.go
@@ -34,7 +34,7 @@ func init() {
const RAW_TRIGGER_JOB_TABLE = "gitlab_api_trigger_job"
var CollectApiTriggerJobsMeta = plugin.SubTaskMeta{
- Name: "collectApiTriggerJobs",
+ Name: "Collect Trigger Jobs",
EntryPoint: CollectApiTriggerJobs,
EnabledByDefault: false,
Description: "Collect job data from gitlab api, supports both timeFilter and diffSync.",
diff --git a/backend/plugins/gitlab/tasks/trigger_job_extractor.go b/backend/plugins/gitlab/tasks/trigger_job_extractor.go
index f9b64eeb4..f429f9b4d 100644
--- a/backend/plugins/gitlab/tasks/trigger_job_extractor.go
+++ b/backend/plugins/gitlab/tasks/trigger_job_extractor.go
@@ -19,11 +19,12 @@ package tasks
import (
"encoding/json"
+ "time"
+
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/gitlab/models"
- "time"
)
func init() {
@@ -46,7 +47,7 @@ type ApiTriggerJob struct {
}
var ExtractApiTriggerJobsMeta = plugin.SubTaskMeta{
- Name: "extractApiTriggerJobs",
+ Name: "Extract Trigger Jobs",
EntryPoint: ExtractApiTriggerJobs,
EnabledByDefault: true,
Description: "Extract raw Gitlab trigger jobs data into tool layer table GitlabPipeline",
diff --git a/backend/python/pydevlake/pydevlake/subtasks.py b/backend/python/pydevlake/pydevlake/subtasks.py
index c1552f5b6..d3b549e0b 100644
--- a/backend/python/pydevlake/pydevlake/subtasks.py
+++ b/backend/python/pydevlake/pydevlake/subtasks.py
@@ -34,7 +34,8 @@ class Subtask:
@property
def name(self):
- return f'{self.verb.lower()}{self.stream.plugin_name.capitalize()}{self.stream.name.capitalize()}'
+ #return f'{self.verb.lower()}{self.stream.plugin_name.capitalize()}{self.stream.name.capitalize()}'
+ return f'{self.verb.capitalize()} {self.stream.name.capitalize()}'
@property
def description(self):
diff --git a/backend/server/api/router.go b/backend/server/api/router.go
index 188c29e4c..9577b2494 100644
--- a/backend/server/api/router.go
+++ b/backend/server/api/router.go
@@ -48,6 +48,7 @@ func RegisterRouter(r *gin.Engine, basicRes context.BasicRes) {
r.GET("/pipelines/:pipelineId", pipelines.Get)
r.DELETE("/pipelines/:pipelineId", pipelines.Delete)
r.GET("/pipelines/:pipelineId/tasks", task.GetTaskByPipeline)
+ r.GET("/pipelines/:pipelineId/subtasks", task.GetSubtaskByPipeline)
r.POST("/pipelines/:pipelineId/rerun", pipelines.PostRerun)
r.GET("/pipelines/:pipelineId/logging.tar.gz", pipelines.DownloadLogs)
diff --git a/backend/server/api/task/task.go b/backend/server/api/task/task.go
index f43e574a1..594e9d6c3 100644
--- a/backend/server/api/task/task.go
+++ b/backend/server/api/task/task.go
@@ -72,6 +72,30 @@ func GetTaskByPipeline(c *gin.Context) {
shared.ApiOutputSuccess(c, getTaskResponse{Tasks: tasks, Count: len(tasks)}, http.StatusOK)
}
+// GetSubtasksByPipeline return most recent subtasks
+// @Summary Get subtasks, only the most recent subtasks will be returned
+// @Tags framework/tasks
+// @Accept application/json
+// @Param pipelineId path int true "pipelineId"
+// @Success 200 {object} models.SubTasksOuput
+// @Failure 400 {object} shared.ApiBody "Bad Request"
+// @Failure 500 {object} shared.ApiBody "Internal Error"
+// @Router /pipelines/{pipelineId}/subtasks [get]
+func GetSubtaskByPipeline(c *gin.Context) {
+ pipelineId, err := strconv.ParseUint(c.Param("pipelineId"), 10, 64)
+ if err != nil {
+ shared.ApiOutputError(c, errors.BadInput.Wrap(err, "invalid pipeline ID format"))
+ return
+ }
+ subTasksOuput, err := services.GetSubTasksInfo(pipelineId, true, nil)
+ if err != nil {
+ shared.ApiOutputError(c, errors.Default.Wrap(err, "error getting tasks"))
+ return
+ }
+
+ shared.ApiOutputSuccess(c, subTasksOuput, http.StatusOK)
+}
+
// RerunTask rerun the specified task.
// @Summary rerun task
// @Tags framework/tasks
diff --git a/backend/server/services/task.go b/backend/server/services/task.go
index 885014fa3..5640b7a6c 100644
--- a/backend/server/services/task.go
+++ b/backend/server/services/task.go
@@ -20,6 +20,7 @@ package services
import (
"context"
"fmt"
+ "math"
"strings"
"github.com/apache/incubator-devlake/core/dal"
@@ -236,3 +237,147 @@ func RerunTask(taskId uint64) (*models.Task, errors.Error) {
rerunTask.Options = taskOption
return rerunTask, nil
}
+
+// GetSubTasksInfo returns subtask list of the pipeline, only the most recently subtasks would be returned
+func GetSubTasksInfo(pipelineId uint64, shouldSanitize bool, tx dal.Dal) (*models.SubTasksOuput, errors.Error) {
+ if tx == nil {
+ tx = db
+ }
+ var tasks []*models.Task
+ err := tx.All(&tasks, dal.Where("pipeline_id = ?", pipelineId))
+ if err != nil {
+ return nil, err
+ }
+ filterTasks := filterTasks(tasks)
+ var subtasksInfo []models.SubtasksInfo
+ var totalSubtasksCount int64
+ var totalFinishedSubTasksCount int64
+ var count int64
+ var status []string
+ for _, task := range filterTasks {
+ // skip org plugin step
+ if task.Plugin == "org" {
+ continue
+ }
+ subTaskResult := models.SubtasksInfo{
+ ID: task.ID,
+ PipelineID: task.PipelineId,
+ CreatedAt: task.CreatedAt,
+ UpdatedAt: task.UpdatedAt,
+ BeganAt: task.BeganAt,
+ FinishedAt: task.FinishedAt,
+ Plugin: task.Plugin,
+ Status: task.Status,
+ Message: task.Message,
+ ErrorName: task.ErrorName,
+ SpentSeconds: task.SpentSeconds,
+ }
+ if shouldSanitize {
+ taskOption, err := SanitizePluginOption(task.Plugin, task.Options)
+ if err != nil {
+ return nil, errors.Convert(err)
+ }
+ subTaskResult.Options = taskOption
+ }
+
+ subtasks := []*models.Subtask{}
+ err = tx.All(&subtasks, dal.Where("task_id = ?", task.ID))
+ if err != nil {
+ return nil, err
+ }
+ for _, subtask := range subtasks {
+ t := &models.SubtaskDetails{
+ ID: subtask.ID,
+ CreatedAt: subtask.CreatedAt,
+ UpdatedAt: subtask.UpdatedAt,
+ TaskID: subtask.TaskID,
+ Name: subtask.Name,
+ Number: subtask.Number,
+ BeganAt: subtask.BeganAt,
+ FinishedAt: subtask.FinishedAt,
+ SpentSeconds: subtask.SpentSeconds,
+ FinishedRecords: subtask.FinishedRecords,
+ Sequence: subtask.Sequence,
+ IsCollector: subtask.IsCollector,
+ IsFailed: subtask.IsFailed,
+ Message: subtask.Message,
+ }
+ subTaskResult.SubtaskDetails = append(subTaskResult.SubtaskDetails, t)
+ }
+ subtasksInfo = append(subtasksInfo, subTaskResult)
+
+ collectSubtasksCount := errors.Must1(tx.Count(dal.From("_devlake_subtasks"), dal.Where("task_id = ?", task.ID)))
+ totalSubtasksCount += collectSubtasksCount
+ finishedSubTasksCount := errors.Must1(tx.Count(dal.From("_devlake_subtasks"), dal.Where("task_id = ? and finished_at is not null", task.ID)))
+ totalFinishedSubTasksCount += finishedSubTasksCount
+ count++
+
+ status = append(status, task.Status)
+ }
+
+ subTasksOuput := &models.SubTasksOuput{}
+ subTasksOuput.SubtasksInfo = subtasksInfo
+ subTasksOuput.Count = totalSubtasksCount
+
+ completionRateFloat := float64(totalFinishedSubTasksCount) / float64(totalSubtasksCount)
+ roundedCompletionRate := math.Round(completionRateFloat*100) / 100
+ subTasksOuput.CompletionRate = roundedCompletionRate
+
+ subTasksOuput.Status = getTaskStatus(status)
+ subTasksOuput.Count = count
+
+ return subTasksOuput, nil
+}
+
+func filterTasks(tasks []*models.Task) []*models.Task {
+ taskMap := make(map[string]*models.Task)
+
+ for _, task := range tasks {
+ if task.Plugin == "org" {
+ continue
+ }
+
+ if existingTask, ok := taskMap[task.Plugin]; ok {
+ if task.BeganAt != nil && (existingTask.BeganAt == nil || task.BeganAt.After(*existingTask.BeganAt)) {
+ taskMap[task.Plugin] = task
+ }
+ } else {
+ taskMap[task.Plugin] = task
+ }
+ }
+
+ var filteredTasks []*models.Task
+ for _, task := range taskMap {
+ filteredTasks = append(filteredTasks, task)
+ }
+
+ return filteredTasks
+}
+
+func getTaskStatus(statuses []string) string {
+ var status string
+ if len(statuses) == 0 {
+ return status
+ }
+
+ failedCount := 0
+ completedCount := 0
+ for _, s := range statuses {
+ if s == models.TASK_FAILED {
+ failedCount++
+ } else if s == models.TASK_COMPLETED {
+ completedCount++
+ }
+ }
+ if failedCount > 0 && completedCount > 0 {
+ status = "TASK_PARTIAL"
+ } else if failedCount == len(statuses) {
+ status = models.TASK_FAILED
+ } else if completedCount == len(statuses) {
+ status = models.TASK_COMPLETED
+ } else {
+ status = models.TASK_RUNNING
+ }
+
+ return status
+}