You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by ab...@apache.org on 2023/11/16 06:35:27 UTC
(incubator-devlake) branch feat#6088 updated (01f9c656c -> 52e35e393)
This is an automated email from the ASF dual-hosted git repository.
abeizn pushed a change to branch feat#6088
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
omit 01f9c656c feat: support skipCollectors in Advanced mode blueprint
omit 01505fc74 feat: add skip collector at advanced mode when subtask not nil
add 6c7540033 feat(config-ui): support theme setting (#6446)
add c1e0310a6 docs: page and pageSize should not be required (#6452)
add 8c1d80556 fix: syntax error near unexpected token & (#6458)
add 1a8fe13cd fix: close #6456 (#6459)
add b3aa4b3f2 feat(config-ui): use new table component to replace old (#6405)
add c7a03078b fix: jenkins stages durationMillis<0 (#6462)
add ebb699e84 fix(config-ui): show error when initializing connection error (#6471)
add d69a4f216 Change the type of `duration_sec` from int to float (#6465)
add b6d8d8a59 fix(webhook): add migration script when webhooks have no API keys (#6472)
add 7cc6c0491 fix: empty version (#6474)
new 5fcca4320 feat: add skip collector at advanced mode when subtask not nil
new 52e35e393 feat: support skipCollectors in Advanced mode blueprint
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (01f9c656c)
\
N -- N -- N refs/heads/feat#6088 (52e35e393)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
backend/Makefile | 4 +-
.../models/domainlayer/devops/cicd_deployment.go | 2 +-
.../domainlayer/devops/cicd_deployment_commit.go | 2 +-
.../models/domainlayer/devops/cicd_pipeline.go | 2 +-
.../core/models/domainlayer/devops/cicd_task.go | 2 +-
...ipeline.go => 20220905_modify_cicd_pipeline.go} | 0
...cicd_tasks.go => 20220908_modify_cicd_tasks.go} | 0
...s_diffs.go => 20221109_modify_commits_diffs.go} | 0
.../20231115_change_duration_sec_to_float64.go | 139 ++++
backend/core/models/migrationscripts/register.go | 1 +
backend/go.mod | 2 +-
backend/go.sum | 2 +
.../plugins/bamboo/tasks/deploy_build_convertor.go | 2 +-
.../plugins/bamboo/tasks/job_build_convertor.go | 2 +-
.../plugins/bamboo/tasks/plan_build_convertor.go | 2 +-
.../bitbucket/tasks/deployment_convertor.go | 4 +-
.../plugins/bitbucket/tasks/pipeline_convertor.go | 2 +-
.../bitbucket/tasks/pipeline_steps_convertor.go | 2 +-
backend/plugins/circleci/models/job.go | 2 +-
backend/plugins/circleci/models/workflow.go | 2 +-
backend/plugins/circleci/tasks/job_extractor.go | 2 +-
.../plugins/circleci/tasks/workflow_extractor.go | 2 +-
.../dora/tasks/deployment_commits_generator.go | 2 +-
backend/plugins/github/tasks/cicd_job_convertor.go | 2 +-
backend/plugins/github/tasks/cicd_run_convertor.go | 2 +-
.../plugins/github/tasks/deployment_convertor.go | 2 +-
.../snapshot_tables/cicd_deployment_commits.csv | 14 +-
.../e2e/snapshot_tables/cicd_deployments.csv | 14 +-
.../gitlab/e2e/snapshot_tables/cicd_pipelines.csv | 26 +-
.../gitlab/e2e/snapshot_tables/cicd_tasks.csv | 80 +-
.../snapshot_tables/cicd_tasks_no_prod_regex.csv | 80 +-
.../plugins/gitlab/tasks/deployment_convertor.go | 6 +-
backend/plugins/gitlab/tasks/job_convertor.go | 2 +-
backend/plugins/gitlab/tasks/pipeline_convertor.go | 2 +-
.../jenkins/e2e/snapshot_tables/cicd_pipelines.csv | 26 +-
.../jenkins/e2e/snapshot_tables/cicd_tasks.csv | 24 +-
.../snapshot_tables/cicd_tasks_after_stages.csv | 28 +-
backend/plugins/jenkins/e2e/stages_test.go | 4 -
.../plugins/jenkins/tasks/build_cicd_convertor.go | 15 +-
backend/plugins/jenkins/tasks/stage_convertor.go | 12 +-
backend/plugins/webhook/api/deployments.go | 5 +-
.../migrationscripts/20231116_add_api_keys.go | 163 ++++
.../models/migrationscripts/archived/api_keys.go} | 23 +-
.../webhook/models/migrationscripts/register.go | 1 +
backend/scripts/build-plugins.sh | 6 +
backend/scripts/build-server.sh | 2 +
backend/server/api/apikeys/apikeys.go | 9 +-
backend/server/api/project/project.go | 4 +-
config-ui/env.example | 1 +
config-ui/package.json | 1 +
config-ui/src/components/index.ts | 1 -
.../src/components/table/components/content.tsx | 95 ---
config-ui/src/components/table/components/index.ts | 21 -
.../src/components/table/components/loading.tsx | 27 -
.../src/components/table/components/no-data.tsx | 42 -
config-ui/src/components/table/hooks/index.ts | 19 -
.../components/table/hooks/use-row-selection.ts | 93 ---
config-ui/src/components/table/index.ts | 20 -
config-ui/src/components/table/styled.ts | 55 --
config-ui/src/components/table/table.tsx | 56 --
config-ui/src/components/table/types.ts | 27 -
config-ui/src/features/connections/slice.ts | 10 +
config-ui/src/main.tsx | 15 +-
.../pages/blueprint/connection-detail/index.tsx | 8 +-
.../pages/blueprint/detail/configuration-panel.tsx | 7 +-
config-ui/src/pages/blueprint/home/index.tsx | 32 +-
config-ui/src/pages/connection/detail/index.tsx | 16 +-
config-ui/src/pages/project/home/index.tsx | 23 +-
.../plugins/components/connection-list/index.tsx | 12 +-
.../components/scope-config-select/index.tsx | 7 +-
.../src/plugins/register/webhook/connection.tsx | 77 +-
config-ui/src/routes/api-keys/api-keys.tsx | 10 +-
config-ui/src/routes/layout/layout.tsx | 9 +-
config-ui/src/routes/layout/styled.ts | 2 -
config-ui/src/routes/pipeline/components/table.tsx | 112 +--
config-ui/src/vite-env.d.ts | 1 +
config-ui/yarn.lock | 876 ++++++++++++++++++++-
grafana/Dockerfile | 2 +-
78 files changed, 1559 insertions(+), 850 deletions(-)
rename backend/core/models/migrationscripts/{20220905_modfiy_cicd_pipeline.go => 20220905_modify_cicd_pipeline.go} (100%)
rename backend/core/models/migrationscripts/{20220908_modfiy_cicd_tasks.go => 20220908_modify_cicd_tasks.go} (100%)
rename backend/core/models/migrationscripts/{20221109_modfiy_commits_diffs.go => 20221109_modify_commits_diffs.go} (100%)
create mode 100644 backend/core/models/migrationscripts/20231115_change_duration_sec_to_float64.go
create mode 100644 backend/plugins/webhook/models/migrationscripts/20231116_add_api_keys.go
copy backend/{core/models/api_key.go => plugins/webhook/models/migrationscripts/archived/api_keys.go} (70%)
delete mode 100644 config-ui/src/components/table/components/content.tsx
delete mode 100644 config-ui/src/components/table/components/index.ts
delete mode 100644 config-ui/src/components/table/components/loading.tsx
delete mode 100644 config-ui/src/components/table/components/no-data.tsx
delete mode 100644 config-ui/src/components/table/hooks/index.ts
delete mode 100644 config-ui/src/components/table/hooks/use-row-selection.ts
delete mode 100644 config-ui/src/components/table/index.ts
delete mode 100644 config-ui/src/components/table/styled.ts
delete mode 100644 config-ui/src/components/table/table.tsx
delete mode 100644 config-ui/src/components/table/types.ts
(incubator-devlake) 02/02: feat: support skipCollectors in Advanced mode blueprint
Posted by ab...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
abeizn pushed a commit to branch feat#6088
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit 52e35e393ce3e0c3c77f118650fe04735669b6ef
Author: abeizn <zi...@merico.dev>
AuthorDate: Thu Nov 16 14:34:18 2023 +0800
feat: support skipCollectors in Advanced mode blueprint
---
backend/core/runner/run_task.go | 7 ++++++-
backend/server/services/blueprint.go | 22 +---------------------
backend/server/services/blueprint_makeplan_v200.go | 17 +++++++++++++++++
3 files changed, 24 insertions(+), 22 deletions(-)
diff --git a/backend/core/runner/run_task.go b/backend/core/runner/run_task.go
index a04e65fe6..6ee04eccb 100644
--- a/backend/core/runner/run_task.go
+++ b/backend/core/runner/run_task.go
@@ -20,6 +20,7 @@ package runner
import (
gocontext "context"
"fmt"
+ "strings"
"time"
"github.com/apache/incubator-devlake/core/context"
@@ -214,8 +215,12 @@ func RunPluginSubTasks(
}
}
- // make sure `Required` subtasks are always enabled
+ // 1. make sure `Collect` subtasks skip if `SkipCollectors` is true
+ // 2. make sure `Required` subtasks are always enabled
for _, subtaskMeta := range subtaskMetas {
+ if syncPolicy != nil && syncPolicy.SkipCollectors && strings.Contains(strings.ToLower(subtaskMeta.Name), "collect") {
+ subtasksFlag[subtaskMeta.Name] = false
+ }
if subtaskMeta.Required {
subtasksFlag[subtaskMeta.Name] = true
}
diff --git a/backend/server/services/blueprint.go b/backend/server/services/blueprint.go
index 7363f229a..34eef95d9 100644
--- a/backend/server/services/blueprint.go
+++ b/backend/server/services/blueprint.go
@@ -285,11 +285,7 @@ func createPipelineByBlueprint(blueprint *models.Blueprint, syncPolicy *models.S
return nil, err
}
} else {
- if syncPolicy != nil && syncPolicy.SkipCollectors {
- plan = removeCollectorTasks(blueprint.Plan)
- } else {
- plan = blueprint.Plan
- }
+ plan = blueprint.Plan
}
newPipeline := models.NewPipeline{}
@@ -392,19 +388,3 @@ func TriggerBlueprint(id uint64, syncPolicy *models.SyncPolicy) (*models.Pipelin
return createPipelineByBlueprint(blueprint, syncPolicy)
}
-
-func removeCollectorTasks(plan models.PipelinePlan) models.PipelinePlan {
- for j, stage := range plan {
- for k, task := range stage {
- newSubtasks := make([]string, 0, len(task.Subtasks))
- for _, subtask := range task.Subtasks {
- if !strings.Contains(strings.ToLower(subtask), "collect") {
- newSubtasks = append(newSubtasks, subtask)
- }
- }
- task.Subtasks = newSubtasks
- plan[j][k] = task
- }
- }
- return plan
-}
diff --git a/backend/server/services/blueprint_makeplan_v200.go b/backend/server/services/blueprint_makeplan_v200.go
index cc9a2d30a..045061e59 100644
--- a/backend/server/services/blueprint_makeplan_v200.go
+++ b/backend/server/services/blueprint_makeplan_v200.go
@@ -20,6 +20,7 @@ package services
import (
"encoding/json"
"fmt"
+ "strings"
"github.com/apache/incubator-devlake/core/errors"
coreModels "github.com/apache/incubator-devlake/core/models"
@@ -138,3 +139,19 @@ func GeneratePlanJsonV200(
)
return plan, err
}
+
+func removeCollectorTasks(plan coreModels.PipelinePlan) coreModels.PipelinePlan {
+ for j, stage := range plan {
+ for k, task := range stage {
+ newSubtasks := make([]string, 0, len(task.Subtasks))
+ for _, subtask := range task.Subtasks {
+ if !strings.Contains(strings.ToLower(subtask), "collect") {
+ newSubtasks = append(newSubtasks, subtask)
+ }
+ }
+ task.Subtasks = newSubtasks
+ plan[j][k] = task
+ }
+ }
+ return plan
+}
(incubator-devlake) 01/02: feat: add skip collector at advanced mode when subtask not nil
Posted by ab...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
abeizn pushed a commit to branch feat#6088
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit 5fcca4320f5b239ffe856730a1cd60c8ad265550
Author: abeizn <zi...@merico.dev>
AuthorDate: Tue Nov 14 16:40:05 2023 +0800
feat: add skip collector at advanced mode when subtask not nil
---
backend/server/services/blueprint.go | 22 ++++++++-
backend/server/services/blueprint_makeplan_v200.go | 14 +-----
backend/server/services/blueprint_test.go | 55 ++++++++++++++++++++++
3 files changed, 77 insertions(+), 14 deletions(-)
diff --git a/backend/server/services/blueprint.go b/backend/server/services/blueprint.go
index 34eef95d9..7363f229a 100644
--- a/backend/server/services/blueprint.go
+++ b/backend/server/services/blueprint.go
@@ -285,7 +285,11 @@ func createPipelineByBlueprint(blueprint *models.Blueprint, syncPolicy *models.S
return nil, err
}
} else {
- plan = blueprint.Plan
+ if syncPolicy != nil && syncPolicy.SkipCollectors {
+ plan = removeCollectorTasks(blueprint.Plan)
+ } else {
+ plan = blueprint.Plan
+ }
}
newPipeline := models.NewPipeline{}
@@ -388,3 +392,19 @@ func TriggerBlueprint(id uint64, syncPolicy *models.SyncPolicy) (*models.Pipelin
return createPipelineByBlueprint(blueprint, syncPolicy)
}
+
+func removeCollectorTasks(plan models.PipelinePlan) models.PipelinePlan {
+ for j, stage := range plan {
+ for k, task := range stage {
+ newSubtasks := make([]string, 0, len(task.Subtasks))
+ for _, subtask := range task.Subtasks {
+ if !strings.Contains(strings.ToLower(subtask), "collect") {
+ newSubtasks = append(newSubtasks, subtask)
+ }
+ }
+ task.Subtasks = newSubtasks
+ plan[j][k] = task
+ }
+ }
+ return plan
+}
diff --git a/backend/server/services/blueprint_makeplan_v200.go b/backend/server/services/blueprint_makeplan_v200.go
index 69e11bfc2..cc9a2d30a 100644
--- a/backend/server/services/blueprint_makeplan_v200.go
+++ b/backend/server/services/blueprint_makeplan_v200.go
@@ -20,7 +20,6 @@ package services
import (
"encoding/json"
"fmt"
- "strings"
"github.com/apache/incubator-devlake/core/errors"
coreModels "github.com/apache/incubator-devlake/core/models"
@@ -73,18 +72,7 @@ func GeneratePlanJsonV200(
// skip collectors
if skipCollectors {
for i, plan := range sourcePlans {
- for j, stage := range plan {
- for k, task := range stage {
- newSubtasks := make([]string, 0, len(task.Subtasks))
- for _, subtask := range task.Subtasks {
- if !strings.Contains(strings.ToLower(subtask), "collect") {
- newSubtasks = append(newSubtasks, subtask)
- }
- }
- task.Subtasks = newSubtasks
- sourcePlans[i][j][k] = task
- }
- }
+ sourcePlans[i] = removeCollectorTasks(plan)
}
// remove gitextractor plugin if it's not the only task
diff --git a/backend/server/services/blueprint_test.go b/backend/server/services/blueprint_test.go
index 3a4fc95ff..f0a607047 100644
--- a/backend/server/services/blueprint_test.go
+++ b/backend/server/services/blueprint_test.go
@@ -92,3 +92,58 @@ func TestParallelizePipelineTasks(t *testing.T) {
ParallelizePipelinePlans(plan1, plan2, plan3),
)
}
+
+func TestRemoveCollectorTasks(t *testing.T) {
+ plan1 := coreModels.PipelinePlan{
+ {
+ {
+ Plugin: "github",
+ Subtasks: []string{
+ "CollectApiPipelines",
+ "ExtractApiPipelines",
+ "collectApiPipelineDetails",
+ "extractApiPipelineDetails",
+ "collectApiJobs",
+ "extractApiJobs",
+ "collectAccounts",
+ "extractAccounts",
+ "ConvertAccounts",
+ "convertApiProject",
+ "convertPipelines",
+ "convertPipelineCommits",
+ "convertJobs",
+ },
+ },
+ },
+ {
+ {
+ Plugin: "starrocks",
+ Subtasks: []string{},
+ },
+ },
+ }
+ assert.Equal(t, coreModels.PipelinePlan{
+ {
+ {
+ Plugin: "github",
+ Subtasks: []string{
+ "ExtractApiPipelines",
+ "extractApiPipelineDetails",
+ "extractApiJobs",
+ "extractAccounts",
+ "ConvertAccounts",
+ "convertApiProject",
+ "convertPipelines",
+ "convertPipelineCommits",
+ "convertJobs",
+ },
+ },
+ },
+ {
+ {
+ Plugin: "starrocks",
+ Subtasks: []string{},
+ },
+ },
+ }, removeCollectorTasks(plan1))
+}