You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by kl...@apache.org on 2023/06/27 13:08:55 UTC
[incubator-devlake] branch main updated: fix: _raw_data_params are missing for some scope (#5584)
This is an automated email from the ASF dual-hosted git repository.
klesh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 7d6c931ec fix: _raw_data_params are missing for some scope (#5584)
7d6c931ec is described below
commit 7d6c931ec81fc0f00df5447644bbbc16c068293d
Author: Liang Zhang <li...@merico.dev>
AuthorDate: Tue Jun 27 21:08:49 2023 +0800
fix: _raw_data_params are missing for some scope (#5584)
---
backend/server/services/blueprint_makeplan_v200.go | 53 ++++++----------------
.../services/blueprint_makeplan_v200_test.go | 4 +-
2 files changed, 15 insertions(+), 42 deletions(-)
diff --git a/backend/server/services/blueprint_makeplan_v200.go b/backend/server/services/blueprint_makeplan_v200.go
index 6b84d11d8..e557ddb72 100644
--- a/backend/server/services/blueprint_makeplan_v200.go
+++ b/backend/server/services/blueprint_makeplan_v200.go
@@ -35,51 +35,26 @@ func GeneratePlanJsonV200(
metrics map[string]json.RawMessage,
skipCollectors bool,
) (plugin.PipelinePlan, errors.Error) {
- // generate plan and collect scopes
- plan, scopes, err := genPlanJsonV200(projectName, syncPolicy, sources, metrics, skipCollectors)
- if err != nil {
- return nil, err
- }
- // save scopes to database
- if len(scopes) > 0 {
- for _, scope := range scopes {
- err = db.CreateOrUpdate(scope)
- if err != nil {
- scopeInfo := fmt.Sprintf("[Id:%s][Name:%s][TableName:%s]", scope.ScopeId(), scope.ScopeName(), scope.TableName())
- return nil, errors.Default.Wrap(err, fmt.Sprintf("failed to create scopes:[%s]", scopeInfo))
- }
- }
- }
- return plan, err
-}
-
-func genPlanJsonV200(
- projectName string,
- syncPolicy plugin.BlueprintSyncPolicy,
- sources *models.BlueprintSettings,
- metrics map[string]json.RawMessage,
- skipCollectors bool,
-) (plugin.PipelinePlan, []plugin.Scope, errors.Error) {
connections := make([]*plugin.BlueprintConnectionV200, 0)
err := errors.Convert(json.Unmarshal(sources.Connections, &connections))
if err != nil {
- return nil, nil, err
+ return nil, err
}
// make plan for data-source plugins fist. generate plan for each
- // connections, then merge them into one legitimate plan and collect the
+ // connection, then merge them into one legitimate plan and collect the
// scopes produced by the data-source plugins
sourcePlans := make([]plugin.PipelinePlan, len(connections))
scopes := make([]plugin.Scope, 0, len(connections))
for i, connection := range connections {
if len(connection.Scopes) == 0 && connection.Plugin != `webhook` && connection.Plugin != `jenkins` {
// webhook needn't scopes
- // jenkins may upgrade from v100 and its' scope is empty
- return nil, nil, errors.Default.New(fmt.Sprintf("connections[%d].scopes is empty", i))
+ // jenkins may upgrade from v100 and its scope is empty
+ return nil, errors.Default.New(fmt.Sprintf("connections[%d].scopes is empty", i))
}
p, err := plugin.GetPlugin(connection.Plugin)
if err != nil {
- return nil, nil, err
+ return nil, err
}
if pluginBp, ok := p.(plugin.DataSourcePluginBlueprintV200); ok {
var pluginScopes []plugin.Scope
@@ -89,13 +64,13 @@ func genPlanJsonV200(
syncPolicy,
)
if err != nil {
- return nil, nil, err
+ return nil, err
}
// collect scopes for the project. a github repository may produce
// 2 scopes, 1 repo and 1 board
scopes = append(scopes, pluginScopes...)
} else {
- return nil, nil, errors.Default.New(
+ return nil, errors.Default.New(
fmt.Sprintf("plugin %s does not support DataSourcePluginBlueprintV200", connection.Plugin),
)
}
@@ -143,7 +118,7 @@ func genPlanJsonV200(
for metricPluginName, metricPluginOptJson := range metrics {
p, err := plugin.GetPlugin(metricPluginName)
if err != nil {
- return nil, nil, err
+ return nil, err
}
if pluginBp, ok := p.(plugin.MetricPluginBlueprintV200); ok {
// If we enable one metric plugin, even if it has nil option, we still process it
@@ -152,11 +127,11 @@ func genPlanJsonV200(
}
metricPlans[i], err = pluginBp.MakeMetricPluginPipelinePlanV200(projectName, metricPluginOptJson)
if err != nil {
- return nil, nil, err
+ return nil, err
}
- i += 1
+ i++
} else {
- return nil, nil, errors.Default.New(
+ return nil, errors.Default.New(
fmt.Sprintf("plugin %s does not support MetricPluginBlueprintV200", metricPluginName),
)
}
@@ -165,12 +140,12 @@ func genPlanJsonV200(
if projectName != "" {
p, err := plugin.GetPlugin("org")
if err != nil {
- return nil, nil, err
+ return nil, err
}
if pluginBp, ok := p.(plugin.ProjectMapper); ok {
planForProjectMapping, err = pluginBp.MapProject(projectName, scopes)
if err != nil {
- return nil, nil, err
+ return nil, err
}
}
}
@@ -179,5 +154,5 @@ func genPlanJsonV200(
ParallelizePipelinePlans(sourcePlans...),
ParallelizePipelinePlans(metricPlans...),
)
- return plan, scopes, err
+ return plan, err
}
diff --git a/backend/server/services/blueprint_makeplan_v200_test.go b/backend/server/services/blueprint_makeplan_v200_test.go
index 743c67742..a0fc4e6c1 100644
--- a/backend/server/services/blueprint_makeplan_v200_test.go
+++ b/backend/server/services/blueprint_makeplan_v200_test.go
@@ -83,7 +83,6 @@ func TestMakePlanV200(t *testing.T) {
expectedPlan = append(expectedPlan, orgPlan...)
expectedPlan = append(expectedPlan, githubOutputPlan...)
expectedPlan = append(expectedPlan, doraOutputPlan...)
- expectedScopes := append(make([]plugin.Scope, 0), githubOutputScopes...)
// plugin registration
plugin.RegisterPlugin(githubName, github)
@@ -102,9 +101,8 @@ func TestMakePlanV200(t *testing.T) {
doraName: nil,
}
- plan, scopes, err := genPlanJsonV200(projectName, syncPolicy, sources, metrics, false)
+ plan, err := GeneratePlanJsonV200(projectName, syncPolicy, sources, metrics, false)
assert.Nil(t, err)
assert.Equal(t, expectedPlan, plan)
- assert.Equal(t, expectedScopes, scopes)
}