You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by kl...@apache.org on 2023/06/27 13:08:55 UTC

[incubator-devlake] branch main updated: fix: _raw_data_params are missing for some scope (#5584)

This is an automated email from the ASF dual-hosted git repository.

klesh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git


The following commit(s) were added to refs/heads/main by this push:
     new 7d6c931ec fix: _raw_data_params are missing for some scope (#5584)
7d6c931ec is described below

commit 7d6c931ec81fc0f00df5447644bbbc16c068293d
Author: Liang Zhang <li...@merico.dev>
AuthorDate: Tue Jun 27 21:08:49 2023 +0800

    fix: _raw_data_params are missing for some scope (#5584)
---
 backend/server/services/blueprint_makeplan_v200.go | 53 ++++++----------------
 .../services/blueprint_makeplan_v200_test.go       |  4 +-
 2 files changed, 15 insertions(+), 42 deletions(-)

diff --git a/backend/server/services/blueprint_makeplan_v200.go b/backend/server/services/blueprint_makeplan_v200.go
index 6b84d11d8..e557ddb72 100644
--- a/backend/server/services/blueprint_makeplan_v200.go
+++ b/backend/server/services/blueprint_makeplan_v200.go
@@ -35,51 +35,26 @@ func GeneratePlanJsonV200(
 	metrics map[string]json.RawMessage,
 	skipCollectors bool,
 ) (plugin.PipelinePlan, errors.Error) {
-	// generate plan and collect scopes
-	plan, scopes, err := genPlanJsonV200(projectName, syncPolicy, sources, metrics, skipCollectors)
-	if err != nil {
-		return nil, err
-	}
-	// save scopes to database
-	if len(scopes) > 0 {
-		for _, scope := range scopes {
-			err = db.CreateOrUpdate(scope)
-			if err != nil {
-				scopeInfo := fmt.Sprintf("[Id:%s][Name:%s][TableName:%s]", scope.ScopeId(), scope.ScopeName(), scope.TableName())
-				return nil, errors.Default.Wrap(err, fmt.Sprintf("failed to create scopes:[%s]", scopeInfo))
-			}
-		}
-	}
-	return plan, err
-}
-
-func genPlanJsonV200(
-	projectName string,
-	syncPolicy plugin.BlueprintSyncPolicy,
-	sources *models.BlueprintSettings,
-	metrics map[string]json.RawMessage,
-	skipCollectors bool,
-) (plugin.PipelinePlan, []plugin.Scope, errors.Error) {
 	connections := make([]*plugin.BlueprintConnectionV200, 0)
 	err := errors.Convert(json.Unmarshal(sources.Connections, &connections))
 	if err != nil {
-		return nil, nil, err
+		return nil, err
 	}
 	// make plan for data-source plugins fist. generate plan for each
-	// connections, then merge them into one legitimate plan and collect the
+	// connection, then merge them into one legitimate plan and collect the
 	// scopes produced by the data-source plugins
 	sourcePlans := make([]plugin.PipelinePlan, len(connections))
 	scopes := make([]plugin.Scope, 0, len(connections))
 	for i, connection := range connections {
 		if len(connection.Scopes) == 0 && connection.Plugin != `webhook` && connection.Plugin != `jenkins` {
 			// webhook needn't scopes
-			// jenkins may upgrade from v100 and its' scope is empty
-			return nil, nil, errors.Default.New(fmt.Sprintf("connections[%d].scopes is empty", i))
+			// jenkins may upgrade from v100 and its scope is empty
+			return nil, errors.Default.New(fmt.Sprintf("connections[%d].scopes is empty", i))
 		}
 
 		p, err := plugin.GetPlugin(connection.Plugin)
 		if err != nil {
-			return nil, nil, err
+			return nil, err
 		}
 		if pluginBp, ok := p.(plugin.DataSourcePluginBlueprintV200); ok {
 			var pluginScopes []plugin.Scope
@@ -89,13 +64,13 @@ func genPlanJsonV200(
 				syncPolicy,
 			)
 			if err != nil {
-				return nil, nil, err
+				return nil, err
 			}
 			// collect scopes for the project. a github repository may produce
 			// 2 scopes, 1 repo and 1 board
 			scopes = append(scopes, pluginScopes...)
 		} else {
-			return nil, nil, errors.Default.New(
+			return nil, errors.Default.New(
 				fmt.Sprintf("plugin %s does not support DataSourcePluginBlueprintV200", connection.Plugin),
 			)
 		}
@@ -143,7 +118,7 @@ func genPlanJsonV200(
 	for metricPluginName, metricPluginOptJson := range metrics {
 		p, err := plugin.GetPlugin(metricPluginName)
 		if err != nil {
-			return nil, nil, err
+			return nil, err
 		}
 		if pluginBp, ok := p.(plugin.MetricPluginBlueprintV200); ok {
 			// If we enable one metric plugin, even if it has nil option, we still process it
@@ -152,11 +127,11 @@ func genPlanJsonV200(
 			}
 			metricPlans[i], err = pluginBp.MakeMetricPluginPipelinePlanV200(projectName, metricPluginOptJson)
 			if err != nil {
-				return nil, nil, err
+				return nil, err
 			}
-			i += 1
+			i++
 		} else {
-			return nil, nil, errors.Default.New(
+			return nil, errors.Default.New(
 				fmt.Sprintf("plugin %s does not support MetricPluginBlueprintV200", metricPluginName),
 			)
 		}
@@ -165,12 +140,12 @@ func genPlanJsonV200(
 	if projectName != "" {
 		p, err := plugin.GetPlugin("org")
 		if err != nil {
-			return nil, nil, err
+			return nil, err
 		}
 		if pluginBp, ok := p.(plugin.ProjectMapper); ok {
 			planForProjectMapping, err = pluginBp.MapProject(projectName, scopes)
 			if err != nil {
-				return nil, nil, err
+				return nil, err
 			}
 		}
 	}
@@ -179,5 +154,5 @@ func genPlanJsonV200(
 		ParallelizePipelinePlans(sourcePlans...),
 		ParallelizePipelinePlans(metricPlans...),
 	)
-	return plan, scopes, err
+	return plan, err
 }
diff --git a/backend/server/services/blueprint_makeplan_v200_test.go b/backend/server/services/blueprint_makeplan_v200_test.go
index 743c67742..a0fc4e6c1 100644
--- a/backend/server/services/blueprint_makeplan_v200_test.go
+++ b/backend/server/services/blueprint_makeplan_v200_test.go
@@ -83,7 +83,6 @@ func TestMakePlanV200(t *testing.T) {
 	expectedPlan = append(expectedPlan, orgPlan...)
 	expectedPlan = append(expectedPlan, githubOutputPlan...)
 	expectedPlan = append(expectedPlan, doraOutputPlan...)
-	expectedScopes := append(make([]plugin.Scope, 0), githubOutputScopes...)
 
 	// plugin registration
 	plugin.RegisterPlugin(githubName, github)
@@ -102,9 +101,8 @@ func TestMakePlanV200(t *testing.T) {
 		doraName: nil,
 	}
 
-	plan, scopes, err := genPlanJsonV200(projectName, syncPolicy, sources, metrics, false)
+	plan, err := GeneratePlanJsonV200(projectName, syncPolicy, sources, metrics, false)
 	assert.Nil(t, err)
 
 	assert.Equal(t, expectedPlan, plan)
-	assert.Equal(t, expectedScopes, scopes)
 }