You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by ab...@apache.org on 2023/09/04 07:06:59 UTC
[incubator-devlake] branch main updated: refactor: convert bp.settings into multiple tables (#6006)
This is an automated email from the ASF dual-hosted git repository.
abeizn pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new d75eb02da refactor: convert bp.settings into multiple tables (#6006)
d75eb02da is described below
commit d75eb02da4ea853860100fa2fd151dd013a8ee8f
Author: Klesh Wong <zh...@merico.dev>
AuthorDate: Mon Sep 4 15:06:55 2023 +0800
refactor: convert bp.settings into multiple tables (#6006)
---
backend/core/models/blueprint.go | 154 +++++--------------
.../20230829_normalize_bp_settings.go | 120 +++++++++++++++
.../models/migrationscripts/archived/blueprint.go | 21 +++
.../migrationscripts/archived/iso8601time.go | 171 +++++++++++++++++++++
backend/core/models/migrationscripts/register.go | 1 +
backend/core/models/pipeline.go | 68 +++++---
.../pipeline_test.go} | 5 +-
backend/core/models/task.go | 23 ++-
backend/core/plugin/plugin_blueprint.go | 29 +---
backend/core/plugin/plugin_pipeline.go | 49 ------
.../helpers/pluginhelper/api/connection_helper.go | 8 +-
.../pluginhelper/api/scope_generic_helper.go | 49 +-----
.../pluginhelper/services/blueprint_helper.go | 140 +++++++++--------
backend/plugins/bamboo/api/blueprint_V200_test.go | 10 +-
backend/plugins/bamboo/api/blueprint_v200.go | 29 ++--
backend/plugins/bamboo/impl/impl.go | 7 +-
.../plugins/bitbucket/api/blueprint_V200_test.go | 17 +-
backend/plugins/bitbucket/api/blueprint_v200.go | 32 ++--
backend/plugins/bitbucket/impl/impl.go | 6 +-
backend/plugins/dora/impl/impl.go | 5 +-
backend/plugins/dora/impl/impl_test.go | 10 +-
backend/plugins/github/api/blueprint_V200_test.go | 17 +-
backend/plugins/github/api/blueprint_v200.go | 44 ++++--
backend/plugins/github/impl/impl.go | 10 +-
backend/plugins/gitlab/api/blueprint_V200_test.go | 12 +-
backend/plugins/gitlab/api/blueprint_v200.go | 39 ++---
backend/plugins/gitlab/impl/impl.go | 10 +-
backend/plugins/jenkins/api/blueprint_v200.go | 30 ++--
backend/plugins/jenkins/api/blueprint_v200_test.go | 16 +-
backend/plugins/jenkins/impl/impl.go | 8 +-
backend/plugins/jira/api/blueprint_v200.go | 30 ++--
backend/plugins/jira/api/blueprint_v200_test.go | 15 +-
backend/plugins/jira/impl/impl.go | 7 +-
backend/plugins/org/impl/impl.go | 9 +-
backend/plugins/pagerduty/api/blueprint_v200.go | 27 ++--
backend/plugins/pagerduty/impl/impl.go | 8 +-
backend/plugins/sonarqube/api/blueprint_v200.go | 30 ++--
.../plugins/sonarqube/api/blueprint_v200_test.go | 15 +-
backend/plugins/sonarqube/impl/impl.go | 8 +-
backend/plugins/tapd/api/blueprint_v200.go | 30 ++--
backend/plugins/tapd/api/blueprint_v200_test.go | 15 +-
backend/plugins/tapd/impl/impl.go | 7 +-
backend/plugins/teambition/api/blueprint200.go | 30 ++--
backend/plugins/teambition/impl/impl.go | 7 +-
backend/plugins/trello/api/blueprint_v200.go | 29 ++--
backend/plugins/trello/impl/impl.go | 7 +-
backend/plugins/webhook/api/blueprint_v200.go | 3 +-
backend/plugins/webhook/impl/impl.go | 7 +-
backend/plugins/zentao/api/blueprint_V200_test.go | 20 +--
backend/plugins/zentao/api/blueprint_v200.go | 24 +--
backend/plugins/zentao/impl/impl.go | 7 +-
backend/server/services/blueprint.go | 82 +++-------
backend/server/services/blueprint_makeplan_v200.go | 30 ++--
.../services/blueprint_makeplan_v200_test.go | 28 ++--
backend/server/services/blueprint_test.go | 55 +------
backend/server/services/pipeline.go | 3 +-
backend/server/services/remote/models/models.go | 2 +-
.../services/remote/plugin/plugin_extensions.go | 13 +-
backend/test/e2e/manual/azuredevops/azure_test.go | 11 +-
backend/test/e2e/manual/gitlab/gitlab_test.go | 11 +-
.../test/e2e/manual/pagerduty/pagerduty_test.go | 11 +-
backend/test/e2e/remote/helper.go | 9 +-
backend/test/e2e/remote/python_plugin_test.go | 2 +-
backend/test/helper/api.go | 12 +-
backend/test/helper/models.go | 3 +-
65 files changed, 991 insertions(+), 756 deletions(-)
diff --git a/backend/core/models/blueprint.go b/backend/core/models/blueprint.go
index 4566d97aa..19d5efdf4 100644
--- a/backend/core/models/blueprint.go
+++ b/backend/core/models/blueprint.go
@@ -18,12 +18,9 @@ limitations under the License.
package models
import (
- "encoding/json"
"time"
- "github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models/common"
- "github.com/apache/incubator-devlake/core/plugin"
)
const (
@@ -33,133 +30,60 @@ const (
// @Description CronConfig
type Blueprint struct {
- Name string `json:"name" validate:"required"`
- ProjectName string `json:"projectName" gorm:"type:varchar(255)"`
- Mode string `json:"mode" gorm:"varchar(20)" validate:"required,oneof=NORMAL ADVANCED"`
- Plan plugin.PipelinePlan `json:"plan" gorm:"serializer:encdec"`
- Enable bool `json:"enable"`
- CronConfig string `json:"cronConfig" format:"* * * * *" example:"0 0 * * 1"`
- IsManual bool `json:"isManual"`
- SkipOnFail bool `json:"skipOnFail"`
- Labels []string `json:"labels" gorm:"-"`
- Settings json.RawMessage `json:"settings" swaggertype:"array,string" example:"please check api: /blueprints/<PLUGIN_NAME>/blueprint-setting" gorm:"serializer:encdec"`
+ Name string `json:"name" validate:"required"`
+ ProjectName string `json:"projectName" gorm:"type:varchar(255)"`
+ Mode string `json:"mode" gorm:"varchar(20)" validate:"required,oneof=NORMAL ADVANCED"`
+ Plan PipelinePlan `json:"plan" gorm:"serializer:encdec"`
+ Enable bool `json:"enable"`
+ CronConfig string `json:"cronConfig" format:"* * * * *" example:"0 0 * * 1"`
+ IsManual bool `json:"isManual"`
+ SkipOnFail bool `json:"skipOnFail"`
+ BeforePlan PipelinePlan `json:"beforePlan" gorm:"serializer:encdec"`
+ AfterPlan PipelinePlan `json:"afterPlan" gorm:"serializer:encdec"`
+ TimeAfter *time.Time `json:"timeAfter"`
+ Labels []string `json:"labels" gorm:"-"`
+ Connections []*BlueprintConnection `json:"connections" gorm:"-"`
common.Model `swaggerignore:"true"`
}
-type BlueprintSettings struct {
- Version string `json:"version" validate:"required,semver,oneof=1.0.0"`
- TimeAfter *time.Time `json:"timeAfter"`
- Connections json.RawMessage `json:"connections" validate:"required"`
- BeforePlan json.RawMessage `json:"before_plan"`
- AfterPlan json.RawMessage `json:"after_plan"`
-}
-
-// UnmarshalConnections unmarshals the connections on this BlueprintSettings reference
-func (bps *BlueprintSettings) UnmarshalConnections() ([]*plugin.BlueprintConnectionV200, errors.Error) {
- var connections []*plugin.BlueprintConnectionV200
- if bps.Connections == nil {
- return nil, nil
- }
- err := json.Unmarshal(bps.Connections, &connections)
- if err != nil {
- return nil, errors.Default.Wrap(err, `unmarshal connections fail`)
- }
- return connections, nil
-}
-
-// UpdateConnections updates the connections on this BlueprintSettings reference according to the updater function
-func (bps *BlueprintSettings) UpdateConnections(updater func(c *plugin.BlueprintConnectionV200) errors.Error) errors.Error {
- conns, err := bps.UnmarshalConnections()
- if err != nil {
- return err
- }
- for i, conn := range conns {
- err = updater(conn)
- if err != nil {
- return err
- }
- if conn.Scopes == nil {
- conn.Scopes = []*plugin.BlueprintScopeV200{} //UI expects this to be []
- }
- conns[i] = conn
- }
- bps.Connections, err = errors.Convert01(json.Marshal(&conns))
- if err != nil {
- return err
- }
- return nil
+func (Blueprint) TableName() string {
+ return "_devlake_blueprints"
}
-// UnmarshalSettings unmarshals the BlueprintSettings on the Blueprint
-func (bp *Blueprint) UnmarshalSettings() (BlueprintSettings, errors.Error) {
- var settings BlueprintSettings
- err := errors.Convert(json.Unmarshal(bp.Settings, &settings))
- if err != nil {
- return settings, errors.Default.Wrap(err, `unmarshal settings fail`)
- }
- return settings, nil
+type BlueprintLabel struct {
+ CreatedAt time.Time `json:"createdAt"`
+ UpdatedAt time.Time `json:"updatedAt"`
+ BlueprintId uint64 `json:"blueprint_id" gorm:"primaryKey"`
+ Name string `json:"name" gorm:"primaryKey;index"`
}
-// GetConnections Gets all the blueprint connections for this blueprint
-func (bp *Blueprint) GetConnections() ([]*plugin.BlueprintConnectionV200, errors.Error) {
- settings, err := bp.UnmarshalSettings()
- if err != nil {
- return nil, err
- }
- conns, err := settings.UnmarshalConnections()
- if err != nil {
- return nil, err
- }
- return conns, nil
+func (BlueprintLabel) TableName() string {
+ return "_devlake_blueprint_labels"
}
-// UpdateSettings updates the blueprint instance with this settings reference
-func (bp *Blueprint) UpdateSettings(settings *BlueprintSettings) errors.Error {
- if settings.Connections == nil {
- bp.Settings = nil
- } else {
- settingsRaw, err := errors.Convert01(json.Marshal(settings))
- if err != nil {
- return err
- }
- bp.Settings = settingsRaw
- }
- return nil
+type BlueprintConnection struct {
+ BlueprintId uint64 `json:"-" gorm:"primaryKey" validate:"required"`
+ PluginName string `json:"pluginName" gorm:"primaryKey;type:varchar(255)" validate:"required"`
+ ConnectionId uint64 `json:"connectionId" gorm:"primaryKey" validate:"required"`
+ Scopes []*BlueprintScope `json:"scopes" gorm:"-"`
}
-// GetScopes Gets all the scopes for a given connection for this blueprint. Returns an empty slice if none found.
-func (bp *Blueprint) GetScopes(connectionId uint64, pluginName string) ([]*plugin.BlueprintScopeV200, errors.Error) {
- conns, err := bp.GetConnections()
- if err != nil {
- return nil, err
- }
- visited := map[string]any{}
- var result []*plugin.BlueprintScopeV200
- for _, conn := range conns {
- if conn.ConnectionId != connectionId || conn.Plugin != pluginName {
- continue
- }
- for _, scope := range conn.Scopes {
- if _, ok := visited[scope.Id]; !ok {
- result = append(result, scope)
- visited[scope.Id] = true
- }
- }
- }
- return result, nil
+func (BlueprintConnection) TableName() string {
+ return "_devlake_blueprint_connections"
}
-func (Blueprint) TableName() string {
- return "_devlake_blueprints"
+type BlueprintScope struct {
+ BlueprintId uint64 `json:"-" gorm:"primaryKey" validate:"required"`
+ PluginName string `json:"-" gorm:"primaryKey;type:varchar(255)" validate:"required"`
+ ConnectionId uint64 `json:"-" gorm:"primaryKey" validate:"required"`
+ ScopeId string `json:"scopeId" gorm:"primaryKey;type:varchar(255)" validate:"required"`
}
-type DbBlueprintLabel struct {
- CreatedAt time.Time `json:"createdAt"`
- UpdatedAt time.Time `json:"updatedAt"`
- BlueprintId uint64 `json:"blueprint_id" gorm:"primaryKey"`
- Name string `json:"name" gorm:"primaryKey;index"`
+func (BlueprintScope) TableName() string {
+ return "_devlake_blueprint_scopes"
}
-func (DbBlueprintLabel) TableName() string {
- return "_devlake_blueprint_labels"
+type BlueprintSyncPolicy struct {
+ SkipOnFail bool `json:"skipOnFail"`
+ TimeAfter *time.Time `json:"timeAfter"`
}
diff --git a/backend/core/models/migrationscripts/20230829_normalize_bp_settings.go b/backend/core/models/migrationscripts/20230829_normalize_bp_settings.go
new file mode 100644
index 000000000..881dcb9bb
--- /dev/null
+++ b/backend/core/models/migrationscripts/20230829_normalize_bp_settings.go
@@ -0,0 +1,120 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+ "encoding/json"
+ "time"
+
+ "github.com/apache/incubator-devlake/core/context"
+ "github.com/apache/incubator-devlake/core/dal"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/models/migrationscripts/archived"
+ "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/helpers/migrationhelper"
+)
+
+var _ plugin.MigrationScript = (*normalizeBpSettings)(nil)
+
+type normalizeBpSettings struct{}
+
+type blueprint20230829 struct {
+ ID uint64 `gorm:"primaryKey"`
+ Settings string
+ BeforePlan json.RawMessage
+ AfterPlan json.RawMessage
+ TimeAfter *time.Time
+}
+
+func (blueprint20230829) TableName() string {
+ return "_devlake_blueprints"
+}
+
+type blueprintSettings20230829 struct {
+ BeforePlan json.RawMessage `json:"before_plan"`
+ AfterPlan json.RawMessage `json:"after_plan"`
+ TimeAfter *archived.Iso8601Time `json:"timeAfter"`
+ Connections []struct {
+ Plugin string `json:"plugin"`
+ ConnectionID uint64 `json:"connectionId"`
+ Scopes []struct {
+ ID string `json:"id"`
+ } `json:"scopes"`
+ } `json:"connections"`
+}
+
+func (script *normalizeBpSettings) Up(basicRes context.BasicRes) errors.Error {
+ err := migrationhelper.AutoMigrateTables(
+ basicRes,
+ new(blueprint20230829),
+ new(archived.BlueprintConnection),
+ new(archived.BlueprintScope),
+ )
+ if err != nil {
+ return err
+ }
+ encKey := basicRes.GetConfig(plugin.EncodeKeyEnvStr)
+ if encKey == "" {
+ return errors.BadInput.New("invalid encKey")
+ }
+ db := basicRes.GetDal()
+ bp := &blueprint20230829{}
+ cursor := errors.Must1(db.Cursor(dal.From("_devlake_blueprints")))
+ defer cursor.Close()
+
+ for cursor.Next() {
+ // load row
+ errors.Must(db.Fetch(cursor, bp))
+ // decrypt and unmarshal settings
+ settingsJson := errors.Must1(plugin.Decrypt(encKey, bp.Settings))
+ settings := &blueprintSettings20230829{}
+ errors.Must(json.Unmarshal([]byte(settingsJson), settings))
+ // update bp fields
+ bp.BeforePlan = settings.BeforePlan
+ bp.AfterPlan = settings.AfterPlan
+ bp.TimeAfter = archived.Iso8601TimeToTime(settings.TimeAfter)
+ errors.Must(db.Update(bp))
+ // create bp connections and scopes records
+ for _, conn := range settings.Connections {
+ errors.Must(db.CreateOrUpdate(&archived.BlueprintConnection{
+ BlueprintId: bp.ID,
+ PluginName: conn.Plugin,
+ ConnectionId: conn.ConnectionID,
+ }))
+ for _, scope := range conn.Scopes {
+ errors.Must(db.CreateOrUpdate(&archived.BlueprintScope{
+ BlueprintId: bp.ID,
+ PluginName: conn.Plugin,
+ ConnectionId: conn.ConnectionID,
+ ScopeId: scope.ID,
+ }))
+ }
+ }
+ }
+ // drop settings column
+ errors.Must(db.DropColumns("_devlake_blueprints", "settings"))
+ return nil
+}
+
+func (*normalizeBpSettings) Version() uint64 {
+ return 20230829145125
+}
+
+func (*normalizeBpSettings) Name() string {
+ return "normalize bp settings to multiple tables"
+}
diff --git a/backend/core/models/migrationscripts/archived/blueprint.go b/backend/core/models/migrationscripts/archived/blueprint.go
index 9bf4c68dd..9ba226f22 100644
--- a/backend/core/models/migrationscripts/archived/blueprint.go
+++ b/backend/core/models/migrationscripts/archived/blueprint.go
@@ -32,3 +32,24 @@ type Blueprint struct {
func (Blueprint) TableName() string {
return "_devlake_blueprints"
}
+
+type BlueprintConnection struct {
+ BlueprintId uint64 `gorm:"primaryKey"`
+ PluginName string `gorm:"primaryKey;type:varchar(255)"`
+ ConnectionId uint64 `gorm:"primaryKey"`
+}
+
+func (BlueprintConnection) TableName() string {
+ return "_devlake_blueprint_connections"
+}
+
+type BlueprintScope struct {
+ BlueprintId uint64 `gorm:"primaryKey"`
+ PluginName string `gorm:"primaryKey;type:varchar(255)"`
+ ConnectionId uint64 `gorm:"primaryKey"`
+ ScopeId string `gorm:"primaryKey;type:varchar(255)"`
+}
+
+func (BlueprintScope) TableName() string {
+ return "_devlake_blueprint_scopes"
+}
diff --git a/backend/core/models/migrationscripts/archived/iso8601time.go b/backend/core/models/migrationscripts/archived/iso8601time.go
new file mode 100644
index 000000000..84af885d0
--- /dev/null
+++ b/backend/core/models/migrationscripts/archived/iso8601time.go
@@ -0,0 +1,171 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package archived
+
+import (
+ "database/sql/driver"
+ "fmt"
+ "regexp"
+ "strings"
+ "time"
+)
+
+/*
+parse iso8601 datetime format for json.Unmarshal
+
+declare your field type as:
+
+type Foo struct {
+ Created Iso8601Time
+}
+
+foo := &Foo{}
+err := errors.ConvertError(json.Unmarshal("{\"created\": \"2021-02-19T01:53:35.340+0800\"}", foo))
+var time time.Time
+time = foo.Created.ToTime()
+*/
+
+// DateTimeFormatItem FIXME ...
+// TODO: move this to helper
+type DateTimeFormatItem struct {
+ Matcher *regexp.Regexp
+ Format string
+}
+
+// DateTimeFormats FIXME ...
+var DateTimeFormats []DateTimeFormatItem
+
+func init() {
+ DateTimeFormats = []DateTimeFormatItem{
+ {
+ Matcher: regexp.MustCompile(`[+-][\d]{4}$`),
+ Format: "2006-01-02T15:04:05-0700",
+ },
+ {
+ Matcher: regexp.MustCompile(`[\d]{3}[+-][\d]{2}:[\d]{2}$`),
+ Format: "2006-01-02T15:04:05.000-07:00",
+ },
+ {
+ Matcher: regexp.MustCompile(`[+-][\d]{2}:[\d]{2}$`),
+ Format: "2006-01-02T15:04:05-07:00",
+ },
+ {
+ Matcher: regexp.MustCompile(`[\d]{4}-[\d]{2}-[\d]{2} [\d]{2}:[\d]{2}:[\d]{2}$`),
+ Format: "2006-01-02 15:04:05",
+ },
+ {
+ Matcher: regexp.MustCompile(`[+-][\d]{2}-[\d]{2}$`),
+ Format: "2006-01-02",
+ },
+ }
+}
+
+// Iso8601Time is type time.Time
+type Iso8601Time struct {
+ Time time.Time
+ format string
+}
+
+func (jt *Iso8601Time) String() string {
+ format := jt.format
+ if format == "" {
+ format = DateTimeFormats[0].Format
+ }
+ return jt.Time.Format(format)
+}
+
+// MarshalJSON FIXME ...
+func (jt Iso8601Time) MarshalJSON() ([]byte, error) {
+ return []byte(fmt.Sprintf(`"%s"`, jt.String())), nil
+}
+
+// UnmarshalJSON FIXME ...
+func (jt *Iso8601Time) UnmarshalJSON(b []byte) error {
+ timeString := string(b)
+ if timeString == "null" {
+ return nil
+ }
+ if strings.Contains(timeString, "0000-00-00") {
+ return nil
+ }
+ timeString = strings.Trim(timeString, `"`)
+ t, err := ConvertStringToTime(timeString)
+ if err != nil {
+ return err
+ }
+ jt.Time = t
+ return nil
+}
+
+// ToTime FIXME ...
+func (jt *Iso8601Time) ToTime() time.Time {
+ return jt.Time
+}
+
+// ToNullableTime FIXME ...
+func (jt *Iso8601Time) ToNullableTime() *time.Time {
+ if jt == nil {
+ return nil
+ }
+ return &jt.Time
+}
+
+// ConvertStringToTime FIXME ...
+func ConvertStringToTime(timeString string) (t time.Time, err error) {
+ for _, formatItem := range DateTimeFormats {
+ if formatItem.Matcher.MatchString(timeString) {
+ return time.Parse(formatItem.Format, timeString)
+ }
+ }
+ return time.Parse(time.RFC3339, timeString)
+}
+
+// Iso8601TimeToTime FIXME ...
+func Iso8601TimeToTime(iso8601Time *Iso8601Time) *time.Time {
+ if iso8601Time == nil {
+ return nil
+ }
+ t := iso8601Time.ToTime()
+ return &t
+}
+
+// Value FIXME ...
+func (jt *Iso8601Time) Value() (driver.Value, error) {
+ if jt == nil {
+ return nil, nil
+ }
+ var zeroTime time.Time
+ t := jt.Time
+ if t.UnixNano() == zeroTime.UnixNano() {
+ return nil, nil
+ }
+ return t, nil
+}
+
+// Scan FIXME ...
+func (jt *Iso8601Time) Scan(v interface{}) error {
+ value, ok := v.(time.Time)
+ if ok {
+ *jt = Iso8601Time{
+ Time: value,
+ format: time.RFC3339,
+ }
+ return nil
+ }
+ return fmt.Errorf("can not convert %v to timestamp", v)
+}
diff --git a/backend/core/models/migrationscripts/register.go b/backend/core/models/migrationscripts/register.go
index eee037c60..2f372318c 100644
--- a/backend/core/models/migrationscripts/register.go
+++ b/backend/core/models/migrationscripts/register.go
@@ -91,5 +91,6 @@ func All() []plugin.MigrationScript {
new(tasksUsesJSON),
new(modifyCicdPipelinesToText),
new(dropTapStateTable),
+ new(normalizeBpSettings),
}
}
diff --git a/backend/core/models/pipeline.go b/backend/core/models/pipeline.go
index bec361943..92824b8a1 100644
--- a/backend/core/models/pipeline.go
+++ b/backend/core/models/pipeline.go
@@ -21,34 +21,64 @@ import (
"time"
"github.com/apache/incubator-devlake/core/models/common"
- "github.com/apache/incubator-devlake/core/plugin"
)
+type GenericPipelineTask[T any] struct {
+ Plugin string `json:"plugin" binding:"required"`
+ Subtasks []string `json:"subtasks"`
+ Options T `json:"options"`
+}
+
+type GenericPipelineStage[T any] []*GenericPipelineTask[T]
+type GenericPipelinePlan[T any] []GenericPipelineStage[T]
+
+// PipelineTask represents a smallest unit of execution inside a PipelinePlan
+type PipelineTask GenericPipelineTask[map[string]interface{}]
+
+// PipelineStage consist of multiple PipelineTasks, they will be executed in parallel
+type PipelineStage []*PipelineTask
+
+// PipelinePlan consist of multiple PipelineStages, they will be executed in sequential order
+type PipelinePlan []PipelineStage
+
+// IsEmpty checks if a PipelinePlan is empty
+func (plan PipelinePlan) IsEmpty() bool {
+ if len(plan) == 0 {
+ return true
+ }
+ for _, stage := range plan {
+ if len(stage) > 0 {
+ return false
+ }
+ }
+ return true
+}
+
type Pipeline struct {
common.Model
- Name string `json:"name" gorm:"index"`
- BlueprintId uint64 `json:"blueprintId"`
- Plan plugin.PipelinePlan `json:"plan" gorm:"serializer:encdec"`
- TotalTasks int `json:"totalTasks"`
- FinishedTasks int `json:"finishedTasks"`
- BeganAt *time.Time `json:"beganAt"`
- FinishedAt *time.Time `json:"finishedAt" gorm:"index"`
- Status string `json:"status"`
- Message string `json:"message"`
- ErrorName string `json:"errorName"`
- SpentSeconds int `json:"spentSeconds"`
- Stage int `json:"stage"`
- Labels []string `json:"labels" gorm:"-"`
- SkipOnFail bool `json:"skipOnFail"`
+ Name string `json:"name" gorm:"index"`
+ BlueprintId uint64 `json:"blueprintId"`
+ Plan PipelinePlan `json:"plan" gorm:"serializer:encdec"`
+ TotalTasks int `json:"totalTasks"`
+ FinishedTasks int `json:"finishedTasks"`
+ BeganAt *time.Time `json:"beganAt"`
+ FinishedAt *time.Time `json:"finishedAt" gorm:"index"`
+ Status string `json:"status"`
+ Message string `json:"message"`
+ ErrorName string `json:"errorName"`
+ SpentSeconds int `json:"spentSeconds"`
+ Stage int `json:"stage"`
+ Labels []string `json:"labels" gorm:"-"`
+ SkipOnFail bool `json:"skipOnFail"`
}
// We use a 2D array because the request body must be an array of a set of tasks
// to be executed concurrently, while each set is to be executed sequentially.
type NewPipeline struct {
- Name string `json:"name"`
- Plan plugin.PipelinePlan `json:"plan" swaggertype:"array,string" example:"please check api /pipelines/<PLUGIN_NAME>/pipeline-plan"`
- Labels []string `json:"labels"`
- SkipOnFail bool `json:"skipOnFail"`
+ Name string `json:"name"`
+ Plan PipelinePlan `json:"plan" swaggertype:"array,string" example:"please check api /pipelines/<PLUGIN_NAME>/pipeline-plan"`
+ Labels []string `json:"labels"`
+ SkipOnFail bool `json:"skipOnFail"`
BlueprintId uint64
}
diff --git a/backend/core/plugin/plugin_blueprint_test.go b/backend/core/models/pipeline_test.go
similarity index 98%
rename from backend/core/plugin/plugin_blueprint_test.go
rename to backend/core/models/pipeline_test.go
index e19ebe86b..af5a4f1e0 100644
--- a/backend/core/plugin/plugin_blueprint_test.go
+++ b/backend/core/models/pipeline_test.go
@@ -15,11 +15,12 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
-package plugin
+package models
import (
- "github.com/stretchr/testify/assert"
"testing"
+
+ "github.com/stretchr/testify/assert"
)
func TestPipelinePlan_IsEmpty(t *testing.T) {
diff --git a/backend/core/models/task.go b/backend/core/models/task.go
index 336d11f39..6fba0f760 100644
--- a/backend/core/models/task.go
+++ b/backend/core/models/task.go
@@ -21,7 +21,6 @@ import (
"time"
"github.com/apache/incubator-devlake/core/models/common"
- plugin "github.com/apache/incubator-devlake/core/plugin"
)
const (
@@ -45,6 +44,15 @@ type TaskProgressDetail struct {
SubTaskNumber int `json:"subTaskNumber"`
}
+type NewTask struct {
+ // Plugin name
+ *PipelineTask
+ PipelineId uint64 `json:"-"`
+ PipelineRow int `json:"-"`
+ PipelineCol int `json:"-"`
+ IsRerun bool `json:"-"`
+}
+
type Task struct {
common.Model
Plugin string `json:"plugin" gorm:"index"`
@@ -65,13 +73,8 @@ type Task struct {
SpentSeconds int `json:"spentSeconds"`
}
-type NewTask struct {
- // Plugin name
- *plugin.PipelineTask
- PipelineId uint64 `json:"-"`
- PipelineRow int `json:"-"`
- PipelineCol int `json:"-"`
- IsRerun bool `json:"-"`
+func (Task) TableName() string {
+ return "_devlake_tasks"
}
type Subtask struct {
@@ -84,10 +87,6 @@ type Subtask struct {
SpentSeconds int64 `json:"spentSeconds"`
}
-func (Task) TableName() string {
- return "_devlake_tasks"
-}
-
func (Subtask) TableName() string {
return "_devlake_subtasks"
}
diff --git a/backend/core/plugin/plugin_blueprint.go b/backend/core/plugin/plugin_blueprint.go
index e86379f5b..6831f90f0 100644
--- a/backend/core/plugin/plugin_blueprint.go
+++ b/backend/core/plugin/plugin_blueprint.go
@@ -19,9 +19,9 @@ package plugin
import (
"encoding/json"
- "time"
"github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/models"
)
/*
@@ -71,23 +71,12 @@ step 3: framework should maintain the project_mapping table based on the []Scope
type DataSourcePluginBlueprintV200 interface {
MakeDataSourcePipelinePlanV200(
connectionId uint64,
- scopes []*BlueprintScopeV200,
- syncPolicy BlueprintSyncPolicy,
- ) (PipelinePlan, []Scope, errors.Error)
+ scopes []*models.BlueprintScope,
+ syncPolicy models.BlueprintSyncPolicy,
+ ) (models.PipelinePlan, []Scope, errors.Error)
}
// BlueprintConnectionV200 contains the pluginName/connectionId and related Scopes,
-type BlueprintConnectionV200 struct {
- Plugin string `json:"plugin" validate:"required"`
- ConnectionId uint64 `json:"connectionId" validate:"required"`
- Scopes []*BlueprintScopeV200 `json:"scopes" validate:"required"`
-}
-
-// BlueprintScopeV200 contains the `id` and `name` for a specific scope
-type BlueprintScopeV200 struct {
- Id string `json:"id"`
- Name string `json:"name"`
-}
// MetricPluginBlueprintV200 is similar to the DataSourcePluginBlueprintV200
// but for Metric Plugin, take dora as an example, it doens't have any scope,
@@ -96,12 +85,12 @@ type BlueprintScopeV200 struct {
// right Deployment keep in mind it would be called IFF the plugin was enabled
// for the project.
type MetricPluginBlueprintV200 interface {
- MakeMetricPluginPipelinePlanV200(projectName string, options json.RawMessage) (PipelinePlan, errors.Error)
+ MakeMetricPluginPipelinePlanV200(projectName string, options json.RawMessage) (models.PipelinePlan, errors.Error)
}
// ProjectMapper is implemented by the plugin org, which binding project and scopes
type ProjectMapper interface {
- MapProject(projectName string, scopes []Scope) (PipelinePlan, errors.Error)
+ MapProject(projectName string, scopes []Scope) (models.PipelinePlan, errors.Error)
}
// CompositeDataSourcePluginBlueprintV200 is for unit test
@@ -128,9 +117,3 @@ type CompositeProjectMapper interface {
PluginMeta
ProjectMapper
}
-
-type BlueprintSyncPolicy struct {
- Version string `json:"version" validate:"required,semver,oneof=1.0.0"`
- SkipOnFail bool `json:"skipOnFail"`
- TimeAfter *time.Time `json:"timeAfter"`
-}
diff --git a/backend/core/plugin/plugin_pipeline.go b/backend/core/plugin/plugin_pipeline.go
deleted file mode 100644
index 750a0b858..000000000
--- a/backend/core/plugin/plugin_pipeline.go
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package plugin
-
-type GenericPipelineTask[T any] struct {
- Plugin string `json:"plugin" binding:"required"`
- Subtasks []string `json:"subtasks"`
- Options T `json:"options"`
-}
-
-type GenericPipelineStage[T any] []*GenericPipelineTask[T]
-type GenericPipelinePlan[T any] []GenericPipelineStage[T]
-
-// PipelineTask represents a smallest unit of execution inside a PipelinePlan
-type PipelineTask GenericPipelineTask[map[string]interface{}]
-
-// PipelineStage consist of multiple PipelineTasks, they will be executed in parallel
-type PipelineStage []*PipelineTask
-
-// PipelinePlan consist of multiple PipelineStages, they will be executed in sequential order
-type PipelinePlan []PipelineStage
-
-// IsEmpty checks if a PipelinePlan is empty
-func (plan PipelinePlan) IsEmpty() bool {
- if len(plan) == 0 {
- return true
- }
- for _, stage := range plan {
- if len(stage) > 0 {
- return false
- }
- }
- return true
-}
diff --git a/backend/helpers/pluginhelper/api/connection_helper.go b/backend/helpers/pluginhelper/api/connection_helper.go
index 6ae72755d..af5b8885e 100644
--- a/backend/helpers/pluginhelper/api/connection_helper.go
+++ b/backend/helpers/pluginhelper/api/connection_helper.go
@@ -19,9 +19,10 @@ package api
import (
"fmt"
+ "strconv"
+
"github.com/apache/incubator-devlake/helpers/pluginhelper/services"
"github.com/apache/incubator-devlake/server/api/shared"
- "strconv"
"github.com/apache/incubator-devlake/core/context"
"github.com/apache/incubator-devlake/core/dal"
@@ -124,10 +125,7 @@ func (c *ConnectionApiHelper) List(connections interface{}) errors.Error {
// Delete connection.
func (c *ConnectionApiHelper) deleteConnection(connection interface{}) (*services.BlueprintProjectPairs, errors.Error) {
connectionId := reflectField(connection, "ID").Uint()
- referencingBps, err := c.bpManager.GetBlueprintsByConnection(c.pluginName, connectionId)
- if err != nil {
- return nil, err
- }
+ referencingBps := c.bpManager.GetBlueprintsByConnection(c.pluginName, connectionId)
if len(referencingBps) > 0 {
return services.NewBlueprintProjectPairs(referencingBps), errors.Conflict.New("Found one or more blueprint/project references to this connection")
}
diff --git a/backend/helpers/pluginhelper/api/scope_generic_helper.go b/backend/helpers/pluginhelper/api/scope_generic_helper.go
index 264ac1738..c77c6f727 100644
--- a/backend/helpers/pluginhelper/api/scope_generic_helper.go
+++ b/backend/helpers/pluginhelper/api/scope_generic_helper.go
@@ -248,26 +248,8 @@ func (gs *GenericScopeApiHelper[Conn, Scope, ScopeConfig]) GetScopes(input *plug
}
// return empty array rather than nil in case of no scopes
if len(apiScopes) > 0 && params.loadBlueprints {
- // fetch blueprints for all scopes in one call since all bps must be loaded to determine which ones are associated with the scopes
- // TODO: split bp.settings into separate tables and load only the ones needed
- var scopeIds []string
for _, apiScope := range apiScopes {
- // scopeId := fmt.Sprintf("%v", reflectField(apiScope.Scope, gs.reflectionParams.ScopeIdFieldName).Interface())
- scopeIds = append(scopeIds, apiScope.Scope.ScopeId())
- }
- blueprintMap := errors.Must1(gs.bpManager.GetBlueprintsByScopes(params.connectionId, params.plugin, scopeIds...))
- for _, apiScope := range apiScopes {
- if bps, ok := blueprintMap[apiScope.Scope.ScopeId()]; ok {
- apiScope.Blueprints = bps
- delete(blueprintMap, apiScope.Scope.ScopeId())
- }
- }
- if len(blueprintMap) > 0 {
- var danglingIds []string
- for bpId := range blueprintMap {
- danglingIds = append(danglingIds, bpId)
- }
- gs.log.Warn(nil, "The following dangling scopes were found: %v", danglingIds)
+ apiScope.Blueprints = gs.bpManager.GetBlueprintsByScopeId(params.connectionId, params.plugin, apiScope.Scope.ScopeId())
}
}
return &ScopeListRes[Scope, ScopeConfig]{
@@ -295,13 +277,7 @@ func (gs *GenericScopeApiHelper[Conn, Scope, ScopeConfig]) GetScope(input *plugi
}
scopeRes := apiScopes[0]
if params.loadBlueprints {
- blueprintMap, err := gs.bpManager.GetBlueprintsByScopes(params.connectionId, params.plugin, params.scopeId)
- if err != nil {
- return nil, errors.Default.Wrap(err, fmt.Sprintf("error getting blueprints for scope with scope ID %s", params.scopeId))
- }
- if len(blueprintMap) == 1 {
- scopeRes.Blueprints = blueprintMap[params.scopeId]
- }
+ scopeRes.Blueprints = gs.bpManager.GetBlueprintsByScopeId(params.connectionId, params.plugin, params.scopeId)
}
return scopeRes, nil
}
@@ -338,9 +314,12 @@ func (gs *GenericScopeApiHelper[Conn, Scope, ScopeConfig]) DeleteScope(input *pl
}
if !params.deleteDataOnly {
- if refs, err := gs.getScopeReferences(params.connectionId, params.scopeId); err != nil || refs != nil {
- if err != nil {
- return nil, err
+ blueprints := gs.bpManager.GetBlueprintsByScopeId(params.connectionId, params.plugin, params.scopeId)
+ if len(blueprints) > 0 {
+ refs = &serviceHelper.BlueprintProjectPairs{}
+ for _, bp := range blueprints {
+ refs.Blueprints = append(refs.Blueprints, bp.Name)
+ refs.Projects = append(refs.Projects, bp.ProjectName)
}
return refs, errors.Conflict.New("Found one or more references to this scope")
}
@@ -373,18 +352,6 @@ func (gs *GenericScopeApiHelper[Conn, Scope, ScopeConfig]) addScopeConfig(scopes
return apiScopes, nil
}
-func (gs *GenericScopeApiHelper[Conn, Scope, ScopeConfig]) getScopeReferences(connectionId uint64, scopeId string) (*serviceHelper.BlueprintProjectPairs, errors.Error) {
- blueprintMap, err := gs.bpManager.GetBlueprintsByScopes(connectionId, gs.plugin, scopeId)
- if err != nil {
- return nil, err
- }
- blueprints := blueprintMap[scopeId]
- if len(blueprints) == 0 {
- return nil, nil
- }
- return serviceHelper.NewBlueprintProjectPairs(blueprints), nil
-}
-
func (gs *GenericScopeApiHelper[Conn, Scope, ScopeConfig]) extractFromReqParam(input *plugin.ApiResourceInput, withScopeId bool) (*requestParams, errors.Error) {
connectionId, err := strconv.ParseUint(input.Params["connectionId"], 10, 64)
if err != nil {
diff --git a/backend/helpers/pluginhelper/services/blueprint_helper.go b/backend/helpers/pluginhelper/services/blueprint_helper.go
index 33b21180f..aae0535f7 100644
--- a/backend/helpers/pluginhelper/services/blueprint_helper.go
+++ b/backend/helpers/pluginhelper/services/blueprint_helper.go
@@ -72,14 +72,14 @@ func (b *BlueprintManager) SaveDbBlueprint(blueprint *models.Blueprint) errors.E
if err != nil {
return errors.Default.Wrap(err, "error creating DB blueprint")
}
- err = b.db.Delete(&models.DbBlueprintLabel{}, dal.Where(`blueprint_id = ?`, blueprint.ID))
+ err = b.db.Delete(&models.BlueprintLabel{}, dal.Where(`blueprint_id = ?`, blueprint.ID))
if err != nil {
return errors.Default.Wrap(err, "error delete DB blueprint's old labelModels")
}
if len(blueprint.Labels) > 0 {
- blueprintLabels := make([]*models.DbBlueprintLabel, 0)
+ blueprintLabels := make([]*models.BlueprintLabel, 0)
for i := range blueprint.Labels {
- blueprintLabels = append(blueprintLabels, &models.DbBlueprintLabel{
+ blueprintLabels = append(blueprintLabels, &models.BlueprintLabel{
BlueprintId: blueprint.ID,
Name: blueprint.Labels[i],
})
@@ -89,6 +89,20 @@ func (b *BlueprintManager) SaveDbBlueprint(blueprint *models.Blueprint) errors.E
return errors.Default.Wrap(err, "error creating DB blueprint's labelModels")
}
}
+ errors.Must(b.db.Delete(&models.BlueprintConnection{}, dal.Where("blueprint_id = ?", blueprint.ID)))
+ errors.Must(b.db.Delete(&models.BlueprintScope{}, dal.Where("blueprint_id = ?", blueprint.ID)))
+ for _, conn := range blueprint.Connections {
+ conn.BlueprintId = blueprint.ID
+ // validate
+ errors.Must(b.db.Create(conn))
+ for _, scope := range conn.Scopes {
+ scope.BlueprintId = blueprint.ID
+ scope.PluginName = conn.PluginName
+ scope.ConnectionId = conn.ConnectionId
+ // validate
+ errors.Must(b.db.Create(scope))
+ }
+ }
return nil
}
@@ -108,6 +122,7 @@ const (
func (b *BlueprintManager) GetDbBlueprints(query *GetBlueprintQuery) ([]*models.Blueprint, int64, errors.Error) {
// process query parameters
clauses := []dal.Clause{dal.From(&models.Blueprint{})}
+ // clauses := []dal.Clause{dal.From("_devlake_blueprints")}
if query.Enable != nil {
clauses = append(clauses, dal.Where("enable = ?", *query.Enable))
}
@@ -142,10 +157,8 @@ func (b *BlueprintManager) GetDbBlueprints(query *GetBlueprintQuery) ([]*models.
}
// count total records
- count, err := b.db.Count(clauses...)
- if err != nil {
- return nil, 0, err
- }
+ // var count int64
+ count := errors.Must1(b.db.Count(clauses...))
clauses = append(clauses, dal.Orderby("id DESC"))
// load paginated blueprints from database
if query.SkipRecords != 0 {
@@ -155,14 +168,11 @@ func (b *BlueprintManager) GetDbBlueprints(query *GetBlueprintQuery) ([]*models.
clauses = append(clauses, dal.Limit(query.PageSize))
}
dbBlueprints := make([]*models.Blueprint, 0)
- err = b.db.All(&dbBlueprints, clauses...)
- if err != nil {
- return nil, 0, errors.Default.Wrap(err, "error getting DB count of blueprints")
- }
+ errors.Must(b.db.All(&dbBlueprints, clauses...))
// load labels for blueprints
for _, dbBlueprint := range dbBlueprints {
- err = b.fillBlueprintDetail(dbBlueprint)
+ err := b.fillBlueprintDetail(dbBlueprint)
if err != nil {
return nil, 0, err
}
@@ -189,46 +199,38 @@ func (b *BlueprintManager) GetDbBlueprint(blueprintId uint64) (*models.Blueprint
}
// GetBlueprintsByScopes returns all blueprints that have these scopeIds and this connection Id
-func (b *BlueprintManager) GetBlueprintsByScopes(connectionId uint64, pluginName string, scopeIds ...string) (map[string][]*models.Blueprint, errors.Error) {
- bps, _, err := b.GetDbBlueprints(&GetBlueprintQuery{Mode: "NORMAL"})
- if err != nil {
- return nil, err
- }
- scopeMap := map[string][]*models.Blueprint{}
- for _, bp := range bps {
- scopes, err := bp.GetScopes(connectionId, pluginName)
- if err != nil {
- return nil, err
- }
- for _, scope := range scopes {
- if contains(scopeIds, scope.Id) {
- scopeMap[scope.Id] = append(scopeMap[scope.Id], bp)
- }
- }
- }
- return scopeMap, nil
+func (b *BlueprintManager) GetBlueprintsByScopeId(connectionId uint64, pluginName string, scopeId string) []*models.Blueprint {
+ blueprints := make([]*models.Blueprint, 0)
+ errors.Must(b.db.All(
+ &blueprints,
+ dal.From("_devlake_blueprints bp"),
+ dal.Join("JOIN _devlake_blueprint_scopes sc ON sc.blueprint_id = bp.id"),
+ dal.Where(
+ "mode = ? AND sc.connection_id = ? AND sc.plugin_name = ? AND sc.scope_id = ?",
+ "NORMAL",
+ connectionId,
+ pluginName,
+ scopeId,
+ ),
+ ))
+ return blueprints
}
// GetBlueprintsByScopes returns all blueprints that have these scopeIds and this connection Id
-func (b *BlueprintManager) GetBlueprintsByConnection(plugin string, connectionId uint64) ([]*models.Blueprint, errors.Error) {
- bps, _, err := b.GetDbBlueprints(&GetBlueprintQuery{})
- if err != nil {
- return nil, err
- }
- var filteredBps []*models.Blueprint
- for _, bp := range bps {
- connections, err := bp.GetConnections()
- if err != nil {
- return nil, err
- }
- for _, connection := range connections {
- if connection.ConnectionId == connectionId && connection.Plugin == plugin {
- filteredBps = append(filteredBps, bp)
- break
- }
- }
- }
- return filteredBps, nil
+func (b *BlueprintManager) GetBlueprintsByConnection(plugin string, connectionId uint64) []*models.Blueprint {
+ blueprints := make([]*models.Blueprint, 0)
+ errors.Must(b.db.All(
+ &blueprints,
+ dal.From("_devlake_blueprints bp"),
+ dal.Join("JOIN _devlake_blueprint_connections conn ON conn.blueprint_id = bp.id"),
+ dal.Where(
+ "mode = ? AND conn.connection_id = ? AND conn.plugin_name = ?",
+ "NORMAL",
+ connectionId,
+ plugin,
+ ),
+ ))
+ return blueprints
}
// GetDbBlueprintByProjectName returns the detail of a given projectName
@@ -260,7 +262,7 @@ func (b *BlueprintManager) DeleteBlueprint(id uint64) errors.Error {
}
}
}()
- err = tx.Delete(&models.DbBlueprintLabel{}, dal.Where("blueprint_id = ?", id))
+ err = tx.Delete(&models.BlueprintLabel{}, dal.Where("blueprint_id = ?", id))
if err != nil {
return err
}
@@ -272,22 +274,36 @@ func (b *BlueprintManager) DeleteBlueprint(id uint64) errors.Error {
if err != nil {
return err
}
+ errors.Must(tx.Delete(&models.BlueprintConnection{}, dal.Where("blueprint_id = ?", id)))
+ errors.Must(tx.Delete(&models.BlueprintScope{}, dal.Where("blueprint_id = ?", id)))
return tx.Commit()
}
func (b *BlueprintManager) fillBlueprintDetail(blueprint *models.Blueprint) errors.Error {
- err := b.db.Pluck("name", &blueprint.Labels, dal.From(&models.DbBlueprintLabel{}), dal.Where("blueprint_id = ?", blueprint.ID))
- if err != nil {
- return errors.Internal.Wrap(err, "error getting the blueprint labels from database")
+ errors.Must(
+ b.db.Pluck(
+ "name",
+ &blueprint.Labels, dal.From(&models.BlueprintLabel{}), dal.Where("blueprint_id = ?", blueprint.ID),
+ ),
+ )
+ errors.Must(
+ b.db.All(
+ &blueprint.Connections,
+ dal.Where("blueprint_id = ?", blueprint.ID),
+ ),
+ )
+ for _, conn := range blueprint.Connections {
+ errors.Must(
+ b.db.All(
+ &conn.Scopes,
+ dal.Where(
+ "blueprint_id = ? AND plugin_name = ? AND connection_id = ?",
+ blueprint.ID,
+ conn.PluginName,
+ conn.ConnectionId,
+ ),
+ ),
+ )
}
return nil
}
-
-func contains(list []string, target string) bool {
- for _, t := range list {
- if t == target {
- return true
- }
- }
- return false
-}
diff --git a/backend/plugins/bamboo/api/blueprint_V200_test.go b/backend/plugins/bamboo/api/blueprint_V200_test.go
index 113a5e738..ee70a2cae 100644
--- a/backend/plugins/bamboo/api/blueprint_V200_test.go
+++ b/backend/plugins/bamboo/api/blueprint_V200_test.go
@@ -20,6 +20,7 @@ package api
import (
"testing"
+ coreModels "github.com/apache/incubator-devlake/core/models"
mockdal "github.com/apache/incubator-devlake/mocks/core/dal"
mockplugin "github.com/apache/incubator-devlake/mocks/core/plugin"
@@ -47,11 +48,10 @@ func TestMakeDataSourcePipelinePlanV200(t *testing.T) {
const testScopeConfigName string = "bamboo scope config"
const testProxy string = ""
- syncPolicy := &plugin.BlueprintSyncPolicy{}
- bpScopes := []*plugin.BlueprintScopeV200{
+ syncPolicy := &coreModels.BlueprintSyncPolicy{}
+ bpScopes := []*coreModels.BlueprintScope{
{
- Id: testKey,
- Name: testName,
+ ScopeId: testKey,
},
}
@@ -100,7 +100,7 @@ func TestMakeDataSourcePipelinePlanV200(t *testing.T) {
tasks.ConvertPlansMeta,
}
- var expectPlans = plugin.PipelinePlan{
+ var expectPlans = coreModels.PipelinePlan{
{
{
Plugin: "bamboo",
diff --git a/backend/plugins/bamboo/api/blueprint_v200.go b/backend/plugins/bamboo/api/blueprint_v200.go
index 2c5429c31..811db19ba 100644
--- a/backend/plugins/bamboo/api/blueprint_v200.go
+++ b/backend/plugins/bamboo/api/blueprint_v200.go
@@ -20,6 +20,7 @@ package api
import (
"fmt"
+ coreModels "github.com/apache/incubator-devlake/core/models"
"github.com/apache/incubator-devlake/plugins/bamboo/models"
"github.com/apache/incubator-devlake/core/errors"
@@ -34,9 +35,9 @@ import (
func MakePipelinePlanV200(
subtaskMetas []plugin.SubTaskMeta,
connectionId uint64,
- scope []*plugin.BlueprintScopeV200,
- syncPolicy *plugin.BlueprintSyncPolicy,
-) (plugin.PipelinePlan, []plugin.Scope, errors.Error) {
+ scope []*coreModels.BlueprintScope,
+ syncPolicy *coreModels.BlueprintSyncPolicy,
+) (coreModels.PipelinePlan, []plugin.Scope, errors.Error) {
var err errors.Error
connection := new(models.BambooConnection)
err1 := connectionHelper.FirstById(connection, connectionId)
@@ -57,14 +58,14 @@ func MakePipelinePlanV200(
return pp, sc, nil
}
-func makeScopeV200(connectionId uint64, scopes []*plugin.BlueprintScopeV200) ([]plugin.Scope, errors.Error) {
+func makeScopeV200(connectionId uint64, scopes []*coreModels.BlueprintScope) ([]plugin.Scope, errors.Error) {
sc := make([]plugin.Scope, 0, len(scopes))
for _, scope := range scopes {
- id := didgen.NewDomainIdGenerator(&models.BambooPlan{}).Generate(connectionId, scope.Id)
+ id := didgen.NewDomainIdGenerator(&models.BambooPlan{}).Generate(connectionId, scope.ScopeId)
// get project from db
- project, scopeConfig, err := scopeHelper.DbHelper().GetScopeAndConfig(connectionId, scope.Id)
+ project, scopeConfig, err := scopeHelper.DbHelper().GetScopeAndConfig(connectionId, scope.ScopeId)
if err != nil {
return nil, err
}
@@ -82,15 +83,15 @@ func makeScopeV200(connectionId uint64, scopes []*plugin.BlueprintScopeV200) ([]
func makePipelinePlanV200(
subtaskMetas []plugin.SubTaskMeta,
- scopes []*plugin.BlueprintScopeV200,
- connection *models.BambooConnection, syncPolicy *plugin.BlueprintSyncPolicy,
-) (plugin.PipelinePlan, errors.Error) {
- plans := make(plugin.PipelinePlan, 0, len(scopes))
+ scopes []*coreModels.BlueprintScope,
+ connection *models.BambooConnection, syncPolicy *coreModels.BlueprintSyncPolicy,
+) (coreModels.PipelinePlan, errors.Error) {
+ plans := make(coreModels.PipelinePlan, 0, len(scopes))
for _, scope := range scopes {
- var stage plugin.PipelineStage
+ var stage coreModels.PipelineStage
var err errors.Error
// get project
- _, scopeConfig, err := scopeHelper.DbHelper().GetScopeAndConfig(connection.ID, scope.Id)
+ _, scopeConfig, err := scopeHelper.DbHelper().GetScopeAndConfig(connection.ID, scope.ScopeId)
if err != nil {
return nil, err
}
@@ -98,7 +99,7 @@ func makePipelinePlanV200(
// bamboo main part
options := make(map[string]interface{})
options["connectionId"] = connection.ID
- options["planKey"] = scope.Id
+ options["planKey"] = scope.ScopeId
options["scopeConfigId"] = scopeConfig.ID
// construct subtasks
@@ -107,7 +108,7 @@ func makePipelinePlanV200(
return nil, err
}
- stage = append(stage, &plugin.PipelineTask{
+ stage = append(stage, &coreModels.PipelineTask{
Plugin: "bamboo",
Subtasks: subtasks,
Options: options,
diff --git a/backend/plugins/bamboo/impl/impl.go b/backend/plugins/bamboo/impl/impl.go
index e60335ac2..cd31c2bbb 100644
--- a/backend/plugins/bamboo/impl/impl.go
+++ b/backend/plugins/bamboo/impl/impl.go
@@ -23,6 +23,7 @@ import (
"github.com/apache/incubator-devlake/core/context"
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
+ coreModels "github.com/apache/incubator-devlake/core/models"
"github.com/apache/incubator-devlake/core/models/domainlayer/devops"
"github.com/apache/incubator-devlake/core/plugin"
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
@@ -64,7 +65,11 @@ func (p Bamboo) ScopeConfig() dal.Tabler {
return &models.BambooScopeConfig{}
}
-func (p Bamboo) MakeDataSourcePipelinePlanV200(connectionId uint64, scopes []*plugin.BlueprintScopeV200, syncPolicy plugin.BlueprintSyncPolicy) (plugin.PipelinePlan, []plugin.Scope, errors.Error) {
+func (p Bamboo) MakeDataSourcePipelinePlanV200(
+ connectionId uint64,
+ scopes []*coreModels.BlueprintScope,
+ syncPolicy coreModels.BlueprintSyncPolicy,
+) (coreModels.PipelinePlan, []plugin.Scope, errors.Error) {
return api.MakePipelinePlanV200(p.SubTaskMetas(), connectionId, scopes, &syncPolicy)
}
diff --git a/backend/plugins/bitbucket/api/blueprint_V200_test.go b/backend/plugins/bitbucket/api/blueprint_V200_test.go
index 4a2efb735..5de59d64b 100644
--- a/backend/plugins/bitbucket/api/blueprint_V200_test.go
+++ b/backend/plugins/bitbucket/api/blueprint_V200_test.go
@@ -20,6 +20,7 @@ package api
import (
"testing"
+ coreModels "github.com/apache/incubator-devlake/core/models"
"github.com/apache/incubator-devlake/core/models/common"
"github.com/apache/incubator-devlake/core/models/domainlayer"
"github.com/apache/incubator-devlake/core/models/domainlayer/code"
@@ -62,21 +63,21 @@ func TestMakeDataSourcePipelinePlanV200(t *testing.T) {
// Refresh Global Variables and set the sql mock
mockBasicRes(t)
- bs := &plugin.BlueprintScopeV200{
- Id: "1",
+ bs := &coreModels.BlueprintScope{
+ ScopeId: "1",
}
- bpScopes := make([]*plugin.BlueprintScopeV200, 0)
+ bpScopes := make([]*coreModels.BlueprintScope, 0)
bpScopes = append(bpScopes, bs)
- syncPolicy := &plugin.BlueprintSyncPolicy{}
+ syncPolicy := &coreModels.BlueprintSyncPolicy{}
- plan := make(plugin.PipelinePlan, len(bpScopes))
+ plan := make(coreModels.PipelinePlan, len(bpScopes))
plan, err = makeDataSourcePipelinePlanV200(nil, plan, bpScopes, connection, syncPolicy)
assert.Nil(t, err)
scopes, err := makeScopesV200(bpScopes, connection)
assert.Nil(t, err)
- expectPlan := plugin.PipelinePlan{
- plugin.PipelineStage{
+ expectPlan := coreModels.PipelinePlan{
+ coreModels.PipelineStage{
{
Plugin: "bitbucket",
Subtasks: []string{},
@@ -95,7 +96,7 @@ func TestMakeDataSourcePipelinePlanV200(t *testing.T) {
},
},
},
- plugin.PipelineStage{
+ coreModels.PipelineStage{
{
Plugin: "refdiff",
Options: map[string]interface{}{
diff --git a/backend/plugins/bitbucket/api/blueprint_v200.go b/backend/plugins/bitbucket/api/blueprint_v200.go
index d9e6ca54d..cfa165b99 100644
--- a/backend/plugins/bitbucket/api/blueprint_v200.go
+++ b/backend/plugins/bitbucket/api/blueprint_v200.go
@@ -22,6 +22,7 @@ import (
"time"
"github.com/apache/incubator-devlake/core/errors"
+ coreModels "github.com/apache/incubator-devlake/core/models"
"github.com/apache/incubator-devlake/core/models/domainlayer"
"github.com/apache/incubator-devlake/core/models/domainlayer/code"
"github.com/apache/incubator-devlake/core/models/domainlayer/devops"
@@ -34,7 +35,12 @@ import (
"github.com/apache/incubator-devlake/plugins/bitbucket/tasks"
)
-func MakeDataSourcePipelinePlanV200(subtaskMetas []plugin.SubTaskMeta, connectionId uint64, bpScopes []*plugin.BlueprintScopeV200, syncPolicy *plugin.BlueprintSyncPolicy) (plugin.PipelinePlan, []plugin.Scope, errors.Error) {
+func MakeDataSourcePipelinePlanV200(
+ subtaskMetas []plugin.SubTaskMeta,
+ connectionId uint64,
+ bpScopes []*coreModels.BlueprintScope,
+ syncPolicy *coreModels.BlueprintSyncPolicy,
+) (coreModels.PipelinePlan, []plugin.Scope, errors.Error) {
// get the connection info for url
connection := &models.BitbucketConnection{}
err := connectionHelper.FirstById(connection, connectionId)
@@ -42,7 +48,7 @@ func MakeDataSourcePipelinePlanV200(subtaskMetas []plugin.SubTaskMeta, connectio
return nil, nil, err
}
- plan := make(plugin.PipelinePlan, len(bpScopes))
+ plan := make(coreModels.PipelinePlan, len(bpScopes))
plan, err = makeDataSourcePipelinePlanV200(subtaskMetas, plan, bpScopes, connection, syncPolicy)
if err != nil {
return nil, nil, err
@@ -57,18 +63,18 @@ func MakeDataSourcePipelinePlanV200(subtaskMetas []plugin.SubTaskMeta, connectio
func makeDataSourcePipelinePlanV200(
subtaskMetas []plugin.SubTaskMeta,
- plan plugin.PipelinePlan,
- bpScopes []*plugin.BlueprintScopeV200,
+ plan coreModels.PipelinePlan,
+ bpScopes []*coreModels.BlueprintScope,
connection *models.BitbucketConnection,
- syncPolicy *plugin.BlueprintSyncPolicy,
-) (plugin.PipelinePlan, errors.Error) {
+ syncPolicy *coreModels.BlueprintSyncPolicy,
+) (coreModels.PipelinePlan, errors.Error) {
for i, bpScope := range bpScopes {
stage := plan[i]
if stage == nil {
- stage = plugin.PipelineStage{}
+ stage = coreModels.PipelineStage{}
}
// get repo and scope config from db
- repo, scopeConfig, err := scopeHelper.DbHelper().GetScopeAndConfig(connection.ID, bpScope.Id)
+ repo, scopeConfig, err := scopeHelper.DbHelper().GetScopeAndConfig(connection.ID, bpScope.ScopeId)
if err != nil {
return nil, err
}
@@ -81,7 +87,7 @@ func makeDataSourcePipelinePlanV200(
}
refdiffOp := scopeConfig.Refdiff
refdiffOp["repoId"] = didgen.NewDomainIdGenerator(&models.BitbucketRepo{}).Generate(connection.ID, repo.BitbucketId)
- plan[j] = plugin.PipelineStage{
+ plan[j] = coreModels.PipelineStage{
{
Plugin: "refdiff",
Options: refdiffOp,
@@ -107,7 +113,7 @@ func makeDataSourcePipelinePlanV200(
if err != nil {
return nil, err
}
- stage = append(stage, &plugin.PipelineTask{
+ stage = append(stage, &coreModels.PipelineTask{
Plugin: "bitbucket",
Subtasks: subtasks,
Options: options,
@@ -123,7 +129,7 @@ func makeDataSourcePipelinePlanV200(
return nil, err
}
cloneUrl.User = url.UserPassword(connection.Username, connection.Password)
- stage = append(stage, &plugin.PipelineTask{
+ stage = append(stage, &coreModels.PipelineTask{
Plugin: "gitextractor",
Options: map[string]interface{}{
"url": cloneUrl.String(),
@@ -139,10 +145,10 @@ func makeDataSourcePipelinePlanV200(
return plan, nil
}
-func makeScopesV200(bpScopes []*plugin.BlueprintScopeV200, connection *models.BitbucketConnection) ([]plugin.Scope, errors.Error) {
+func makeScopesV200(bpScopes []*coreModels.BlueprintScope, connection *models.BitbucketConnection) ([]plugin.Scope, errors.Error) {
scopes := make([]plugin.Scope, 0)
for _, bpScope := range bpScopes {
- repo, scopeConfig, err := scopeHelper.DbHelper().GetScopeAndConfig(connection.ID, bpScope.Id)
+ repo, scopeConfig, err := scopeHelper.DbHelper().GetScopeAndConfig(connection.ID, bpScope.ScopeId)
if err != nil {
return nil, err
}
diff --git a/backend/plugins/bitbucket/impl/impl.go b/backend/plugins/bitbucket/impl/impl.go
index 812e37b8b..ca5ef0836 100644
--- a/backend/plugins/bitbucket/impl/impl.go
+++ b/backend/plugins/bitbucket/impl/impl.go
@@ -24,6 +24,7 @@ import (
"github.com/apache/incubator-devlake/core/context"
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
+ coreModels "github.com/apache/incubator-devlake/core/models"
"github.com/apache/incubator-devlake/core/models/domainlayer/devops"
"github.com/apache/incubator-devlake/core/plugin"
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
@@ -198,7 +199,10 @@ func (p Bitbucket) MigrationScripts() []plugin.MigrationScript {
return migrationscripts.All()
}
-func (p Bitbucket) MakeDataSourcePipelinePlanV200(connectionId uint64, scopes []*plugin.BlueprintScopeV200, syncPolicy plugin.BlueprintSyncPolicy) (pp plugin.PipelinePlan, sc []plugin.Scope, err errors.Error) {
+func (p Bitbucket) MakeDataSourcePipelinePlanV200(
+ connectionId uint64,
+ scopes []*coreModels.BlueprintScope,
+ syncPolicy coreModels.BlueprintSyncPolicy) (pp coreModels.PipelinePlan, sc []plugin.Scope, err errors.Error) {
return api.MakeDataSourcePipelinePlanV200(p.SubTaskMetas(), connectionId, scopes, &syncPolicy)
}
diff --git a/backend/plugins/dora/impl/impl.go b/backend/plugins/dora/impl/impl.go
index 7f76b2bcc..c456f13a4 100644
--- a/backend/plugins/dora/impl/impl.go
+++ b/backend/plugins/dora/impl/impl.go
@@ -22,6 +22,7 @@ import (
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
+ coreModels "github.com/apache/incubator-devlake/core/models"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/plugins/dora/models/migrationscripts"
"github.com/apache/incubator-devlake/plugins/dora/tasks"
@@ -115,13 +116,13 @@ func (p Dora) MigrationScripts() []plugin.MigrationScript {
return migrationscripts.All()
}
-func (p Dora) MakeMetricPluginPipelinePlanV200(projectName string, options json.RawMessage) (plugin.PipelinePlan, errors.Error) {
+func (p Dora) MakeMetricPluginPipelinePlanV200(projectName string, options json.RawMessage) (coreModels.PipelinePlan, errors.Error) {
op := &tasks.DoraOptions{}
err := json.Unmarshal(options, op)
if err != nil {
return nil, errors.Default.WrapRaw(err)
}
- plan := plugin.PipelinePlan{
+ plan := coreModels.PipelinePlan{
{
{
Plugin: "dora",
diff --git a/backend/plugins/dora/impl/impl_test.go b/backend/plugins/dora/impl/impl_test.go
index c34679d1f..587cc517e 100644
--- a/backend/plugins/dora/impl/impl_test.go
+++ b/backend/plugins/dora/impl/impl_test.go
@@ -21,7 +21,7 @@ import (
"encoding/json"
"testing"
- "github.com/apache/incubator-devlake/core/plugin"
+ coreModels "github.com/apache/incubator-devlake/core/models"
"github.com/stretchr/testify/assert"
)
@@ -36,8 +36,8 @@ func TestMakeMetricPluginPipelinePlanV200(t *testing.T) {
optionJson, err := json.Marshal(option)
assert.Nil(t, err)
plan, err := dora.MakeMetricPluginPipelinePlanV200(projectName, optionJson)
- doraOutputPlan := plugin.PipelinePlan{
- plugin.PipelineStage{
+ doraOutputPlan := coreModels.PipelinePlan{
+ coreModels.PipelineStage{
{
Plugin: "dora",
Subtasks: []string{
@@ -47,14 +47,14 @@ func TestMakeMetricPluginPipelinePlanV200(t *testing.T) {
Options: map[string]interface{}{"projectName": projectName},
},
},
- plugin.PipelineStage{
+ coreModels.PipelineStage{
{
Plugin: "refdiff",
Subtasks: []string{"calculateDeploymentCommitsDiff"},
Options: map[string]interface{}{"projectName": projectName},
},
},
- plugin.PipelineStage{
+ coreModels.PipelineStage{
{
Plugin: "dora",
Subtasks: []string{
diff --git a/backend/plugins/github/api/blueprint_V200_test.go b/backend/plugins/github/api/blueprint_V200_test.go
index 4f1542304..dd1712ec6 100644
--- a/backend/plugins/github/api/blueprint_V200_test.go
+++ b/backend/plugins/github/api/blueprint_V200_test.go
@@ -20,6 +20,7 @@ package api
import (
"testing"
+ coreModels "github.com/apache/incubator-devlake/core/models"
"github.com/apache/incubator-devlake/core/models/common"
"github.com/apache/incubator-devlake/core/models/domainlayer"
"github.com/apache/incubator-devlake/core/models/domainlayer/code"
@@ -63,21 +64,21 @@ func TestMakeDataSourcePipelinePlanV200(t *testing.T) {
// Refresh Global Variables and set the sql mock
mockBasicRes(t)
- bs := &plugin.BlueprintScopeV200{
- Id: "1",
+ bs := &coreModels.BlueprintScope{
+ ScopeId: "1",
}
- bpScopes := make([]*plugin.BlueprintScopeV200, 0)
+ bpScopes := make([]*coreModels.BlueprintScope, 0)
bpScopes = append(bpScopes, bs)
- syncPolicy := &plugin.BlueprintSyncPolicy{}
+ syncPolicy := &coreModels.BlueprintSyncPolicy{}
- plan := make(plugin.PipelinePlan, len(bpScopes))
+ plan := make(coreModels.PipelinePlan, len(bpScopes))
plan, err = makeDataSourcePipelinePlanV200(nil, plan, bpScopes, connection, syncPolicy)
assert.Nil(t, err)
scopes, err := makeScopesV200(bpScopes, connection)
assert.Nil(t, err)
- expectPlan := plugin.PipelinePlan{
- plugin.PipelineStage{
+ expectPlan := coreModels.PipelinePlan{
+ coreModels.PipelineStage{
{
Plugin: "github",
Subtasks: []string{},
@@ -97,7 +98,7 @@ func TestMakeDataSourcePipelinePlanV200(t *testing.T) {
},
},
},
- plugin.PipelineStage{
+ coreModels.PipelineStage{
{
Plugin: "refdiff",
Options: map[string]interface{}{
diff --git a/backend/plugins/github/api/blueprint_v200.go b/backend/plugins/github/api/blueprint_v200.go
index 19de30132..eaa62a8c4 100644
--- a/backend/plugins/github/api/blueprint_v200.go
+++ b/backend/plugins/github/api/blueprint_v200.go
@@ -29,6 +29,7 @@ import (
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
+ coreModels "github.com/apache/incubator-devlake/core/models"
"github.com/apache/incubator-devlake/core/models/domainlayer"
"github.com/apache/incubator-devlake/core/models/domainlayer/code"
"github.com/apache/incubator-devlake/core/models/domainlayer/devops"
@@ -42,7 +43,12 @@ import (
"github.com/apache/incubator-devlake/plugins/github/tasks"
)
-func MakeDataSourcePipelinePlanV200(subtaskMetas []plugin.SubTaskMeta, connectionId uint64, bpScopes []*plugin.BlueprintScopeV200, syncPolicy *plugin.BlueprintSyncPolicy) (plugin.PipelinePlan, []plugin.Scope, errors.Error) {
+func MakeDataSourcePipelinePlanV200(
+ subtaskMetas []plugin.SubTaskMeta,
+ connectionId uint64,
+ bpScopes []*coreModels.BlueprintScope,
+ syncPolicy *coreModels.BlueprintSyncPolicy,
+) (coreModels.PipelinePlan, []plugin.Scope, errors.Error) {
// get the connection info for url
connection := &models.GithubConnection{}
err := connectionHelper.FirstById(connection, connectionId)
@@ -57,7 +63,7 @@ func MakeDataSourcePipelinePlanV200(subtaskMetas []plugin.SubTaskMeta, connectio
return nil, nil, err
}
- plan := make(plugin.PipelinePlan, len(bpScopes))
+ plan := make(coreModels.PipelinePlan, len(bpScopes))
plan, err = makeDataSourcePipelinePlanV200(subtaskMetas, plan, bpScopes, connection, syncPolicy)
if err != nil {
return nil, nil, err
@@ -72,18 +78,18 @@ func MakeDataSourcePipelinePlanV200(subtaskMetas []plugin.SubTaskMeta, connectio
func makeDataSourcePipelinePlanV200(
subtaskMetas []plugin.SubTaskMeta,
- plan plugin.PipelinePlan,
- bpScopes []*plugin.BlueprintScopeV200,
+ plan coreModels.PipelinePlan,
+ bpScopes []*coreModels.BlueprintScope,
connection *models.GithubConnection,
- syncPolicy *plugin.BlueprintSyncPolicy,
-) (plugin.PipelinePlan, errors.Error) {
+ syncPolicy *coreModels.BlueprintSyncPolicy,
+) (coreModels.PipelinePlan, errors.Error) {
for i, bpScope := range bpScopes {
stage := plan[i]
if stage == nil {
- stage = plugin.PipelineStage{}
+ stage = coreModels.PipelineStage{}
}
// get repo and scope config from db
- githubRepo, scopeConfig, err := scopeHelper.DbHelper().GetScopeAndConfig(connection.ID, bpScope.Id)
+ githubRepo, scopeConfig, err := scopeHelper.DbHelper().GetScopeAndConfig(connection.ID, bpScope.ScopeId)
if err != nil {
return nil, err
}
@@ -96,7 +102,7 @@ func makeDataSourcePipelinePlanV200(
}
refdiffOp := scopeConfig.Refdiff
refdiffOp["repoId"] = didgen.NewDomainIdGenerator(&models.GithubRepo{}).Generate(connection.ID, githubRepo.GithubId)
- plan[j] = plugin.PipelineStage{
+ plan[j] = coreModels.PipelineStage{
{
Plugin: "refdiff",
Options: refdiffOp,
@@ -131,7 +137,7 @@ func makeDataSourcePipelinePlanV200(
}
token := strings.Split(connection.Token, ",")[0]
cloneUrl.User = url.UserPassword("git", token)
- stage = append(stage, &plugin.PipelineTask{
+ stage = append(stage, &coreModels.PipelineTask{
Plugin: "gitextractor",
Options: map[string]interface{}{
"url": cloneUrl.String(),
@@ -147,14 +153,14 @@ func makeDataSourcePipelinePlanV200(
return plan, nil
}
-func makeScopesV200(bpScopes []*plugin.BlueprintScopeV200, connection *models.GithubConnection) ([]plugin.Scope, errors.Error) {
+func makeScopesV200(bpScopes []*coreModels.BlueprintScope, connection *models.GithubConnection) ([]plugin.Scope, errors.Error) {
scopes := make([]plugin.Scope, 0)
for _, bpScope := range bpScopes {
githubRepo := &models.GithubRepo{}
// get repo from db
- err := basicRes.GetDal().First(githubRepo, dal.Where(`connection_id = ? AND github_id = ?`, connection.ID, bpScope.Id))
+ err := basicRes.GetDal().First(githubRepo, dal.Where(`connection_id = ? AND github_id = ?`, connection.ID, bpScope.ScopeId))
if err != nil {
- return nil, errors.Default.Wrap(err, fmt.Sprintf("fail to find repo%s", bpScope.Id))
+ return nil, errors.Default.Wrap(err, fmt.Sprintf("fail to find repo%s", bpScope.ScopeId))
}
scopeConfig := &models.GithubScopeConfig{}
@@ -204,7 +210,13 @@ func makeScopesV200(bpScopes []*plugin.BlueprintScopeV200, connection *models.Gi
return scopes, nil
}
-func addGithub(subtaskMetas []plugin.SubTaskMeta, connection *models.GithubConnection, entities []string, stage plugin.PipelineStage, options map[string]interface{}) (plugin.PipelineStage, errors.Error) {
+func addGithub(
+ subtaskMetas []plugin.SubTaskMeta,
+ connection *models.GithubConnection,
+ entities []string,
+ stage coreModels.PipelineStage,
+ options map[string]interface{},
+) (coreModels.PipelineStage, errors.Error) {
// construct github(graphql) task
if connection.EnableGraphql {
// FIXME this need fix when 2 plugins merged
@@ -217,7 +229,7 @@ func addGithub(subtaskMetas []plugin.SubTaskMeta, connection *models.GithubConne
if err != nil {
return nil, err
}
- stage = append(stage, &plugin.PipelineTask{
+ stage = append(stage, &coreModels.PipelineTask{
Plugin: "github_graphql",
Subtasks: subtasks,
Options: options,
@@ -230,7 +242,7 @@ func addGithub(subtaskMetas []plugin.SubTaskMeta, connection *models.GithubConne
if err != nil {
return nil, err
}
- stage = append(stage, &plugin.PipelineTask{
+ stage = append(stage, &coreModels.PipelineTask{
Plugin: "github",
Subtasks: subtasks,
Options: options,
diff --git a/backend/plugins/github/impl/impl.go b/backend/plugins/github/impl/impl.go
index 0151d81de..97cbe9364 100644
--- a/backend/plugins/github/impl/impl.go
+++ b/backend/plugins/github/impl/impl.go
@@ -19,12 +19,14 @@ package impl
import (
"fmt"
- "github.com/apache/incubator-devlake/helpers/pluginhelper/subtaskmeta/sorter"
"time"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/subtaskmeta/sorter"
+
"github.com/apache/incubator-devlake/core/context"
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
+ coreModels "github.com/apache/incubator-devlake/core/models"
"github.com/apache/incubator-devlake/core/models/domainlayer/devops"
"github.com/apache/incubator-devlake/core/plugin"
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
@@ -221,7 +223,11 @@ func (p Github) ApiResources() map[string]map[string]plugin.ApiResourceHandler {
}
}
-func (p Github) MakeDataSourcePipelinePlanV200(connectionId uint64, scopes []*plugin.BlueprintScopeV200, syncPolicy plugin.BlueprintSyncPolicy) (pp plugin.PipelinePlan, sc []plugin.Scope, err errors.Error) {
+func (p Github) MakeDataSourcePipelinePlanV200(
+ connectionId uint64,
+ scopes []*coreModels.BlueprintScope,
+ syncPolicy coreModels.BlueprintSyncPolicy,
+) (pp coreModels.PipelinePlan, sc []plugin.Scope, err errors.Error) {
return api.MakeDataSourcePipelinePlanV200(p.SubTaskMetas(), connectionId, scopes, &syncPolicy)
}
diff --git a/backend/plugins/gitlab/api/blueprint_V200_test.go b/backend/plugins/gitlab/api/blueprint_V200_test.go
index 9241095a6..da2825aab 100644
--- a/backend/plugins/gitlab/api/blueprint_V200_test.go
+++ b/backend/plugins/gitlab/api/blueprint_V200_test.go
@@ -22,6 +22,7 @@ import (
"testing"
"time"
+ coreModels "github.com/apache/incubator-devlake/core/models"
mockdal "github.com/apache/incubator-devlake/mocks/core/dal"
mockplugin "github.com/apache/incubator-devlake/mocks/core/plugin"
@@ -51,11 +52,10 @@ func TestMakeDataSourcePipelinePlanV200(t *testing.T) {
const testScopeConfigName string = "gitlab scope config"
const testProxy string = ""
- syncPolicy := &plugin.BlueprintSyncPolicy{}
- bpScopes := []*plugin.BlueprintScopeV200{
+ syncPolicy := &coreModels.BlueprintSyncPolicy{}
+ bpScopes := []*coreModels.BlueprintScope{
{
- Id: strconv.Itoa(testID),
- Name: testName,
+ ScopeId: strconv.Itoa(testID),
},
}
@@ -118,7 +118,7 @@ func TestMakeDataSourcePipelinePlanV200(t *testing.T) {
tasks.ExtractApiPipelinesMeta,
}
- var expectPlans = plugin.PipelinePlan{
+ var expectPlans = coreModels.PipelinePlan{
{
{
Plugin: "gitlab",
@@ -175,7 +175,7 @@ func TestMakeDataSourcePipelinePlanV200(t *testing.T) {
var err errors.Error
- // register gitlab plugin for NewDomainIdGenerator
+ // register gitlab coreModels for NewDomainIdGenerator
mockMeta := mockplugin.NewPluginMeta(t)
mockMeta.On("RootPkgPath").Return("github.com/apache/incubator-devlake/plugins/gitlab")
mockMeta.On("Name").Return("dummy").Maybe()
diff --git a/backend/plugins/gitlab/api/blueprint_v200.go b/backend/plugins/gitlab/api/blueprint_v200.go
index c7339689a..f4513951b 100644
--- a/backend/plugins/gitlab/api/blueprint_v200.go
+++ b/backend/plugins/gitlab/api/blueprint_v200.go
@@ -31,6 +31,7 @@ import (
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/utils"
+ coreModels "github.com/apache/incubator-devlake/core/models"
"github.com/apache/incubator-devlake/core/models/domainlayer/code"
"github.com/apache/incubator-devlake/core/models/domainlayer/devops"
"github.com/apache/incubator-devlake/core/models/domainlayer/didgen"
@@ -44,9 +45,9 @@ import (
func MakePipelinePlanV200(
subtaskMetas []plugin.SubTaskMeta,
connectionId uint64,
- scope []*plugin.BlueprintScopeV200,
- syncPolicy *plugin.BlueprintSyncPolicy,
-) (plugin.PipelinePlan, []plugin.Scope, errors.Error) {
+ scope []*coreModels.BlueprintScope,
+ syncPolicy *coreModels.BlueprintSyncPolicy,
+) (coreModels.PipelinePlan, []plugin.Scope, errors.Error) {
var err errors.Error
connection := new(models.GitlabConnection)
err1 := connectionHelper.FirstById(connection, connectionId)
@@ -67,18 +68,18 @@ func MakePipelinePlanV200(
return pp, sc, nil
}
-func makeScopeV200(connectionId uint64, scopes []*plugin.BlueprintScopeV200) ([]plugin.Scope, errors.Error) {
+func makeScopeV200(connectionId uint64, scopes []*coreModels.BlueprintScope) ([]plugin.Scope, errors.Error) {
sc := make([]plugin.Scope, 0, 3*len(scopes))
for _, scope := range scopes {
- intScopeId, err1 := strconv.Atoi(scope.Id)
+ intScopeId, err1 := strconv.Atoi(scope.ScopeId)
if err1 != nil {
- return nil, errors.Default.Wrap(err1, fmt.Sprintf("Failed to strconv.Atoi for scope.Id [%s]", scope.Id))
+ return nil, errors.Default.Wrap(err1, fmt.Sprintf("Failed to strconv.Atoi for scope.Id [%s]", scope.ScopeId))
}
id := didgen.NewDomainIdGenerator(&models.GitlabProject{}).Generate(connectionId, intScopeId)
// get repo from db
- gitlabProject, scopeConfig, err := scopeHelper.DbHelper().GetScopeAndConfig(connectionId, scope.Id)
+ gitlabProject, scopeConfig, err := scopeHelper.DbHelper().GetScopeAndConfig(connectionId, scope.ScopeId)
if err != nil {
return nil, err
}
@@ -112,23 +113,23 @@ func makeScopeV200(connectionId uint64, scopes []*plugin.BlueprintScopeV200) ([]
func makePipelinePlanV200(
subtaskMetas []plugin.SubTaskMeta,
- scopes []*plugin.BlueprintScopeV200,
- connection *models.GitlabConnection, syncPolicy *plugin.BlueprintSyncPolicy,
-) (plugin.PipelinePlan, errors.Error) {
- plans := make(plugin.PipelinePlan, 0, 3*len(scopes))
+ scopes []*coreModels.BlueprintScope,
+ connection *models.GitlabConnection, syncPolicy *coreModels.BlueprintSyncPolicy,
+) (coreModels.PipelinePlan, errors.Error) {
+ plans := make(coreModels.PipelinePlan, 0, 3*len(scopes))
for _, scope := range scopes {
- var stage plugin.PipelineStage
+ var stage coreModels.PipelineStage
var err errors.Error
// get repo
- gitlabProject, scopeConfig, err := scopeHelper.DbHelper().GetScopeAndConfig(connection.ID, scope.Id)
+ gitlabProject, scopeConfig, err := scopeHelper.DbHelper().GetScopeAndConfig(connection.ID, scope.ScopeId)
if err != nil {
return nil, err
}
// get int scopeId
- intScopeId, err1 := strconv.Atoi(scope.Id)
+ intScopeId, err1 := strconv.Atoi(scope.ScopeId)
if err != nil {
- return nil, errors.Default.Wrap(err1, fmt.Sprintf("Failed to strconv.Atoi for scope.Id [%s]", scope.Id))
+ return nil, errors.Default.Wrap(err1, fmt.Sprintf("Failed to strconv.Atoi for scope.Id [%s]", scope.ScopeId))
}
// gitlab main part
@@ -146,7 +147,7 @@ func makePipelinePlanV200(
return nil, err
}
- stage = append(stage, &plugin.PipelineTask{
+ stage = append(stage, &coreModels.PipelineTask{
Plugin: "gitlab",
Subtasks: subtasks,
Options: options,
@@ -159,7 +160,7 @@ func makePipelinePlanV200(
return nil, err
}
cloneUrl.User = url.UserPassword("git", connection.Token)
- stage = append(stage, &plugin.PipelineTask{
+ stage = append(stage, &coreModels.PipelineTask{
Plugin: "gitextractor",
Options: map[string]interface{}{
"url": cloneUrl.String(),
@@ -174,11 +175,11 @@ func makePipelinePlanV200(
// refdiff part
if scopeConfig.Refdiff != nil {
- task := &plugin.PipelineTask{
+ task := &coreModels.PipelineTask{
Plugin: "refdiff",
Options: scopeConfig.Refdiff,
}
- plans = append(plans, plugin.PipelineStage{task})
+ plans = append(plans, coreModels.PipelineStage{task})
}
}
return plans, nil
diff --git a/backend/plugins/gitlab/impl/impl.go b/backend/plugins/gitlab/impl/impl.go
index 7053f7b19..0a9008fa4 100644
--- a/backend/plugins/gitlab/impl/impl.go
+++ b/backend/plugins/gitlab/impl/impl.go
@@ -19,12 +19,14 @@ package impl
import (
"fmt"
- "github.com/apache/incubator-devlake/helpers/pluginhelper/subtaskmeta/sorter"
"time"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/subtaskmeta/sorter"
+
"github.com/apache/incubator-devlake/core/context"
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
+ coreModels "github.com/apache/incubator-devlake/core/models"
"github.com/apache/incubator-devlake/core/models/domainlayer/devops"
"github.com/apache/incubator-devlake/core/plugin"
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
@@ -72,7 +74,11 @@ func (p Gitlab) ScopeConfig() dal.Tabler {
return &models.GitlabScopeConfig{}
}
-func (p Gitlab) MakeDataSourcePipelinePlanV200(connectionId uint64, scopes []*plugin.BlueprintScopeV200, syncPolicy plugin.BlueprintSyncPolicy) (plugin.PipelinePlan, []plugin.Scope, errors.Error) {
+func (p Gitlab) MakeDataSourcePipelinePlanV200(
+ connectionId uint64,
+ scopes []*coreModels.BlueprintScope,
+ syncPolicy coreModels.BlueprintSyncPolicy,
+) (coreModels.PipelinePlan, []plugin.Scope, errors.Error) {
return api.MakePipelinePlanV200(p.SubTaskMetas(), connectionId, scopes, &syncPolicy)
}
diff --git a/backend/plugins/jenkins/api/blueprint_v200.go b/backend/plugins/jenkins/api/blueprint_v200.go
index 82ac7ad64..00ba31fd1 100644
--- a/backend/plugins/jenkins/api/blueprint_v200.go
+++ b/backend/plugins/jenkins/api/blueprint_v200.go
@@ -21,6 +21,7 @@ import (
"time"
"github.com/apache/incubator-devlake/core/errors"
+ coreModels "github.com/apache/incubator-devlake/core/models"
"github.com/apache/incubator-devlake/core/models/domainlayer"
"github.com/apache/incubator-devlake/core/models/domainlayer/devops"
"github.com/apache/incubator-devlake/core/models/domainlayer/didgen"
@@ -30,8 +31,13 @@ import (
"github.com/apache/incubator-devlake/plugins/jenkins/models"
)
-func MakeDataSourcePipelinePlanV200(subtaskMetas []plugin.SubTaskMeta, connectionId uint64, bpScopes []*plugin.BlueprintScopeV200, syncPolicy *plugin.BlueprintSyncPolicy) (plugin.PipelinePlan, []plugin.Scope, errors.Error) {
- plan := make(plugin.PipelinePlan, len(bpScopes))
+func MakeDataSourcePipelinePlanV200(
+ subtaskMetas []plugin.SubTaskMeta,
+ connectionId uint64,
+ bpScopes []*coreModels.BlueprintScope,
+ syncPolicy *coreModels.BlueprintSyncPolicy,
+) (coreModels.PipelinePlan, []plugin.Scope, errors.Error) {
+ plan := make(coreModels.PipelinePlan, len(bpScopes))
plan, err := makeDataSourcePipelinePlanV200(subtaskMetas, plan, bpScopes, connectionId, syncPolicy)
if err != nil {
return nil, nil, err
@@ -46,20 +52,20 @@ func MakeDataSourcePipelinePlanV200(subtaskMetas []plugin.SubTaskMeta, connectio
func makeDataSourcePipelinePlanV200(
subtaskMetas []plugin.SubTaskMeta,
- plan plugin.PipelinePlan,
- bpScopes []*plugin.BlueprintScopeV200,
+ plan coreModels.PipelinePlan,
+ bpScopes []*coreModels.BlueprintScope,
connectionId uint64,
- syncPolicy *plugin.BlueprintSyncPolicy,
-) (plugin.PipelinePlan, errors.Error) {
+ syncPolicy *coreModels.BlueprintSyncPolicy,
+) (coreModels.PipelinePlan, errors.Error) {
for i, bpScope := range bpScopes {
stage := plan[i]
if stage == nil {
- stage = plugin.PipelineStage{}
+ stage = coreModels.PipelineStage{}
}
// construct task options for Jenkins
options := make(map[string]interface{})
- options["scopeId"] = bpScope.Id
+ options["scopeId"] = bpScope.ScopeId
options["connectionId"] = connectionId
if syncPolicy.TimeAfter != nil {
@@ -67,7 +73,7 @@ func makeDataSourcePipelinePlanV200(
}
// get scope config from db
- _, scopeConfig, err := scopeHelper.DbHelper().GetScopeAndConfig(connectionId, bpScope.Id)
+ _, scopeConfig, err := scopeHelper.DbHelper().GetScopeAndConfig(connectionId, bpScope.ScopeId)
if err != nil {
return nil, err
}
@@ -76,7 +82,7 @@ func makeDataSourcePipelinePlanV200(
if err != nil {
return nil, err
}
- stage = append(stage, &plugin.PipelineTask{
+ stage = append(stage, &coreModels.PipelineTask{
Plugin: "jenkins",
Subtasks: subtasks,
Options: options,
@@ -88,13 +94,13 @@ func makeDataSourcePipelinePlanV200(
}
func makeScopesV200(
- bpScopes []*plugin.BlueprintScopeV200,
+ bpScopes []*coreModels.BlueprintScope,
connectionId uint64,
) ([]plugin.Scope, errors.Error) {
scopes := make([]plugin.Scope, 0)
for _, bpScope := range bpScopes {
// get job and scope config from db
- jenkinsJob, scopeConfig, err := scopeHelper.DbHelper().GetScopeAndConfig(connectionId, bpScope.Id)
+ jenkinsJob, scopeConfig, err := scopeHelper.DbHelper().GetScopeAndConfig(connectionId, bpScope.ScopeId)
if err != nil {
return nil, err
}
diff --git a/backend/plugins/jenkins/api/blueprint_v200_test.go b/backend/plugins/jenkins/api/blueprint_v200_test.go
index db4c4fa08..463662365 100644
--- a/backend/plugins/jenkins/api/blueprint_v200_test.go
+++ b/backend/plugins/jenkins/api/blueprint_v200_test.go
@@ -20,6 +20,7 @@ package api
import (
"testing"
+ coreModels "github.com/apache/incubator-devlake/core/models"
"github.com/apache/incubator-devlake/core/models/common"
"github.com/apache/incubator-devlake/core/models/domainlayer"
"github.com/apache/incubator-devlake/core/models/domainlayer/devops"
@@ -38,24 +39,23 @@ func TestMakeDataSourcePipelinePlanV200(t *testing.T) {
mockMeta.On("Name").Return("jenkins").Maybe()
err := plugin.RegisterPlugin("jenkins", mockMeta)
assert.Nil(t, err)
- bs := &plugin.BlueprintScopeV200{
- Id: "a/b/ccc",
- Name: "",
+ bs := &coreModels.BlueprintScope{
+ ScopeId: "a/b/ccc",
}
- syncPolicy := &plugin.BlueprintSyncPolicy{}
- bpScopes := make([]*plugin.BlueprintScopeV200, 0)
+ syncPolicy := &coreModels.BlueprintSyncPolicy{}
+ bpScopes := make([]*coreModels.BlueprintScope, 0)
bpScopes = append(bpScopes, bs)
mockBasicRes(t)
- plan := make(plugin.PipelinePlan, len(bpScopes))
+ plan := make(coreModels.PipelinePlan, len(bpScopes))
plan, err = makeDataSourcePipelinePlanV200(nil, plan, bpScopes, 1, syncPolicy)
assert.Nil(t, err)
scopes, err := makeScopesV200(bpScopes, 1)
assert.Nil(t, err)
- expectPlan := plugin.PipelinePlan{
- plugin.PipelineStage{
+ expectPlan := coreModels.PipelinePlan{
+ coreModels.PipelineStage{
{
Plugin: "jenkins",
Subtasks: []string{},
diff --git a/backend/plugins/jenkins/impl/impl.go b/backend/plugins/jenkins/impl/impl.go
index bc44bec07..123e7ddb5 100644
--- a/backend/plugins/jenkins/impl/impl.go
+++ b/backend/plugins/jenkins/impl/impl.go
@@ -22,6 +22,8 @@ import (
"strings"
"time"
+ coreModels "github.com/apache/incubator-devlake/core/models"
+
"github.com/apache/incubator-devlake/core/context"
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/models/domainlayer/devops"
@@ -168,7 +170,11 @@ func (p Jenkins) MigrationScripts() []plugin.MigrationScript {
return migrationscripts.All()
}
-func (p Jenkins) MakeDataSourcePipelinePlanV200(connectionId uint64, scopes []*plugin.BlueprintScopeV200, syncPolicy plugin.BlueprintSyncPolicy) (pp plugin.PipelinePlan, sc []plugin.Scope, err errors.Error) {
+func (p Jenkins) MakeDataSourcePipelinePlanV200(
+ connectionId uint64,
+ scopes []*coreModels.BlueprintScope,
+ syncPolicy coreModels.BlueprintSyncPolicy,
+) (pp coreModels.PipelinePlan, sc []plugin.Scope, err errors.Error) {
return api.MakeDataSourcePipelinePlanV200(p.SubTaskMetas(), connectionId, scopes, &syncPolicy)
}
diff --git a/backend/plugins/jira/api/blueprint_v200.go b/backend/plugins/jira/api/blueprint_v200.go
index cd4783c74..737c77d33 100644
--- a/backend/plugins/jira/api/blueprint_v200.go
+++ b/backend/plugins/jira/api/blueprint_v200.go
@@ -21,6 +21,7 @@ import (
"time"
"github.com/apache/incubator-devlake/core/errors"
+ coreModels "github.com/apache/incubator-devlake/core/models"
"github.com/apache/incubator-devlake/core/models/domainlayer"
"github.com/apache/incubator-devlake/core/models/domainlayer/didgen"
"github.com/apache/incubator-devlake/core/models/domainlayer/ticket"
@@ -30,8 +31,13 @@ import (
"github.com/apache/incubator-devlake/plugins/jira/models"
)
-func MakeDataSourcePipelinePlanV200(subtaskMetas []plugin.SubTaskMeta, connectionId uint64, bpScopes []*plugin.BlueprintScopeV200, syncPolicy *plugin.BlueprintSyncPolicy) (plugin.PipelinePlan, []plugin.Scope, errors.Error) {
- plan := make(plugin.PipelinePlan, len(bpScopes))
+func MakeDataSourcePipelinePlanV200(
+ subtaskMetas []plugin.SubTaskMeta,
+ connectionId uint64,
+ bpScopes []*coreModels.BlueprintScope,
+ syncPolicy *coreModels.BlueprintSyncPolicy,
+) (coreModels.PipelinePlan, []plugin.Scope, errors.Error) {
+ plan := make(coreModels.PipelinePlan, len(bpScopes))
plan, err := makeDataSourcePipelinePlanV200(subtaskMetas, plan, bpScopes, connectionId, syncPolicy)
if err != nil {
return nil, nil, err
@@ -46,26 +52,26 @@ func MakeDataSourcePipelinePlanV200(subtaskMetas []plugin.SubTaskMeta, connectio
func makeDataSourcePipelinePlanV200(
subtaskMetas []plugin.SubTaskMeta,
- plan plugin.PipelinePlan,
- bpScopes []*plugin.BlueprintScopeV200,
+ plan coreModels.PipelinePlan,
+ bpScopes []*coreModels.BlueprintScope,
connectionId uint64,
- syncPolicy *plugin.BlueprintSyncPolicy,
-) (plugin.PipelinePlan, errors.Error) {
+ syncPolicy *coreModels.BlueprintSyncPolicy,
+) (coreModels.PipelinePlan, errors.Error) {
for i, bpScope := range bpScopes {
stage := plan[i]
if stage == nil {
- stage = plugin.PipelineStage{}
+ stage = coreModels.PipelineStage{}
}
// construct task options for Jira
options := make(map[string]interface{})
- options["scopeId"] = bpScope.Id
+ options["scopeId"] = bpScope.ScopeId
options["connectionId"] = connectionId
if syncPolicy.TimeAfter != nil {
options["timeAfter"] = syncPolicy.TimeAfter.Format(time.RFC3339)
}
// get scope config from db
- _, scopeConfig, err := scopeHelper.DbHelper().GetScopeAndConfig(connectionId, bpScope.Id)
+ _, scopeConfig, err := scopeHelper.DbHelper().GetScopeAndConfig(connectionId, bpScope.ScopeId)
if err != nil {
return nil, err
}
@@ -74,7 +80,7 @@ func makeDataSourcePipelinePlanV200(
if err != nil {
return nil, err
}
- stage = append(stage, &plugin.PipelineTask{
+ stage = append(stage, &coreModels.PipelineTask{
Plugin: "jira",
Subtasks: subtasks,
Options: options,
@@ -86,13 +92,13 @@ func makeDataSourcePipelinePlanV200(
}
func makeScopesV200(
- bpScopes []*plugin.BlueprintScopeV200,
+ bpScopes []*coreModels.BlueprintScope,
connectionId uint64,
) ([]plugin.Scope, errors.Error) {
scopes := make([]plugin.Scope, 0)
for _, bpScope := range bpScopes {
// get board and scope config from db
- jiraBoard, scopeConfig, err := scopeHelper.DbHelper().GetScopeAndConfig(connectionId, bpScope.Id)
+ jiraBoard, scopeConfig, err := scopeHelper.DbHelper().GetScopeAndConfig(connectionId, bpScope.ScopeId)
if err != nil {
return nil, err
}
diff --git a/backend/plugins/jira/api/blueprint_v200_test.go b/backend/plugins/jira/api/blueprint_v200_test.go
index 0d4e8aa7d..e8e64fe17 100644
--- a/backend/plugins/jira/api/blueprint_v200_test.go
+++ b/backend/plugins/jira/api/blueprint_v200_test.go
@@ -20,6 +20,7 @@ package api
import (
"testing"
+ coreModels "github.com/apache/incubator-devlake/core/models"
"github.com/apache/incubator-devlake/core/models/common"
"github.com/apache/incubator-devlake/core/models/domainlayer"
"github.com/apache/incubator-devlake/core/models/domainlayer/ticket"
@@ -38,13 +39,13 @@ func TestMakeDataSourcePipelinePlanV200(t *testing.T) {
mockMeta.On("Name").Return("jira").Maybe()
err := plugin.RegisterPlugin("jira", mockMeta)
assert.Nil(t, err)
- bs := &plugin.BlueprintScopeV200{
- Id: "10",
+ bs := &coreModels.BlueprintScope{
+ ScopeId: "10",
}
- syncPolicy := &plugin.BlueprintSyncPolicy{}
- bpScopes := make([]*plugin.BlueprintScopeV200, 0)
+ syncPolicy := &coreModels.BlueprintSyncPolicy{}
+ bpScopes := make([]*coreModels.BlueprintScope, 0)
bpScopes = append(bpScopes, bs)
- plan := make(plugin.PipelinePlan, len(bpScopes))
+ plan := make(coreModels.PipelinePlan, len(bpScopes))
mockBasicRes(t)
plan, err = makeDataSourcePipelinePlanV200(nil, plan, bpScopes, uint64(1), syncPolicy)
@@ -52,8 +53,8 @@ func TestMakeDataSourcePipelinePlanV200(t *testing.T) {
scopes, err := makeScopesV200(bpScopes, uint64(1))
assert.Nil(t, err)
- expectPlan := plugin.PipelinePlan{
- plugin.PipelineStage{
+ expectPlan := coreModels.PipelinePlan{
+ coreModels.PipelineStage{
{
Plugin: "jira",
Subtasks: []string{},
diff --git a/backend/plugins/jira/impl/impl.go b/backend/plugins/jira/impl/impl.go
index b2a39bbd9..f74d71986 100644
--- a/backend/plugins/jira/impl/impl.go
+++ b/backend/plugins/jira/impl/impl.go
@@ -25,6 +25,7 @@ import (
"github.com/apache/incubator-devlake/core/context"
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
+ coreModels "github.com/apache/incubator-devlake/core/models"
"github.com/apache/incubator-devlake/core/plugin"
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/jira/api"
@@ -261,7 +262,11 @@ func (p Jira) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]int
return taskData, nil
}
-func (p Jira) MakeDataSourcePipelinePlanV200(connectionId uint64, scopes []*plugin.BlueprintScopeV200, syncPolicy plugin.BlueprintSyncPolicy) (pp plugin.PipelinePlan, sc []plugin.Scope, err errors.Error) {
+func (p Jira) MakeDataSourcePipelinePlanV200(
+ connectionId uint64,
+ scopes []*coreModels.BlueprintScope,
+ syncPolicy coreModels.BlueprintSyncPolicy,
+) (pp coreModels.PipelinePlan, sc []plugin.Scope, err errors.Error) {
return api.MakeDataSourcePipelinePlanV200(p.SubTaskMetas(), connectionId, scopes, &syncPolicy)
}
diff --git a/backend/plugins/org/impl/impl.go b/backend/plugins/org/impl/impl.go
index 4ee6f9637..cd4ffa7fd 100644
--- a/backend/plugins/org/impl/impl.go
+++ b/backend/plugins/org/impl/impl.go
@@ -21,6 +21,7 @@ import (
"github.com/apache/incubator-devlake/core/context"
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
+ coreModels "github.com/apache/incubator-devlake/core/models"
"github.com/apache/incubator-devlake/core/plugin"
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/org/api"
@@ -63,9 +64,9 @@ func (p Org) SubTaskMetas() []plugin.SubTaskMeta {
}
}
-func (p Org) MapProject(projectName string, scopes []plugin.Scope) (plugin.PipelinePlan, errors.Error) {
- var plan plugin.PipelinePlan
- var stage plugin.PipelineStage
+func (p Org) MapProject(projectName string, scopes []plugin.Scope) (coreModels.PipelinePlan, errors.Error) {
+ var plan coreModels.PipelinePlan
+ var stage coreModels.PipelineStage
// construct task options for Org
options := make(map[string]interface{})
@@ -75,7 +76,7 @@ func (p Org) MapProject(projectName string, scopes []plugin.Scope) (plugin.Pipel
if err != nil {
return nil, err
}
- stage = append(stage, &plugin.PipelineTask{
+ stage = append(stage, &coreModels.PipelineTask{
Plugin: "org",
Subtasks: subtasks,
Options: options,
diff --git a/backend/plugins/pagerduty/api/blueprint_v200.go b/backend/plugins/pagerduty/api/blueprint_v200.go
index 72ca47313..5087a1ff0 100644
--- a/backend/plugins/pagerduty/api/blueprint_v200.go
+++ b/backend/plugins/pagerduty/api/blueprint_v200.go
@@ -21,6 +21,7 @@ import (
"time"
"github.com/apache/incubator-devlake/core/errors"
+ coreModels "github.com/apache/incubator-devlake/core/models"
"github.com/apache/incubator-devlake/core/models/domainlayer"
"github.com/apache/incubator-devlake/core/models/domainlayer/didgen"
"github.com/apache/incubator-devlake/core/models/domainlayer/ticket"
@@ -31,8 +32,12 @@ import (
"github.com/apache/incubator-devlake/plugins/pagerduty/tasks"
)
-func MakeDataSourcePipelinePlanV200(subtaskMetas []plugin.SubTaskMeta, connectionId uint64, bpScopes []*plugin.BlueprintScopeV200, syncPolicy *plugin.BlueprintSyncPolicy,
-) (plugin.PipelinePlan, []plugin.Scope, errors.Error) {
+func MakeDataSourcePipelinePlanV200(
+ subtaskMetas []plugin.SubTaskMeta,
+ connectionId uint64,
+ bpScopes []*coreModels.BlueprintScope,
+ syncPolicy *coreModels.BlueprintSyncPolicy,
+) (coreModels.PipelinePlan, []plugin.Scope, errors.Error) {
// get the connection info for url
connection := &models.PagerDutyConnection{}
err := connectionHelper.FirstById(connection, connectionId)
@@ -40,7 +45,7 @@ func MakeDataSourcePipelinePlanV200(subtaskMetas []plugin.SubTaskMeta, connectio
return nil, nil, err
}
- plan := make(plugin.PipelinePlan, len(bpScopes))
+ plan := make(coreModels.PipelinePlan, len(bpScopes))
plan, err = makeDataSourcePipelinePlanV200(subtaskMetas, plan, bpScopes, connection, syncPolicy)
if err != nil {
return nil, nil, err
@@ -55,14 +60,14 @@ func MakeDataSourcePipelinePlanV200(subtaskMetas []plugin.SubTaskMeta, connectio
func makeDataSourcePipelinePlanV200(
subtaskMetas []plugin.SubTaskMeta,
- plan plugin.PipelinePlan,
- bpScopes []*plugin.BlueprintScopeV200,
+ plan coreModels.PipelinePlan,
+ bpScopes []*coreModels.BlueprintScope,
connection *models.PagerDutyConnection,
- syncPolicy *plugin.BlueprintSyncPolicy,
-) (plugin.PipelinePlan, errors.Error) {
+ syncPolicy *coreModels.BlueprintSyncPolicy,
+) (coreModels.PipelinePlan, errors.Error) {
for i, bpScope := range bpScopes {
// get board and scope config from db
- service, scopeConfig, err := scopeHelper.DbHelper().GetScopeAndConfig(connection.ID, bpScope.Id)
+ service, scopeConfig, err := scopeHelper.DbHelper().GetScopeAndConfig(connection.ID, bpScope.ScopeId)
if err != nil {
return nil, err
}
@@ -85,7 +90,7 @@ func makeDataSourcePipelinePlanV200(
if err != nil {
return nil, err
}
- stage := []*plugin.PipelineTask{
+ stage := []*coreModels.PipelineTask{
{
Plugin: "pagerduty",
Subtasks: subtasks,
@@ -97,11 +102,11 @@ func makeDataSourcePipelinePlanV200(
return plan, nil
}
-func makeScopesV200(bpScopes []*plugin.BlueprintScopeV200, connection *models.PagerDutyConnection) ([]plugin.Scope, errors.Error) {
+func makeScopesV200(bpScopes []*coreModels.BlueprintScope, connection *models.PagerDutyConnection) ([]plugin.Scope, errors.Error) {
scopes := make([]plugin.Scope, 0)
for _, bpScope := range bpScopes {
// get board and scope config from db
- service, scopeConfig, err := scopeHelper.DbHelper().GetScopeAndConfig(connection.ID, bpScope.Id)
+ service, scopeConfig, err := scopeHelper.DbHelper().GetScopeAndConfig(connection.ID, bpScope.ScopeId)
if err != nil {
return nil, err
}
diff --git a/backend/plugins/pagerduty/impl/impl.go b/backend/plugins/pagerduty/impl/impl.go
index f2293d33a..0314ddb9a 100644
--- a/backend/plugins/pagerduty/impl/impl.go
+++ b/backend/plugins/pagerduty/impl/impl.go
@@ -24,6 +24,7 @@ import (
"github.com/apache/incubator-devlake/core/context"
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
+ coreModels "github.com/apache/incubator-devlake/core/models"
"github.com/apache/incubator-devlake/core/plugin"
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/pagerduty/api"
@@ -173,8 +174,11 @@ func (p PagerDuty) ApiResources() map[string]map[string]plugin.ApiResourceHandle
}
}
-func (p PagerDuty) MakeDataSourcePipelinePlanV200(connectionId uint64, scopes []*plugin.BlueprintScopeV200, syncPolicy plugin.BlueprintSyncPolicy,
-) (plugin.PipelinePlan, []plugin.Scope, errors.Error) {
+func (p PagerDuty) MakeDataSourcePipelinePlanV200(
+ connectionId uint64,
+ scopes []*coreModels.BlueprintScope,
+ syncPolicy coreModels.BlueprintSyncPolicy,
+) (coreModels.PipelinePlan, []plugin.Scope, errors.Error) {
return api.MakeDataSourcePipelinePlanV200(p.SubTaskMetas(), connectionId, scopes, &syncPolicy)
}
diff --git a/backend/plugins/sonarqube/api/blueprint_v200.go b/backend/plugins/sonarqube/api/blueprint_v200.go
index dd6656217..fa4825ce3 100644
--- a/backend/plugins/sonarqube/api/blueprint_v200.go
+++ b/backend/plugins/sonarqube/api/blueprint_v200.go
@@ -24,6 +24,7 @@ import (
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
+ coreModels "github.com/apache/incubator-devlake/core/models"
"github.com/apache/incubator-devlake/core/models/domainlayer"
"github.com/apache/incubator-devlake/core/models/domainlayer/codequality"
"github.com/apache/incubator-devlake/core/models/domainlayer/didgen"
@@ -33,8 +34,13 @@ import (
"github.com/apache/incubator-devlake/plugins/sonarqube/models"
)
-func MakeDataSourcePipelinePlanV200(subtaskMetas []plugin.SubTaskMeta, connectionId uint64, bpScopes []*plugin.BlueprintScopeV200, syncPolicy *plugin.BlueprintSyncPolicy) (plugin.PipelinePlan, []plugin.Scope, errors.Error) {
- plan := make(plugin.PipelinePlan, len(bpScopes))
+func MakeDataSourcePipelinePlanV200(
+ subtaskMetas []plugin.SubTaskMeta,
+ connectionId uint64,
+ bpScopes []*coreModels.BlueprintScope,
+ syncPolicy *coreModels.BlueprintSyncPolicy,
+) (coreModels.PipelinePlan, []plugin.Scope, errors.Error) {
+ plan := make(coreModels.PipelinePlan, len(bpScopes))
plan, err := makeDataSourcePipelinePlanV200(subtaskMetas, plan, bpScopes, connectionId, syncPolicy)
if err != nil {
return nil, nil, err
@@ -49,26 +55,26 @@ func MakeDataSourcePipelinePlanV200(subtaskMetas []plugin.SubTaskMeta, connectio
func makeDataSourcePipelinePlanV200(
subtaskMetas []plugin.SubTaskMeta,
- plan plugin.PipelinePlan,
- bpScopes []*plugin.BlueprintScopeV200,
+ plan coreModels.PipelinePlan,
+ bpScopes []*coreModels.BlueprintScope,
connectionId uint64,
- syncPolicy *plugin.BlueprintSyncPolicy,
-) (plugin.PipelinePlan, errors.Error) {
+ syncPolicy *coreModels.BlueprintSyncPolicy,
+) (coreModels.PipelinePlan, errors.Error) {
for i, bpScope := range bpScopes {
stage := plan[i]
if stage == nil {
- stage = plugin.PipelineStage{}
+ stage = coreModels.PipelineStage{}
}
// construct task options for Sonarqube
options := make(map[string]interface{})
options["connectionId"] = connectionId
- options["projectKey"] = bpScope.Id
+ options["projectKey"] = bpScope.ScopeId
subtasks, err := helper.MakePipelinePlanSubtasks(subtaskMetas, plugin.DOMAIN_TYPES)
if err != nil {
return nil, err
}
- stage = append(stage, &plugin.PipelineTask{
+ stage = append(stage, &coreModels.PipelineTask{
Plugin: "sonarqube",
Subtasks: subtasks,
Options: options,
@@ -79,16 +85,16 @@ func makeDataSourcePipelinePlanV200(
return plan, nil
}
-func makeScopesV200(bpScopes []*plugin.BlueprintScopeV200, connectionId uint64) ([]plugin.Scope, errors.Error) {
+func makeScopesV200(bpScopes []*coreModels.BlueprintScope, connectionId uint64) ([]plugin.Scope, errors.Error) {
scopes := make([]plugin.Scope, 0)
for _, bpScope := range bpScopes {
sonarqubeProject := &models.SonarqubeProject{}
// get repo from db
err := basicRes.GetDal().First(sonarqubeProject,
dal.Where("connection_id = ? and project_key = ?",
- connectionId, bpScope.Id))
+ connectionId, bpScope.ScopeId))
if err != nil {
- return nil, errors.Default.Wrap(err, fmt.Sprintf("fail to find sonarqube project %s", bpScope.Id))
+ return nil, errors.Default.Wrap(err, fmt.Sprintf("fail to find sonarqube project %s", bpScope.ScopeId))
}
// add board to scopes
// if utils.StringsContains(bpScope.Entities, plugin.DOMAIN_TYPE_CODE_QUALITY) {
diff --git a/backend/plugins/sonarqube/api/blueprint_v200_test.go b/backend/plugins/sonarqube/api/blueprint_v200_test.go
index 009fe9a9b..f9d9f760e 100644
--- a/backend/plugins/sonarqube/api/blueprint_v200_test.go
+++ b/backend/plugins/sonarqube/api/blueprint_v200_test.go
@@ -20,6 +20,7 @@ package api
import (
"testing"
+ coreModels "github.com/apache/incubator-devlake/core/models"
"github.com/apache/incubator-devlake/core/models/domainlayer"
"github.com/apache/incubator-devlake/core/models/domainlayer/codequality"
"github.com/apache/incubator-devlake/core/plugin"
@@ -37,13 +38,13 @@ func TestMakeDataSourcePipelinePlanV200(t *testing.T) {
mockMeta.On("Name").Return("sonarqube").Maybe()
err := plugin.RegisterPlugin("sonarqube", mockMeta)
assert.Nil(t, err)
- bs := &plugin.BlueprintScopeV200{
- Id: "f5a50c63-2e8f-4107-9014-853f6f467757",
+ bs := &coreModels.BlueprintScope{
+ ScopeId: "f5a50c63-2e8f-4107-9014-853f6f467757",
}
- syncPolicy := &plugin.BlueprintSyncPolicy{}
- bpScopes := make([]*plugin.BlueprintScopeV200, 0)
+ syncPolicy := &coreModels.BlueprintSyncPolicy{}
+ bpScopes := make([]*coreModels.BlueprintScope, 0)
bpScopes = append(bpScopes, bs)
- plan := make(plugin.PipelinePlan, len(bpScopes))
+ plan := make(coreModels.PipelinePlan, len(bpScopes))
plan, err = makeDataSourcePipelinePlanV200(nil, plan, bpScopes, uint64(1), syncPolicy)
assert.Nil(t, err)
basicRes = NewMockBasicRes()
@@ -51,8 +52,8 @@ func TestMakeDataSourcePipelinePlanV200(t *testing.T) {
scopes, err := makeScopesV200(bpScopes, uint64(1))
assert.Nil(t, err)
- expectPlan := plugin.PipelinePlan{
- plugin.PipelineStage{
+ expectPlan := coreModels.PipelinePlan{
+ coreModels.PipelineStage{
{
Plugin: "sonarqube",
Subtasks: []string{},
diff --git a/backend/plugins/sonarqube/impl/impl.go b/backend/plugins/sonarqube/impl/impl.go
index 514554339..5ec0919df 100644
--- a/backend/plugins/sonarqube/impl/impl.go
+++ b/backend/plugins/sonarqube/impl/impl.go
@@ -21,6 +21,8 @@ import (
"fmt"
"time"
+ coreModels "github.com/apache/incubator-devlake/core/models"
+
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/context"
@@ -195,7 +197,11 @@ func (p Sonarqube) ApiResources() map[string]map[string]plugin.ApiResourceHandle
}
}
-func (p Sonarqube) MakeDataSourcePipelinePlanV200(connectionId uint64, scopes []*plugin.BlueprintScopeV200, syncPolicy plugin.BlueprintSyncPolicy) (pp plugin.PipelinePlan, sc []plugin.Scope, err errors.Error) {
+func (p Sonarqube) MakeDataSourcePipelinePlanV200(
+ connectionId uint64,
+ scopes []*coreModels.BlueprintScope,
+ syncPolicy coreModels.BlueprintSyncPolicy,
+) (pp coreModels.PipelinePlan, sc []plugin.Scope, err errors.Error) {
return api.MakeDataSourcePipelinePlanV200(p.SubTaskMetas(), connectionId, scopes, &syncPolicy)
}
diff --git a/backend/plugins/tapd/api/blueprint_v200.go b/backend/plugins/tapd/api/blueprint_v200.go
index 513eb28b8..82fe90214 100644
--- a/backend/plugins/tapd/api/blueprint_v200.go
+++ b/backend/plugins/tapd/api/blueprint_v200.go
@@ -22,6 +22,7 @@ import (
"time"
"github.com/apache/incubator-devlake/core/errors"
+ coreModels "github.com/apache/incubator-devlake/core/models"
"github.com/apache/incubator-devlake/core/models/domainlayer"
"github.com/apache/incubator-devlake/core/models/domainlayer/didgen"
"github.com/apache/incubator-devlake/core/models/domainlayer/ticket"
@@ -31,8 +32,13 @@ import (
"github.com/apache/incubator-devlake/plugins/tapd/models"
)
-func MakeDataSourcePipelinePlanV200(subtaskMetas []plugin.SubTaskMeta, connectionId uint64, bpScopes []*plugin.BlueprintScopeV200, syncPolicy *plugin.BlueprintSyncPolicy) (plugin.PipelinePlan, []plugin.Scope, errors.Error) {
- plan := make(plugin.PipelinePlan, len(bpScopes))
+func MakeDataSourcePipelinePlanV200(
+ subtaskMetas []plugin.SubTaskMeta,
+ connectionId uint64,
+ bpScopes []*coreModels.BlueprintScope,
+ syncPolicy *coreModels.BlueprintSyncPolicy,
+) (coreModels.PipelinePlan, []plugin.Scope, errors.Error) {
+ plan := make(coreModels.PipelinePlan, len(bpScopes))
plan, err := makeDataSourcePipelinePlanV200(subtaskMetas, plan, bpScopes, connectionId, syncPolicy)
if err != nil {
return nil, nil, err
@@ -47,19 +53,19 @@ func MakeDataSourcePipelinePlanV200(subtaskMetas []plugin.SubTaskMeta, connectio
func makeDataSourcePipelinePlanV200(
subtaskMetas []plugin.SubTaskMeta,
- plan plugin.PipelinePlan,
- bpScopes []*plugin.BlueprintScopeV200,
+ plan coreModels.PipelinePlan,
+ bpScopes []*coreModels.BlueprintScope,
connectionId uint64,
- syncPolicy *plugin.BlueprintSyncPolicy,
-) (plugin.PipelinePlan, errors.Error) {
+ syncPolicy *coreModels.BlueprintSyncPolicy,
+) (coreModels.PipelinePlan, errors.Error) {
for i, bpScope := range bpScopes {
stage := plan[i]
if stage == nil {
- stage = plugin.PipelineStage{}
+ stage = coreModels.PipelineStage{}
}
// construct task options for tapd
options := make(map[string]interface{})
- intNum, err := errors.Convert01(strconv.Atoi(bpScope.Id))
+ intNum, err := errors.Convert01(strconv.Atoi(bpScope.ScopeId))
if err != nil {
return nil, err
}
@@ -69,7 +75,7 @@ func makeDataSourcePipelinePlanV200(
options["timeAfter"] = syncPolicy.TimeAfter.Format(time.RFC3339)
}
- _, scopeConfig, err := scopeHelper.DbHelper().GetScopeAndConfig(connectionId, bpScope.Id)
+ _, scopeConfig, err := scopeHelper.DbHelper().GetScopeAndConfig(connectionId, bpScope.ScopeId)
if err != nil {
return nil, err
}
@@ -78,7 +84,7 @@ func makeDataSourcePipelinePlanV200(
if err != nil {
return nil, err
}
- stage = append(stage, &plugin.PipelineTask{
+ stage = append(stage, &coreModels.PipelineTask{
Plugin: "tapd",
Subtasks: subtasks,
Options: options,
@@ -90,14 +96,14 @@ func makeDataSourcePipelinePlanV200(
}
func makeScopesV200(
- bpScopes []*plugin.BlueprintScopeV200,
+ bpScopes []*coreModels.BlueprintScope,
connectionId uint64) ([]plugin.Scope, errors.Error,
) {
scopes := make([]plugin.Scope, 0)
for _, bpScope := range bpScopes {
// get workspace and scope config from db
- tapdWorkspace, scopeConfig, err := scopeHelper.DbHelper().GetScopeAndConfig(connectionId, bpScope.Id)
+ tapdWorkspace, scopeConfig, err := scopeHelper.DbHelper().GetScopeAndConfig(connectionId, bpScope.ScopeId)
if err != nil {
return nil, err
}
diff --git a/backend/plugins/tapd/api/blueprint_v200_test.go b/backend/plugins/tapd/api/blueprint_v200_test.go
index 6466f06c9..c7aa2647b 100644
--- a/backend/plugins/tapd/api/blueprint_v200_test.go
+++ b/backend/plugins/tapd/api/blueprint_v200_test.go
@@ -20,6 +20,7 @@ package api
import (
"testing"
+ coreModels "github.com/apache/incubator-devlake/core/models"
"github.com/apache/incubator-devlake/core/models/common"
"github.com/apache/incubator-devlake/core/models/domainlayer"
"github.com/apache/incubator-devlake/core/models/domainlayer/ticket"
@@ -38,13 +39,13 @@ func TestMakeDataSourcePipelinePlanV200(t *testing.T) {
mockMeta.On("Name").Return("dummy").Maybe()
err := plugin.RegisterPlugin("tapd", mockMeta)
assert.Nil(t, err)
- bs := &plugin.BlueprintScopeV200{
- Id: "10",
+ bs := &coreModels.BlueprintScope{
+ ScopeId: "10",
}
- syncPolicy := &plugin.BlueprintSyncPolicy{}
- bpScopes := make([]*plugin.BlueprintScopeV200, 0)
+ syncPolicy := &coreModels.BlueprintSyncPolicy{}
+ bpScopes := make([]*coreModels.BlueprintScope, 0)
bpScopes = append(bpScopes, bs)
- plan := make(plugin.PipelinePlan, len(bpScopes))
+ plan := make(coreModels.PipelinePlan, len(bpScopes))
mockBasicRes(t)
plan, err = makeDataSourcePipelinePlanV200(nil, plan, bpScopes, uint64(1), syncPolicy)
@@ -52,8 +53,8 @@ func TestMakeDataSourcePipelinePlanV200(t *testing.T) {
scopes, err := makeScopesV200(bpScopes, uint64(1))
assert.Nil(t, err)
- expectPlan := plugin.PipelinePlan{
- plugin.PipelineStage{
+ expectPlan := coreModels.PipelinePlan{
+ coreModels.PipelineStage{
{
Plugin: "tapd",
Subtasks: []string{},
diff --git a/backend/plugins/tapd/impl/impl.go b/backend/plugins/tapd/impl/impl.go
index b77ca2d21..50fbb43c3 100644
--- a/backend/plugins/tapd/impl/impl.go
+++ b/backend/plugins/tapd/impl/impl.go
@@ -24,6 +24,7 @@ import (
"github.com/apache/incubator-devlake/core/context"
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
+ coreModels "github.com/apache/incubator-devlake/core/models"
"github.com/apache/incubator-devlake/core/plugin"
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/tapd/api"
@@ -271,7 +272,11 @@ func (p Tapd) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]int
return taskData, nil
}
-func (p Tapd) MakeDataSourcePipelinePlanV200(connectionId uint64, scopes []*plugin.BlueprintScopeV200, syncPolicy plugin.BlueprintSyncPolicy) (pp plugin.PipelinePlan, sc []plugin.Scope, err errors.Error) {
+func (p Tapd) MakeDataSourcePipelinePlanV200(
+ connectionId uint64,
+ scopes []*coreModels.BlueprintScope,
+ syncPolicy coreModels.BlueprintSyncPolicy,
+) (pp coreModels.PipelinePlan, sc []plugin.Scope, err errors.Error) {
return api.MakeDataSourcePipelinePlanV200(p.SubTaskMetas(), connectionId, scopes, &syncPolicy)
}
diff --git a/backend/plugins/teambition/api/blueprint200.go b/backend/plugins/teambition/api/blueprint200.go
index 104a2c671..52e8fbb92 100644
--- a/backend/plugins/teambition/api/blueprint200.go
+++ b/backend/plugins/teambition/api/blueprint200.go
@@ -25,6 +25,7 @@ import (
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
+ coreModels "github.com/apache/incubator-devlake/core/models"
"github.com/apache/incubator-devlake/core/models/domainlayer"
"github.com/apache/incubator-devlake/core/models/domainlayer/didgen"
"github.com/apache/incubator-devlake/core/models/domainlayer/ticket"
@@ -32,8 +33,13 @@ import (
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
)
-func MakeDataSourcePipelinePlanV200(subtaskMetas []plugin.SubTaskMeta, connectionId uint64, bpScopes []*plugin.BlueprintScopeV200, syncPolicy *plugin.BlueprintSyncPolicy) (plugin.PipelinePlan, []plugin.Scope, errors.Error) {
- plan := make(plugin.PipelinePlan, len(bpScopes))
+func MakeDataSourcePipelinePlanV200(
+ subtaskMetas []plugin.SubTaskMeta,
+ connectionId uint64,
+ bpScopes []*coreModels.BlueprintScope,
+ syncPolicy *coreModels.BlueprintSyncPolicy,
+) (coreModels.PipelinePlan, []plugin.Scope, errors.Error) {
+ plan := make(coreModels.PipelinePlan, len(bpScopes))
plan, err := makeDataSourcePipelinePlanV200(subtaskMetas, plan, bpScopes, connectionId, syncPolicy)
if err != nil {
return nil, nil, err
@@ -48,19 +54,19 @@ func MakeDataSourcePipelinePlanV200(subtaskMetas []plugin.SubTaskMeta, connectio
func makeDataSourcePipelinePlanV200(
subtaskMetas []plugin.SubTaskMeta,
- plan plugin.PipelinePlan,
- bpScopes []*plugin.BlueprintScopeV200,
+ plan coreModels.PipelinePlan,
+ bpScopes []*coreModels.BlueprintScope,
connectionId uint64,
- syncPolicy *plugin.BlueprintSyncPolicy,
-) (plugin.PipelinePlan, errors.Error) {
+ syncPolicy *coreModels.BlueprintSyncPolicy,
+) (coreModels.PipelinePlan, errors.Error) {
for i, bpScope := range bpScopes {
stage := plan[i]
if stage == nil {
- stage = plugin.PipelineStage{}
+ stage = coreModels.PipelineStage{}
}
// construct task options for teambition
options := make(map[string]interface{})
- options["projectId"] = bpScope.Id
+ options["projectId"] = bpScope.ScopeId
options["connectionId"] = connectionId
if syncPolicy.TimeAfter != nil {
options["timeAfter"] = syncPolicy.TimeAfter.Format(time.RFC3339)
@@ -70,7 +76,7 @@ func makeDataSourcePipelinePlanV200(
if err != nil {
return nil, err
}
- stage = append(stage, &plugin.PipelineTask{
+ stage = append(stage, &coreModels.PipelineTask{
Plugin: "teambition",
Subtasks: subtasks,
Options: options,
@@ -81,16 +87,16 @@ func makeDataSourcePipelinePlanV200(
return plan, nil
}
-func makeScopesV200(bpScopes []*plugin.BlueprintScopeV200, connectionId uint64) ([]plugin.Scope, errors.Error) {
+func makeScopesV200(bpScopes []*coreModels.BlueprintScope, connectionId uint64) ([]plugin.Scope, errors.Error) {
scopes := make([]plugin.Scope, 0)
for _, bpScope := range bpScopes {
project := &models.TeambitionProject{}
// get project from db
err := basicRes.GetDal().First(project,
dal.Where(`connection_id = ? and id = ?`,
- connectionId, bpScope.Id))
+ connectionId, bpScope.ScopeId))
if err != nil {
- return nil, errors.Default.Wrap(err, fmt.Sprintf("fail to find project %s", bpScope.Id))
+ return nil, errors.Default.Wrap(err, fmt.Sprintf("fail to find project %s", bpScope.ScopeId))
}
// add board to scopes
// if utils.StringsContains(bpScope.Entities, plugin.DOMAIN_TYPE_TICKET) {
diff --git a/backend/plugins/teambition/impl/impl.go b/backend/plugins/teambition/impl/impl.go
index d1a660426..e499a0294 100644
--- a/backend/plugins/teambition/impl/impl.go
+++ b/backend/plugins/teambition/impl/impl.go
@@ -24,6 +24,7 @@ import (
"github.com/apache/incubator-devlake/core/context"
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
+ coreModels "github.com/apache/incubator-devlake/core/models"
"github.com/apache/incubator-devlake/core/plugin"
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/teambition/api"
@@ -118,7 +119,11 @@ func (p Teambition) SubTaskMetas() []plugin.SubTaskMeta {
}
}
-func (p Teambition) MakeDataSourcePipelinePlanV200(connectionId uint64, scopes []*plugin.BlueprintScopeV200, syncPolicy plugin.BlueprintSyncPolicy) (pp plugin.PipelinePlan, sc []plugin.Scope, err errors.Error) {
+func (p Teambition) MakeDataSourcePipelinePlanV200(
+ connectionId uint64,
+ scopes []*coreModels.BlueprintScope,
+ syncPolicy coreModels.BlueprintSyncPolicy,
+) (pp coreModels.PipelinePlan, sc []plugin.Scope, err errors.Error) {
return api.MakeDataSourcePipelinePlanV200(p.SubTaskMetas(), connectionId, scopes, &syncPolicy)
}
diff --git a/backend/plugins/trello/api/blueprint_v200.go b/backend/plugins/trello/api/blueprint_v200.go
index 784a2e332..d2b79329b 100644
--- a/backend/plugins/trello/api/blueprint_v200.go
+++ b/backend/plugins/trello/api/blueprint_v200.go
@@ -28,18 +28,24 @@ import (
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/utils"
+ coreModels "github.com/apache/incubator-devlake/core/models"
"github.com/apache/incubator-devlake/core/models/domainlayer/didgen"
"github.com/apache/incubator-devlake/core/plugin"
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
)
-func MakePipelinePlanV200(subtaskMetas []plugin.SubTaskMeta, connectionId uint64, scope []*plugin.BlueprintScopeV200, syncPolicy *plugin.BlueprintSyncPolicy) (plugin.PipelinePlan, []plugin.Scope, errors.Error) {
+func MakePipelinePlanV200(
+ subtaskMetas []plugin.SubTaskMeta,
+ connectionId uint64,
+ scope []*coreModels.BlueprintScope,
+ syncPolicy *coreModels.BlueprintSyncPolicy,
+) (coreModels.PipelinePlan, []plugin.Scope, errors.Error) {
scopes, err := makeScopeV200(connectionId, scope)
if err != nil {
return nil, nil, err
}
- plan := make(plugin.PipelinePlan, len(scope))
+ plan := make(coreModels.PipelinePlan, len(scope))
plan, err = makePipelinePlanV200(subtaskMetas, plan, scope, connectionId, syncPolicy)
if err != nil {
return nil, nil, err
@@ -48,11 +54,11 @@ func MakePipelinePlanV200(subtaskMetas []plugin.SubTaskMeta, connectionId uint64
return plan, scopes, nil
}
-func makeScopeV200(connectionId uint64, scopes []*plugin.BlueprintScopeV200) ([]plugin.Scope, errors.Error) {
+func makeScopeV200(connectionId uint64, scopes []*coreModels.BlueprintScope) ([]plugin.Scope, errors.Error) {
sc := make([]plugin.Scope, 0, len(scopes))
for _, scope := range scopes {
- trelloBoard, scopeConfig, err := scopeHelper.DbHelper().GetScopeAndConfig(connectionId, scope.Id)
+ trelloBoard, scopeConfig, err := scopeHelper.DbHelper().GetScopeAndConfig(connectionId, scope.ScopeId)
if err != nil {
return nil, err
}
@@ -71,22 +77,27 @@ func makeScopeV200(connectionId uint64, scopes []*plugin.BlueprintScopeV200) ([]
return sc, nil
}
-func makePipelinePlanV200(subtaskMetas []plugin.SubTaskMeta, plan plugin.PipelinePlan, scopes []*plugin.BlueprintScopeV200, connectionId uint64, syncPolicy *plugin.BlueprintSyncPolicy) (plugin.PipelinePlan, errors.Error) {
+func makePipelinePlanV200(
+ subtaskMetas []plugin.SubTaskMeta,
+ plan coreModels.PipelinePlan,
+ scopes []*coreModels.BlueprintScope,
+ connectionId uint64, syncPolicy *coreModels.BlueprintSyncPolicy,
+) (coreModels.PipelinePlan, errors.Error) {
for i, scope := range scopes {
stage := plan[i]
if stage == nil {
- stage = plugin.PipelineStage{}
+ stage = coreModels.PipelineStage{}
}
// construct task options for trello
options := make(map[string]interface{})
options["connectionId"] = connectionId
- options["scopeId"] = scope.Id
+ options["scopeId"] = scope.ScopeId
if syncPolicy.TimeAfter != nil {
options["timeAfter"] = syncPolicy.TimeAfter.Format(time.RFC3339)
}
- _, scopeConfig, err := scopeHelper.DbHelper().GetScopeAndConfig(connectionId, scope.Id)
+ _, scopeConfig, err := scopeHelper.DbHelper().GetScopeAndConfig(connectionId, scope.ScopeId)
if err != nil {
return nil, err
}
@@ -96,7 +107,7 @@ func makePipelinePlanV200(subtaskMetas []plugin.SubTaskMeta, plan plugin.Pipelin
return nil, err
}
- stage = append(stage, &plugin.PipelineTask{
+ stage = append(stage, &coreModels.PipelineTask{
Plugin: "trello",
Subtasks: subtasks,
Options: options,
diff --git a/backend/plugins/trello/impl/impl.go b/backend/plugins/trello/impl/impl.go
index e20e5ad86..532c69794 100644
--- a/backend/plugins/trello/impl/impl.go
+++ b/backend/plugins/trello/impl/impl.go
@@ -23,6 +23,7 @@ import (
"github.com/apache/incubator-devlake/core/context"
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
+ coreModels "github.com/apache/incubator-devlake/core/models"
"github.com/apache/incubator-devlake/core/plugin"
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/trello/api"
@@ -191,7 +192,11 @@ func (p Trello) ApiResources() map[string]map[string]plugin.ApiResourceHandler {
}
}
-func (p Trello) MakeDataSourcePipelinePlanV200(connectionId uint64, scopes []*plugin.BlueprintScopeV200, syncPolicy plugin.BlueprintSyncPolicy) (plugin.PipelinePlan, []plugin.Scope, errors.Error) {
+func (p Trello) MakeDataSourcePipelinePlanV200(
+ connectionId uint64,
+ scopes []*coreModels.BlueprintScope,
+ syncPolicy coreModels.BlueprintSyncPolicy,
+) (coreModels.PipelinePlan, []plugin.Scope, errors.Error) {
return api.MakePipelinePlanV200(p.SubTaskMetas(), connectionId, scopes, &syncPolicy)
}
diff --git a/backend/plugins/webhook/api/blueprint_v200.go b/backend/plugins/webhook/api/blueprint_v200.go
index 006beaded..eaa6be35d 100644
--- a/backend/plugins/webhook/api/blueprint_v200.go
+++ b/backend/plugins/webhook/api/blueprint_v200.go
@@ -21,6 +21,7 @@ import (
"fmt"
"github.com/apache/incubator-devlake/core/errors"
+ coreModels "github.com/apache/incubator-devlake/core/models"
"github.com/apache/incubator-devlake/core/models/domainlayer"
"github.com/apache/incubator-devlake/core/models/domainlayer/devops"
"github.com/apache/incubator-devlake/core/models/domainlayer/ticket"
@@ -28,7 +29,7 @@ import (
"github.com/apache/incubator-devlake/plugins/webhook/models"
)
-func MakeDataSourcePipelinePlanV200(connectionId uint64) (plugin.PipelinePlan, []plugin.Scope, errors.Error) {
+func MakeDataSourcePipelinePlanV200(connectionId uint64) (coreModels.PipelinePlan, []plugin.Scope, errors.Error) {
// get the connection info for url
connection := &models.WebhookConnection{}
err := connectionHelper.FirstById(connection, connectionId)
diff --git a/backend/plugins/webhook/impl/impl.go b/backend/plugins/webhook/impl/impl.go
index 7056154f5..46a01715e 100644
--- a/backend/plugins/webhook/impl/impl.go
+++ b/backend/plugins/webhook/impl/impl.go
@@ -21,6 +21,7 @@ import (
"github.com/apache/incubator-devlake/core/context"
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
+ coreModels "github.com/apache/incubator-devlake/core/models"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/plugins/webhook/api"
"github.com/apache/incubator-devlake/plugins/webhook/models"
@@ -58,7 +59,11 @@ func (p Webhook) GetTablesInfo() []dal.Tabler {
}
}
-func (p Webhook) MakeDataSourcePipelinePlanV200(connectionId uint64, _ []*plugin.BlueprintScopeV200, _ plugin.BlueprintSyncPolicy) (pp plugin.PipelinePlan, sc []plugin.Scope, err errors.Error) {
+func (p Webhook) MakeDataSourcePipelinePlanV200(
+ connectionId uint64,
+ _ []*coreModels.BlueprintScope,
+ _ coreModels.BlueprintSyncPolicy,
+) (pp coreModels.PipelinePlan, sc []plugin.Scope, err errors.Error) {
return api.MakeDataSourcePipelinePlanV200(connectionId)
}
diff --git a/backend/plugins/zentao/api/blueprint_V200_test.go b/backend/plugins/zentao/api/blueprint_V200_test.go
index 114908fc8..b5fe6b10f 100644
--- a/backend/plugins/zentao/api/blueprint_V200_test.go
+++ b/backend/plugins/zentao/api/blueprint_V200_test.go
@@ -20,6 +20,8 @@ package api
import (
"testing"
+ coreModels "github.com/apache/incubator-devlake/core/models"
+
"github.com/apache/incubator-devlake/core/models/common"
"github.com/apache/incubator-devlake/core/models/domainlayer"
"github.com/apache/incubator-devlake/core/models/domainlayer/ticket"
@@ -61,22 +63,22 @@ func TestMakeDataSourcePipelinePlanV200(t *testing.T) {
// Refresh Global Variables and set the sql mock
mockBasicRes(t)
- bs := &plugin.BlueprintScopeV200{
- Id: "1",
+ bs := &coreModels.BlueprintScope{
+ ScopeId: "1",
}
- /*bs2 := &plugin.BlueprintScopeV200{
+ /*bs2 := &coreModels.BlueprintScope{
Id: "product/1",
}*/
- bpScopes := make([]*plugin.BlueprintScopeV200, 0)
+ bpScopes := make([]*coreModels.BlueprintScope, 0)
bpScopes = append(bpScopes, bs)
- syncPolicy := &plugin.BlueprintSyncPolicy{}
+ syncPolicy := &coreModels.BlueprintSyncPolicy{}
- plan := make(plugin.PipelinePlan, len(bpScopes))
+ plan := make(coreModels.PipelinePlan, len(bpScopes))
plan, scopes, err := makePipelinePlanV200(nil, plan, bpScopes, connection, syncPolicy)
assert.Nil(t, err)
- expectPlan := plugin.PipelinePlan{
- plugin.PipelineStage{
+ expectPlan := coreModels.PipelinePlan{
+ coreModels.PipelineStage{
{
Plugin: "zentao",
Subtasks: []string{},
@@ -86,7 +88,7 @@ func TestMakeDataSourcePipelinePlanV200(t *testing.T) {
},
},
},
- /*plugin.PipelineStage{
+ /*coreModels.PipelineStage{
{
Plugin: "zentao",
Subtasks: []string{},
diff --git a/backend/plugins/zentao/api/blueprint_v200.go b/backend/plugins/zentao/api/blueprint_v200.go
index 6c772e631..6d53f0b76 100644
--- a/backend/plugins/zentao/api/blueprint_v200.go
+++ b/backend/plugins/zentao/api/blueprint_v200.go
@@ -21,6 +21,7 @@ import (
"time"
"github.com/apache/incubator-devlake/core/errors"
+ coreModels "github.com/apache/incubator-devlake/core/models"
"github.com/apache/incubator-devlake/core/models/domainlayer"
"github.com/apache/incubator-devlake/core/models/domainlayer/didgen"
"github.com/apache/incubator-devlake/core/models/domainlayer/ticket"
@@ -31,7 +32,12 @@ import (
"github.com/apache/incubator-devlake/plugins/zentao/tasks"
)
-func MakeDataSourcePipelinePlanV200(subtaskMetas []plugin.SubTaskMeta, connectionId uint64, bpScopes []*plugin.BlueprintScopeV200, syncPolicy *plugin.BlueprintSyncPolicy) (plugin.PipelinePlan, []plugin.Scope, errors.Error) {
+func MakeDataSourcePipelinePlanV200(
+ subtaskMetas []plugin.SubTaskMeta,
+ connectionId uint64,
+ bpScopes []*coreModels.BlueprintScope,
+ syncPolicy *coreModels.BlueprintSyncPolicy,
+) (coreModels.PipelinePlan, []plugin.Scope, errors.Error) {
// get the connection info for url
connection := &models.ZentaoConnection{}
err := connectionHelper.FirstById(connection, connectionId)
@@ -39,7 +45,7 @@ func MakeDataSourcePipelinePlanV200(subtaskMetas []plugin.SubTaskMeta, connectio
return nil, nil, err
}
- plan := make(plugin.PipelinePlan, len(bpScopes))
+ plan := make(coreModels.PipelinePlan, len(bpScopes))
plan, scopes, err := makePipelinePlanV200(subtaskMetas, plan, bpScopes, connection, syncPolicy)
if err != nil {
return nil, nil, err
@@ -50,16 +56,16 @@ func MakeDataSourcePipelinePlanV200(subtaskMetas []plugin.SubTaskMeta, connectio
func makePipelinePlanV200(
subtaskMetas []plugin.SubTaskMeta,
- plan plugin.PipelinePlan,
- bpScopes []*plugin.BlueprintScopeV200,
+ plan coreModels.PipelinePlan,
+ bpScopes []*coreModels.BlueprintScope,
connection *models.ZentaoConnection,
- syncPolicy *plugin.BlueprintSyncPolicy,
-) (plugin.PipelinePlan, []plugin.Scope, errors.Error) {
+ syncPolicy *coreModels.BlueprintSyncPolicy,
+) (coreModels.PipelinePlan, []plugin.Scope, errors.Error) {
domainScopes := make([]plugin.Scope, 0)
for i, bpScope := range bpScopes {
stage := plan[i]
if stage == nil {
- stage = plugin.PipelineStage{}
+ stage = coreModels.PipelineStage{}
}
// construct task options
op := &tasks.ZentaoOptions{
@@ -68,7 +74,7 @@ func makePipelinePlanV200(
var entities []string
- project, scopeConfig, err := projectScopeHelper.DbHelper().GetScopeAndConfig(connection.ID, bpScope.Id)
+ project, scopeConfig, err := projectScopeHelper.DbHelper().GetScopeAndConfig(connection.ID, bpScope.ScopeId)
if err != nil {
return nil, nil, err
}
@@ -117,7 +123,7 @@ func makePipelinePlanV200(
if err != nil {
return nil, nil, err
}
- stage = append(stage, &plugin.PipelineTask{
+ stage = append(stage, &coreModels.PipelineTask{
Plugin: "zentao",
Subtasks: subtasks,
Options: options,
diff --git a/backend/plugins/zentao/impl/impl.go b/backend/plugins/zentao/impl/impl.go
index d90652c1b..d316b9e90 100644
--- a/backend/plugins/zentao/impl/impl.go
+++ b/backend/plugins/zentao/impl/impl.go
@@ -23,6 +23,7 @@ import (
"github.com/apache/incubator-devlake/core/context"
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
+ coreModels "github.com/apache/incubator-devlake/core/models"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/core/runner"
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
@@ -284,7 +285,11 @@ func (p Zentao) ApiResources() map[string]map[string]plugin.ApiResourceHandler {
}
}
-func (p Zentao) MakeDataSourcePipelinePlanV200(connectionId uint64, scopes []*plugin.BlueprintScopeV200, syncPolicy plugin.BlueprintSyncPolicy) (pp plugin.PipelinePlan, sc []plugin.Scope, err errors.Error) {
+func (p Zentao) MakeDataSourcePipelinePlanV200(
+ connectionId uint64,
+ scopes []*coreModels.BlueprintScope,
+ syncPolicy coreModels.BlueprintSyncPolicy,
+) (pp coreModels.PipelinePlan, sc []plugin.Scope, err errors.Error) {
return api.MakeDataSourcePipelinePlanV200(p.SubTaskMetas(), connectionId, scopes, &syncPolicy)
}
diff --git a/backend/server/services/blueprint.go b/backend/server/services/blueprint.go
index 29aa62631..b644a807d 100644
--- a/backend/server/services/blueprint.go
+++ b/backend/server/services/blueprint.go
@@ -27,7 +27,6 @@ import (
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models"
- "github.com/apache/incubator-devlake/core/plugin"
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/impls/logruslog"
"github.com/robfig/cron/v3"
@@ -124,9 +123,6 @@ func GetBlueprintByProjectName(projectName string) (*models.Blueprint, errors.Er
}
func validateBlueprintAndMakePlan(blueprint *models.Blueprint) errors.Error {
- if len(blueprint.Settings) == 0 {
- blueprint.Settings = nil
- }
// validation
err := vld.Struct(blueprint)
if err != nil {
@@ -266,7 +262,7 @@ func ReloadBlueprints(c *cron.Cron) errors.Error {
}
func createPipelineByBlueprint(blueprint *models.Blueprint, skipCollectors bool) (*models.Pipeline, errors.Error) {
- var plan plugin.PipelinePlan
+ var plan models.PipelinePlan
var err errors.Error
if blueprint.Mode == models.BLUEPRINT_MODE_NORMAL {
plan, err = MakePlanForBlueprint(blueprint, skipCollectors)
@@ -310,68 +306,34 @@ func createPipelineByBlueprint(blueprint *models.Blueprint, skipCollectors bool)
}
// MakePlanForBlueprint generates pipeline plan by version
-func MakePlanForBlueprint(blueprint *models.Blueprint, skipCollectors bool) (plugin.PipelinePlan, errors.Error) {
- bpSettings := new(models.BlueprintSettings)
- err := errors.Convert(json.Unmarshal(blueprint.Settings, bpSettings))
- if err != nil {
- return nil, errors.Default.Wrap(err, fmt.Sprintf("settings:%s", string(blueprint.Settings)))
- }
-
- bpSyncPolicy := plugin.BlueprintSyncPolicy{}
- bpSyncPolicy.TimeAfter = bpSettings.TimeAfter
-
- var plan plugin.PipelinePlan
- switch bpSettings.Version {
- case "1.0.0":
- return nil, errors.BadInput.New("Blueprint v1.0.0 had been deprecated, please se v2.0.0 instead")
- case "2.0.0":
- // load project metric plugins and convert it to a map
- metrics := make(map[string]json.RawMessage)
- projectMetrics := make([]models.ProjectMetricSetting, 0)
- if blueprint.ProjectName != "" {
- err = db.All(&projectMetrics, dal.Where("project_name = ? AND enable = ?", blueprint.ProjectName, true))
- if err != nil {
- return nil, err
- }
- for _, projectMetric := range projectMetrics {
- metrics[projectMetric.PluginName] = json.RawMessage(projectMetric.PluginOption)
- }
- }
- plan, err = GeneratePlanJsonV200(blueprint.ProjectName, bpSyncPolicy, bpSettings, metrics, skipCollectors)
- default:
- return nil, errors.Default.New(fmt.Sprintf("unknown version of blueprint settings: %s", bpSettings.Version))
- }
- if err != nil {
- return nil, err
- }
- return WrapPipelinePlans(bpSettings.BeforePlan, plan, bpSettings.AfterPlan)
-}
-
-// WrapPipelinePlans merges multiple pipelines and append before and after pipeline
-func WrapPipelinePlans(beforePlanJson json.RawMessage, mainPlan plugin.PipelinePlan, afterPlanJson json.RawMessage) (plugin.PipelinePlan, errors.Error) {
- beforePipelinePlan := plugin.PipelinePlan{}
- afterPipelinePlan := plugin.PipelinePlan{}
-
- if beforePlanJson != nil {
- err := errors.Convert(json.Unmarshal(beforePlanJson, &beforePipelinePlan))
+func MakePlanForBlueprint(blueprint *models.Blueprint, skipCollectors bool) (models.PipelinePlan, errors.Error) {
+ bpSyncPolicy := models.BlueprintSyncPolicy{}
+ bpSyncPolicy.TimeAfter = blueprint.TimeAfter
+
+ var plan models.PipelinePlan
+ // load project metric plugins and convert it to a map
+ metrics := make(map[string]json.RawMessage)
+ projectMetrics := make([]models.ProjectMetricSetting, 0)
+ if blueprint.ProjectName != "" {
+ err := db.All(&projectMetrics, dal.Where("project_name = ? AND enable = ?", blueprint.ProjectName, true))
if err != nil {
return nil, err
}
- }
- if afterPlanJson != nil {
- err := errors.Convert(json.Unmarshal(afterPlanJson, &afterPipelinePlan))
- if err != nil {
- return nil, err
+ for _, projectMetric := range projectMetrics {
+ metrics[projectMetric.PluginName] = json.RawMessage(projectMetric.PluginOption)
}
}
-
- return SequencializePipelinePlans(beforePipelinePlan, mainPlan, afterPipelinePlan), nil
+ plan, err := GeneratePlanJsonV200(blueprint.ProjectName, bpSyncPolicy, blueprint.Connections, metrics, skipCollectors)
+ if err != nil {
+ return nil, err
+ }
+ return SequencializePipelinePlans(blueprint.BeforePlan, plan, blueprint.AfterPlan), nil
}
// ParallelizePipelinePlans merges multiple pipelines into one unified plan
// by assuming they can be executed in parallel
-func ParallelizePipelinePlans(plans ...plugin.PipelinePlan) plugin.PipelinePlan {
- merged := make(plugin.PipelinePlan, 0)
+func ParallelizePipelinePlans(plans ...models.PipelinePlan) models.PipelinePlan {
+ merged := make(models.PipelinePlan, 0)
// iterate all pipelineTasks and try to merge them into `merged`
for _, plan := range plans {
// add all stages from plan to merged
@@ -388,8 +350,8 @@ func ParallelizePipelinePlans(plans ...plugin.PipelinePlan) plugin.PipelinePlan
// SequencializePipelinePlans merges multiple pipelines into one unified plan
// by assuming they must be executed in sequencial order
-func SequencializePipelinePlans(plans ...plugin.PipelinePlan) plugin.PipelinePlan {
- merged := make(plugin.PipelinePlan, 0)
+func SequencializePipelinePlans(plans ...models.PipelinePlan) models.PipelinePlan {
+ merged := make(models.PipelinePlan, 0)
// iterate all pipelineTasks and try to merge them into `merged`
for _, plan := range plans {
merged = append(merged, plan...)
diff --git a/backend/server/services/blueprint_makeplan_v200.go b/backend/server/services/blueprint_makeplan_v200.go
index e557ddb72..175b6d3c1 100644
--- a/backend/server/services/blueprint_makeplan_v200.go
+++ b/backend/server/services/blueprint_makeplan_v200.go
@@ -23,36 +23,32 @@ import (
"strings"
"github.com/apache/incubator-devlake/core/errors"
- "github.com/apache/incubator-devlake/core/models"
+ coreModels "github.com/apache/incubator-devlake/core/models"
"github.com/apache/incubator-devlake/core/plugin"
)
// GeneratePlanJsonV200 generates pipeline plan according v2.0.0 definition
func GeneratePlanJsonV200(
projectName string,
- syncPolicy plugin.BlueprintSyncPolicy,
- sources *models.BlueprintSettings,
+ syncPolicy coreModels.BlueprintSyncPolicy,
+ connections []*coreModels.BlueprintConnection,
metrics map[string]json.RawMessage,
skipCollectors bool,
-) (plugin.PipelinePlan, errors.Error) {
- connections := make([]*plugin.BlueprintConnectionV200, 0)
- err := errors.Convert(json.Unmarshal(sources.Connections, &connections))
- if err != nil {
- return nil, err
- }
- // make plan for data-source plugins fist. generate plan for each
+) (coreModels.PipelinePlan, errors.Error) {
+ var err errors.Error
+ // make plan for data-source coreModels fist. generate plan for each
// connection, then merge them into one legitimate plan and collect the
// scopes produced by the data-source plugins
- sourcePlans := make([]plugin.PipelinePlan, len(connections))
+ sourcePlans := make([]coreModels.PipelinePlan, len(connections))
scopes := make([]plugin.Scope, 0, len(connections))
for i, connection := range connections {
- if len(connection.Scopes) == 0 && connection.Plugin != `webhook` && connection.Plugin != `jenkins` {
+ if len(connection.Scopes) == 0 && connection.PluginName != `webhook` && connection.PluginName != `jenkins` {
// webhook needn't scopes
// jenkins may upgrade from v100 and its scope is empty
return nil, errors.Default.New(fmt.Sprintf("connections[%d].scopes is empty", i))
}
- p, err := plugin.GetPlugin(connection.Plugin)
+ p, err := plugin.GetPlugin(connection.PluginName)
if err != nil {
return nil, err
}
@@ -71,7 +67,7 @@ func GeneratePlanJsonV200(
scopes = append(scopes, pluginScopes...)
} else {
return nil, errors.Default.New(
- fmt.Sprintf("plugin %s does not support DataSourcePluginBlueprintV200", connection.Plugin),
+ fmt.Sprintf("plugin %s does not support DataSourcePluginBlueprintV200", connection.PluginName),
)
}
}
@@ -96,7 +92,7 @@ func GeneratePlanJsonV200(
// remove gitextractor plugin if it's not the only task
for i, plan := range sourcePlans {
for j, stage := range plan {
- newStage := make(plugin.PipelineStage, 0, len(stage))
+ newStage := make(coreModels.PipelineStage, 0, len(stage))
hasGitExtractor := false
for _, task := range stage {
if task.Plugin != "gitextractor" {
@@ -113,7 +109,7 @@ func GeneratePlanJsonV200(
}
// make plans for metric plugins
- metricPlans := make([]plugin.PipelinePlan, len(metrics))
+ metricPlans := make([]coreModels.PipelinePlan, len(metrics))
i := 0
for metricPluginName, metricPluginOptJson := range metrics {
p, err := plugin.GetPlugin(metricPluginName)
@@ -136,7 +132,7 @@ func GeneratePlanJsonV200(
)
}
}
- var planForProjectMapping plugin.PipelinePlan
+ var planForProjectMapping coreModels.PipelinePlan
if projectName != "" {
p, err := plugin.GetPlugin("org")
if err != nil {
diff --git a/backend/server/services/blueprint_makeplan_v200_test.go b/backend/server/services/blueprint_makeplan_v200_test.go
index a0fc4e6c1..ce9c3cf8b 100644
--- a/backend/server/services/blueprint_makeplan_v200_test.go
+++ b/backend/server/services/blueprint_makeplan_v200_test.go
@@ -21,7 +21,7 @@ import (
"encoding/json"
"testing"
- "github.com/apache/incubator-devlake/core/models"
+ coreModels "github.com/apache/incubator-devlake/core/models"
"github.com/apache/incubator-devlake/core/models/domainlayer"
"github.com/apache/incubator-devlake/core/models/domainlayer/code"
"github.com/apache/incubator-devlake/core/models/domainlayer/ticket"
@@ -36,12 +36,12 @@ func TestMakePlanV200(t *testing.T) {
githubName := "TestMakePlanV200-github" // mimic github
// mock github plugin as a data source plugin
githubConnId := uint64(1)
- syncPolicy := plugin.BlueprintSyncPolicy{}
- githubScopes := []*plugin.BlueprintScopeV200{
- {Id: "", Name: "apache/incubator-devlake"},
- {Id: "", Name: "apache/incubator-devlake-website"},
+ syncPolicy := coreModels.BlueprintSyncPolicy{}
+ githubScopes := []*coreModels.BlueprintScope{
+ {ScopeId: "github:GithubRepo:1:123"},
+ {ScopeId: "github:GithubRepo:1:321"},
}
- githubOutputPlan := plugin.PipelinePlan{
+ githubOutputPlan := coreModels.PipelinePlan{
{
{Plugin: githubName, Options: map[string]interface{}{"name": "apache/incubator-devlake"}},
{Plugin: "gitextractor", Options: map[string]interface{}{"url": "http://gihub.com/apache/incubator-devlake.git"}},
@@ -60,7 +60,7 @@ func TestMakePlanV200(t *testing.T) {
// mock dora plugin as a metric plugin
doraName := "TestMakePlanV200-dora"
- doraOutputPlan := plugin.PipelinePlan{
+ doraOutputPlan := coreModels.PipelinePlan{
{
{Plugin: "refdiff", Subtasks: []string{"calculateProjectDeploymentCommitsDiff"}, Options: map[string]interface{}{"projectName": projectName}},
{Plugin: doraName},
@@ -71,7 +71,7 @@ func TestMakePlanV200(t *testing.T) {
// mock org plugin
org := new(mockplugin.CompositeProjectMapper)
- orgPlan := plugin.PipelinePlan{
+ orgPlan := coreModels.PipelinePlan{
{
{Plugin: "org", Subtasks: []string{"setProjectMapping"}, Options: map[string]interface{}{"projectMappings": []interface{}{tasks.NewProjectMapping(projectName, githubOutputScopes)}}},
},
@@ -79,7 +79,7 @@ func TestMakePlanV200(t *testing.T) {
org.On("MapProject", projectName, githubOutputScopes).Return(orgPlan, nil)
// expectation, establish expectation before any code being launch to avoid unwanted modification
- expectedPlan := make(plugin.PipelinePlan, 0)
+ expectedPlan := make(coreModels.PipelinePlan, 0)
expectedPlan = append(expectedPlan, orgPlan...)
expectedPlan = append(expectedPlan, githubOutputPlan...)
expectedPlan = append(expectedPlan, doraOutputPlan...)
@@ -90,18 +90,14 @@ func TestMakePlanV200(t *testing.T) {
plugin.RegisterPlugin("org", org)
// put them together and call GeneratePlanJsonV200
- connections, _ := json.Marshal([]*plugin.BlueprintConnectionV200{
- {Plugin: githubName, ConnectionId: githubConnId, Scopes: githubScopes},
- })
- sources := &models.BlueprintSettings{
- Version: "2.0.0",
- Connections: connections,
+ connections := []*coreModels.BlueprintConnection{
+ {PluginName: githubName, ConnectionId: githubConnId, Scopes: githubScopes},
}
metrics := map[string]json.RawMessage{
doraName: nil,
}
- plan, err := GeneratePlanJsonV200(projectName, syncPolicy, sources, metrics, false)
+ plan, err := GeneratePlanJsonV200(projectName, syncPolicy, connections, metrics, false)
assert.Nil(t, err)
assert.Equal(t, expectedPlan, plan)
diff --git a/backend/server/services/blueprint_test.go b/backend/server/services/blueprint_test.go
index d50680ad1..3a4fc95ff 100644
--- a/backend/server/services/blueprint_test.go
+++ b/backend/server/services/blueprint_test.go
@@ -18,15 +18,14 @@ limitations under the License.
package services
import (
- "encoding/json"
- "github.com/apache/incubator-devlake/core/plugin"
"testing"
+ coreModels "github.com/apache/incubator-devlake/core/models"
"github.com/stretchr/testify/assert"
)
func TestParallelizePipelineTasks(t *testing.T) {
- plan1 := plugin.PipelinePlan{
+ plan1 := coreModels.PipelinePlan{
{
{Plugin: "github"},
{Plugin: "gitlab"},
@@ -37,13 +36,13 @@ func TestParallelizePipelineTasks(t *testing.T) {
},
}
- plan2 := plugin.PipelinePlan{
+ plan2 := coreModels.PipelinePlan{
{
{Plugin: "jira"},
},
}
- plan3 := plugin.PipelinePlan{
+ plan3 := coreModels.PipelinePlan{
{
{Plugin: "jenkins"},
},
@@ -59,7 +58,7 @@ func TestParallelizePipelineTasks(t *testing.T) {
assert.Equal(t, plan2, ParallelizePipelinePlans(plan2))
assert.Equal(
t,
- plugin.PipelinePlan{
+ coreModels.PipelinePlan{
{
{Plugin: "github"},
{Plugin: "gitlab"},
@@ -74,7 +73,7 @@ func TestParallelizePipelineTasks(t *testing.T) {
)
assert.Equal(
t,
- plugin.PipelinePlan{
+ coreModels.PipelinePlan{
{
{Plugin: "github"},
{Plugin: "gitlab"},
@@ -93,45 +92,3 @@ func TestParallelizePipelineTasks(t *testing.T) {
ParallelizePipelinePlans(plan1, plan2, plan3),
)
}
-
-func TestWrapPipelinePlans(t *testing.T) {
- beforePlan2 := json.RawMessage(`[[{"plugin":"github"},{"plugin":"gitlab"}],[{"plugin":"gitextractor1"},{"plugin":"gitextractor2"}]]`)
-
- mainPlan := plugin.PipelinePlan{
- {
- {Plugin: "jira"},
- },
- }
-
- afterPlan2 := json.RawMessage(`[[{"plugin":"jenkins"}],[{"plugin":"jenkins"}]]`)
-
- result1, err1 := WrapPipelinePlans(nil, mainPlan, nil)
- assert.Nil(t, err1)
- assert.Equal(t, mainPlan, result1)
-
- result2, err2 := WrapPipelinePlans(beforePlan2, mainPlan, afterPlan2)
- assert.Nil(t, err2)
- assert.Equal(t, plugin.PipelinePlan{
- {
- {Plugin: "github"},
- {Plugin: "gitlab"},
- },
- {
- {Plugin: "gitextractor1"},
- {Plugin: "gitextractor2"},
- },
- {
- {Plugin: "jira"},
- },
- {
- {Plugin: "jenkins"},
- },
- {
- {Plugin: "jenkins"},
- },
- }, result2)
-
- result3, err3 := WrapPipelinePlans(json.RawMessage("[]"), mainPlan, json.RawMessage("[]"))
- assert.Nil(t, err3)
- assert.Equal(t, mainPlan, result3)
-}
diff --git a/backend/server/services/pipeline.go b/backend/server/services/pipeline.go
index 4dcccfd22..8e72901ed 100644
--- a/backend/server/services/pipeline.go
+++ b/backend/server/services/pipeline.go
@@ -29,7 +29,6 @@ import (
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models"
- "github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/core/utils"
"github.com/apache/incubator-devlake/helpers/dbhelper"
"github.com/apache/incubator-devlake/impls/logruslog"
@@ -509,7 +508,7 @@ func RerunPipeline(pipelineId uint64, task *models.Task) (tasks []*models.Task,
}
// create new task
rerunTask, err := CreateTask(&models.NewTask{
- PipelineTask: &plugin.PipelineTask{
+ PipelineTask: &models.PipelineTask{
Plugin: t.Plugin,
Subtasks: t.Subtasks,
Options: t.Options,
diff --git a/backend/server/services/remote/models/models.go b/backend/server/services/remote/models/models.go
index 17523cf78..00621301b 100644
--- a/backend/server/services/remote/models/models.go
+++ b/backend/server/services/remote/models/models.go
@@ -138,7 +138,7 @@ type DynamicDomainScope struct {
}
type PipelineData struct {
- Plan plugin.PipelinePlan `json:"plan"`
+ Plan models.PipelinePlan `json:"plan"`
Scopes []DynamicDomainScope `json:"scopes"`
}
diff --git a/backend/server/services/remote/plugin/plugin_extensions.go b/backend/server/services/remote/plugin/plugin_extensions.go
index b9aa47cb7..bd331f5db 100644
--- a/backend/server/services/remote/plugin/plugin_extensions.go
+++ b/backend/server/services/remote/plugin/plugin_extensions.go
@@ -21,6 +21,7 @@ import (
"encoding/json"
"github.com/apache/incubator-devlake/core/errors"
+ coreModels "github.com/apache/incubator-devlake/core/models"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/server/services/remote/bridge"
"github.com/apache/incubator-devlake/server/services/remote/models"
@@ -35,11 +36,15 @@ type (
}
)
-func (p remoteMetricPlugin) MakeMetricPluginPipelinePlanV200(projectName string, options json.RawMessage) (plugin.PipelinePlan, errors.Error) {
- return nil, errors.Internal.New("Remote metric plugins not supported")
+func (p remoteMetricPlugin) MakeMetricPluginPipelinePlanV200(projectName string, options json.RawMessage) (coreModels.PipelinePlan, errors.Error) {
+ return nil, errors.Internal.New("Remote metric coreModels not supported")
}
-func (p remoteDatasourcePlugin) MakeDataSourcePipelinePlanV200(connectionId uint64, bpScopes []*plugin.BlueprintScopeV200, syncPolicy plugin.BlueprintSyncPolicy) (plugin.PipelinePlan, []plugin.Scope, errors.Error) {
+func (p remoteDatasourcePlugin) MakeDataSourcePipelinePlanV200(
+ connectionId uint64,
+ bpScopes []*coreModels.BlueprintScope,
+ syncPolicy coreModels.BlueprintSyncPolicy,
+) (coreModels.PipelinePlan, []plugin.Scope, errors.Error) {
connection := p.connectionTabler.New()
err := p.connHelper.FirstById(connection, connectionId)
if err != nil {
@@ -49,7 +54,7 @@ func (p remoteDatasourcePlugin) MakeDataSourcePipelinePlanV200(connectionId uint
db := basicRes.GetDal()
var toolScopeConfigPairs = make([]interface{}, len(bpScopes))
for i, bpScope := range bpScopes {
- toolScope, scopeConfig, err := p.getScopeAndConfig(db, connectionId, bpScope.Id)
+ toolScope, scopeConfig, err := p.getScopeAndConfig(db, connectionId, bpScope.ScopeId)
if err != nil {
return nil, nil, err
}
diff --git a/backend/test/e2e/manual/azuredevops/azure_test.go b/backend/test/e2e/manual/azuredevops/azure_test.go
index 14eb0564f..d24c94c4e 100644
--- a/backend/test/e2e/manual/azuredevops/azure_test.go
+++ b/backend/test/e2e/manual/azuredevops/azure_test.go
@@ -110,17 +110,16 @@ func TestAzure(t *testing.T) {
require.Equal(t, repoConfig.ID, scope.ScopeConfigId)
}
// create bp_scopes from the scopes
- var bpScopes []*plugin.BlueprintScopeV200
+ var bpScopes []*models.BlueprintScope
for _, scope := range scopes {
- bpScopes = append(bpScopes, &plugin.BlueprintScopeV200{
- Id: scope.Id,
- Name: scope.Name,
+ bpScopes = append(bpScopes, &models.BlueprintScope{
+ ScopeId: scope.Id,
})
}
// create the bp
bp := client.CreateBasicBlueprintV2(connection.Name, &helper.BlueprintV2Config{
- Connection: &plugin.BlueprintConnectionV200{
- Plugin: azurePlugin,
+ Connection: &models.BlueprintConnection{
+ PluginName: azurePlugin,
ConnectionId: connection.ID,
Scopes: bpScopes,
},
diff --git a/backend/test/e2e/manual/gitlab/gitlab_test.go b/backend/test/e2e/manual/gitlab/gitlab_test.go
index e0a0d0b7e..eca43ba84 100644
--- a/backend/test/e2e/manual/gitlab/gitlab_test.go
+++ b/backend/test/e2e/manual/gitlab/gitlab_test.go
@@ -129,17 +129,16 @@ func TestGitlabPlugin(t *testing.T) {
})
projects := client.ListProjects()
require.Equal(t, 1, len(projects.Projects))
- var scopes []*plugin.BlueprintScopeV200
+ var scopes []*models.BlueprintScope
for _, scope := range listedScopes {
project := helper.Cast[pluginmodels.GitlabProject](scope.Scope)
- scopes = append(scopes, &plugin.BlueprintScopeV200{
- Id: fmt.Sprintf("%s", project.ScopeId()),
- Name: "blueprint-v200",
+ scopes = append(scopes, &models.BlueprintScope{
+ ScopeId: fmt.Sprintf("%s", project.ScopeId()),
})
}
bp := client.CreateBasicBlueprintV2(connection.Name, &helper.BlueprintV2Config{
- Connection: &plugin.BlueprintConnectionV200{
- Plugin: pluginName,
+ Connection: &models.BlueprintConnection{
+ PluginName: pluginName,
ConnectionId: connection.ID,
Scopes: scopes,
},
diff --git a/backend/test/e2e/manual/pagerduty/pagerduty_test.go b/backend/test/e2e/manual/pagerduty/pagerduty_test.go
index 9afce0ff8..c0dfc50d7 100644
--- a/backend/test/e2e/manual/pagerduty/pagerduty_test.go
+++ b/backend/test/e2e/manual/pagerduty/pagerduty_test.go
@@ -78,16 +78,15 @@ func TestPagerDutyPlugin(t *testing.T) {
require.True(t, len(createdScopes) > 0)
uniqueString := time.Now().Format(time.RFC3339)
outputProject := createProject(client, fmt.Sprintf("pagerduty-project-%s-%s", pluginName, uniqueString))
- var bpScopes []*plugin.BlueprintScopeV200
+ var bpScopes []*models.BlueprintScope
for _, scope := range createdScopes {
- bpScopes = append(bpScopes, &plugin.BlueprintScopeV200{
- Id: scope.Id,
- Name: fmt.Sprintf("pagerduty-blueprint-v200-%s", uniqueString),
+ bpScopes = append(bpScopes, &models.BlueprintScope{
+ ScopeId: scope.Id,
})
}
bp := client.CreateBasicBlueprintV2(connection.Name, &helper.BlueprintV2Config{
- Connection: &plugin.BlueprintConnectionV200{
- Plugin: pluginName,
+ Connection: &models.BlueprintConnection{
+ PluginName: pluginName,
ConnectionId: connection.ID,
Scopes: bpScopes,
},
diff --git a/backend/test/e2e/remote/helper.go b/backend/test/e2e/remote/helper.go
index 7ef6a1644..71e30d507 100644
--- a/backend/test/e2e/remote/helper.go
+++ b/backend/test/e2e/remote/helper.go
@@ -147,13 +147,12 @@ func CreateTestBlueprints(t *testing.T, client *helper.DevlakeClient, count int)
blueprint := client.CreateBasicBlueprintV2(
fmt.Sprintf("Test blueprint %d", i),
&helper.BlueprintV2Config{
- Connection: &plugin.BlueprintConnectionV200{
- Plugin: "fake",
+ Connection: &models.BlueprintConnection{
+ PluginName: "fake",
ConnectionId: connection.ID,
- Scopes: []*plugin.BlueprintScopeV200{
+ Scopes: []*models.BlueprintScope{
{
- Id: scope.Id,
- Name: "Test scope",
+ ScopeId: scope.Id,
},
},
},
diff --git a/backend/test/e2e/remote/python_plugin_test.go b/backend/test/e2e/remote/python_plugin_test.go
index 83b4a9fa7..d14f3ca57 100644
--- a/backend/test/e2e/remote/python_plugin_test.go
+++ b/backend/test/e2e/remote/python_plugin_test.go
@@ -153,7 +153,7 @@ func TestRunPipeline(t *testing.T) {
scope := CreateTestScope(client, scopeConfig, conn.ID)
pipeline := client.RunPipeline(models.NewPipeline{
Name: "remote_test",
- Plan: []plugin.PipelineStage{
+ Plan: []models.PipelineStage{
{
{
Plugin: PLUGIN_NAME,
diff --git a/backend/test/helper/api.go b/backend/test/helper/api.go
index 21aad1fd3..5caf817d0 100644
--- a/backend/test/helper/api.go
+++ b/backend/test/helper/api.go
@@ -27,7 +27,6 @@ import (
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models"
- "github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/server/api/blueprints"
apiProject "github.com/apache/incubator-devlake/server/api/project"
"github.com/apache/incubator-devlake/server/api/shared"
@@ -79,13 +78,6 @@ func (d *DevlakeClient) DeleteConnection(pluginName string, connectionId uint64)
// CreateBasicBlueprintV2 FIXME
func (d *DevlakeClient) CreateBasicBlueprintV2(name string, config *BlueprintV2Config) models.Blueprint {
- settings := &models.BlueprintSettings{
- Version: "2.0.0",
- TimeAfter: config.TimeAfter,
- Connections: ToJson([]*plugin.BlueprintConnectionV200{
- config.Connection,
- }),
- }
blueprint := models.Blueprint{
Name: name,
ProjectName: config.ProjectName,
@@ -96,7 +88,9 @@ func (d *DevlakeClient) CreateBasicBlueprintV2(name string, config *BlueprintV2C
IsManual: true,
SkipOnFail: config.SkipOnFail,
Labels: []string{"test-label"},
- Settings: ToJson(settings),
+ Connections: []*models.BlueprintConnection{
+ config.Connection,
+ },
}
d.testCtx.Helper()
blueprint = sendHttpRequest[models.Blueprint](d.testCtx, d.timeout, &testContext{
diff --git a/backend/test/helper/models.go b/backend/test/helper/models.go
index 03903212a..1596a3c65 100644
--- a/backend/test/helper/models.go
+++ b/backend/test/helper/models.go
@@ -23,7 +23,6 @@ import (
"github.com/apache/incubator-devlake/core/config"
"github.com/apache/incubator-devlake/core/models"
- "github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
)
@@ -62,7 +61,7 @@ type Connection struct {
}
type BlueprintV2Config struct {
- Connection *plugin.BlueprintConnectionV200
+ Connection *models.BlueprintConnection
TimeAfter *time.Time
SkipOnFail bool
ProjectName string