You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by kl...@apache.org on 2022/06/20 06:52:24 UTC
[incubator-devlake] branch main updated: feat: add `mode` to blueprint entity (#2228)
This is an automated email from the ASF dual-hosted git repository.
klesh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new a7dbf425 feat: add `mode` to blueprint entity (#2228)
a7dbf425 is described below
commit a7dbf4253fc1d8b7af2908e2af78b5f7312da29f
Author: Klesh Wong <zh...@merico.dev>
AuthorDate: Mon Jun 20 14:52:20 2022 +0800
feat: add `mode` to blueprint entity (#2228)
* feat: add `mode` to blueprint entity
* feat: schedule blueprints with cron_config != MANUAL
* feat: disable NORMAL mode
* feat: separate Manual option from cronConfig
---
api/blueprints/blueprints.go | 10 +--
models/blueprint.go | 9 +-
models/migrationscripts/register.go | 2 +-
.../updateSchemas20220616.go} | 36 +++++---
services/blueprint.go | 99 ++++++++++------------
services/init.go | 3 +
6 files changed, 84 insertions(+), 75 deletions(-)
diff --git a/api/blueprints/blueprints.go b/api/blueprints/blueprints.go
index 28463cb3..4bd5bb93 100644
--- a/api/blueprints/blueprints.go
+++ b/api/blueprints/blueprints.go
@@ -164,17 +164,13 @@ func Patch(c *gin.Context) {
shared.ApiOutputError(c, err, http.StatusBadRequest)
return
}
- blueprint, err := services.GetBlueprint(id)
- if err != nil {
- shared.ApiOutputError(c, err, http.StatusBadRequest)
- return
- }
- err = c.ShouldBind(blueprint)
+ var body map[string]interface{}
+ err = c.ShouldBind(&body)
if err != nil {
shared.ApiOutputError(c, err, http.StatusBadRequest)
return
}
- err = services.UpdateBlueprint(blueprint)
+ blueprint, err := services.PatchBlueprint(id, body)
if err != nil {
shared.ApiOutputError(c, err, http.StatusBadRequest)
return
diff --git a/models/blueprint.go b/models/blueprint.go
index 1368645b..9ae9052e 100644
--- a/models/blueprint.go
+++ b/models/blueprint.go
@@ -22,11 +22,16 @@ import (
"gorm.io/datatypes"
)
+const BLUEPRINT_MODE_NORMAL = "NORMAL"
+const BLUEPRINT_MODE_ADVANCED = "ADVANCED"
+
type Blueprint struct {
Name string `json:"name" validate:"required"`
- Tasks datatypes.JSON `json:"tasks" validate:"required"`
+ Mode string `json:"mode" gorm:"varchar(20)" validate:"required,oneof=NORMAL ADVANCED"`
+ Tasks datatypes.JSON `json:"tasks"`
Enable bool `json:"enable"`
- CronConfig string `json:"cronConfig" validate:"required"`
+ CronConfig string `json:"cronConfig"`
+ IsManual bool `json:"isManual"`
common.Model
}
diff --git a/models/migrationscripts/register.go b/models/migrationscripts/register.go
index d9adfcbb..3f1c1dec 100644
--- a/models/migrationscripts/register.go
+++ b/models/migrationscripts/register.go
@@ -27,6 +27,6 @@ func All() []migration.Script {
new(updateSchemas20220513), new(updateSchemas20220524), new(updateSchemas20220526),
new(updateSchemas20220527), new(updateSchemas20220528), new(updateSchemas20220601),
new(updateSchemas20220602), new(updateSchemas20220612), new(updateSchemas20220613),
- new(updateSchemas20220614), new(updateSchemas2022061402),
+ new(updateSchemas20220614), new(updateSchemas2022061402), new(updateSchemas20220616),
}
}
diff --git a/models/blueprint.go b/models/migrationscripts/updateSchemas20220616.go
similarity index 56%
copy from models/blueprint.go
copy to models/migrationscripts/updateSchemas20220616.go
index 1368645b..dcb84c32 100644
--- a/models/blueprint.go
+++ b/models/migrationscripts/updateSchemas20220616.go
@@ -15,21 +15,37 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
-package models
+package migrationscripts
import (
- "github.com/apache/incubator-devlake/models/common"
- "gorm.io/datatypes"
+ "context"
+
+ "gorm.io/gorm"
)
-type Blueprint struct {
- Name string `json:"name" validate:"required"`
- Tasks datatypes.JSON `json:"tasks" validate:"required"`
- Enable bool `json:"enable"`
- CronConfig string `json:"cronConfig" validate:"required"`
- common.Model
+type Blueprint20220616 struct {
+ Mode string `json:"mode" gorm:"varchar(20)" validate:"required,oneof=NORMAL ADVANCED"`
+ IsManual bool `json:"isManual"`
}
-func (Blueprint) TableName() string {
+func (Blueprint20220616) TableName() string {
return "_devlake_blueprints"
}
+
+type updateSchemas20220616 struct{}
+
+func (*updateSchemas20220616) Up(ctx context.Context, db *gorm.DB) error {
+ err := db.Migrator().AutoMigrate(&Blueprint20220616{})
+ if err != nil {
+ return err
+ }
+ return nil
+}
+
+func (*updateSchemas20220616) Version() uint64 {
+ return 20220616110537
+}
+
+func (*updateSchemas20220616) Name() string {
+ return "add mode field to blueprint"
+}
diff --git a/services/blueprint.go b/services/blueprint.go
index 561a4b72..4055c739 100644
--- a/services/blueprint.go
+++ b/services/blueprint.go
@@ -25,6 +25,7 @@ import (
"github.com/apache/incubator-devlake/logger"
"github.com/apache/incubator-devlake/models"
"github.com/go-playground/validator/v10"
+ "github.com/mitchellh/mapstructure"
"github.com/robfig/cron/v3"
"gorm.io/gorm"
)
@@ -89,86 +90,71 @@ func GetBlueprint(blueprintId uint64) (*models.Blueprint, error) {
return blueprint, nil
}
-/*
-func ModifyBlueprint(newBlueprint *models.EditBlueprint) (*models.Blueprint, error) {
- _, err := cron.ParseStandard(newBlueprint.CronConfig)
- if err != nil {
- return nil, fmt.Errorf("invalid cronConfig: %w", err)
+func validateBlueprint(blueprint *models.Blueprint) error {
+ // TODO: implement NORMAL mode
+ if blueprint.Mode == models.BLUEPRINT_MODE_NORMAL {
+ return fmt.Errorf("NORMAL mode is yet to be implemented")
}
-
- blueprint := models.Blueprint{}
- err = db.Model(&models.Blueprint{}).
- Where("id = ?", newBlueprint.BlueprintId).Limit(1).Find(&blueprint).Error
+ // validation
+ err := vld.Struct(blueprint)
if err != nil {
- return nil, err
- }
- // update cronConfig
- if newBlueprint.CronConfig != "" {
- blueprint.CronConfig = newBlueprint.CronConfig
+ return err
}
- // update tasks
- if newBlueprint.Tasks != nil {
- blueprint.Tasks, err = json.Marshal(newBlueprint.Tasks)
+ if blueprint.CronConfig != "" {
+ _, err = cron.ParseStandard(blueprint.CronConfig)
if err != nil {
- return nil, err
+ return fmt.Errorf("invalid cronConfig: %w", err)
}
+ } else if blueprint.IsManual == false {
+ return fmt.Errorf("cronConfig is required for Automated blueprint")
}
- blueprint.Enable = newBlueprint.Enable
-
- err = db.Model(&models.Blueprint{}).
- Clauses(clause.OnConflict{UpdateAll: true}).Create(&blueprint).Error
- if err != nil {
- return nil, errors.InternalError
- }
- err = ReloadBlueprints(cronManager)
- if err != nil {
- return nil, errors.InternalError
+ if blueprint.Mode == models.BLUEPRINT_MODE_ADVANCED {
+ tasks := make([][]models.NewTask, 0)
+ err = json.Unmarshal(blueprint.Tasks, &tasks)
+ if err != nil {
+ return fmt.Errorf("invalid tasks: %w", err)
+ }
+ // tasks should not be empty
+ if len(tasks) == 0 || len(tasks[0]) == 0 {
+ return fmt.Errorf("empty tasks")
+ }
}
- return &blueprint, nil
+ // TODO: validate each of every task object
+ return nil
}
-*/
-func validateBlueprint(blueprint *models.Blueprint) error {
- // validation
- err := vld.Struct(blueprint)
- if err != nil {
- return err
- }
- _, err = cron.ParseStandard(blueprint.CronConfig)
+func PatchBlueprint(id uint64, body map[string]interface{}) (*models.Blueprint, error) {
+ // load record from db
+ blueprint, err := GetBlueprint(id)
if err != nil {
- return fmt.Errorf("invalid cronConfig: %w", err)
+ return nil, err
}
- tasks := make([][]models.NewTask, 0)
- err = json.Unmarshal(blueprint.Tasks, &tasks)
+ originMode := blueprint.Mode
+ err = mapstructure.Decode(body, blueprint)
if err != nil {
- return fmt.Errorf("invalid tasks: %w", err)
+ return nil, err
}
- // tasks should not be empty
- if len(tasks) == 0 || len(tasks[0]) == 0 {
- return fmt.Errorf("empty tasks")
+ // make sure mode is not being update
+ if originMode != blueprint.Mode {
+ return nil, fmt.Errorf("mode is not updatable")
}
- // TODO: validate each of every task object
- return nil
-}
-
-func UpdateBlueprint(blueprint *models.Blueprint) error {
// validation
- err := validateBlueprint(blueprint)
+ err = validateBlueprint(blueprint)
if err != nil {
- return err
+ return nil, err
}
// save
err = db.Save(blueprint).Error
if err != nil {
- return errors.InternalError
+ return nil, errors.InternalError
}
// reload schedule
err = ReloadBlueprints(cronManager)
if err != nil {
- return errors.InternalError
+ return nil, errors.InternalError
}
// done
- return nil
+ return blueprint, nil
}
func DeleteBlueprint(id uint64) error {
@@ -185,7 +171,9 @@ func DeleteBlueprint(id uint64) error {
func ReloadBlueprints(c *cron.Cron) error {
blueprints := make([]*models.Blueprint, 0)
- err := db.Model(&models.Blueprint{}).Where("enable = ?", true).Find(&blueprints).Error
+ err := db.Model(&models.Blueprint{}).
+ Where("enable = ? AND is_manual = ?", true, false).
+ Find(&blueprints).Error
if err != nil {
panic(err)
}
@@ -227,5 +215,6 @@ func ReloadBlueprints(c *cron.Cron) error {
if len(blueprints) > 0 {
c.Start()
}
+ log.Info("total %d blueprints were scheduled", len(blueprints))
return nil
}
diff --git a/services/init.go b/services/init.go
index 4843e887..dd70c17b 100644
--- a/services/init.go
+++ b/services/init.go
@@ -21,6 +21,7 @@ import (
"context"
"github.com/apache/incubator-devlake/models/migrationscripts"
+ "github.com/apache/incubator-devlake/plugins/core"
"time"
@@ -36,10 +37,12 @@ import (
var cfg *viper.Viper
var db *gorm.DB
var cronManager *cron.Cron
+var log core.Logger
func init() {
var err error
cfg = config.GetConfig()
+ log = logger.Global
db, err = runner.NewGormDb(cfg, logger.Global.Nested("db"))
location := cron.WithLocation(time.UTC)
cronManager = cron.New(location)