You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by wa...@apache.org on 2022/11/14 02:51:51 UTC
[incubator-devlake] branch release-v0.14 updated: fix(framework): simplify migration scripts (#3727)
This is an automated email from the ASF dual-hosted git repository.
warren pushed a commit to branch release-v0.14
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/release-v0.14 by this push:
new a1ee08f30 fix(framework): simplify migration scripts (#3727)
a1ee08f30 is described below
commit a1ee08f30a18dc1f32414a1add15e2b909f06d00
Author: Warren Chen <yi...@merico.dev>
AuthorDate: Mon Nov 14 10:51:47 2022 +0800
fix(framework): simplify migration scripts (#3727)
---
.../migrationscripts/20220903_encrypt_blueprint.go | 95 +++++++++-------------
.../migrationscripts/20220904_encrypt_pipeline.go | 86 +++++++-------------
2 files changed, 68 insertions(+), 113 deletions(-)
diff --git a/models/migrationscripts/20220903_encrypt_blueprint.go b/models/migrationscripts/20220903_encrypt_blueprint.go
index 0115aa397..a53d2b92d 100644
--- a/models/migrationscripts/20220903_encrypt_blueprint.go
+++ b/models/migrationscripts/20220903_encrypt_blueprint.go
@@ -19,7 +19,6 @@ package migrationscripts
import (
"context"
- "encoding/json"
"github.com/apache/incubator-devlake/config"
"github.com/apache/incubator-devlake/errors"
"github.com/apache/incubator-devlake/models/migrationscripts/archived"
@@ -27,96 +26,76 @@ import (
"gorm.io/gorm"
)
-type Blueprint0903Before struct {
- NewPlan string `json:"plan"`
- NewSettings string `json:"settings"`
-}
-
-func (Blueprint0903Before) TableName() string {
- return "_devlake_blueprints"
-}
-
-type Blueprint0903Temp struct {
- NewPlan string `json:"plan"`
- NewSettings string `json:"settings"`
- Name string `json:"name" validate:"required"`
- Mode string `json:"mode" gorm:"varchar(20)" validate:"required,oneof=NORMAL ADVANCED"`
- Plan json.RawMessage
- Enable bool `json:"enable"`
- //please check this https://crontab.guru/ for detail
- CronConfig string `json:"cronConfig" format:"* * * * *" example:"0 0 * * 1"`
- IsManual bool `json:"isManual"`
- Settings json.RawMessage
- archived.Model `swaggerignore:"true"`
-}
-
-func (Blueprint0903Temp) TableName() string {
- return "_devlake_blueprints"
-}
-
-type Blueprint0903TempAfter struct {
+type BlueprintEncryption0904 struct {
+ archived.Model
Plan string
+ RawPlan string
Settings string
- NewPlan string
- NewSettings string
+ RawSettings string
}
-func (Blueprint0903TempAfter) TableName() string {
+func (BlueprintEncryption0904) TableName() string {
return "_devlake_blueprints"
}
type encryptBLueprint struct{}
func (*encryptBLueprint) Up(ctx context.Context, db *gorm.DB) errors.Error {
- err := db.AutoMigrate(&Blueprint0903Before{})
+ c := config.GetConfig()
+ encKey := c.GetString(core.EncodeKeyEnvStr)
+ if encKey == "" {
+ return errors.BadInput.New("invalid encKey")
+ }
+
+ blueprint := &BlueprintEncryption0904{}
+ err := db.Migrator().RenameColumn(blueprint, "plan", "raw_plan")
if err != nil {
return errors.Convert(err)
}
- var result *gorm.DB
- var blueprintList []Blueprint0903Temp
- result = db.Find(&blueprintList)
- if result.Error != nil {
- return errors.Convert(result.Error)
+ err = db.Migrator().RenameColumn(blueprint, "settings", "raw_settings")
+ if err != nil {
+ return errors.Convert(err)
+ }
+ err = db.AutoMigrate(blueprint)
+ if err != nil {
+ return errors.Convert(err)
}
- // Encrypt all blueprints.plan&settings which had been stored before v0.14
- for _, v := range blueprintList {
- c := config.GetConfig()
- encKey := c.GetString(core.EncodeKeyEnvStr)
- if encKey == "" {
- return errors.BadInput.New("invalid encKey")
+ // Encrypt all blueprints.plan which had been stored before v0.14
+ cursor, err := db.Model(blueprint).Rows()
+ if err != nil {
+ return errors.Convert(err)
+ }
+ defer cursor.Close()
+
+ for cursor.Next() {
+ err = db.ScanRows(cursor, blueprint)
+ if err != nil {
+ return errors.Convert(err)
}
- v.NewPlan, err = core.Encrypt(encKey, string(v.Plan))
+ blueprint.Plan, err = core.Encrypt(encKey, blueprint.RawPlan)
if err != nil {
return errors.Convert(err)
}
- v.NewSettings, err = core.Encrypt(encKey, string(v.Settings))
+ blueprint.Settings, err = core.Encrypt(encKey, blueprint.RawSettings)
if err != nil {
return errors.Convert(err)
}
- err = errors.Convert(db.Save(&v).Error)
+ err = errors.Convert(db.Save(blueprint).Error)
if err != nil {
return errors.Convert(err)
}
}
- err = db.Migrator().DropColumn(&Blueprint0903Temp{}, "plan")
- if err != nil {
- return errors.Convert(err)
- }
- err = db.Migrator().DropColumn(&Blueprint0903Temp{}, "settings")
- if err != nil {
- return errors.Convert(err)
- }
- err = db.Migrator().RenameColumn(&Blueprint0903TempAfter{}, "new_plan", "plan")
+ err = db.Migrator().DropColumn(blueprint, "raw_plan")
if err != nil {
return errors.Convert(err)
}
- err = db.Migrator().RenameColumn(&Blueprint0903TempAfter{}, "new_settings", "settings")
+ err = db.Migrator().DropColumn(blueprint, "raw_settings")
if err != nil {
return errors.Convert(err)
}
- _ = db.Find(&blueprintList)
+ _ = db.First(blueprint)
return nil
}
diff --git a/models/migrationscripts/20220904_encrypt_pipeline.go b/models/migrationscripts/20220904_encrypt_pipeline.go
index 6b77a8beb..4352f9eb3 100644
--- a/models/migrationscripts/20220904_encrypt_pipeline.go
+++ b/models/migrationscripts/20220904_encrypt_pipeline.go
@@ -19,95 +19,71 @@ package migrationscripts
import (
"context"
+ "github.com/apache/incubator-devlake/models/migrationscripts/archived"
+
"github.com/apache/incubator-devlake/config"
"github.com/apache/incubator-devlake/plugins/core"
- "time"
"github.com/apache/incubator-devlake/errors"
- "github.com/apache/incubator-devlake/models/common"
- "gorm.io/datatypes"
"gorm.io/gorm"
)
-type Pipeline0904TempBefore struct {
- NewPlan string
-}
-
-func (Pipeline0904TempBefore) TableName() string {
- return "_devlake_pipelines"
-}
-
-type Pipeline0904Temp struct {
- common.Model
- Name string `json:"name" gorm:"index"`
- BlueprintId uint64 `json:"blueprintId"`
- NewPlan string `json:"plan" encrypt:"yes"`
- TotalTasks int `json:"totalTasks"`
- FinishedTasks int `json:"finishedTasks"`
- BeganAt *time.Time `json:"beganAt"`
- FinishedAt *time.Time `json:"finishedAt" gorm:"index"`
- Status string `json:"status"`
- Message string `json:"message"`
- SpentSeconds int `json:"spentSeconds"`
- Stage int `json:"stage"`
- Plan datatypes.JSON
-}
-
-func (Pipeline0904Temp) TableName() string {
- return "_devlake_pipelines"
-}
-
-type Pipeline0904TempAfter struct {
- NewPlan string
+type PipelineEncryption0904 struct {
+ archived.Model
Plan string
+ RawPlan string
}
-func (Pipeline0904TempAfter) TableName() string {
+func (PipelineEncryption0904) TableName() string {
return "_devlake_pipelines"
}
type encryptPipeline struct{}
func (*encryptPipeline) Up(ctx context.Context, db *gorm.DB) errors.Error {
- err := db.AutoMigrate(&Pipeline0904TempBefore{})
+ c := config.GetConfig()
+ encKey := c.GetString(core.EncodeKeyEnvStr)
+ if encKey == "" {
+ return errors.BadInput.New("invalid encKey")
+ }
+
+ pipeline := &PipelineEncryption0904{}
+ err := db.Migrator().RenameColumn(pipeline, "plan", "raw_plan")
+ if err != nil {
+ return errors.Convert(err)
+ }
+ err = db.AutoMigrate(pipeline)
if err != nil {
return errors.Convert(err)
}
- var result *gorm.DB
- var pipelineList []Pipeline0904Temp
- result = db.Find(&pipelineList)
- if result.Error != nil {
- return errors.Convert(result.Error)
+ // Encrypt all pipelines.plan which had been stored before v0.14
+ cursor, err := db.Model(pipeline).Rows()
+ if err != nil {
+ return errors.Convert(err)
}
+ defer cursor.Close()
- // Encrypt all pipelines.plan&settings which had been stored before v0.14
- for _, v := range pipelineList {
- c := config.GetConfig()
- encKey := c.GetString(core.EncodeKeyEnvStr)
- if encKey == "" {
- return errors.BadInput.New("invalid encKey")
+ for cursor.Next() {
+ err = db.ScanRows(cursor, pipeline)
+ if err != nil {
+ return errors.Convert(err)
}
- v.NewPlan, err = core.Encrypt(encKey, string(v.Plan))
+ pipeline.Plan, err = core.Encrypt(encKey, pipeline.RawPlan)
if err != nil {
return errors.Convert(err)
}
-
- err = errors.Convert(db.Save(&v).Error)
+ err = errors.Convert(db.Save(pipeline).Error)
if err != nil {
return errors.Convert(err)
}
}
- err = db.Migrator().DropColumn(&Pipeline0904Temp{}, "plan")
- if err != nil {
- return errors.Convert(err)
- }
- err = db.Migrator().RenameColumn(&Pipeline0904TempAfter{}, "new_plan", "plan")
+ err = db.Migrator().DropColumn(pipeline, "raw_plan")
if err != nil {
return errors.Convert(err)
}
- _ = db.Find(&pipelineList)
+ _ = db.First(pipeline)
return nil
}