You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by GitBox <gi...@apache.org> on 2022/11/14 03:48:49 UTC
[GitHub] [incubator-devlake] warren830 opened a new pull request, #3728: fix(framework): fix migration scripts
warren830 opened a new pull request, #3728:
URL: https://github.com/apache/incubator-devlake/pull/3728
# Summary
In pr: https://github.com/apache/incubator-devlake/pull/3727, we fixed bugs for #3713 but just a hotfix for v0.14, this pr is to resolve the problem in main branch , in addition, for all fields with `datatypes.JSON`, we changed to `json.RawMessage` as pg would treat these two types with difference.
Lastly, we tested through the following steps:
1. deploy devlake v0.12 by docker-compose
2. created several blueprints and pipelines
3. shutdown v0.12 devlake
4. deploy 0.14 devlake and shutdown
5. deploy new code
6. check blueprints and pipelines through config-ui
### Does this close any open issues?
Closes #3713
### Screenshots
Include any relevant screenshots here.
### Other Information
Any other information that is important to this PR.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@devlake.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [incubator-devlake] warren830 commented on pull request #3728: fix(framework): fix migration scripts
Posted by GitBox <gi...@apache.org>.
warren830 commented on PR #3728:
URL: https://github.com/apache/incubator-devlake/pull/3728#issuecomment-1313182696
> `TransformColumns` is packed with rename column operations. I think we don't need to rename it manually. The doc of `TransformColumns` is not very clear about that, we should provide an example to help developer understand the usage
Fixed, I will submit a pr for that
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@devlake.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [incubator-devlake] warren830 commented on a diff in pull request #3728: fix(framework): fix migration scripts
Posted by GitBox <gi...@apache.org>.
warren830 commented on code in PR #3728:
URL: https://github.com/apache/incubator-devlake/pull/3728#discussion_r1021151408
##########
models/migrationscripts/20220903_encrypt_blueprint.go:
##########
@@ -30,63 +28,59 @@ var _ core.MigrationScript = (*encryptBlueprint)(nil)
type encryptBlueprint struct{}
-type blueprint20220903Before struct {
- Name string `json:"name" validate:"required"`
- Mode string `json:"mode" gorm:"varchar(20)" validate:"required,oneof=NORMAL ADVANCED"`
- Plan json.RawMessage `json:"plan"`
- Enable bool `json:"enable"`
- CronConfig string `json:"cronConfig" format:"* * * * *" example:"0 0 * * 1"`
- IsManual bool `json:"isManual"`
- Settings json.RawMessage `json:"settings" swaggertype:"array,string" example:"please check api: /blueprints/<PLUGIN_NAME>/blueprint-setting"`
- archived.Model `swaggerignore:"true"`
+type BlueprintEncryption0904Before struct {
+ archived.Model
+ Plan string
+ Settings string
+}
+
+func (BlueprintEncryption0904Before) TableName() string {
Review Comment:
fixed
##########
models/migrationscripts/20220903_encrypt_blueprint.go:
##########
@@ -30,63 +28,59 @@ var _ core.MigrationScript = (*encryptBlueprint)(nil)
type encryptBlueprint struct{}
-type blueprint20220903Before struct {
- Name string `json:"name" validate:"required"`
- Mode string `json:"mode" gorm:"varchar(20)" validate:"required,oneof=NORMAL ADVANCED"`
- Plan json.RawMessage `json:"plan"`
- Enable bool `json:"enable"`
- CronConfig string `json:"cronConfig" format:"* * * * *" example:"0 0 * * 1"`
- IsManual bool `json:"isManual"`
- Settings json.RawMessage `json:"settings" swaggertype:"array,string" example:"please check api: /blueprints/<PLUGIN_NAME>/blueprint-setting"`
- archived.Model `swaggerignore:"true"`
+type BlueprintEncryption0904Before struct {
+ archived.Model
+ Plan string
+ Settings string
+}
+
+func (BlueprintEncryption0904Before) TableName() string {
+ return "_devlake_blueprints"
+}
+
+type BlueprintEncryption0904After struct {
Review Comment:
fixed
##########
models/migrationscripts/20220903_encrypt_blueprint.go:
##########
@@ -30,63 +28,59 @@ var _ core.MigrationScript = (*encryptBlueprint)(nil)
type encryptBlueprint struct{}
-type blueprint20220903Before struct {
- Name string `json:"name" validate:"required"`
- Mode string `json:"mode" gorm:"varchar(20)" validate:"required,oneof=NORMAL ADVANCED"`
- Plan json.RawMessage `json:"plan"`
- Enable bool `json:"enable"`
- CronConfig string `json:"cronConfig" format:"* * * * *" example:"0 0 * * 1"`
- IsManual bool `json:"isManual"`
- Settings json.RawMessage `json:"settings" swaggertype:"array,string" example:"please check api: /blueprints/<PLUGIN_NAME>/blueprint-setting"`
- archived.Model `swaggerignore:"true"`
+type BlueprintEncryption0904Before struct {
+ archived.Model
+ Plan string
+ Settings string
+}
+
+func (BlueprintEncryption0904Before) TableName() string {
+ return "_devlake_blueprints"
+}
+
+type BlueprintEncryption0904After struct {
+ archived.Model
+ Plan string
+ Settings string
}
-type blueprint20220903After struct {
- /* unchanged part */
- Name string `json:"name" validate:"required"`
- Mode string `json:"mode" gorm:"varchar(20)" validate:"required,oneof=NORMAL ADVANCED"`
- Enable bool `json:"enable"`
- CronConfig string `json:"cronConfig" format:"* * * * *" example:"0 0 * * 1"`
- IsManual bool `json:"isManual"`
- archived.Model `swaggerignore:"true"`
- /* changed part */
- Plan string `json:"plan"`
- Settings string `json:"settings"`
+func (BlueprintEncryption0904After) TableName() string {
+ return "_devlake_blueprints"
}
func (script *encryptBlueprint) Up(basicRes core.BasicRes) errors.Error {
+ db := basicRes.GetDal()
encKey := basicRes.GetConfig(core.EncodeKeyEnvStr)
if encKey == "" {
return errors.BadInput.New("invalid encKey")
}
-
- return migrationhelper.TransformTable(
+ blueprint := &BlueprintEncryption0904After{}
+ err := migrationhelper.TransformColumns(
basicRes,
script,
"_devlake_blueprints",
- func(s *blueprint20220903Before) (*blueprint20220903After, errors.Error) {
- encryptedPlan, err := core.Encrypt(encKey, string(s.Plan))
+ []string{"plan", "settings"},
+ func(src *BlueprintEncryption0904Before) (*BlueprintEncryption0904After, errors.Error) {
+ plan, err := core.Encrypt(encKey, src.Plan)
if err != nil {
return nil, err
}
- encryptedSettings, err := core.Encrypt(encKey, string(s.Settings))
+ settings, err := core.Encrypt(encKey, src.Settings)
if err != nil {
return nil, err
}
-
- dst := &blueprint20220903After{
- Name: s.Name,
- Mode: s.Mode,
- Enable: s.Enable,
- CronConfig: s.CronConfig,
- IsManual: s.IsManual,
- Model: archived.Model{ID: s.ID},
- Plan: encryptedPlan,
- Settings: encryptedSettings,
- }
- return dst, nil
+ return &BlueprintEncryption0904After{
+ Model: src.Model,
+ Plan: plan,
+ Settings: settings,
+ }, nil
},
)
+ if err != nil {
+ return err
+ }
+ _ = db.First(blueprint)
Review Comment:
fixed
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@devlake.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [incubator-devlake] klesh merged pull request #3728: fix(framework): fix migration scripts
Posted by GitBox <gi...@apache.org>.
klesh merged PR #3728:
URL: https://github.com/apache/incubator-devlake/pull/3728
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@devlake.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [incubator-devlake] klesh commented on a diff in pull request #3728: fix(framework): fix migration scripts
Posted by GitBox <gi...@apache.org>.
klesh commented on code in PR #3728:
URL: https://github.com/apache/incubator-devlake/pull/3728#discussion_r1021138533
##########
models/migrationscripts/20220903_encrypt_blueprint.go:
##########
@@ -30,63 +28,59 @@ var _ core.MigrationScript = (*encryptBlueprint)(nil)
type encryptBlueprint struct{}
-type blueprint20220903Before struct {
- Name string `json:"name" validate:"required"`
- Mode string `json:"mode" gorm:"varchar(20)" validate:"required,oneof=NORMAL ADVANCED"`
- Plan json.RawMessage `json:"plan"`
- Enable bool `json:"enable"`
- CronConfig string `json:"cronConfig" format:"* * * * *" example:"0 0 * * 1"`
- IsManual bool `json:"isManual"`
- Settings json.RawMessage `json:"settings" swaggertype:"array,string" example:"please check api: /blueprints/<PLUGIN_NAME>/blueprint-setting"`
- archived.Model `swaggerignore:"true"`
+type BlueprintEncryption0904Before struct {
+ archived.Model
+ Plan string
+ Settings string
+}
+
+func (BlueprintEncryption0904Before) TableName() string {
+ return "_devlake_blueprints"
+}
+
+type BlueprintEncryption0904After struct {
Review Comment:
Looks like they are identical, we can use one struct instead
##########
models/migrationscripts/20220903_encrypt_blueprint.go:
##########
@@ -30,63 +28,59 @@ var _ core.MigrationScript = (*encryptBlueprint)(nil)
type encryptBlueprint struct{}
-type blueprint20220903Before struct {
- Name string `json:"name" validate:"required"`
- Mode string `json:"mode" gorm:"varchar(20)" validate:"required,oneof=NORMAL ADVANCED"`
- Plan json.RawMessage `json:"plan"`
- Enable bool `json:"enable"`
- CronConfig string `json:"cronConfig" format:"* * * * *" example:"0 0 * * 1"`
- IsManual bool `json:"isManual"`
- Settings json.RawMessage `json:"settings" swaggertype:"array,string" example:"please check api: /blueprints/<PLUGIN_NAME>/blueprint-setting"`
- archived.Model `swaggerignore:"true"`
+type BlueprintEncryption0904Before struct {
+ archived.Model
+ Plan string
+ Settings string
+}
+
+func (BlueprintEncryption0904Before) TableName() string {
Review Comment:
No needed
##########
models/migrationscripts/20220903_encrypt_blueprint.go:
##########
@@ -30,63 +28,59 @@ var _ core.MigrationScript = (*encryptBlueprint)(nil)
type encryptBlueprint struct{}
-type blueprint20220903Before struct {
- Name string `json:"name" validate:"required"`
- Mode string `json:"mode" gorm:"varchar(20)" validate:"required,oneof=NORMAL ADVANCED"`
- Plan json.RawMessage `json:"plan"`
- Enable bool `json:"enable"`
- CronConfig string `json:"cronConfig" format:"* * * * *" example:"0 0 * * 1"`
- IsManual bool `json:"isManual"`
- Settings json.RawMessage `json:"settings" swaggertype:"array,string" example:"please check api: /blueprints/<PLUGIN_NAME>/blueprint-setting"`
- archived.Model `swaggerignore:"true"`
+type BlueprintEncryption0904Before struct {
+ archived.Model
+ Plan string
+ Settings string
+}
+
+func (BlueprintEncryption0904Before) TableName() string {
+ return "_devlake_blueprints"
+}
+
+type BlueprintEncryption0904After struct {
+ archived.Model
+ Plan string
+ Settings string
}
-type blueprint20220903After struct {
- /* unchanged part */
- Name string `json:"name" validate:"required"`
- Mode string `json:"mode" gorm:"varchar(20)" validate:"required,oneof=NORMAL ADVANCED"`
- Enable bool `json:"enable"`
- CronConfig string `json:"cronConfig" format:"* * * * *" example:"0 0 * * 1"`
- IsManual bool `json:"isManual"`
- archived.Model `swaggerignore:"true"`
- /* changed part */
- Plan string `json:"plan"`
- Settings string `json:"settings"`
+func (BlueprintEncryption0904After) TableName() string {
+ return "_devlake_blueprints"
}
func (script *encryptBlueprint) Up(basicRes core.BasicRes) errors.Error {
+ db := basicRes.GetDal()
encKey := basicRes.GetConfig(core.EncodeKeyEnvStr)
if encKey == "" {
return errors.BadInput.New("invalid encKey")
}
-
- return migrationhelper.TransformTable(
+ blueprint := &BlueprintEncryption0904After{}
+ err := migrationhelper.TransformColumns(
basicRes,
script,
"_devlake_blueprints",
- func(s *blueprint20220903Before) (*blueprint20220903After, errors.Error) {
- encryptedPlan, err := core.Encrypt(encKey, string(s.Plan))
+ []string{"plan", "settings"},
+ func(src *BlueprintEncryption0904Before) (*BlueprintEncryption0904After, errors.Error) {
+ plan, err := core.Encrypt(encKey, src.Plan)
if err != nil {
return nil, err
}
- encryptedSettings, err := core.Encrypt(encKey, string(s.Settings))
+ settings, err := core.Encrypt(encKey, src.Settings)
if err != nil {
return nil, err
}
-
- dst := &blueprint20220903After{
- Name: s.Name,
- Mode: s.Mode,
- Enable: s.Enable,
- CronConfig: s.CronConfig,
- IsManual: s.IsManual,
- Model: archived.Model{ID: s.ID},
- Plan: encryptedPlan,
- Settings: encryptedSettings,
- }
- return dst, nil
+ return &BlueprintEncryption0904After{
+ Model: src.Model,
+ Plan: plan,
+ Settings: settings,
+ }, nil
},
)
+ if err != nil {
+ return err
+ }
+ _ = db.First(blueprint)
Review Comment:
Don't do it here, do it inside the helper.
##########
models/migrationscripts/20220904_encrypt_pipeline.go:
##########
@@ -18,81 +18,62 @@ limitations under the License.
package migrationscripts
import (
- "time"
-
"github.com/apache/incubator-devlake/errors"
"github.com/apache/incubator-devlake/helpers/migrationhelper"
- "github.com/apache/incubator-devlake/models/common"
"github.com/apache/incubator-devlake/models/migrationscripts/archived"
"github.com/apache/incubator-devlake/plugins/core"
- "gorm.io/datatypes"
)
var _ core.MigrationScript = (*encryptPipeline)(nil)
type encryptPipeline struct{}
-type pipeline20220904Before struct {
+type PipelineEncryption0904Before struct {
+ archived.Model
+ Plan string
+}
+
+func (PipelineEncryption0904Before) TableName() string {
+ return "_devlake_pipelines"
+}
+
+type PipelineEncryption0904After struct {
archived.Model
- Name string `json:"name" gorm:"index"`
- BlueprintId uint64 `json:"blueprintId"`
- Plan datatypes.JSON `json:"plan"` // target field
- TotalTasks int `json:"totalTasks"`
- FinishedTasks int `json:"finishedTasks"`
- BeganAt *time.Time `json:"beganAt"`
- FinishedAt *time.Time `json:"finishedAt" gorm:"index"`
- Status string `json:"status"`
- Message string `json:"message"`
- SpentSeconds int `json:"spentSeconds"`
- Stage int `json:"stage"`
+ Plan string
}
-type pipeline0904After struct {
- common.Model
- Name string `json:"name" gorm:"index"`
- BlueprintId uint64 `json:"blueprintId"`
- Plan string `json:"plan" encrypt:"yes"` // target field
- TotalTasks int `json:"totalTasks"`
- FinishedTasks int `json:"finishedTasks"`
- BeganAt *time.Time `json:"beganAt"`
- FinishedAt *time.Time `json:"finishedAt" gorm:"index"`
- Status string `json:"status"`
- Message string `json:"message"`
- SpentSeconds int `json:"spentSeconds"`
- Stage int `json:"stage"`
+func (PipelineEncryption0904After) TableName() string {
+ return "_devlake_pipelines"
}
func (script *encryptPipeline) Up(basicRes core.BasicRes) errors.Error {
+ db := basicRes.GetDal()
encKey := basicRes.GetConfig(core.EncodeKeyEnvStr)
if encKey == "" {
return errors.BadInput.New("invalid encKey")
}
-
- return migrationhelper.TransformTable(
+ pipeline := &PipelineEncryption0904After{}
+ err := migrationhelper.TransformColumns(
basicRes,
script,
"_devlake_pipelines",
- func(s *pipeline20220904Before) (*pipeline0904After, errors.Error) {
- encryptedPlan, err := core.Encrypt(encKey, string(s.Plan))
+ []string{"plan"},
+ func(src *PipelineEncryption0904Before) (*PipelineEncryption0904After, errors.Error) {
+ plan, err := core.Encrypt(encKey, src.Plan)
if err != nil {
return nil, err
}
-
- dst := &pipeline0904After{
- Name: s.Name,
- BlueprintId: s.BlueprintId,
- FinishedTasks: s.FinishedTasks,
- BeganAt: s.BeganAt,
- FinishedAt: s.FinishedAt,
- Status: s.Status,
- Message: s.Message,
- SpentSeconds: s.SpentSeconds,
- Stage: s.Stage,
- Plan: encryptedPlan,
- }
- return dst, nil
+ return &PipelineEncryption0904After{
+ Model: src.Model,
+ Plan: plan,
+ }, nil
},
)
+ if err != nil {
+ return err
+ }
+ _ = db.First(pipeline)
Review Comment:
Same as above
##########
models/migrationscripts/20220903_encrypt_blueprint.go:
##########
@@ -30,63 +28,59 @@ var _ core.MigrationScript = (*encryptBlueprint)(nil)
type encryptBlueprint struct{}
-type blueprint20220903Before struct {
- Name string `json:"name" validate:"required"`
- Mode string `json:"mode" gorm:"varchar(20)" validate:"required,oneof=NORMAL ADVANCED"`
- Plan json.RawMessage `json:"plan"`
- Enable bool `json:"enable"`
- CronConfig string `json:"cronConfig" format:"* * * * *" example:"0 0 * * 1"`
- IsManual bool `json:"isManual"`
- Settings json.RawMessage `json:"settings" swaggertype:"array,string" example:"please check api: /blueprints/<PLUGIN_NAME>/blueprint-setting"`
- archived.Model `swaggerignore:"true"`
+type BlueprintEncryption0904Before struct {
+ archived.Model
+ Plan string
+ Settings string
+}
+
+func (BlueprintEncryption0904Before) TableName() string {
+ return "_devlake_blueprints"
+}
+
+type BlueprintEncryption0904After struct {
+ archived.Model
+ Plan string
+ Settings string
}
-type blueprint20220903After struct {
- /* unchanged part */
- Name string `json:"name" validate:"required"`
- Mode string `json:"mode" gorm:"varchar(20)" validate:"required,oneof=NORMAL ADVANCED"`
- Enable bool `json:"enable"`
- CronConfig string `json:"cronConfig" format:"* * * * *" example:"0 0 * * 1"`
- IsManual bool `json:"isManual"`
- archived.Model `swaggerignore:"true"`
- /* changed part */
- Plan string `json:"plan"`
- Settings string `json:"settings"`
+func (BlueprintEncryption0904After) TableName() string {
Review Comment:
Not needed
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@devlake.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [incubator-devlake] warren830 commented on a diff in pull request #3728: fix(framework): fix migration scripts
Posted by GitBox <gi...@apache.org>.
warren830 commented on code in PR #3728:
URL: https://github.com/apache/incubator-devlake/pull/3728#discussion_r1021151537
##########
models/migrationscripts/20220904_encrypt_pipeline.go:
##########
@@ -18,81 +18,62 @@ limitations under the License.
package migrationscripts
import (
- "time"
-
"github.com/apache/incubator-devlake/errors"
"github.com/apache/incubator-devlake/helpers/migrationhelper"
- "github.com/apache/incubator-devlake/models/common"
"github.com/apache/incubator-devlake/models/migrationscripts/archived"
"github.com/apache/incubator-devlake/plugins/core"
- "gorm.io/datatypes"
)
var _ core.MigrationScript = (*encryptPipeline)(nil)
type encryptPipeline struct{}
-type pipeline20220904Before struct {
+type PipelineEncryption0904Before struct {
+ archived.Model
+ Plan string
+}
+
+func (PipelineEncryption0904Before) TableName() string {
+ return "_devlake_pipelines"
+}
+
+type PipelineEncryption0904After struct {
archived.Model
- Name string `json:"name" gorm:"index"`
- BlueprintId uint64 `json:"blueprintId"`
- Plan datatypes.JSON `json:"plan"` // target field
- TotalTasks int `json:"totalTasks"`
- FinishedTasks int `json:"finishedTasks"`
- BeganAt *time.Time `json:"beganAt"`
- FinishedAt *time.Time `json:"finishedAt" gorm:"index"`
- Status string `json:"status"`
- Message string `json:"message"`
- SpentSeconds int `json:"spentSeconds"`
- Stage int `json:"stage"`
+ Plan string
}
-type pipeline0904After struct {
- common.Model
- Name string `json:"name" gorm:"index"`
- BlueprintId uint64 `json:"blueprintId"`
- Plan string `json:"plan" encrypt:"yes"` // target field
- TotalTasks int `json:"totalTasks"`
- FinishedTasks int `json:"finishedTasks"`
- BeganAt *time.Time `json:"beganAt"`
- FinishedAt *time.Time `json:"finishedAt" gorm:"index"`
- Status string `json:"status"`
- Message string `json:"message"`
- SpentSeconds int `json:"spentSeconds"`
- Stage int `json:"stage"`
+func (PipelineEncryption0904After) TableName() string {
+ return "_devlake_pipelines"
}
func (script *encryptPipeline) Up(basicRes core.BasicRes) errors.Error {
+ db := basicRes.GetDal()
encKey := basicRes.GetConfig(core.EncodeKeyEnvStr)
if encKey == "" {
return errors.BadInput.New("invalid encKey")
}
-
- return migrationhelper.TransformTable(
+ pipeline := &PipelineEncryption0904After{}
+ err := migrationhelper.TransformColumns(
basicRes,
script,
"_devlake_pipelines",
- func(s *pipeline20220904Before) (*pipeline0904After, errors.Error) {
- encryptedPlan, err := core.Encrypt(encKey, string(s.Plan))
+ []string{"plan"},
+ func(src *PipelineEncryption0904Before) (*PipelineEncryption0904After, errors.Error) {
+ plan, err := core.Encrypt(encKey, src.Plan)
if err != nil {
return nil, err
}
-
- dst := &pipeline0904After{
- Name: s.Name,
- BlueprintId: s.BlueprintId,
- FinishedTasks: s.FinishedTasks,
- BeganAt: s.BeganAt,
- FinishedAt: s.FinishedAt,
- Status: s.Status,
- Message: s.Message,
- SpentSeconds: s.SpentSeconds,
- Stage: s.Stage,
- Plan: encryptedPlan,
- }
- return dst, nil
+ return &PipelineEncryption0904After{
+ Model: src.Model,
+ Plan: plan,
+ }, nil
},
)
+ if err != nil {
+ return err
+ }
+ _ = db.First(pipeline)
Review Comment:
fixed
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@devlake.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org