You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by ab...@apache.org on 2022/10/20 11:01:29 UTC
[incubator-devlake] branch main updated: refactor: migration scripts of AE, Azure, bitbucket (#3509)
This is an automated email from the ASF dual-hosted git repository.
abeizn pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new d979b154 refactor: migration scripts of AE, Azure, bitbucket (#3509)
d979b154 is described below
commit d979b15483a6eb5bb832abfa313503274813d24a
Author: mindlesscloud <li...@merico.dev>
AuthorDate: Thu Oct 20 19:01:24 2022 +0800
refactor: migration scripts of AE, Azure, bitbucket (#3509)
---
plugins/ae/impl/impl.go | 7 +--
.../migrationscripts/20220714_add_init_tables.go | 46 +++++++-------
plugins/ae/models/migrationscripts/register.go | 8 +--
plugins/azure/impl/impl.go | 6 +-
.../migrationscripts/20220825_add_init_tables.go | 28 +++------
plugins/azure/models/migrationscripts/register.go | 8 +--
plugins/bitbucket/impl/impl.go | 5 +-
.../migrationscripts/20220803_add_init_tables.go | 70 +++-------------------
.../20220914_add_pipeline_tables.go | 7 +--
.../20221013_add_deployment_tables.go | 11 +---
...4_add_RepoId_CommitSha_field_pipeline_tables.go | 42 ++++---------
.../bitbucket/models/migrationscripts/register.go | 10 ++--
12 files changed, 77 insertions(+), 171 deletions(-)
diff --git a/plugins/ae/impl/impl.go b/plugins/ae/impl/impl.go
index ec97da43..da5c9bfd 100644
--- a/plugins/ae/impl/impl.go
+++ b/plugins/ae/impl/impl.go
@@ -19,9 +19,8 @@ package impl
import (
"fmt"
- "github.com/apache/incubator-devlake/errors"
- "github.com/apache/incubator-devlake/migration"
+ "github.com/apache/incubator-devlake/errors"
"github.com/apache/incubator-devlake/plugins/ae/api"
"github.com/apache/incubator-devlake/plugins/ae/models"
"github.com/apache/incubator-devlake/plugins/ae/models/migrationscripts"
@@ -36,7 +35,7 @@ var _ core.PluginMeta = (*AE)(nil)
var _ core.PluginInit = (*AE)(nil)
var _ core.PluginTask = (*AE)(nil)
var _ core.PluginApi = (*AE)(nil)
-var _ core.Migratable = (*AE)(nil)
+var _ core.PluginMigration = (*AE)(nil)
var _ core.CloseablePluginTask = (*AE)(nil)
type AE struct{}
@@ -100,7 +99,7 @@ func (plugin AE) RootPkgPath() string {
return "github.com/apache/incubator-devlake/plugins/ae"
}
-func (plugin AE) MigrationScripts() []migration.Script {
+func (plugin AE) MigrationScripts() []core.MigrationScript {
return migrationscripts.All()
}
diff --git a/plugins/ae/models/migrationscripts/20220714_add_init_tables.go b/plugins/ae/models/migrationscripts/20220714_add_init_tables.go
index cc11ec8d..1ae21f7e 100644
--- a/plugins/ae/models/migrationscripts/20220714_add_init_tables.go
+++ b/plugins/ae/models/migrationscripts/20220714_add_init_tables.go
@@ -18,68 +18,66 @@ limitations under the License.
package migrationscripts
import (
- "context"
+ "github.com/apache/incubator-devlake/config"
"github.com/apache/incubator-devlake/errors"
-
+ "github.com/apache/incubator-devlake/helpers/migrationhelper"
"github.com/apache/incubator-devlake/plugins/ae/models/migrationscripts/archived"
"github.com/apache/incubator-devlake/plugins/core"
"github.com/apache/incubator-devlake/plugins/helper"
- "gorm.io/gorm"
)
-type addInitTables struct {
- config core.ConfigGetter
-}
-
-func (u *addInitTables) SetConfigGetter(config core.ConfigGetter) {
- u.config = config
-}
+type addInitTables20220714 struct{}
-func (u *addInitTables) Up(ctx context.Context, db *gorm.DB) errors.Error {
- err := db.Migrator().DropTable(
+func (u *addInitTables20220714) Up(basicRes core.BasicRes) errors.Error {
+ err := basicRes.GetDal().DropTables(
&archived.AECommit{},
&archived.AEProject{},
"_raw_ae_project",
"_raw_ae_commits",
)
if err != nil {
- return errors.Convert(err)
+ return err
}
- err = db.Migrator().AutoMigrate(
+ err = migrationhelper.AutoMigrateTables(
+ basicRes,
&archived.AECommit{},
&archived.AEProject{},
&archived.AeConnection{},
)
if err != nil {
- return errors.Convert(err)
+ return err
}
- encodeKey := u.config.GetString(core.EncodeKeyEnvStr)
+ c := config.GetConfig()
+ encodeKey := c.GetString(core.EncodeKeyEnvStr)
connection := &archived.AeConnection{}
- connection.Endpoint = u.config.GetString("AE_ENDPOINT")
- connection.Proxy = u.config.GetString("AE_PROXY")
- connection.SecretKey = u.config.GetString("AE_SECRET_KEY")
- connection.AppId = u.config.GetString("AE_APP_ID")
+ connection.Endpoint = c.GetString("AE_ENDPOINT")
+ connection.Proxy = c.GetString("AE_PROXY")
+ connection.SecretKey = c.GetString("AE_SECRET_KEY")
+ connection.AppId = c.GetString("AE_APP_ID")
connection.Name = "AE"
if connection.Endpoint != "" && connection.AppId != "" && connection.SecretKey != "" && encodeKey != "" {
err = helper.UpdateEncryptFields(connection, func(plaintext string) (string, errors.Error) {
return core.Encrypt(encodeKey, plaintext)
})
if err != nil {
- return errors.Default.Wrap(err, "error encrypting fields")
+ return err
}
// update from .env and save to db
- db.Create(connection)
+ err = basicRes.GetDal().Create(connection)
+ if err != nil {
+ return err
+ }
}
return nil
}
-func (*addInitTables) Version() uint64 {
+func (*addInitTables20220714) Version() uint64 {
return 20220714201133
}
-func (*addInitTables) Name() string {
+func (*addInitTables20220714) Name() string {
return "AE init schemas"
}
diff --git a/plugins/ae/models/migrationscripts/register.go b/plugins/ae/models/migrationscripts/register.go
index c1365f7d..f5e9d97a 100644
--- a/plugins/ae/models/migrationscripts/register.go
+++ b/plugins/ae/models/migrationscripts/register.go
@@ -18,12 +18,12 @@ limitations under the License.
package migrationscripts
import (
- "github.com/apache/incubator-devlake/migration"
+ "github.com/apache/incubator-devlake/plugins/core"
)
// All return all the migration scripts
-func All() []migration.Script {
- return []migration.Script{
- new(addInitTables),
+func All() []core.MigrationScript {
+ return []core.MigrationScript{
+ new(addInitTables20220714),
}
}
diff --git a/plugins/azure/impl/impl.go b/plugins/azure/impl/impl.go
index 0f306dc8..5b55d488 100644
--- a/plugins/azure/impl/impl.go
+++ b/plugins/azure/impl/impl.go
@@ -19,15 +19,14 @@ package impl
import (
"fmt"
+
"github.com/apache/incubator-devlake/errors"
- "github.com/apache/incubator-devlake/migration"
"github.com/apache/incubator-devlake/plugins/azure/api"
"github.com/apache/incubator-devlake/plugins/azure/models"
"github.com/apache/incubator-devlake/plugins/azure/models/migrationscripts"
"github.com/apache/incubator-devlake/plugins/azure/tasks"
"github.com/apache/incubator-devlake/plugins/core"
"github.com/apache/incubator-devlake/plugins/helper"
-
"github.com/spf13/viper"
"gorm.io/gorm"
)
@@ -38,6 +37,7 @@ var _ core.PluginInit = (*Azure)(nil)
var _ core.PluginTask = (*Azure)(nil)
var _ core.PluginApi = (*Azure)(nil)
var _ core.CloseablePluginTask = (*Azure)(nil)
+var _ core.PluginMigration = (*Azure)(nil)
// PluginEntry exports for Framework to search and load
var PluginEntry Azure //nolint
@@ -117,7 +117,7 @@ func (plugin Azure) ApiResources() map[string]map[string]core.ApiResourceHandler
}
}
-func (plugin Azure) MigrationScripts() []migration.Script {
+func (plugin Azure) MigrationScripts() []core.MigrationScript {
return migrationscripts.All()
}
diff --git a/plugins/azure/models/migrationscripts/20220825_add_init_tables.go b/plugins/azure/models/migrationscripts/20220825_add_init_tables.go
index 47f80dea..af1ef0df 100644
--- a/plugins/azure/models/migrationscripts/20220825_add_init_tables.go
+++ b/plugins/azure/models/migrationscripts/20220825_add_init_tables.go
@@ -18,37 +18,27 @@ limitations under the License.
package migrationscripts
import (
- "context"
"github.com/apache/incubator-devlake/errors"
-
+ "github.com/apache/incubator-devlake/helpers/migrationhelper"
"github.com/apache/incubator-devlake/plugins/azure/models/migrationscripts/archived"
- "gorm.io/gorm"
+ "github.com/apache/incubator-devlake/plugins/core"
)
-type addInitTables struct{}
+type addInitTables20220825 struct{}
-func (*addInitTables) Up(ctx context.Context, db *gorm.DB) errors.Error {
- if !db.Migrator().HasTable(&archived.AzureConnection{}) {
- err := db.Migrator().AutoMigrate(&archived.AzureConnection{})
- if err != nil {
- return errors.Convert(err)
- }
- }
- err := db.Migrator().AutoMigrate(
+func (*addInitTables20220825) Up(basicRes core.BasicRes) errors.Error {
+ return migrationhelper.AutoMigrateTables(
+ basicRes,
+ &archived.AzureConnection{},
&archived.AzureRepo{},
&archived.AzureBuildDefinition{},
)
- if err != nil {
- return errors.Convert(err)
- }
-
- return nil
}
-func (*addInitTables) Version() uint64 {
+func (*addInitTables20220825) Version() uint64 {
return 20220825231237
}
-func (*addInitTables) Name() string {
+func (*addInitTables20220825) Name() string {
return "Azure init schemas"
}
diff --git a/plugins/azure/models/migrationscripts/register.go b/plugins/azure/models/migrationscripts/register.go
index c1365f7d..8e64d44b 100644
--- a/plugins/azure/models/migrationscripts/register.go
+++ b/plugins/azure/models/migrationscripts/register.go
@@ -18,12 +18,12 @@ limitations under the License.
package migrationscripts
import (
- "github.com/apache/incubator-devlake/migration"
+ "github.com/apache/incubator-devlake/plugins/core"
)
// All return all the migration scripts
-func All() []migration.Script {
- return []migration.Script{
- new(addInitTables),
+func All() []core.MigrationScript {
+ return []core.MigrationScript{
+ new(addInitTables20220825),
}
}
diff --git a/plugins/bitbucket/impl/impl.go b/plugins/bitbucket/impl/impl.go
index 8619687a..9f23a181 100644
--- a/plugins/bitbucket/impl/impl.go
+++ b/plugins/bitbucket/impl/impl.go
@@ -19,7 +19,6 @@ package impl
import (
"github.com/apache/incubator-devlake/errors"
- "github.com/apache/incubator-devlake/migration"
"github.com/apache/incubator-devlake/plugins/bitbucket/api"
"github.com/apache/incubator-devlake/plugins/bitbucket/models"
"github.com/apache/incubator-devlake/plugins/bitbucket/models/migrationscripts"
@@ -34,7 +33,7 @@ var _ core.PluginMeta = (*Bitbucket)(nil)
var _ core.PluginInit = (*Bitbucket)(nil)
var _ core.PluginTask = (*Bitbucket)(nil)
var _ core.PluginApi = (*Bitbucket)(nil)
-var _ core.Migratable = (*Bitbucket)(nil)
+var _ core.PluginMigration = (*Bitbucket)(nil)
var _ core.PluginBlueprintV100 = (*Bitbucket)(nil)
type Bitbucket string
@@ -105,7 +104,7 @@ func (plugin Bitbucket) RootPkgPath() string {
return "github.com/apache/incubator-devlake/plugins/bitbucket"
}
-func (plugin Bitbucket) MigrationScripts() []migration.Script {
+func (plugin Bitbucket) MigrationScripts() []core.MigrationScript {
return migrationscripts.All()
}
diff --git a/plugins/bitbucket/models/migrationscripts/20220803_add_init_tables.go b/plugins/bitbucket/models/migrationscripts/20220803_add_init_tables.go
index ca8b59d6..a722ac76 100644
--- a/plugins/bitbucket/models/migrationscripts/20220803_add_init_tables.go
+++ b/plugins/bitbucket/models/migrationscripts/20220803_add_init_tables.go
@@ -18,20 +18,16 @@ limitations under the License.
package migrationscripts
import (
- "context"
- "encoding/base64"
- "github.com/apache/incubator-devlake/config"
"github.com/apache/incubator-devlake/errors"
+ "github.com/apache/incubator-devlake/helpers/migrationhelper"
"github.com/apache/incubator-devlake/plugins/bitbucket/models/migrationscripts/archived"
"github.com/apache/incubator-devlake/plugins/core"
- "gorm.io/gorm"
- "strings"
)
-type addInitTables struct{}
+type addInitTables20220803 struct{}
-func (*addInitTables) Up(ctx context.Context, db *gorm.DB) errors.Error {
- err := db.Migrator().DropTable(
+func (script *addInitTables20220803) Up(basicRes core.BasicRes) errors.Error {
+ err := basicRes.GetDal().DropTables(
//history table
&archived.BitbucketRepo{},
&archived.BitbucketRepoCommit{},
@@ -43,10 +39,11 @@ func (*addInitTables) Up(ctx context.Context, db *gorm.DB) errors.Error {
&archived.BitbucketIssueComment{},
)
if err != nil {
- return errors.Convert(err)
+ return err
}
- err = db.Migrator().AutoMigrate(
+ return migrationhelper.AutoMigrateTables(
+ basicRes,
&archived.BitbucketRepo{},
&archived.BitbucketRepoCommit{},
&archived.BitbucketConnection{},
@@ -57,61 +54,12 @@ func (*addInitTables) Up(ctx context.Context, db *gorm.DB) errors.Error {
&archived.BitbucketPrComment{},
&archived.BitbucketIssueComment{},
)
- if err != nil {
- return errors.Convert(err)
- }
-
- var result *gorm.DB
- var bitbucketConns []archived.BitbucketConnection
- result = db.Find(&bitbucketConns)
- if result.Error != nil {
- return errors.Convert(result.Error)
- }
-
- for _, v := range bitbucketConns {
- conn := &archived.BitbucketConnection{}
- conn.ID = v.ID
- conn.Name = v.Name
- conn.Endpoint = v.Endpoint
- conn.Proxy = v.Proxy
- conn.RateLimitPerHour = v.RateLimitPerHour
-
- c := config.GetConfig()
- encKey := c.GetString(core.EncodeKeyEnvStr)
- if encKey == "" {
- return errors.BadInput.New("bitbucket invalid encKey")
- }
- var auth string
- if auth, err = core.Decrypt(encKey, v.BasicAuth.GetEncodedToken()); err != nil {
- return errors.Convert(err)
- }
- var pk []byte
- pk, err = base64.StdEncoding.DecodeString(auth)
- if err != nil {
- return errors.Default.Wrap(err, "error creating connection entry for BitBucket")
- }
- originInfo := strings.Split(string(pk), ":")
- if len(originInfo) == 2 {
- conn.Username = originInfo[0]
- conn.Password, err = core.Encrypt(encKey, originInfo[1])
- if err != nil {
- return errors.Convert(err)
- }
- // create
- tx := db.Create(&conn)
- if tx.Error != nil {
- return errors.Default.Wrap(tx.Error, "error adding connection to DB")
- }
- }
- }
-
- return nil
}
-func (*addInitTables) Version() uint64 {
+func (*addInitTables20220803) Version() uint64 {
return 20220803220824
}
-func (*addInitTables) Name() string {
+func (*addInitTables20220803) Name() string {
return "Bitbucket init schema 20220803"
}
diff --git a/plugins/bitbucket/models/migrationscripts/20220914_add_pipeline_tables.go b/plugins/bitbucket/models/migrationscripts/20220914_add_pipeline_tables.go
index 45db9377..d9a40069 100644
--- a/plugins/bitbucket/models/migrationscripts/20220914_add_pipeline_tables.go
+++ b/plugins/bitbucket/models/migrationscripts/20220914_add_pipeline_tables.go
@@ -18,16 +18,15 @@ limitations under the License.
package migrationscripts
import (
- "context"
"github.com/apache/incubator-devlake/errors"
"github.com/apache/incubator-devlake/plugins/bitbucket/models/migrationscripts/archived"
- "gorm.io/gorm"
+ "github.com/apache/incubator-devlake/plugins/core"
)
type addPipeline20220914 struct{}
-func (*addPipeline20220914) Up(ctx context.Context, db *gorm.DB) errors.Error {
- err := db.Migrator().AutoMigrate(&archived.BitbucketPipeline{})
+func (*addPipeline20220914) Up(basicRes core.BasicRes) errors.Error {
+ err := basicRes.GetDal().AutoMigrate(&archived.BitbucketPipeline{})
if err != nil {
return errors.Convert(err)
}
diff --git a/plugins/bitbucket/models/migrationscripts/20221013_add_deployment_tables.go b/plugins/bitbucket/models/migrationscripts/20221013_add_deployment_tables.go
index 11ab6d7b..10a158a6 100644
--- a/plugins/bitbucket/models/migrationscripts/20221013_add_deployment_tables.go
+++ b/plugins/bitbucket/models/migrationscripts/20221013_add_deployment_tables.go
@@ -18,20 +18,15 @@ limitations under the License.
package migrationscripts
import (
- "context"
"github.com/apache/incubator-devlake/errors"
"github.com/apache/incubator-devlake/plugins/bitbucket/models/migrationscripts/archived"
- "gorm.io/gorm"
+ "github.com/apache/incubator-devlake/plugins/core"
)
type addDeployment20221013 struct{}
-func (*addDeployment20221013) Up(ctx context.Context, db *gorm.DB) errors.Error {
- err := db.Migrator().AutoMigrate(&archived.BitbucketDeployment{})
- if err != nil {
- return errors.Convert(err)
- }
- return nil
+func (*addDeployment20221013) Up(basicRes core.BasicRes) errors.Error {
+ return basicRes.GetDal().AutoMigrate(&archived.BitbucketDeployment{})
}
func (*addDeployment20221013) Version() uint64 {
diff --git a/plugins/bitbucket/models/migrationscripts/20221014_add_RepoId_CommitSha_field_pipeline_tables.go b/plugins/bitbucket/models/migrationscripts/20221014_add_RepoId_CommitSha_field_pipeline_tables.go
index 9d520fb1..e9eea34f 100644
--- a/plugins/bitbucket/models/migrationscripts/20221014_add_RepoId_CommitSha_field_pipeline_tables.go
+++ b/plugins/bitbucket/models/migrationscripts/20221014_add_RepoId_CommitSha_field_pipeline_tables.go
@@ -18,51 +18,29 @@ limitations under the License.
package migrationscripts
import (
- "context"
- "time"
-
"github.com/apache/incubator-devlake/errors"
- "github.com/apache/incubator-devlake/models/migrationscripts/archived"
- "gorm.io/gorm"
+ "github.com/apache/incubator-devlake/plugins/core"
)
-type Task20221014 struct {
- ConnectionId uint64 `gorm:"primaryKey"`
- BitbucketId string `gorm:"primaryKey"`
- Status string `gorm:"type:varchar(100)"`
- Result string `gorm:"type:varchar(100)"`
- RefName string `gorm:"type:varchar(255)"`
- RepoId string `gorm:"type:varchar(255)"`
- CommitSha string `gorm:"type:varchar(255)"`
- WebUrl string `gorm:"type:varchar(255)"`
- DurationInSeconds uint64
- BitbucketCreatedOn *time.Time
- BitbucketCompleteOn *time.Time
- archived.NoPKModel
+type bitbucketPipeline20221014 struct {
+ RepoId string `gorm:"type:varchar(255)"`
+ CommitSha string `gorm:"type:varchar(255)"`
}
-func (Task20221014) TableName() string {
+func (bitbucketPipeline20221014) TableName() string {
return "_tool_bitbucket_pipelines"
}
-type addRepoIdAndCommitShaField struct{}
+type addRepoIdAndCommitShaField20221014 struct{}
-func (*addRepoIdAndCommitShaField) Up(ctx context.Context, db *gorm.DB) errors.Error {
- err := db.Migrator().AddColumn(Task20221014{}, "repo_id")
- if err != nil {
- return errors.Convert(err)
- }
- err = db.Migrator().AddColumn(Task20221014{}, "commit_sha")
- if err != nil {
- return errors.Convert(err)
- }
- return nil
+func (*addRepoIdAndCommitShaField20221014) Up(basicRes core.BasicRes) errors.Error {
+ return basicRes.GetDal().AutoMigrate(&bitbucketPipeline20221014{})
}
-func (*addRepoIdAndCommitShaField) Version() uint64 {
+func (*addRepoIdAndCommitShaField20221014) Version() uint64 {
return 20221014114623
}
-func (*addRepoIdAndCommitShaField) Name() string {
+func (*addRepoIdAndCommitShaField20221014) Name() string {
return "add column `repo_id` and `commit_sha` at _tool_bitbucket_pipelines"
}
diff --git a/plugins/bitbucket/models/migrationscripts/register.go b/plugins/bitbucket/models/migrationscripts/register.go
index 758a4e73..d6d7fc8e 100644
--- a/plugins/bitbucket/models/migrationscripts/register.go
+++ b/plugins/bitbucket/models/migrationscripts/register.go
@@ -18,15 +18,15 @@ limitations under the License.
package migrationscripts
import (
- "github.com/apache/incubator-devlake/migration"
+ "github.com/apache/incubator-devlake/plugins/core"
)
// All return all the migration scripts
-func All() []migration.Script {
- return []migration.Script{
- new(addInitTables),
+func All() []core.MigrationScript {
+ return []core.MigrationScript{
+ new(addInitTables20220803),
new(addPipeline20220914),
new(addDeployment20221013),
- new(addRepoIdAndCommitShaField),
+ new(addRepoIdAndCommitShaField20221014),
}
}