You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by kl...@apache.org on 2022/10/08 11:32:24 UTC

[incubator-devlake] branch main updated: refactor: migration helper (#3169)

This is an automated email from the ASF dual-hosted git repository.

klesh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git


The following commit(s) were added to refs/heads/main by this push:
     new 7b0199a6 refactor: migration helper (#3169)
7b0199a6 is described below

commit 7b0199a6d8557d4f18b73200d619a96d10df0163
Author: mappjzc <zh...@merico.dev>
AuthorDate: Sat Oct 8 19:32:20 2022 +0800

    refactor: migration helper (#3169)
    
    * refactor: migration helper
    
    Add ReBuildTable
    Add ReBuildTableWithOutBak
    
    Nddtfjiang <zh...@merico.dev>
    
    * refactor: add param checking
    
    Add paramCheckingForReBuildTable
    Add paramCheckingForReBuildTableWithOutBak
    Add paramCheckingForReBuildTableShare
    
    Nddtfjiang <zh...@merico.dev>
    
    * refactor: rename func
    
    rename to TransformRowsInPlace
    rename to TransformRowsBetweenTables
    
    Nddtfjiang <zh...@merico.dev>
    
    * refactor: change gorm db to dal
    
    change gorm db to dal.
    
    Nddtfjiang <zh...@merico.dev>
---
 helpers/migrationhelper/helper.go | 187 ++++++++++++++++++++++++++++++++++++++
 impl/dalgorm/dalgorm.go           |  21 ++++-
 plugins/core/dal/dal.go           |   7 +-
 3 files changed, 213 insertions(+), 2 deletions(-)

diff --git a/helpers/migrationhelper/helper.go b/helpers/migrationhelper/helper.go
new file mode 100644
index 00000000..2191a1c9
--- /dev/null
+++ b/helpers/migrationhelper/helper.go
@@ -0,0 +1,187 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationhelper
+
+import (
+	"fmt"
+	"reflect"
+
+	"github.com/apache/incubator-devlake/errors"
+	"github.com/apache/incubator-devlake/plugins/core/dal"
+	"github.com/apache/incubator-devlake/plugins/gitlab/api"
+	"github.com/apache/incubator-devlake/plugins/helper"
+	"gorm.io/gorm/schema"
+)
+
+// TransformRowsInPlace method can be used when we need to change the table structure and reprocess all the data in the table.
+func TransformRowsInPlace(db dal.Dal, src schema.Tabler, bak schema.Tabler, dst schema.Tabler, callback_transform func(src schema.Tabler) schema.Tabler) (errs errors.Error) {
+	var err error
+
+	// param cheking
+	errs = paramCheckingForTransformRowsInPlace(db, src, bak, dst, callback_transform)
+	if errs != nil {
+		return errors.Default.Wrap(errs, "TransformRowsInPlace param cheking error")
+	}
+
+	// rename the src to bak for cache src table
+	err = db.RenameTable(src, bak)
+	if err != nil {
+		return errors.Default.Wrap(err, fmt.Sprintf("error no rename [%s] to [%s]", src.TableName(), bak.TableName()))
+	}
+
+	// rollback for rename back
+	defer func() {
+		if errs != nil {
+			err = db.RenameTable(bak, src)
+			if err != nil {
+				errs = errors.Default.Wrap(err, fmt.Sprintf("fail to rollback table [%s] , you must to rollback by yourself. %s", bak.TableName(), err.Error()))
+			}
+		}
+	}()
+
+	return TransformRowsBetweenTables(db, bak, dst, callback_transform)
+}
+
+// TransformRowsBetweenTables method can be used when we need to change the table structure and reprocess all the data in the table.
+// It request the src table and the dst table with different table name.
+func TransformRowsBetweenTables(db dal.Dal, src schema.Tabler, dst schema.Tabler, callback_transform func(src schema.Tabler) schema.Tabler) (errs errors.Error) {
+	var err error
+
+	errs = paramCheckingForTransformRowsBetweenTables(db, src, dst, callback_transform)
+	if errs != nil {
+		return errors.Default.Wrap(errs, "TransformRowsBetweenTables param cheking error")
+	}
+
+	// create new commit_files table
+	err = db.AutoMigrate(dst)
+	if err != nil {
+		return errors.Default.Wrap(err, fmt.Sprintf("error on auto migrate [%s]", dst.TableName()))
+	}
+
+	// rollback for create new table
+	defer func() {
+		if errs != nil {
+			err = db.DropTable(dst)
+			if err != nil {
+				errs = errors.Default.Wrap(err, fmt.Sprintf("fail to rollback table [%s] , you must to rollback by yourself. %s", dst.TableName(), err.Error()))
+			}
+		}
+	}()
+
+	// update src id to dst id and write to the dst table
+	cursor, err := db.Cursor(
+		dal.From(src.TableName()),
+	)
+	if err != nil {
+		return errors.Default.Wrap(err, fmt.Sprintf("error on select [%s]]", src.TableName()))
+	}
+	defer cursor.Close()
+
+	// caculate and save the data to new table
+	batch, err := helper.NewBatchSave(api.BasicRes, reflect.TypeOf(dst), 200)
+	if err != nil {
+		return errors.Default.Wrap(err, fmt.Sprintf("error getting batch from table [%s]", dst.TableName()))
+	}
+
+	defer batch.Close()
+	for cursor.Next() {
+		err = db.Fetch(cursor, src)
+		if err != nil {
+			return errors.Default.Wrap(err, fmt.Sprintf("error scan rows from table [%s]", src.TableName()))
+		}
+
+		cf := callback_transform(src)
+
+		err = batch.Add(&cf)
+		if err != nil {
+			return errors.Default.Wrap(err, fmt.Sprintf("error on [%s] batch add", dst.TableName()))
+		}
+	}
+
+	// drop the src table
+	err = db.DropTable(src)
+	if err != nil {
+		return errors.Default.Wrap(err, fmt.Sprintf("error no drop [%s]", src.TableName()))
+	}
+
+	return nil
+}
+
+// paramCheckingForTransformRowsInPlace check the params of TransformRowsInPlace
+func paramCheckingForTransformRowsInPlace(db dal.Dal, src schema.Tabler, bak schema.Tabler, dst schema.Tabler, callback_transform func(src schema.Tabler) schema.Tabler) (errs errors.Error) {
+	errs = paramCheckingShare(db, dst, callback_transform)
+	if errs != nil {
+		return errs
+	}
+
+	if src == nil {
+		return errors.Default.New("can not working with param src nil")
+	}
+
+	if bak == nil {
+		return errors.Default.New("can not working with param bak nil")
+	}
+
+	if dst.TableName() == bak.TableName() {
+		return errors.Default.New(fmt.Sprintf("the bak and dst can not use the same table name [%s][%s].",
+			bak.TableName(), dst.TableName()))
+	}
+
+	if src.TableName() == bak.TableName() {
+		return errors.Default.New(fmt.Sprintf("the src and bak can not use the same table name [%s][%s].",
+			src.TableName(), bak.TableName()))
+	}
+
+	return nil
+}
+
+// paramCheckingForTransformRowsBetweenTables check the params of ReBuildTableWithOutBak
+func paramCheckingForTransformRowsBetweenTables(db dal.Dal, src schema.Tabler, dst schema.Tabler, callback_transform func(src schema.Tabler) schema.Tabler) (errs errors.Error) {
+	errs = paramCheckingShare(db, dst, callback_transform)
+	if errs != nil {
+		return errs
+	}
+
+	if src == nil {
+		return errors.Default.New("can not working with param src nil")
+	}
+
+	if src.TableName() == dst.TableName() {
+		return errors.Default.New(fmt.Sprintf("src and dst can not use the same table name [%s][%s].",
+			src.TableName(), dst.TableName()))
+	}
+
+	return nil
+}
+
+// paramCheckingShare check the Share part params of TransformRowsBetweenTables and TransformRowsInPlace
+func paramCheckingShare(db dal.Dal, dst schema.Tabler, callback_transform func(src schema.Tabler) schema.Tabler) (errs errors.Error) {
+	if db == nil {
+		return errors.Default.New("can not working with param db nil")
+	}
+
+	if dst == nil {
+		return errors.Default.New("can not working with param dst nil")
+	}
+
+	if callback_transform == nil {
+		return errors.Default.New("can not working with param callback_transform nil")
+	}
+
+	return nil
+}
diff --git a/impl/dalgorm/dalgorm.go b/impl/dalgorm/dalgorm.go
index 9d16556b..003c565f 100644
--- a/impl/dalgorm/dalgorm.go
+++ b/impl/dalgorm/dalgorm.go
@@ -19,10 +19,11 @@ package dalgorm
 
 import (
 	"database/sql"
-	"github.com/apache/incubator-devlake/errors"
 	"reflect"
 	"strings"
 
+	"github.com/apache/incubator-devlake/errors"
+
 	"github.com/apache/incubator-devlake/plugins/core/dal"
 	"github.com/apache/incubator-devlake/utils"
 	"gorm.io/gorm"
@@ -220,3 +221,21 @@ func (d *Dalgorm) AllTables() ([]string, errors.Error) {
 func NewDalgorm(db *gorm.DB) *Dalgorm {
 	return &Dalgorm{db}
 }
+
+// RenameTable rename the oldName table to newName
+func (d *Dalgorm) RenameTable(oldName interface{}, newName interface{}) errors.Error {
+	err := d.db.Migrator().RenameTable(oldName, newName)
+	if err != nil {
+		return errors.Default.New(err.Error())
+	}
+	return nil
+}
+
+// DropTable drop the table
+func (d *Dalgorm) DropTable(dst ...interface{}) errors.Error {
+	err := d.db.Migrator().DropTable(dst)
+	if err != nil {
+		return errors.Default.New(err.Error())
+	}
+	return nil
+}
diff --git a/plugins/core/dal/dal.go b/plugins/core/dal/dal.go
index 257eaf5a..01bf9fa6 100644
--- a/plugins/core/dal/dal.go
+++ b/plugins/core/dal/dal.go
@@ -19,9 +19,10 @@ package dal
 
 import (
 	"database/sql"
-	"github.com/apache/incubator-devlake/errors"
 	"reflect"
 
+	"github.com/apache/incubator-devlake/errors"
+
 	"gorm.io/gorm/schema"
 )
 
@@ -88,6 +89,10 @@ type Dal interface {
 	GetColumns(dst schema.Tabler, filter func(columnMeta ColumnMeta) bool) (cms []ColumnMeta, err errors.Error)
 	// GetPrimarykeyFields get the PrimaryKey from `gorm` tag
 	GetPrimaryKeyFields(t reflect.Type) []reflect.StructField
+	// RenameTable rename the oldName table to newName
+	RenameTable(oldName interface{}, newName interface{}) errors.Error
+	// DropTable drop the table
+	DropTable(dst ...interface{}) errors.Error
 }
 
 // GetColumnNames returns table Column Names in database