You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by kl...@apache.org on 2022/06/06 09:24:09 UTC
[incubator-devlake] branch main updated: add migration for directrun; update feishu model (#2030)
This is an automated email from the ASF dual-hosted git repository.
klesh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 108e393b add migration for directrun; update feishu model (#2030)
108e393b is described below
commit 108e393bf0c2909d4d491dce3017b278eddea219
Author: likyh <l...@likyh.com>
AuthorDate: Mon Jun 6 17:24:04 2022 +0800
add migration for directrun; update feishu model (#2030)
* add migration for directrun; update feishu model
* fix: fix primary read bug when time is primary key
* open needRecursion
* delete needRecursion
Co-authored-by: linyh <ya...@meri.co>
---
plugins/feishu/feishu.go | 4 +-
plugins/feishu/models/meeting_top_user_item.go | 13 ++--
.../migrationscripts/updateSchemas20220526.go | 90 ++++++++++++++++++++++
.../tasks/meeting_top_user_item_extractor.go | 7 +-
plugins/helper/batch_save.go | 27 +++----
runner/directrun.go | 11 +++
6 files changed, 123 insertions(+), 29 deletions(-)
diff --git a/plugins/feishu/feishu.go b/plugins/feishu/feishu.go
index 2902cd32..b5a5828f 100644
--- a/plugins/feishu/feishu.go
+++ b/plugins/feishu/feishu.go
@@ -73,7 +73,9 @@ func (plugin Feishu) RootPkgPath() string {
}
func (plugin Feishu) MigrationScripts() []migration.Script {
- return []migration.Script{new(migrationscripts.InitSchemas)}
+ return []migration.Script{
+ new(migrationscripts.InitSchemas), new(migrationscripts.UpdateSchemas20220524),
+ }
}
func (plugin Feishu) ApiResources() map[string]map[string]core.ApiResourceHandler {
diff --git a/plugins/feishu/models/meeting_top_user_item.go b/plugins/feishu/models/meeting_top_user_item.go
index 989090e7..9b50c0c0 100644
--- a/plugins/feishu/models/meeting_top_user_item.go
+++ b/plugins/feishu/models/meeting_top_user_item.go
@@ -23,13 +23,12 @@ import (
)
type FeishuMeetingTopUserItem struct {
- common.Model `json:"-"`
- StartTime time.Time
- MeetingCount string `json:"meeting_count" gorm:"type:varchar(255)"`
- MeetingDuration string `json:"meeting_duration" gorm:"type:varchar(255)"`
- Name string `json:"name" gorm:"type:varchar(255)"`
- UserType int64 `json:"user_type"`
- common.RawDataOrigin
+ common.NoPKModel `json:"-"`
+ StartTime time.Time `gorm:"primaryKey"`
+ Name string `json:"name" gorm:"primaryKey;type:varchar(255)"`
+ MeetingCount string `json:"meeting_count" gorm:"type:varchar(255)"`
+ MeetingDuration string `json:"meeting_duration" gorm:"type:varchar(255)"`
+ UserType int64 `json:"user_type"`
}
func (FeishuMeetingTopUserItem) TableName() string {
diff --git a/plugins/feishu/models/migrationscripts/updateSchemas20220526.go b/plugins/feishu/models/migrationscripts/updateSchemas20220526.go
new file mode 100644
index 00000000..634046d4
--- /dev/null
+++ b/plugins/feishu/models/migrationscripts/updateSchemas20220526.go
@@ -0,0 +1,90 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+ "context"
+ "github.com/apache/incubator-devlake/models/common"
+ "github.com/apache/incubator-devlake/plugins/feishu/models/migrationscripts/archived"
+ "gorm.io/gorm/clause"
+ "time"
+
+ "gorm.io/gorm"
+)
+
+type FeishuMeetingTopUserItem20220524Temp struct {
+ common.NoPKModel `json:"-"`
+ StartTime time.Time `gorm:"primaryKey"`
+ Name string `json:"name" gorm:"primaryKey;type:varchar(255)"`
+ MeetingCount string `json:"meeting_count" gorm:"type:varchar(255)"`
+ MeetingDuration string `json:"meeting_duration" gorm:"type:varchar(255)"`
+ UserType int64 `json:"user_type"`
+}
+
+func (FeishuMeetingTopUserItem20220524Temp) TableName() string {
+ return "_tool_feishu_meeting_top_user_items_tmp"
+}
+
+type FeishuMeetingTopUserItem20220524 struct {
+}
+
+func (FeishuMeetingTopUserItem20220524) TableName() string {
+ return "_tool_feishu_meeting_top_user_items"
+}
+
+type UpdateSchemas20220524 struct{}
+
+func (*UpdateSchemas20220524) Up(ctx context.Context, db *gorm.DB) error {
+ cursor, err := db.Model(archived.FeishuMeetingTopUserItem{}).Rows()
+ if err != nil {
+ return err
+ }
+ defer cursor.Close()
+ // 1. create a temporary table to store unique records
+ err = db.Migrator().CreateTable(FeishuMeetingTopUserItem20220524Temp{})
+ if err != nil {
+ return err
+ }
+ // 2. dedupe records and insert into the temporary table
+ for cursor.Next() {
+ inputRow := FeishuMeetingTopUserItem20220524Temp{}
+ err := db.ScanRows(cursor, &inputRow)
+ if err != nil {
+ return err
+ }
+ err = db.Clauses(clause.OnConflict{UpdateAll: true}).Create(inputRow).Error
+ if err != nil {
+ return err
+ }
+ }
+ // 3. drop old table
+ err = db.Migrator().DropTable(archived.FeishuMeetingTopUserItem{})
+ if err != nil {
+ return err
+ }
+ // 4. rename the temporary table to the old table
+ return db.Migrator().RenameTable(FeishuMeetingTopUserItem20220524Temp{}, FeishuMeetingTopUserItem20220524{})
+}
+
+func (*UpdateSchemas20220524) Version() uint64 {
+ return 20220524000001
+}
+
+func (*UpdateSchemas20220524) Name() string {
+ return "change primary column `id` to start_time+name"
+}
diff --git a/plugins/feishu/tasks/meeting_top_user_item_extractor.go b/plugins/feishu/tasks/meeting_top_user_item_extractor.go
index 6459447d..fd4c3ca5 100644
--- a/plugins/feishu/tasks/meeting_top_user_item_extractor.go
+++ b/plugins/feishu/tasks/meeting_top_user_item_extractor.go
@@ -27,8 +27,7 @@ import (
var _ core.SubTaskEntryPoint = ExtractMeetingTopUserItem
func ExtractMeetingTopUserItem(taskCtx core.SubTaskContext) error {
-
- exetractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{
+ extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{
RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
Ctx: taskCtx,
Params: FeishuApiParams{
@@ -62,12 +61,12 @@ func ExtractMeetingTopUserItem(taskCtx core.SubTaskContext) error {
return err
}
- return exetractor.Execute()
+ return extractor.Execute()
}
var ExtractMeetingTopUserItemMeta = core.SubTaskMeta{
Name: "extractMeetingTopUserItem",
EntryPoint: ExtractMeetingTopUserItem,
EnabledByDefault: true,
- Description: "Extrat raw top user meeting data into tool layer table feishu_meeting_top_user_item",
+ Description: "Extract raw top user meeting data into tool layer table feishu_meeting_top_user_item",
}
diff --git a/plugins/helper/batch_save.go b/plugins/helper/batch_save.go
index 832dec83..2f9d20b7 100644
--- a/plugins/helper/batch_save.go
+++ b/plugins/helper/batch_save.go
@@ -112,14 +112,10 @@ func hasPrimaryKey(ifv reflect.Type) bool {
}
for i := 0; i < ifv.NumField(); i++ {
v := ifv.Field(i)
- switch v.Type.Kind() {
- case reflect.Struct:
- ok := hasPrimaryKey(v.Type)
- if ok {
- return true
- }
- default:
- if ok := isPrimaryKey(v); ok {
+ if ok := isPrimaryKey(v); ok {
+ return true
+ } else if v.Type.Kind() == reflect.Struct {
+ if ok := hasPrimaryKey(v.Type); ok {
return true
}
}
@@ -135,18 +131,15 @@ func getPrimaryKeyValue(iface interface{}) string {
}
for i := 0; i < ifv.NumField(); i++ {
v := ifv.Field(i)
- switch v.Kind() {
- case reflect.Struct:
- s := getPrimaryKeyValue(v.Interface())
+ if isPrimaryKey(ifv.Type().Field(i)) {
+ s := fmt.Sprintf("%v", v.Interface())
if s != "" {
ss = append(ss, s)
}
- default:
- if isPrimaryKey(ifv.Type().Field(i)) {
- s := fmt.Sprintf("%v", v.Interface())
- if s != "" {
- ss = append(ss, s)
- }
+ } else if v.Kind() == reflect.Struct {
+ s := getPrimaryKeyValue(v.Interface())
+ if s != "" {
+ ss = append(ss, s)
}
}
}
diff --git a/runner/directrun.go b/runner/directrun.go
index 98e7b095..5e5851e0 100644
--- a/runner/directrun.go
+++ b/runner/directrun.go
@@ -21,6 +21,7 @@ import (
"context"
"github.com/apache/incubator-devlake/config"
"github.com/apache/incubator-devlake/logger"
+ "github.com/apache/incubator-devlake/migration"
"github.com/apache/incubator-devlake/plugins/core"
"github.com/spf13/cobra"
"os"
@@ -64,6 +65,16 @@ func DirectRun(cmd *cobra.Command, args []string, pluginTask core.PluginTask, op
panic(err)
}
+ // collect migration and run
+ migration.Init(db)
+ if migratable, ok := pluginTask.(core.Migratable); ok {
+ migration.Register(migratable.MigrationScripts(), cmd.Use)
+ }
+ err = migration.Execute(context.Background())
+ if err != nil {
+ panic(err)
+ }
+
ctx, cancel := context.WithCancel(context.Background())
sigc := make(chan os.Signal, 1)
signal.Notify(sigc, syscall.SIGTSTP)