(incubator-devlake) branch main updated: feat: starrocsk plugin add table config (#8286)

[kl...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

klesh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git


The following commit(s) were added to refs/heads/main by this push:
     new afcb66ba3 feat: starrocsk plugin add table config (#8286)
afcb66ba3 is described below

commit afcb66ba34dc6e28be93c978d8c273c9b64ead21
Author: long2ice <ji...@merico.dev>
AuthorDate: Wed Feb 5 16:41:21 2025 +0800

    feat: starrocsk plugin add table config (#8286)
---
 backend/plugins/starrocks/tasks/task_data.go | 1 +
 backend/plugins/starrocks/tasks/tasks.go     | 8 +++++++-
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/backend/plugins/starrocks/tasks/task_data.go b/backend/plugins/starrocks/tasks/task_data.go
index f178cba6a..c90598ef2 100644
--- a/backend/plugins/starrocks/tasks/task_data.go
+++ b/backend/plugins/starrocks/tasks/task_data.go
@@ -20,6 +20,7 @@ package tasks
 type TableConfig struct {
 	IncludedColumns []string `mapstructure:"included_columns"`
 	ExcludedColumns []string `mapstructure:"excluded_columns"`
+	Where           string   `mapstructure:"where"`
 }
 
 type StarRocksConfig struct {
diff --git a/backend/plugins/starrocks/tasks/tasks.go b/backend/plugins/starrocks/tasks/tasks.go
index b6c60e03d..79fbe5d2a 100644
--- a/backend/plugins/starrocks/tasks/tasks.go
+++ b/backend/plugins/starrocks/tasks/tasks.go
@@ -269,13 +269,18 @@ func copyDataToDst(dc *DataConfigParams, columnMap map[string]string, orderBy st
 	table := dc.SrcTableName
 	starrocksTable := dc.DestTableName
 	starrocksTmpTable := fmt.Sprintf("%s_tmp", starrocksTable)
-
+	tableConfig, ok := config.TableConfigs[table]
+	where := ""
+	if ok {
+		where = tableConfig.Where
+	}
 	var offset int
 	var err error
 	var rows dal.Rows
 	rows, err = db.Cursor(
 		dal.From(table),
 		dal.Orderby(orderBy),
+		dal.Where(where),
 	)
 	if err != nil {
 		if strings.Contains(err.Error(), "cached plan must not change result type") {
@@ -283,6 +288,7 @@ func copyDataToDst(dc *DataConfigParams, columnMap map[string]string, orderBy st
 			rows, err = db.Cursor(
 				dal.From(table),
 				dal.Orderby(orderBy),
+				dal.Where(where),
 			)
 			if err != nil {
 				return err