You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by zh...@apache.org on 2022/09/09 07:52:57 UTC
[incubator-devlake] branch main updated: feat: implement API for plugin `customize` (#2911)
This is an automated email from the ASF dual-hosted git repository.
zhangliang2022 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 075e3d3b feat: implement API for plugin `customize` (#2911)
075e3d3b is described below
commit 075e3d3b480e407aede5a0893bd61de9b0fd051d
Author: mindlesscloud <li...@merico.dev>
AuthorDate: Fri Sep 9 15:52:53 2022 +0800
feat: implement API for plugin `customize` (#2911)
* feat: add new plugin
* feat: sub-task ExtractCustomizedFields for plugin customize
* refactor: new task Options
* fix: update swagger docs
* fix: fix e2e test
* refactor: use join statement reduce query times
* refactor: switch to new error type
---
go.mod | 3 +
go.sum | 6 +
impl/dalgorm/dalgorm.go | 10 ++
plugins/core/dal/dal.go | 4 +
plugins/customize/README.md | 1 +
plugins/customize/api/api.go | 163 +++++++++++++++++++++
plugins/customize/api/blueprint.go | 55 +++++++
plugins/customize/api/swagger.go | 52 +++++++
plugins/customize/customize.go | 42 ++++++
plugins/customize/e2e/extract_fields_test.go | 62 ++++++++
.../e2e/raw_tables/_raw_jira_api_issues.csv | 31 ++++
plugins/customize/e2e/raw_tables/issues.csv | 31 ++++
plugins/customize/e2e/snapshot_tables/issues.csv | 31 ++++
plugins/customize/impl/impl.go | 88 +++++++++++
plugins/customize/models/model.go | 26 ++++
.../customize/tasks/customized_fields_extractor.go | 134 +++++++++++++++++
plugins/customize/tasks/task_data.go | 33 +++++
17 files changed, 772 insertions(+)
diff --git a/go.mod b/go.mod
index c99ecc60..1b88ccd5 100644
--- a/go.mod
+++ b/go.mod
@@ -25,6 +25,7 @@ require (
github.com/stretchr/testify v1.7.0
github.com/swaggo/gin-swagger v1.4.3
github.com/swaggo/swag v1.8.3
+ github.com/tidwall/gjson v1.14.3
github.com/viant/afs v1.16.0
github.com/x-cray/logrus-prefixed-formatter v0.5.2
go.temporal.io/api v1.7.1-0.20220223032354-6e6fe738916a
@@ -113,6 +114,8 @@ require (
github.com/spf13/pflag v1.0.6-0.20200504143853-81378bbcd8a1 // indirect
github.com/stretchr/objx v0.3.0 // indirect
github.com/subosito/gotenv v1.2.0 // indirect
+ github.com/tidwall/match v1.1.1 // indirect
+ github.com/tidwall/pretty v1.2.0 // indirect
github.com/ugorji/go/codec v1.2.6 // indirect
github.com/xanzy/ssh-agent v0.3.0 // indirect
go.uber.org/atomic v1.9.0 // indirect
diff --git a/go.sum b/go.sum
index dff4852e..367862a6 100644
--- a/go.sum
+++ b/go.sum
@@ -676,6 +676,12 @@ github.com/swaggo/gin-swagger v1.4.3/go.mod h1:hBg6tGeKJsUu/P79BH+WGUR8nq2LuGE0O
github.com/swaggo/swag v1.8.1/go.mod h1:ugemnJsPZm/kRwFUnzBlbHRd0JY9zE1M4F+uy2pAaPQ=
github.com/swaggo/swag v1.8.3 h1:3pZSSCQ//gAH88lfmxM3Cd1+JCsxV8Md6f36b9hrZ5s=
github.com/swaggo/swag v1.8.3/go.mod h1:jMLeXOOmYyjk8PvHTsXBdrubsNd9gUJTTCzL5iBnseg=
+github.com/tidwall/gjson v1.14.3 h1:9jvXn7olKEHU1S9vwoMGliaT8jq1vJ7IH/n9zD9Dnlw=
+github.com/tidwall/gjson v1.14.3/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
+github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
+github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
+github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs=
+github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=
github.com/ugorji/go v1.2.6/go.mod h1:anCg0y61KIhDlPZmnH+so+RQbysYVyDko0IMgJv0Nn0=
diff --git a/impl/dalgorm/dalgorm.go b/impl/dalgorm/dalgorm.go
index 82adf354..2b41cfe6 100644
--- a/impl/dalgorm/dalgorm.go
+++ b/impl/dalgorm/dalgorm.go
@@ -162,6 +162,16 @@ func (d *Dalgorm) GetColumns(dst schema.Tabler, filter func(columnMeta dal.Colum
return cms, nil
}
+// AddColumn add one column for the table
+func (d *Dalgorm) AddColumn(table, columnName, columnType string) error {
+ return d.Exec("ALTER TABLE ? ADD ? ?", clause.Table{Name: table}, clause.Column{Name: columnName}, clause.Expr{SQL: columnType})
+}
+
+// DropColumn drop one column from the table
+func (d *Dalgorm) DropColumn(table, columnName string) error {
+ return d.Exec("ALTER TABLE ? DROP COLUMN ?", clause.Table{Name: table}, clause.Column{Name: columnName})
+}
+
// GetPrimaryKeyFields get the PrimaryKey from `gorm` tag
func (d *Dalgorm) GetPrimaryKeyFields(t reflect.Type) []reflect.StructField {
return utils.WalkFields(t, func(field *reflect.StructField) bool {
diff --git a/plugins/core/dal/dal.go b/plugins/core/dal/dal.go
index 11ab5d85..810609c9 100644
--- a/plugins/core/dal/dal.go
+++ b/plugins/core/dal/dal.go
@@ -49,6 +49,10 @@ type ColumnMeta interface {
type Dal interface {
// AutoMigrate runs auto migration for given entity
AutoMigrate(entity interface{}, clauses ...Clause) error
+ // AddColumn add column for the table
+ AddColumn(table, columnName, columnType string) error
+ // DropColumn drop column from the table
+ DropColumn(table, columnName string) error
// Exec executes raw sql query
Exec(query string, params ...interface{}) error
// RawCursor executes raw sql query and returns a database cursor
diff --git a/plugins/customize/README.md b/plugins/customize/README.md
new file mode 100644
index 00000000..825e35f0
--- /dev/null
+++ b/plugins/customize/README.md
@@ -0,0 +1 @@
+Please see details in the [Apache DevLake website](https://devlake.apache.org/docs/Plugins/customize)
\ No newline at end of file
diff --git a/plugins/customize/api/api.go b/plugins/customize/api/api.go
new file mode 100644
index 00000000..08312c43
--- /dev/null
+++ b/plugins/customize/api/api.go
@@ -0,0 +1,163 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package api
+
+import (
+ "fmt"
+ "net/http"
+ "strings"
+
+ "github.com/apache/incubator-devlake/errors"
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/core/dal"
+ "github.com/apache/incubator-devlake/plugins/customize/models"
+)
+
+type field struct {
+ ColumnName string `json:"columnName"`
+ ColumnType string `json:"columnType"`
+}
+
+func getFields(d dal.Dal, tbl string) ([]field, error) {
+ columns, err := d.GetColumns(&models.Table{tbl}, func(columnMeta dal.ColumnMeta) bool {
+ return strings.HasPrefix(columnMeta.Name(), "x_")
+ })
+ if err != nil {
+ return nil, errors.Default.Wrap(err, "GetColumns error")
+ }
+ var result []field
+ for _, col := range columns {
+ result = append(result, field{
+ ColumnName: col.Name(),
+ ColumnType: "VARCHAR(255)",
+ })
+ }
+ return result, nil
+}
+func checkField(d dal.Dal, table, field string) (bool, error) {
+ if !strings.HasPrefix(field, "x_") {
+ return false, errors.Default.New("column name should start with `x_`")
+ }
+ fields, err := getFields(d, table)
+ if err != nil {
+ return false, err
+ }
+ for _, fld := range fields {
+ if fld.ColumnName == field {
+ return true, nil
+ }
+ }
+ return false, nil
+}
+
+func CreateField(d dal.Dal, table, field string) error {
+ exists, err := checkField(d, table, field)
+ if err != nil {
+ return err
+ }
+ if exists {
+ return nil
+ }
+ err = d.AddColumn(table, field, "VARCHAR(255)")
+ if err != nil {
+ return errors.Default.Wrap(err, "AddColumn error")
+ }
+ return nil
+}
+
+func deleteField(d dal.Dal, table, field string) error {
+ exists, err := checkField(d, table, field)
+ if err != nil {
+ return err
+ }
+ if !exists {
+ return nil
+ }
+ err = d.DropColumn(table, field)
+ if err != nil {
+ return errors.Default.Wrap(err, "DropColumn error")
+ }
+ return nil
+}
+
+type input struct {
+ Name string `json:"name" example:"x_new_column"`
+}
+type Handlers struct {
+ dal dal.Dal
+}
+
+func NewHandlers(dal dal.Dal) *Handlers {
+ return &Handlers{dal: dal}
+}
+
+// ListFields return all customized fields
+// @Summary return all customized fields
+// @Description return all customized fields
+// @Tags plugins/customize
+// @Success 200 {object} shared.ApiBody "Success"
+// @Failure 400 {string} errcode.Error "Bad Request"
+// @Failure 500 {string} errcode.Error "Internel Error"
+// @Router /plugins/customize/{table}/fields [GET]
+func (h *Handlers) ListFields(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) {
+ fields, err := getFields(h.dal, input.Params["table"])
+ if err != nil {
+ return &core.ApiResourceOutput{Status: http.StatusBadRequest}, errors.Default.Wrap(err, "getFields error")
+ }
+ return &core.ApiResourceOutput{Body: fields, Status: http.StatusOK}, nil
+}
+
+// CreateFields create a customized field
+// @Summary create a customized field
+// @Description create a customized field
+// @Tags plugins/customize
+// @Param request body input true "request body"
+// @Success 200 {object} shared.ApiBody "Success"
+// @Failure 400 {string} errcode.Error "Bad Request"
+// @Failure 500 {string} errcode.Error "Internel Error"
+// @Router /plugins/customize/{table}/fields [POST]
+func (h *Handlers) CreateFields(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) {
+ table := input.Params["table"]
+ fld, ok := input.Body["name"].(string)
+ if !ok {
+ return &core.ApiResourceOutput{Status: http.StatusBadRequest}, fmt.Errorf("the name is not string")
+ }
+ err := CreateField(h.dal, table, fld)
+ if err != nil {
+ return nil, errors.Default.Wrap(err, "CreateField error")
+ }
+ return &core.ApiResourceOutput{Body: field{fld, "varchar(255)"}, Status: http.StatusOK}, nil
+}
+
+// DeleteField delete a customized fields
+// @Summary return all customized fields
+// @Description return all customized fields
+// @Tags plugins/customize
+// @Success 200 {object} shared.ApiBody "Success"
+// @Failure 400 {string} errcode.Error "Bad Request"
+// @Failure 500 {string} errcode.Error "Internel Error"
+// @Router /plugins/customize/{table}/fields [DELETE]
+func (h *Handlers) DeleteField(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) {
+ table := input.Params["table"]
+ fld := input.Params["field"]
+ err := deleteField(h.dal, table, fld)
+ if err != nil {
+ return &core.ApiResourceOutput{Status: http.StatusBadRequest}, errors.Default.Wrap(err, "deleteField error")
+ }
+ return &core.ApiResourceOutput{Status: http.StatusOK}, nil
+}
diff --git a/plugins/customize/api/blueprint.go b/plugins/customize/api/blueprint.go
new file mode 100644
index 00000000..2e12ca5e
--- /dev/null
+++ b/plugins/customize/api/blueprint.go
@@ -0,0 +1,55 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package api
+
+import (
+ "encoding/json"
+
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/helper"
+ "github.com/apache/incubator-devlake/plugins/jira/tasks"
+)
+
+func MakePipelinePlan(subtaskMetas []core.SubTaskMeta, connectionId uint64, scope []*core.BlueprintScopeV100) (core.PipelinePlan, error) {
+ var err error
+ plan := make(core.PipelinePlan, len(scope))
+ for i, scopeElem := range scope {
+ taskOptions := make(map[string]interface{})
+ err = json.Unmarshal(scopeElem.Options, &taskOptions)
+ if err != nil {
+ return nil, err
+ }
+ _, err := tasks.DecodeAndValidateTaskOptions(taskOptions)
+ if err != nil {
+ return nil, err
+ }
+ // subtasks
+ subtasks, err := helper.MakePipelinePlanSubtasks(subtaskMetas, scopeElem.Entities)
+ if err != nil {
+ return nil, err
+ }
+ plan[i] = core.PipelineStage{
+ {
+ Plugin: "customize",
+ Subtasks: subtasks,
+ Options: taskOptions,
+ },
+ }
+ }
+ return plan, nil
+}
diff --git a/plugins/customize/api/swagger.go b/plugins/customize/api/swagger.go
new file mode 100644
index 00000000..7d8d3be1
--- /dev/null
+++ b/plugins/customize/api/swagger.go
@@ -0,0 +1,52 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package api
+
+import "github.com/apache/incubator-devlake/plugins/customize/tasks"
+
+// @Summary blueprints setting for customize
+// @Description blueprint setting for customize
+// @Tags plugins/customize
+// @Accept application/json
+// @Param blueprint-setting body blueprintSetting true "json"
+// @Router /blueprints/customize/blueprint-setting [post]
+func _() {}
+
+type blueprintSetting []struct {
+ Version string `json:"version" example:"1.0.0"`
+ Connections []struct {
+ Plugin string `json:"plugin" example:"customize"`
+ Scope []struct {
+ Options tasks.Options `json:"options"`
+ } `json:"scope"`
+ } `json:"connections"`
+}
+
+// @Summary pipelines plan for customize
+// @Description pipelines plan for customize
+// @Tags plugins/customize
+// @Accept application/json
+// @Param pipeline-plan body pipelinePlan true "json"
+// @Router /pipelines/customize/pipeline-plan [post]
+func _() {}
+
+type pipelinePlan [][]struct {
+ Plugin string `json:"plugin"`
+ Subtasks []string `json:"subtasks"`
+ Options tasks.Options `json:"options"`
+}
diff --git a/plugins/customize/customize.go b/plugins/customize/customize.go
new file mode 100644
index 00000000..644c8359
--- /dev/null
+++ b/plugins/customize/customize.go
@@ -0,0 +1,42 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main // must be main for plugin entry point
+
+import (
+ "github.com/apache/incubator-devlake/plugins/customize/impl"
+ "github.com/apache/incubator-devlake/plugins/customize/tasks"
+ "github.com/apache/incubator-devlake/runner"
+ "github.com/spf13/cobra"
+)
+
+var PluginEntry impl.Customize //nolint
+
+// standalone mode for debugging
+func main() {
+ cmd := &cobra.Command{Use: "customize"}
+ cmd.Run = func(c *cobra.Command, args []string) {
+ runner.DirectRun(c, args, PluginEntry, map[string]interface{}{"transformationRules": []tasks.MappingRules{{
+ Table: "issues",
+ RawDataTable: "_raw_jira_api_issues",
+ RawDataParams: "{\"ConnectionId\":1,\"BoardId\":8}",
+ Mapping: map[string]string{"x_test": "fields.timespent"},
+ }},
+ })
+ }
+ runner.RunCmd(cmd)
+}
diff --git a/plugins/customize/e2e/extract_fields_test.go b/plugins/customize/e2e/extract_fields_test.go
new file mode 100644
index 00000000..10a1bbc3
--- /dev/null
+++ b/plugins/customize/e2e/extract_fields_test.go
@@ -0,0 +1,62 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+ "testing"
+
+ "github.com/apache/incubator-devlake/helpers/e2ehelper"
+ "github.com/apache/incubator-devlake/models/domainlayer/ticket"
+ "github.com/apache/incubator-devlake/plugins/customize/api"
+ "github.com/apache/incubator-devlake/plugins/customize/impl"
+ "github.com/apache/incubator-devlake/plugins/customize/tasks"
+)
+
+func TestBoardDataFlow(t *testing.T) {
+ var plugin impl.Customize
+ dataflowTester := e2ehelper.NewDataFlowTester(t, "customize", plugin)
+
+ taskData := &tasks.TaskData{
+ Options: &tasks.Options{[]tasks.MappingRules{{
+ Table: "issues",
+ RawDataTable: "_raw_jira_api_issues",
+ RawDataParams: "{\"ConnectionId\":1,\"BoardId\":8}",
+ Mapping: map[string]string{"x_test": "fields.created"},
+ }}}}
+
+ // import raw data table
+ dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_jira_api_issues.csv", "_raw_jira_api_issues")
+ dataflowTester.ImportCsvIntoTabler("./raw_tables/issues.csv", &ticket.Issue{})
+ err := api.CreateField(dataflowTester.Dal, "issues", "x_test")
+ if err != nil {
+ t.Fatal(err)
+ }
+ // verify extension fields extraction
+ dataflowTester.Subtask(tasks.ExtractCustomizedFieldsMeta, taskData)
+ dataflowTester.VerifyTable(
+ ticket.Issue{},
+ "./snapshot_tables/issues.csv",
+ []string{
+ "id",
+ "_raw_data_params",
+ "_raw_data_table",
+ "_raw_data_id",
+ "x_test",
+ },
+ )
+}
diff --git a/plugins/customize/e2e/raw_tables/_raw_jira_api_issues.csv b/plugins/customize/e2e/raw_tables/_raw_jira_api_issues.csv
new file mode 100644
index 00000000..8b546b68
--- /dev/null
+++ b/plugins/customize/e2e/raw_tables/_raw_jira_api_issues.csv
@@ -0,0 +1,31 @@
+"id","params","data","url","input","created_at"
+1701,"{""ConnectionId"":1,""BoardId"":8}","{""expand"":""operations,versionedRepresentations,editmeta,changelog,renderedFields"",""id"":""10063"",""self"":""https://merico.atlassian.net/rest/agile/1.0/issue/10063"",""key"":""EE-1"",""changelog"":{""startAt"":0,""maxResults"":25,""total"":25,""histories"":[{""id"":""90706"",""author"":{""self"":""https://merico.atlassian.net/rest/api/2/user?accountId=5e9711ba34f7b90c0fbc37d3"",""accountId"":""5e9711ba34f7b90c0fbc37d3"",""avatarUrls"":{""4 [...]
+1702,"{""ConnectionId"":1,""BoardId"":8}","{""expand"":""operations,versionedRepresentations,editmeta,changelog,renderedFields"",""id"":""10064"",""self"":""https://merico.atlassian.net/rest/agile/1.0/issue/10064"",""key"":""EE-2"",""changelog"":{""startAt"":0,""maxResults"":24,""total"":24,""histories"":[{""id"":""90429"",""author"":{""self"":""https://merico.atlassian.net/rest/api/2/user?accountId=5e9711ba34f7b90c0fbc37d3"",""accountId"":""5e9711ba34f7b90c0fbc37d3"",""avatarUrls"":{""4 [...]
+1703,"{""ConnectionId"":1,""BoardId"":8}","{""expand"":""operations,versionedRepresentations,editmeta,changelog,renderedFields"",""id"":""10065"",""self"":""https://merico.atlassian.net/rest/agile/1.0/issue/10065"",""key"":""EE-3"",""changelog"":{""startAt"":0,""maxResults"":26,""total"":26,""histories"":[{""id"":""90430"",""author"":{""self"":""https://merico.atlassian.net/rest/api/2/user?accountId=5e9711ba34f7b90c0fbc37d3"",""accountId"":""5e9711ba34f7b90c0fbc37d3"",""avatarUrls"":{""4 [...]
+1704,"{""ConnectionId"":1,""BoardId"":8}","{""expand"":""operations,versionedRepresentations,editmeta,changelog,renderedFields"",""id"":""10066"",""self"":""https://merico.atlassian.net/rest/agile/1.0/issue/10066"",""key"":""EE-4"",""changelog"":{""startAt"":0,""maxResults"":23,""total"":23,""histories"":[{""id"":""90722"",""author"":{""self"":""https://merico.atlassian.net/rest/api/2/user?accountId=5e9711ba34f7b90c0fbc37d3"",""accountId"":""5e9711ba34f7b90c0fbc37d3"",""avatarUrls"":{""4 [...]
+1705,"{""ConnectionId"":1,""BoardId"":8}","{""expand"":""operations,versionedRepresentations,editmeta,changelog,renderedFields"",""id"":""10067"",""self"":""https://merico.atlassian.net/rest/agile/1.0/issue/10067"",""key"":""EE-5"",""changelog"":{""startAt"":0,""maxResults"":18,""total"":18,""histories"":[{""id"":""295895"",""author"":{""self"":""https://merico.atlassian.net/rest/api/2/user?accountId=5ecfbd0c2490cf0c09e2e598"",""accountId"":""5ecfbd0c2490cf0c09e2e598"",""avatarUrls"":{"" [...]
+1706,"{""ConnectionId"":1,""BoardId"":8}","{""expand"":""operations,versionedRepresentations,editmeta,changelog,renderedFields"",""id"":""10068"",""self"":""https://merico.atlassian.net/rest/agile/1.0/issue/10068"",""key"":""EE-6"",""changelog"":{""startAt"":0,""maxResults"":17,""total"":17,""histories"":[{""id"":""90473"",""author"":{""self"":""https://merico.atlassian.net/rest/api/2/user?accountId=5e9711ba34f7b90c0fbc37d3"",""accountId"":""5e9711ba34f7b90c0fbc37d3"",""avatarUrls"":{""4 [...]
+1707,"{""ConnectionId"":1,""BoardId"":8}","{""expand"":""operations,versionedRepresentations,editmeta,changelog,renderedFields"",""id"":""10070"",""self"":""https://merico.atlassian.net/rest/agile/1.0/issue/10070"",""key"":""EE-8"",""changelog"":{""startAt"":0,""maxResults"":17,""total"":17,""histories"":[{""id"":""295892"",""author"":{""self"":""https://merico.atlassian.net/rest/api/2/user?accountId=5ecfbd0c2490cf0c09e2e598"",""accountId"":""5ecfbd0c2490cf0c09e2e598"",""avatarUrls"":{"" [...]
+1708,"{""ConnectionId"":1,""BoardId"":8}","{""expand"":""operations,versionedRepresentations,editmeta,changelog,renderedFields"",""id"":""10071"",""self"":""https://merico.atlassian.net/rest/agile/1.0/issue/10071"",""key"":""EE-9"",""changelog"":{""startAt"":0,""maxResults"":18,""total"":18,""histories"":[{""id"":""296002"",""author"":{""self"":""https://merico.atlassian.net/rest/api/2/user?accountId=5ecfbd0c2490cf0c09e2e598"",""accountId"":""5ecfbd0c2490cf0c09e2e598"",""avatarUrls"":{"" [...]
+1709,"{""ConnectionId"":1,""BoardId"":8}","{""expand"":""operations,versionedRepresentations,editmeta,changelog,renderedFields"",""id"":""10072"",""self"":""https://merico.atlassian.net/rest/agile/1.0/issue/10072"",""key"":""EE-10"",""changelog"":{""startAt"":0,""maxResults"":17,""total"":17,""histories"":[{""id"":""295894"",""author"":{""self"":""https://merico.atlassian.net/rest/api/2/user?accountId=5ecfbd0c2490cf0c09e2e598"",""accountId"":""5ecfbd0c2490cf0c09e2e598"",""avatarUrls"":{" [...]
+1710,"{""ConnectionId"":1,""BoardId"":8}","{""expand"":""operations,versionedRepresentations,editmeta,changelog,renderedFields"",""id"":""10076"",""self"":""https://merico.atlassian.net/rest/agile/1.0/issue/10076"",""key"":""EE-14"",""changelog"":{""startAt"":0,""maxResults"":18,""total"":18,""histories"":[{""id"":""295897"",""author"":{""self"":""https://merico.atlassian.net/rest/api/2/user?accountId=5ecfbd0c2490cf0c09e2e598"",""accountId"":""5ecfbd0c2490cf0c09e2e598"",""avatarUrls"":{" [...]
+1711,"{""ConnectionId"":1,""BoardId"":8}","{""expand"":""operations,versionedRepresentations,editmeta,changelog,renderedFields"",""id"":""10077"",""self"":""https://merico.atlassian.net/rest/agile/1.0/issue/10077"",""key"":""EE-15"",""changelog"":{""startAt"":0,""maxResults"":20,""total"":20,""histories"":[{""id"":""296102"",""author"":{""self"":""https://merico.atlassian.net/rest/api/2/user?accountId=5ecfbd0c2490cf0c09e2e598"",""accountId"":""5ecfbd0c2490cf0c09e2e598"",""avatarUrls"":{" [...]
+1712,"{""ConnectionId"":1,""BoardId"":8}","{""expand"":""operations,versionedRepresentations,editmeta,changelog,renderedFields"",""id"":""10078"",""self"":""https://merico.atlassian.net/rest/agile/1.0/issue/10078"",""key"":""EE-16"",""changelog"":{""startAt"":0,""maxResults"":18,""total"":18,""histories"":[{""id"":""295898"",""author"":{""self"":""https://merico.atlassian.net/rest/api/2/user?accountId=5ecfbd0c2490cf0c09e2e598"",""accountId"":""5ecfbd0c2490cf0c09e2e598"",""avatarUrls"":{" [...]
+1713,"{""ConnectionId"":1,""BoardId"":8}","{""expand"":""operations,versionedRepresentations,editmeta,changelog,renderedFields"",""id"":""10079"",""self"":""https://merico.atlassian.net/rest/agile/1.0/issue/10079"",""key"":""EE-17"",""changelog"":{""startAt"":0,""maxResults"":31,""total"":31,""histories"":[{""id"":""296582"",""author"":{""self"":""https://merico.atlassian.net/rest/api/2/user?accountId=5ecfbd0c2490cf0c09e2e598"",""accountId"":""5ecfbd0c2490cf0c09e2e598"",""avatarUrls"":{" [...]
+1714,"{""ConnectionId"":1,""BoardId"":8}","{""expand"":""operations,versionedRepresentations,editmeta,changelog,renderedFields"",""id"":""10081"",""self"":""https://merico.atlassian.net/rest/agile/1.0/issue/10081"",""key"":""EE-19"",""changelog"":{""startAt"":0,""maxResults"":17,""total"":17,""histories"":[{""id"":""90485"",""author"":{""self"":""https://merico.atlassian.net/rest/api/2/user?accountId=5e9711ba34f7b90c0fbc37d3"",""accountId"":""5e9711ba34f7b90c0fbc37d3"",""avatarUrls"":{"" [...]
+1715,"{""ConnectionId"":1,""BoardId"":8}","{""expand"":""operations,versionedRepresentations,editmeta,changelog,renderedFields"",""id"":""10082"",""self"":""https://merico.atlassian.net/rest/agile/1.0/issue/10082"",""key"":""EE-20"",""changelog"":{""startAt"":0,""maxResults"":14,""total"":14,""histories"":[{""id"":""142717"",""author"":{""self"":""https://merico.atlassian.net/rest/api/2/user?accountId=5ecfbd0984083c0c12e5af8f"",""accountId"":""5ecfbd0984083c0c12e5af8f"",""avatarUrls"":{" [...]
+1716,"{""ConnectionId"":1,""BoardId"":8}","{""expand"":""operations,versionedRepresentations,editmeta,changelog,renderedFields"",""id"":""10085"",""self"":""https://merico.atlassian.net/rest/agile/1.0/issue/10085"",""key"":""EE-23"",""changelog"":{""startAt"":0,""maxResults"":20,""total"":20,""histories"":[{""id"":""123813"",""author"":{""self"":""https://merico.atlassian.net/rest/api/2/user?accountId=5e9711ba34f7b90c0fbc37d3"",""accountId"":""5e9711ba34f7b90c0fbc37d3"",""avatarUrls"":{" [...]
+1717,"{""ConnectionId"":1,""BoardId"":8}","{""expand"":""operations,versionedRepresentations,editmeta,changelog,renderedFields"",""id"":""10086"",""self"":""https://merico.atlassian.net/rest/agile/1.0/issue/10086"",""key"":""EE-24"",""changelog"":{""startAt"":0,""maxResults"":11,""total"":11,""histories"":[{""id"":""90454"",""author"":{""self"":""https://merico.atlassian.net/rest/api/2/user?accountId=5e9711ba34f7b90c0fbc37d3"",""accountId"":""5e9711ba34f7b90c0fbc37d3"",""avatarUrls"":{"" [...]
+1718,"{""ConnectionId"":1,""BoardId"":8}","{""expand"":""operations,versionedRepresentations,editmeta,changelog,renderedFields"",""id"":""10087"",""self"":""https://merico.atlassian.net/rest/agile/1.0/issue/10087"",""key"":""EE-25"",""changelog"":{""startAt"":0,""maxResults"":17,""total"":17,""histories"":[{""id"":""90481"",""author"":{""self"":""https://merico.atlassian.net/rest/api/2/user?accountId=5e9711ba34f7b90c0fbc37d3"",""accountId"":""5e9711ba34f7b90c0fbc37d3"",""avatarUrls"":{"" [...]
+1719,"{""ConnectionId"":1,""BoardId"":8}","{""expand"":""operations,versionedRepresentations,editmeta,changelog,renderedFields"",""id"":""10088"",""self"":""https://merico.atlassian.net/rest/agile/1.0/issue/10088"",""key"":""EE-26"",""changelog"":{""startAt"":0,""maxResults"":14,""total"":14,""histories"":[{""id"":""90445"",""author"":{""self"":""https://merico.atlassian.net/rest/api/2/user?accountId=5e9711ba34f7b90c0fbc37d3"",""accountId"":""5e9711ba34f7b90c0fbc37d3"",""avatarUrls"":{"" [...]
+1720,"{""ConnectionId"":1,""BoardId"":8}","{""expand"":""operations,versionedRepresentations,editmeta,changelog,renderedFields"",""id"":""10089"",""self"":""https://merico.atlassian.net/rest/agile/1.0/issue/10089"",""key"":""EE-27"",""changelog"":{""startAt"":0,""maxResults"":15,""total"":15,""histories"":[{""id"":""90451"",""author"":{""self"":""https://merico.atlassian.net/rest/api/2/user?accountId=5e9711ba34f7b90c0fbc37d3"",""accountId"":""5e9711ba34f7b90c0fbc37d3"",""avatarUrls"":{"" [...]
+1721,"{""ConnectionId"":1,""BoardId"":8}","{""expand"":""operations,versionedRepresentations,editmeta,changelog,renderedFields"",""id"":""10090"",""self"":""https://merico.atlassian.net/rest/agile/1.0/issue/10090"",""key"":""EE-28"",""changelog"":{""startAt"":0,""maxResults"":14,""total"":14,""histories"":[{""id"":""90642"",""author"":{""self"":""https://merico.atlassian.net/rest/api/2/user?accountId=5e9711ba34f7b90c0fbc37d3"",""accountId"":""5e9711ba34f7b90c0fbc37d3"",""avatarUrls"":{"" [...]
+1722,"{""ConnectionId"":1,""BoardId"":8}","{""expand"":""operations,versionedRepresentations,editmeta,changelog,renderedFields"",""id"":""10091"",""self"":""https://merico.atlassian.net/rest/agile/1.0/issue/10091"",""key"":""EE-29"",""changelog"":{""startAt"":0,""maxResults"":15,""total"":15,""histories"":[{""id"":""90443"",""author"":{""self"":""https://merico.atlassian.net/rest/api/2/user?accountId=5e9711ba34f7b90c0fbc37d3"",""accountId"":""5e9711ba34f7b90c0fbc37d3"",""avatarUrls"":{"" [...]
+1723,"{""ConnectionId"":1,""BoardId"":8}","{""expand"":""operations,versionedRepresentations,editmeta,changelog,renderedFields"",""id"":""10092"",""self"":""https://merico.atlassian.net/rest/agile/1.0/issue/10092"",""key"":""EE-30"",""changelog"":{""startAt"":0,""maxResults"":11,""total"":11,""histories"":[{""id"":""90434"",""author"":{""self"":""https://merico.atlassian.net/rest/api/2/user?accountId=5e9711ba34f7b90c0fbc37d3"",""accountId"":""5e9711ba34f7b90c0fbc37d3"",""avatarUrls"":{"" [...]
+1724,"{""ConnectionId"":1,""BoardId"":8}","{""expand"":""operations,versionedRepresentations,editmeta,changelog,renderedFields"",""id"":""10093"",""self"":""https://merico.atlassian.net/rest/agile/1.0/issue/10093"",""key"":""EE-31"",""changelog"":{""startAt"":0,""maxResults"":15,""total"":15,""histories"":[{""id"":""90439"",""author"":{""self"":""https://merico.atlassian.net/rest/api/2/user?accountId=5e9711ba34f7b90c0fbc37d3"",""accountId"":""5e9711ba34f7b90c0fbc37d3"",""avatarUrls"":{"" [...]
+1725,"{""ConnectionId"":1,""BoardId"":8}","{""expand"":""operations,versionedRepresentations,editmeta,changelog,renderedFields"",""id"":""10094"",""self"":""https://merico.atlassian.net/rest/agile/1.0/issue/10094"",""key"":""EE-32"",""changelog"":{""startAt"":0,""maxResults"":14,""total"":14,""histories"":[{""id"":""90440"",""author"":{""self"":""https://merico.atlassian.net/rest/api/2/user?accountId=5e9711ba34f7b90c0fbc37d3"",""accountId"":""5e9711ba34f7b90c0fbc37d3"",""avatarUrls"":{"" [...]
+1726,"{""ConnectionId"":1,""BoardId"":8}","{""expand"":""operations,versionedRepresentations,editmeta,changelog,renderedFields"",""id"":""10095"",""self"":""https://merico.atlassian.net/rest/agile/1.0/issue/10095"",""key"":""EE-33"",""changelog"":{""startAt"":0,""maxResults"":10,""total"":10,""histories"":[{""id"":""90446"",""author"":{""self"":""https://merico.atlassian.net/rest/api/2/user?accountId=5e9711ba34f7b90c0fbc37d3"",""accountId"":""5e9711ba34f7b90c0fbc37d3"",""avatarUrls"":{"" [...]
+1727,"{""ConnectionId"":1,""BoardId"":8}","{""expand"":""operations,versionedRepresentations,editmeta,changelog,renderedFields"",""id"":""10096"",""self"":""https://merico.atlassian.net/rest/agile/1.0/issue/10096"",""key"":""EE-34"",""changelog"":{""startAt"":0,""maxResults"":16,""total"":16,""histories"":[{""id"":""90482"",""author"":{""self"":""https://merico.atlassian.net/rest/api/2/user?accountId=5e9711ba34f7b90c0fbc37d3"",""accountId"":""5e9711ba34f7b90c0fbc37d3"",""avatarUrls"":{"" [...]
+1728,"{""ConnectionId"":1,""BoardId"":8}","{""expand"":""operations,versionedRepresentations,editmeta,changelog,renderedFields"",""id"":""10097"",""self"":""https://merico.atlassian.net/rest/agile/1.0/issue/10097"",""key"":""EE-35"",""changelog"":{""startAt"":0,""maxResults"":14,""total"":14,""histories"":[{""id"":""90448"",""author"":{""self"":""https://merico.atlassian.net/rest/api/2/user?accountId=5e9711ba34f7b90c0fbc37d3"",""accountId"":""5e9711ba34f7b90c0fbc37d3"",""avatarUrls"":{"" [...]
+1729,"{""ConnectionId"":1,""BoardId"":8}","{""expand"":""operations,versionedRepresentations,editmeta,changelog,renderedFields"",""id"":""10098"",""self"":""https://merico.atlassian.net/rest/agile/1.0/issue/10098"",""key"":""EE-36"",""changelog"":{""startAt"":0,""maxResults"":15,""total"":15,""histories"":[{""id"":""90442"",""author"":{""self"":""https://merico.atlassian.net/rest/api/2/user?accountId=5e9711ba34f7b90c0fbc37d3"",""accountId"":""5e9711ba34f7b90c0fbc37d3"",""avatarUrls"":{"" [...]
+1730,"{""ConnectionId"":,""BoardId"":8}","{""expand"":""operations,versionedRepresentations,editmeta,changelog,renderedFields"",""id"":""10099"",""self"":""https://merico.atlassian.net/rest/agile/1.0/issue/10099"",""key"":""EE-37"",""changelog"":{""startAt"":0,""maxResults"":13,""total"":13,""histories"":[{""id"":""90441"",""author"":{""self"":""https://merico.atlassian.net/rest/api/2/user?accountId=5e9711ba34f7b90c0fbc37d3"",""accountId"":""5e9711ba34f7b90c0fbc37d3"",""avatarUrls"":{""4 [...]
diff --git a/plugins/customize/e2e/raw_tables/issues.csv b/plugins/customize/e2e/raw_tables/issues.csv
new file mode 100644
index 00000000..8130ef26
--- /dev/null
+++ b/plugins/customize/e2e/raw_tables/issues.csv
@@ -0,0 +1,31 @@
+"id","created_at","updated_at","_raw_data_params","_raw_data_table","_raw_data_id","_raw_data_remark","url","icon_url","issue_key","title","description","epic_key","type","status","original_status","story_point","resolution_date","created_date","updated_date","lead_time_minutes","parent_issue_id","priority","original_estimate_minutes","time_spent_minutes","time_remaining_minutes","creator_id","creator_name","assignee_id","assignee_name","severity","component","deployment_id"
+"jira:JiraIssue:1:10063","2022-08-01 08:18:48.762","2022-08-23 13:12:41.015","{""ConnectionId"":1,""BoardId"":8}","_raw_jira_api_issues",1701,"","https://merico.atlassian.net/browse/EE-1","https://merico.atlassian.net/rest/api/2/universal_avatar/view/type/issuetype/avatar/10315?size=medium","EE-1","四个排序图：测试/注释覆盖度、复用度、模块性","","","STORY","DONE","已完成",0,"2020-06-19 06:31:18.495","2020-06-12 00:13:13.360","2021-03-28 08:06:08.713",10458,"","Medium",0,0,0,"jira:JiraAccount:1:5e9711ba34f7b90c [...]
+"jira:JiraIssue:1:10064","2022-08-01 08:18:48.762","2022-08-23 13:12:41.015","{""ConnectionId"":1,""BoardId"":8}","_raw_jira_api_issues",1702,"","https://merico.atlassian.net/browse/EE-2","https://merico.atlassian.net/rest/api/2/universal_avatar/view/type/issuetype/avatar/10315?size=medium","EE-2","问题堆叠分布排序图","","","STORY","DONE","已完成",0,"2020-06-23 10:20:58.999","2020-06-12 00:15:36.123","2021-03-28 08:05:55.016",16445,"","Medium",0,0,0,"jira:JiraAccount:1:5e9711ba34f7b90c0fbc37d3","Ra [...]
+"jira:JiraIssue:1:10065","2022-08-01 08:18:48.762","2022-08-23 13:12:41.015","{""ConnectionId"":1,""BoardId"":8}","_raw_jira_api_issues",1703,"","https://merico.atlassian.net/browse/EE-3","https://merico.atlassian.net/rest/api/2/universal_avatar/view/type/issuetype/avatar/10315?size=medium","EE-3","问题积压图率","","","STORY","DONE","已完成",0,"2020-06-23 10:21:11.996","2020-06-12 00:15:41.600","2021-03-28 08:05:55.061",16445,"","Medium",0,0,0,"jira:JiraAccount:1:5e9711ba34f7b90c0fbc37d3","Ranki [...]
+"jira:JiraIssue:1:10066","2022-08-01 08:18:48.762","2022-08-23 13:12:41.015","{""ConnectionId"":1,""BoardId"":8}","_raw_jira_api_issues",1704,"","https://merico.atlassian.net/browse/EE-4","https://merico.atlassian.net/rest/api/2/universal_avatar/view/type/issuetype/avatar/10315?size=medium","EE-4","问题分布的帕累托图","","","STORY","DONE","已完成",0,"2020-06-23 10:21:23.562","2020-06-12 00:15:46.144","2021-03-28 08:06:09.535",16445,"","Medium",0,0,0,"jira:JiraAccount:1:5e9711ba34f7b90c0fbc37d3","Ra [...]
+"jira:JiraIssue:1:10067","2022-08-01 08:18:48.762","2022-08-23 13:12:41.015","{""ConnectionId"":1,""BoardId"":8}","_raw_jira_api_issues",1705,"","https://merico.atlassian.net/browse/EE-5","https://merico.atlassian.net/rest/api/2/universal_avatar/view/type/issuetype/avatar/10318?size=medium","EE-5","通用技术任务","","","TASK","DONE","已完成",0,"2020-06-18 04:06:00.747","2020-06-12 00:16:44.157","2021-03-28 08:05:54.622",8869,"","Medium",0,0,0,"jira:JiraAccount:1:5e9711ba34f7b90c0fbc37d3","Rankin [...]
+"jira:JiraIssue:1:10068","2022-08-01 08:18:48.762","2022-08-23 13:12:41.015","{""ConnectionId"":1,""BoardId"":8}","_raw_jira_api_issues",1706,"","https://merico.atlassian.net/browse/EE-6","https://merico.atlassian.net/rest/api/2/universal_avatar/view/type/issuetype/avatar/10315?size=medium","EE-6","变异系数、生产率的四象限图","","","STORY","DONE","已完成",0,"2020-06-16 11:56:14.433","2020-06-12 00:17:26.986","2021-03-28 08:05:56.750",6458,"","Medium",0,0,0,"jira:JiraAccount:1:5e9711ba34f7b90c0fbc37d3", [...]
+"jira:JiraIssue:1:10070","2022-08-01 08:18:48.762","2022-08-23 13:12:41.015","{""ConnectionId"":1,""BoardId"":8}","_raw_jira_api_issues",1707,"","https://merico.atlassian.net/browse/EE-8","https://merico.atlassian.net/rest/api/2/universal_avatar/view/type/issuetype/avatar/10318?size=medium","EE-8","多团队支持","","","TASK","DONE","已完成",0,"2020-07-08 17:11:45.201","2020-06-12 00:18:58.050","2021-03-28 08:05:54.576",38452,"","Medium",0,0,0,"jira:JiraAccount:1:5e9711ba34f7b90c0fbc37d3","Rankin [...]
+"jira:JiraIssue:1:10071","2022-08-01 08:18:48.762","2022-08-23 13:12:41.015","{""ConnectionId"":1,""BoardId"":8}","_raw_jira_api_issues",1708,"","https://merico.atlassian.net/browse/EE-9","https://merico.atlassian.net/rest/api/2/universal_avatar/view/type/issuetype/avatar/10318?size=medium","EE-9","Common backend 拆分","","","TASK","DONE","已完成",0,"2020-07-08 17:12:05.663","2020-06-12 00:19:17.336","2021-03-28 08:06:20.165",38452,"","Medium",1680,0,0,"jira:JiraAccount:1:5e9711ba34f7b90c0fbc [...]
+"jira:JiraIssue:1:10072","2022-08-01 08:18:48.762","2022-08-23 13:12:41.015","{""ConnectionId"":1,""BoardId"":8}","_raw_jira_api_issues",1709,"","https://merico.atlassian.net/browse/EE-10","https://merico.atlassian.net/rest/api/2/universal_avatar/view/type/issuetype/avatar/10318?size=medium","EE-10","部署SaaS版本","","","TASK","DONE","已完成",0,"2020-07-08 17:11:55.247","2020-06-12 00:19:24.637","2021-03-28 08:05:54.472",38452,"","Medium",0,0,0,"jira:JiraAccount:1:5e9711ba34f7b90c0fbc37d3","Ra [...]
+"jira:JiraIssue:1:10076","2022-08-01 08:18:48.762","2022-08-23 13:12:41.015","{""ConnectionId"":1,""BoardId"":8}","_raw_jira_api_issues",1710,"","https://merico.atlassian.net/browse/EE-14","https://merico.atlassian.net/rest/api/2/universal_avatar/view/type/issuetype/avatar/10318?size=medium","EE-14","调整文件夹结构","","","TASK","DONE","已完成",0,"2020-06-15 08:59:51.304","2020-06-12 00:24:25.922","2021-03-28 08:05:56.152",4835,"","Medium",240,600,0,"jira:JiraAccount:1:5e9711ba34f7b90c0fbc37d3"," [...]
+"jira:JiraIssue:1:10077","2022-08-01 08:18:48.762","2022-08-23 13:12:41.015","{""ConnectionId"":1,""BoardId"":8}","_raw_jira_api_issues",1711,"","https://merico.atlassian.net/browse/EE-15","https://merico.atlassian.net/rest/api/2/universal_avatar/view/type/issuetype/avatar/10318?size=medium","EE-15","路由权限控制","","","TASK","DONE","已完成",0,"2020-06-15 09:00:26.956","2020-06-12 00:24:39.624","2021-03-28 08:06:01.995",4835,"","Medium",240,480,0,"jira:JiraAccount:1:5e9711ba34f7b90c0fbc37d3","Ra [...]
+"jira:JiraIssue:1:10078","2022-08-01 08:18:48.762","2022-08-23 13:12:41.015","{""ConnectionId"":1,""BoardId"":8}","_raw_jira_api_issues",1712,"","https://merico.atlassian.net/browse/EE-16","https://merico.atlassian.net/rest/api/2/universal_avatar/view/type/issuetype/avatar/10318?size=medium","EE-16","优化前端 webpack 开发阶段构建速度","","","TASK","DONE","已完成",0,"2020-06-15 09:01:44.159","2020-06-12 00:24:49.017","2021-03-28 08:05:55.863",4836,"","Medium",0,60,0,"jira:JiraAccount:1:5e9711ba34f7b90c [...]
+"jira:JiraIssue:1:10079","2022-08-01 08:18:48.762","2022-08-23 13:12:41.015","{""ConnectionId"":1,""BoardId"":8}","_raw_jira_api_issues",1713,"","https://merico.atlassian.net/browse/EE-17","https://merico.atlassian.net/rest/api/2/universal_avatar/view/type/issuetype/avatar/10318?size=medium","EE-17","新的错误处理机制","","","TASK","DONE","已完成",0,"2020-07-22 07:25:29.104","2020-06-12 00:24:56.048","2021-03-28 08:05:54.426",58020,"","Medium",360,120,0,"jira:JiraAccount:1:5e9711ba34f7b90c0fbc37d3" [...]
+"jira:JiraIssue:1:10081","2022-08-01 08:18:48.762","2022-08-23 13:12:41.015","{""ConnectionId"":1,""BoardId"":8}","_raw_jira_api_issues",1714,"","https://merico.atlassian.net/browse/EE-19","https://merico.atlassian.net/rest/api/2/universal_avatar/view/type/issuetype/avatar/10315?size=medium","EE-19","LDAP需要支持TLS和证书","","","STORY","DONE","已完成",0,"2020-06-18 08:34:11.117","2020-06-12 00:28:00.241","2021-03-28 08:05:57.326",9126,"","Medium",0,0,0,"jira:JiraAccount:1:5e9711ba34f7b90c0fbc37d [...]
+"jira:JiraIssue:1:10082","2022-08-01 08:18:48.762","2022-08-23 13:12:41.015","{""ConnectionId"":1,""BoardId"":8}","_raw_jira_api_issues",1715,"","https://merico.atlassian.net/browse/EE-20","https://merico.atlassian.net/rest/api/2/universal_avatar/view/type/issuetype/avatar/10315?size=medium","EE-20","团队首页垂直化","","","STORY","DONE","已完成",0,"2020-06-17 07:25:54.370","2020-06-12 00:29:43.677","2021-08-06 06:14:54.647",7616,"","Medium",0,0,0,"jira:JiraAccount:1:5e9711ba34f7b90c0fbc37d3","Rank [...]
+"jira:JiraIssue:1:10085","2022-08-01 08:18:48.762","2022-08-23 13:12:41.015","{""ConnectionId"":1,""BoardId"":8}","_raw_jira_api_issues",1716,"","https://merico.atlassian.net/browse/EE-23","https://merico.atlassian.net/rest/api/2/universal_avatar/view/type/issuetype/avatar/10303?size=medium","EE-23","批量删除事故","","","BUG","DONE","已完成",0,"2020-06-15 09:07:56.798","2020-06-12 00:33:57.204","2021-03-28 08:05:57.095",4833,"","Medium",0,60,0,"jira:JiraAccount:1:5e9711ba34f7b90c0fbc37d3","Ranki [...]
+"jira:JiraIssue:1:10086","2022-08-01 08:18:48.762","2022-08-23 13:12:41.015","{""ConnectionId"":1,""BoardId"":8}","_raw_jira_api_issues",1717,"","https://merico.atlassian.net/browse/EE-24","https://merico.atlassian.net/rest/api/2/universal_avatar/view/type/issuetype/avatar/10315?size=medium","EE-24","LDAP支持自定义的证书上传","","","STORY","DONE","已完成",0,"2020-06-12 07:17:28.659","2020-06-12 00:35:15.489","2021-03-28 08:05:55.819",402,"","Medium",0,0,0,"jira:JiraAccount:1:5e9711ba34f7b90c0fbc37d3 [...]
+"jira:JiraIssue:1:10087","2022-08-01 08:18:48.762","2022-08-23 13:12:41.015","{""ConnectionId"":1,""BoardId"":8}","_raw_jira_api_issues",1718,"","https://merico.atlassian.net/browse/EE-25","https://merico.atlassian.net/rest/api/2/universal_avatar/view/type/issuetype/avatar/10316?size=medium","EE-25","组件封装及Demo","","","SUB-TASK","DONE","已完成",0,"2020-06-18 04:02:22.350","2020-06-12 00:40:54.210","2021-03-28 08:05:57.189",8841,"jira:JiraIssue:1:10063","Medium",240,0,0,"jira:JiraAccount:1:5 [...]
+"jira:JiraIssue:1:10088","2022-08-01 08:18:48.762","2022-08-23 13:12:41.015","{""ConnectionId"":1,""BoardId"":8}","_raw_jira_api_issues",1719,"","https://merico.atlassian.net/browse/EE-26","https://merico.atlassian.net/rest/api/2/universal_avatar/view/type/issuetype/avatar/10316?size=medium","EE-26","定接口","","","SUB-TASK","DONE","已完成",0,"2020-06-15 09:06:51.438","2020-06-12 00:41:01.683","2021-03-28 08:05:55.208",4825,"jira:JiraIssue:1:10063","Medium",0,0,0,"jira:JiraAccount:1:5e9711ba3 [...]
+"jira:JiraIssue:1:10089","2022-08-01 08:18:48.762","2022-08-23 13:12:41.015","{""ConnectionId"":1,""BoardId"":8}","_raw_jira_api_issues",1720,"","https://merico.atlassian.net/browse/EE-27","https://merico.atlassian.net/rest/api/2/universal_avatar/view/type/issuetype/avatar/10316?size=medium","EE-27","提供后端接口","","","SUB-TASK","DONE","已完成",0,"2020-06-19 06:31:31.662","2020-06-12 00:41:16.622","2021-03-28 08:05:55.498",10430,"jira:JiraIssue:1:10063","Medium",660,0,0,"jira:JiraAccount:1:5e9 [...]
+"jira:JiraIssue:1:10090","2022-08-01 08:18:48.762","2022-08-23 13:12:41.015","{""ConnectionId"":1,""BoardId"":8}","_raw_jira_api_issues",1721,"","https://merico.atlassian.net/browse/EE-28","https://merico.atlassian.net/rest/api/2/universal_avatar/view/type/issuetype/avatar/10316?size=medium","EE-28","数据填充与联调","","","SUB-TASK","DONE","已完成",0,"2020-06-18 04:03:04.637","2020-06-12 00:41:36.317","2021-03-28 08:06:05.443",8841,"jira:JiraIssue:1:10063","Medium",360,0,0,"jira:JiraAccount:1:5e9 [...]
+"jira:JiraIssue:1:10091","2022-08-01 08:18:48.762","2022-08-23 13:12:41.015","{""ConnectionId"":1,""BoardId"":8}","_raw_jira_api_issues",1722,"","https://merico.atlassian.net/browse/EE-29","https://merico.atlassian.net/rest/api/2/universal_avatar/view/type/issuetype/avatar/10316?size=medium","EE-29","组件封装及Demo","","","SUB-TASK","DONE","已完成",0,"2020-06-18 04:03:30.760","2020-06-12 00:48:29.035","2021-03-28 08:05:55.731",8835,"jira:JiraIssue:1:10064","Medium",360,0,0,"jira:JiraAccount:1:5 [...]
+"jira:JiraIssue:1:10092","2022-08-01 08:18:48.762","2022-08-23 13:12:41.015","{""ConnectionId"":1,""BoardId"":8}","_raw_jira_api_issues",1723,"","https://merico.atlassian.net/browse/EE-30","https://merico.atlassian.net/rest/api/2/universal_avatar/view/type/issuetype/avatar/10316?size=medium","EE-30","定接口","","","SUB-TASK","DONE","已完成",0,"2020-06-15 09:06:40.206","2020-06-12 00:48:39.803","2021-03-28 08:05:55.159",4818,"jira:JiraIssue:1:10064","Medium",0,0,0,"jira:JiraAccount:1:5e9711ba3 [...]
+"jira:JiraIssue:1:10093","2022-08-01 08:18:48.762","2022-08-23 13:12:41.015","{""ConnectionId"":1,""BoardId"":8}","_raw_jira_api_issues",1724,"","https://merico.atlassian.net/browse/EE-31","https://merico.atlassian.net/rest/api/2/universal_avatar/view/type/issuetype/avatar/10316?size=medium","EE-31","后端接口","","","SUB-TASK","DONE","已完成",0,"2020-06-19 07:35:31.762","2020-06-12 00:48:46.751","2021-03-28 08:05:55.544",10486,"jira:JiraIssue:1:10064","Medium",120,0,0,"jira:JiraAccount:1:5e971 [...]
+"jira:JiraIssue:1:10094","2022-08-01 08:18:48.762","2022-08-23 13:12:41.015","{""ConnectionId"":1,""BoardId"":8}","_raw_jira_api_issues",1725,"","https://merico.atlassian.net/browse/EE-32","https://merico.atlassian.net/rest/api/2/universal_avatar/view/type/issuetype/avatar/10316?size=medium","EE-32","数据填充与联调","","","SUB-TASK","DONE","已完成",0,"2020-06-18 04:03:48.818","2020-06-12 00:48:53.279","2021-03-28 08:05:55.592",8834,"jira:JiraIssue:1:10064","Medium",360,0,0,"jira:JiraAccount:1:5e9 [...]
+"jira:JiraIssue:1:10095","2022-08-01 08:18:48.762","2022-08-23 13:12:41.015","{""ConnectionId"":1,""BoardId"":8}","_raw_jira_api_issues",1726,"","https://merico.atlassian.net/browse/EE-33","https://merico.atlassian.net/rest/api/2/universal_avatar/view/type/issuetype/avatar/10316?size=medium","EE-33","准备测试用例","","","SUB-TASK","DONE","已完成",0,"2020-06-19 06:32:19.340","2020-06-12 00:50:45.674","2021-03-28 08:05:55.253",10421,"jira:JiraIssue:1:10063","Medium",0,0,0,"jira:JiraAccount:1:5e9711 [...]
+"jira:JiraIssue:1:10096","2022-08-01 08:18:48.762","2022-08-23 13:12:41.015","{""ConnectionId"":1,""BoardId"":8}","_raw_jira_api_issues",1727,"","https://merico.atlassian.net/browse/EE-34","https://merico.atlassian.net/rest/api/2/universal_avatar/view/type/issuetype/avatar/10316?size=medium","EE-34"," 组件封装及Demo","","","SUB-TASK","DONE","已完成",0,"2020-06-18 04:04:05.951","2020-06-12 00:51:57.807","2021-03-28 08:05:57.232",8832,"jira:JiraIssue:1:10065","Medium",240,0,0,"jira:JiraAccount:1: [...]
+"jira:JiraIssue:1:10097","2022-08-01 08:18:48.762","2022-08-23 13:12:41.015","{""ConnectionId"":1,""BoardId"":8}","_raw_jira_api_issues",1728,"","https://merico.atlassian.net/browse/EE-35","https://merico.atlassian.net/rest/api/2/universal_avatar/view/type/issuetype/avatar/10316?size=medium","EE-35","定接口","","","SUB-TASK","DONE","已完成",0,"2020-06-15 09:06:30.942","2020-06-12 00:52:04.767","2021-03-28 08:05:55.353",4814,"jira:JiraIssue:1:10065","Medium",0,0,0,"jira:JiraAccount:1:5e9711ba34 [...]
+"jira:JiraIssue:1:10098","2022-08-01 08:18:48.762","2022-08-23 13:12:41.015","{""ConnectionId"":1,""BoardId"":8}","_raw_jira_api_issues",1729,"","https://merico.atlassian.net/browse/EE-36","https://merico.atlassian.net/rest/api/2/universal_avatar/view/type/issuetype/avatar/10316?size=medium","EE-36","后端接口","","","SUB-TASK","DONE","已完成",0,"2020-06-19 07:35:44.696","2020-06-12 00:52:12.678","2021-03-28 08:05:55.685",10483,"jira:JiraIssue:1:10065","Medium",120,0,0,"jira:JiraAccount:1:5e9711 [...]
+"jira:JiraIssue:1:10099","2022-08-01 08:18:48.762","2022-08-23 13:12:41.015","{""ConnectionId"":1,""BoardId"":8}","_raw_jira_api_issues",1730,"","https://merico.atlassian.net/browse/EE-37","https://merico.atlassian.net/rest/api/2/universal_avatar/view/type/issuetype/avatar/10316?size=medium","EE-37","数据填充与联调 ","","","SUB-TASK","DONE","已完成",0,"2020-06-18 04:04:31.261","2020-06-12 00:52:23.456","2021-03-28 08:05:55.638",8832,"jira:JiraIssue:1:10065","Medium",0,0,0,"jira:JiraAccount:1:5e971 [...]
diff --git a/plugins/customize/e2e/snapshot_tables/issues.csv b/plugins/customize/e2e/snapshot_tables/issues.csv
new file mode 100644
index 00000000..2a662778
--- /dev/null
+++ b/plugins/customize/e2e/snapshot_tables/issues.csv
@@ -0,0 +1,31 @@
+id,_raw_data_params,_raw_data_table,_raw_data_id,x_test
+jira:JiraIssue:1:10063,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_issues,1701,2020-06-12T08:13:13.360+0800
+jira:JiraIssue:1:10064,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_issues,1702,2020-06-12T08:15:36.123+0800
+jira:JiraIssue:1:10065,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_issues,1703,2020-06-12T08:15:41.600+0800
+jira:JiraIssue:1:10066,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_issues,1704,2020-06-12T08:15:46.144+0800
+jira:JiraIssue:1:10067,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_issues,1705,2020-06-12T08:16:44.157+0800
+jira:JiraIssue:1:10068,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_issues,1706,2020-06-12T08:17:26.986+0800
+jira:JiraIssue:1:10070,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_issues,1707,2020-06-12T08:18:58.050+0800
+jira:JiraIssue:1:10071,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_issues,1708,2020-06-12T08:19:17.336+0800
+jira:JiraIssue:1:10072,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_issues,1709,2020-06-12T08:19:24.637+0800
+jira:JiraIssue:1:10076,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_issues,1710,2020-06-12T08:24:25.922+0800
+jira:JiraIssue:1:10077,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_issues,1711,2020-06-12T08:24:39.624+0800
+jira:JiraIssue:1:10078,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_issues,1712,2020-06-12T08:24:49.017+0800
+jira:JiraIssue:1:10079,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_issues,1713,2020-06-12T08:24:56.048+0800
+jira:JiraIssue:1:10081,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_issues,1714,2020-06-12T08:28:00.241+0800
+jira:JiraIssue:1:10082,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_issues,1715,2020-06-12T08:29:43.677+0800
+jira:JiraIssue:1:10085,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_issues,1716,2020-06-12T08:33:57.204+0800
+jira:JiraIssue:1:10086,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_issues,1717,2020-06-12T08:35:15.489+0800
+jira:JiraIssue:1:10087,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_issues,1718,2020-06-12T08:40:54.210+0800
+jira:JiraIssue:1:10088,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_issues,1719,2020-06-12T08:41:01.683+0800
+jira:JiraIssue:1:10089,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_issues,1720,2020-06-12T08:41:16.622+0800
+jira:JiraIssue:1:10090,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_issues,1721,2020-06-12T08:41:36.317+0800
+jira:JiraIssue:1:10091,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_issues,1722,2020-06-12T08:48:29.035+0800
+jira:JiraIssue:1:10092,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_issues,1723,2020-06-12T08:48:39.803+0800
+jira:JiraIssue:1:10093,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_issues,1724,2020-06-12T08:48:46.751+0800
+jira:JiraIssue:1:10094,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_issues,1725,2020-06-12T08:48:53.279+0800
+jira:JiraIssue:1:10095,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_issues,1726,2020-06-12T08:50:45.674+0800
+jira:JiraIssue:1:10096,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_issues,1727,2020-06-12T08:51:57.807+0800
+jira:JiraIssue:1:10097,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_issues,1728,2020-06-12T08:52:04.767+0800
+jira:JiraIssue:1:10098,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_issues,1729,2020-06-12T08:52:12.678+0800
+jira:JiraIssue:1:10099,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_issues,1730,2020-06-12T08:52:23.456+0800
diff --git a/plugins/customize/impl/impl.go b/plugins/customize/impl/impl.go
new file mode 100644
index 00000000..e5e5abdc
--- /dev/null
+++ b/plugins/customize/impl/impl.go
@@ -0,0 +1,88 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package impl
+
+import (
+ "fmt"
+
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/customize/api"
+ "github.com/apache/incubator-devlake/plugins/customize/tasks"
+ "github.com/apache/incubator-devlake/plugins/helper"
+ "github.com/mitchellh/mapstructure"
+ "github.com/spf13/viper"
+ "gorm.io/gorm"
+)
+
+var _ core.PluginMeta = (*Customize)(nil)
+var _ core.PluginInit = (*Customize)(nil)
+var _ core.PluginApi = (*Customize)(nil)
+
+type Customize struct {
+ handlers *api.Handlers
+}
+
+func (plugin *Customize) Init(config *viper.Viper, logger core.Logger, db *gorm.DB) error {
+ basicRes := helper.NewDefaultBasicRes(config, logger, db)
+ plugin.handlers = api.NewHandlers(basicRes.GetDal())
+ return nil
+}
+
+func (plugin Customize) SubTaskMetas() []core.SubTaskMeta {
+ return []core.SubTaskMeta{
+ tasks.ExtractCustomizedFieldsMeta,
+ }
+}
+
+func (plugin Customize) MakePipelinePlan(connectionId uint64, scope []*core.BlueprintScopeV100) (core.PipelinePlan, error) {
+ return api.MakePipelinePlan(plugin.SubTaskMetas(), connectionId, scope)
+}
+func (plugin Customize) PrepareTaskData(taskCtx core.TaskContext, options map[string]interface{}) (interface{}, error) {
+ var op tasks.Options
+ var err error
+ logger := taskCtx.GetLogger()
+ logger.Debug("%v", options)
+ err = mapstructure.Decode(options, &op)
+ if err != nil {
+ return nil, fmt.Errorf("could not decode Jira options: %v", err)
+ }
+ taskData := &tasks.TaskData{
+ Options: &op,
+ }
+ return taskData, nil
+}
+
+func (plugin Customize) Description() string {
+ return "To customize table fields"
+}
+
+func (plugin Customize) RootPkgPath() string {
+ return "github.com/apache/incubator-devlake/plugins/customize"
+}
+
+func (plugin *Customize) ApiResources() map[string]map[string]core.ApiResourceHandler {
+ return map[string]map[string]core.ApiResourceHandler{
+ ":table/fields": {
+ "GET": plugin.handlers.ListFields,
+ "POST": plugin.handlers.CreateFields,
+ },
+ ":table/fields/:field": {
+ "DELETE": plugin.handlers.DeleteField,
+ },
+ }
+}
diff --git a/plugins/customize/models/model.go b/plugins/customize/models/model.go
new file mode 100644
index 00000000..8a251450
--- /dev/null
+++ b/plugins/customize/models/model.go
@@ -0,0 +1,26 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package models
+
+type Table struct {
+ Name string
+}
+
+func (t *Table) TableName() string {
+ return t.Name
+}
diff --git a/plugins/customize/tasks/customized_fields_extractor.go b/plugins/customize/tasks/customized_fields_extractor.go
new file mode 100644
index 00000000..6bf1f9ce
--- /dev/null
+++ b/plugins/customize/tasks/customized_fields_extractor.go
@@ -0,0 +1,134 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "context"
+ "fmt"
+ "github.com/apache/incubator-devlake/errors"
+ "strings"
+
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/core/dal"
+ "github.com/apache/incubator-devlake/plugins/customize/models"
+ "github.com/tidwall/gjson"
+)
+
+var _ core.SubTaskEntryPoint = ExtractCustomizedFields
+
+var ExtractCustomizedFieldsMeta = core.SubTaskMeta{Name: "extractCustomizedFields",
+ EntryPoint: ExtractCustomizedFields,
+ EnabledByDefault: true,
+ Description: "extract customized fields",
+}
+
+// ExtractCustomizedFields extracts fields from raw data tables and assigns to domain layer tables
+func ExtractCustomizedFields(taskCtx core.SubTaskContext) error {
+ data := taskCtx.GetData().(*TaskData)
+ if data == nil || data.Options == nil {
+ return nil
+ }
+ d := taskCtx.GetDal()
+ var err error
+ for _, rule := range data.Options.TransformationRules {
+ err = extractCustomizedFields(taskCtx.GetContext(), d, rule.Table, rule.RawDataTable, rule.RawDataParams, rule.Mapping)
+ if err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func extractCustomizedFields(ctx context.Context, d dal.Dal, table, rawTable, rawDataParams string, extractor map[string]string) error {
+ pkFields, err := dal.GetPrimarykeyColumns(d, &models.Table{table})
+ if err != nil {
+ return err
+ }
+ rawDataField := fmt.Sprintf("%s.data", rawTable)
+ // `fields` only include `_raw_data_id` and primary keys coming from the domain layer table, and `data` coming from the raw layer
+ fields := []string{fmt.Sprintf("%s.%s", table, "_raw_data_id")}
+ fields = append(fields, rawDataField)
+ for _, field := range pkFields {
+ fields = append(fields, fmt.Sprintf("%s.%s", table, field.Name()))
+ }
+ clauses := []dal.Clause{
+ dal.Select(strings.Join(fields, ", ")),
+ dal.From(table),
+ dal.Join(fmt.Sprintf(" LEFT JOIN %s ON %s._raw_data_id = %s.id", rawTable, table, rawTable)),
+ dal.Where("_raw_data_table = ? AND _raw_data_params = ?", rawTable, rawDataParams),
+ }
+ rows, err := d.Cursor(clauses...)
+ if err != nil {
+ return err
+ }
+ defer rows.Close()
+
+ for rows.Next() {
+ select {
+ case <-ctx.Done():
+ return ctx.Err()
+ default:
+ }
+ row := make(map[string]interface{})
+ updates := make(map[string]string)
+ err = d.Fetch(rows, &row)
+ if err != nil {
+ return err
+ }
+ switch blob := row["data"].(type) {
+ case []byte:
+ for field, path := range extractor {
+ updates[field] = gjson.GetBytes(blob, path).String()
+ }
+ case string:
+ for field, path := range extractor {
+ updates[field] = gjson.Get(blob, path).String()
+ }
+ default:
+ return nil
+ }
+
+ if len(updates) > 0 {
+ // remove columns that are not primary key
+ delete(row, "_raw_data_id")
+ delete(row, "data")
+ query, params := mkUpdate(table, updates, row)
+ err = d.Exec(query, params...)
+ if err != nil {
+ return errors.Default.Wrap(err, "Exec SQL error")
+ }
+ }
+ }
+ return nil
+}
+
+func mkUpdate(table string, updates map[string]string, pk map[string]interface{}) (string, []interface{}) {
+ var params []interface{}
+ stat := fmt.Sprintf("UPDATE %s SET ", table)
+ var uu []string
+ for field, value := range updates {
+ uu = append(uu, fmt.Sprintf("%s = ?", field))
+ params = append(params, value)
+ }
+ var ww []string
+ for field, value := range pk {
+ ww = append(ww, fmt.Sprintf("%s = ?", field))
+ params = append(params, value)
+ }
+ return stat + strings.Join(uu, ", ") + " WHERE " + strings.Join(ww, " AND "), params
+}
diff --git a/plugins/customize/tasks/task_data.go b/plugins/customize/tasks/task_data.go
new file mode 100644
index 00000000..62b69d7e
--- /dev/null
+++ b/plugins/customize/tasks/task_data.go
@@ -0,0 +1,33 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+type MappingRules struct {
+ Table string `json:"table" example:"issues"`
+ RawDataTable string `json:"_raw_data_table" example:"_raw_jira_api_issues"`
+ RawDataParams string `json:"_raw_data_params" example:"{\"ConnectionId\":1,\"BoardId\":8}"`
+ Mapping map[string]string `json:"mapping" example:"x_text:fields.created"`
+}
+
+type Options struct {
+ TransformationRules []MappingRules `json:"transformationRules"`
+}
+
+type TaskData struct {
+ Options *Options
+}