You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by kl...@apache.org on 2023/05/25 12:17:47 UTC
[incubator-devlake] branch main updated: Zentao changelog (#5257)
This is an automated email from the ASF dual-hosted git repository.
klesh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new ceefbdb04 Zentao changelog (#5257)
ceefbdb04 is described below
commit ceefbdb04a3afac49b42104954b574b8544261ad
Author: mappjzc <zh...@merico.dev>
AuthorDate: Thu May 25 20:17:42 2023 +0800
Zentao changelog (#5257)
* feat: zentao support dbget for changelog
Add DBGetActionHistory for Zentao
Nddtfjiang <zh...@merico.dev>
* feat: add dburl support
Add connection support for dbget.
Add blueprint support for dbget.
Nddtfjiang <zh...@merico.dev>
* feat: add e2e test for changelog
Add TestZentaoDbGetDataFlow.
Fix some struct error.
Fix unit test error.
Nddtfjiang <zh...@merico.dev>
* fix: fix e2e not support read
Fix e2e not support read field.
Nddtfjiang <zh...@merico.dev>
* fix: fix for review
Removed BaseDbConfigReader.
Nddtfjiang <zh...@merico.dev>
* fix: fix for review
Remove blueprint dburl option
Add db setting in connection
Nddtfjiang <zh...@merico.dev>
---
backend/core/runner/db.go | 7 +-
backend/plugins/zentao/api/blueprint_v200.go | 5 +-
backend/plugins/zentao/e2e/changelog_test.go | 95 ++++++++++++++++
.../plugins/zentao/e2e/raw_tables/zt_action.csv | 15 +++
.../plugins/zentao/e2e/raw_tables/zt_history.csv | 29 +++++
.../e2e/snapshot_tables/_tool_zentao_changelog.csv | 13 +++
.../_tool_zentao_changelog_detail.csv | 29 +++++
backend/plugins/zentao/impl/impl.go | 37 ++++++-
.../plugins/zentao/models/archived/changelog.go | 67 +++++++++++
.../plugins/zentao/models/archived/connection.go | 13 ---
backend/plugins/zentao/models/changelog.go | 67 +++++++++++
backend/plugins/zentao/models/connection.go | 5 +
.../20230519_add_init_changelog_tables.go | 64 +++++++++++
.../zentao/models/migrationscripts/register.go | 1 +
backend/plugins/zentao/models/product.go | 2 +-
backend/plugins/zentao/models/remote_db.go | 96 ++++++++++++++++
backend/plugins/zentao/tasks/changelog_dbget.go | 122 +++++++++++++++++++++
backend/plugins/zentao/tasks/task_data.go | 4 +
18 files changed, 650 insertions(+), 21 deletions(-)
diff --git a/backend/core/runner/db.go b/backend/core/runner/db.go
index bca6cc153..abbfe2277 100644
--- a/backend/core/runner/db.go
+++ b/backend/core/runner/db.go
@@ -19,6 +19,10 @@ package runner
import (
"fmt"
+ "net/url"
+ "strings"
+ "time"
+
"github.com/apache/incubator-devlake/core/config"
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
@@ -27,9 +31,6 @@ import (
"gorm.io/driver/postgres"
"gorm.io/gorm"
gormLogger "gorm.io/gorm/logger"
- "net/url"
- "strings"
- "time"
)
// NewGormDb creates a new *gorm.DB and set it up properly
diff --git a/backend/plugins/zentao/api/blueprint_v200.go b/backend/plugins/zentao/api/blueprint_v200.go
index 6d0805247..0352fb1f3 100644
--- a/backend/plugins/zentao/api/blueprint_v200.go
+++ b/backend/plugins/zentao/api/blueprint_v200.go
@@ -19,6 +19,9 @@ package api
import (
"fmt"
+ "strings"
+ "time"
+
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models/domainlayer"
@@ -30,8 +33,6 @@ import (
"github.com/apache/incubator-devlake/plugins/zentao/models"
"github.com/apache/incubator-devlake/plugins/zentao/tasks"
"github.com/go-playground/validator/v10"
- "strings"
- "time"
)
func MakeDataSourcePipelinePlanV200(subtaskMetas []plugin.SubTaskMeta, connectionId uint64, bpScopes []*plugin.BlueprintScopeV200, syncPolicy *plugin.BlueprintSyncPolicy) (plugin.PipelinePlan, []plugin.Scope, errors.Error) {
diff --git a/backend/plugins/zentao/e2e/changelog_test.go b/backend/plugins/zentao/e2e/changelog_test.go
new file mode 100644
index 000000000..aed72b615
--- /dev/null
+++ b/backend/plugins/zentao/e2e/changelog_test.go
@@ -0,0 +1,95 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+ "testing"
+
+ "github.com/apache/incubator-devlake/core/config"
+ "github.com/apache/incubator-devlake/core/models/common"
+ "github.com/apache/incubator-devlake/core/runner"
+ "github.com/apache/incubator-devlake/helpers/e2ehelper"
+ "github.com/apache/incubator-devlake/impls/dalgorm"
+ "github.com/apache/incubator-devlake/plugins/zentao/impl"
+ "github.com/apache/incubator-devlake/plugins/zentao/models"
+ "github.com/apache/incubator-devlake/plugins/zentao/tasks"
+ "github.com/spf13/viper"
+)
+
+func TestZentaoDbGetDataFlow(t *testing.T) {
+
+ var zentao impl.Zentao
+ dataflowTester := e2ehelper.NewDataFlowTester(t, "zentao", zentao)
+ cfg := config.GetConfig()
+
+ taskData := &tasks.ZentaoTaskData{
+ Options: &tasks.ZentaoOptions{
+ ConnectionId: 1,
+ ProjectId: 0,
+ ProductId: 1,
+ },
+ }
+
+ dataflowTester.ImportCsvIntoTabler("./raw_tables/zt_action.csv", models.ZentaoRemoteDbAction{})
+ dataflowTester.ImportCsvIntoTabler("./raw_tables/zt_history.csv", models.ZentaoRemoteDbHistory{})
+
+ v := viper.New()
+ v.Set("DB_URL", cfg.GetString(`E2E_DB_URL`))
+ v.Set("DB_LOGGING_LEVEL", cfg.GetString("DB_LOGGING_LEVEL"))
+ v.Set("DB_IDLE_CONNS", cfg.GetInt("DB_IDLE_CONNS"))
+ v.Set("DbMaxConns", cfg.GetInt("DB_MAX_CONNS"))
+
+ rgorm, err := runner.NewGormDb(v, dataflowTester.Log)
+ if err != nil {
+ return
+ }
+ taskData.RemoteDb = dalgorm.NewDalgorm(rgorm)
+
+ // verify conversion
+ dataflowTester.FlushTabler(&models.ZentaoChangelog{})
+ dataflowTester.FlushTabler(&models.ZentaoChangelogDetail{})
+ dataflowTester.Subtask(tasks.DBGetChangelogMeta, taskData)
+
+ dataflowTester.VerifyTable(
+ models.ZentaoChangelog{},
+ "./snapshot_tables/_tool_zentao_changelog.csv",
+ e2ehelper.ColumnWithRawData(
+ "connection_id",
+ "id",
+ "object_id",
+ "execution",
+ "actor",
+ "action",
+ "extra",
+ "object_type",
+ "project",
+ "product",
+ "vision",
+ "comment",
+ "efforted",
+ "date",
+ ),
+ )
+
+ dataflowTester.VerifyTableWithOptions(
+ &models.ZentaoChangelogDetail{},
+ e2ehelper.TableOptions{
+ CSVRelPath: "./snapshot_tables/_tool_zentao_changelog_detail.csv",
+ IgnoreTypes: []interface{}{common.NoPKModel{}},
+ })
+}
diff --git a/backend/plugins/zentao/e2e/raw_tables/zt_action.csv b/backend/plugins/zentao/e2e/raw_tables/zt_action.csv
new file mode 100644
index 000000000..de443fd57
--- /dev/null
+++ b/backend/plugins/zentao/e2e/raw_tables/zt_action.csv
@@ -0,0 +1,15 @@
+"id","objectType","objectID","product","project","execution","actor","action","date","comment","extra","read","vision","efforted"
+112,"execution",1,",1,",0,1,"admin","edited","2021-04-28 11:06:58","","","0","rnd",0
+113,"execution",2,",1,",0,2,"admin","edited","2021-04-28 11:08:02","","","0","rnd",0
+114,"bug",1,",1,",0,1,"admin","edited","2021-04-28 11:09:08","","","0","rnd",0
+115,"bug",2,",1,",0,1,"admin","edited","2021-04-28 11:09:08","","","0","rnd",0
+116,"bug",3,",1,",0,1,"admin","edited","2021-04-28 11:09:08","","","0","rnd",0
+117,"bug",4,",1,",0,1,"admin","edited","2021-04-28 11:09:08","","","0","rnd",0
+118,"testtask",1,",1,",0,1,"admin","edited","2021-04-28 11:10:06","","","0","rnd",0
+119,"productplan",1,",1,",0,0,"admin","edited","2021-04-28 11:11:10","","","0","rnd",0
+120,"execution",1,",1,",0,1,"admin","started","2021-04-28 11:12:22","","","0","rnd",0
+165,"task",10,",1,",7,1,"dev1","finished","2021-04-28 13:17:54","","","0","rnd",0
+166,"task",11,",1,",7,1,"dev1","started","2021-04-28 13:18:45","","","0","rnd",0
+167,"user",4,",0,",0,0,"dev1","logout","2021-04-28 13:19:30","","","0","rnd",0
+168,"user",5,",0,",0,0,"dev2","login","2021-04-28 13:19:33","","","0","rnd",0
+169,"task",14,",1,",7,1,"dev2","started","2021-04-28 13:19:54","","","0","rnd",0
\ No newline at end of file
diff --git a/backend/plugins/zentao/e2e/raw_tables/zt_history.csv b/backend/plugins/zentao/e2e/raw_tables/zt_history.csv
new file mode 100644
index 000000000..5a70ae859
--- /dev/null
+++ b/backend/plugins/zentao/e2e/raw_tables/zt_history.csv
@@ -0,0 +1,29 @@
+"id","action","field","old","new","diff"
+68,112,"begin","2012-06-05","2020-06-05",""
+69,112,"end","2012-12-04","2021-12-04",""
+70,112,"days","184","391",""
+71,112,"status","done","wait",""
+72,113,"begin","2013-06-05","2021-06-05",""
+73,113,"end","2014-06-04","2022-06-04",""
+74,113,"days","365","260",""
+75,114,"type","interface","codeerror",""
+76,114,"pri","0","1",""
+77,115,"pri","0","2",""
+78,116,"pri","0","1",""
+79,117,"pri","0","1",""
+80,118,"build","trunk","1",""
+81,118,"begin","2012-06-05","2020-06-05",""
+82,118,"end","2013-06-21","2021-06-21",""
+83,119,"begin","2000-01-01","2020-06-05",""
+84,119,"end","2015-01-01","2021-06-04",""
+85,120,"status","wait","doing",""
+91,165,"consumed","0","8",""
+92,165,"realStarted","0000-00-00 00:00:00","2021-04-28 13:17:54",""
+93,165,"finishedDate","","2021-04-01 16:00",""
+94,165,"status","wait","done",""
+95,165,"left","7","0",""
+96,165,"finishedBy","","dev1",""
+97,166,"realStarted","0000-00-00 00:00:00","2021-04-01 16:00",""
+98,166,"status","wait","doing",""
+99,169,"realStarted","0000-00-00 00:00:00","2021-04-01 08:00",""
+100,169,"status","wait","doing",""
\ No newline at end of file
diff --git a/backend/plugins/zentao/e2e/snapshot_tables/_tool_zentao_changelog.csv b/backend/plugins/zentao/e2e/snapshot_tables/_tool_zentao_changelog.csv
new file mode 100644
index 000000000..052473571
--- /dev/null
+++ b/backend/plugins/zentao/e2e/snapshot_tables/_tool_zentao_changelog.csv
@@ -0,0 +1,13 @@
+connection_id,id,object_id,execution,actor,action,extra,object_type,project,product,vision,comment,efforted,date,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
+1,112,1,1,admin,112,,execution,0,1,rnd,,0,2021-04-28T11:06:58.000+00:00,,,0,
+1,113,2,2,admin,113,,execution,0,1,rnd,,0,2021-04-28T11:08:02.000+00:00,,,0,
+1,114,1,1,admin,114,,bug,0,1,rnd,,0,2021-04-28T11:09:08.000+00:00,,,0,
+1,115,2,1,admin,115,,bug,0,1,rnd,,0,2021-04-28T11:09:08.000+00:00,,,0,
+1,116,3,1,admin,116,,bug,0,1,rnd,,0,2021-04-28T11:09:08.000+00:00,,,0,
+1,117,4,1,admin,117,,bug,0,1,rnd,,0,2021-04-28T11:09:08.000+00:00,,,0,
+1,118,1,1,admin,118,,testtask,0,1,rnd,,0,2021-04-28T11:10:06.000+00:00,,,0,
+1,119,1,0,admin,119,,productplan,0,1,rnd,,0,2021-04-28T11:11:10.000+00:00,,,0,
+1,120,1,1,admin,120,,execution,0,1,rnd,,0,2021-04-28T11:12:22.000+00:00,,,0,
+1,165,10,1,dev1,165,,task,7,1,rnd,,0,2021-04-28T13:17:54.000+00:00,,,0,
+1,166,11,1,dev1,166,,task,7,1,rnd,,0,2021-04-28T13:18:45.000+00:00,,,0,
+1,169,14,1,dev2,169,,task,7,1,rnd,,0,2021-04-28T13:19:54.000+00:00,,,0,
diff --git a/backend/plugins/zentao/e2e/snapshot_tables/_tool_zentao_changelog_detail.csv b/backend/plugins/zentao/e2e/snapshot_tables/_tool_zentao_changelog_detail.csv
new file mode 100644
index 000000000..88aee2bc1
--- /dev/null
+++ b/backend/plugins/zentao/e2e/snapshot_tables/_tool_zentao_changelog_detail.csv
@@ -0,0 +1,29 @@
+connection_id,id,changelog_id,field,old,new,diff
+1,68,68,begin,2012-06-05,2020-06-05,
+1,69,69,end,2012-12-04,2021-12-04,
+1,70,70,days,184,391,
+1,71,71,status,done,wait,
+1,72,72,begin,2013-06-05,2021-06-05,
+1,73,73,end,2014-06-04,2022-06-04,
+1,74,74,days,365,260,
+1,75,75,type,interface,codeerror,
+1,76,76,pri,0,1,
+1,77,77,pri,0,2,
+1,78,78,pri,0,1,
+1,79,79,pri,0,1,
+1,80,80,build,trunk,1,
+1,81,81,begin,2012-06-05,2020-06-05,
+1,82,82,end,2013-06-21,2021-06-21,
+1,83,83,begin,2000-01-01,2020-06-05,
+1,84,84,end,2015-01-01,2021-06-04,
+1,85,85,status,wait,doing,
+1,91,91,consumed,0,8,
+1,92,92,realStarted,0000-00-00 00:00:00,2021-04-28 13:17:54,
+1,93,93,finishedDate,,2021-04-01 16:00,
+1,94,94,status,wait,done,
+1,95,95,left,7,0,
+1,96,96,finishedBy,,dev1,
+1,97,97,realStarted,0000-00-00 00:00:00,2021-04-01 16:00,
+1,98,98,status,wait,doing,
+1,99,99,realStarted,0000-00-00 00:00:00,2021-04-01 08:00,
+1,100,100,status,wait,doing,
diff --git a/backend/plugins/zentao/impl/impl.go b/backend/plugins/zentao/impl/impl.go
index 244984330..56b8257e0 100644
--- a/backend/plugins/zentao/impl/impl.go
+++ b/backend/plugins/zentao/impl/impl.go
@@ -23,11 +23,14 @@ import (
"github.com/apache/incubator-devlake/core/context"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/core/runner"
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "github.com/apache/incubator-devlake/impls/dalgorm"
"github.com/apache/incubator-devlake/plugins/zentao/api"
"github.com/apache/incubator-devlake/plugins/zentao/models"
"github.com/apache/incubator-devlake/plugins/zentao/models/migrationscripts"
"github.com/apache/incubator-devlake/plugins/zentao/tasks"
+ "github.com/spf13/viper"
)
// make sure interface is implemented
@@ -54,6 +57,7 @@ func (p Zentao) SubTaskMetas() []plugin.SubTaskMeta {
return []plugin.SubTaskMeta{
tasks.ConvertProductMeta,
tasks.ConvertProjectMeta,
+ tasks.DBGetChangelogMeta,
tasks.CollectExecutionMeta,
tasks.ExtractExecutionMeta,
tasks.ConvertExecutionMeta,
@@ -95,10 +99,39 @@ func (p Zentao) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]i
return nil, errors.Default.Wrap(err, "unable to get Zentao API client instance: %v")
}
- return &tasks.ZentaoTaskData{
+ data := &tasks.ZentaoTaskData{
Options: op,
ApiClient: apiClient,
- }, nil
+ }
+
+ if connection.DbUrl != "" {
+ if connection.DbLoggingLevel == "" {
+ connection.DbLoggingLevel = taskCtx.GetConfig("DB_LOGGING_LEVEL")
+ }
+
+ if connection.DbIdleConns == 0 {
+ connection.DbIdleConns = taskCtx.GetConfigReader().GetInt("DB_IDLE_CONNS")
+ }
+
+ if connection.DbMaxConns == 0 {
+ connection.DbMaxConns = taskCtx.GetConfigReader().GetInt("DB_MAX_CONNS")
+ }
+
+ v := viper.New()
+ v.Set("DB_URL", connection.DbUrl)
+ v.Set("DB_LOGGING_LEVEL", connection.DbLoggingLevel)
+ v.Set("DB_IDLE_CONNS", connection.DbIdleConns)
+ v.Set("DbMaxConns", connection.DbMaxConns)
+
+ rgorm, err := runner.NewGormDb(v, taskCtx.GetLogger())
+ if err != nil {
+ return nil, errors.Default.Wrap(err, fmt.Sprintf("failed to connect to the zentao remote databases %s", connection.DbUrl))
+ }
+
+ data.RemoteDb = dalgorm.NewDalgorm(rgorm)
+ }
+
+ return data, nil
}
// PkgPath information lost when compiled as plugin(.so)
diff --git a/backend/plugins/zentao/models/archived/changelog.go b/backend/plugins/zentao/models/archived/changelog.go
new file mode 100644
index 000000000..249f1b15f
--- /dev/null
+++ b/backend/plugins/zentao/models/archived/changelog.go
@@ -0,0 +1,67 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package archived
+
+import (
+ "time"
+
+ "github.com/apache/incubator-devlake/core/models/migrationscripts/archived"
+)
+
+type ZentaoChangelog struct {
+ archived.NoPKModel `json:"-"`
+ ConnectionId uint64 `json:"connectionId" mapstructure:"connectionId" gorm:"primaryKey;type:BIGINT NOT NULL"`
+ Id int64 `json:"id" mapstructure:"id" gorm:"primaryKey;type:BIGINT NOT NULL"`
+ ObjectId int `json:"objectId" mapstructure:"objectId" gorm:"index; NOT NULL"`
+ Execution int `json:"execution" mapstructure:"execution" `
+ Actor string `json:"actor" mapstructure:"actor" `
+ Action string `json:"action" mapstructure:"action"`
+ Extra string `json:"extra" mapstructure:"extra"`
+ ObjectType string `json:"objectType" mapstructure:"objectType"`
+ Project int `json:"project" mapstructure:"project"`
+ Product string `json:"product" mapstructure:"product"`
+ Vision string `json:"vision" mapstructure:"vision"`
+ Comment string `json:"comment" mapstructure:"comment"`
+ Efforted string `json:"efforted" mapstructure:"efforted"`
+ Date time.Time `json:"date" mapstructure:"date"`
+ Read string `json:"read" mapstructure:"read"`
+}
+
+func (ZentaoChangelog) TableName() string {
+ return "_tool_zentao_changelog"
+}
+
+type ZentaoChangelogDetail struct {
+ archived.NoPKModel `json:"-"`
+ ConnectionId uint64 `json:"connectionId" mapstructure:"connectionId" gorm:"primaryKey;type:BIGINT NOT NULL"`
+ Id int64 `json:"id" mapstructure:"id" gorm:"primaryKey;type:BIGINT NOT NULL"`
+ ChangelogId int64 `json:"changelogId" mapstructure:"changelogId" gorm:"primaryKey;type:BIGINT NOT NULL"`
+ Field string `json:"field" mapstructure:"field"`
+ Old string `json:"old" mapstructure:"old"`
+ New string `json:"new" mapstructure:"new"`
+ Diff string `json:"diff" mapstructure:"diff"`
+}
+
+func (ZentaoChangelogDetail) TableName() string {
+ return "_tool_zentao_changelog_detail"
+}
+
+type ZentaoChangelogCom struct {
+ Changelog *ZentaoChangelog
+ ChangelogDetail *ZentaoChangelogDetail
+}
diff --git a/backend/plugins/zentao/models/archived/connection.go b/backend/plugins/zentao/models/archived/connection.go
index 3b6caa485..dab806d98 100644
--- a/backend/plugins/zentao/models/archived/connection.go
+++ b/backend/plugins/zentao/models/archived/connection.go
@@ -33,19 +33,6 @@ type TestConnectionRequest struct {
BasicAuth `mapstructure:",squash"`
}
-// This object conforms to what the frontend currently expects.
-type ZentaoResponse struct {
- Name string `json:"name"`
- ID int64 `json:"id"`
- ZentaoConnection
-}
-
-// Using User because it requires authentication.
-type ApiUserResponse struct {
- Id int64
- Name string `json:"name"`
-}
-
func (ZentaoConnection) TableName() string {
return "_tool_zentao_connections"
}
diff --git a/backend/plugins/zentao/models/changelog.go b/backend/plugins/zentao/models/changelog.go
new file mode 100644
index 000000000..152b0bdff
--- /dev/null
+++ b/backend/plugins/zentao/models/changelog.go
@@ -0,0 +1,67 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package models
+
+import (
+ "time"
+
+ "github.com/apache/incubator-devlake/core/models/common"
+)
+
+type ZentaoChangelog struct {
+ common.NoPKModel `json:"-"`
+ ConnectionId uint64 `json:"connectionId" mapstructure:"connectionId" gorm:"primaryKey;type:BIGINT NOT NULL"`
+ Id int64 `json:"id" mapstructure:"id" gorm:"primaryKey;type:BIGINT NOT NULL"`
+ ObjectId int `json:"objectId" mapstructure:"objectId" gorm:"index; NOT NULL"`
+ Execution int `json:"execution" mapstructure:"execution" `
+ Actor string `json:"actor" mapstructure:"actor" `
+ Action string `json:"action" mapstructure:"action"`
+ Extra string `json:"extra" mapstructure:"extra"`
+ ObjectType string `json:"objectType" mapstructure:"objectType"`
+ Project int `json:"project" mapstructure:"project"`
+ Product int `json:"product" mapstructure:"product"`
+ Vision string `json:"vision" mapstructure:"vision"`
+ Comment string `json:"comment" mapstructure:"comment"`
+ Efforted string `json:"efforted" mapstructure:"efforted"`
+ Date time.Time `json:"date" mapstructure:"date"`
+ Read string `json:"read" mapstructure:"read"`
+}
+
+func (ZentaoChangelog) TableName() string {
+ return "_tool_zentao_changelog"
+}
+
+type ZentaoChangelogDetail struct {
+ common.NoPKModel `json:"-"`
+ ConnectionId uint64 `json:"connectionId" mapstructure:"connectionId" gorm:"primaryKey;type:BIGINT NOT NULL"`
+ Id int64 `json:"id" mapstructure:"id" gorm:"primaryKey;type:BIGINT NOT NULL"`
+ ChangelogId int64 `json:"changelogId" mapstructure:"changelogId" gorm:"primaryKey;type:BIGINT NOT NULL"`
+ Field string `json:"field" mapstructure:"field"`
+ Old string `json:"old" mapstructure:"old"`
+ New string `json:"new" mapstructure:"new"`
+ Diff string `json:"diff" mapstructure:"diff"`
+}
+
+func (ZentaoChangelogDetail) TableName() string {
+ return "_tool_zentao_changelog_detail"
+}
+
+type ZentaoChangelogCom struct {
+ Changelog *ZentaoChangelog
+ ChangelogDetail *ZentaoChangelogDetail
+}
diff --git a/backend/plugins/zentao/models/connection.go b/backend/plugins/zentao/models/connection.go
index 23a1b58c9..cd39e6fbd 100644
--- a/backend/plugins/zentao/models/connection.go
+++ b/backend/plugins/zentao/models/connection.go
@@ -60,6 +60,11 @@ func (connection ZentaoConn) PrepareApiClient(apiClient apihelperabstract.ApiCli
type ZentaoConn struct {
helper.RestConnection `mapstructure:",squash"`
helper.BasicAuth `mapstructure:",squash"`
+
+ DbUrl string `mapstructure:"dbUrl" json:"dbUrl" gorm:"serializer:encdec"`
+ DbIdleConns int `json:"dbIdleConns" mapstructure:"dbIdleConns"`
+ DbLoggingLevel string `json:"dbLoggingLevel" mapstructure:"dbLoggingLevel"`
+ DbMaxConns int `json:"dbMaxConns" mapstructure:"dbMaxConns"`
}
// ZentaoConnection holds ZentaoConn plus ID/Name for database storage
diff --git a/backend/plugins/zentao/models/migrationscripts/20230519_add_init_changelog_tables.go b/backend/plugins/zentao/models/migrationscripts/20230519_add_init_changelog_tables.go
new file mode 100644
index 000000000..ec8c676c6
--- /dev/null
+++ b/backend/plugins/zentao/models/migrationscripts/20230519_add_init_changelog_tables.go
@@ -0,0 +1,64 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+ "github.com/apache/incubator-devlake/core/context"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/helpers/migrationhelper"
+ "github.com/apache/incubator-devlake/plugins/zentao/models/archived"
+)
+
+type addInitChangelogTables struct{}
+
+// This object conforms to what the frontend currently sends.
+type ZentaoConnection20230522 struct {
+ DbUrl string `mapstructure:"dbUrl" json:"dbUrl" gorm:"serializer:encdec"`
+ DbIdleConns int `json:"dbIdleConns" mapstructure:"dbIdleConns"`
+ DbLoggingLevel string `json:"dbLoggingLevel" mapstructure:"dbLoggingLevel"`
+ DbMaxConns int `json:"dbMaxConns" mapstructure:"dbMaxConns"`
+}
+
+func (ZentaoConnection20230522) TableName() string {
+ return "_tool_zentao_connections"
+}
+
+func (*addInitChangelogTables) Up(basicRes context.BasicRes) errors.Error {
+ db := basicRes.GetDal()
+ err := db.DropTables(
+ &archived.ZentaoChangelog{},
+ &archived.ZentaoChangelogDetail{},
+ )
+ if err != nil {
+ return err
+ }
+ return migrationhelper.AutoMigrateTables(
+ basicRes,
+ &archived.ZentaoChangelog{},
+ &archived.ZentaoChangelogDetail{},
+ &ZentaoConnection20230522{},
+ )
+}
+
+func (*addInitChangelogTables) Version() uint64 {
+ return 20230525000001
+}
+
+func (*addInitChangelogTables) Name() string {
+ return "zentao init changelog schemas"
+}
diff --git a/backend/plugins/zentao/models/migrationscripts/register.go b/backend/plugins/zentao/models/migrationscripts/register.go
index ec054748c..e184d69bc 100644
--- a/backend/plugins/zentao/models/migrationscripts/register.go
+++ b/backend/plugins/zentao/models/migrationscripts/register.go
@@ -25,5 +25,6 @@ import (
func All() []plugin.MigrationScript {
return []plugin.MigrationScript{
new(addInitTables),
+ new(addInitChangelogTables),
}
}
diff --git a/backend/plugins/zentao/models/product.go b/backend/plugins/zentao/models/product.go
index c67fdc39e..521286294 100644
--- a/backend/plugins/zentao/models/product.go
+++ b/backend/plugins/zentao/models/product.go
@@ -114,7 +114,7 @@ func (res ZentaoProductRes) ConvertApiScope() plugin.ToolLayerScope {
type ZentaoProduct struct {
common.NoPKModel `json:"-"`
- ConnectionId uint64 `json:"connectionid" mapstructure:"connectionid" gorm:"primaryKey;type:BIGINT NOT NULL"`
+ ConnectionId uint64 `json:"connectionId" mapstructure:"connectionId" gorm:"primaryKey;type:BIGINT NOT NULL"`
Id int64 `json:"id" mapstructure:"id" gorm:"primaryKey;type:BIGINT NOT NULL"`
Program int `json:"program" mapstructure:"program"`
Name string `json:"name" mapstructure:"name"`
diff --git a/backend/plugins/zentao/models/remote_db.go b/backend/plugins/zentao/models/remote_db.go
new file mode 100644
index 000000000..bb5439458
--- /dev/null
+++ b/backend/plugins/zentao/models/remote_db.go
@@ -0,0 +1,96 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package models
+
+import (
+ "time"
+)
+
+type ZentaoRemoteDbHistoryBase struct {
+ Field string `gorm:"column:field"`
+ Old string `gorm:"column:old"`
+ New string `gorm:"column:new"`
+ Diff string `gorm:"column:diff"`
+}
+
+type ZentaoRemoteDbHistory struct {
+ Id int `gorm:"column:id"`
+ Action int `gorm:"column:action"`
+ ZentaoRemoteDbHistoryBase
+}
+
+func (ZentaoRemoteDbHistory) TableName() string {
+ return "zt_history"
+}
+
+type ZentaoRemoteDbAction struct {
+ Id int `gorm:"column:id"`
+ ObjectType string `gorm:"column:objectType"`
+ ObjectId int `gorm:"column:objectID"`
+ Product string `gorm:"column:product"`
+ Project int `gorm:"column:project"`
+ Execution int `gorm:"column:execution"`
+ Actor string `gorm:"column:actor"`
+ Action string `gorm:"column:action"`
+ Date time.Time `gorm:"column:date"`
+ Comment string `gorm:"column:comment"`
+ Extra string `gorm:"column:extra"`
+ Read string `gorm:"column:read"`
+ Vision string `gorm:"column:vision"`
+ Efforted string `gorm:"column:efforted"`
+}
+
+func (ZentaoRemoteDbAction) TableName() string {
+ return "zt_action"
+}
+
+type ZentaoRemoteDbActionHistory struct {
+ ZentaoRemoteDbAction
+ ZentaoRemoteDbHistoryBase
+
+ ActionId int `gorm:"column:aid"`
+ HistoryId int `gorm:"column:hid"`
+}
+
+func (ah *ZentaoRemoteDbActionHistory) Convert() *ZentaoChangelogCom {
+ return &ZentaoChangelogCom{
+ &ZentaoChangelog{
+ Id: int64(ah.ActionId),
+ ObjectId: ah.ObjectId,
+ Execution: ah.Execution,
+ Actor: ah.Actor,
+ Action: ah.Action,
+ Extra: ah.Extra,
+ ObjectType: ah.ObjectType,
+ Project: ah.Project,
+ Vision: ah.Vision,
+ Comment: ah.Comment,
+ Efforted: ah.Efforted,
+ Date: ah.Date,
+ Read: ah.Read,
+ },
+ &ZentaoChangelogDetail{
+ Id: int64(ah.HistoryId),
+ ChangelogId: int64(ah.Id),
+ Field: ah.Field,
+ Old: ah.Old,
+ New: ah.New,
+ Diff: ah.Diff,
+ },
+ }
+}
diff --git a/backend/plugins/zentao/tasks/changelog_dbget.go b/backend/plugins/zentao/tasks/changelog_dbget.go
new file mode 100644
index 000000000..64d15630f
--- /dev/null
+++ b/backend/plugins/zentao/tasks/changelog_dbget.go
@@ -0,0 +1,122 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "fmt"
+ "reflect"
+
+ "github.com/apache/incubator-devlake/core/dal"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "github.com/apache/incubator-devlake/plugins/zentao/models"
+)
+
+var _ plugin.SubTaskEntryPoint = DBGetActionHistory
+
+func DBGetActionHistory(taskCtx plugin.SubTaskContext) errors.Error {
+ data := taskCtx.GetData().(*ZentaoTaskData)
+
+ // skip if no RemoteDb
+ if data.RemoteDb == nil {
+ return nil
+ }
+
+ divider := api.NewBatchSaveDivider(taskCtx, 500, "", "")
+ defer func() {
+ err1 := divider.Close()
+ if err1 != nil {
+ panic(err1)
+ }
+ }()
+
+ return dBGetActionHistory(data, func(zcc *models.ZentaoChangelogCom) errors.Error {
+ batch, err := divider.ForType(reflect.TypeOf(zcc.Changelog))
+ if err != nil {
+ return err
+ }
+ zcc.Changelog.ConnectionId = data.Options.ConnectionId
+ zcc.Changelog.Product = int(data.Options.ProductId)
+ err = batch.Add(zcc.Changelog)
+ if err != nil {
+ return err
+ }
+ if zcc.ChangelogDetail.Id != 0 {
+ batch, err = divider.ForType(reflect.TypeOf(zcc.ChangelogDetail))
+ if err != nil {
+ return err
+ }
+ zcc.ChangelogDetail.ConnectionId = data.Options.ConnectionId
+ err = batch.Add(zcc.ChangelogDetail)
+ if err != nil {
+ return err
+ }
+ }
+ return nil
+ })
+}
+
+var DBGetChangelogMeta = plugin.SubTaskMeta{
+ Name: "DBGetChangelog",
+ EntryPoint: DBGetActionHistory,
+ EnabledByDefault: true,
+ Description: "get action and history data to be changelog from Zentao databases",
+ DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET},
+}
+
+// it is work for zentao version 18.3
+func dBGetActionHistory(data *ZentaoTaskData, callback func(*models.ZentaoChangelogCom) errors.Error) errors.Error {
+ rdb := data.RemoteDb
+ atn := (models.ZentaoRemoteDbAction{}).TableName()
+ htn := (models.ZentaoRemoteDbHistory{}).TableName()
+
+ clause := []dal.Clause{
+ dal.Select(fmt.Sprintf("*,%s.id aid,%s.id hid ", atn, htn)),
+ dal.From(atn),
+ }
+
+ if data.Options.ProductId != 0 {
+ clause = append(clause, dal.Where(fmt.Sprintf("%s.product = ?", atn), fmt.Sprintf(",%d,", data.Options.ProductId)))
+ }
+ if data.Options.ProjectId != 0 {
+ clause = append(clause, dal.Where(fmt.Sprintf("%s.project = ?", atn), data.Options.ProjectId))
+ }
+ clause = append(clause, dal.Join(fmt.Sprintf("LEFT JOIN %s on %s.action = %s.id", htn, htn, atn)))
+
+ cursor, err := rdb.Cursor(clause...)
+ if err != nil {
+ return err
+ }
+ defer cursor.Close()
+
+ for cursor.Next() {
+ actionHistory := &models.ZentaoRemoteDbActionHistory{}
+ err = rdb.Fetch(cursor, actionHistory)
+ if err != nil {
+ return err
+ }
+
+ err = callback(actionHistory.Convert())
+ if err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
diff --git a/backend/plugins/zentao/tasks/task_data.go b/backend/plugins/zentao/tasks/task_data.go
index 83d47e7fe..406276f2e 100644
--- a/backend/plugins/zentao/tasks/task_data.go
+++ b/backend/plugins/zentao/tasks/task_data.go
@@ -19,6 +19,8 @@ package tasks
import (
"fmt"
+
+ "github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/mitchellh/mapstructure"
@@ -41,10 +43,12 @@ type ZentaoOptions struct {
TimeAfter string `json:"timeAfter" mapstructure:"timeAfter,omitempty"`
//TransformationRuleId uint64 `json:"transformationZentaoeId" mapstructure:"transformationRuleId,omitempty"`
//*models.ZentaoTransformationRule `mapstructure:"transformationRules,omitempty" json:"transformationRules"`
+
}
type ZentaoTaskData struct {
Options *ZentaoOptions
+ RemoteDb dal.Dal
ApiClient *helper.ApiAsyncClient
}