You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by wa...@apache.org on 2022/09/13 06:31:50 UTC
[incubator-devlake] branch main updated: fix(framework): add board_repo record for git* issues
This is an automated email from the ASF dual-hosted git repository.
warren pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new f1a63baf fix(framework): add board_repo record for git* issues
f1a63baf is described below
commit f1a63baf9cd12bc2da39c2249bffc0c7dc816c87
Author: Yingchu Chen <yi...@merico.dev>
AuthorDate: Tue Sep 13 12:36:28 2022 +0800
fix(framework): add board_repo record for git* issues
closes #3056
---
models/domainlayer/crossdomain/board_repo.go | 3 ++
.../20220913_modfiy_board_repos.go} | 34 ++++++++++++++++++----
models/migrationscripts/register.go | 1 +
plugins/gitee/tasks/repo_convertor.go | 7 +++++
plugins/github/e2e/repo_test.go | 11 +++++++
plugins/github/e2e/snapshot_tables/board_repos.csv | 2 ++
plugins/github/tasks/repo_convertor.go | 6 ++++
plugins/gitlab/e2e/project_test.go | 23 +++++++++++++++
plugins/gitlab/e2e/snapshot_tables/board_repos.csv | 2 ++
plugins/gitlab/e2e/snapshot_tables/boards.csv | 2 ++
plugins/gitlab/tasks/project_convertor.go | 12 +++++++-
11 files changed, 97 insertions(+), 6 deletions(-)
diff --git a/models/domainlayer/crossdomain/board_repo.go b/models/domainlayer/crossdomain/board_repo.go
index 003ca7ee..251acc94 100644
--- a/models/domainlayer/crossdomain/board_repo.go
+++ b/models/domainlayer/crossdomain/board_repo.go
@@ -17,9 +17,12 @@ limitations under the License.
package crossdomain
+import "github.com/apache/incubator-devlake/models/common"
+
type BoardRepo struct {
BoardId string `gorm:"primaryKey;type:varchar(255)"`
RepoId string `gorm:"primaryKey;type:varchar(255)"`
+ common.NoPKModel
}
func (BoardRepo) TableName() string {
diff --git a/models/domainlayer/crossdomain/board_repo.go b/models/migrationscripts/20220913_modfiy_board_repos.go
similarity index 58%
copy from models/domainlayer/crossdomain/board_repo.go
copy to models/migrationscripts/20220913_modfiy_board_repos.go
index 003ca7ee..7b677ace 100644
--- a/models/domainlayer/crossdomain/board_repo.go
+++ b/models/migrationscripts/20220913_modfiy_board_repos.go
@@ -15,13 +15,37 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
-package crossdomain
+package migrationscripts
-type BoardRepo struct {
- BoardId string `gorm:"primaryKey;type:varchar(255)"`
- RepoId string `gorm:"primaryKey;type:varchar(255)"`
+import (
+ "context"
+ "github.com/apache/incubator-devlake/models/migrationscripts/archived"
+
+ "gorm.io/gorm"
+)
+
+type modifyBoardRepos struct{}
+
+func (*modifyBoardRepos) Up(ctx context.Context, db *gorm.DB) error {
+ err := db.Migrator().AutoMigrate(BoardRepo0913{})
+ if err != nil {
+ return err
+ }
+ return nil
+}
+
+func (*modifyBoardRepos) Version() uint64 {
+ return 20220913232735
+}
+
+func (*modifyBoardRepos) Name() string {
+ return "modify board repos"
+}
+
+type BoardRepo0913 struct {
+ archived.NoPKModel
}
-func (BoardRepo) TableName() string {
+func (BoardRepo0913) TableName() string {
return "board_repos"
}
diff --git a/models/migrationscripts/register.go b/models/migrationscripts/register.go
index ceef2660..c6a090a4 100644
--- a/models/migrationscripts/register.go
+++ b/models/migrationscripts/register.go
@@ -43,5 +43,6 @@ func All() []migration.Script {
new(encryptBLueprint),
new(encryptPipeline),
new(modifyCICDTasks),
+ new(modifyBoardRepos),
}
}
diff --git a/plugins/gitee/tasks/repo_convertor.go b/plugins/gitee/tasks/repo_convertor.go
index 029b6542..1b959cd1 100644
--- a/plugins/gitee/tasks/repo_convertor.go
+++ b/plugins/gitee/tasks/repo_convertor.go
@@ -19,6 +19,7 @@ package tasks
import (
"fmt"
+ "github.com/apache/incubator-devlake/models/domainlayer/crossdomain"
"reflect"
"github.com/apache/incubator-devlake/plugins/core/dal"
@@ -85,9 +86,15 @@ func ConvertRepo(taskCtx core.SubTaskContext) error {
CreatedDate: &repository.CreatedDate,
}
+ domainBoardRepo := &crossdomain.BoardRepo{
+ BoardId: repoIdGen.Generate(data.Options.ConnectionId, repository.GiteeId),
+ RepoId: repoIdGen.Generate(data.Options.ConnectionId, repository.GiteeId),
+ }
+
return []interface{}{
domainRepository,
domainBoard,
+ domainBoardRepo,
}, nil
},
})
diff --git a/plugins/github/e2e/repo_test.go b/plugins/github/e2e/repo_test.go
index a3ac451b..fc218a6b 100644
--- a/plugins/github/e2e/repo_test.go
+++ b/plugins/github/e2e/repo_test.go
@@ -18,6 +18,7 @@ limitations under the License.
package e2e
import (
+ "github.com/apache/incubator-devlake/models/domainlayer/crossdomain"
"testing"
"github.com/apache/incubator-devlake/models/domainlayer/code"
@@ -90,6 +91,7 @@ func TestRepoDataFlow(t *testing.T) {
// verify extraction
dataflowTester.FlushTabler(&code.Repo{})
dataflowTester.FlushTabler(&ticket.Board{})
+ dataflowTester.FlushTabler(&crossdomain.BoardRepo{})
dataflowTester.Subtask(tasks.ConvertRepoMeta, taskData)
dataflowTester.VerifyTable(
code.Repo{},
@@ -118,4 +120,13 @@ func TestRepoDataFlow(t *testing.T) {
"created_date",
},
)
+
+ dataflowTester.VerifyTable(
+ crossdomain.BoardRepo{},
+ "./snapshot_tables/board_repos.csv",
+ []string{
+ "board_id",
+ "repo_id",
+ },
+ )
}
diff --git a/plugins/github/e2e/snapshot_tables/board_repos.csv b/plugins/github/e2e/snapshot_tables/board_repos.csv
new file mode 100644
index 00000000..f0a12154
--- /dev/null
+++ b/plugins/github/e2e/snapshot_tables/board_repos.csv
@@ -0,0 +1,2 @@
+board_id,repo_id
+github:GithubRepo:1:134018330,github:GithubRepo:1:134018330
diff --git a/plugins/github/tasks/repo_convertor.go b/plugins/github/tasks/repo_convertor.go
index ac3f90bc..a6353f89 100644
--- a/plugins/github/tasks/repo_convertor.go
+++ b/plugins/github/tasks/repo_convertor.go
@@ -19,6 +19,7 @@ package tasks
import (
"fmt"
+ "github.com/apache/incubator-devlake/models/domainlayer/crossdomain"
"reflect"
"github.com/apache/incubator-devlake/plugins/core/dal"
@@ -93,9 +94,14 @@ func ConvertRepo(taskCtx core.SubTaskContext) error {
CreatedDate: &repository.CreatedDate,
}
+ domainBoardRepo := &crossdomain.BoardRepo{
+ BoardId: repoIdGen.Generate(data.Options.ConnectionId, repository.GithubId),
+ RepoId: repoIdGen.Generate(data.Options.ConnectionId, repository.GithubId),
+ }
return []interface{}{
domainRepository,
domainBoard,
+ domainBoardRepo,
}, nil
},
})
diff --git a/plugins/gitlab/e2e/project_test.go b/plugins/gitlab/e2e/project_test.go
index ac40e6f0..0b6ebfdb 100644
--- a/plugins/gitlab/e2e/project_test.go
+++ b/plugins/gitlab/e2e/project_test.go
@@ -18,6 +18,8 @@ limitations under the License.
package e2e
import (
+ "github.com/apache/incubator-devlake/models/domainlayer/crossdomain"
+ "github.com/apache/incubator-devlake/models/domainlayer/ticket"
"testing"
"github.com/apache/incubator-devlake/helpers/e2ehelper"
@@ -74,6 +76,8 @@ func TestGitlabProjectDataFlow(t *testing.T) {
// verify conversion
dataflowTester.FlushTabler(&code.Repo{})
+ dataflowTester.FlushTabler(&ticket.Board{})
+ dataflowTester.FlushTabler(&crossdomain.BoardRepo{})
dataflowTester.Subtask(tasks.ConvertProjectMeta, taskData)
dataflowTester.VerifyTable(
code.Repo{},
@@ -95,4 +99,23 @@ func TestGitlabProjectDataFlow(t *testing.T) {
"deleted",
},
)
+ dataflowTester.VerifyTable(
+ ticket.Board{},
+ "./snapshot_tables/boards.csv",
+ []string{
+ "id",
+ "name",
+ "description",
+ "url",
+ "created_date",
+ },
+ )
+ dataflowTester.VerifyTable(
+ crossdomain.BoardRepo{},
+ "./snapshot_tables/board_repos.csv",
+ []string{
+ "board_id",
+ "repo_id",
+ },
+ )
}
diff --git a/plugins/gitlab/e2e/snapshot_tables/board_repos.csv b/plugins/gitlab/e2e/snapshot_tables/board_repos.csv
new file mode 100644
index 00000000..837a6c03
--- /dev/null
+++ b/plugins/gitlab/e2e/snapshot_tables/board_repos.csv
@@ -0,0 +1,2 @@
+board_id,repo_id
+gitlab:GitlabProject:1:12345678,gitlab:GitlabProject:1:12345678
diff --git a/plugins/gitlab/e2e/snapshot_tables/boards.csv b/plugins/gitlab/e2e/snapshot_tables/boards.csv
new file mode 100644
index 00000000..b65e4644
--- /dev/null
+++ b/plugins/gitlab/e2e/snapshot_tables/boards.csv
@@ -0,0 +1,2 @@
+id,name,description,url,created_date
+gitlab:GitlabProject:1:12345678,Snowflake Spend,This is a dbt package for understanding the cost your Snowflake Data Warehouse is accruing.,https://gitlab.com/gitlab-data/snowflake_spend,2019-06-20T14:14:45.108+00:00
diff --git a/plugins/gitlab/tasks/project_convertor.go b/plugins/gitlab/tasks/project_convertor.go
index c1124ec2..59d271e3 100644
--- a/plugins/gitlab/tasks/project_convertor.go
+++ b/plugins/gitlab/tasks/project_convertor.go
@@ -18,6 +18,7 @@ limitations under the License.
package tasks
import (
+ "github.com/apache/incubator-devlake/models/domainlayer/crossdomain"
"reflect"
"github.com/apache/incubator-devlake/plugins/core/dal"
@@ -62,10 +63,11 @@ func ConvertApiProjects(taskCtx core.SubTaskContext) error {
domainRepository := convertToRepositoryModel(gitlabProject)
domainBoard := convertToBoardModel(gitlabProject)
-
+ domainBoardRepo := convertToBoardRepoModel(gitlabProject)
return []interface{}{
domainRepository,
domainBoard,
+ domainBoardRepo,
}, nil
},
})
@@ -103,3 +105,11 @@ func convertToBoardModel(project *models.GitlabProject) *ticket.Board {
}
return domainBoard
}
+
+func convertToBoardRepoModel(project *models.GitlabProject) *crossdomain.BoardRepo {
+ domainBoardRepo := &crossdomain.BoardRepo{
+ BoardId: didgen.NewDomainIdGenerator(project).Generate(project.ConnectionId, project.GitlabId),
+ RepoId: didgen.NewDomainIdGenerator(project).Generate(project.ConnectionId, project.GitlabId),
+ }
+ return domainBoardRepo
+}