You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by zh...@apache.org on 2022/06/16 09:31:25 UTC
[incubator-devlake] branch main updated: refactor(gitlab): add conn id to project (#2212)
This is an automated email from the ASF dual-hosted git repository.
zhangliang2022 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 2b5e58dd refactor(gitlab): add conn id to project (#2212)
2b5e58dd is described below
commit 2b5e58dd0b51934e0dab1020e25dd0cd8539c1c1
Author: Warren Chen <yi...@merico.dev>
AuthorDate: Thu Jun 16 17:31:21 2022 +0800
refactor(gitlab): add conn id to project (#2212)
---
config/config.go | 13 ++++++-------
plugins/gitlab/e2e/project_test.go | 5 +++--
.../gitlab/e2e/tables/_raw_gitlab_api_projects.csv | 2 +-
plugins/gitlab/e2e/tables/_tool_gitlab_projects.csv | 2 +-
plugins/gitlab/e2e/tables/repos.csv | 2 +-
plugins/gitlab/gitlab.go | 5 ++++-
plugins/gitlab/impl/impl.go | 20 +++++++++++++++++++-
plugins/gitlab/models/connection.go | 4 +---
plugins/gitlab/tasks/pipeline_collector.go | 1 +
plugins/gitlab/tasks/project_extractor.go | 4 ++--
plugins/gitlab/tasks/shared.go | 10 ++++++----
11 files changed, 45 insertions(+), 23 deletions(-)
diff --git a/config/config.go b/config/config.go
index 9e57cf96..aeab9def 100644
--- a/config/config.go
+++ b/config/config.go
@@ -45,13 +45,12 @@ func initConfig(v *viper.Viper) {
v.SetConfigType("env")
envPath := getEnvPath()
// AddConfigPath adds a path for Viper to search for the config file in.
- if envPath == "" {
- v.AddConfigPath("$PWD/../..")
- v.AddConfigPath("..")
- v.AddConfigPath(".")
- } else {
- v.AddConfigPath(envPath)
- }
+ v.AddConfigPath("$PWD/../..")
+ v.AddConfigPath("$PWD/../../..")
+ v.AddConfigPath("..")
+ v.AddConfigPath(".")
+ v.AddConfigPath(envPath)
+
}
func getConfigName() string {
diff --git a/plugins/gitlab/e2e/project_test.go b/plugins/gitlab/e2e/project_test.go
index c7342ef1..6c094b46 100644
--- a/plugins/gitlab/e2e/project_test.go
+++ b/plugins/gitlab/e2e/project_test.go
@@ -27,7 +27,7 @@ import (
"github.com/apache/incubator-devlake/plugins/gitlab/tasks"
)
-func TestGitlabDataFlow(t *testing.T) {
+func TestGitlabProjectDataFlow(t *testing.T) {
var gitlab impl.Gitlab
dataflowTester := e2ehelper.NewDataFlowTester(t, "gitlab", gitlab)
@@ -40,7 +40,8 @@ func TestGitlabDataFlow(t *testing.T) {
}
// import raw data table
- dataflowTester.ImportCsvIntoRawTable("./tables/_raw_gitlab_api_projects.csv", "_raw_gitlab_api_project")
+ dataflowTester.ImportCsvIntoRawTable("./tables/_raw_gitlab_api_projects.csv",
+ "_raw_gitlab_api_project")
// verify extraction
dataflowTester.FlushTabler(&models.GitlabProject{})
diff --git a/plugins/gitlab/e2e/tables/_raw_gitlab_api_projects.csv b/plugins/gitlab/e2e/tables/_raw_gitlab_api_projects.csv
index 9b5559c3..438acac6 100644
--- a/plugins/gitlab/e2e/tables/_raw_gitlab_api_projects.csv
+++ b/plugins/gitlab/e2e/tables/_raw_gitlab_api_projects.csv
@@ -1,2 +1,2 @@
"id","params","data","url","input","created_at"
-2,"{""ProjectId"":3472737}","{""id"": 3472737, ""name"": ""inkscape"", ""path"": ""inkscape"", ""_links"": {""self"": ""https://gitlab.com/api/v4/projects/3472737"", ""events"": ""https://gitlab.com/api/v4/projects/3472737/events"", ""issues"": ""https://gitlab.com/api/v4/projects/3472737/issues"", ""labels"": ""https://gitlab.com/api/v4/projects/3472737/labels"", ""members"": ""https://gitlab.com/api/v4/projects/3472737/members"", ""repo_branches"": ""https://gitlab.com/api/v4/projects/ [...]
+2,"{""ConnectionId"":1,""ProjectId"":3472737}","{""id"": 3472737, ""name"": ""inkscape"", ""path"": ""inkscape"", ""_links"": {""self"": ""https://gitlab.com/api/v4/projects/3472737"", ""events"": ""https://gitlab.com/api/v4/projects/3472737/events"", ""issues"": ""https://gitlab.com/api/v4/projects/3472737/issues"", ""labels"": ""https://gitlab.com/api/v4/projects/3472737/labels"", ""members"": ""https://gitlab.com/api/v4/projects/3472737/members"", ""repo_branches"": ""https://gitlab.c [...]
diff --git a/plugins/gitlab/e2e/tables/_tool_gitlab_projects.csv b/plugins/gitlab/e2e/tables/_tool_gitlab_projects.csv
index 4c8690a9..99991a96 100644
--- a/plugins/gitlab/e2e/tables/_tool_gitlab_projects.csv
+++ b/plugins/gitlab/e2e/tables/_tool_gitlab_projects.csv
@@ -1,2 +1,2 @@
connection_id,gitlab_id,name,description,default_branch,path_with_namespace,web_url,creator_id,visibility,open_issues_count,star_count,forked_from_project_id,forked_from_project_web_url,created_date,updated_date,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
-0,3472737,inkscape,Inkscape vector image editor,master,inkscape/inkscape,https://gitlab.com/inkscape/inkscape,0,public,1472,2627,0,,2017-06-09T14:16:35.615+00:00,2022-05-05T08:26:20.527+00:00,"{""ProjectId"":3472737}",_raw_gitlab_api_project,2,
+1,3472737,inkscape,Inkscape vector image editor,master,inkscape/inkscape,https://gitlab.com/inkscape/inkscape,0,public,1472,2627,0,,2017-06-09T14:16:35.615+00:00,2022-05-05T08:26:20.527+00:00,"{""ConnectionId"":1,""ProjectId"":3472737}",_raw_gitlab_api_project,2,
diff --git a/plugins/gitlab/e2e/tables/repos.csv b/plugins/gitlab/e2e/tables/repos.csv
index 0ec8467f..0ae80177 100644
--- a/plugins/gitlab/e2e/tables/repos.csv
+++ b/plugins/gitlab/e2e/tables/repos.csv
@@ -1,2 +1,2 @@
id,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark,name,url,description,owner_id,language,forked_from,created_date,updated_date,deleted
-gitlab:GitlabProject:0:3472737,"{""ProjectId"":3472737}",_raw_gitlab_api_project,2,,inkscape,https://gitlab.com/inkscape/inkscape,Inkscape vector image editor,,,,2017-06-09T14:16:35.615+00:00,2022-05-05T08:26:20.527+00:00,0
+gitlab:GitlabProject:1:3472737,"{""ConnectionId"":1,""ProjectId"":3472737}",_raw_gitlab_api_project,2,,inkscape,https://gitlab.com/inkscape/inkscape,Inkscape vector image editor,,,,2017-06-09T14:16:35.615+00:00,2022-05-05T08:26:20.527+00:00,0
diff --git a/plugins/gitlab/gitlab.go b/plugins/gitlab/gitlab.go
index 88ce9f10..462a4e4c 100644
--- a/plugins/gitlab/gitlab.go
+++ b/plugins/gitlab/gitlab.go
@@ -30,11 +30,14 @@ var PluginEntry impl.Gitlab //nolint
func main() {
gitlabCmd := &cobra.Command{Use: "gitlab"}
projectId := gitlabCmd.Flags().IntP("project-id", "p", 0, "gitlab project id")
+ connectionId := gitlabCmd.Flags().Uint64P("connection-id", "c", 0, "gitlab connection id")
_ = gitlabCmd.MarkFlagRequired("project-id")
+ _ = gitlabCmd.MarkFlagRequired("connection-id")
gitlabCmd.Run = func(cmd *cobra.Command, args []string) {
runner.DirectRun(cmd, args, PluginEntry, map[string]interface{}{
- "projectId": *projectId,
+ "projectId": *projectId,
+ "connectionId": *connectionId,
})
}
diff --git a/plugins/gitlab/impl/impl.go b/plugins/gitlab/impl/impl.go
index 18dfb3e8..c80082e9 100644
--- a/plugins/gitlab/impl/impl.go
+++ b/plugins/gitlab/impl/impl.go
@@ -18,11 +18,14 @@ limitations under the License.
package impl
import (
+ "fmt"
"github.com/apache/incubator-devlake/migration"
"github.com/apache/incubator-devlake/plugins/core"
"github.com/apache/incubator-devlake/plugins/gitlab/api"
+ "github.com/apache/incubator-devlake/plugins/gitlab/models"
"github.com/apache/incubator-devlake/plugins/gitlab/models/migrationscripts"
"github.com/apache/incubator-devlake/plugins/gitlab/tasks"
+ "github.com/apache/incubator-devlake/plugins/helper"
"github.com/mitchellh/mapstructure"
"github.com/spf13/viper"
"gorm.io/gorm"
@@ -82,8 +85,23 @@ func (plugin Gitlab) PrepareTaskData(taskCtx core.TaskContext, options map[strin
if err != nil {
return nil, err
}
-
+ if op.ConnectionId == 0 {
+ return nil, fmt.Errorf("connectionId is invalid")
+ }
+ connection := &models.GitlabConnection{}
+ connectionHelper := helper.NewConnectionHelper(
+ taskCtx,
+ nil,
+ )
+ if err != nil {
+ return nil, err
+ }
+ err = connectionHelper.FirstById(connection, op.ConnectionId)
+ if err != nil {
+ return nil, err
+ }
apiClient, err := tasks.NewGitlabApiClient(taskCtx)
+
if err != nil {
return nil, err
}
diff --git a/plugins/gitlab/models/connection.go b/plugins/gitlab/models/connection.go
index a9ae6072..23112d60 100644
--- a/plugins/gitlab/models/connection.go
+++ b/plugins/gitlab/models/connection.go
@@ -17,9 +17,7 @@ limitations under the License.
package models
-import (
- "github.com/apache/incubator-devlake/plugins/helper"
-)
+import "github.com/apache/incubator-devlake/plugins/helper"
// This object conforms to what the frontend currently sends.
type GitlabConnection struct {
diff --git a/plugins/gitlab/tasks/pipeline_collector.go b/plugins/gitlab/tasks/pipeline_collector.go
index 3dfcff4d..bc9c135c 100644
--- a/plugins/gitlab/tasks/pipeline_collector.go
+++ b/plugins/gitlab/tasks/pipeline_collector.go
@@ -39,6 +39,7 @@ func CollectApiPipelines(taskCtx core.SubTaskContext) error {
ApiClient: data.ApiClient,
PageSize: 100,
Incremental: false,
+ Concurrency: 10,
UrlTemplate: "projects/{{ .Params.ProjectId }}/pipelines",
Query: GetQuery,
ResponseParser: GetRawMessageFromResponse,
diff --git a/plugins/gitlab/tasks/project_extractor.go b/plugins/gitlab/tasks/project_extractor.go
index d33eb207..da953ee6 100644
--- a/plugins/gitlab/tasks/project_extractor.go
+++ b/plugins/gitlab/tasks/project_extractor.go
@@ -32,7 +32,7 @@ var ExtractProjectMeta = core.SubTaskMeta{
}
func ExtractApiProject(taskCtx core.SubTaskContext) error {
- rawDataSubTaskArgs, _ := CreateRawDataSubTaskArgs(taskCtx, RAW_PROJECT_TABLE)
+ rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_PROJECT_TABLE)
extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{
RawDataSubTaskArgs: *rawDataSubTaskArgs,
@@ -44,7 +44,7 @@ func ExtractApiProject(taskCtx core.SubTaskContext) error {
return nil, err
}
gitlabProject := convertProject(gitlabApiProject)
-
+ gitlabProject.ConnectionId = data.Options.ConnectionId
results := make([]interface{}, 0, 1)
results = append(results, gitlabProject)
diff --git a/plugins/gitlab/tasks/shared.go b/plugins/gitlab/tasks/shared.go
index f29b5260..66be35f3 100644
--- a/plugins/gitlab/tasks/shared.go
+++ b/plugins/gitlab/tasks/shared.go
@@ -32,7 +32,8 @@ import (
)
type GitlabApiParams struct {
- ProjectId int
+ ConnectionId uint64
+ ProjectId int
}
type GitlabInput struct {
@@ -86,7 +87,8 @@ func CreateRawDataSubTaskArgs(taskCtx core.SubTaskContext, Table string) (*helpe
RawDataSubTaskArgs := &helper.RawDataSubTaskArgs{
Ctx: taskCtx,
Params: GitlabApiParams{
- ProjectId: data.Options.ProjectId,
+ ProjectId: data.Options.ProjectId,
+ ConnectionId: data.Options.ConnectionId,
},
Table: Table,
}
@@ -100,8 +102,8 @@ func GetMergeRequestsIterator(taskCtx core.SubTaskContext) (*helper.DalCursorIte
dal.Select("gmr.gitlab_id, gmr.iid"),
dal.From("_tool_gitlab_merge_requests gmr"),
dal.Where(
- `gmr.project_id = ?`,
- data.Options.ProjectId,
+ `gmr.project_id = ? and gmr.connection_id = ?`,
+ data.Options.ProjectId, data.Options.ConnectionId,
),
}
// construct the input iterator