You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by kl...@apache.org on 2023/12/01 03:06:43 UTC
(incubator-devlake) branch kw-5519-dshelper created (now 0b034ae3a)
This is an automated email from the ASF dual-hosted git repository.
klesh pushed a change to branch kw-5519-dshelper
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
at 0b034ae3a refactor: gitlab adopts new remote api helper
This branch includes the following new commits:
new 0b034ae3a refactor: gitlab adopts new remote api helper
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(incubator-devlake) 01/01: refactor: gitlab adopts new remote api helper
Posted by kl...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
klesh pushed a commit to branch kw-5519-dshelper
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit 0b034ae3aab8ac0a6464747c44aaafc191b7c0eb
Author: Klesh Wong <zh...@merico.dev>
AuthorDate: Fri Dec 1 11:06:32 2023 +0800
refactor: gitlab adopts new remote api helper
---
.../api/ds_remote_api_scope_list_helper.go | 2 +-
backend/plugins/github/api/remote.go | 3 +-
backend/plugins/gitlab/api/init.go | 23 +-
backend/plugins/gitlab/api/proxy.go | 39 ---
backend/plugins/gitlab/api/remote.go | 322 ++++++++++++---------
backend/plugins/gitlab/models/connection.go | 2 +
backend/plugins/gitlab/models/project.go | 8 -
7 files changed, 204 insertions(+), 195 deletions(-)
diff --git a/backend/helpers/pluginhelper/api/ds_remote_api_scope_list_helper.go b/backend/helpers/pluginhelper/api/ds_remote_api_scope_list_helper.go
index 58e7db20e..60634c53d 100644
--- a/backend/helpers/pluginhelper/api/ds_remote_api_scope_list_helper.go
+++ b/backend/helpers/pluginhelper/api/ds_remote_api_scope_list_helper.go
@@ -65,7 +65,7 @@ func (rsl *DsRemoteApiScopeListHelper[C, S, P]) Get(input *plugin.ApiResourceInp
}
groupId := input.Query.Get("groupId")
pageInfo := new(P)
- pageToken := input.Params["pageToken"]
+ pageToken := input.Query.Get("pageToken")
// decode page token, we use pageToken because the pagination strategy varies from plugin to plugin
// some may use `page` and `size` while some may adopt `offset` and `limit`, even some may use `cursor`
if pageToken != "" {
diff --git a/backend/plugins/github/api/remote.go b/backend/plugins/github/api/remote.go
index f9f0176bd..16d128b05 100644
--- a/backend/plugins/github/api/remote.go
+++ b/backend/plugins/github/api/remote.go
@@ -226,9 +226,10 @@ func searchGithubRepos(
}
func toRepoModel(r *repo) dsmodels.DsRemoteApiScopeListEntry[models.GithubRepo] {
+ parentId := fmt.Sprintf("%v", r.Owner.Login)
return dsmodels.DsRemoteApiScopeListEntry[models.GithubRepo]{
Type: api.RAS_ENTRY_TYPE_SCOPE,
- ParentId: &r.Owner.Login,
+ ParentId: &parentId,
Id: fmt.Sprintf("%v", r.ID),
Name: fmt.Sprintf("%v", r.Name),
FullName: fmt.Sprintf("%v", r.FullName),
diff --git a/backend/plugins/gitlab/api/init.go b/backend/plugins/gitlab/api/init.go
index 02bf69694..570239e07 100644
--- a/backend/plugins/gitlab/api/init.go
+++ b/backend/plugins/gitlab/api/init.go
@@ -27,13 +27,14 @@ import (
var vld *validator.Validate
-var dsHelper *api.DsHelper[models.GitlabConnection, models.GitlabProject, models.GitlabScopeConfig]
-
-var connectionHelper *api.ConnectionApiHelper
-var remoteHelper *api.RemoteApiHelper[models.GitlabConnection, models.GitlabProject, models.GitlabApiProject, models.GroupResponse]
var basicRes context.BasicRes
+var dsHelper *api.DsHelper[models.GitlabConnection, models.GitlabProject, models.GitlabScopeConfig]
+var raProxy *api.DsRemoteApiProxyHelper[models.GitlabConnection]
+var raScopeList *api.DsRemoteApiScopeListHelper[models.GitlabConnection, models.GitlabProject, GitlabRemotePagination]
+var raScopeSearch *api.DsRemoteApiScopeSearchHelper[models.GitlabConnection, models.GitlabProject]
func Init(br context.BasicRes, p plugin.PluginMeta) {
+ vld = validator.New()
basicRes = br
dsHelper = api.NewDataSourceHelper[
models.GitlabConnection,
@@ -49,15 +50,7 @@ func Init(br context.BasicRes, p plugin.PluginMeta) {
nil,
)
// TODO: remove connectionHelper and refactor remoteHelper
- vld = validator.New()
- connectionHelper = api.NewConnectionHelper(
- basicRes,
- vld,
- p.Name(),
- )
- remoteHelper = api.NewRemoteHelper[models.GitlabConnection, models.GitlabProject, models.GitlabApiProject, models.GroupResponse](
- basicRes,
- vld,
- connectionHelper,
- )
+ raProxy = api.NewDsRemoteApiProxyHelper[models.GitlabConnection](dsHelper.ConnApi.ModelApiHelper)
+ raScopeList = api.NewDsRemoteApiScopeListHelper[models.GitlabConnection, models.GitlabProject, GitlabRemotePagination](raProxy, listGitlabRemoteScopes)
+ raScopeSearch = api.NewDsRemoteApiScopeSearchHelper[models.GitlabConnection, models.GitlabProject](raProxy, searchGitlabScopes)
}
diff --git a/backend/plugins/gitlab/api/proxy.go b/backend/plugins/gitlab/api/proxy.go
deleted file mode 100644
index f9ee83910..000000000
--- a/backend/plugins/gitlab/api/proxy.go
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package api
-
-import (
- "time"
-
- "github.com/apache/incubator-devlake/core/errors"
- "github.com/apache/incubator-devlake/core/plugin"
- "github.com/apache/incubator-devlake/plugins/gitlab/models"
-)
-
-const (
- TimeOut = 10 * time.Second
-)
-
-func Proxy(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
- connection := &models.GitlabConnection{}
- err := connectionHelper.First(connection, input.Params)
- if err != nil {
- return nil, err
- }
- return remoteHelper.ProxyApiGet(connection, input.Params["path"], input.Query)
-}
diff --git a/backend/plugins/gitlab/api/remote.go b/backend/plugins/gitlab/api/remote.go
index 25f1cfe46..edc1b8090 100644
--- a/backend/plugins/gitlab/api/remote.go
+++ b/backend/plugins/gitlab/api/remote.go
@@ -18,18 +18,165 @@ limitations under the License.
package api
import (
- gocontext "context"
"fmt"
"net/http"
"net/url"
+ "strings"
- "github.com/apache/incubator-devlake/core/context"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ dsmodels "github.com/apache/incubator-devlake/helpers/pluginhelper/api/models"
"github.com/apache/incubator-devlake/plugins/gitlab/models"
)
+type GitlabRemotePagination struct {
+ Page int `json:"page" mapstructure:"page"`
+ PerPage int `json:"per_page" mapstructure:"per_page"`
+ Step string `json:"step" mapstructure:"step"`
+}
+
+func (p GitlabRemotePagination) ToQuery() url.Values {
+ return url.Values{
+ "page": {fmt.Sprintf("%v", p.Page)},
+ "per_page": {fmt.Sprintf("%v", p.PerPage)},
+ }
+}
+
+func listGitlabRemoteScopes(
+ connection *models.GitlabConnection,
+ apiClient plugin.ApiClient,
+ groupId string,
+ page GitlabRemotePagination,
+) (
+ children []dsmodels.DsRemoteApiScopeListEntry[models.GitlabProject],
+ nextPage *GitlabRemotePagination,
+ err errors.Error,
+) {
+ if page.Page == 0 {
+ page.Page = 1
+ }
+ if page.PerPage == 0 {
+ page.PerPage = 100
+ }
+ if page.Step == "" {
+ page.Step = "group"
+ }
+
+ // load all groups unless groupId is user's own account
+ if page.Step == "group" && !strings.HasPrefix(groupId, "users/") {
+ children, nextPage, err = listGitlabRemoteGroups(connection, apiClient, groupId, page)
+ if err != nil {
+ return
+ }
+ // no more groups
+ if nextPage == nil {
+ nextPage = &GitlabRemotePagination{
+ Page: 1,
+ PerPage: page.PerPage,
+ Step: "project",
+ }
+ }
+ } else {
+ // load all project under the group or user's own account
+ children, nextPage, err = listGitlabRemoteProjects(connection, apiClient, groupId, page)
+ }
+ return
+}
+
+func listGitlabRemoteGroups(
+ connection *models.GitlabConnection,
+ apiClient plugin.ApiClient,
+ groupId string,
+ page GitlabRemotePagination,
+) (
+ children []dsmodels.DsRemoteApiScopeListEntry[models.GitlabProject],
+ nextPage *GitlabRemotePagination,
+ err errors.Error,
+) {
+ apiPath := ""
+ query := page.ToQuery()
+ var res *http.Response
+ if groupId == "" && page.Page == 1 {
+ // make users own account as a group
+ children = append(children, dsmodels.DsRemoteApiScopeListEntry[models.GitlabProject]{
+ Type: api.RAS_ENTRY_TYPE_GROUP,
+ Id: fmt.Sprintf("users/%v", apiClient.GetData("UserId")),
+ Name: apiClient.GetData("UserName").(string),
+ FullName: apiClient.GetData("UserName").(string),
+ })
+ }
+ if groupId == "" {
+ apiPath = "groups"
+ query.Set("top_level_only", "true")
+ } else {
+ apiPath = fmt.Sprintf("groups/%s/subgroups", groupId)
+ }
+ res, err = apiClient.Get(apiPath, query, nil)
+ var resGroups []models.GroupResponse
+ errors.Must(api.UnmarshalResponse(res, &resGroups))
+ for _, group := range resGroups {
+ children = append(children, dsmodels.DsRemoteApiScopeListEntry[models.GitlabProject]{
+ Type: api.RAS_ENTRY_TYPE_GROUP,
+ Id: fmt.Sprintf("%v", group.Id),
+ Name: group.Name,
+ FullName: group.FullPath,
+ })
+ }
+ nextPage = getNextPage(&page, res)
+ return
+}
+
+func listGitlabRemoteProjects(
+ connection *models.GitlabConnection,
+ apiClient plugin.ApiClient,
+ groupId string,
+ page GitlabRemotePagination,
+) (
+ children []dsmodels.DsRemoteApiScopeListEntry[models.GitlabProject],
+ nextPage *GitlabRemotePagination,
+ err errors.Error,
+) {
+ apiPath := ""
+ query := page.ToQuery()
+ query.Set("archived", "false")
+ query.Set("min_access_level", "20")
+ //
+ if strings.HasPrefix(groupId, "users/") {
+ apiPath = fmt.Sprintf("%s/projects", groupId)
+ } else {
+ apiPath = fmt.Sprintf("/groups/%s/projects", groupId)
+ }
+ res, err := apiClient.Get(apiPath, query, nil)
+ var resProjects []models.GitlabApiProject
+ errors.Must(api.UnmarshalResponse(res, &resProjects))
+ for _, project := range resProjects {
+ children = append(children, toProjectModel(&project))
+ }
+ nextPage = getNextPage(&page, res)
+ return
+}
+
+func toProjectModel(project *models.GitlabApiProject) dsmodels.DsRemoteApiScopeListEntry[models.GitlabProject] {
+ return dsmodels.DsRemoteApiScopeListEntry[models.GitlabProject]{
+ Type: api.RAS_ENTRY_TYPE_SCOPE,
+ Id: fmt.Sprintf("%v", project.GitlabId),
+ Name: project.Name,
+ FullName: project.PathWithNamespace,
+ }
+}
+
+func getNextPage(page *GitlabRemotePagination, res *http.Response) *GitlabRemotePagination {
+ if res.Header.Get("x-next-page") == "" {
+ return nil
+ }
+ return &GitlabRemotePagination{
+ Page: page.Page + 1,
+ PerPage: page.PerPage,
+ Step: page.Step,
+ }
+}
+
// RemoteScopes list all available scope for users
// @Summary list all available scope for users
// @Description list all available scope for users
@@ -43,107 +190,39 @@ import (
// @Failure 500 {object} shared.ApiBody "Internal Error"
// @Router /plugins/gitlab/connections/{connectionId}/remote-scopes [GET]
func RemoteScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
- var resGroup []models.GroupResponse
- return remoteHelper.GetScopesFromRemote(input,
- func(basicRes context.BasicRes, gid string, queryData *api.RemoteQueryData, connection models.GitlabConnection) ([]models.GroupResponse, errors.Error) {
- apiClient, err := api.NewApiClientFromConnection(gocontext.TODO(), basicRes, &connection)
- if err != nil {
- return nil, errors.BadInput.Wrap(err, "failed to get create apiClient")
- }
- query := initialQuery(queryData)
- var res *http.Response
- if gid == "" {
- query.Set("top_level_only", "true")
- res, err = apiClient.Get("groups", query, nil)
- if err != nil {
- return nil, err
- }
- } else {
- if gid[:6] == "group:" {
- gid = gid[6:]
- }
- res, err = apiClient.Get(fmt.Sprintf("groups/%s/subgroups", gid), query, nil)
- if err != nil {
- return nil, err
- }
- }
- err = api.UnmarshalResponse(res, &resGroup)
- if err != nil {
- return nil, err
- }
- return resGroup, err
- },
- func(basicRes context.BasicRes, gid string, queryData *api.RemoteQueryData, connection models.GitlabConnection) ([]models.GitlabApiProject, errors.Error) {
- apiClient, err := api.NewApiClientFromConnection(gocontext.TODO(), basicRes, &connection)
- if err != nil {
- return nil, errors.BadInput.Wrap(err, "failed to get create apiClient")
- }
- query := initialQuery(queryData)
- var res *http.Response
- var resBody []models.GitlabApiProject
- if gid == "" {
- var resProjects []models.GitlabApiProject
- if connection.Endpoint == models.GitlabCloudEndPoint {
- res, err = apiClient.Get(fmt.Sprintf("users/%d/projects", apiClient.GetData("UserId")), query, nil)
- } else {
- res, err = apiClient.Get("/projects", query, nil)
- }
- if err != nil {
- return nil, err
- }
- err = api.UnmarshalResponse(res, &resProjects)
- if err != nil {
- return nil, err
- }
-
- for _, project := range resProjects {
- skipProject := false
- for _, group := range resGroup {
- if project.Namespace.Name == group.Name && project.Namespace.Path == group.Path {
- skipProject = true
- break
- }
- }
- if !skipProject {
- resBody = append(resBody, project)
- }
- }
- } else {
- if gid[:6] == "group:" {
- gid = gid[6:]
- }
- res, err = apiClient.Get(fmt.Sprintf("/groups/%s/projects", gid), query, nil)
- if err != nil {
- return nil, err
- }
- err = api.UnmarshalResponse(res, &resBody)
- if err != nil {
- return nil, err
- }
- }
-
- // Filter out projects where the user only has Guest permission and archived projects
- var filteredProjects []models.GitlabApiProject
- for _, project := range resBody {
- membersURL := fmt.Sprintf("/projects/%d/members/%d", project.GitlabId, apiClient.GetData("UserId"))
- membersRes, err := apiClient.Get(membersURL, nil, nil)
- if err != nil {
- return nil, err
- }
- var member models.GitlabMember
- err = api.UnmarshalResponse(membersRes, &member)
- if err != nil {
- return nil, err
- }
- if member.AccessLevel != 10 && !project.Archived {
- filteredProjects = append(filteredProjects, project)
- }
- }
-
- resBody = filteredProjects
+ return raScopeList.Get(input)
+}
- return resBody, err
- })
+func searchGitlabScopes(
+ apiClient plugin.ApiClient,
+ params *dsmodels.DsRemoteApiScopeSearchParams,
+) (
+ children []dsmodels.DsRemoteApiScopeListEntry[models.GitlabProject],
+ err errors.Error,
+) {
+ res, err := apiClient.Get(
+ "projects",
+ url.Values{
+ "search": []string{params.Search},
+ "page": []string{fmt.Sprintf("%v", params.Page)},
+ "per_page": []string{fmt.Sprintf("%v", params.PageSize)},
+ "archived": {"false"},
+ "min_access_level": {"20"},
+ },
+ nil,
+ )
+ if err != nil {
+ return nil, err
+ }
+ var resBody []models.GitlabApiProject
+ err = api.UnmarshalResponse(res, &resBody)
+ if err != nil {
+ return nil, err
+ }
+ for _, r := range resBody {
+ children = append(children, toProjectModel(&r))
+ }
+ return
}
// SearchRemoteScopes use the Search API and only return project
@@ -160,35 +239,16 @@ func RemoteScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, er
// @Failure 500 {object} shared.ApiBody "Internal Error"
// @Router /plugins/gitlab/connections/{connectionId}/search-remote-scopes [GET]
func SearchRemoteScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
- return remoteHelper.SearchRemoteScopes(input,
- func(basicRes context.BasicRes, queryData *api.RemoteQueryData, connection models.GitlabConnection) ([]models.GitlabApiProject, errors.Error) {
- apiClient, err := api.NewApiClientFromConnection(gocontext.TODO(), basicRes, &connection)
- if err != nil {
- return nil, errors.BadInput.Wrap(err, "failed to get create apiClient")
- }
- query := initialQuery(queryData)
- query.Set("search", queryData.Search[0])
- // request search
- res, err := apiClient.Get("projects", query, nil)
- if err != nil {
- return nil, err
- }
- var resBody []models.GitlabApiProject
- err = api.UnmarshalResponse(res, &resBody)
- if err != nil {
- return nil, err
- }
- for i := 0; i < len(resBody); i++ {
- // as we need to set PathWithNamespace to name in SearchRemoteScopes, but interface.ScopeName will return name, so we switch it
- resBody[i].Name, resBody[i].PathWithNamespace = resBody[i].PathWithNamespace, resBody[i].Name
- }
- return resBody, err
- })
+ return raScopeSearch.Get(input)
}
-func initialQuery(queryData *api.RemoteQueryData) url.Values {
- query := url.Values{}
- query.Set("page", fmt.Sprintf("%v", queryData.Page))
- query.Set("per_page", fmt.Sprintf("%v", queryData.PerPage))
- return query
+// Proxy is a proxy to Gitlab API
+// @Summary Proxy to Gitlab API
+// @Description Proxy to Gitlab API
+// @Tags plugins/gitlab
+// @Param connectionId path int true "connection ID"
+// @Param path path string true "path to Gitlab API"
+// @Router /plugins/gitlab/connections/{connectionId}/proxy/{path} [GET]
+func Proxy(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
+ return raProxy.Proxy(input)
}
diff --git a/backend/plugins/gitlab/models/connection.go b/backend/plugins/gitlab/models/connection.go
index f9c15f2af..aad7c6b98 100644
--- a/backend/plugins/gitlab/models/connection.go
+++ b/backend/plugins/gitlab/models/connection.go
@@ -34,6 +34,7 @@ type GitlabConn struct {
const GitlabCloudEndPoint string = "https://gitlab.com/api/v4/"
const GitlabApiClientData_UserId string = "UserId"
+const GitlabApiClientData_UserName string = "UserName"
const GitlabApiClientData_ApiVersion string = "ApiVersion"
// this function is used to rewrite the same function of AccessToken
@@ -109,6 +110,7 @@ func (conn *GitlabConn) PrepareApiClient(apiClient plugin.ApiClient) errors.Erro
}
apiClient.SetData(GitlabApiClientData_UserId, userResBody.Id)
+ apiClient.SetData(GitlabApiClientData_UserName, userResBody.Name)
apiClient.SetData(GitlabApiClientData_ApiVersion, versionResBody.Version)
return nil
diff --git a/backend/plugins/gitlab/models/project.go b/backend/plugins/gitlab/models/project.go
index 73b1a311e..54908af74 100644
--- a/backend/plugins/gitlab/models/project.go
+++ b/backend/plugins/gitlab/models/project.go
@@ -152,14 +152,6 @@ type GroupResponse struct {
ParentId *int `json:"parent_id"`
}
-func (p GroupResponse) GroupId() string {
- return "group:" + strconv.Itoa(p.Id)
-}
-
-func (p GroupResponse) GroupName() string {
- return p.Name
-}
-
type GitlabApiParams struct {
ConnectionId uint64
ProjectId int