You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by kl...@apache.org on 2023/02/10 14:02:10 UTC
[incubator-devlake] branch main updated: Surpport Gitlab v11 surpport (#4359)
This is an automated email from the ASF dual-hosted git repository.
klesh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 4026548c8 Surpport Gitlab v11 surpport (#4359)
4026548c8 is described below
commit 4026548c8cc81f99848d56a0d5d73fa58404e628
Author: mappjzc <zh...@merico.dev>
AuthorDate: Fri Feb 10 22:02:05 2023 +0800
Surpport Gitlab v11 surpport (#4359)
* feat: gitlab v11 surpport
Add ExtractApiPipelineDetails
Add CollectApiPipelineDetails
Add ExtractAccountDetails
Add CollectAccountDetails
Add ExtractApiMergeRequestDetails
Add CollectApiMergeRequestDetails
Add NewApiClientFromConnectionWithTest
updated gitlab connection creating prograss
Nddtfjiang <zh...@merico.dev>
* fix: fix for review
fix for review
Nddtfjiang <zh...@merico.dev>
* feat: gitlab connection adding
Add PrepareApiClient for gitlab connection
Add Version for gitlab connection
Nddtfjiang <zh...@merico.dev>
* fix: add apiClient data
Add ApiClient data for gitlab
Nddtfjiang <zh...@merico.dev>
* feat: add is detail required
Add IsDetailRequired to _tool_gitlab_merge_requests
Add IsDetailRequired to _tool_gitlab_pipelines
Nddtfjiang <zh...@merico.dev>
---
backend/go.mod | 2 +
backend/go.sum | 4 +
backend/helpers/pluginhelper/api/api_client.go | 47 ++++++++++-
.../api/apihelperabstract/api_client_abstract.go | 2 +
backend/plugins/gitlab/api/blueprint.go | 13 +--
backend/plugins/gitlab/api/connection.go | 31 ++++---
backend/plugins/gitlab/impl/impl.go | 10 ++-
backend/plugins/gitlab/models/connection.go | 92 ++++++++++++++++++--
.../20230210_add_is_detail_required.go | 58 +++++++++++++
.../gitlab/models/migrationscripts/register.go | 1 +
backend/plugins/gitlab/models/mr.go | 4 +-
backend/plugins/gitlab/models/pipeline.go | 5 +-
backend/plugins/gitlab/models/project.go | 3 +-
backend/plugins/gitlab/tasks/account_collector.go | 15 +++-
...nt_collector.go => account_detail_collector.go} | 84 ++++++++++++++-----
...nt_extractor.go => account_detail_extractor.go} | 21 +++--
backend/plugins/gitlab/tasks/account_extractor.go | 9 +-
backend/plugins/gitlab/tasks/api_client.go | 21 +++--
.../plugins/gitlab/tasks/mr_detail_collector.go | 89 ++++++++++++++++++++
.../{mr_extractor.go => mr_detail_extractor.go} | 82 ++----------------
backend/plugins/gitlab/tasks/mr_extractor.go | 39 +++++++--
.../gitlab/tasks/pipeline_detail_collector.go | 98 ++++++++++++++++++++++
...e_extractor.go => pipeline_detail_extractor.go} | 62 ++++----------
backend/plugins/gitlab/tasks/pipeline_extractor.go | 35 +++++---
backend/plugins/gitlab/tasks/shared.go | 25 +++++-
25 files changed, 635 insertions(+), 217 deletions(-)
diff --git a/backend/go.mod b/backend/go.mod
index 6605d9227..774212bb3 100644
--- a/backend/go.mod
+++ b/backend/go.mod
@@ -55,6 +55,7 @@ require (
github.com/cockroachdb/redact v1.1.3 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
+ github.com/dlclark/regexp2 v1.8.0
github.com/emirpasic/gods v1.12.0 // indirect
github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a // indirect
github.com/fsnotify/fsnotify v1.5.1 // indirect
@@ -119,6 +120,7 @@ require (
github.com/ugorji/go/codec v1.2.6 // indirect
github.com/xanzy/ssh-agent v0.3.0 // indirect
go.uber.org/atomic v1.9.0 // indirect
+ golang.org/x/mod v0.8.0
golang.org/x/net v0.1.0 // indirect
golang.org/x/sys v0.1.0 // indirect
golang.org/x/term v0.1.0 // indirect
diff --git a/backend/go.sum b/backend/go.sum
index 5fea80b14..6ee6cae5b 100644
--- a/backend/go.sum
+++ b/backend/go.sum
@@ -131,6 +131,8 @@ github.com/denisenkom/go-mssqldb v0.9.0/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27N
github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4=
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
+github.com/dlclark/regexp2 v1.8.0 h1:rJD5HeGIT/2b5CDk63FVCwZA3qgYElfg+oQK7uH5pfE=
+github.com/dlclark/regexp2 v1.8.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM=
github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg=
@@ -818,6 +820,8 @@ golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.6.0 h1:b9gGHsz9/HhJ3HF5DHQytPpuwocVTChQJK3AvoLRD5I=
+golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8=
+golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
diff --git a/backend/helpers/pluginhelper/api/api_client.go b/backend/helpers/pluginhelper/api/api_client.go
index 96e4bae83..b7bda0bd5 100644
--- a/backend/helpers/pluginhelper/api/api_client.go
+++ b/backend/helpers/pluginhelper/api/api_client.go
@@ -22,6 +22,7 @@ import (
gocontext "context"
"crypto/tls"
"encoding/json"
+ "encoding/xml"
"fmt"
"io"
"net/http"
@@ -29,6 +30,7 @@ import (
"reflect"
"regexp"
"strings"
+ "sync"
"time"
"unicode/utf8"
@@ -45,9 +47,12 @@ var ErrIgnoreAndContinue = errors.Default.New("ignore and continue")
// ApiClient is designed for simple api requests
type ApiClient struct {
- client *http.Client
- endpoint string
- headers map[string]string
+ client *http.Client
+ endpoint string
+ headers map[string]string
+ data map[string]interface{}
+ data_mutex sync.Mutex
+
beforeRequest common.ApiClientBeforeRequest
afterResponse common.ApiClientAfterResponse
ctx gocontext.Context
@@ -67,6 +72,7 @@ func NewApiClientFromConnection(
if err != nil {
return nil, err
}
+
// if connection needs to prepare the ApiClient, i.e. fetch token for future requests
if prepareApiClient, ok := connection.(aha.PrepareApiClient); ok {
err = prepareApiClient.PrepareApiClient(apiClient)
@@ -74,12 +80,14 @@ func NewApiClientFromConnection(
return nil, err
}
}
+
// if connection requires authorization
if authenticator, ok := connection.(aha.ApiAuthenticator); ok {
apiClient.SetBeforeFunction(func(req *http.Request) errors.Error {
return authenticator.SetupAuthentication(req)
})
}
+
return apiClient, nil
}
@@ -160,6 +168,7 @@ func (apiClient *ApiClient) Setup(
apiClient.client = &http.Client{Timeout: timeout}
apiClient.SetEndpoint(endpoint)
apiClient.SetHeaders(headers)
+ apiClient.data = map[string]interface{}{}
}
// SetEndpoint FIXME ...
@@ -182,6 +191,24 @@ func (apiClient *ApiClient) GetTimeout() time.Duration {
return apiClient.client.Timeout
}
+// SetData FIXME ...
+func (apiClient *ApiClient) SetData(name string, data interface{}) {
+ apiClient.data_mutex.Lock()
+ defer apiClient.data_mutex.Unlock()
+
+ apiClient.data[name] = data
+}
+
+// GetData FIXME ...
+func (apiClient *ApiClient) GetData(name string) interface{} {
+ apiClient.data_mutex.Lock()
+ defer apiClient.data_mutex.Unlock()
+
+ data := apiClient.data[name]
+
+ return data
+}
+
// SetHeaders FIXME ...
func (apiClient *ApiClient) SetHeaders(headers map[string]string) {
apiClient.headers = headers
@@ -353,6 +380,20 @@ func UnmarshalResponse(res *http.Response, v interface{}) errors.Error {
return nil
}
+// UnmarshalResponseXML FIXME ...
+func UnmarshalResponseXML(res *http.Response, v interface{}) errors.Error {
+ defer res.Body.Close()
+ resBody, err := io.ReadAll(res.Body)
+ if err != nil {
+ return errors.Default.Wrap(err, fmt.Sprintf("error reading response from %s", res.Request.URL.String()))
+ }
+ err = errors.Convert(xml.Unmarshal(resBody, &v))
+ if err != nil {
+ return errors.Default.Wrap(err, fmt.Sprintf("error decoding XML response from %s: raw response: %s", res.Request.URL.String(), string(resBody)))
+ }
+ return nil
+}
+
// GetURIStringPointer FIXME ...
func GetURIStringPointer(baseUrl string, relativePath string, query url.Values) (*string, errors.Error) {
// If the base URL doesn't end with a slash, and has a relative path attached
diff --git a/backend/helpers/pluginhelper/api/apihelperabstract/api_client_abstract.go b/backend/helpers/pluginhelper/api/apihelperabstract/api_client_abstract.go
index 8399ff448..617652fb8 100644
--- a/backend/helpers/pluginhelper/api/apihelperabstract/api_client_abstract.go
+++ b/backend/helpers/pluginhelper/api/apihelperabstract/api_client_abstract.go
@@ -26,6 +26,8 @@ import (
// ApiClientAbstract defines the functionalities needed by all plugins for Synchronized API Request
type ApiClientAbstract interface {
+ SetData(name string, data interface{})
+ GetData(name string) interface{}
SetHeaders(headers map[string]string)
Get(path string, query url.Values, headers http.Header) (*http.Response, errors.Error)
Post(path string, query url.Values, body interface{}, headers http.Header) (*http.Response, errors.Error)
diff --git a/backend/plugins/gitlab/api/blueprint.go b/backend/plugins/gitlab/api/blueprint.go
index df9a96be3..ffa3fc600 100644
--- a/backend/plugins/gitlab/api/blueprint.go
+++ b/backend/plugins/gitlab/api/blueprint.go
@@ -24,8 +24,6 @@ import (
"io"
"net/http"
"net/url"
- "strings"
- "time"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models/domainlayer/didgen"
@@ -44,21 +42,16 @@ func MakePipelinePlan(subtaskMetas []plugin.SubTaskMeta, connectionId uint64, sc
if err != nil {
return nil, err
}
- token := strings.Split(connection.Token, ",")[0]
- apiClient, err := api.NewApiClient(
+ apiClient, err := api.NewApiClientFromConnection(
context.TODO(),
- connection.Endpoint,
- map[string]string{
- "Authorization": fmt.Sprintf("Bearer %s", token),
- },
- 10*time.Second,
- connection.Proxy,
basicRes,
+ connection,
)
if err != nil {
return nil, err
}
+
plan, err := makePipelinePlan(subtaskMetas, scope, apiClient, connection)
if err != nil {
return nil, err
diff --git a/backend/plugins/gitlab/api/connection.go b/backend/plugins/gitlab/api/connection.go
index c9ca68d21..65867ec8a 100644
--- a/backend/plugins/gitlab/api/connection.go
+++ b/backend/plugins/gitlab/api/connection.go
@@ -25,13 +25,19 @@ import (
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/gitlab/models"
+ "github.com/apache/incubator-devlake/server/api/shared"
)
+type GitlabTestConnResponse struct {
+ shared.ApiBody
+ Connection *models.GitlabConn
+}
+
// @Summary test gitlab connection
// @Description Test gitlab Connection
// @Tags plugins/gitlab
// @Param body body models.GitlabConn true "json body"
-// @Success 200 {object} shared.ApiBody "Success"
+// @Success 200 {object} GitlabTestConnResponse "Success"
// @Failure 400 {string} errcode.Error "Bad Request"
// @Failure 500 {string} errcode.Error "Internal Error"
// @Router /plugins/gitlab/test [POST]
@@ -43,26 +49,17 @@ func TestConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput,
return nil, err
}
- // test connection
- apiClient, err := api.NewApiClientFromConnection(context.TODO(), basicRes, &connection)
+ _, err = api.NewApiClientFromConnection(context.TODO(), basicRes, &connection)
if err != nil {
- return nil, errors.Convert(err)
+ return nil, err
}
- res, err := apiClient.Get("user", nil, nil)
- if err != nil {
- return nil, errors.Convert(err)
- }
- resBody := &models.ApiUserResponse{}
- err = api.UnmarshalResponse(res, resBody)
- if err != nil {
- return nil, errors.Convert(err)
- }
+ body := GitlabTestConnResponse{}
+ body.Success = true
+ body.Message = "success"
+ body.Connection = &connection
- if res.StatusCode != http.StatusOK {
- return nil, errors.HttpStatus(res.StatusCode).New("unexpected status code while testing connection")
- }
- return nil, nil
+ return &plugin.ApiResourceOutput{Body: body, Status: http.StatusOK}, nil
}
// @Summary create gitlab connection
diff --git a/backend/plugins/gitlab/impl/impl.go b/backend/plugins/gitlab/impl/impl.go
index e87f65fbe..ecb399ee9 100644
--- a/backend/plugins/gitlab/impl/impl.go
+++ b/backend/plugins/gitlab/impl/impl.go
@@ -19,6 +19,8 @@ package impl
import (
"fmt"
+ "time"
+
"github.com/apache/incubator-devlake/core/context"
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
@@ -28,7 +30,6 @@ import (
"github.com/apache/incubator-devlake/plugins/gitlab/models"
"github.com/apache/incubator-devlake/plugins/gitlab/models/migrationscripts"
"github.com/apache/incubator-devlake/plugins/gitlab/tasks"
- "time"
)
var _ interface {
@@ -98,17 +99,23 @@ func (p Gitlab) SubTaskMetas() []plugin.SubTaskMeta {
tasks.ExtractApiIssuesMeta,
tasks.CollectApiMergeRequestsMeta,
tasks.ExtractApiMergeRequestsMeta,
+ tasks.CollectApiMergeRequestDetailsMeta,
+ tasks.CollectApiMergeRequestDetailsMeta,
tasks.CollectApiMrNotesMeta,
tasks.ExtractApiMrNotesMeta,
tasks.CollectApiMrCommitsMeta,
tasks.ExtractApiMrCommitsMeta,
tasks.CollectApiPipelinesMeta,
tasks.ExtractApiPipelinesMeta,
+ tasks.CollectApiPipelineDetailsMeta,
+ tasks.ExtractApiPipelineDetailsMeta,
tasks.CollectApiJobsMeta,
tasks.ExtractApiJobsMeta,
tasks.EnrichMergeRequestsMeta,
tasks.CollectAccountsMeta,
tasks.ExtractAccountsMeta,
+ tasks.CollectAccountDetailsMeta,
+ tasks.ExtractAccountDetailsMeta,
tasks.ConvertAccountsMeta,
tasks.ConvertProjectMeta,
tasks.ConvertApiMergeRequestsMeta,
@@ -146,6 +153,7 @@ func (p Gitlab) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]i
if err != nil {
return nil, errors.BadInput.Wrap(err, "connection not found")
}
+
apiClient, err := tasks.NewGitlabApiClient(taskCtx, connection)
if err != nil {
return nil, err
diff --git a/backend/plugins/gitlab/models/connection.go b/backend/plugins/gitlab/models/connection.go
index 31cee6b05..1dcb19863 100644
--- a/backend/plugins/gitlab/models/connection.go
+++ b/backend/plugins/gitlab/models/connection.go
@@ -18,19 +18,96 @@ limitations under the License.
package models
import (
- helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "fmt"
+ "net/http"
+
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/api/apihelperabstract"
)
// GitlabConn holds the essential information to connect to the Gitlab API
type GitlabConn struct {
- helper.RestConnection `mapstructure:",squash"`
- helper.AccessToken `mapstructure:",squash"`
+ api.RestConnection `mapstructure:",squash"`
+ api.AccessToken `mapstructure:",squash"`
+}
+
+const GitlabApiClientData_UserId string = "UserId"
+const GitlabApiClientData_ApiVersion string = "ApiVersion"
+
+// this function is used to rewrite the same function of AccessToken
+func (conn *GitlabConn) SetupAuthentication(request *http.Request) errors.Error {
+ return nil
+}
+
+// PrepareApiClient test api and set the IsPrivateToken,version,UserId and so on.
+func (conn *GitlabConn) PrepareApiClient(apiClient apihelperabstract.ApiClientAbstract) errors.Error {
+ header1 := http.Header{}
+ header1.Set("Authorization", fmt.Sprintf("Bearer %v", conn.Token))
+ // test request for access token
+ userResBody := &ApiUserResponse{}
+ res, err := apiClient.Get("user", nil, header1)
+ if res.StatusCode != http.StatusUnauthorized {
+ if err != nil {
+ return errors.Convert(err)
+ }
+ err = api.UnmarshalResponse(res, userResBody)
+ if err != nil {
+ return errors.Convert(err)
+ }
+
+ if res.StatusCode != http.StatusOK {
+ return errors.HttpStatus(res.StatusCode).New("unexpected status code while testing connection")
+ }
+ apiClient.SetHeaders(map[string]string{
+ "Authorization": fmt.Sprintf("Bearer %v", conn.Token),
+ })
+ } else {
+ header2 := http.Header{}
+ header2.Set("Private-Token", conn.Token)
+ res, err = apiClient.Get("user", nil, header2)
+ if err != nil {
+ return errors.Convert(err)
+ }
+ err = api.UnmarshalResponse(res, userResBody)
+ if err != nil {
+ return errors.Convert(err)
+ }
+
+ if res.StatusCode != http.StatusOK {
+ return errors.HttpStatus(res.StatusCode).New("unexpected status code while testing connection[PrivateToken]")
+ }
+ apiClient.SetHeaders(map[string]string{
+ "Private-Token": conn.Token,
+ })
+ }
+ // get gitlab version
+ versionResBody := &ApiVersionResponse{}
+ res, err = apiClient.Get("version", nil, nil)
+ if err != nil {
+ return errors.Convert(err)
+ }
+
+ err = api.UnmarshalResponse(res, versionResBody)
+ if err != nil {
+ return errors.Convert(err)
+ }
+
+ // add v for semver compare
+ if versionResBody.Version[0] != 'v' {
+ versionResBody.Version = "v" + versionResBody.Version
+ }
+
+ apiClient.SetData(GitlabApiClientData_UserId, userResBody.Id)
+ apiClient.SetData(GitlabApiClientData_ApiVersion, versionResBody.Version)
+
+ return nil
}
// GitlabConnection holds GitlabConn plus ID/Name for database storage
type GitlabConnection struct {
- helper.BaseConnection `mapstructure:",squash"`
- GitlabConn `mapstructure:",squash"`
+ api.BaseConnection `mapstructure:",squash"`
+ GitlabConn `mapstructure:",squash"`
}
// This object conforms to what the frontend currently expects.
@@ -40,6 +117,11 @@ type GitlabResponse struct {
GitlabConnection
}
+type ApiVersionResponse struct {
+ Version string `json:"version"`
+ Revision string `json:"revision"`
+}
+
// Using User because it requires authentication.
type ApiUserResponse struct {
Id int
diff --git a/backend/plugins/gitlab/models/migrationscripts/20230210_add_is_detail_required.go b/backend/plugins/gitlab/models/migrationscripts/20230210_add_is_detail_required.go
new file mode 100644
index 000000000..96222836c
--- /dev/null
+++ b/backend/plugins/gitlab/models/migrationscripts/20230210_add_is_detail_required.go
@@ -0,0 +1,58 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+ "github.com/apache/incubator-devlake/core/context"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/helpers/migrationhelper"
+)
+
+type gitlabMergeRequests20230210 struct {
+ IsDetailRequired bool
+}
+
+func (gitlabMergeRequests20230210) TableName() string {
+ return "_tool_gitlab_merge_requests"
+}
+
+type gitlabPipelines20230210 struct {
+ IsDetailRequired bool
+}
+
+func (gitlabPipelines20230210) TableName() string {
+ return "_tool_gitlab_pipelines"
+}
+
+type addIsDetailRequired20230210 struct{}
+
+func (script *addIsDetailRequired20230210) Up(basicRes context.BasicRes) errors.Error {
+ return migrationhelper.AutoMigrateTables(
+ basicRes,
+ &gitlabMergeRequests20230210{},
+ &gitlabPipelines20230210{},
+ )
+}
+
+func (*addIsDetailRequired20230210) Version() uint64 {
+ return 20230210161031
+}
+
+func (*addIsDetailRequired20230210) Name() string {
+ return "add IsDetailRequired to _tool_gitlab_merge_requests and _tool_gitlab_pipelines"
+}
diff --git a/backend/plugins/gitlab/models/migrationscripts/register.go b/backend/plugins/gitlab/models/migrationscripts/register.go
index 41d42782e..562e48032 100644
--- a/backend/plugins/gitlab/models/migrationscripts/register.go
+++ b/backend/plugins/gitlab/models/migrationscripts/register.go
@@ -31,5 +31,6 @@ func All() []plugin.MigrationScript {
new(fixDurationToFloat8),
new(addTransformationRule20221125),
new(addStdTypeToIssue221230),
+ new(addIsDetailRequired20230210),
}
}
diff --git a/backend/plugins/gitlab/models/mr.go b/backend/plugins/gitlab/models/mr.go
index 1bf1dfbe2..05dff8fd4 100644
--- a/backend/plugins/gitlab/models/mr.go
+++ b/backend/plugins/gitlab/models/mr.go
@@ -18,8 +18,9 @@ limitations under the License.
package models
import (
- "github.com/apache/incubator-devlake/core/models/common"
"time"
+
+ "github.com/apache/incubator-devlake/core/models/common"
)
type GitlabMergeRequest struct {
@@ -34,6 +35,7 @@ type GitlabMergeRequest struct {
WebUrl string `gorm:"type:varchar(255)"`
UserNotesCount int
WorkInProgress bool
+ IsDetailRequired bool
SourceBranch string `gorm:"type:varchar(255)"`
TargetBranch string `gorm:"type:varchar(255)"`
MergeCommitSha string `gorm:"type:varchar(255)"`
diff --git a/backend/plugins/gitlab/models/pipeline.go b/backend/plugins/gitlab/models/pipeline.go
index 825beb4c4..c66e78a24 100644
--- a/backend/plugins/gitlab/models/pipeline.go
+++ b/backend/plugins/gitlab/models/pipeline.go
@@ -18,8 +18,9 @@ limitations under the License.
package models
import (
- "github.com/apache/incubator-devlake/core/models/common"
"time"
+
+ "github.com/apache/incubator-devlake/core/models/common"
)
type GitlabPipeline struct {
@@ -39,6 +40,8 @@ type GitlabPipeline struct {
FinishedAt *time.Time
Coverage string
+ IsDetailRequired bool
+
common.NoPKModel
}
diff --git a/backend/plugins/gitlab/models/project.go b/backend/plugins/gitlab/models/project.go
index 120f2db9d..24db88ce2 100644
--- a/backend/plugins/gitlab/models/project.go
+++ b/backend/plugins/gitlab/models/project.go
@@ -18,8 +18,9 @@ limitations under the License.
package models
import (
- "github.com/apache/incubator-devlake/core/models/common"
"time"
+
+ "github.com/apache/incubator-devlake/core/models/common"
)
type GitlabProject struct {
diff --git a/backend/plugins/gitlab/tasks/account_collector.go b/backend/plugins/gitlab/tasks/account_collector.go
index 4f6f11723..26b26a70d 100644
--- a/backend/plugins/gitlab/tasks/account_collector.go
+++ b/backend/plugins/gitlab/tasks/account_collector.go
@@ -19,11 +19,14 @@ package tasks
import (
"encoding/json"
+ "net/http"
+ "net/url"
+
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
- "net/http"
- "net/url"
+ "github.com/apache/incubator-devlake/plugins/gitlab/models"
+ "golang.org/x/mod/semver"
)
const RAW_USER_TABLE = "gitlab_api_users"
@@ -41,10 +44,16 @@ func CollectAccounts(taskCtx plugin.SubTaskContext) errors.Error {
logger := taskCtx.GetLogger()
logger.Info("collect gitlab users")
+ // it means we can not use /members/all to get the data
+ urlTemplate := "/projects/{{ .Params.ProjectId }}/members/all"
+ if semver.Compare(data.ApiClient.GetData(models.GitlabApiClientData_ApiVersion).(string), "v13.11") < 0 {
+ urlTemplate = "/projects/{{ .Params.ProjectId }}/members/"
+ }
+
collector, err := api.NewApiCollector(api.ApiCollectorArgs{
RawDataSubTaskArgs: *rawDataSubTaskArgs,
ApiClient: data.ApiClient,
- UrlTemplate: "/projects/{{ .Params.ProjectId }}/members/all",
+ UrlTemplate: urlTemplate,
//PageSize: 100,
Query: func(reqData *api.RequestData) (url.Values, errors.Error) {
query := url.Values{}
diff --git a/backend/plugins/gitlab/tasks/account_collector.go b/backend/plugins/gitlab/tasks/account_detail_collector.go
similarity index 50%
copy from backend/plugins/gitlab/tasks/account_collector.go
copy to backend/plugins/gitlab/tasks/account_detail_collector.go
index 4f6f11723..c10d96450 100644
--- a/backend/plugins/gitlab/tasks/account_collector.go
+++ b/backend/plugins/gitlab/tasks/account_detail_collector.go
@@ -18,33 +18,49 @@ limitations under the License.
package tasks
import (
- "encoding/json"
+ "net/url"
+ "reflect"
+
+ "github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
- "net/http"
- "net/url"
+ "github.com/apache/incubator-devlake/plugins/gitlab/models"
+ "golang.org/x/mod/semver"
)
-const RAW_USER_TABLE = "gitlab_api_users"
+const RAW_USER_DETAIL_TABLE = "gitlab_api_user_details"
-var CollectAccountsMeta = plugin.SubTaskMeta{
- Name: "collectAccounts",
- EntryPoint: CollectAccounts,
+var CollectAccountDetailsMeta = plugin.SubTaskMeta{
+ Name: "collectAccountDetails",
+ EntryPoint: CollectAccountDetails,
EnabledByDefault: true,
- Description: "collect gitlab users",
+ Description: "collect gitlab user details",
DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS},
}
-func CollectAccounts(taskCtx plugin.SubTaskContext) errors.Error {
- rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_USER_TABLE)
+func CollectAccountDetails(taskCtx plugin.SubTaskContext) errors.Error {
+
+ rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_USER_DETAIL_TABLE)
logger := taskCtx.GetLogger()
- logger.Info("collect gitlab users")
+ logger.Info("collect gitlab user details")
+
+ if !NeedAccountDetails(data.ApiClient) {
+ logger.Info("Don't need collect gitlab user details,skip")
+ return nil
+ }
+
+ iterator, err := GetAccountsIterator(taskCtx)
+ if err != nil {
+ return err
+ }
+ defer iterator.Close()
collector, err := api.NewApiCollector(api.ApiCollectorArgs{
RawDataSubTaskArgs: *rawDataSubTaskArgs,
ApiClient: data.ApiClient,
- UrlTemplate: "/projects/{{ .Params.ProjectId }}/members/all",
+ UrlTemplate: "/projects/{{ .Params.ProjectId }}/members/{{ .Input.GitlabId }}",
+ Input: iterator,
//PageSize: 100,
Query: func(reqData *api.RequestData) (url.Values, errors.Error) {
query := url.Values{}
@@ -54,14 +70,7 @@ func CollectAccounts(taskCtx plugin.SubTaskContext) errors.Error {
return query, nil
},
- ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) {
- var items []json.RawMessage
- err := api.UnmarshalResponse(res, &items)
- if err != nil {
- return nil, err
- }
- return items, nil
- },
+ ResponseParser: GetOneRawMessageFromResponse,
})
if err != nil {
@@ -71,3 +80,38 @@ func CollectAccounts(taskCtx plugin.SubTaskContext) errors.Error {
return collector.Execute()
}
+
+// checking if we need detail data
+func NeedAccountDetails(apiClient *api.ApiAsyncClient) bool {
+ if apiClient == nil {
+ return false
+ }
+
+ if version, ok := apiClient.GetData(models.GitlabApiClientData_ApiVersion).(string); ok {
+ if semver.Compare(version, "v13.11") < 0 && version != "" {
+ return true
+ }
+ }
+
+ return false
+}
+
+func GetAccountsIterator(taskCtx plugin.SubTaskContext) (*api.DalCursorIterator, errors.Error) {
+ db := taskCtx.GetDal()
+ data := taskCtx.GetData().(*GitlabTaskData)
+ clauses := []dal.Clause{
+ dal.Select("ga.gitlab_id,ga.gitlab_id as iid"),
+ dal.From("_tool_gitlab_accounts ga"),
+ dal.Where(
+ `ga.connection_id = ?`,
+ data.Options.ConnectionId,
+ ),
+ }
+ // construct the input iterator
+ cursor, err := db.Cursor(clauses...)
+ if err != nil {
+ return nil, err
+ }
+
+ return api.NewDalCursorIterator(db, cursor, reflect.TypeOf(GitlabInput{}))
+}
diff --git a/backend/plugins/gitlab/tasks/account_extractor.go b/backend/plugins/gitlab/tasks/account_detail_extractor.go
similarity index 79%
copy from backend/plugins/gitlab/tasks/account_extractor.go
copy to backend/plugins/gitlab/tasks/account_detail_extractor.go
index 38dd497e8..e16616232 100644
--- a/backend/plugins/gitlab/tasks/account_extractor.go
+++ b/backend/plugins/gitlab/tasks/account_detail_extractor.go
@@ -19,22 +19,31 @@ package tasks
import (
"encoding/json"
+
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/gitlab/models"
)
-var ExtractAccountsMeta = plugin.SubTaskMeta{
- Name: "extractAccounts",
- EntryPoint: ExtractAccounts,
+var ExtractAccountDetailsMeta = plugin.SubTaskMeta{
+ Name: "extractAccountDetails",
+ EntryPoint: ExtractAccountDetails,
EnabledByDefault: true,
- Description: "Extract raw workspace data into tool layer table _tool_gitlab_accounts",
+ Description: "Extract detail raw workspace data into tool layer table _tool_gitlab_accounts",
DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS},
}
-func ExtractAccounts(taskCtx plugin.SubTaskContext) errors.Error {
- rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_USER_TABLE)
+func ExtractAccountDetails(taskCtx plugin.SubTaskContext) errors.Error {
+ rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_USER_DETAIL_TABLE)
+
+ logger := taskCtx.GetLogger()
+ logger.Info("Extract gitlab user details")
+
+ if !NeedAccountDetails(data.ApiClient) {
+ logger.Info("Don't need Extract gitlab user details,skip")
+ return nil
+ }
extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{
RawDataSubTaskArgs: *rawDataSubTaskArgs,
diff --git a/backend/plugins/gitlab/tasks/account_extractor.go b/backend/plugins/gitlab/tasks/account_extractor.go
index 38dd497e8..56df7fdbb 100644
--- a/backend/plugins/gitlab/tasks/account_extractor.go
+++ b/backend/plugins/gitlab/tasks/account_extractor.go
@@ -19,6 +19,7 @@ package tasks
import (
"encoding/json"
+
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
@@ -57,6 +58,7 @@ func ExtractAccounts(taskCtx plugin.SubTaskContext) errors.Error {
WebUrl: userRes.WebUrl,
}
results = append(results, GitlabAccount)
+
return results, nil
},
})
@@ -65,5 +67,10 @@ func ExtractAccounts(taskCtx plugin.SubTaskContext) errors.Error {
return err
}
- return extractor.Execute()
+ err = extractor.Execute()
+ if err != nil {
+ return err
+ }
+
+ return nil
}
diff --git a/backend/plugins/gitlab/tasks/api_client.go b/backend/plugins/gitlab/tasks/api_client.go
index df4517ba0..5010f8733 100644
--- a/backend/plugins/gitlab/tasks/api_client.go
+++ b/backend/plugins/gitlab/tasks/api_client.go
@@ -28,13 +28,11 @@ import (
"github.com/apache/incubator-devlake/plugins/gitlab/models"
)
-func NewGitlabApiClient(taskCtx plugin.TaskContext, connection *models.GitlabConnection) (*api.ApiAsyncClient, errors.Error) {
- // create synchronize api client so we can calculate api rate limit dynamically
- apiClient, err := api.NewApiClientFromConnection(taskCtx.GetContext(), taskCtx, connection)
- if err != nil {
- return nil, err
- }
-
+func CreateGitlabAsyncApiClient(
+ taskCtx plugin.TaskContext,
+ apiClient *api.ApiClient,
+ connection *models.GitlabConnection,
+) (*api.ApiAsyncClient, errors.Error) {
// create rate limit calculator
rateLimiter := &api.ApiRateLimitCalculator{
UserRateLimitPerHour: connection.RateLimitPerHour,
@@ -62,6 +60,15 @@ func NewGitlabApiClient(taskCtx plugin.TaskContext, connection *models.GitlabCon
return asyncApiClient, nil
}
+func NewGitlabApiClient(taskCtx plugin.TaskContext, connection *models.GitlabConnection) (*api.ApiAsyncClient, errors.Error) {
+ apiClient, err := api.NewApiClientFromConnection(taskCtx.GetContext(), taskCtx, connection)
+ if err != nil {
+ return nil, err
+ }
+
+ return CreateGitlabAsyncApiClient(taskCtx, apiClient, connection)
+}
+
func ignoreHTTPStatus403(res *http.Response) errors.Error {
if res.StatusCode == http.StatusForbidden {
return api.ErrIgnoreAndContinue
diff --git a/backend/plugins/gitlab/tasks/mr_detail_collector.go b/backend/plugins/gitlab/tasks/mr_detail_collector.go
new file mode 100644
index 000000000..3d3ff9ee9
--- /dev/null
+++ b/backend/plugins/gitlab/tasks/mr_detail_collector.go
@@ -0,0 +1,89 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "reflect"
+
+ "github.com/apache/incubator-devlake/core/dal"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
+ helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+)
+
+const RAW_MERGE_REQUEST_DETAIL_TABLE = "gitlab_api_merge_request_details"
+
+var CollectApiMergeRequestDetailsMeta = plugin.SubTaskMeta{
+ Name: "collectApiMergeRequestDetails",
+ EntryPoint: CollectApiMergeRequestDetails,
+ EnabledByDefault: true,
+ Description: "Collect merge request Details data from gitlab api",
+ DomainTypes: []string{plugin.DOMAIN_TYPE_CODE_REVIEW},
+}
+
+func CollectApiMergeRequestDetails(taskCtx plugin.SubTaskContext) errors.Error {
+ rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_MERGE_REQUEST_DETAIL_TABLE)
+ collectorWithState, err := helper.NewApiCollectorWithState(*rawDataSubTaskArgs, data.CreatedDateAfter)
+ if err != nil {
+ return err
+ }
+
+ iterator, err := GetMergeRequestDetailsIterator(taskCtx, collectorWithState)
+ if err != nil {
+ return err
+ }
+
+ err = collectorWithState.InitCollector(helper.ApiCollectorArgs{
+ ApiClient: data.ApiClient,
+ PageSize: 100,
+ Incremental: false,
+ Input: iterator,
+ UrlTemplate: "projects/{{ .Params.ProjectId }}/merge_requests/{{ .Input.Iid }}",
+ Query: GetQuery,
+ GetTotalPages: GetTotalPagesFromResponse,
+ ResponseParser: GetOneRawMessageFromResponse,
+ })
+ if err != nil {
+ return err
+ }
+
+ return collectorWithState.Execute()
+}
+
+func GetMergeRequestDetailsIterator(taskCtx plugin.SubTaskContext, collectorWithState *helper.ApiCollectorStateManager) (*helper.DalCursorIterator, errors.Error) {
+ db := taskCtx.GetDal()
+ data := taskCtx.GetData().(*GitlabTaskData)
+ clauses := []dal.Clause{
+ dal.Select("gmr.gitlab_id, gmr.iid"),
+ dal.From("_tool_gitlab_merge_requests gmr"),
+ dal.Where(
+ `gmr.project_id = ? and gmr.connection_id = ? and gmr.is_detail_required = ?`,
+ data.Options.ProjectId, data.Options.ConnectionId, true,
+ ),
+ }
+ if collectorWithState.CreatedDateAfter != nil {
+ clauses = append(clauses, dal.Where("gitlab_created_at > ?", *collectorWithState.CreatedDateAfter))
+ }
+ // construct the input iterator
+ cursor, err := db.Cursor(clauses...)
+ if err != nil {
+ return nil, err
+ }
+
+ return helper.NewDalCursorIterator(db, cursor, reflect.TypeOf(GitlabInput{}))
+}
diff --git a/backend/plugins/gitlab/tasks/mr_extractor.go b/backend/plugins/gitlab/tasks/mr_detail_extractor.go
similarity index 58%
copy from backend/plugins/gitlab/tasks/mr_extractor.go
copy to backend/plugins/gitlab/tasks/mr_detail_extractor.go
index 07dd6c004..3808b6473 100644
--- a/backend/plugins/gitlab/tasks/mr_extractor.go
+++ b/backend/plugins/gitlab/tasks/mr_detail_extractor.go
@@ -19,63 +19,24 @@ package tasks
import (
"encoding/json"
+ "regexp"
+
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/gitlab/models"
- "regexp"
)
-type MergeRequestRes struct {
- GitlabId int `json:"id"`
- Iid int
- ProjectId int `json:"project_id"`
- SourceProjectId int `json:"source_project_id"`
- TargetProjectId int `json:"target_project_id"`
- State string
- Title string
- Description string
- WebUrl string `json:"web_url"`
- UserNotesCount int `json:"user_notes_count"`
- WorkInProgress bool `json:"work_in_progress"`
- SourceBranch string `json:"source_branch"`
- TargetBranch string `json:"target_branch"`
- GitlabCreatedAt api.Iso8601Time `json:"created_at"`
- MergedAt *api.Iso8601Time `json:"merged_at"`
- ClosedAt *api.Iso8601Time `json:"closed_at"`
- MergeCommitSha string `json:"merge_commit_sha"`
- MergedBy struct {
- Username string `json:"username"`
- } `json:"merged_by"`
- Author struct {
- Id int `json:"id"`
- Username string `json:"username"`
- }
- Reviewers []Reviewer
- FirstCommentTime api.Iso8601Time
- Labels []string `json:"labels"`
-}
-
-type Reviewer struct {
- GitlabId int `json:"id"`
- MergeRequestId int
- Name string
- Username string
- State string
- AvatarUrl string `json:"avatar_url"`
- WebUrl string `json:"web_url"`
-}
-
-var ExtractApiMergeRequestsMeta = plugin.SubTaskMeta{
- Name: "extractApiMergeRequests",
- EntryPoint: ExtractApiMergeRequests,
+var ExtractApiMergeRequestDetailsMeta = plugin.SubTaskMeta{
+ Name: "extractApiMergeRequestDetails",
+ EntryPoint: ExtractApiMergeRequestDetails,
EnabledByDefault: true,
- Description: "Extract raw merge requests data into tool layer table GitlabMergeRequest and GitlabReviewer",
+ Description: "Extract raw merge request Details data into tool layer table GitlabMergeRequest and GitlabReviewer",
DomainTypes: []string{plugin.DOMAIN_TYPE_CODE_REVIEW},
}
-func ExtractApiMergeRequests(taskCtx plugin.SubTaskContext) errors.Error {
- rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_MERGE_REQUEST_TABLE)
+func ExtractApiMergeRequestDetails(taskCtx plugin.SubTaskContext) errors.Error {
+ rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_MERGE_REQUEST_DETAIL_TABLE)
config := data.Options.GitlabTransformationRule
var labelTypeRegex *regexp.Regexp
var labelComponentRegex *regexp.Regexp
@@ -109,6 +70,7 @@ func ExtractApiMergeRequests(taskCtx plugin.SubTaskContext) errors.Error {
}
results := make([]interface{}, 0, len(mr.Reviewers)+1)
gitlabMergeRequest.ConnectionId = data.Options.ConnectionId
+ gitlabMergeRequest.IsDetailRequired = true
results = append(results, gitlabMergeRequest)
for _, label := range mr.Labels {
results = append(results, &models.GitlabMrLabel{
@@ -157,29 +119,3 @@ func ExtractApiMergeRequests(taskCtx plugin.SubTaskContext) errors.Error {
return extractor.Execute()
}
-
-func convertMergeRequest(mr *MergeRequestRes) (*models.GitlabMergeRequest, errors.Error) {
- gitlabMergeRequest := &models.GitlabMergeRequest{
- GitlabId: mr.GitlabId,
- Iid: mr.Iid,
- ProjectId: mr.ProjectId,
- SourceProjectId: mr.SourceProjectId,
- TargetProjectId: mr.TargetProjectId,
- State: mr.State,
- Title: mr.Title,
- Description: mr.Description,
- WebUrl: mr.WebUrl,
- UserNotesCount: mr.UserNotesCount,
- WorkInProgress: mr.WorkInProgress,
- SourceBranch: mr.SourceBranch,
- TargetBranch: mr.TargetBranch,
- MergeCommitSha: mr.MergeCommitSha,
- MergedAt: api.Iso8601TimeToTime(mr.MergedAt),
- GitlabCreatedAt: mr.GitlabCreatedAt.ToTime(),
- ClosedAt: api.Iso8601TimeToTime(mr.ClosedAt),
- MergedByUsername: mr.MergedBy.Username,
- AuthorUsername: mr.Author.Username,
- AuthorUserId: mr.Author.Id,
- }
- return gitlabMergeRequest, nil
-}
diff --git a/backend/plugins/gitlab/tasks/mr_extractor.go b/backend/plugins/gitlab/tasks/mr_extractor.go
index 07dd6c004..13d07af30 100644
--- a/backend/plugins/gitlab/tasks/mr_extractor.go
+++ b/backend/plugins/gitlab/tasks/mr_extractor.go
@@ -19,11 +19,13 @@ package tasks
import (
"encoding/json"
+ "regexp"
+ "strings"
+
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/gitlab/models"
- "regexp"
)
type MergeRequestRes struct {
@@ -80,24 +82,28 @@ func ExtractApiMergeRequests(taskCtx plugin.SubTaskContext) errors.Error {
var labelTypeRegex *regexp.Regexp
var labelComponentRegex *regexp.Regexp
var prType = config.PrType
- var err error
+
+ var err1 error
if len(prType) > 0 {
- labelTypeRegex, err = regexp.Compile(prType)
- if err != nil {
- return errors.Default.Wrap(err, "regexp Compile prType failed")
+ labelTypeRegex, err1 = regexp.Compile(prType)
+ if err1 != nil {
+ return errors.Default.Wrap(err1, "regexp Compile prType failed")
}
}
var prComponent = config.PrComponent
if len(prComponent) > 0 {
- labelComponentRegex, err = regexp.Compile(prComponent)
- if err != nil {
- return errors.Default.Wrap(err, "regexp Compile prComponent failed")
+ labelComponentRegex, err1 = regexp.Compile(prComponent)
+ if err1 != nil {
+ return errors.Default.Wrap(err1, "regexp Compile prComponent failed")
}
}
+
extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{
RawDataSubTaskArgs: *rawDataSubTaskArgs,
Extract: func(row *api.RawData) ([]interface{}, errors.Error) {
+
mr := &MergeRequestRes{}
+ s := string(row.Data)
err := errors.Convert(json.Unmarshal(row.Data, mr))
if err != nil {
return nil, err
@@ -107,6 +113,15 @@ func ExtractApiMergeRequests(taskCtx plugin.SubTaskContext) errors.Error {
if err != nil {
return nil, err
}
+
+ // if we can not find merged_at and closed_at info in the detail
+ // we need get detail for gitlab v11
+ if !strings.Contains(s, "\"merged_at\":") {
+ if !strings.Contains(s, "\"closed_at\":") {
+ gitlabMergeRequest.IsDetailRequired = true
+ }
+ }
+
results := make([]interface{}, 0, len(mr.Reviewers)+1)
gitlabMergeRequest.ConnectionId = data.Options.ConnectionId
results = append(results, gitlabMergeRequest)
@@ -155,7 +170,12 @@ func ExtractApiMergeRequests(taskCtx plugin.SubTaskContext) errors.Error {
return errors.Convert(err)
}
- return extractor.Execute()
+ err = extractor.Execute()
+ if err != nil {
+ return err
+ }
+
+ return nil
}
func convertMergeRequest(mr *MergeRequestRes) (*models.GitlabMergeRequest, errors.Error) {
@@ -171,6 +191,7 @@ func convertMergeRequest(mr *MergeRequestRes) (*models.GitlabMergeRequest, error
WebUrl: mr.WebUrl,
UserNotesCount: mr.UserNotesCount,
WorkInProgress: mr.WorkInProgress,
+ IsDetailRequired: false,
SourceBranch: mr.SourceBranch,
TargetBranch: mr.TargetBranch,
MergeCommitSha: mr.MergeCommitSha,
diff --git a/backend/plugins/gitlab/tasks/pipeline_detail_collector.go b/backend/plugins/gitlab/tasks/pipeline_detail_collector.go
new file mode 100644
index 000000000..69977ae3f
--- /dev/null
+++ b/backend/plugins/gitlab/tasks/pipeline_detail_collector.go
@@ -0,0 +1,98 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "net/url"
+ "reflect"
+
+ "github.com/apache/incubator-devlake/core/dal"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
+ helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+)
+
+const RAW_PIPELINE_DETAILS_TABLE = "gitlab_api_pipeline_details"
+
+var CollectApiPipelineDetailsMeta = plugin.SubTaskMeta{
+ Name: "collectApiPipelineDetails",
+ EntryPoint: CollectApiPipelineDetails,
+ EnabledByDefault: true,
+ Description: "Collect pipeline details data from gitlab api",
+ DomainTypes: []string{plugin.DOMAIN_TYPE_CICD},
+}
+
+func CollectApiPipelineDetails(taskCtx plugin.SubTaskContext) errors.Error {
+ rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_PIPELINE_DETAILS_TABLE)
+ collectorWithState, err := helper.NewApiCollectorWithState(*rawDataSubTaskArgs, data.CreatedDateAfter)
+ if err != nil {
+ return err
+ }
+
+ incremental := collectorWithState.IsIncremental()
+
+ iterator, err := GetPipelinesIterator(taskCtx, collectorWithState)
+ if err != nil {
+ return err
+ }
+ defer iterator.Close()
+
+ err = collectorWithState.InitCollector(helper.ApiCollectorArgs{
+ RawDataSubTaskArgs: *rawDataSubTaskArgs,
+ ApiClient: data.ApiClient,
+ PageSize: 100,
+ Input: iterator,
+ Incremental: incremental,
+ UrlTemplate: "projects/{{ .Params.ProjectId }}/pipelines/{{ .Input.GitlabId }}",
+ Query: func(reqData *helper.RequestData) (url.Values, errors.Error) {
+ query := url.Values{}
+ query.Set("with_stats", "true")
+ return query, nil
+ },
+ ResponseParser: GetOneRawMessageFromResponse,
+ AfterResponse: ignoreHTTPStatus403, // ignore 403 for CI/CD disable
+ })
+ if err != nil {
+ return err
+ }
+
+ return collectorWithState.Execute()
+}
+
+func GetPipelinesIterator(taskCtx plugin.SubTaskContext, collectorWithState *helper.ApiCollectorStateManager) (*helper.DalCursorIterator, errors.Error) {
+ db := taskCtx.GetDal()
+ data := taskCtx.GetData().(*GitlabTaskData)
+ clauses := []dal.Clause{
+ dal.Select("gp.gitlab_id,gp.gitlab_id as iid"),
+ dal.From("_tool_gitlab_pipelines gp"),
+ dal.Where(
+ `gp.project_id = ? and gp.connection_id = ? and gp.is_detail_required = ?`,
+ data.Options.ProjectId, data.Options.ConnectionId, true,
+ ),
+ }
+ if collectorWithState.CreatedDateAfter != nil {
+ clauses = append(clauses, dal.Where("gitlab_created_at > ?", *collectorWithState.CreatedDateAfter))
+ }
+ // construct the input iterator
+ cursor, err := db.Cursor(clauses...)
+ if err != nil {
+ return nil, err
+ }
+
+ return helper.NewDalCursorIterator(db, cursor, reflect.TypeOf(GitlabInput{}))
+}
diff --git a/backend/plugins/gitlab/tasks/pipeline_extractor.go b/backend/plugins/gitlab/tasks/pipeline_detail_extractor.go
similarity index 60%
copy from backend/plugins/gitlab/tasks/pipeline_extractor.go
copy to backend/plugins/gitlab/tasks/pipeline_detail_extractor.go
index 3aa47a80e..fe13bf69b 100644
--- a/backend/plugins/gitlab/tasks/pipeline_extractor.go
+++ b/backend/plugins/gitlab/tasks/pipeline_detail_extractor.go
@@ -19,50 +19,23 @@ package tasks
import (
"encoding/json"
+
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/gitlab/models"
)
-type ApiDetailedStatus struct {
- Icon string
- Text string
- Label string
- Group string
- Tooltip string
- HasDetails bool `json:"has_details"`
- DetailsPath string `json:"details_path"`
- Favicon string
-}
-
-type ApiPipeline struct {
- Id int `json:"id"`
- Ref string
- Sha string
- Status string
- Tag bool
- Duration int
- WebUrl string `json:"web_url"`
-
- CreatedAt *api.Iso8601Time `json:"created_at"`
- UpdatedAt *api.Iso8601Time `json:"updated_at"`
- StartedAt *api.Iso8601Time `json:"started_at"`
- FinishedAt *api.Iso8601Time `json:"finished_at"`
-
- ApiDetailedStatus
-}
-
-var ExtractApiPipelinesMeta = plugin.SubTaskMeta{
- Name: "extractApiPipelines",
- EntryPoint: ExtractApiPipelines,
+var ExtractApiPipelineDetailsMeta = plugin.SubTaskMeta{
+ Name: "extractApiPipelineDetails",
+ EntryPoint: ExtractApiPipelineDetails,
EnabledByDefault: true,
- Description: "Extract raw pipelines data into tool layer table GitlabPipeline",
+ Description: "Extract raw pipeline details data into tool layer table GitlabPipeline",
DomainTypes: []string{plugin.DOMAIN_TYPE_CICD},
}
-func ExtractApiPipelines(taskCtx plugin.SubTaskContext) errors.Error {
- rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_PIPELINE_TABLE)
+func ExtractApiPipelineDetails(taskCtx plugin.SubTaskContext) errors.Error {
+ rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_PIPELINE_DETAILS_TABLE)
extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{
RawDataSubTaskArgs: *rawDataSubTaskArgs,
@@ -78,16 +51,17 @@ func ExtractApiPipelines(taskCtx plugin.SubTaskContext) errors.Error {
gitlabApiPipeline.Duration = int(gitlabApiPipeline.UpdatedAt.ToTime().Sub(gitlabApiPipeline.CreatedAt.ToTime()).Seconds())
}
gitlabPipeline := &models.GitlabPipeline{
- GitlabId: gitlabApiPipeline.Id,
- ProjectId: data.Options.ProjectId,
- WebUrl: gitlabApiPipeline.WebUrl,
- Status: gitlabApiPipeline.Status,
- GitlabCreatedAt: api.Iso8601TimeToTime(gitlabApiPipeline.CreatedAt),
- GitlabUpdatedAt: api.Iso8601TimeToTime(gitlabApiPipeline.UpdatedAt),
- StartedAt: api.Iso8601TimeToTime(gitlabApiPipeline.StartedAt),
- FinishedAt: api.Iso8601TimeToTime(gitlabApiPipeline.FinishedAt),
- Duration: gitlabApiPipeline.Duration,
- ConnectionId: data.Options.ConnectionId,
+ GitlabId: gitlabApiPipeline.Id,
+ ProjectId: data.Options.ProjectId,
+ WebUrl: gitlabApiPipeline.WebUrl,
+ Status: gitlabApiPipeline.Status,
+ GitlabCreatedAt: api.Iso8601TimeToTime(gitlabApiPipeline.CreatedAt),
+ GitlabUpdatedAt: api.Iso8601TimeToTime(gitlabApiPipeline.UpdatedAt),
+ StartedAt: api.Iso8601TimeToTime(gitlabApiPipeline.StartedAt),
+ FinishedAt: api.Iso8601TimeToTime(gitlabApiPipeline.FinishedAt),
+ Duration: gitlabApiPipeline.Duration,
+ ConnectionId: data.Options.ConnectionId,
+ IsDetailRequired: true,
}
if err != nil {
return nil, err
diff --git a/backend/plugins/gitlab/tasks/pipeline_extractor.go b/backend/plugins/gitlab/tasks/pipeline_extractor.go
index 3aa47a80e..6b1ad5875 100644
--- a/backend/plugins/gitlab/tasks/pipeline_extractor.go
+++ b/backend/plugins/gitlab/tasks/pipeline_extractor.go
@@ -19,6 +19,7 @@ package tasks
import (
"encoding/json"
+
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
@@ -77,20 +78,23 @@ func ExtractApiPipelines(taskCtx plugin.SubTaskContext) errors.Error {
if gitlabApiPipeline.UpdatedAt != nil && gitlabApiPipeline.CreatedAt != nil {
gitlabApiPipeline.Duration = int(gitlabApiPipeline.UpdatedAt.ToTime().Sub(gitlabApiPipeline.CreatedAt.ToTime()).Seconds())
}
+
gitlabPipeline := &models.GitlabPipeline{
- GitlabId: gitlabApiPipeline.Id,
- ProjectId: data.Options.ProjectId,
- WebUrl: gitlabApiPipeline.WebUrl,
- Status: gitlabApiPipeline.Status,
- GitlabCreatedAt: api.Iso8601TimeToTime(gitlabApiPipeline.CreatedAt),
- GitlabUpdatedAt: api.Iso8601TimeToTime(gitlabApiPipeline.UpdatedAt),
- StartedAt: api.Iso8601TimeToTime(gitlabApiPipeline.StartedAt),
- FinishedAt: api.Iso8601TimeToTime(gitlabApiPipeline.FinishedAt),
- Duration: gitlabApiPipeline.Duration,
- ConnectionId: data.Options.ConnectionId,
+ GitlabId: gitlabApiPipeline.Id,
+ ProjectId: data.Options.ProjectId,
+ WebUrl: gitlabApiPipeline.WebUrl,
+ Status: gitlabApiPipeline.Status,
+ GitlabCreatedAt: api.Iso8601TimeToTime(gitlabApiPipeline.CreatedAt),
+ GitlabUpdatedAt: api.Iso8601TimeToTime(gitlabApiPipeline.UpdatedAt),
+ StartedAt: api.Iso8601TimeToTime(gitlabApiPipeline.StartedAt),
+ FinishedAt: api.Iso8601TimeToTime(gitlabApiPipeline.FinishedAt),
+ Duration: gitlabApiPipeline.Duration,
+ ConnectionId: data.Options.ConnectionId,
+ IsDetailRequired: false,
}
- if err != nil {
- return nil, err
+
+ if gitlabApiPipeline.CreatedAt == nil && gitlabApiPipeline.UpdatedAt == nil {
+ gitlabPipeline.IsDetailRequired = true
}
pipelineProject := &models.GitlabPipelineProject{
@@ -112,5 +116,10 @@ func ExtractApiPipelines(taskCtx plugin.SubTaskContext) errors.Error {
return err
}
- return extractor.Execute()
+ err = extractor.Execute()
+ if err != nil {
+ return err
+ }
+
+ return nil
}
diff --git a/backend/plugins/gitlab/tasks/shared.go b/backend/plugins/gitlab/tasks/shared.go
index 0088a94e1..0d94e3e1a 100644
--- a/backend/plugins/gitlab/tasks/shared.go
+++ b/backend/plugins/gitlab/tasks/shared.go
@@ -20,8 +20,6 @@ package tasks
import (
"encoding/json"
"fmt"
- "github.com/apache/incubator-devlake/core/dal"
- "github.com/apache/incubator-devlake/core/errors"
"io"
"net/http"
"net/url"
@@ -29,6 +27,9 @@ import (
"strconv"
"time"
+ "github.com/apache/incubator-devlake/core/dal"
+ "github.com/apache/incubator-devlake/core/errors"
+
"github.com/apache/incubator-devlake/core/plugin"
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
)
@@ -75,6 +76,26 @@ func GetRawMessageFromResponse(res *http.Response) ([]json.RawMessage, errors.Er
return rawMessages, nil
}
+func GetOneRawMessageFromResponse(res *http.Response) ([]json.RawMessage, errors.Error) {
+ rawMessage := json.RawMessage{}
+
+ if res == nil {
+ return nil, errors.Default.New("res is nil")
+ }
+ defer res.Body.Close()
+ resBody, err := io.ReadAll(res.Body)
+ if err != nil {
+ return nil, errors.Default.Wrap(err, fmt.Sprintf("error reading response from %s", res.Request.URL.String()))
+ }
+
+ err = errors.Convert(json.Unmarshal(resBody, &rawMessage))
+ if err != nil {
+ return nil, errors.Default.Wrap(err, fmt.Sprintf("error decoding response from %s. raw response was: %s", res.Request.URL.String(), string(resBody)))
+ }
+
+ return []json.RawMessage{rawMessage}, nil
+}
+
func GetRawMessageCreatedAtAfter(createDateAfter *time.Time) func(res *http.Response) ([]json.RawMessage, errors.Error) {
type ApiModel struct {
CreatedAt *helper.Iso8601Time `json:"created_at"`