You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by zh...@apache.org on 2022/06/14 11:21:11 UTC
[incubator-devlake] branch main updated: refactor(gitlab): multi conn (#2147)
This is an automated email from the ASF dual-hosted git repository.
zhangliang2022 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 04e4a01d refactor(gitlab): multi conn (#2147)
04e4a01d is described below
commit 04e4a01d94c13a98b240628205f3ce3123f2a3ab
Author: Warren Chen <yi...@merico.dev>
AuthorDate: Tue Jun 14 19:21:07 2022 +0800
refactor(gitlab): multi conn (#2147)
---
plugins/gitlab/api/connection.go | 97 ++++++++++++----------
.../archived/tag.go => api/init.go} | 28 ++++---
plugins/gitlab/impl/impl.go | 11 +--
plugins/gitlab/models/commit.go | 1 +
plugins/gitlab/models/connection.go | 32 +++++--
plugins/gitlab/models/issue.go | 1 +
plugins/gitlab/models/issue_label.go | 5 +-
.../models/migrationscripts/archived/commit.go | 4 +-
.../models/migrationscripts/archived/connection.go | 72 ++++++++++++++++
.../issue.go} | 42 +---------
.../{ => migrationscripts/archived}/issue_label.go | 13 ++-
.../archived/{merge_request.go => mr.go} | 12 +--
.../{merge_request_comment.go => mr_comment.go} | 5 +-
.../{merge_request_commit.go => mr_commit.go} | 5 +-
.../archived/{merge_request_note.go => mr_note.go} | 7 +-
.../models/migrationscripts/archived/pipeline.go | 5 +-
.../models/migrationscripts/archived/project.go | 5 +-
.../{project_commits.go => project_commit.go} | 6 +-
.../models/migrationscripts/archived/reviewer.go | 6 +-
.../gitlab/models/migrationscripts/archived/tag.go | 6 +-
.../models/migrationscripts/archived/user.go | 9 +-
.../gitlab/models/migrationscripts/init_schema.go | 76 ++++++++++++++++-
.../migrationscripts/updateSchemas20220510.go | 48 -----------
plugins/gitlab/models/mr.go | 7 +-
plugins/gitlab/models/mr_comment.go | 2 +
plugins/gitlab/models/mr_commit.go | 1 +
plugins/gitlab/models/mr_note.go | 2 +
plugins/gitlab/models/pipeline.go | 2 +
plugins/gitlab/models/project.go | 2 +
plugins/gitlab/models/project_commit.go | 2 +
plugins/gitlab/models/reviewer.go | 2 +
plugins/gitlab/models/tag.go | 2 +
plugins/gitlab/models/user.go | 5 +-
33 files changed, 316 insertions(+), 207 deletions(-)
diff --git a/plugins/gitlab/api/connection.go b/plugins/gitlab/api/connection.go
index fa87719c..db6b569d 100644
--- a/plugins/gitlab/api/connection.go
+++ b/plugins/gitlab/api/connection.go
@@ -19,23 +19,17 @@ package api
import (
"fmt"
- "github.com/apache/incubator-devlake/config"
- "github.com/apache/incubator-devlake/plugins/gitlab/models"
"net/http"
"time"
"github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/gitlab/models"
"github.com/apache/incubator-devlake/plugins/helper"
- "github.com/go-playground/validator/v10"
"github.com/mitchellh/mapstructure"
)
-var vld = validator.New()
-
-/*
-POST /plugins/gitlab/test
-*/
func TestConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) {
+
// decode
var err error
var connection models.TestConnectionRequest
@@ -48,11 +42,12 @@ func TestConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput, erro
if err != nil {
return nil, err
}
+
// test connection
apiClient, err := helper.NewApiClient(
connection.Endpoint,
map[string]string{
- "Authorization": fmt.Sprintf("Bearer %v", connection.Auth),
+ "Authorization": fmt.Sprintf("Bearer %v", connection.Token),
},
3*time.Second,
connection.Proxy,
@@ -61,6 +56,7 @@ func TestConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput, erro
if err != nil {
return nil, err
}
+
res, err := apiClient.Get("user", nil, nil)
if err != nil {
return nil, err
@@ -78,68 +74,77 @@ func TestConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput, erro
}
/*
-PATCH /plugins/gitlab/connections/:connectionId
+POST /plugins/gitlab/connections
+{
+ "name": "gitlab data connection name",
+ "endpoint": "gitlab api endpoint, i.e. https://gitlab.com/api/v4/",
+ "token": "gitlab api access token"
+}
*/
-func PatchConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) {
- v := config.GetConfig()
+func PostConnections(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) {
+ // update from request and save to database
connection := &models.GitlabConnection{}
- err := helper.EncodeStruct(v, connection, "env")
+ err := connectionHelper.Create(connection, input)
if err != nil {
return nil, err
}
- // update from request and save to .env
- err = helper.DecodeStruct(v, connection, input.Body, "env")
+ return &core.ApiResourceOutput{Body: connection, Status: http.StatusOK}, nil
+}
+
+/*
+PATCH /plugins/gitlab/connections/:connectionId
+{
+ "name": "gitlab data connection name",
+ "endpoint": "gitlab api endpoint, i.e. https://gitlab.com/api/v4/",
+ "token": "gitlab api access token"
+}
+*/
+func PatchConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) {
+ connection := &models.GitlabConnection{}
+ err := connectionHelper.Patch(connection, input)
if err != nil {
return nil, err
}
- err = config.WriteConfig(v)
+ return &core.ApiResourceOutput{Body: connection}, nil
+}
+
+/*
+DELETE /plugins/gitlab/connections/:connectionId
+*/
+func DeleteConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) {
+ connection := &models.GitlabConnection{}
+ err := connectionHelper.First(connection, input.Params)
if err != nil {
return nil, err
}
- response := models.GitlabResponse{
- GitlabConnection: *connection,
- Name: "Gitlab",
- ID: 1,
- }
- return &core.ApiResourceOutput{Body: response, Status: http.StatusOK}, nil
+ err = connectionHelper.Delete(connection)
+ return &core.ApiResourceOutput{Body: connection}, err
}
/*
GET /plugins/gitlab/connections
*/
func ListConnections(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) {
- // RETURN ONLY 1 SOURCE (FROM ENV) until multi-connection is developed.
- v := config.GetConfig()
- connection := &models.GitlabConnection{}
-
- err := helper.EncodeStruct(v, connection, "env")
+ var connections []models.GitlabConnection
+ err := connectionHelper.List(&connections)
if err != nil {
return nil, err
}
- response := models.GitlabResponse{
- GitlabConnection: *connection,
- Name: "Gitlab",
- ID: 1,
- }
-
- return &core.ApiResourceOutput{Body: []models.GitlabResponse{response}}, nil
+ return &core.ApiResourceOutput{Body: connections, Status: http.StatusOK}, nil
}
/*
GET /plugins/gitlab/connections/:connectionId
+
+
+{
+ "name": "gitlab data connection name",
+ "endpoint": "gitlab api endpoint, i.e. https://gitlab.com/api/v4/",
+ "token": "gitlab api access token"
+}
*/
func GetConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) {
- // RETURN ONLY 1 SOURCE FROM ENV (Ignore ID until multi-connection is developed.)
- v := config.GetConfig()
connection := &models.GitlabConnection{}
- err := helper.EncodeStruct(v, connection, "env")
- if err != nil {
- return nil, err
- }
- response := &models.GitlabResponse{
- GitlabConnection: *connection,
- Name: "Gitlab",
- ID: 1,
- }
- return &core.ApiResourceOutput{Body: response}, nil
+ err := connectionHelper.First(connection, input.Params)
+ return &core.ApiResourceOutput{Body: connection}, err
}
diff --git a/plugins/gitlab/models/migrationscripts/archived/tag.go b/plugins/gitlab/api/init.go
similarity index 59%
copy from plugins/gitlab/models/migrationscripts/archived/tag.go
copy to plugins/gitlab/api/init.go
index c621810a..6774e148 100644
--- a/plugins/gitlab/models/migrationscripts/archived/tag.go
+++ b/plugins/gitlab/api/init.go
@@ -15,21 +15,25 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
-package archived
+package api
import (
- "github.com/apache/incubator-devlake/models/migrationscripts/archived"
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/helper"
+ "github.com/go-playground/validator/v10"
+ "github.com/spf13/viper"
+ "gorm.io/gorm"
)
-type GitlabTag struct {
- Name string `gorm:"primaryKey;type:varchar(60)"`
- Message string
- Target string `gorm:"type:varchar(255)"`
- Protected bool
- ReleaseDescription string
- archived.NoPKModel
-}
+var vld *validator.Validate
+var connectionHelper *helper.ConnectionApiHelper
+var basicRes core.BasicRes
-func (GitlabTag) TableName() string {
- return "_tool_gitlab_tags"
+func Init(config *viper.Viper, logger core.Logger, database *gorm.DB) {
+ basicRes = helper.NewDefaultBasicRes(config, logger, database)
+ vld = validator.New()
+ connectionHelper = helper.NewConnectionHelper(
+ basicRes,
+ vld,
+ )
}
diff --git a/plugins/gitlab/impl/impl.go b/plugins/gitlab/impl/impl.go
index 2c48224b..18dfb3e8 100644
--- a/plugins/gitlab/impl/impl.go
+++ b/plugins/gitlab/impl/impl.go
@@ -37,6 +37,7 @@ var _ core.Migratable = (*Gitlab)(nil)
type Gitlab string
func (plugin Gitlab) Init(config *viper.Viper, logger core.Logger, db *gorm.DB) error {
+ api.Init(config, logger, db)
return nil
}
@@ -100,8 +101,6 @@ func (plugin Gitlab) RootPkgPath() string {
func (plugin Gitlab) MigrationScripts() []migration.Script {
return []migration.Script{
new(migrationscripts.InitSchemas),
- new(migrationscripts.UpdateSchemas20220510),
- new(migrationscripts.UpdateSchemas20220525),
}
}
@@ -111,11 +110,13 @@ func (plugin Gitlab) ApiResources() map[string]map[string]core.ApiResourceHandle
"POST": api.TestConnection,
},
"connections": {
- "GET": api.ListConnections,
+ "POST": api.PostConnections,
+ "GET": api.ListConnections,
},
"connections/:connectionId": {
- "GET": api.GetConnection,
- "PATCH": api.PatchConnection,
+ "PATCH": api.PatchConnection,
+ "DELETE": api.DeleteConnection,
+ "GET": api.GetConnection,
},
}
}
diff --git a/plugins/gitlab/models/commit.go b/plugins/gitlab/models/commit.go
index 9378b557..dd5b0301 100644
--- a/plugins/gitlab/models/commit.go
+++ b/plugins/gitlab/models/commit.go
@@ -27,6 +27,7 @@ type GitlabCommit struct {
Sha string `gorm:"primaryKey;type:varchar(40)"`
Title string
Message string
+ ConnectionId uint64 `gorm:"primaryKey"`
ShortId string `gorm:"type:varchar(255)"`
AuthorName string `gorm:"type:varchar(255)"`
AuthorEmail string `gorm:"type:varchar(255)"`
diff --git a/plugins/gitlab/models/connection.go b/plugins/gitlab/models/connection.go
index 9ffc6a3e..a9ae6072 100644
--- a/plugins/gitlab/models/connection.go
+++ b/plugins/gitlab/models/connection.go
@@ -17,11 +17,20 @@ limitations under the License.
package models
+import (
+ "github.com/apache/incubator-devlake/plugins/helper"
+)
+
// This object conforms to what the frontend currently sends.
type GitlabConnection struct {
- Endpoint string `mapstructure:"endpoint" validate:"required" env:"GITLAB_ENDPOINT" json:"endpoint"`
- Auth string `mapstructure:"auth" validate:"required" env:"GITLAB_AUTH" json:"auth"`
- Proxy string `mapstructure:"proxy" env:"GITLAB_PROXY" json:"proxy"`
+ helper.RestConnection `mapstructure:",squash"`
+ helper.AccessToken `mapstructure:",squash"`
+}
+
+type TestConnectionRequest struct {
+ Endpoint string `json:"endpoint"`
+ Proxy string `json:"proxy"`
+ helper.AccessToken `mapstructure:",squash"`
}
// This object conforms to what the frontend currently expects.
@@ -37,8 +46,17 @@ type ApiUserResponse struct {
Name string `json:"name"`
}
-type TestConnectionRequest struct {
- Endpoint string `json:"endpoint" validate:"required"`
- Auth string `json:"auth" validate:"required"`
- Proxy string `json:"proxy"`
+type Config struct {
+ MrType string `mapstructure:"MrType" env:"GITLAB_PR_TYPE" json:"MrType"`
+ MrComponent string `mapstructure:"MrComponent" env:"GITLAB_PR_COMPONENT" json:"MrComponent"`
+ IssueSeverity string `mapstructure:"issueSeverity" env:"GITLAB_ISSUE_SEVERITY" json:"issueSeverity"`
+ IssuePriority string `mapstructure:"issuePriority" env:"GITLAB_ISSUE_PRIORITY" json:"issuePriority"`
+ IssueComponent string `mapstructure:"issueComponent" env:"GITLAB_ISSUE_COMPONENT" json:"issueComponent"`
+ IssueTypeBug string `mapstructure:"issueTypeBug" env:"GITLAB_ISSUE_TYPE_BUG" json:"issueTypeBug"`
+ IssueTypeIncident string `mapstructure:"issueTypeIncident" env:"GITLAB_ISSUE_TYPE_INCIDENT" json:"issueTypeIncident"`
+ IssueTypeRequirement string `mapstructure:"issueTypeRequirement" env:"GITLAB_ISSUE_TYPE_REQUIREMENT" json:"issueTypeRequirement"`
+}
+
+func (GitlabConnection) TableName() string {
+ return "_tool_gitlab_connections"
}
diff --git a/plugins/gitlab/models/issue.go b/plugins/gitlab/models/issue.go
index 2432824d..d0d5cc31 100644
--- a/plugins/gitlab/models/issue.go
+++ b/plugins/gitlab/models/issue.go
@@ -23,6 +23,7 @@ import (
)
type GitlabIssue struct {
+ ConnectionId uint64 `gorm:"primaryKey"`
GitlabId int `gorm:"primaryKey"`
ProjectId int `gorm:"index"`
Number int `gorm:"index;comment:Used in API requests ex. api/repo/1/issue/<THIS_NUMBER>"`
diff --git a/plugins/gitlab/models/issue_label.go b/plugins/gitlab/models/issue_label.go
index 17441766..b99b7f35 100644
--- a/plugins/gitlab/models/issue_label.go
+++ b/plugins/gitlab/models/issue_label.go
@@ -25,8 +25,9 @@ import (
// Pull Requests are considered Issues in GitHub.
type GitlabIssueLabel struct {
- IssueId int `gorm:"primaryKey;autoIncrement:false"`
- LabelName string `gorm:"primaryKey;type:varchar(255)"`
+ ConnectionId uint64 `gorm:"primaryKey"`
+ IssueId int `gorm:"primaryKey;autoIncrement:false"`
+ LabelName string `gorm:"primaryKey;type:varchar(255)"`
common.NoPKModel
}
diff --git a/plugins/gitlab/models/migrationscripts/archived/commit.go b/plugins/gitlab/models/migrationscripts/archived/commit.go
index 17c2cbcc..a9793411 100644
--- a/plugins/gitlab/models/migrationscripts/archived/commit.go
+++ b/plugins/gitlab/models/migrationscripts/archived/commit.go
@@ -18,15 +18,15 @@ limitations under the License.
package archived
import (
- "time"
-
"github.com/apache/incubator-devlake/models/migrationscripts/archived"
+ "time"
)
type GitlabCommit struct {
Sha string `gorm:"primaryKey;type:varchar(40)"`
Title string
Message string
+ ConnectionId uint64 `gorm:"primaryKey"`
ShortId string `gorm:"type:varchar(255)"`
AuthorName string `gorm:"type:varchar(255)"`
AuthorEmail string `gorm:"type:varchar(255)"`
diff --git a/plugins/gitlab/models/migrationscripts/archived/connection.go b/plugins/gitlab/models/migrationscripts/archived/connection.go
new file mode 100644
index 00000000..0de3012c
--- /dev/null
+++ b/plugins/gitlab/models/migrationscripts/archived/connection.go
@@ -0,0 +1,72 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package archived
+
+import (
+ "github.com/apache/incubator-devlake/models/migrationscripts/archived"
+)
+
+// This object conforms to what the frontend currently sends.
+type GitlabConnection struct {
+ RestConnection `mapstructure:",squash"`
+ AccessToken `mapstructure:",squash"`
+}
+
+type RestConnection struct {
+ BaseConnection `mapstructure:",squash"`
+ Endpoint string `mapstructure:"endpoint" validate:"required" json:"endpoint"`
+ Proxy string `mapstructure:"proxy" json:"proxy"`
+ RateLimit int `comment:"api request rate limt per hour" json:"rateLimit"`
+}
+
+type BaseConnection struct {
+ Name string `gorm:"type:varchar(100);uniqueIndex" json:"name" validate:"required"`
+ archived.Model
+}
+
+type AccessToken struct {
+ Token string `mapstructure:"token" validate:"required" json:"token" encrypt:"yes"`
+}
+
+// This object conforms to what the frontend currently expects.
+type GitlabResponse struct {
+ Name string `json:"name"`
+ ID int `json:"id"`
+ GitlabConnection
+}
+
+// Using User because it requires authentication.
+type ApiUserResponse struct {
+ Id int
+ Name string `json:"name"`
+}
+
+type Config struct {
+ MrType string `mapstructure:"MrType" env:"GITLAB_PR_TYPE" json:"MrType"`
+ MrComponent string `mapstructure:"MrComponent" env:"GITLAB_PR_COMPONENT" json:"MrComponent"`
+ IssueSeverity string `mapstructure:"issueSeverity" env:"GITLAB_ISSUE_SEVERITY" json:"issueSeverity"`
+ IssuePriority string `mapstructure:"issuePriority" env:"GITLAB_ISSUE_PRIORITY" json:"issuePriority"`
+ IssueComponent string `mapstructure:"issueComponent" env:"GITLAB_ISSUE_COMPONENT" json:"issueComponent"`
+ IssueTypeBug string `mapstructure:"issueTypeBug" env:"GITLAB_ISSUE_TYPE_BUG" json:"issueTypeBug"`
+ IssueTypeIncident string `mapstructure:"issueTypeIncident" env:"GITLAB_ISSUE_TYPE_INCIDENT" json:"issueTypeIncident"`
+ IssueTypeRequirement string `mapstructure:"issueTypeRequirement" env:"GITLAB_ISSUE_TYPE_REQUIREMENT" json:"issueTypeRequirement"`
+}
+
+func (GitlabConnection) TableName() string {
+ return "_tool_gitlab_connections"
+}
diff --git a/plugins/gitlab/models/migrationscripts/update_schemas20220525.go b/plugins/gitlab/models/migrationscripts/archived/issue.go
similarity index 66%
rename from plugins/gitlab/models/migrationscripts/update_schemas20220525.go
rename to plugins/gitlab/models/migrationscripts/archived/issue.go
index 5df572bd..0f8dec41 100644
--- a/plugins/gitlab/models/migrationscripts/update_schemas20220525.go
+++ b/plugins/gitlab/models/migrationscripts/archived/issue.go
@@ -15,18 +15,15 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
-package migrationscripts
+package archived
import (
- "context"
"github.com/apache/incubator-devlake/models/migrationscripts/archived"
- "gorm.io/gorm"
"time"
)
-type UpdateSchemas20220525 struct{}
-
-type GitlabIssue20220525 struct {
+type GitlabIssue struct {
+ ConnectionId uint64 `gorm:"primaryKey"`
GitlabId int `gorm:"primaryKey"`
ProjectId int `gorm:"index"`
Number int `gorm:"index;comment:Used in API requests ex. api/repo/1/issue/<THIS_NUMBER>"`
@@ -50,37 +47,6 @@ type GitlabIssue20220525 struct {
archived.NoPKModel
}
-func (GitlabIssue20220525) TableName() string {
+func (GitlabIssue) TableName() string {
return "_tool_gitlab_issues"
}
-
-type GitlabIssueLabel20220525 struct {
- IssueId int `gorm:"primaryKey;autoIncrement:false"`
- LabelName string `gorm:"primaryKey;type:varchar(255)"`
- archived.NoPKModel
-}
-
-func (GitlabIssueLabel20220525) TableName() string {
- return "_tool_gitlab_issue_labels"
-}
-
-func (*UpdateSchemas20220525) Up(ctx context.Context, db *gorm.DB) error {
- err := db.Migrator().AutoMigrate(GitlabIssue20220525{}, GitlabIssueLabel20220525{})
- if err != nil {
- return err
- }
-
- return nil
-}
-
-func (*UpdateSchemas20220525) Version() uint64 {
- return 20220510212344
-}
-
-func (*UpdateSchemas20220525) Owner() string {
- return "Gitlab"
-}
-
-func (*UpdateSchemas20220525) Name() string {
- return "add gitlab issue"
-}
diff --git a/plugins/gitlab/models/issue_label.go b/plugins/gitlab/models/migrationscripts/archived/issue_label.go
similarity index 78%
copy from plugins/gitlab/models/issue_label.go
copy to plugins/gitlab/models/migrationscripts/archived/issue_label.go
index 17441766..28fc538f 100644
--- a/plugins/gitlab/models/issue_label.go
+++ b/plugins/gitlab/models/migrationscripts/archived/issue_label.go
@@ -15,19 +15,18 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
-package models
+package archived
-import (
- "github.com/apache/incubator-devlake/models/common"
-)
+import "github.com/apache/incubator-devlake/models/migrationscripts/archived"
// Please note that Issue Labels can also apply to Pull Requests.
// Pull Requests are considered Issues in GitHub.
type GitlabIssueLabel struct {
- IssueId int `gorm:"primaryKey;autoIncrement:false"`
- LabelName string `gorm:"primaryKey;type:varchar(255)"`
- common.NoPKModel
+ ConnectionId uint64 `gorm:"primaryKey"`
+ IssueId int `gorm:"primaryKey;autoIncrement:false"`
+ LabelName string `gorm:"primaryKey;type:varchar(255)"`
+ archived.NoPKModel
}
func (GitlabIssueLabel) TableName() string {
diff --git a/plugins/gitlab/models/migrationscripts/archived/merge_request.go b/plugins/gitlab/models/migrationscripts/archived/mr.go
similarity index 87%
rename from plugins/gitlab/models/migrationscripts/archived/merge_request.go
rename to plugins/gitlab/models/migrationscripts/archived/mr.go
index 05133b06..0a600039 100644
--- a/plugins/gitlab/models/migrationscripts/archived/merge_request.go
+++ b/plugins/gitlab/models/migrationscripts/archived/mr.go
@@ -18,15 +18,15 @@ limitations under the License.
package archived
import (
- "time"
-
"github.com/apache/incubator-devlake/models/migrationscripts/archived"
+ "time"
)
type GitlabMergeRequest struct {
- GitlabId int `gorm:"primaryKey"`
- Iid int `gorm:"index"`
- ProjectId int `gorm:"index"`
+ ConnectionId uint64 `gorm:"primaryKey"`
+ GitlabId int `gorm:"primaryKey"`
+ Iid int `gorm:"index"`
+ ProjectId int `gorm:"index"`
SourceProjectId int
TargetProjectId int
State string `gorm:"type:varchar(255)"`
@@ -46,7 +46,7 @@ type GitlabMergeRequest struct {
AuthorUserId int
Component string `gorm:"type:varchar(255)"`
FirstCommentTime *time.Time `gorm:"comment:Time when the first comment occurred"`
- ReviewRounds int
+ ReviewRounds int `gorm:"comment:How many rounds of review this MR went through"`
archived.NoPKModel
}
diff --git a/plugins/gitlab/models/migrationscripts/archived/merge_request_comment.go b/plugins/gitlab/models/migrationscripts/archived/mr_comment.go
similarity index 97%
rename from plugins/gitlab/models/migrationscripts/archived/merge_request_comment.go
rename to plugins/gitlab/models/migrationscripts/archived/mr_comment.go
index 71291055..e06902f8 100644
--- a/plugins/gitlab/models/migrationscripts/archived/merge_request_comment.go
+++ b/plugins/gitlab/models/migrationscripts/archived/mr_comment.go
@@ -18,12 +18,13 @@ limitations under the License.
package archived
import (
- "time"
-
"github.com/apache/incubator-devlake/models/migrationscripts/archived"
+ "time"
)
type GitlabMergeRequestComment struct {
+ ConnectionId uint64 `gorm:"primaryKey"`
+
GitlabId int `gorm:"primaryKey"`
MergeRequestId int `gorm:"index"`
MergeRequestIid int `gorm:"comment:Used in API requests ex. /api/merge_requests/<THIS_IID>"`
diff --git a/plugins/gitlab/models/migrationscripts/archived/merge_request_commit.go b/plugins/gitlab/models/migrationscripts/archived/mr_commit.go
similarity index 92%
rename from plugins/gitlab/models/migrationscripts/archived/merge_request_commit.go
rename to plugins/gitlab/models/migrationscripts/archived/mr_commit.go
index 397f6d2c..5716e7bb 100644
--- a/plugins/gitlab/models/migrationscripts/archived/merge_request_commit.go
+++ b/plugins/gitlab/models/migrationscripts/archived/mr_commit.go
@@ -17,9 +17,7 @@ limitations under the License.
package archived
-import (
- "github.com/apache/incubator-devlake/models/migrationscripts/archived"
-)
+import "github.com/apache/incubator-devlake/models/migrationscripts/archived"
// This Model is intended to save commits that are associated to a merge request
// for the purpose of calculating PR Review Rounds and other metrics that
@@ -28,6 +26,7 @@ import (
// Thus a "Merge Request Commit" needs to be considered as distinct from a "Commit"
type GitlabMergeRequestCommit struct {
+ ConnectionId uint64 `gorm:"primaryKey"`
CommitSha string `gorm:"primaryKey;type:varchar(40)"`
MergeRequestId int `gorm:"primaryKey;autoIncrement:false"`
archived.NoPKModel
diff --git a/plugins/gitlab/models/migrationscripts/archived/merge_request_note.go b/plugins/gitlab/models/migrationscripts/archived/mr_note.go
similarity index 93%
rename from plugins/gitlab/models/migrationscripts/archived/merge_request_note.go
rename to plugins/gitlab/models/migrationscripts/archived/mr_note.go
index 39a5d23c..4c69c374 100644
--- a/plugins/gitlab/models/migrationscripts/archived/merge_request_note.go
+++ b/plugins/gitlab/models/migrationscripts/archived/mr_note.go
@@ -18,12 +18,13 @@ limitations under the License.
package archived
import (
- "time"
-
"github.com/apache/incubator-devlake/models/migrationscripts/archived"
+ "time"
)
type GitlabMergeRequestNote struct {
+ ConnectionId uint64 `gorm:"primaryKey"`
+
GitlabId int `gorm:"primaryKey"`
MergeRequestId int `gorm:"index"`
MergeRequestIid int `gorm:"comment:Used in API requests ex. /api/merge_requests/<THIS_IID>"`
@@ -33,7 +34,7 @@ type GitlabMergeRequestNote struct {
GitlabCreatedAt time.Time
Confidential bool
Resolvable bool `gorm:"comment:Is or is not review comment"`
- System bool `gorm:"comment:Is or is not auto-generated vs. human generated"`
+ IsSystem bool `gorm:"comment:Is or is not auto-generated vs. human generated"`
archived.NoPKModel
}
diff --git a/plugins/gitlab/models/migrationscripts/archived/pipeline.go b/plugins/gitlab/models/migrationscripts/archived/pipeline.go
index 98223142..409bd747 100644
--- a/plugins/gitlab/models/migrationscripts/archived/pipeline.go
+++ b/plugins/gitlab/models/migrationscripts/archived/pipeline.go
@@ -18,12 +18,13 @@ limitations under the License.
package archived
import (
- "time"
-
"github.com/apache/incubator-devlake/models/migrationscripts/archived"
+ "time"
)
type GitlabPipeline struct {
+ ConnectionId uint64 `gorm:"primaryKey"`
+
GitlabId int `gorm:"primaryKey"`
ProjectId int `gorm:"index"`
GitlabCreatedAt time.Time
diff --git a/plugins/gitlab/models/migrationscripts/archived/project.go b/plugins/gitlab/models/migrationscripts/archived/project.go
index 398a3f3e..cf73721e 100644
--- a/plugins/gitlab/models/migrationscripts/archived/project.go
+++ b/plugins/gitlab/models/migrationscripts/archived/project.go
@@ -18,12 +18,13 @@ limitations under the License.
package archived
import (
- "time"
-
"github.com/apache/incubator-devlake/models/migrationscripts/archived"
+ "time"
)
type GitlabProject struct {
+ ConnectionId uint64 `gorm:"primaryKey"`
+
GitlabId int `gorm:"primaryKey"`
Name string `gorm:"type:varchar(255)"`
Description string
diff --git a/plugins/gitlab/models/migrationscripts/archived/project_commits.go b/plugins/gitlab/models/migrationscripts/archived/project_commit.go
similarity index 89%
rename from plugins/gitlab/models/migrationscripts/archived/project_commits.go
rename to plugins/gitlab/models/migrationscripts/archived/project_commit.go
index 033e3d13..0d218079 100644
--- a/plugins/gitlab/models/migrationscripts/archived/project_commits.go
+++ b/plugins/gitlab/models/migrationscripts/archived/project_commit.go
@@ -17,11 +17,11 @@ limitations under the License.
package archived
-import (
- "github.com/apache/incubator-devlake/models/migrationscripts/archived"
-)
+import "github.com/apache/incubator-devlake/models/migrationscripts/archived"
type GitlabProjectCommit struct {
+ ConnectionId uint64 `gorm:"primaryKey"`
+
GitlabProjectId int `gorm:"primaryKey"`
CommitSha string `gorm:"primaryKey;type:varchar(40)"`
archived.NoPKModel
diff --git a/plugins/gitlab/models/migrationscripts/archived/reviewer.go b/plugins/gitlab/models/migrationscripts/archived/reviewer.go
index eb58084d..5285341b 100644
--- a/plugins/gitlab/models/migrationscripts/archived/reviewer.go
+++ b/plugins/gitlab/models/migrationscripts/archived/reviewer.go
@@ -17,11 +17,11 @@ limitations under the License.
package archived
-import (
- "github.com/apache/incubator-devlake/models/migrationscripts/archived"
-)
+import "github.com/apache/incubator-devlake/models/migrationscripts/archived"
type GitlabReviewer struct {
+ ConnectionId uint64 `gorm:"primaryKey"`
+
GitlabId int `gorm:"primaryKey"`
MergeRequestId int `gorm:"index"`
ProjectId int `gorm:"index"`
diff --git a/plugins/gitlab/models/migrationscripts/archived/tag.go b/plugins/gitlab/models/migrationscripts/archived/tag.go
index c621810a..000c3d04 100644
--- a/plugins/gitlab/models/migrationscripts/archived/tag.go
+++ b/plugins/gitlab/models/migrationscripts/archived/tag.go
@@ -17,11 +17,11 @@ limitations under the License.
package archived
-import (
- "github.com/apache/incubator-devlake/models/migrationscripts/archived"
-)
+import "github.com/apache/incubator-devlake/models/migrationscripts/archived"
type GitlabTag struct {
+ ConnectionId uint64 `gorm:"primaryKey"`
+
Name string `gorm:"primaryKey;type:varchar(60)"`
Message string
Target string `gorm:"type:varchar(255)"`
diff --git a/plugins/gitlab/models/migrationscripts/archived/user.go b/plugins/gitlab/models/migrationscripts/archived/user.go
index 1b3f404b..0132721a 100644
--- a/plugins/gitlab/models/migrationscripts/archived/user.go
+++ b/plugins/gitlab/models/migrationscripts/archived/user.go
@@ -17,13 +17,12 @@ limitations under the License.
package archived
-import (
- "github.com/apache/incubator-devlake/models/migrationscripts/archived"
-)
+import "github.com/apache/incubator-devlake/models/migrationscripts/archived"
type GitlabUser struct {
- Email string `gorm:"primaryKey;type:varchar(255)"`
- Name string `gorm:"type:varchar(255)"`
+ ConnectionId uint64 `gorm:"primaryKey"`
+ Email string `gorm:"primaryKey;type:varchar(255)"`
+ Name string `gorm:"type:varchar(255)"`
archived.NoPKModel
}
diff --git a/plugins/gitlab/models/migrationscripts/init_schema.go b/plugins/gitlab/models/migrationscripts/init_schema.go
index ef76e3fd..a826d8aa 100644
--- a/plugins/gitlab/models/migrationscripts/init_schema.go
+++ b/plugins/gitlab/models/migrationscripts/init_schema.go
@@ -19,15 +19,57 @@ package migrationscripts
import (
"context"
-
+ "fmt"
+ "github.com/apache/incubator-devlake/config"
+ "github.com/apache/incubator-devlake/plugins/core"
"github.com/apache/incubator-devlake/plugins/gitlab/models/migrationscripts/archived"
"gorm.io/gorm"
+ "gorm.io/gorm/clause"
)
type InitSchemas struct{}
func (*InitSchemas) Up(ctx context.Context, db *gorm.DB) error {
- return db.Migrator().AutoMigrate(
+ rawTableList := []string{
+ "_raw_gitlab_api_children_on_pipeline",
+ "_raw_gitlab_api_commit",
+ "_raw_gitlab_api_issues",
+ "_raw_gitlab_api_merge_request_commits",
+ "_raw_gitlab_api_merge_request_notes",
+ "_raw_gitlab_api_merge_requests",
+ "_raw_gitlab_api_pipeline",
+ "_raw_gitlab_api_project",
+ "_raw_gitlab_api_tag",
+ }
+ for _, v := range rawTableList {
+ err := db.Exec(fmt.Sprintf("DROP TABLE IF EXISTS %s CASCADE", v)).Error
+ if err != nil {
+ return err
+ }
+ }
+
+ err := db.Migrator().DropTable(
+ &archived.GitlabProject{},
+ &archived.GitlabMergeRequest{},
+ &archived.GitlabCommit{},
+ &archived.GitlabTag{},
+ &archived.GitlabProjectCommit{},
+ &archived.GitlabPipeline{},
+ &archived.GitlabReviewer{},
+ &archived.GitlabMergeRequestNote{},
+ &archived.GitlabMergeRequestCommit{},
+ &archived.GitlabMergeRequestComment{},
+ &archived.GitlabUser{},
+ &archived.GitlabConnection{},
+ &archived.GitlabIssue{},
+ &archived.GitlabIssueLabel{},
+ )
+
+ if err != nil {
+ return err
+ }
+
+ err = db.Migrator().AutoMigrate(
&archived.GitlabProject{},
&archived.GitlabMergeRequest{},
&archived.GitlabCommit{},
@@ -39,11 +81,39 @@ func (*InitSchemas) Up(ctx context.Context, db *gorm.DB) error {
&archived.GitlabMergeRequestCommit{},
&archived.GitlabMergeRequestComment{},
&archived.GitlabUser{},
+ &archived.GitlabConnection{},
+ &archived.GitlabIssue{},
+ &archived.GitlabIssueLabel{},
)
+
+ if err != nil {
+ return err
+ }
+
+ conn := &archived.GitlabConnection{}
+ v := config.GetConfig()
+ encKey := v.GetString(core.EncodeKeyEnvStr)
+
+ conn.Name = "init gitlab connection"
+ conn.ID = 1
+ conn.Endpoint = v.GetString("GITLAB_ENDPOINT")
+ conn.Token, err = core.Encrypt(encKey, v.GetString("GITLAB_AUTH"))
+ if err != nil {
+ return err
+ }
+ conn.Proxy = v.GetString("GITLAB_PROXY")
+ conn.RateLimit = v.GetInt("GITLAB_API_REQUESTS_PER_HOUR")
+ fmt.Println(conn.Endpoint)
+ err = db.Clauses(clause.OnConflict{DoNothing: true}).Create(conn).Error
+
+ if err != nil {
+ return err
+ }
+ return nil
}
func (*InitSchemas) Version() uint64 {
- return 20220407201136
+ return 20220614231236
}
func (*InitSchemas) Name() string {
diff --git a/plugins/gitlab/models/migrationscripts/updateSchemas20220510.go b/plugins/gitlab/models/migrationscripts/updateSchemas20220510.go
deleted file mode 100644
index 18896cab..00000000
--- a/plugins/gitlab/models/migrationscripts/updateSchemas20220510.go
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package migrationscripts
-
-import (
- "context"
- "github.com/apache/incubator-devlake/plugins/gitlab/models/migrationscripts/archived"
-
- "gorm.io/gorm"
-)
-
-type UpdateSchemas20220510 struct{}
-
-func (*UpdateSchemas20220510) Up(ctx context.Context, db *gorm.DB) error {
- err := db.Migrator().RenameColumn(archived.GitlabMergeRequestNote{}, "system", "is_system")
- if err != nil {
- return err
- }
-
- return nil
-}
-
-func (*UpdateSchemas20220510) Version() uint64 {
- return 20220510212344
-}
-
-func (*UpdateSchemas20220510) Owner() string {
- return "Gitlab"
-}
-
-func (*UpdateSchemas20220510) Name() string {
- return "Change key word system to is_system"
-}
diff --git a/plugins/gitlab/models/mr.go b/plugins/gitlab/models/mr.go
index 55ea5e46..a857c488 100644
--- a/plugins/gitlab/models/mr.go
+++ b/plugins/gitlab/models/mr.go
@@ -24,9 +24,10 @@ import (
)
type GitlabMergeRequest struct {
- GitlabId int `gorm:"primaryKey"`
- Iid int `gorm:"index"`
- ProjectId int `gorm:"index"`
+ ConnectionId uint64 `gorm:"primaryKey"`
+ GitlabId int `gorm:"primaryKey"`
+ Iid int `gorm:"index"`
+ ProjectId int `gorm:"index"`
SourceProjectId int
TargetProjectId int
State string `gorm:"type:varchar(255)"`
diff --git a/plugins/gitlab/models/mr_comment.go b/plugins/gitlab/models/mr_comment.go
index ccd0c341..64851af0 100644
--- a/plugins/gitlab/models/mr_comment.go
+++ b/plugins/gitlab/models/mr_comment.go
@@ -23,6 +23,8 @@ import (
)
type GitlabMergeRequestComment struct {
+ ConnectionId uint64 `gorm:"primaryKey"`
+
GitlabId int `gorm:"primaryKey"`
MergeRequestId int `gorm:"index"`
MergeRequestIid int `gorm:"comment:Used in API requests ex. /api/merge_requests/<THIS_IID>"`
diff --git a/plugins/gitlab/models/mr_commit.go b/plugins/gitlab/models/mr_commit.go
index 6d88cfa2..55fc3b3f 100644
--- a/plugins/gitlab/models/mr_commit.go
+++ b/plugins/gitlab/models/mr_commit.go
@@ -28,6 +28,7 @@ import (
// Thus a "Merge Request Commit" needs to be considered as distinct from a "Commit"
type GitlabMergeRequestCommit struct {
+ ConnectionId uint64 `gorm:"primaryKey"`
CommitSha string `gorm:"primaryKey;type:varchar(40)"`
MergeRequestId int `gorm:"primaryKey;autoIncrement:false"`
common.NoPKModel
diff --git a/plugins/gitlab/models/mr_note.go b/plugins/gitlab/models/mr_note.go
index 3b857072..b649b78d 100644
--- a/plugins/gitlab/models/mr_note.go
+++ b/plugins/gitlab/models/mr_note.go
@@ -24,6 +24,8 @@ import (
)
type GitlabMergeRequestNote struct {
+ ConnectionId uint64 `gorm:"primaryKey"`
+
GitlabId int `gorm:"primaryKey"`
MergeRequestId int `gorm:"index"`
MergeRequestIid int `gorm:"comment:Used in API requests ex. /api/merge_requests/<THIS_IID>"`
diff --git a/plugins/gitlab/models/pipeline.go b/plugins/gitlab/models/pipeline.go
index 425844ac..2c16a74c 100644
--- a/plugins/gitlab/models/pipeline.go
+++ b/plugins/gitlab/models/pipeline.go
@@ -23,6 +23,8 @@ import (
)
type GitlabPipeline struct {
+ ConnectionId uint64 `gorm:"primaryKey"`
+
GitlabId int `gorm:"primaryKey"`
ProjectId int `gorm:"index"`
GitlabCreatedAt time.Time
diff --git a/plugins/gitlab/models/project.go b/plugins/gitlab/models/project.go
index 04183963..1bcb63ec 100644
--- a/plugins/gitlab/models/project.go
+++ b/plugins/gitlab/models/project.go
@@ -24,6 +24,8 @@ import (
)
type GitlabProject struct {
+ ConnectionId uint64 `gorm:"primaryKey"`
+
GitlabId int `gorm:"primaryKey"`
Name string `gorm:"type:varchar(255)"`
Description string
diff --git a/plugins/gitlab/models/project_commit.go b/plugins/gitlab/models/project_commit.go
index 4512f602..549f8742 100644
--- a/plugins/gitlab/models/project_commit.go
+++ b/plugins/gitlab/models/project_commit.go
@@ -20,6 +20,8 @@ package models
import "github.com/apache/incubator-devlake/models/common"
type GitlabProjectCommit struct {
+ ConnectionId uint64 `gorm:"primaryKey"`
+
GitlabProjectId int `gorm:"primaryKey"`
CommitSha string `gorm:"primaryKey;type:varchar(40)"`
common.NoPKModel
diff --git a/plugins/gitlab/models/reviewer.go b/plugins/gitlab/models/reviewer.go
index 232d67e3..4728e598 100644
--- a/plugins/gitlab/models/reviewer.go
+++ b/plugins/gitlab/models/reviewer.go
@@ -22,6 +22,8 @@ import (
)
type GitlabReviewer struct {
+ ConnectionId uint64 `gorm:"primaryKey"`
+
GitlabId int `gorm:"primaryKey"`
MergeRequestId int `gorm:"index"`
ProjectId int `gorm:"index"`
diff --git a/plugins/gitlab/models/tag.go b/plugins/gitlab/models/tag.go
index fa47b4db..68f6262b 100644
--- a/plugins/gitlab/models/tag.go
+++ b/plugins/gitlab/models/tag.go
@@ -22,6 +22,8 @@ import (
)
type GitlabTag struct {
+ ConnectionId uint64 `gorm:"primaryKey"`
+
Name string `gorm:"primaryKey;type:varchar(60)"`
Message string
Target string `gorm:"type:varchar(255)"`
diff --git a/plugins/gitlab/models/user.go b/plugins/gitlab/models/user.go
index 1e233737..3e61d794 100644
--- a/plugins/gitlab/models/user.go
+++ b/plugins/gitlab/models/user.go
@@ -20,8 +20,9 @@ package models
import "github.com/apache/incubator-devlake/models/common"
type GitlabUser struct {
- Email string `gorm:"primaryKey;type:varchar(255)"`
- Name string `gorm:"type:varchar(255)"`
+ ConnectionId uint64 `gorm:"primaryKey"`
+ Email string `gorm:"primaryKey;type:varchar(255)"`
+ Name string `gorm:"type:varchar(255)"`
common.NoPKModel
}