You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by GitBox <gi...@apache.org> on 2022/06/09 09:34:16 UTC
[GitHub] [incubator-devlake] warren830 opened a new pull request, #2147: refactor(gitlab): gitlab multi conn
warren830 opened a new pull request, #2147:
URL: https://github.com/apache/incubator-devlake/pull/2147
# Summary
Modify backend to support multi connection for gitlab
### Does this close any open issues?
Please mention the issues here.
### Screenshots
Include any relevant screenshots here.
### Other Information
Any other information that is important to this PR.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@devlake.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [incubator-devlake] warren830 commented on a diff in pull request #2147: refactor(gitlab): gitlab multi conn
Posted by GitBox <gi...@apache.org>.
warren830 commented on code in PR #2147:
URL: https://github.com/apache/incubator-devlake/pull/2147#discussion_r896566824
##########
plugins/gitlab/models/migrationscripts/init_schema.go:
##########
@@ -39,11 +66,55 @@ func (*InitSchemas) Up(ctx context.Context, db *gorm.DB) error {
&archived.GitlabMergeRequestCommit{},
&archived.GitlabMergeRequestComment{},
&archived.GitlabUser{},
+ &archived.GitlabConnection{},
+ &archived.GitlabIssue{},
+ &archived.GitlabIssueLabel{},
)
+
+ if err != nil {
+ return err
+ }
+
+ err = db.Migrator().AutoMigrate(
+ &archived.GitlabProject{},
+ &archived.GitlabMergeRequest{},
+ &archived.GitlabCommit{},
+ &archived.GitlabTag{},
+ &archived.GitlabProjectCommit{},
+ &archived.GitlabPipeline{},
+ &archived.GitlabReviewer{},
+ &archived.GitlabMergeRequestNote{},
+ &archived.GitlabMergeRequestCommit{},
+ &archived.GitlabMergeRequestComment{},
+ &archived.GitlabUser{},
+ &archived.GitlabConnection{},
+ &archived.GitlabIssue{},
+ &archived.GitlabIssueLabel{},
+ )
+
+ if err != nil {
+ return err
+ }
+
+ conn := &archived.GitlabConnection{}
+ v := config.GetConfig()
+ conn.Name = "first connection"
+ conn.ID = 1
+ conn.Endpoint = v.GetString("GITLAB_ENDPOINT")
+ conn.Token = v.GetString("GITLAB_AUTH")
+ conn.Proxy = v.GetString("GITLAB_PROXY")
+ conn.RateLimit = v.GetInt("GITLAB_API_REQUESTS_PER_HOUR")
+ fmt.Println(conn.Endpoint)
+ err = db.Clauses(clause.OnConflict{DoNothing: true}).Create(conn).Error
+
+ if err != nil {
+ return err
+ }
+ return nil
}
func (*InitSchemas) Version() uint64 {
- return 20220407201136
+ return 202206147201136
Review Comment:
My mistake…
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@devlake.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [incubator-devlake] likyh commented on a diff in pull request #2147: refactor(gitlab): gitlab multi conn
Posted by GitBox <gi...@apache.org>.
likyh commented on code in PR #2147:
URL: https://github.com/apache/incubator-devlake/pull/2147#discussion_r896507496
##########
plugins/gitlab/models/connection.go:
##########
@@ -17,11 +17,20 @@ limitations under the License.
package models
+import (
+ "github.com/apache/incubator-devlake/plugins/helper"
+)
+
// This object conforms to what the frontend currently sends.
type GitlabConnection struct {
- Endpoint string `mapstructure:"endpoint" validate:"required" env:"GITLAB_ENDPOINT" json:"endpoint"`
- Auth string `mapstructure:"auth" validate:"required" env:"GITLAB_AUTH" json:"auth"`
- Proxy string `mapstructure:"proxy" env:"GITLAB_PROXY" json:"proxy"`
+ helper.RestConnection `mapstructure:",squash"`
+ helper.AccessToken `mapstructure:",squash"`
Review Comment:
write column instead of using the helper.
##########
plugins/gitlab/models/migrationscripts/init_schema.go:
##########
@@ -19,15 +19,42 @@ package migrationscripts
import (
"context"
-
+ "fmt"
+ "github.com/apache/incubator-devlake/config"
"github.com/apache/incubator-devlake/plugins/gitlab/models/migrationscripts/archived"
"gorm.io/gorm"
+ "gorm.io/gorm/clause"
)
type InitSchemas struct{}
+type RawMr struct {
+}
+
+func (r RawMr) tableName() string {
+ return "_raw_gitlab_merge_requests"
+}
+
func (*InitSchemas) Up(ctx context.Context, db *gorm.DB) error {
- return db.Migrator().AutoMigrate(
+ rawTableList := []string{
+ "_raw_gitlab_api_children_on_pipeline",
+ "_raw_gitlab_api_commit",
+ "_raw_gitlab_api_issues",
+ "_raw_gitlab_api_merge_request_commits",
+ "_raw_gitlab_api_merge_request_notes",
+ "_raw_gitlab_api_merge_requests",
+ "_raw_gitlab_api_pipeline",
+ "_raw_gitlab_api_project",
+ "_raw_gitlab_api_tag",
+ }
+ for _, v := range rawTableList {
+ err := db.Exec(fmt.Sprintf("DROP TABLE IF EXISTS %s CASCADE", v)).Error
+ if err != nil {
+ return err
+ }
+ }
+
+ err := db.Migrator().DropTable(
Review Comment:
Will droping a not exist table got the error?
##########
plugins/gitlab/models/migrationscripts/init_schema.go:
##########
@@ -19,15 +19,42 @@ package migrationscripts
import (
"context"
-
+ "fmt"
+ "github.com/apache/incubator-devlake/config"
"github.com/apache/incubator-devlake/plugins/gitlab/models/migrationscripts/archived"
"gorm.io/gorm"
+ "gorm.io/gorm/clause"
)
type InitSchemas struct{}
+type RawMr struct {
+}
+
+func (r RawMr) tableName() string {
+ return "_raw_gitlab_merge_requests"
+}
+
func (*InitSchemas) Up(ctx context.Context, db *gorm.DB) error {
- return db.Migrator().AutoMigrate(
+ rawTableList := []string{
+ "_raw_gitlab_api_children_on_pipeline",
+ "_raw_gitlab_api_commit",
+ "_raw_gitlab_api_issues",
+ "_raw_gitlab_api_merge_request_commits",
+ "_raw_gitlab_api_merge_request_notes",
+ "_raw_gitlab_api_merge_requests",
+ "_raw_gitlab_api_pipeline",
+ "_raw_gitlab_api_project",
+ "_raw_gitlab_api_tag",
+ }
+ for _, v := range rawTableList {
+ err := db.Exec(fmt.Sprintf("DROP TABLE IF EXISTS %s CASCADE", v)).Error
+ if err != nil {
+ return err
+ }
+ }
+
+ err := db.Migrator().DropTable(
Review Comment:
Do you drop all tables that may appear? I notice that these are fewer tables than AutoMigrate.
##########
plugins/gitlab/models/migrationscripts/init_schema.go:
##########
@@ -39,11 +66,55 @@ func (*InitSchemas) Up(ctx context.Context, db *gorm.DB) error {
&archived.GitlabMergeRequestCommit{},
&archived.GitlabMergeRequestComment{},
&archived.GitlabUser{},
+ &archived.GitlabConnection{},
+ &archived.GitlabIssue{},
+ &archived.GitlabIssueLabel{},
)
+
+ if err != nil {
+ return err
+ }
+
+ err = db.Migrator().AutoMigrate(
+ &archived.GitlabProject{},
+ &archived.GitlabMergeRequest{},
+ &archived.GitlabCommit{},
+ &archived.GitlabTag{},
+ &archived.GitlabProjectCommit{},
+ &archived.GitlabPipeline{},
+ &archived.GitlabReviewer{},
+ &archived.GitlabMergeRequestNote{},
+ &archived.GitlabMergeRequestCommit{},
+ &archived.GitlabMergeRequestComment{},
+ &archived.GitlabUser{},
+ &archived.GitlabConnection{},
+ &archived.GitlabIssue{},
+ &archived.GitlabIssueLabel{},
+ )
+
+ if err != nil {
+ return err
+ }
+
+ conn := &archived.GitlabConnection{}
+ v := config.GetConfig()
+ conn.Name = "first connection"
+ conn.ID = 1
+ conn.Endpoint = v.GetString("GITLAB_ENDPOINT")
+ conn.Token = v.GetString("GITLAB_AUTH")
+ conn.Proxy = v.GetString("GITLAB_PROXY")
+ conn.RateLimit = v.GetInt("GITLAB_API_REQUESTS_PER_HOUR")
+ fmt.Println(conn.Endpoint)
+ err = db.Clauses(clause.OnConflict{DoNothing: true}).Create(conn).Error
+
+ if err != nil {
+ return err
+ }
+ return nil
}
func (*InitSchemas) Version() uint64 {
- return 20220407201136
+ return 202206147201136
Review Comment:
There is one more digit here. Mistakes? And what does `7201136` mean? use the count of migration scripts (000001) instead?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@devlake.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [incubator-devlake] warren830 commented on a diff in pull request #2147: refactor(gitlab): gitlab multi conn
Posted by GitBox <gi...@apache.org>.
warren830 commented on code in PR #2147:
URL: https://github.com/apache/incubator-devlake/pull/2147#discussion_r896499881
##########
plugins/gitlab/models/connection.go:
##########
@@ -17,11 +17,13 @@ limitations under the License.
package models
+import "github.com/apache/incubator-devlake/plugins/helper"
+
// This object conforms to what the frontend currently sends.
type GitlabConnection struct {
- Endpoint string `mapstructure:"endpoint" validate:"required" env:"GITLAB_ENDPOINT" json:"endpoint"`
- Auth string `mapstructure:"auth" validate:"required" env:"GITLAB_AUTH" json:"auth"`
- Proxy string `mapstructure:"proxy" env:"GITLAB_PROXY" json:"proxy"`
+ helper.RestConnection `mapstructure:",squash"`
+ helper.AccessToken `mapstructure:",squash"`
+ Config `mapstructure:",squash"`
Review Comment:
OK，fixed
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@devlake.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [incubator-devlake] warren830 commented on a diff in pull request #2147: refactor(gitlab): gitlab multi conn
Posted by GitBox <gi...@apache.org>.
warren830 commented on code in PR #2147:
URL: https://github.com/apache/incubator-devlake/pull/2147#discussion_r896554033
##########
plugins/gitlab/models/connection.go:
##########
@@ -17,11 +17,20 @@ limitations under the License.
package models
+import (
+ "github.com/apache/incubator-devlake/plugins/helper"
+)
+
// This object conforms to what the frontend currently sends.
type GitlabConnection struct {
- Endpoint string `mapstructure:"endpoint" validate:"required" env:"GITLAB_ENDPOINT" json:"endpoint"`
- Auth string `mapstructure:"auth" validate:"required" env:"GITLAB_AUTH" json:"auth"`
- Proxy string `mapstructure:"proxy" env:"GITLAB_PROXY" json:"proxy"`
+ helper.RestConnection `mapstructure:",squash"`
+ helper.AccessToken `mapstructure:",squash"`
Review Comment:
this is in models, not in archived
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@devlake.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [incubator-devlake] warren830 commented on a diff in pull request #2147: refactor(gitlab): gitlab multi conn
Posted by GitBox <gi...@apache.org>.
warren830 commented on code in PR #2147:
URL: https://github.com/apache/incubator-devlake/pull/2147#discussion_r896553673
##########
plugins/gitlab/models/migrationscripts/init_schema.go:
##########
@@ -19,15 +19,42 @@ package migrationscripts
import (
"context"
-
+ "fmt"
+ "github.com/apache/incubator-devlake/config"
"github.com/apache/incubator-devlake/plugins/gitlab/models/migrationscripts/archived"
"gorm.io/gorm"
+ "gorm.io/gorm/clause"
)
type InitSchemas struct{}
+type RawMr struct {
+}
+
+func (r RawMr) tableName() string {
+ return "_raw_gitlab_merge_requests"
+}
+
func (*InitSchemas) Up(ctx context.Context, db *gorm.DB) error {
- return db.Migrator().AutoMigrate(
+ rawTableList := []string{
+ "_raw_gitlab_api_children_on_pipeline",
+ "_raw_gitlab_api_commit",
+ "_raw_gitlab_api_issues",
+ "_raw_gitlab_api_merge_request_commits",
+ "_raw_gitlab_api_merge_request_notes",
+ "_raw_gitlab_api_merge_requests",
+ "_raw_gitlab_api_pipeline",
+ "_raw_gitlab_api_project",
+ "_raw_gitlab_api_tag",
+ }
+ for _, v := range rawTableList {
+ err := db.Exec(fmt.Sprintf("DROP TABLE IF EXISTS %s CASCADE", v)).Error
+ if err != nil {
+ return err
+ }
+ }
+
+ err := db.Migrator().DropTable(
Review Comment:
> Will droping a not exist table got the error?
Nope, cause it's drop if exists
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@devlake.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [incubator-devlake] klesh commented on a diff in pull request #2147: refactor(gitlab): gitlab multi conn
Posted by GitBox <gi...@apache.org>.
klesh commented on code in PR #2147:
URL: https://github.com/apache/incubator-devlake/pull/2147#discussion_r895848728
##########
plugins/gitlab/models/connection.go:
##########
@@ -17,11 +17,13 @@ limitations under the License.
package models
+import "github.com/apache/incubator-devlake/plugins/helper"
+
// This object conforms to what the frontend currently sends.
type GitlabConnection struct {
- Endpoint string `mapstructure:"endpoint" validate:"required" env:"GITLAB_ENDPOINT" json:"endpoint"`
- Auth string `mapstructure:"auth" validate:"required" env:"GITLAB_AUTH" json:"auth"`
- Proxy string `mapstructure:"proxy" env:"GITLAB_PROXY" json:"proxy"`
+ helper.RestConnection `mapstructure:",squash"`
+ helper.AccessToken `mapstructure:",squash"`
+ Config `mapstructure:",squash"`
Review Comment:
No, we dont store Config to Connection anymore, we load them from `task.options`
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@devlake.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [incubator-devlake] mindlesscloud merged pull request #2147: refactor(gitlab): gitlab multi conn
Posted by GitBox <gi...@apache.org>.
mindlesscloud merged PR #2147:
URL: https://github.com/apache/incubator-devlake/pull/2147
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@devlake.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [incubator-devlake] warren830 commented on a diff in pull request #2147: refactor(gitlab): gitlab multi conn
Posted by GitBox <gi...@apache.org>.
warren830 commented on code in PR #2147:
URL: https://github.com/apache/incubator-devlake/pull/2147#discussion_r896553423
##########
plugins/gitlab/models/migrationscripts/init_schema.go:
##########
@@ -19,15 +19,42 @@ package migrationscripts
import (
"context"
-
+ "fmt"
+ "github.com/apache/incubator-devlake/config"
"github.com/apache/incubator-devlake/plugins/gitlab/models/migrationscripts/archived"
"gorm.io/gorm"
+ "gorm.io/gorm/clause"
)
type InitSchemas struct{}
+type RawMr struct {
+}
+
+func (r RawMr) tableName() string {
+ return "_raw_gitlab_merge_requests"
+}
+
func (*InitSchemas) Up(ctx context.Context, db *gorm.DB) error {
- return db.Migrator().AutoMigrate(
+ rawTableList := []string{
+ "_raw_gitlab_api_children_on_pipeline",
+ "_raw_gitlab_api_commit",
+ "_raw_gitlab_api_issues",
+ "_raw_gitlab_api_merge_request_commits",
+ "_raw_gitlab_api_merge_request_notes",
+ "_raw_gitlab_api_merge_requests",
+ "_raw_gitlab_api_pipeline",
+ "_raw_gitlab_api_project",
+ "_raw_gitlab_api_tag",
+ }
+ for _, v := range rawTableList {
+ err := db.Exec(fmt.Sprintf("DROP TABLE IF EXISTS %s CASCADE", v)).Error
+ if err != nil {
+ return err
+ }
+ }
+
+ err := db.Migrator().DropTable(
Review Comment:
I only dropped tables belong to gitlab including all raw and tool tables.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@devlake.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [incubator-devlake] likyh commented on a diff in pull request #2147: refactor(gitlab): gitlab multi conn
Posted by GitBox <gi...@apache.org>.
likyh commented on code in PR #2147:
URL: https://github.com/apache/incubator-devlake/pull/2147#discussion_r896558323
##########
plugins/gitlab/models/connection.go:
##########
@@ -17,11 +17,20 @@ limitations under the License.
package models
+import (
+ "github.com/apache/incubator-devlake/plugins/helper"
+)
+
// This object conforms to what the frontend currently sends.
type GitlabConnection struct {
- Endpoint string `mapstructure:"endpoint" validate:"required" env:"GITLAB_ENDPOINT" json:"endpoint"`
- Auth string `mapstructure:"auth" validate:"required" env:"GITLAB_AUTH" json:"auth"`
- Proxy string `mapstructure:"proxy" env:"GITLAB_PROXY" json:"proxy"`
+ helper.RestConnection `mapstructure:",squash"`
+ helper.AccessToken `mapstructure:",squash"`
Review Comment:
ok
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@devlake.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org