You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by kl...@apache.org on 2022/06/13 15:12:07 UTC
[incubator-devlake] branch main updated: save github connection in db and support create multi connection (#2128)
This is an automated email from the ASF dual-hosted git repository.
klesh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new c7e483ee save github connection in db and support create multi connection (#2128)
c7e483ee is described below
commit c7e483ee38290977a24d674bb21c9b0d4b875782
Author: likyh <l...@likyh.com>
AuthorDate: Mon Jun 13 23:12:02 2022 +0800
save github connection in db and support create multi connection (#2128)
* save github connection in db and support create multi connection
* append
* use helper.AccessToken in github connections
* delete reading config in github plugin
Co-authored-by: linyh <ya...@meri.co>
---
.env.example | 21 +------
config-ui/src/data/Providers.js | 2 +-
config-ui/src/data/integrations.jsx | 2 +-
config-ui/src/hooks/useConnectionManager.jsx | 6 +-
config-ui/src/hooks/useSettingsManager.jsx | 2 +-
.../pages/configure/connections/AddConnection.jsx | 7 +--
.../pages/configure/connections/ConnectionForm.jsx | 10 ++--
.../src/pages/configure/integration/manage.jsx | 4 +-
plugins/github/api/connection.go | 67 +++++++++-------------
plugins/github/api/init.go | 42 ++++++++++++++
plugins/github/github.go | 24 ++++++--
plugins/github/models/connection.go | 45 +++++++--------
.../updateSchemas20220608.go} | 67 ++++++++++++++--------
plugins/github/tasks/api_client.go | 23 ++------
plugins/github/tasks/issue_extractor.go | 28 ++-------
plugins/github/tasks/pr_extractor.go | 8 +--
plugins/helper/connection.go | 2 +-
plugins/icla/plugin_main.go | 9 +--
18 files changed, 184 insertions(+), 185 deletions(-)
diff --git a/.env.example b/.env.example
index a031fb91..a7a161d8 100644
--- a/.env.example
+++ b/.env.example
@@ -78,25 +78,10 @@ FEISHU_ENDPOINT=https://open.feishu.cn/open-apis/vc/v1/
# GitHub configuration #
########################
-GITHUB_ENDPOINT=https://api.github.com/
-GITHUB_AUTH=
-GITHUB_PROXY=
-GITHUB_API_REQUESTS_PER_HOUR=
-# GITHUB_PR_TYPE=type/(.*)$ the program will extract the value in (), in this example, you will get "refactor" from "type/refactor"
-GITHUB_PR_TYPE='type/(.*)$'
-# GITHUB_PR_COMPONENT=component/(.*)$ the program will extract the value in (), in this example, you will get "plugins" from "component/plugins"
-GITHUB_PR_COMPONENT='component/(.*)$'
-# GITHUB_ISSUE_SEVERITY=severity/(.*)$ the program will extract the value in (), in this example, you will get "refactor" from "type/refactor"
-GITHUB_ISSUE_SEVERITY='severity/(.*)$'
-# GITHUB_ISSUE_COMPONENT=component/(.*)$ the program will extract the value in (), in this example, you will get "refactor" from "type/refactor"
-GITHUB_ISSUE_COMPONENT='component/(.*)$'
-GITHUB_ISSUE_PRIORITY='^(highest|high|medium|low)$'
-GITHUB_ISSUE_TYPE_BUG='^(bug|failure|error)$'
-GITHUB_ISSUE_TYPE_REQUIREMENT='^(feat|feature|proposal|requirement)$'
-GITHUB_ISSUE_TYPE_INCIDENT=
-# GITHUB_PR_BODY_CLOSE_PATTERN='(?mi)(fix|close|resolve|fixes|closes|resolves|fixed|closed|resolved)[\s]*.*(((and )?(#|https:\/\/github.com\/%s\/%s\/issues\/)\d+[ ]*)+)'
+# GitHub configuration has been migrated into DB #
+# FIXME It's very special because no defined in config, so it will be deleted in next PR
GITHUB_PR_BODY_CLOSE_PATTERN='(?mi)(fix|close|resolve|fixes|closes|resolves|fixed|closed|resolved)[\s]*.*(((and )?(#|https:\/\/github.com\/%s\/%s\/issues\/)\d+[ ]*)+)'
-# GITHUB_PR_TITLE_PATTERN='.*\(#(\d+)\)'
+# FIXME this config use in refdiff
GITHUB_PR_TITLE_PATTERN='.*\(#(\d+)\)'
##########################
diff --git a/config-ui/src/data/Providers.js b/config-ui/src/data/Providers.js
index 384f1746..96140f56 100644
--- a/config-ui/src/data/Providers.js
+++ b/config-ui/src/data/Providers.js
@@ -63,7 +63,7 @@ const ProviderConnectionLimits = {
gitlab: 1,
jenkins: 1,
// jira: null, // (Multi-connection, no-limit)
- github: 1
+ // github: null
}
// NOTE: Not all fields may be referenced/displayed for a provider,
diff --git a/config-ui/src/data/integrations.jsx b/config-ui/src/data/integrations.jsx
index 0566ebdf..9912dffc 100644
--- a/config-ui/src/data/integrations.jsx
+++ b/config-ui/src/data/integrations.jsx
@@ -91,7 +91,7 @@ const integrationsData = [
id: Providers.GITHUB,
type: ProviderTypes.INTEGRATION,
enabled: true,
- multiConnection: false,
+ multiConnection: true,
name: ProviderLabels.GITHUB,
icon: <GitHubProvider className='providerIconSvg' width='30' height='30' style={{ float: 'left', marginTop: '5px' }} />,
iconDashboard: <GitHubProvider className='providerIconSvg' width='48' height='48' />,
diff --git a/config-ui/src/hooks/useConnectionManager.jsx b/config-ui/src/hooks/useConnectionManager.jsx
index f9116919..a9c7bfff 100644
--- a/config-ui/src/hooks/useConnectionManager.jsx
+++ b/config-ui/src/hooks/useConnectionManager.jsx
@@ -128,7 +128,7 @@ function useConnectionManager ({
connectionPayload = { name: name, endpoint: endpointUrl, username: username, password: password, proxy: proxy, ...connectionPayload }
break
case Providers.GITHUB:
- connectionPayload = { name: name, endpoint: endpointUrl, auth: token, proxy: proxy, ...connectionPayload }
+ connectionPayload = { name: name, endpoint: endpointUrl, token, proxy: proxy, ...connectionPayload }
break
case Providers.JENKINS:
// eslint-disable-next-line max-len
@@ -329,7 +329,7 @@ function useConnectionManager ({
endpoint: c.Endpoint || c.endpoint,
username: c.username,
password: c.password,
- auth: c.basicAuthEncoded || c.auth,
+ auth: c.basicAuthEncoded || c.token,
proxy: c.Proxy || c.Proxy
}
const onSuccess = (res) => {
@@ -380,7 +380,7 @@ function useConnectionManager ({
setProxy(activeConnection.Proxy || activeConnection.proxy)
break
case Providers.GITHUB:
- setToken(activeConnection.basicAuthEncoded || activeConnection.auth)
+ setToken(activeConnection.basicAuthEncoded || activeConnection.token)
setProxy(activeConnection.Proxy || activeConnection.proxy)
break
case Providers.JIRA:
diff --git a/config-ui/src/hooks/useSettingsManager.jsx b/config-ui/src/hooks/useSettingsManager.jsx
index bdfb635d..35fe331d 100644
--- a/config-ui/src/hooks/useSettingsManager.jsx
+++ b/config-ui/src/hooks/useSettingsManager.jsx
@@ -49,7 +49,7 @@ function useSettingsManager ({
...connectionPayload,
name: connection.name,
endpoint: connection.endpoint,
- auth: connection.auth,
+ auth: connection.token,
proxy: connection.proxy || connection.Proxy
}
break
diff --git a/config-ui/src/pages/configure/connections/AddConnection.jsx b/config-ui/src/pages/configure/connections/AddConnection.jsx
index 7e81106d..88a15d09 100644
--- a/config-ui/src/pages/configure/connections/AddConnection.jsx
+++ b/config-ui/src/pages/configure/connections/AddConnection.jsx
@@ -45,7 +45,8 @@ export default function AddConnection () {
const [activeProvider, setActiveProvider] = useState(integrationsData.find(p => p.id === providerId))
const {
- testConnection, saveConnection,
+ testConnection,
+ saveConnection,
errors,
isSaving,
isTesting,
@@ -101,15 +102,13 @@ export default function AddConnection () {
if (activeProvider && activeProvider.id) {
fetchAllConnections()
switch (activeProvider.id) {
- case Providers.GITHUB:
- setName(ProviderLabels.GITHUB)
- break
case Providers.GITLAB:
setName(ProviderLabels.GITLAB)
break
case Providers.JENKINS:
setName(ProviderLabels.JENKINS)
break
+ case Providers.GITHUB:
case Providers.JIRA:
default:
setName('')
diff --git a/config-ui/src/pages/configure/connections/ConnectionForm.jsx b/config-ui/src/pages/configure/connections/ConnectionForm.jsx
index 2e657c13..697719ab 100644
--- a/config-ui/src/pages/configure/connections/ConnectionForm.jsx
+++ b/config-ui/src/pages/configure/connections/ConnectionForm.jsx
@@ -159,7 +159,7 @@ export default function ConnectionForm (props) {
backgroundColor: '#f0f0f0'
}}
>
- <p className='warning-message' intent={Intent.WARNING}>
+ <p className='warning-message'>
<Icon icon='warning-sign' size='16' color={Colors.GRAY1} style={{ marginRight: '5px' }} />
<strong>CONNECTION SOURCES LIMITED</strong><br />
You may only add <Tag intent={Intent.PRIMARY}>{sourceLimits[activeProvider.id]}</Tag> instance(s) at this time,
@@ -181,7 +181,7 @@ export default function ConnectionForm (props) {
border: showLimitWarning ? 'inherit' : 0
}}
>
- <p className='warning-message' intent={Intent.WARNING}>
+ <p className='warning-message'>
<Icon icon='error' size='16' color={Colors.RED4} style={{ marginRight: '5px' }} />
<strong>UNABLE TO SAVE CONNECTION ({name !== '' ? name : 'BLANK'})</strong><br />
</p>
@@ -197,7 +197,7 @@ export default function ConnectionForm (props) {
<div className='formContainer'>
<FormGroup
disabled={isTesting || isSaving || isLocked}
- readOnly={[Providers.GITHUB, Providers.GITLAB, Providers.JENKINS].includes(activeProvider.id)}
+ readOnly={[Providers.GITLAB, Providers.JENKINS].includes(activeProvider.id)}
label=''
inline={true}
labelFor='connection-name'
@@ -216,14 +216,14 @@ export default function ConnectionForm (props) {
id='connection-name'
inputRef={connectionNameRef}
disabled={isTesting || isSaving || isLocked}
- readOnly={[Providers.GITHUB, Providers.GITLAB, Providers.JENKINS].includes(activeProvider.id)}
+ readOnly={[Providers.GITLAB, Providers.JENKINS].includes(activeProvider.id)}
placeholder={placeholders ? placeholders.name : 'Enter Instance Name'}
value={name}
onChange={(e) => onNameChange(e.target.value)}
// className={`input connection-name-input ${fieldHasError('Connection') ? 'invalid-field' : ''}`}
// className='input connection-name-input'
className={`input connection-name-input ${stateErrored === 'connection-name' ? 'invalid-field' : ''}`}
- leftIcon={[Providers.GITHUB, Providers.GITLAB, Providers.JENKINS].includes(activeProvider.id) ? 'lock' : null}
+ leftIcon={[Providers.GITLAB, Providers.JENKINS].includes(activeProvider.id) ? 'lock' : null}
inline={true}
rightElement={(
<InputValidationError
diff --git a/config-ui/src/pages/configure/integration/manage.jsx b/config-ui/src/pages/configure/integration/manage.jsx
index 87afe6d3..6f1987ca 100644
--- a/config-ui/src/pages/configure/integration/manage.jsx
+++ b/config-ui/src/pages/configure/integration/manage.jsx
@@ -264,7 +264,7 @@ export default function ManageIntegration () {
<table className='bp3-html-table bp3-html-table-bordered connections-table' style={{ width: '100%' }}>
<thead>
<tr>
- {activeProvider.id === Providers.JIRA && (<th>ID</th>)}
+ {!sourceLimits[activeProvider.id] && (<th>ID</th>)}
<th>Connection Name</th>
<th>Endpoint</th>
<th>Status</th>
@@ -278,7 +278,7 @@ export default function ManageIntegration () {
// eslint-disable-next-line max-len
className={getTestedConnection(connection) && getTestedConnection(connection).status !== 1 ? 'connection-offline' : 'connection-online'}
>
- {activeProvider.id === Providers.JIRA && (
+ {!sourceLimits[activeProvider.id] && (
<td
style={{ cursor: 'pointer' }}
className='cell-name'
diff --git a/plugins/github/api/connection.go b/plugins/github/api/connection.go
index 170d3669..0e468dc4 100644
--- a/plugins/github/api/connection.go
+++ b/plugins/github/api/connection.go
@@ -19,23 +19,17 @@ package api
import (
"fmt"
- "github.com/apache/incubator-devlake/config"
"github.com/apache/incubator-devlake/plugins/github/models"
"net/http"
"strings"
"time"
"github.com/apache/incubator-devlake/plugins/helper"
- "github.com/go-playground/validator/v10"
"github.com/mitchellh/mapstructure"
"github.com/apache/incubator-devlake/plugins/core"
)
-type ApiUserPublicEmailResponse []models.PublicEmail
-
-var vld = validator.New()
-
/*
POST /plugins/github/test
*/
@@ -109,68 +103,63 @@ func TestConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput, erro
}
/*
-PATCH /plugins/github/connections/:connectionId
+POST /plugins/github/connections
*/
-func PatchConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) {
- v := config.GetConfig()
+func PostConnections(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) {
connection := &models.GithubConnection{}
- err := helper.EncodeStruct(v, connection, "env")
+ err := connectionHelper.Create(connection, input)
if err != nil {
return nil, err
}
- // update from request and save to .env
- err = helper.DecodeStruct(v, connection, input.Body, "env")
+ return &core.ApiResourceOutput{Body: connection, Status: http.StatusOK}, nil
+}
+
+/*
+PATCH /plugins/github/connections/:connectionId
+*/
+func PatchConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) {
+ connection := &models.GithubConnection{}
+ err := connectionHelper.Patch(connection, input)
if err != nil {
return nil, err
}
- err = config.WriteConfig(v)
+ return &core.ApiResourceOutput{Body: connection, Status: http.StatusOK}, nil
+}
+
+/*
+DELETE /plugins/github/connections/:connectionId
+*/
+func DeleteConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) {
+ connection := &models.GithubConnection{}
+ err := connectionHelper.First(connection, input.Params)
if err != nil {
return nil, err
}
- response := models.GithubResponse{
- GithubConnection: *connection,
- Name: "Github",
- ID: 1,
- }
- return &core.ApiResourceOutput{Body: response, Status: http.StatusOK}, nil
+ err = connectionHelper.Delete(connection)
+ return &core.ApiResourceOutput{Body: connection}, err
}
/*
GET /plugins/github/connections
*/
func ListConnections(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) {
- // RETURN ONLY 1 SOURCE (FROM ENV) until multi-connection is developed.
- v := config.GetConfig()
- connection := &models.GithubConnection{}
-
- err := helper.EncodeStruct(v, connection, "env")
+ var connections []models.GithubConnection
+ err := connectionHelper.List(&connections)
if err != nil {
return nil, err
}
- response := models.GithubResponse{
- GithubConnection: *connection,
- Name: "Github",
- ID: 1,
- }
- return &core.ApiResourceOutput{Body: []models.GithubResponse{response}}, nil
+ return &core.ApiResourceOutput{Body: connections}, nil
}
/*
GET /plugins/github/connections/:connectionId
*/
func GetConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) {
- // RETURN ONLY 1 SOURCE FROM ENV (Ignore ID until multi-connection is developed.)
- v := config.GetConfig()
connection := &models.GithubConnection{}
- err := helper.EncodeStruct(v, connection, "env")
+ err := connectionHelper.First(connection, input.Params)
if err != nil {
return nil, err
}
- response := &models.GithubResponse{
- GithubConnection: *connection,
- Name: "Github",
- ID: 1,
- }
- return &core.ApiResourceOutput{Body: response}, nil
+ return &core.ApiResourceOutput{Body: connection}, err
}
diff --git a/plugins/github/api/init.go b/plugins/github/api/init.go
new file mode 100644
index 00000000..aef88dcb
--- /dev/null
+++ b/plugins/github/api/init.go
@@ -0,0 +1,42 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package api
+
+import (
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/github/models"
+ "github.com/apache/incubator-devlake/plugins/helper"
+ "github.com/go-playground/validator/v10"
+ "github.com/spf13/viper"
+ "gorm.io/gorm"
+)
+
+type ApiUserPublicEmailResponse []models.PublicEmail
+
+var vld *validator.Validate
+var connectionHelper *helper.ConnectionApiHelper
+var basicRes core.BasicRes
+
+func Init(config *viper.Viper, logger core.Logger, database *gorm.DB) {
+ basicRes = helper.NewDefaultBasicRes(config, logger, database)
+ vld = validator.New()
+ connectionHelper = helper.NewConnectionHelper(
+ basicRes,
+ vld,
+ )
+}
diff --git a/plugins/github/github.go b/plugins/github/github.go
index 765a4d28..93ddf23a 100644
--- a/plugins/github/github.go
+++ b/plugins/github/github.go
@@ -19,10 +19,10 @@ package main // must be main for plugin entry point
import (
"fmt"
-
"github.com/apache/incubator-devlake/migration"
"github.com/apache/incubator-devlake/plugins/core"
"github.com/apache/incubator-devlake/plugins/github/api"
+ "github.com/apache/incubator-devlake/plugins/github/models"
"github.com/apache/incubator-devlake/plugins/github/models/migrationscripts"
"github.com/apache/incubator-devlake/plugins/github/tasks"
"github.com/apache/incubator-devlake/runner"
@@ -41,6 +41,7 @@ var _ core.Migratable = (*Github)(nil)
type Github struct{}
func (plugin Github) Init(config *viper.Viper, logger core.Logger, db *gorm.DB) error {
+ api.Init(config, logger, db)
return nil
}
@@ -95,7 +96,16 @@ func (plugin Github) PrepareTaskData(taskCtx core.TaskContext, options map[strin
if op.Repo == "" {
return nil, fmt.Errorf("repo is required for GitHub execution")
}
- apiClient, err := tasks.CreateApiClient(taskCtx)
+
+ // find the only github now
+ // FIXME to query by connection id when multi connection support
+ connection := &models.GithubConnection{}
+ err = taskCtx.GetDb().First(connection, "name = ?", `GitHub`).Error
+ if err != nil {
+ return err, nil
+ }
+
+ apiClient, err := tasks.CreateApiClient(taskCtx, connection)
if err != nil {
return nil, err
}
@@ -113,7 +123,7 @@ func (plugin Github) RootPkgPath() string {
func (plugin Github) MigrationScripts() []migration.Script {
return []migration.Script{
new(migrationscripts.InitSchemas), new(migrationscripts.UpdateSchemas20220509),
- new(migrationscripts.UpdateSchemas20220524),
+ new(migrationscripts.UpdateSchemas20220524), new(migrationscripts.UpdateSchemas20220608),
}
}
@@ -123,11 +133,13 @@ func (plugin Github) ApiResources() map[string]map[string]core.ApiResourceHandle
"POST": api.TestConnection,
},
"connections": {
- "GET": api.ListConnections,
+ "POST": api.PostConnections,
+ "GET": api.ListConnections,
},
"connections/:connectionId": {
- "GET": api.GetConnection,
- "PATCH": api.PatchConnection,
+ "GET": api.GetConnection,
+ "PATCH": api.PatchConnection,
+ "DELETE": api.DeleteConnection,
},
}
}
diff --git a/plugins/github/models/connection.go b/plugins/github/models/connection.go
index b100c547..34303c91 100644
--- a/plugins/github/models/connection.go
+++ b/plugins/github/models/connection.go
@@ -17,31 +17,32 @@ limitations under the License.
package models
-type GithubConnection struct {
- Endpoint string `mapstructure:"endpoint" validate:"required" env:"GITHUB_ENDPOINT" json:"endpoint"`
- Auth string `mapstructure:"auth" validate:"required" env:"GITHUB_AUTH" json:"auth"`
- Proxy string `mapstructure:"proxy" env:"GITHUB_PROXY" json:"proxy"`
+import "github.com/apache/incubator-devlake/plugins/helper"
- Config `mapstructure:",squash"`
+type TestConnectionRequest struct {
+ Endpoint string `json:"endpoint" validate:"required,url"`
+ Auth string `json:"auth" validate:"required"`
+ Proxy string `json:"proxy"`
}
-type Config struct {
- PrType string `mapstructure:"prType" env:"GITHUB_PR_TYPE" json:"prType"`
- PrComponent string `mapstructure:"prComponent" env:"GITHUB_PR_COMPONENT" json:"prComponent"`
- IssueSeverity string `mapstructure:"issueSeverity" env:"GITHUB_ISSUE_SEVERITY" json:"issueSeverity"`
- IssuePriority string `mapstructure:"issuePriority" env:"GITHUB_ISSUE_PRIORITY" json:"issuePriority"`
- IssueComponent string `mapstructure:"issueComponent" env:"GITHUB_ISSUE_COMPONENT" json:"issueComponent"`
- IssueTypeBug string `mapstructure:"issueTypeBug" env:"GITHUB_ISSUE_TYPE_BUG" json:"issueTypeBug"`
- IssueTypeIncident string `mapstructure:"issueTypeIncident" env:"GITHUB_ISSUE_TYPE_INCIDENT" json:"issueTypeIncident"`
- IssueTypeRequirement string `mapstructure:"issueTypeRequirement" env:"GITHUB_ISSUE_TYPE_REQUIREMENT" json:"issueTypeRequirement"`
+type GithubConnection struct {
+ helper.RestConnection `mapstructure:",squash"`
+ helper.AccessToken `mapstructure:",squash"`
}
-// This object conforms to what the frontend currently expects.
-type GithubResponse struct {
- Name string `json:"name"`
- ID int `json:"id"`
+type Config struct {
+ PrType string `mapstructure:"prType" json:"prType"`
+ PrComponent string `mapstructure:"prComponent" json:"prComponent"`
+ IssueSeverity string `mapstructure:"issueSeverity" json:"issueSeverity"`
+ IssuePriority string `mapstructure:"issuePriority" json:"issuePriority"`
+ IssueComponent string `mapstructure:"issueComponent" json:"issueComponent"`
+ IssueTypeBug string `mapstructure:"issueTypeBug" json:"issueTypeBug"`
+ IssueTypeIncident string `mapstructure:"issueTypeIncident" json:"issueTypeIncident"`
+ IssueTypeRequirement string `mapstructure:"issueTypeRequirement" json:"issueTypeRequirement"`
+}
- GithubConnection
+func (GithubConnection) TableName() string {
+ return "_tool_github_connections"
}
// Using Public Email because it requires authentication, and it is public information anyway.
@@ -52,9 +53,3 @@ type PublicEmail struct {
Verified bool
Visibility string
}
-
-type TestConnectionRequest struct {
- Endpoint string `json:"endpoint" validate:"required,url"`
- Auth string `json:"auth" validate:"required"`
- Proxy string `json:"proxy"`
-}
diff --git a/plugins/github/models/connection.go b/plugins/github/models/migrationscripts/updateSchemas20220608.go
similarity index 52%
copy from plugins/github/models/connection.go
copy to plugins/github/models/migrationscripts/updateSchemas20220608.go
index b100c547..679579b4 100644
--- a/plugins/github/models/connection.go
+++ b/plugins/github/models/migrationscripts/updateSchemas20220608.go
@@ -15,17 +15,28 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
-package models
+package migrationscripts
-type GithubConnection struct {
- Endpoint string `mapstructure:"endpoint" validate:"required" env:"GITHUB_ENDPOINT" json:"endpoint"`
- Auth string `mapstructure:"auth" validate:"required" env:"GITHUB_AUTH" json:"auth"`
- Proxy string `mapstructure:"proxy" env:"GITHUB_PROXY" json:"proxy"`
+import (
+ "context"
+ "github.com/apache/incubator-devlake/config"
+ "github.com/apache/incubator-devlake/models/migrationscripts/archived"
+ "github.com/apache/incubator-devlake/plugins/helper"
+ "gorm.io/gorm"
+)
- Config `mapstructure:",squash"`
+type GithubConnection20220608 struct {
+ archived.Model
+ Name string `gorm:"type:varchar(100);uniqueIndex" json:"name" validate:"required"`
+ Endpoint string `mapstructure:"endpoint" env:"GITHUB_ENDPOINT" validate:"required"`
+ Proxy string `mapstructure:"proxy" env:"GITHUB_PROXY"`
+ RateLimit int `comment:"api request rate limit per hour"`
+ Token string `mapstructure:"token" validate:"required" env:"GITHUB_AUTH"`
+
+ Config20220608 `mapstructure:",squash"`
}
-type Config struct {
+type Config20220608 struct {
PrType string `mapstructure:"prType" env:"GITHUB_PR_TYPE" json:"prType"`
PrComponent string `mapstructure:"prComponent" env:"GITHUB_PR_COMPONENT" json:"prComponent"`
IssueSeverity string `mapstructure:"issueSeverity" env:"GITHUB_ISSUE_SEVERITY" json:"issueSeverity"`
@@ -36,25 +47,35 @@ type Config struct {
IssueTypeRequirement string `mapstructure:"issueTypeRequirement" env:"GITHUB_ISSUE_TYPE_REQUIREMENT" json:"issueTypeRequirement"`
}
-// This object conforms to what the frontend currently expects.
-type GithubResponse struct {
- Name string `json:"name"`
- ID int `json:"id"`
+func (GithubConnection20220608) TableName() string {
+ return "_tool_github_connections"
+}
+
+type UpdateSchemas20220608 struct{}
- GithubConnection
+func (*UpdateSchemas20220608) Up(ctx context.Context, db *gorm.DB) error {
+ err := db.Migrator().CreateTable(GithubConnection20220608{})
+ if err != nil {
+ return err
+ }
+ v := config.GetConfig()
+ connection := &GithubConnection20220608{}
+ err = helper.EncodeStruct(v, connection, "env")
+ connection.Name = `GitHub`
+ if err != nil {
+ return err
+ }
+ // update from .env and save to db
+ if connection.Endpoint != `` && connection.Token != `` {
+ db.Create(connection)
+ }
+ return nil
}
-// Using Public Email because it requires authentication, and it is public information anyway.
-// We're not using email information for anything here.
-type PublicEmail struct {
- Email string
- Primary bool
- Verified bool
- Visibility string
+func (*UpdateSchemas20220608) Version() uint64 {
+ return 20220608000003
}
-type TestConnectionRequest struct {
- Endpoint string `json:"endpoint" validate:"required,url"`
- Auth string `json:"auth" validate:"required"`
- Proxy string `json:"proxy"`
+func (*UpdateSchemas20220608) Name() string {
+ return "Add connection for github"
}
diff --git a/plugins/github/tasks/api_client.go b/plugins/github/tasks/api_client.go
index aa0e9aa0..3c0a5c7f 100644
--- a/plugins/github/tasks/api_client.go
+++ b/plugins/github/tasks/api_client.go
@@ -19,6 +19,7 @@ package tasks
import (
"fmt"
+ "github.com/apache/incubator-devlake/plugins/github/models"
"net/http"
"strconv"
"strings"
@@ -26,28 +27,14 @@ import (
"github.com/apache/incubator-devlake/plugins/core"
"github.com/apache/incubator-devlake/plugins/helper"
- "github.com/apache/incubator-devlake/utils"
)
-func CreateApiClient(taskCtx core.TaskContext) (*helper.ApiAsyncClient, error) {
+func CreateApiClient(taskCtx core.TaskContext, connection *models.GithubConnection) (*helper.ApiAsyncClient, error) {
// load configuration
- endpoint := taskCtx.GetConfig("GITHUB_ENDPOINT")
- if endpoint == "" {
- return nil, fmt.Errorf("endpint is required")
- }
- proxy := taskCtx.GetConfig("GITHUB_PROXY")
- userRateLimit, err := utils.StrToIntOr(taskCtx.GetConfig("GITHUB_API_REQUESTS_PER_HOUR"), 0)
- if err != nil {
- return nil, err
- }
- auth := taskCtx.GetConfig("GITHUB_AUTH")
- if auth == "" {
- return nil, fmt.Errorf("GITHUB_AUTH is required")
- }
- tokens := strings.Split(auth, ",")
+ tokens := strings.Split(connection.Token, ",")
tokenIndex := 0
// create synchronize api client so we can calculate api rate limit dynamically
- apiClient, err := helper.NewApiClient(endpoint, nil, 0, proxy, taskCtx.GetContext())
+ apiClient, err := helper.NewApiClient(connection.Endpoint, nil, 0, connection.Proxy, taskCtx.GetContext())
if err != nil {
return nil, err
}
@@ -67,7 +54,7 @@ func CreateApiClient(taskCtx core.TaskContext) (*helper.ApiAsyncClient, error) {
// create rate limit calculator
rateLimiter := &helper.ApiRateLimitCalculator{
- UserRateLimitPerHour: userRateLimit,
+ UserRateLimitPerHour: connection.RateLimit,
Method: http.MethodGet,
DynamicRateLimit: func(res *http.Response) (int, time.Duration, error) {
/* calculate by number of remaining requests
diff --git a/plugins/github/tasks/issue_extractor.go b/plugins/github/tasks/issue_extractor.go
index f11a608e..8581112f 100644
--- a/plugins/github/tasks/issue_extractor.go
+++ b/plugins/github/tasks/issue_extractor.go
@@ -72,44 +72,26 @@ func ExtractApiIssues(taskCtx core.SubTaskContext) error {
var issueTypeRequirementRegex *regexp.Regexp
var issueTypeIncidentRegex *regexp.Regexp
var issueSeverity = config.IssueSeverity
- if issueSeverity == "" {
- issueSeverity = taskCtx.GetConfig("GITHUB_ISSUE_SEVERITY")
- }
- var issueComponent = config.IssueComponent
- if issueComponent == "" {
- issueComponent = taskCtx.GetConfig("GITHUB_ISSUE_COMPONENT")
- }
- var issuePriority = config.IssuePriority
- if issuePriority == "" {
- issuePriority = taskCtx.GetConfig("GITHUB_ISSUE_PRIORITY")
- }
- var issueTypeBug = config.IssueTypeBug
- if issueTypeBug == "" {
- issueTypeBug = taskCtx.GetConfig("GITHUB_ISSUE_TYPE_BUG")
- }
- var issueTypeRequirement = config.IssueTypeRequirement
- if issueTypeRequirement == "" {
- issueTypeRequirement = taskCtx.GetConfig("GITHUB_ISSUE_TYPE_REQUIREMENT")
- }
- var issueTypeIncident = config.IssueTypeIncident
- if issueTypeIncident == "" {
- issueTypeIncident = taskCtx.GetConfig("GITHUB_ISSUE_TYPE_INCIDENT")
- }
if len(issueSeverity) > 0 {
issueSeverityRegex = regexp.MustCompile(issueSeverity)
}
+ var issueComponent = config.IssueComponent
if len(issueComponent) > 0 {
issueComponentRegex = regexp.MustCompile(issueComponent)
}
+ var issuePriority = config.IssuePriority
if len(issuePriority) > 0 {
issuePriorityRegex = regexp.MustCompile(issuePriority)
}
+ var issueTypeBug = config.IssueTypeBug
if len(issueTypeBug) > 0 {
issueTypeBugRegex = regexp.MustCompile(issueTypeBug)
}
+ var issueTypeRequirement = config.IssueTypeRequirement
if len(issueTypeRequirement) > 0 {
issueTypeRequirementRegex = regexp.MustCompile(issueTypeRequirement)
}
+ var issueTypeIncident = config.IssueTypeIncident
if len(issueTypeIncident) > 0 {
issueTypeIncidentRegex = regexp.MustCompile(issueTypeIncident)
}
diff --git a/plugins/github/tasks/pr_extractor.go b/plugins/github/tasks/pr_extractor.go
index db6e18be..69ba2b65 100644
--- a/plugins/github/tasks/pr_extractor.go
+++ b/plugins/github/tasks/pr_extractor.go
@@ -72,16 +72,10 @@ func ExtractApiPullRequests(taskCtx core.SubTaskContext) error {
var labelTypeRegex *regexp.Regexp
var labelComponentRegex *regexp.Regexp
var prType = config.PrType
- if prType == "" {
- prType = taskCtx.GetConfig("GITHUB_PR_TYPE")
- }
- var prComponent = config.PrComponent
- if prComponent == "" {
- prComponent = taskCtx.GetConfig("GITHUB_PR_COMPONENT")
- }
if len(prType) > 0 {
labelTypeRegex = regexp.MustCompile(prType)
}
+ var prComponent = config.PrComponent
if len(prComponent) > 0 {
labelComponentRegex = regexp.MustCompile(prComponent)
}
diff --git a/plugins/helper/connection.go b/plugins/helper/connection.go
index 26379be5..f0c3f8d8 100644
--- a/plugins/helper/connection.go
+++ b/plugins/helper/connection.go
@@ -53,7 +53,7 @@ type RestConnection struct {
BaseConnection `mapstructure:",squash"`
Endpoint string `mapstructure:"endpoint" validate:"required" json:"endpoint"`
Proxy string `mapstructure:"proxy" json:"proxy"`
- RateLimit int `comment:"api request rate limt per hour" json:"rateLimit"`
+ RateLimit int `comment:"api request rate limit per hour" json:"rateLimit"`
}
type ConnectionApiHelper struct {
diff --git a/plugins/icla/plugin_main.go b/plugins/icla/plugin_main.go
index 9e6271bb..8d43f3d8 100644
--- a/plugins/icla/plugin_main.go
+++ b/plugins/icla/plugin_main.go
@@ -89,15 +89,8 @@ func (plugin Icla) ApiResources() map[string]map[string]core.ApiResourceHandler
// standalone mode for debugging
func main() {
cmd := &cobra.Command{Use: "icla"}
-
- // TODO add your cmd flag if necessary
- // yourFlag := cmd.Flags().IntP("yourFlag", "y", 8, "TODO add description here")
- // _ = cmd.MarkFlagRequired("yourFlag")
-
cmd.Run = func(cmd *cobra.Command, args []string) {
- runner.DirectRun(cmd, args, PluginEntry, map[string]interface{}{
- // TODO add more custom params here
- })
+ runner.DirectRun(cmd, args, PluginEntry, map[string]interface{}{})
}
runner.RunCmd(cmd)
}