You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by kl...@apache.org on 2023/01/30 03:13:09 UTC
[incubator-devlake] branch main updated: Multiple Authorization Methods Support Mechanism (JIRA supports PAT/BasicAuth) (#4260)
This is an automated email from the ASF dual-hosted git repository.
klesh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 77feb2f5d Multiple Authorization Methods Support Mechanism (JIRA supports PAT/BasicAuth) (#4260)
77feb2f5d is described below
commit 77feb2f5d864f153e623504788ea731bcb098126
Author: Klesh Wong <zh...@merico.dev>
AuthorDate: Mon Jan 30 11:13:04 2023 +0800
Multiple Authorization Methods Support Mechanism (JIRA supports PAT/BasicAuth) (#4260)
* chore: add trim trailing whitespace
* feat: jira supports multiple authentication methods
* fix: lint
* fix: JiraConnection.AuthMethod should default to `BasicAuth`
---
.editorconfig | 3 +
backend/helpers/pluginhelper/api/api_client.go | 34 +++-
.../api/apihelperabstract/connection_abstract.go | 77 +++++++++
backend/helpers/pluginhelper/api/connection.go | 174 ++-------------------
.../helpers/pluginhelper/api/connection_auths.go | 167 ++++++++++++++++++++
.../api/{connection.go => connection_helper.go} | 111 +++----------
.../helpers/pluginhelper/api/connection_test.go | 110 -------------
backend/plugins/ae/api/connection.go | 26 +--
backend/plugins/ae/models/connection.go | 42 ++++-
.../migrationscripts/20220714_add_init_tables.go | 7 -
.../models/migrationscripts/archived/connection.go | 2 +-
backend/plugins/ae/tasks/api_client.go | 22 +--
backend/plugins/azure/models/connection.go | 1 +
backend/plugins/bitbucket/api/blueprint_test.go | 21 +--
backend/plugins/bitbucket/models/connection.go | 1 +
backend/plugins/feishu/models/connection.go | 1 +
.../migrationscripts/20220714_add_init_tables.go | 7 -
backend/plugins/gitee/api/blueprint_test.go | 19 +--
backend/plugins/gitee/models/connection.go | 1 +
backend/plugins/github/api/blueprint_V200_test.go | 15 +-
backend/plugins/github/api/blueprint_test.go | 19 +--
backend/plugins/github/models/connection.go | 1 +
.../migrationscripts/20220715_add_init_tables.go | 7 -
backend/plugins/gitlab/api/blueprint_V200_test.go | 29 ++--
backend/plugins/gitlab/api/blueprint_test.go | 31 ++--
backend/plugins/gitlab/api/connection.go | 20 +--
backend/plugins/gitlab/models/connection.go | 13 +-
backend/plugins/jenkins/api/blueprint_v100_test.go | 15 +-
backend/plugins/jenkins/models/connection.go | 1 +
backend/plugins/jira/api/connection.go | 38 ++---
backend/plugins/jira/models/connection.go | 26 ++-
.../migrationscripts/20230129_add_multi_auth.go | 57 +++++++
.../jira/models/migrationscripts/register.go | 1 +
backend/plugins/sonarqube/models/connection.go | 1 +
backend/plugins/tapd/models/connection.go | 1 +
backend/plugins/zentao/models/connection.go | 1 +
36 files changed, 546 insertions(+), 556 deletions(-)
diff --git a/.editorconfig b/.editorconfig
index 91adc1af3..7006295bf 100644
--- a/.editorconfig
+++ b/.editorconfig
@@ -1,3 +1,6 @@
+[*]
+trim_trailing_whitespace = true
+
[*.go]
indent_style = tab
indent_size = 4
diff --git a/backend/helpers/pluginhelper/api/api_client.go b/backend/helpers/pluginhelper/api/api_client.go
index ba11abf45..4b403f61f 100644
--- a/backend/helpers/pluginhelper/api/api_client.go
+++ b/backend/helpers/pluginhelper/api/api_client.go
@@ -23,11 +23,6 @@ import (
"crypto/tls"
"encoding/json"
"fmt"
- "github.com/apache/incubator-devlake/core/context"
- "github.com/apache/incubator-devlake/core/errors"
- "github.com/apache/incubator-devlake/core/log"
- "github.com/apache/incubator-devlake/core/utils"
- "github.com/apache/incubator-devlake/helpers/pluginhelper/common"
"io"
"net/http"
"net/url"
@@ -35,6 +30,13 @@ import (
"strings"
"time"
"unicode/utf8"
+
+ "github.com/apache/incubator-devlake/core/context"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/log"
+ "github.com/apache/incubator-devlake/core/utils"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/api/apihelperabstract"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/common"
)
// ErrIgnoreAndContinue is a error which should be ignored
@@ -60,7 +62,27 @@ type ApiClient struct {
logger log.Logger
}
-// NewApiClient FIXME ...
+// NewApiClientFromConnection creates ApiClient based on given connection.
+// The connection must
+func NewApiClientFromConnection(
+ ctx gocontext.Context,
+ br context.BasicRes,
+ connection apihelperabstract.ApiConnection,
+) (*ApiClient, errors.Error) {
+ apiClient, err := NewApiClient(ctx, connection.GetEndpoint(), nil, 0, connection.GetProxy(), br)
+ if err != nil {
+ return nil, err
+ }
+ // if connection requires authorization
+ if authenticator, ok := connection.(apihelperabstract.ApiAuthenticator); ok {
+ apiClient.SetBeforeFunction(func(req *http.Request) errors.Error {
+ return authenticator.SetupAuthentication(req)
+ })
+ }
+ return apiClient, nil
+}
+
+// NewApiClient creates a new synchronize ApiClient
func NewApiClient(
ctx gocontext.Context,
endpoint string,
diff --git a/backend/helpers/pluginhelper/api/apihelperabstract/connection_abstract.go b/backend/helpers/pluginhelper/api/apihelperabstract/connection_abstract.go
new file mode 100644
index 000000000..273cf90c6
--- /dev/null
+++ b/backend/helpers/pluginhelper/api/apihelperabstract/connection_abstract.go
@@ -0,0 +1,77 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package apihelperabstract
+
+import (
+ "net/http"
+
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/go-playground/validator/v10"
+)
+
+// ApiConnection represents a API Connection
+type ApiConnection interface {
+ GetEndpoint() string
+ GetProxy() string
+ GetRateLimitPerHour() int
+}
+
+// ApiAuthenticator is to be implemented by a Concreate Connection if Authorization is required
+type ApiAuthenticator interface {
+ // SetupAuthentication is a hook function for connection to set up authentication for the HTTP request
+ // before sending it to the server
+ SetupAuthentication(request *http.Request) errors.Error
+}
+
+// ConnectionValidator represents the API Connection would validate its fields with customized logic
+type ConnectionValidator interface {
+ ValidateConnection(connection interface{}, valdator *validator.Validate) errors.Error
+}
+
+// MultiAuth
+const (
+ AUTH_METHOD_BASIC = "BasicAuth"
+ AUTH_METHOD_TOKEN = "AccessToken"
+ AUTH_METHOD_APPKEY = "AppKey"
+)
+
+var ALL_AUTH = map[string]bool{
+ AUTH_METHOD_BASIC: true,
+ AUTH_METHOD_TOKEN: true,
+ AUTH_METHOD_APPKEY: true,
+}
+
+// MultiAuthenticator represents the API Connection supports multiple authorization methods
+type MultiAuthenticator interface {
+ GetAuthMethod() string
+}
+
+// BasicAuthenticator represents HTTP Basic Authentication
+type BasicAuthenticator interface {
+ GetBasicAuthenticator() ApiAuthenticator
+}
+
+// AccessTokenAuthenticator represents HTTP Bearer Authentication with Access Token
+type AccessTokenAuthenticator interface {
+ GetAccessTokenAuthenticator() ApiAuthenticator
+}
+
+// AppKeyAuthenticator represents the API Key and Secret authentication mechanism
+type AppKeyAuthenticator interface {
+ GetAppKeyAuthenticator() ApiAuthenticator
+}
diff --git a/backend/helpers/pluginhelper/api/connection.go b/backend/helpers/pluginhelper/api/connection.go
index a8843ae1d..bb05074df 100644
--- a/backend/helpers/pluginhelper/api/connection.go
+++ b/backend/helpers/pluginhelper/api/connection.go
@@ -18,185 +18,33 @@ limitations under the License.
package api
import (
- "encoding/base64"
- "fmt"
- "github.com/apache/incubator-devlake/core/context"
- "github.com/apache/incubator-devlake/core/dal"
- "github.com/apache/incubator-devlake/core/errors"
- "github.com/apache/incubator-devlake/core/log"
"github.com/apache/incubator-devlake/core/models/common"
- "github.com/apache/incubator-devlake/core/plugin"
- "reflect"
- "strconv"
- "strings"
-
- "github.com/go-playground/validator/v10"
)
-// BaseConnection FIXME ...
+// BaseConnection defines basic properties that every connection should have
type BaseConnection struct {
Name string `gorm:"type:varchar(100);uniqueIndex" json:"name" validate:"required"`
common.Model
}
-// BasicAuth FIXME ...
-type BasicAuth struct {
- Username string `mapstructure:"username" validate:"required" json:"username"`
- Password string `mapstructure:"password" validate:"required" json:"password" gorm:"serializer:encdec"`
-}
-
-// GetEncodedToken FIXME ...
-func (ba BasicAuth) GetEncodedToken() string {
- return base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%v:%v", ba.Username, ba.Password)))
-}
-
-// AccessToken FIXME ...
-type AccessToken struct {
- Token string `mapstructure:"token" validate:"required" json:"token" gorm:"serializer:encdec"`
-}
-
-// AppKey FIXME ...
-type AppKey struct {
- AppId string `mapstructure:"app_id" validate:"required" json:"appId"`
- SecretKey string `mapstructure:"secret_key" validate:"required" json:"secretKey" encrypt:"yes"`
-}
-
-// RestConnection FIXME ...
+// RestConnection implements the ApiConnection interface
type RestConnection struct {
- BaseConnection `mapstructure:",squash"`
Endpoint string `mapstructure:"endpoint" validate:"required" json:"endpoint"`
Proxy string `mapstructure:"proxy" json:"proxy"`
RateLimitPerHour int `comment:"api request rate limit per hour" json:"rateLimitPerHour"`
}
-// ConnectionApiHelper is used to write the CURD of connection
-type ConnectionApiHelper struct {
- encKey string
- log log.Logger
- db dal.Dal
- validator *validator.Validate
-}
-
-// NewConnectionHelper FIXME ...
-func NewConnectionHelper(
- basicRes context.BasicRes,
- vld *validator.Validate,
-) *ConnectionApiHelper {
- if vld == nil {
- vld = validator.New()
- }
- return &ConnectionApiHelper{
- encKey: basicRes.GetConfig(plugin.EncodeKeyEnvStr),
- log: basicRes.GetLogger(),
- db: basicRes.GetDal(),
- validator: vld,
- }
-}
-
-// Create a connection record based on request body
-func (c *ConnectionApiHelper) Create(connection interface{}, input *plugin.ApiResourceInput) errors.Error {
- // update fields from request body
- err := c.merge(connection, input.Body)
- if err != nil {
- return err
- }
- return c.save(connection)
-}
-
-// Patch (Modify) a connection record based on request body
-func (c *ConnectionApiHelper) Patch(connection interface{}, input *plugin.ApiResourceInput) errors.Error {
- err := c.First(connection, input.Params)
- if err != nil {
- return err
- }
-
- err = c.merge(connection, input.Body)
- if err != nil {
- return err
- }
- return c.save(connection)
-}
-
-// First finds connection from db by parsing request input and decrypt it
-func (c *ConnectionApiHelper) First(connection interface{}, params map[string]string) errors.Error {
- connectionId := params["connectionId"]
- if connectionId == "" {
- return errors.BadInput.New("missing connectionId")
- }
- id, err := strconv.ParseUint(connectionId, 10, 64)
- if err != nil || id < 1 {
- return errors.BadInput.New("invalid connectionId")
- }
- return c.FirstById(connection, id)
-}
-
-// FirstById finds connection from db by id and decrypt it
-func (c *ConnectionApiHelper) FirstById(connection interface{}, id uint64) errors.Error {
- return c.db.First(connection, dal.Where("id = ?", id))
-}
-
-// List returns all connections with password/token decrypted
-func (c *ConnectionApiHelper) List(connections interface{}) errors.Error {
- return c.db.All(connections)
-}
-
-// Delete connection
-func (c *ConnectionApiHelper) Delete(connection interface{}) errors.Error {
- return c.db.Delete(connection)
-}
-
-func (c *ConnectionApiHelper) merge(connection interface{}, body map[string]interface{}) errors.Error {
- return Decode(body, connection, c.validator)
+// GetEndpoint returns the API endpoint of the connection
+func (rc RestConnection) GetEndpoint() string {
+ return rc.Endpoint
}
-func (c *ConnectionApiHelper) save(connection interface{}) errors.Error {
- err := c.db.CreateOrUpdate(connection)
- if err != nil {
- if strings.Contains(strings.ToLower(err.Error()), "duplicate") {
- return errors.BadInput.Wrap(err, "duplicated Connection Name")
- }
- return err
- }
- return nil
+// GetProxy returns the proxy for the connection
+func (rc RestConnection) GetProxy() string {
+ return rc.Proxy
}
-// UpdateEncryptFields update fields of val with tag `encrypt:"yes|true"`
-func UpdateEncryptFields(val interface{}, update func(in string) (string, errors.Error)) errors.Error {
- v := reflect.ValueOf(val)
- if v.Kind() != reflect.Ptr {
- panic(errors.Default.New(fmt.Sprintf("val is not a pointer: %v", val)))
- }
- e := v.Elem()
- if e.Kind() != reflect.Struct {
- panic(errors.Default.New(fmt.Sprintf("*val is not a struct: %v", val)))
- }
- t := e.Type()
- for i := 0; i < t.NumField(); i++ {
- field := t.Field(i)
- if !field.IsExported() {
- continue
- }
- if field.Type.Kind() == reflect.Struct {
- err := UpdateEncryptFields(e.Field(i).Addr().Interface(), update)
- if err != nil {
- return err
- }
- } else if field.Type.Kind() == reflect.Ptr && field.Type.Elem().Kind() == reflect.Struct {
- fmt.Printf("field : %v\n", e.Field(i).Interface())
- err := UpdateEncryptFields(e.Field(i).Interface(), update)
- if err != nil {
- return err
- }
- } else if field.Type.Kind() == reflect.String {
- tagValue := field.Tag.Get("encrypt")
- if tagValue == "yes" || tagValue == "true" {
- out, err := update(e.Field(i).String())
- if err != nil {
- return err
- }
- e.Field(i).Set(reflect.ValueOf(out))
- }
- }
- }
- return nil
+// GetProxy returns the Rate Limit for the connection
+func (rc RestConnection) GetRateLimitPerHour() int {
+ return rc.RateLimitPerHour
}
diff --git a/backend/helpers/pluginhelper/api/connection_auths.go b/backend/helpers/pluginhelper/api/connection_auths.go
new file mode 100644
index 000000000..6e43c31c2
--- /dev/null
+++ b/backend/helpers/pluginhelper/api/connection_auths.go
@@ -0,0 +1,167 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package api
+
+import (
+ "encoding/base64"
+ "fmt"
+ "net/http"
+ "strings"
+
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/api/apihelperabstract"
+ "github.com/go-playground/validator/v10"
+)
+
+// BasicAuth implements HTTP Basic Authentication
+type BasicAuth struct {
+ Username string `mapstructure:"username" validate:"required" json:"username"`
+ Password string `mapstructure:"password" validate:"required" json:"password" gorm:"serializer:encdec"`
+}
+
+// GetEncodedToken returns encoded bearer token for HTTP Basic Authentication
+func (ba BasicAuth) GetEncodedToken() string {
+ return base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%v:%v", ba.Username, ba.Password)))
+}
+
+// SetupAuthentication sets up the request headers for authentication
+func (ba BasicAuth) SetupAuthentication(request *http.Request) errors.Error {
+ request.Header.Set("Authorization", fmt.Sprintf("Basic %v", ba.GetEncodedToken()))
+ return nil
+}
+
+// GetBasicAuthenticator returns the ApiAuthenticator for setting up the HTTP request
+// it looks odd to return itself with a different type, this is necessary because Callers
+// might call the method from the Outer-Struct(`connection.SetupAuthentication(...)`)
+// which would lead to a Stack Overflow error
+func (ba BasicAuth) GetBasicAuthenticator() apihelperabstract.ApiAuthenticator {
+ return ba
+}
+
+// AccessToken implements HTTP Bearer Authentication with Access Token
+type AccessToken struct {
+ Token string `mapstructure:"token" validate:"required" json:"token" gorm:"serializer:encdec"`
+}
+
+// SetupAuthentication sets up the request headers for authentication
+func (at AccessToken) SetupAuthentication(request *http.Request) errors.Error {
+ request.Header.Set("Authorization", fmt.Sprintf("Bearer %v", at.Token))
+ return nil
+}
+
+// GetAccessTokenAuthenticator returns SetupAuthentication
+func (at AccessToken) GetAccessTokenAuthenticator() apihelperabstract.ApiAuthenticator {
+ return at
+}
+
+// AppKey implements the API Key and Secret authentication mechanism
+type AppKey struct {
+ AppId string `mapstructure:"appId" validate:"required" json:"appId"`
+ SecretKey string `mapstructure:"secretKey" validate:"required" json:"secretKey" gorm:"serializer:encdec"`
+}
+
+// SetupAuthentication sets up the request headers for authentication
+func (ak AppKey) SetupAuthentication(request *http.Request) errors.Error {
+ // no universal way to implement AppKey authentication, plugin should alias AppKey and
+ // define its own implementation
+ panic("not implemented")
+}
+
+// GetAppKeyAuthenticator returns SetupAuthentication
+func (ak AppKey) GetAppKeyAuthenticator() apihelperabstract.ApiAuthenticator {
+ // no universal way to implement AppKey authentication, plugin should alias AppKey and
+ // define its own implementation
+ panic("not implemented")
+}
+
+// MultiAuth implements the MultiAuthenticator interface
+type MultiAuth struct {
+ AuthMethod string `mapstructure:"authMethod" json:"authMethod" validate:"required,oneof=BasicAuth AccessToken AppKey"`
+ apiAuthenticator apihelperabstract.ApiAuthenticator
+}
+
+func (ma MultiAuth) GetApiAuthenticator(connection apihelperabstract.ApiConnection) (apihelperabstract.ApiAuthenticator, errors.Error) {
+ // cache the ApiAuthenticator for performance
+ if ma.apiAuthenticator != nil {
+ return ma.apiAuthenticator, nil
+ }
+ // cache missed
+ switch ma.AuthMethod {
+ case apihelperabstract.AUTH_METHOD_BASIC:
+ basicAuth, ok := connection.(apihelperabstract.BasicAuthenticator)
+ if !ok {
+ return nil, errors.Default.New("connection doesn't support Basic Authentication")
+ }
+ ma.apiAuthenticator = basicAuth.GetBasicAuthenticator()
+ case apihelperabstract.AUTH_METHOD_TOKEN:
+ accessToken, ok := connection.(apihelperabstract.AccessTokenAuthenticator)
+ if !ok {
+ return nil, errors.Default.New("connection doesn't support AccessToken Authentication")
+ }
+ ma.apiAuthenticator = accessToken.GetAccessTokenAuthenticator()
+ case apihelperabstract.AUTH_METHOD_APPKEY:
+ // Note that AppKey Authentication requires complex logic like signing the request with timestamp
+ // so, there is no way to solve them once and for all, each Specific Connection should implement
+ // on its own.
+ appKey, ok := connection.(apihelperabstract.AppKeyAuthenticator)
+ if !ok {
+ return nil, errors.Default.New("connection doesn't support AppKey Authentication")
+ }
+ // check ae/models/connection.go:AeAppKey if you needed an example
+ ma.apiAuthenticator = appKey.GetAppKeyAuthenticator()
+ default:
+ return nil, errors.Default.New("no Authentication Method was specified")
+ }
+ return ma.apiAuthenticator, nil
+}
+
+// SetupAuthenticationForConnection sets up authentication for the specified `req` based on connection
+// Specific Connection should implement IAuthentication and then call this method for MultiAuth to work properly,
+// check jira/models/connection.go:JiraConn if you needed an example
+// Note: this method would be called for each request, so it is performance-sensitive, do NOT use reflection here
+func (ma MultiAuth) SetupAuthenticationForConnection(connection apihelperabstract.ApiConnection, req *http.Request) errors.Error {
+ apiAuthenticator, err := ma.GetApiAuthenticator(connection)
+ if err != nil {
+ return err
+ }
+ return apiAuthenticator.SetupAuthentication(req)
+}
+
+func (ma MultiAuth) ValidateConnection(connection interface{}, v *validator.Validate) errors.Error {
+ // the idea is to filtered out errors from unselected Authentication struct
+ validationErrors := v.Struct(connection).(validator.ValidationErrors)
+ if validationErrors != nil {
+ filteredValidationErrors := make(validator.ValidationErrors, 0)
+ for _, e := range validationErrors {
+ // JiraConnection.JiraConn.BasicAuth.Username
+ ns := strings.Split(e.Namespace(), ".")
+ if len(ns) > 1 {
+ // BasicAuth
+ authName := ns[len(ns)-2]
+ if apihelperabstract.ALL_AUTH[authName] && authName != ma.AuthMethod {
+ continue
+ }
+ filteredValidationErrors = append(filteredValidationErrors, e)
+ }
+ }
+ if len(filteredValidationErrors) > 0 {
+ return errors.BadInput.Wrap(filteredValidationErrors, "validation failed")
+ }
+ }
+ return nil
+}
diff --git a/backend/helpers/pluginhelper/api/connection.go b/backend/helpers/pluginhelper/api/connection_helper.go
similarity index 56%
copy from backend/helpers/pluginhelper/api/connection.go
copy to backend/helpers/pluginhelper/api/connection_helper.go
index a8843ae1d..64760ed95 100644
--- a/backend/helpers/pluginhelper/api/connection.go
+++ b/backend/helpers/pluginhelper/api/connection_helper.go
@@ -18,57 +18,18 @@ limitations under the License.
package api
import (
- "encoding/base64"
- "fmt"
+ "strconv"
+ "strings"
+
"github.com/apache/incubator-devlake/core/context"
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/log"
- "github.com/apache/incubator-devlake/core/models/common"
- "github.com/apache/incubator-devlake/core/plugin"
- "reflect"
- "strconv"
- "strings"
-
+ plugin "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/api/apihelperabstract"
"github.com/go-playground/validator/v10"
)
-// BaseConnection FIXME ...
-type BaseConnection struct {
- Name string `gorm:"type:varchar(100);uniqueIndex" json:"name" validate:"required"`
- common.Model
-}
-
-// BasicAuth FIXME ...
-type BasicAuth struct {
- Username string `mapstructure:"username" validate:"required" json:"username"`
- Password string `mapstructure:"password" validate:"required" json:"password" gorm:"serializer:encdec"`
-}
-
-// GetEncodedToken FIXME ...
-func (ba BasicAuth) GetEncodedToken() string {
- return base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%v:%v", ba.Username, ba.Password)))
-}
-
-// AccessToken FIXME ...
-type AccessToken struct {
- Token string `mapstructure:"token" validate:"required" json:"token" gorm:"serializer:encdec"`
-}
-
-// AppKey FIXME ...
-type AppKey struct {
- AppId string `mapstructure:"app_id" validate:"required" json:"appId"`
- SecretKey string `mapstructure:"secret_key" validate:"required" json:"secretKey" encrypt:"yes"`
-}
-
-// RestConnection FIXME ...
-type RestConnection struct {
- BaseConnection `mapstructure:",squash"`
- Endpoint string `mapstructure:"endpoint" validate:"required" json:"endpoint"`
- Proxy string `mapstructure:"proxy" json:"proxy"`
- RateLimitPerHour int `comment:"api request rate limit per hour" json:"rateLimitPerHour"`
-}
-
// ConnectionApiHelper is used to write the CURD of connection
type ConnectionApiHelper struct {
encKey string
@@ -77,7 +38,7 @@ type ConnectionApiHelper struct {
validator *validator.Validate
}
-// NewConnectionHelper FIXME ...
+// NewConnectionHelper creates a ConnectionHelper for connection management
func NewConnectionHelper(
basicRes context.BasicRes,
vld *validator.Validate,
@@ -132,12 +93,20 @@ func (c *ConnectionApiHelper) First(connection interface{}, params map[string]st
// FirstById finds connection from db by id and decrypt it
func (c *ConnectionApiHelper) FirstById(connection interface{}, id uint64) errors.Error {
- return c.db.First(connection, dal.Where("id = ?", id))
+ err := c.db.First(connection, dal.Where("id = ?", id))
+ if err != nil {
+ return err
+ }
+ return nil
}
// List returns all connections with password/token decrypted
func (c *ConnectionApiHelper) List(connections interface{}) errors.Error {
- return c.db.All(connections)
+ err := c.db.All(connections)
+ if err != nil {
+ return err
+ }
+ return nil
}
// Delete connection
@@ -146,6 +115,13 @@ func (c *ConnectionApiHelper) Delete(connection interface{}) errors.Error {
}
func (c *ConnectionApiHelper) merge(connection interface{}, body map[string]interface{}) errors.Error {
+ if connectionValdiator, ok := connection.(apihelperabstract.ConnectionValidator); ok {
+ err := Decode(body, connection, nil)
+ if err != nil {
+ return err
+ }
+ return connectionValdiator.ValidateConnection(connection, c.validator)
+ }
return Decode(body, connection, c.validator)
}
@@ -159,44 +135,3 @@ func (c *ConnectionApiHelper) save(connection interface{}) errors.Error {
}
return nil
}
-
-// UpdateEncryptFields update fields of val with tag `encrypt:"yes|true"`
-func UpdateEncryptFields(val interface{}, update func(in string) (string, errors.Error)) errors.Error {
- v := reflect.ValueOf(val)
- if v.Kind() != reflect.Ptr {
- panic(errors.Default.New(fmt.Sprintf("val is not a pointer: %v", val)))
- }
- e := v.Elem()
- if e.Kind() != reflect.Struct {
- panic(errors.Default.New(fmt.Sprintf("*val is not a struct: %v", val)))
- }
- t := e.Type()
- for i := 0; i < t.NumField(); i++ {
- field := t.Field(i)
- if !field.IsExported() {
- continue
- }
- if field.Type.Kind() == reflect.Struct {
- err := UpdateEncryptFields(e.Field(i).Addr().Interface(), update)
- if err != nil {
- return err
- }
- } else if field.Type.Kind() == reflect.Ptr && field.Type.Elem().Kind() == reflect.Struct {
- fmt.Printf("field : %v\n", e.Field(i).Interface())
- err := UpdateEncryptFields(e.Field(i).Interface(), update)
- if err != nil {
- return err
- }
- } else if field.Type.Kind() == reflect.String {
- tagValue := field.Tag.Get("encrypt")
- if tagValue == "yes" || tagValue == "true" {
- out, err := update(e.Field(i).String())
- if err != nil {
- return err
- }
- e.Field(i).Set(reflect.ValueOf(out))
- }
- }
- }
- return nil
-}
diff --git a/backend/helpers/pluginhelper/api/connection_test.go b/backend/helpers/pluginhelper/api/connection_test.go
deleted file mode 100644
index 072a41cbe..000000000
--- a/backend/helpers/pluginhelper/api/connection_test.go
+++ /dev/null
@@ -1,110 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package api
-
-import (
- "fmt"
- "github.com/apache/incubator-devlake/core/errors"
- "testing"
- "time"
-
- "github.com/stretchr/testify/assert"
-)
-
-type MockAuth struct {
- Username string
- Password string `encrypt:"yes"`
-}
-
-type MockConnection struct {
- MockAuth
- Name string `mapstructure:"name"`
- BasicAuth string `encrypt:"true"`
- BearToken struct {
- AccessToken string `encrypt:"true"`
- }
- MockAuth2 *MockAuth
- Age int
- Since *time.Time
-}
-
-/*
-func TestMergeFieldsToConnection(t *testing.T) {
- v := &MockConnection{
- Name: "1",
- BearToken: struct {
- AccessToken string "encrypt:\"true\""
- }{
- AccessToken: "2",
- },
- MockAuth: &MockAuth{
- Username: "3",
- Password: "4",
- },
- Age: 5,
- }
- data := make(map[string]interface{})
- data["name"] = "1a"
- data["BasicAuth"] = map[string]interface{}{
- "AccessToken": "2a",
- }
- data["Username"] = "3a"
-
- err := mergeFieldsToConnection(v, data)
- assert.Nil(t, err)
-
- assert.Equal(t, "1a", v.Name)
- assert.Equal(t, "2a", v.BearToken.AccessToken)
- assert.Equal(t, "3a", v.Username)
- assert.Equal(t, "4", v.Password)
- assert.Equal(t, 5, v.Age)
-}
-*/
-
-func TestUpdateEncryptFields(t *testing.T) {
- sinc := time.Now()
- v := &MockConnection{
- MockAuth: MockAuth{
- Username: "1",
- Password: "2",
- },
- Name: "3",
- BearToken: struct {
- AccessToken string `encrypt:"true"`
- }{
- AccessToken: "4",
- },
- MockAuth2: &MockAuth{
- Username: "5",
- Password: "6",
- },
- Age: 7,
- Since: &sinc,
- }
- err := UpdateEncryptFields(v, func(in string) (string, errors.Error) {
- return fmt.Sprintf("%s-asdf", in), nil
- })
- assert.Nil(t, err)
- assert.Equal(t, "1", v.Username)
- assert.Equal(t, "2-asdf", v.Password)
- assert.Equal(t, "3", v.Name)
- assert.Equal(t, "4-asdf", v.BearToken.AccessToken)
- assert.Equal(t, "5", v.MockAuth2.Username)
- assert.Equal(t, "6-asdf", v.MockAuth2.Password)
- assert.Equal(t, 7, v.Age)
-}
diff --git a/backend/plugins/ae/api/connection.go b/backend/plugins/ae/api/connection.go
index 23cebeab2..91d3a0325 100644
--- a/backend/plugins/ae/api/connection.go
+++ b/backend/plugins/ae/api/connection.go
@@ -19,14 +19,13 @@ package api
import (
"context"
- "fmt"
+ "net/http"
+
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/ae/models"
_ "github.com/apache/incubator-devlake/server/api/shared"
- "net/http"
- "time"
)
type ApiMeResponse struct {
@@ -36,7 +35,7 @@ type ApiMeResponse struct {
// @Summary test ae connection
// @Description Test AE Connection
// @Tags plugins/ae
-// @Param body body models.TestConnectionRequest true "json body"
+// @Param body body models.AeConn true "json body"
// @Success 200 {object} shared.ApiBody "Success"
// @Failure 400 {string} errcode.Error "Bad Request"
// @Failure 500 {string} errcode.Error "Internal Error"
@@ -44,30 +43,15 @@ type ApiMeResponse struct {
func TestConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
// decode
var err errors.Error
- var connection models.TestConnectionRequest
+ var connection models.AeConn
if err := api.Decode(input.Body, &connection, vld); err != nil {
return nil, errors.BadInput.Wrap(err, "could not decode request parameters")
}
- // load and process cconfiguration
- endpoint := connection.Endpoint
- appId := connection.AppId
- secretKey := connection.SecretKey
- proxy := connection.Proxy
- apiClient, err := api.NewApiClient(context.TODO(), endpoint, nil, 3*time.Second, proxy, basicRes)
+ apiClient, err := api.NewApiClientFromConnection(context.TODO(), basicRes, connection)
if err != nil {
return nil, err
}
- apiClient.SetBeforeFunction(func(req *http.Request) errors.Error {
- nonceStr := plugin.RandLetterBytes(8)
- timestamp := fmt.Sprintf("%v", time.Now().Unix())
- sign := models.GetSign(req.URL.Query(), appId, secretKey, nonceStr, timestamp)
- req.Header.Set("x-ae-app-id", appId)
- req.Header.Set("x-ae-timestamp", timestamp)
- req.Header.Set("x-ae-nonce-str", nonceStr)
- req.Header.Set("x-ae-sign", sign)
- return nil
- })
res, err := apiClient.Get("projects", nil, nil)
if err != nil {
return nil, err
diff --git a/backend/plugins/ae/models/connection.go b/backend/plugins/ae/models/connection.go
index 17b7de47b..b6249e725 100644
--- a/backend/plugins/ae/models/connection.go
+++ b/backend/plugins/ae/models/connection.go
@@ -21,21 +21,47 @@ import (
"crypto/md5"
"encoding/hex"
"fmt"
- helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "net/http"
"net/url"
"sort"
"strings"
+ "time"
+
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
+ helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/api/apihelperabstract"
)
-type AeConnection struct {
+type AeAppKey helper.AppKey
+
+// SetupAuthentication sets up the HTTP Request Authentication
+func (aak AeAppKey) SetupAuthentication(req *http.Request) errors.Error {
+ nonceStr := plugin.RandLetterBytes(8)
+ timestamp := fmt.Sprintf("%v", time.Now().Unix())
+ sign := signRequest(req.URL.Query(), aak.AppId, aak.SecretKey, nonceStr, timestamp)
+ req.Header.Set("x-ae-app-id", aak.AppId)
+ req.Header.Set("x-ae-timestamp", timestamp)
+ req.Header.Set("x-ae-nonce-str", nonceStr)
+ req.Header.Set("x-ae-sign", sign)
+ return nil
+}
+
+func (aak AeAppKey) GetAppKeyAuthenticator() apihelperabstract.ApiAuthenticator {
+ return aak
+}
+
+// AeConn holds the essential information to connect to the AE API
+type AeConn struct {
helper.RestConnection `mapstructure:",squash"`
- helper.AppKey `mapstructure:",squash"`
+ AeAppKey `mapstructure:",squash"`
}
-type TestConnectionRequest struct {
- Endpoint string `json:"endpoint"`
- Proxy string `json:"proxy"`
- helper.AppKey `mapstructure:",squash"`
+// AeConnection holds AeConn plus ID/Name for database storage
+type AeConnection struct {
+ helper.BaseConnection `mapstructure:",squash"`
+ helper.RestConnection `mapstructure:",squash"`
+ AeAppKey `mapstructure:",squash"`
}
// This object conforms to what the frontend currently expects.
@@ -49,7 +75,7 @@ func (AeConnection) TableName() string {
return "_tool_ae_connections"
}
-func GetSign(query url.Values, appId, secretKey, nonceStr, timestamp string) string {
+func signRequest(query url.Values, appId, secretKey, nonceStr, timestamp string) string {
// clone query because we need to add items
kvs := make([]string, 0, len(query)+3)
kvs = append(kvs, fmt.Sprintf("app_id=%s", appId))
diff --git a/backend/plugins/ae/models/migrationscripts/20220714_add_init_tables.go b/backend/plugins/ae/models/migrationscripts/20220714_add_init_tables.go
index d65299f1c..51060c927 100644
--- a/backend/plugins/ae/models/migrationscripts/20220714_add_init_tables.go
+++ b/backend/plugins/ae/models/migrationscripts/20220714_add_init_tables.go
@@ -23,7 +23,6 @@ import (
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/migrationhelper"
- helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/ae/models/migrationscripts/archived"
)
@@ -59,12 +58,6 @@ func (u *addInitTables20220714) Up(basicRes context.BasicRes) errors.Error {
connection.AppId = c.GetString("AE_APP_ID")
connection.Name = "AE"
if connection.Endpoint != "" && connection.AppId != "" && connection.SecretKey != "" && encodeKey != "" {
- err = helper.UpdateEncryptFields(connection, func(plaintext string) (string, errors.Error) {
- return plugin.Encrypt(encodeKey, plaintext)
- })
- if err != nil {
- return err
- }
// update from .env and save to db
err = basicRes.GetDal().Create(connection)
if err != nil {
diff --git a/backend/plugins/ae/models/migrationscripts/archived/connection.go b/backend/plugins/ae/models/migrationscripts/archived/connection.go
index e76df28e5..59c328800 100644
--- a/backend/plugins/ae/models/migrationscripts/archived/connection.go
+++ b/backend/plugins/ae/models/migrationscripts/archived/connection.go
@@ -33,7 +33,7 @@ type AeConnection struct {
RateLimitPerHour int `comment:"api request rate limit per hour" json:"rateLimit"`
AppId string `mapstructure:"app_id" validate:"required" json:"app_id"`
- SecretKey string `mapstructure:"secret_key" validate:"required" json:"secret_key" encrypt:"yes"`
+ SecretKey string `mapstructure:"secret_key" validate:"required" json:"secret_key" gorm:"serializer:encdec"`
}
func (AeConnection) TableName() string {
diff --git a/backend/plugins/ae/tasks/api_client.go b/backend/plugins/ae/tasks/api_client.go
index d4f9b1f86..3cf0266df 100644
--- a/backend/plugins/ae/tasks/api_client.go
+++ b/backend/plugins/ae/tasks/api_client.go
@@ -18,36 +18,18 @@ limitations under the License.
package tasks
import (
- "fmt"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/ae/models"
- "net/http"
- "time"
)
+// CreateApiClient creates a new asynchronize API Client for AE
func CreateApiClient(taskCtx plugin.TaskContext, connection *models.AeConnection) (*api.ApiAsyncClient, errors.Error) {
- // load and process cconfiguration
- endpoint := connection.Endpoint
- appId := connection.AppId
- secretKey := connection.SecretKey
- proxy := connection.Proxy
-
- apiClient, err := api.NewApiClient(taskCtx.GetContext(), endpoint, nil, 0, proxy, taskCtx)
+ apiClient, err := api.NewApiClientFromConnection(taskCtx.GetContext(), taskCtx, connection)
if err != nil {
return nil, err
}
- apiClient.SetBeforeFunction(func(req *http.Request) errors.Error {
- nonceStr := plugin.RandLetterBytes(8)
- timestamp := fmt.Sprintf("%v", time.Now().Unix())
- sign := models.GetSign(req.URL.Query(), appId, secretKey, nonceStr, timestamp)
- req.Header.Set("x-ae-app-id", appId)
- req.Header.Set("x-ae-timestamp", timestamp)
- req.Header.Set("x-ae-nonce-str", nonceStr)
- req.Header.Set("x-ae-sign", sign)
- return nil
- })
// create ae api client
asyncApiCLient, err := api.CreateAsyncApiClient(taskCtx, apiClient, nil)
diff --git a/backend/plugins/azure/models/connection.go b/backend/plugins/azure/models/connection.go
index da32f6b25..115d6fd4e 100644
--- a/backend/plugins/azure/models/connection.go
+++ b/backend/plugins/azure/models/connection.go
@@ -23,6 +23,7 @@ import (
// This object conforms to what the frontend currently sends.
type AzureConnection struct {
+ helper.BaseConnection `mapstructure:",squash"`
helper.RestConnection `mapstructure:",squash"`
helper.BasicAuth `mapstructure:",squash"`
}
diff --git a/backend/plugins/bitbucket/api/blueprint_test.go b/backend/plugins/bitbucket/api/blueprint_test.go
index d8f6ddb5b..30bfd1be1 100644
--- a/backend/plugins/bitbucket/api/blueprint_test.go
+++ b/backend/plugins/bitbucket/api/blueprint_test.go
@@ -20,6 +20,11 @@ package api
import (
"bytes"
"encoding/json"
+ "io"
+ "net/http"
+ "path"
+ "testing"
+
"github.com/apache/incubator-devlake/core/models/common"
"github.com/apache/incubator-devlake/core/plugin"
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
@@ -29,21 +34,17 @@ import (
"github.com/apache/incubator-devlake/plugins/bitbucket/tasks"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
- "io"
- "net/http"
- "path"
- "testing"
)
func TestMakePipelinePlan(t *testing.T) {
connection := &models.BitbucketConnection{
- RestConnection: helper.RestConnection{
- BaseConnection: helper.BaseConnection{
- Name: "github-test",
- Model: common.Model{
- ID: 1,
- },
+ BaseConnection: helper.BaseConnection{
+ Name: "github-test",
+ Model: common.Model{
+ ID: 1,
},
+ },
+ RestConnection: helper.RestConnection{
Endpoint: "https://TestBitBucket/",
Proxy: "",
RateLimitPerHour: 0,
diff --git a/backend/plugins/bitbucket/models/connection.go b/backend/plugins/bitbucket/models/connection.go
index 951f87ff9..fe9d24a39 100644
--- a/backend/plugins/bitbucket/models/connection.go
+++ b/backend/plugins/bitbucket/models/connection.go
@@ -46,6 +46,7 @@ type TransformationRules struct {
}
type BitbucketConnection struct {
+ helper.BaseConnection `mapstructure:",squash"`
helper.RestConnection `mapstructure:",squash"`
helper.BasicAuth `mapstructure:",squash"`
}
diff --git a/backend/plugins/feishu/models/connection.go b/backend/plugins/feishu/models/connection.go
index 4e5bab8ac..6c5f577c3 100644
--- a/backend/plugins/feishu/models/connection.go
+++ b/backend/plugins/feishu/models/connection.go
@@ -29,6 +29,7 @@ type TestConnectionRequest struct {
}
type FeishuConnection struct {
+ helper.BaseConnection `mapstructure:",squash"`
helper.RestConnection `mapstructure:",squash"`
helper.AppKey `mapstructure:",squash"`
}
diff --git a/backend/plugins/feishu/models/migrationscripts/20220714_add_init_tables.go b/backend/plugins/feishu/models/migrationscripts/20220714_add_init_tables.go
index a31c2f320..e143e59f0 100644
--- a/backend/plugins/feishu/models/migrationscripts/20220714_add_init_tables.go
+++ b/backend/plugins/feishu/models/migrationscripts/20220714_add_init_tables.go
@@ -22,7 +22,6 @@ import (
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/migrationhelper"
- helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/feishu/models/migrationscripts/archived"
)
@@ -56,12 +55,6 @@ func (u *addInitTables) Up(basicRes context.BasicRes) errors.Error {
connection.SecretKey = basicRes.GetConfig(`FEISHU_APPSCRECT`)
connection.Name = `Feishu`
if connection.Endpoint != `` && connection.AppId != `` && connection.SecretKey != `` && encodeKey != `` {
- err = helper.UpdateEncryptFields(connection, func(plaintext string) (string, errors.Error) {
- return plugin.Encrypt(encodeKey, plaintext)
- })
- if err != nil {
- return err
- }
// update from .env and save to db
err = db.CreateIfNotExist(connection)
if err != nil {
diff --git a/backend/plugins/gitee/api/blueprint_test.go b/backend/plugins/gitee/api/blueprint_test.go
index 6c30c6d70..986872d8d 100644
--- a/backend/plugins/gitee/api/blueprint_test.go
+++ b/backend/plugins/gitee/api/blueprint_test.go
@@ -20,6 +20,10 @@ package api
import (
"bytes"
"encoding/json"
+ "io"
+ "net/http"
+ "testing"
+
"github.com/apache/incubator-devlake/core/models/common"
"github.com/apache/incubator-devlake/core/plugin"
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
@@ -29,20 +33,17 @@ import (
"github.com/apache/incubator-devlake/plugins/gitee/tasks"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
- "io"
- "net/http"
- "testing"
)
func TestMakePipelinePlan(t *testing.T) {
connection := &models.GiteeConnection{
- RestConnection: helper.RestConnection{
- BaseConnection: helper.BaseConnection{
- Name: "gitee-test",
- Model: common.Model{
- ID: 1,
- },
+ BaseConnection: helper.BaseConnection{
+ Name: "gitee-test",
+ Model: common.Model{
+ ID: 1,
},
+ },
+ RestConnection: helper.RestConnection{
Endpoint: "https://api.github.com/",
Proxy: "",
RateLimitPerHour: 0,
diff --git a/backend/plugins/gitee/models/connection.go b/backend/plugins/gitee/models/connection.go
index 3373ceb56..b7c72f846 100644
--- a/backend/plugins/gitee/models/connection.go
+++ b/backend/plugins/gitee/models/connection.go
@@ -22,6 +22,7 @@ import (
)
type GiteeConnection struct {
+ helper.BaseConnection `mapstructure:",squash"`
helper.RestConnection `mapstructure:",squash"`
helper.AccessToken `mapstructure:",squash"`
}
diff --git a/backend/plugins/github/api/blueprint_V200_test.go b/backend/plugins/github/api/blueprint_V200_test.go
index 6d0009c56..f17b6edc2 100644
--- a/backend/plugins/github/api/blueprint_V200_test.go
+++ b/backend/plugins/github/api/blueprint_V200_test.go
@@ -18,6 +18,8 @@ limitations under the License.
package api
import (
+ "testing"
+
"github.com/apache/incubator-devlake/core/models/common"
"github.com/apache/incubator-devlake/core/models/domainlayer"
"github.com/apache/incubator-devlake/core/models/domainlayer/code"
@@ -30,18 +32,17 @@ import (
"github.com/apache/incubator-devlake/plugins/github/models"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
- "testing"
)
func TestMakeDataSourcePipelinePlanV200(t *testing.T) {
connection := &models.GithubConnection{
- RestConnection: helper.RestConnection{
- BaseConnection: helper.BaseConnection{
- Name: "github-test",
- Model: common.Model{
- ID: 1,
- },
+ BaseConnection: helper.BaseConnection{
+ Name: "github-test",
+ Model: common.Model{
+ ID: 1,
},
+ },
+ RestConnection: helper.RestConnection{
Endpoint: "https://api.github.com/",
Proxy: "",
RateLimitPerHour: 0,
diff --git a/backend/plugins/github/api/blueprint_test.go b/backend/plugins/github/api/blueprint_test.go
index 6c606ec7e..0add557eb 100644
--- a/backend/plugins/github/api/blueprint_test.go
+++ b/backend/plugins/github/api/blueprint_test.go
@@ -20,6 +20,10 @@ package api
import (
"bytes"
"encoding/json"
+ "io"
+ "net/http"
+ "testing"
+
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models/common"
"github.com/apache/incubator-devlake/core/plugin"
@@ -30,9 +34,6 @@ import (
"github.com/apache/incubator-devlake/plugins/github/tasks"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
- "io"
- "net/http"
- "testing"
)
var repo = &tasks.GithubApiRepo{
@@ -61,13 +62,13 @@ func TestMakePipelinePlan(t *testing.T) {
prepareMockMeta(t)
mockApiClient := prepareMockClient(t, repo)
connection := &models.GithubConnection{
- RestConnection: helper.RestConnection{
- BaseConnection: helper.BaseConnection{
- Name: "github-test",
- Model: common.Model{
- ID: 1,
- },
+ BaseConnection: helper.BaseConnection{
+ Name: "github-test",
+ Model: common.Model{
+ ID: 1,
},
+ },
+ RestConnection: helper.RestConnection{
Endpoint: "https://api.github.com/",
Proxy: "",
RateLimitPerHour: 0,
diff --git a/backend/plugins/github/models/connection.go b/backend/plugins/github/models/connection.go
index 329c6f72e..d5e853cd8 100644
--- a/backend/plugins/github/models/connection.go
+++ b/backend/plugins/github/models/connection.go
@@ -28,6 +28,7 @@ type TestConnectionRequest struct {
}
type GithubConnection struct {
+ helper.BaseConnection `mapstructure:",squash"`
helper.RestConnection `mapstructure:",squash"`
helper.AccessToken `mapstructure:",squash"`
EnableGraphql bool `mapstructure:"enableGraphql" json:"enableGraphql"`
diff --git a/backend/plugins/github/models/migrationscripts/20220715_add_init_tables.go b/backend/plugins/github/models/migrationscripts/20220715_add_init_tables.go
index d6a3bef7d..ff9a0f7fe 100644
--- a/backend/plugins/github/models/migrationscripts/20220715_add_init_tables.go
+++ b/backend/plugins/github/models/migrationscripts/20220715_add_init_tables.go
@@ -22,7 +22,6 @@ import (
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/migrationhelper"
- helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/github/models/migrationscripts/archived"
)
@@ -82,12 +81,6 @@ func (u *addInitTables) Up(basicRes context.BasicRes) errors.Error {
connection.Token = basicRes.GetConfig(`GITHUB_AUTH`)
connection.Name = `GitHub`
if connection.Endpoint != `` && connection.Token != `` && encodeKey != `` {
- err = helper.UpdateEncryptFields(connection, func(plaintext string) (string, errors.Error) {
- return plugin.Encrypt(encodeKey, plaintext)
- })
- if err != nil {
- return err
- }
// update from .env and save to db
err = db.Create(connection)
if err != nil {
diff --git a/backend/plugins/gitlab/api/blueprint_V200_test.go b/backend/plugins/gitlab/api/blueprint_V200_test.go
index 3d03d128f..959ac59d1 100644
--- a/backend/plugins/gitlab/api/blueprint_V200_test.go
+++ b/backend/plugins/gitlab/api/blueprint_V200_test.go
@@ -18,12 +18,13 @@ limitations under the License.
package api
import (
- mockdal "github.com/apache/incubator-devlake/mocks/core/dal"
- mockplugin "github.com/apache/incubator-devlake/mocks/core/plugin"
"strconv"
"testing"
"time"
+ mockdal "github.com/apache/incubator-devlake/mocks/core/dal"
+ mockplugin "github.com/apache/incubator-devlake/mocks/core/plugin"
+
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models/common"
"github.com/apache/incubator-devlake/core/models/domainlayer/code"
@@ -83,19 +84,21 @@ func TestMakeDataSourcePipelinePlanV200(t *testing.T) {
}
var testGitlabConnection = &models.GitlabConnection{
- RestConnection: helper.RestConnection{
- BaseConnection: helper.BaseConnection{
- Name: testName,
- Model: common.Model{
- ID: testConnectionID,
- },
+ BaseConnection: helper.BaseConnection{
+ Name: testName,
+ Model: common.Model{
+ ID: testConnectionID,
},
- Endpoint: testGitlabEndPoint,
- Proxy: testProxy,
- RateLimitPerHour: 0,
},
- AccessToken: helper.AccessToken{
- Token: testToken,
+ GitlabConn: models.GitlabConn{
+ RestConnection: helper.RestConnection{
+ Endpoint: testGitlabEndPoint,
+ Proxy: testProxy,
+ RateLimitPerHour: 0,
+ },
+ AccessToken: helper.AccessToken{
+ Token: testToken,
+ },
},
}
diff --git a/backend/plugins/gitlab/api/blueprint_test.go b/backend/plugins/gitlab/api/blueprint_test.go
index 3f4b56e29..ef619e306 100644
--- a/backend/plugins/gitlab/api/blueprint_test.go
+++ b/backend/plugins/gitlab/api/blueprint_test.go
@@ -20,6 +20,10 @@ package api
import (
"bytes"
"encoding/json"
+ "io"
+ "net/http"
+ "testing"
+
"github.com/apache/incubator-devlake/core/models/common"
"github.com/apache/incubator-devlake/core/plugin"
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
@@ -29,26 +33,25 @@ import (
"github.com/apache/incubator-devlake/plugins/gitlab/tasks"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
- "io"
- "net/http"
- "testing"
)
func TestProcessScope(t *testing.T) {
connection := &models.GitlabConnection{
- RestConnection: helper.RestConnection{
- BaseConnection: helper.BaseConnection{
- Name: "gitlab-test",
- Model: common.Model{
- ID: 1,
- },
+ BaseConnection: helper.BaseConnection{
+ Name: "gitlab-test",
+ Model: common.Model{
+ ID: 1,
},
- Endpoint: "https://gitlab.com/api/v4/",
- Proxy: "",
- RateLimitPerHour: 0,
},
- AccessToken: helper.AccessToken{
- Token: "123",
+ GitlabConn: models.GitlabConn{
+ RestConnection: helper.RestConnection{
+ Endpoint: "https://gitlab.com/api/v4/",
+ Proxy: "",
+ RateLimitPerHour: 0,
+ },
+ AccessToken: helper.AccessToken{
+ Token: "123",
+ },
},
}
mockApiCLient := mockapi.NewApiClientGetter(t)
diff --git a/backend/plugins/gitlab/api/connection.go b/backend/plugins/gitlab/api/connection.go
index 5eeddee2b..f96595668 100644
--- a/backend/plugins/gitlab/api/connection.go
+++ b/backend/plugins/gitlab/api/connection.go
@@ -19,19 +19,18 @@ package api
import (
"context"
- "fmt"
+ "net/http"
+
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/gitlab/models"
- "net/http"
- "time"
)
// @Summary test gitlab connection
// @Description Test gitlab Connection
// @Tags plugins/gitlab
-// @Param body body models.TestConnectionRequest true "json body"
+// @Param body body models.GitlabConn true "json body"
// @Success 200 {object} shared.ApiBody "Success"
// @Failure 400 {string} errcode.Error "Bad Request"
// @Failure 500 {string} errcode.Error "Internal Error"
@@ -39,21 +38,12 @@ import (
func TestConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
// decode
var err errors.Error
- var connection models.TestConnectionRequest
+ var connection models.GitlabConn
if err = api.Decode(input.Body, &connection, vld); err != nil {
return nil, err
}
// test connection
- apiClient, err := api.NewApiClient(
- context.TODO(),
- connection.Endpoint,
- map[string]string{
- "Authorization": fmt.Sprintf("Bearer %v", connection.Token),
- },
- 3*time.Second,
- connection.Proxy,
- basicRes,
- )
+ apiClient, err := api.NewApiClientFromConnection(context.TODO(), basicRes, connection)
if err != nil {
return nil, errors.Convert(err)
}
diff --git a/backend/plugins/gitlab/models/connection.go b/backend/plugins/gitlab/models/connection.go
index c77d75105..0b7b299b7 100644
--- a/backend/plugins/gitlab/models/connection.go
+++ b/backend/plugins/gitlab/models/connection.go
@@ -21,16 +21,17 @@ import (
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
)
-// This object conforms to what the frontend currently sends.
-type GitlabConnection struct {
+// GitlabConn holds the essential information to connect to the Gitlab API
+type GitlabConn struct {
helper.RestConnection `mapstructure:",squash"`
helper.AccessToken `mapstructure:",squash"`
}
-type TestConnectionRequest struct {
- Endpoint string `json:"endpoint"`
- Proxy string `json:"proxy"`
- helper.AccessToken `mapstructure:",squash"`
+// This object conforms to what the frontend currently sends.
+// GitlabConnection holds GitlabConn plus ID/Name for database storage
+type GitlabConnection struct {
+ helper.BaseConnection `mapstructure:",squash"`
+ GitlabConn `mapstructure:",squash"`
}
// This object conforms to what the frontend currently expects.
diff --git a/backend/plugins/jenkins/api/blueprint_v100_test.go b/backend/plugins/jenkins/api/blueprint_v100_test.go
index 12a6f186f..6f766d15f 100644
--- a/backend/plugins/jenkins/api/blueprint_v100_test.go
+++ b/backend/plugins/jenkins/api/blueprint_v100_test.go
@@ -20,11 +20,12 @@ package api
import (
"bytes"
"encoding/json"
- mockapi "github.com/apache/incubator-devlake/mocks/helpers/pluginhelper/api"
"io"
"net/http"
"testing"
+ mockapi "github.com/apache/incubator-devlake/mocks/helpers/pluginhelper/api"
+
"github.com/apache/incubator-devlake/core/models/common"
"github.com/apache/incubator-devlake/core/plugin"
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
@@ -35,13 +36,13 @@ import (
func TestProcessScope(t *testing.T) {
connection := &models.JenkinsConnection{
- RestConnection: helper.RestConnection{
- BaseConnection: helper.BaseConnection{
- Name: "jenkins-test",
- Model: common.Model{
- ID: 1,
- },
+ BaseConnection: helper.BaseConnection{
+ Name: "jenkins-test",
+ Model: common.Model{
+ ID: 1,
},
+ },
+ RestConnection: helper.RestConnection{
Endpoint: "https://api.github.com/",
Proxy: "",
RateLimitPerHour: 0,
diff --git a/backend/plugins/jenkins/models/connection.go b/backend/plugins/jenkins/models/connection.go
index 637d5d113..0fd81082c 100644
--- a/backend/plugins/jenkins/models/connection.go
+++ b/backend/plugins/jenkins/models/connection.go
@@ -23,6 +23,7 @@ import (
// This object conforms to what the frontend currently sends.
type JenkinsConnection struct {
+ helper.BaseConnection `mapstructure:",squash"`
helper.RestConnection `mapstructure:",squash"`
helper.BasicAuth `mapstructure:",squash"`
}
diff --git a/backend/plugins/jira/api/connection.go b/backend/plugins/jira/api/connection.go
index 07b4cc574..41640c588 100644
--- a/backend/plugins/jira/api/connection.go
+++ b/backend/plugins/jira/api/connection.go
@@ -20,20 +20,21 @@ package api
import (
"context"
"fmt"
+ "net/http"
+ "net/url"
+ "strings"
+
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/jira/models"
- "net/http"
- "net/url"
- "strings"
- "time"
+ "github.com/mitchellh/mapstructure"
)
// @Summary test jira connection
// @Description Test Jira Connection
// @Tags plugins/jira
-// @Param body body models.TestConnectionRequest true "json body"
+// @Param body body models.JiraConn true "json body"
// @Success 200 {object} shared.ApiBody "Success"
// @Failure 400 {string} errcode.Error "Bad Request"
// @Failure 500 {string} errcode.Error "Internal Error"
@@ -41,29 +42,24 @@ import (
func TestConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
// decode
var err errors.Error
- var connection models.TestConnectionRequest
- err = api.Decode(input.Body, &connection, vld)
- if err != nil {
- return nil, err
+ var connection models.JiraConn
+ e := mapstructure.Decode(input.Body, &connection)
+ if e != nil {
+ return nil, errors.Convert(e)
+ }
+ e = vld.StructExcept(connection, "BasicAuth", "AccessToken")
+ if e != nil {
+ return nil, errors.Convert(e)
}
// test connection
- apiClient, err := api.NewApiClient(
- context.TODO(),
- connection.Endpoint,
- map[string]string{
- "Authorization": fmt.Sprintf("Basic %v", connection.GetEncodedToken()),
- },
- 3*time.Second,
- connection.Proxy,
- basicRes,
- )
+ apiClient, err := api.NewApiClientFromConnection(context.TODO(), basicRes, connection)
if err != nil {
- return nil, errors.Convert(err)
+ return nil, err
}
// serverInfo checking
res, err := apiClient.Get("api/2/serverInfo", nil, nil)
if err != nil {
- return nil, errors.Convert(err)
+ return nil, err
}
serverInfoFail := "Failed testing the serverInfo: [ " + res.Request.URL.String() + " ]"
// check if `/rest/` was missing
diff --git a/backend/plugins/jira/models/connection.go b/backend/plugins/jira/models/connection.go
index 9e4cce800..eb412c1f2 100644
--- a/backend/plugins/jira/models/connection.go
+++ b/backend/plugins/jira/models/connection.go
@@ -18,6 +18,9 @@ limitations under the License.
package models
import (
+ "net/http"
+
+ "github.com/apache/incubator-devlake/core/errors"
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
)
@@ -27,21 +30,30 @@ type EpicResponse struct {
Value string
}
-type TestConnectionRequest struct {
- Endpoint string `json:"endpoint"`
- Proxy string `json:"proxy"`
- helper.BasicAuth `mapstructure:",squash"`
-}
-
type BoardResponse struct {
Id int
Title string
Value string
}
-type JiraConnection struct {
+// JiraConn holds the essential information to connect to the Jira API
+type JiraConn struct {
helper.RestConnection `mapstructure:",squash"`
+ helper.MultiAuth `mapstructure:",squash"`
helper.BasicAuth `mapstructure:",squash"`
+ helper.AccessToken `mapstructure:",squash"`
+}
+
+// SetupAuthentication implements the `IAuthentication` interface by delegating
+// the actual logic to the `MultiAuth` struct to help us write less code
+func (jc JiraConn) SetupAuthentication(req *http.Request) errors.Error {
+ return jc.MultiAuth.SetupAuthenticationForConnection(&jc, req)
+}
+
+// JiraConnection holds JiraConn plus ID/Name for database storage
+type JiraConnection struct {
+ helper.BaseConnection `mapstructure:",squash"`
+ JiraConn `mapstructure:",squash"`
}
func (JiraConnection) TableName() string {
diff --git a/backend/plugins/jira/models/migrationscripts/20230129_add_multi_auth.go b/backend/plugins/jira/models/migrationscripts/20230129_add_multi_auth.go
new file mode 100644
index 000000000..868eeb918
--- /dev/null
+++ b/backend/plugins/jira/models/migrationscripts/20230129_add_multi_auth.go
@@ -0,0 +1,57 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+ "github.com/apache/incubator-devlake/core/context"
+ "github.com/apache/incubator-devlake/core/dal"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/helpers/migrationhelper"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/api/apihelperabstract"
+)
+
+type jiraMultiAuth20230129 struct {
+ AuthMethod string `gorm:"type:varchar(20)"`
+ Token string `gorm:"type:varchar(255)"`
+}
+
+func (jiraMultiAuth20230129) TableName() string {
+ return "_tool_jira_connections"
+}
+
+type addJiraMultiAuth20230129 struct{}
+
+func (script *addJiraMultiAuth20230129) Up(basicRes context.BasicRes) errors.Error {
+ err := migrationhelper.AutoMigrateTables(basicRes, &jiraMultiAuth20230129{})
+ if err != nil {
+ return err
+ }
+ return basicRes.GetDal().UpdateColumn(
+ &jiraMultiAuth20230129{},
+ "auth_method", apihelperabstract.AUTH_METHOD_BASIC,
+ dal.Where("auth_method IS NULL"),
+ )
+}
+
+func (*addJiraMultiAuth20230129) Version() uint64 {
+ return 20230129115901
+}
+
+func (*addJiraMultiAuth20230129) Name() string {
+ return "add multiauth to _tool_jira_connections"
+}
diff --git a/backend/plugins/jira/models/migrationscripts/register.go b/backend/plugins/jira/models/migrationscripts/register.go
index 6df9925ce..18b1994e6 100644
--- a/backend/plugins/jira/models/migrationscripts/register.go
+++ b/backend/plugins/jira/models/migrationscripts/register.go
@@ -29,5 +29,6 @@ func All() []plugin.MigrationScript {
new(addInitTables20220716),
new(addTransformationRule20221116),
new(addProjectName20221215),
+ new(addJiraMultiAuth20230129),
}
}
diff --git a/backend/plugins/sonarqube/models/connection.go b/backend/plugins/sonarqube/models/connection.go
index dd2978afb..fe60b8179 100644
--- a/backend/plugins/sonarqube/models/connection.go
+++ b/backend/plugins/sonarqube/models/connection.go
@@ -21,6 +21,7 @@ import helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
// This object conforms to what the frontend currently sends.
type SonarqubeConnection struct {
+ helper.BaseConnection `mapstructure:",squash"`
helper.RestConnection `mapstructure:",squash"`
// For sonarqube, we can `use user_token:`
helper.AccessToken `mapstructure:",squash"`
diff --git a/backend/plugins/tapd/models/connection.go b/backend/plugins/tapd/models/connection.go
index 77be688e6..1f99abfbe 100644
--- a/backend/plugins/tapd/models/connection.go
+++ b/backend/plugins/tapd/models/connection.go
@@ -34,6 +34,7 @@ type WorkspaceResponse struct {
}
type TapdConnection struct {
+ helper.BaseConnection `mapstructure:",squash"`
helper.RestConnection `mapstructure:",squash"`
helper.BasicAuth `mapstructure:",squash"`
}
diff --git a/backend/plugins/zentao/models/connection.go b/backend/plugins/zentao/models/connection.go
index de1dedb73..0d71b26b2 100644
--- a/backend/plugins/zentao/models/connection.go
+++ b/backend/plugins/zentao/models/connection.go
@@ -23,6 +23,7 @@ import (
// This object conforms to what the frontend currently sends.
type ZentaoConnection struct {
+ helper.BaseConnection `mapstructure:",squash"`
helper.RestConnection `mapstructure:",squash"`
helper.BasicAuth `mapstructure:",squash"`
}