You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by wa...@apache.org on 2022/06/06 15:49:48 UTC
[incubator-devlake] branch main updated (a27bf451 -> 1980cb39)
This is an automated email from the ASF dual-hosted git repository.
warren pushed a change to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
from a27bf451 feat(frontend): change jira basicauthencode to username/password (#2002)
new f88e5c22 feat(api): change jira basicauthencoded to username/password and extract common utils
new ab0dc9e8 fix(jira): modify migrationscripts to spread basicAuthEncoded to username/password
new 7b485ed0 unit test for connection
new 1980cb39 minor fixes
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
api/shared/api_output.go | 2 +-
config-ui/src/hooks/useConnectionManager.jsx | 2 +-
.../pages/configure/connections/ConnectionForm.jsx | 4 +-
config-ui/src/pages/configure/settings/jira.jsx | 26 +-
config-ui/webpack.config.js | 18 +-
config-ui/webpack.production.config.js | 18 +-
plugins/feishu/tasks/api_client.go | 2 +-
plugins/github/tasks/api_client.go | 2 +-
plugins/gitlab/gitlab.go | 3 +
plugins/gitlab/tasks/api_client.go | 2 +-
plugins/helper/connection.go | 238 ++++++++++++++++
plugins/helper/connection_test.go | 166 +++++++++++
plugins/helper/worker_scheduler_test.go | 2 +-
plugins/jira/api/connection.go | 314 ++-------------------
plugins/jira/api/init.go | 4 -
plugins/jira/api/issue_status_mapping.go | 216 --------------
plugins/jira/api/issue_type_mapping.go | 214 --------------
plugins/jira/api/proxy.go | 16 +-
plugins/jira/jira.go | 26 +-
plugins/jira/models/connection.go | 31 +-
.../migrationscripts/updateSchemas20220505.go | 20 +-
.../migrationscripts/updateSchemas20220601.go | 114 ++++++++
plugins/jira/tasks/api_client.go | 11 +-
plugins/jira/tasks/apiv2models/issue.go | 12 +-
plugins/jira/tasks/issue_extractor.go | 2 +-
plugins/tapd/api/connection.go | 27 --
plugins/tapd/tapd.go | 8 +-
plugins/tapd/tasks/api_client.go | 4 +-
runner/directrun.go | 3 +-
29 files changed, 628 insertions(+), 879 deletions(-)
create mode 100644 plugins/helper/connection.go
create mode 100644 plugins/helper/connection_test.go
delete mode 100644 plugins/jira/api/issue_status_mapping.go
delete mode 100644 plugins/jira/api/issue_type_mapping.go
create mode 100644 plugins/jira/models/migrationscripts/updateSchemas20220601.go
[incubator-devlake] 01/04: feat(api): change jira basicauthencoded to username/password and extract common utils
Posted by wa...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
warren pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit f88e5c22fb6e6699930967ab9c145cab162ec752
Author: Yingchu Chen <yi...@merico.dev>
AuthorDate: Wed May 25 17:24:19 2022 +0800
feat(api): change jira basicauthencoded to username/password and extract common utils
closes #2003
---
api/shared/api_output.go | 2 +-
config-ui/src/hooks/useConnectionManager.jsx | 2 +-
config-ui/src/pages/configure/settings/jira.jsx | 26 +--
plugins/feishu/tasks/api_client.go | 2 +-
plugins/github/tasks/api_client.go | 2 +-
plugins/gitlab/tasks/api_client.go | 2 +-
plugins/helper/connection.go | 217 +++++++++++++++++++++
plugins/jira/api/connection.go | 191 +++---------------
plugins/jira/api/issue_status_mapping.go | 216 --------------------
plugins/jira/api/issue_type_mapping.go | 214 --------------------
plugins/jira/api/proxy.go | 16 +-
plugins/jira/jira.go | 17 +-
plugins/jira/models/connection.go | 16 +-
.../migrationscripts/updateSchemas20220505.go | 20 +-
.../migrationscripts/updateSchemas20220524.go | 72 +++++++
plugins/jira/tasks/api_client.go | 8 +-
plugins/tapd/tasks/api_client.go | 4 +-
17 files changed, 353 insertions(+), 674 deletions(-)
diff --git a/api/shared/api_output.go b/api/shared/api_output.go
index 8e006acd..28e2e630 100644
--- a/api/shared/api_output.go
+++ b/api/shared/api_output.go
@@ -35,7 +35,7 @@ func ApiOutputError(c *gin.Context, err error, status int) {
Message: err.Error(),
})
} else {
- logger.Global.Error("Server Internal Error: %w", err)
+ logger.Global.Error("Server Internal Error: %s", err.Error())
c.JSON(status, &ApiBody{
Success: false,
Message: err.Error(),
diff --git a/config-ui/src/hooks/useConnectionManager.jsx b/config-ui/src/hooks/useConnectionManager.jsx
index a3558df0..f9116919 100644
--- a/config-ui/src/hooks/useConnectionManager.jsx
+++ b/config-ui/src/hooks/useConnectionManager.jsx
@@ -83,7 +83,7 @@ function useConnectionManager ({
let connectionPayload
switch (activeProvider.id) {
case Providers.JIRA:
- connectionPayload = { endpoint: endpointUrl, auth: token, proxy: proxy }
+ connectionPayload = { endpoint: endpointUrl, username: username, password: password, proxy: proxy }
break
case Providers.GITHUB:
connectionPayload = { endpoint: endpointUrl, auth: token, proxy: proxy }
diff --git a/config-ui/src/pages/configure/settings/jira.jsx b/config-ui/src/pages/configure/settings/jira.jsx
index 8305eaa0..cda3defc 100644
--- a/config-ui/src/pages/configure/settings/jira.jsx
+++ b/config-ui/src/pages/configure/settings/jira.jsx
@@ -88,25 +88,6 @@ export default function JiraSettings (props) {
: null
}
- const parseTypeMappings = useCallback((mappings = []) => {
- const GroupedMappings = {
- [MAPPING_TYPES.Requirement]: [],
- [MAPPING_TYPES.Incident]: [],
- [MAPPING_TYPES.Bug]: [],
- }
- Object.entries(mappings).forEach(([tag, typeObj]) => {
- GroupedMappings[typeObj.standardType].push(tag)
- })
- console.log('>>>> PARSED TYPE MAPPINGS ....', GroupedMappings)
- setTypeMappingRequirement(GroupedMappings[MAPPING_TYPES.Requirement])
- setTypeMappingBug(GroupedMappings[MAPPING_TYPES.Bug])
- setTypeMappingIncident(GroupedMappings[MAPPING_TYPES.Incident])
- setRequirementTags(requirementTagsList?.filter(t => GroupedMappings[MAPPING_TYPES.Requirement].includes(t.value)))
- setBugTags(bugTagsList?.filter(t => GroupedMappings[MAPPING_TYPES.Bug].includes(t.value)))
- setIncidentTags(incidentTagsList?.filter(t => GroupedMappings[MAPPING_TYPES.Incident].includes(t.value)))
- return GroupedMappings
- }, [requirementTagsList, bugTagsList, incidentTagsList])
-
useEffect(() => {
const settings = {
epicKeyField: jiraIssueEpicKeyField?.value || '',
@@ -164,13 +145,12 @@ export default function JiraSettings (props) {
console.log('>> CONN SETTINGS OBJECT ', connection)
if (connection && connection.ID) {
// Parse Type Mappings (V2)
- parseTypeMappings(connection.typeMappings)
setStatusMappings([])
setRemoteLinkCommitSha(connection.remotelinkCommitShaPattern)
// setJiraIssueEpicKeyField(fieldsList.find(f => f.value === connection.epicKeyField))
// setJiraIssueStoryPointField(fieldsList.find(f => f.value === connection.storyPointField))
}
- }, [connection, parseTypeMappings])
+ }, [connection])
useEffect(() => {
setTypeMappingRequirement(requirementTags)
@@ -208,10 +188,6 @@ export default function JiraSettings (props) {
setJiraIssueStoryPointField(fieldsList.find(f => f.value === connection.storyPointField))
}, [fieldsList, connection.epicKeyField, connection.storyPointField])
- useEffect(() => {
- parseTypeMappings(connection.typeMappings)
- }, [requirementTagsList, bugTagsList, incidentTagsList, connection.typeMappings, parseTypeMappings])
-
return (
<>
<div className='headlineContainer'>
diff --git a/plugins/feishu/tasks/api_client.go b/plugins/feishu/tasks/api_client.go
index d6b7b7d0..da54b5f9 100644
--- a/plugins/feishu/tasks/api_client.go
+++ b/plugins/feishu/tasks/api_client.go
@@ -92,7 +92,7 @@ func NewFeishuApiClient(taskCtx core.TaskContext) (*helper.ApiAsyncClient, error
apiClient.SetAfterFunction(func(res *http.Response) error {
if res.StatusCode == http.StatusUnauthorized {
- return fmt.Errorf("feishu authentication failed, please check your Bearer Auth Token")
+ return fmt.Errorf("feishu authentication failed, please check your AccessToken")
}
return nil
})
diff --git a/plugins/github/tasks/api_client.go b/plugins/github/tasks/api_client.go
index 94a60e05..aa0e9aa0 100644
--- a/plugins/github/tasks/api_client.go
+++ b/plugins/github/tasks/api_client.go
@@ -60,7 +60,7 @@ func CreateApiClient(taskCtx core.TaskContext) (*helper.ApiAsyncClient, error) {
})
apiClient.SetAfterFunction(func(res *http.Response) error {
if res.StatusCode == http.StatusUnauthorized {
- return fmt.Errorf("authentication failed, please check your Token configuration")
+ return fmt.Errorf("authentication failed, please check your AccessToken configuration")
}
return nil
})
diff --git a/plugins/gitlab/tasks/api_client.go b/plugins/gitlab/tasks/api_client.go
index 0dc8bf64..f4c2b653 100644
--- a/plugins/gitlab/tasks/api_client.go
+++ b/plugins/gitlab/tasks/api_client.go
@@ -54,7 +54,7 @@ func NewGitlabApiClient(taskCtx core.TaskContext) (*helper.ApiAsyncClient, error
}
apiClient.SetAfterFunction(func(res *http.Response) error {
if res.StatusCode == http.StatusUnauthorized {
- return fmt.Errorf("authentication failed, please check your Basic Auth Token")
+ return fmt.Errorf("authentication failed, please check your AccessToken")
}
return nil
})
diff --git a/plugins/helper/connection.go b/plugins/helper/connection.go
new file mode 100644
index 00000000..93974f73
--- /dev/null
+++ b/plugins/helper/connection.go
@@ -0,0 +1,217 @@
+package helper
+
+import (
+ "encoding/base64"
+ "fmt"
+ "github.com/apache/incubator-devlake/config"
+ "github.com/apache/incubator-devlake/models/common"
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/go-playground/validator/v10"
+ "github.com/mitchellh/mapstructure"
+ "gorm.io/gorm"
+ "gorm.io/gorm/clause"
+ "reflect"
+ "strconv"
+)
+
+type BaseConnection struct {
+ Name string `gorm:"type:varchar(100);uniqueIndex" json:"name" validate:"required"`
+ common.Model
+}
+
+type BasicAuth struct {
+ Username string `mapstructure:"username" validate:"required" json:"username"`
+ Password string `mapstructure:"password" validate:"required" json:"password" encrypt:"yes"`
+}
+
+func (ba BasicAuth) GetEncodedToken() string {
+ return base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%v:%v", ba.Username, ba.Password)))
+}
+
+type AccessToken struct {
+ Token string `mapstructure:"token" validate:"required" json:"token" encrypt:"yes"`
+}
+
+type RestConnection struct {
+ BaseConnection `mapstructure:",squash"`
+ Endpoint string `mapstructure:"endpoint" validate:"required" json:"endpoint"`
+ Proxy string `mapstructure:"proxy" json:"proxy"`
+ RateLimit int `comment:"api request rate limt per hour" json:"rateLimit"`
+}
+
+// RefreshAndSaveConnection populate from request input into connection which come from REST functions to connection struct and save to DB
+// and only change value which `data` has
+// mergeFieldsToConnection merges fields from data
+// `connection` is the pointer of a plugin connection
+// `data` is http request input param
+func RefreshAndSaveConnection(connection interface{}, data map[string]interface{}, db *gorm.DB) error {
+ var err error
+ // update fields from request body
+ err = mergeFieldsToConnection(connection, data)
+ if err != nil {
+ return err
+ }
+
+ err = saveToDb(connection, db)
+
+ if err != nil {
+ return err
+ }
+ return nil
+}
+
+func saveToDb(connection interface{}, db *gorm.DB) error {
+ dataVal := reflect.ValueOf(connection)
+ if dataVal.Kind() != reflect.Ptr {
+ panic("entityPtr is not a pointer")
+ }
+
+ dataType := reflect.Indirect(dataVal).Type()
+ fieldName := getEncryptField(dataType, "encrypt")
+ plainPwd := ""
+ err := doEncrypt(dataVal, fieldName)
+ if err != nil {
+ return err
+ }
+ err = db.Clauses(clause.OnConflict{UpdateAll: true}).Save(connection).Error
+ if err != nil {
+ return err
+ }
+
+ err = doDecrypt(dataVal, fieldName)
+ if err != nil {
+ return err
+ }
+ dataVal.Elem().FieldByName(fieldName).Set(reflect.ValueOf(plainPwd))
+
+ return err
+}
+
+// mergeFieldsToConnection will populate all value in map to connection struct and validate the struct
+func mergeFieldsToConnection(specificConnection interface{}, connections ...map[string]interface{}) error {
+ // decode
+ for _, connection := range connections {
+ err := mapstructure.Decode(connection, specificConnection)
+ if err != nil {
+ return err
+ }
+ }
+ // validate
+ vld := validator.New()
+ err := vld.Struct(specificConnection)
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func getEncKey() (string, error) {
+ // encrypt
+ v := config.GetConfig()
+ encKey := v.GetString(core.EncodeKeyEnvStr)
+ if encKey == "" {
+ // Randomly generate a bunch of encryption keys and set them to config
+ encKey = core.RandomEncKey()
+ v.Set(core.EncodeKeyEnvStr, encKey)
+ err := config.WriteConfig(v)
+ if err != nil {
+ return encKey, err
+ }
+ }
+ return encKey, nil
+}
+
+// FindConnectionByInput finds connection from db by parsing request input and decrypt it
+func FindConnectionByInput(input *core.ApiResourceInput, connection interface{}, db *gorm.DB) error {
+ dataVal := reflect.ValueOf(connection)
+ if dataVal.Kind() != reflect.Ptr {
+ return fmt.Errorf("connection is not a pointer")
+ }
+
+ id, err := GetConnectionIdByInputParam(input)
+ if err != nil {
+ return fmt.Errorf("invalid connectionId")
+ }
+
+ err = db.First(connection, id).Error
+ if err != nil {
+ fmt.Printf("--- %s", err.Error())
+ return err
+ }
+
+ dataType := reflect.Indirect(dataVal).Type()
+
+ fieldName := getEncryptField(dataType, "encrypt")
+ return doDecrypt(dataVal, fieldName)
+
+}
+
+// GetConnectionIdByInputParam gets connectionId by parsing request input
+func GetConnectionIdByInputParam(input *core.ApiResourceInput) (uint64, error) {
+ connectionId := input.Params["connectionId"]
+ if connectionId == "" {
+ return 0, fmt.Errorf("missing connectionId")
+ }
+ return strconv.ParseUint(connectionId, 10, 64)
+}
+
+func getEncryptField(t reflect.Type, tag string) string {
+ fieldName := ""
+ for i := 0; i < t.NumField(); i++ {
+ field := t.Field(i)
+ if field.Type.Kind() == reflect.Struct {
+ fieldName = getEncryptField(field.Type, tag)
+ } else {
+ if field.Tag.Get(tag) == "yes" {
+ fieldName = field.Name
+ }
+ }
+ }
+ return fieldName
+}
+
+// DecryptConnection decrypts password/token field for connection
+func DecryptConnection(connection interface{}, fieldName string) error {
+ dataVal := reflect.ValueOf(connection)
+ if dataVal.Kind() != reflect.Ptr {
+ panic("connection is not a pointer")
+ }
+ if len(fieldName) == 0 {
+ dataType := reflect.Indirect(dataVal).Type()
+ fieldName = getEncryptField(dataType, "encrypt")
+ }
+ return doDecrypt(dataVal, fieldName)
+}
+
+func doDecrypt(dataVal reflect.Value, fieldName string) error {
+ encryptCode, err := getEncKey()
+ if err != nil {
+ return err
+ }
+ if len(fieldName) > 0 {
+ decryptStr, err := core.Decrypt(encryptCode, dataVal.Elem().FieldByName(fieldName).String())
+ if err != nil {
+ return err
+ }
+ dataVal.Elem().FieldByName(fieldName).Set(reflect.ValueOf(decryptStr))
+ }
+ return nil
+}
+
+func doEncrypt(dataVal reflect.Value, fieldName string) error {
+ encryptCode, err := getEncKey()
+ if err != nil {
+ return err
+ }
+ if len(fieldName) > 0 {
+ plainPwd := dataVal.Elem().FieldByName(fieldName).String()
+ encyptedStr, err := core.Encrypt(encryptCode, plainPwd)
+
+ if err != nil {
+ return err
+ }
+ dataVal.Elem().FieldByName(fieldName).Set(reflect.ValueOf(encyptedStr))
+ }
+ return nil
+}
diff --git a/plugins/jira/api/connection.go b/plugins/jira/api/connection.go
index 96970d77..aa94537a 100644
--- a/plugins/jira/api/connection.go
+++ b/plugins/jira/api/connection.go
@@ -25,9 +25,6 @@ import (
"strings"
"time"
- "github.com/apache/incubator-devlake/config"
- "github.com/apache/incubator-devlake/models/common"
-
"github.com/apache/incubator-devlake/errors"
"github.com/apache/incubator-devlake/plugins/core"
"github.com/apache/incubator-devlake/plugins/helper"
@@ -52,11 +49,12 @@ func TestConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput, erro
if err != nil {
return nil, err
}
+
// test connection
apiClient, err := helper.NewApiClient(
connection.Endpoint,
map[string]string{
- "Authorization": fmt.Sprintf("Basic %v", connection.Auth),
+ "Authorization": fmt.Sprintf("Basic %v", connection.GetEncodedToken()),
},
3*time.Second,
connection.Proxy,
@@ -100,111 +98,6 @@ func TestConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput, erro
return nil, nil
}
-func findConnectionByInputParam(input *core.ApiResourceInput) (*models.JiraConnection, error) {
- jiraConnectionId, err := getJiraConnectionIdByInputParam(input)
- if err != nil {
- return nil, fmt.Errorf("invalid connectionId")
- }
- return getJiraConnectionById(jiraConnectionId)
-}
-
-func getJiraConnectionIdByInputParam(input *core.ApiResourceInput) (uint64, error) {
- connectionId := input.Params["connectionId"]
- if connectionId == "" {
- return 0, fmt.Errorf("missing connectionId")
- }
- return strconv.ParseUint(connectionId, 10, 64)
-}
-
-func getJiraConnectionById(id uint64) (*models.JiraConnection, error) {
- jiraConnection := &models.JiraConnection{}
- err := db.First(jiraConnection, id).Error
- if err != nil {
- return nil, err
- }
-
- // decrypt
- v := config.GetConfig()
- encKey := v.GetString(core.EncodeKeyEnvStr)
- jiraConnection.BasicAuthEncoded, err = core.Decrypt(encKey, jiraConnection.BasicAuthEncoded)
- if err != nil {
- log.Error("failed to decrypt basic auth: %s", err)
- }
-
- return jiraConnection, nil
-}
-func mergeFieldsToJiraConnection(jiraConnection *models.JiraConnection, connections ...map[string]interface{}) error {
- // decode
- for _, connection := range connections {
- err := mapstructure.Decode(connection, jiraConnection)
- if err != nil {
- return err
- }
- }
-
- // validate
- vld := validator.New()
- err := vld.Struct(jiraConnection)
- if err != nil {
- return err
- }
-
- return nil
-}
-
-func refreshAndSaveJiraConnection(jiraConnection *models.JiraConnection, data map[string]interface{}) error {
- var err error
- // update fields from request body
- err = mergeFieldsToJiraConnection(jiraConnection, data)
- if err != nil {
- return err
- }
-
- encKey, err := getEncKey()
- if err != nil {
- return err
- }
- jiraConnection.BasicAuthEncoded, err = core.Encrypt(encKey, jiraConnection.BasicAuthEncoded)
- if err != nil {
- return err
- }
-
- // transaction for nested operations
- tx := db.Begin()
- defer func() {
- if err != nil {
- tx.Rollback()
- } else {
- tx.Commit()
- }
- }()
- if jiraConnection.ID > 0 {
- err = tx.Save(jiraConnection).Error
- } else {
- err = tx.Create(jiraConnection).Error
- }
- if err != nil {
- if common.IsDuplicateError(err) {
- return fmt.Errorf("jira connection with name %s already exists", jiraConnection.Name)
- }
- return err
- }
- // perform optional operation
- typeMappings := data["typeMappings"]
- if typeMappings != nil {
- err = saveTypeMappings(tx, jiraConnection.ID, typeMappings)
- if err != nil {
- return err
- }
- }
-
- jiraConnection.BasicAuthEncoded, err = core.Decrypt(encKey, jiraConnection.BasicAuthEncoded)
- if err != nil {
- log.Error("failed to decrypt basic auth: %s", err)
- }
- return nil
-}
-
/*
POST /plugins/jira/connections
{
@@ -230,7 +123,7 @@ func PostConnections(input *core.ApiResourceInput) (*core.ApiResourceOutput, err
jiraConnection := &models.JiraConnection{}
// update from request and save to database
- err := refreshAndSaveJiraConnection(jiraConnection, input.Body)
+ err := helper.RefreshAndSaveConnection(jiraConnection, input.Body, db)
if err != nil {
return nil, err
}
@@ -259,14 +152,15 @@ PATCH /plugins/jira/connections/:connectionId
}
*/
func PatchConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) {
+ jiraConnection := &models.JiraConnection{}
// load from db
- jiraConnection, err := findConnectionByInputParam(input)
+ err := helper.FindConnectionByInput(input, jiraConnection, db)
if err != nil {
return nil, err
}
// update from request and save to database
- err = refreshAndSaveJiraConnection(jiraConnection, input.Body)
+ err = helper.RefreshAndSaveConnection(jiraConnection, input.Body, db)
if err != nil {
return nil, err
}
@@ -279,7 +173,7 @@ DELETE /plugins/jira/connections/:connectionId
*/
func DeleteConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) {
// load from db
- jiraConnectionID, err := getJiraConnectionIdByInputParam(input)
+ jiraConnectionID, err := helper.GetConnectionIdByInputParam(input)
if err != nil {
return nil, err
}
@@ -304,22 +198,19 @@ func DeleteConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput, er
GET /plugins/jira/connections
*/
func ListConnections(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) {
- jiraConnections := make([]models.JiraConnection, 0)
+ jiraConnections := make([]*models.JiraConnection, 0)
err := db.Find(&jiraConnections).Error
if err != nil {
return nil, err
}
- encKey, err := getEncKey()
- if err != nil {
- return nil, err
- }
- for i := range jiraConnections {
- jiraConnections[i].BasicAuthEncoded, err = core.Decrypt(encKey, jiraConnections[i].BasicAuthEncoded)
+ for i, _ := range jiraConnections {
+ err = helper.DecryptConnection(jiraConnections[i], "Password")
if err != nil {
- log.Error("failed to decrypt basic auth: %s", err)
+ return nil, err
}
}
- return &core.ApiResourceOutput{Body: jiraConnections}, nil
+
+ return &core.ApiResourceOutput{Body: jiraConnections, Status: http.StatusOK}, nil
}
/*
@@ -345,53 +236,27 @@ GET /plugins/jira/connections/:connectionId
}
*/
func GetConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) {
- jiraConnection, err := findConnectionByInputParam(input)
+ jiraConnection := &models.JiraConnection{}
+ err := helper.FindConnectionByInput(input, jiraConnection, db)
if err != nil {
return nil, err
}
detail := &models.JiraConnectionDetail{
JiraConnection: *jiraConnection,
- TypeMappings: make(map[string]map[string]interface{}),
}
- typeMappings, err := findIssueTypeMappingByConnectionId(jiraConnection.ID)
if err != nil {
return nil, err
}
- for _, jiraTypeMapping := range typeMappings {
- // type mapping
- typeMappingDict := map[string]interface{}{
- "standardType": jiraTypeMapping.StandardType,
- }
- detail.TypeMappings[jiraTypeMapping.UserType] = typeMappingDict
-
- // status mapping
- statusMappings, err := findIssueStatusMappingByConnectionIdAndUserType(
- jiraConnection.ID,
- jiraTypeMapping.UserType,
- )
- if err != nil {
- return nil, err
- }
- if len(statusMappings) == 0 {
- continue
- }
- statusMappingsDict := make(map[string]interface{})
- for _, jiraStatusMapping := range statusMappings {
- statusMappingsDict[jiraStatusMapping.UserStatus] = map[string]interface{}{
- "standardStatus": jiraStatusMapping.StandardStatus,
- }
- }
- typeMappingDict["statusMappings"] = statusMappingsDict
- }
return &core.ApiResourceOutput{Body: detail}, nil
}
// GET /plugins/jira/connections/:connectionId/epics
func GetEpicsByConnectionId(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) {
- jiraConnection, err := findConnectionByInputParam(input)
+ jiraConnection := &models.JiraConnection{}
+ err := helper.FindConnectionByInput(input, jiraConnection, db)
if err != nil {
return nil, err
}
@@ -410,7 +275,11 @@ type GranularitiesResponse struct {
}
func GetGranularitiesByConnectionId(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) {
- jiraConnection, err := findConnectionByInputParam(input)
+ jiraConnection := &models.JiraConnection{}
+ err := helper.FindConnectionByInput(input, jiraConnection, db)
+ if err != nil {
+ return nil, err
+ }
if err != nil {
return nil, err
}
@@ -448,19 +317,3 @@ func GetBoardsByConnectionId(input *core.ApiResourceInput) (*core.ApiResourceOut
}
return &core.ApiResourceOutput{Body: boardResponses}, nil
}
-
-func getEncKey() (string, error) {
- // encrypt
- v := config.GetConfig()
- encKey := v.GetString(core.EncodeKeyEnvStr)
- if encKey == "" {
- // Randomly generate a bunch of encryption keys and set them to config
- encKey = core.RandomEncKey()
- v.Set(core.EncodeKeyEnvStr, encKey)
- err := config.WriteConfig(v)
- if err != nil {
- return encKey, err
- }
- }
- return encKey, nil
-}
diff --git a/plugins/jira/api/issue_status_mapping.go b/plugins/jira/api/issue_status_mapping.go
deleted file mode 100644
index 9caa90b3..00000000
--- a/plugins/jira/api/issue_status_mapping.go
+++ /dev/null
@@ -1,216 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package api
-
-import (
- "fmt"
- "net/http"
-
- "github.com/apache/incubator-devlake/models/common"
-
- "github.com/apache/incubator-devlake/plugins/core"
- "github.com/apache/incubator-devlake/plugins/jira/models"
- "github.com/go-playground/validator/v10"
- "github.com/mitchellh/mapstructure"
- "gorm.io/gorm"
-)
-
-func findIssueStatusMappingFromInput(input *core.ApiResourceInput) (*models.JiraIssueStatusMapping, error) {
- // load type mapping
- jiraIssueTypeMapping, err := findIssueTypeMappingByInputParam(input)
- if err != nil {
- return nil, err
- }
- // load status mapping from db
- userStatus := input.Params["userStatus"]
- if userStatus == "" {
- return nil, fmt.Errorf("missing userStatus")
- }
- jiraIssueStatusMapping := &models.JiraIssueStatusMapping{}
- err = db.First(
- jiraIssueStatusMapping,
- jiraIssueTypeMapping.ConnectionID,
- jiraIssueTypeMapping.UserType,
- userStatus,
- ).Error
- if err != nil {
- return nil, err
- }
-
- return jiraIssueStatusMapping, nil
-}
-
-func mergeFieldsToJiraStatusMapping(
- jiraIssueStatusMapping *models.JiraIssueStatusMapping,
- connections ...map[string]interface{},
-) error {
- // merge fields from connections to jiraIssueStatusMapping
- for _, connection := range connections {
- err := mapstructure.Decode(connection, jiraIssueStatusMapping)
- if err != nil {
- return err
- }
- }
- // validate
- vld := validator.New()
- err := vld.Struct(jiraIssueStatusMapping)
- if err != nil {
- return err
- }
- return nil
-}
-
-func wrapIssueStatusDuplicateErr(err error) error {
- if common.IsDuplicateError(err) {
- return fmt.Errorf("jira issue status mapping already exists")
- }
- return err
-}
-
-func saveStatusMappings(tx *gorm.DB, jiraConnectionId uint64, userType string, statusMappings interface{}) error {
- statusMappingsMap, ok := statusMappings.(map[string]interface{})
- if !ok {
- return fmt.Errorf("statusMappings is not a JSON object: %v", statusMappings)
- }
- err := tx.Where(
- "connection_id = ? AND user_type = ?",
- jiraConnectionId,
- userType).Delete(&models.JiraIssueStatusMapping{}).Error
- if err != nil {
- return err
- }
- for userStatus, statusMapping := range statusMappingsMap {
- statusMappingMap, ok := statusMapping.(map[string]interface{})
- if !ok {
- return fmt.Errorf("statusMapping is not a JSON object: %v", statusMappings)
- }
- jiraIssueStatusMapping := &models.JiraIssueStatusMapping{}
- err = mergeFieldsToJiraStatusMapping(jiraIssueStatusMapping, statusMappingMap, map[string]interface{}{
- "ConnectionID": jiraConnectionId,
- "UserType": userType,
- "UserStatus": userStatus,
- })
- if err != nil {
- return err
- }
- err = tx.Create(jiraIssueStatusMapping).Error
- if err != nil {
- return err
- }
- }
- return nil
-}
-
-func findIssueStatusMappingByConnectionIdAndUserType(
- jiraConnectionId uint64,
- userType string,
-) ([]*models.JiraIssueStatusMapping, error) {
- jiraIssueStatusMappings := make([]*models.JiraIssueStatusMapping, 0)
- err := db.Where(
- "connection_id = ? AND user_type = ?",
- jiraConnectionId,
- userType,
- ).Find(&jiraIssueStatusMappings).Error
- return jiraIssueStatusMappings, err
-}
-
-/*
-POST /plugins/jira/connections/:connectionId/type-mappings/:userType/status-mappings
-{
- "userStatus": "user custom status",
- "standardStatus": "devlake standard status"
-}
-*/
-func PostIssueStatusMappings(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) {
- jiraIssueTypeMapping, err := findIssueTypeMappingByInputParam(input)
- if err != nil {
- return nil, err
- }
- jiraIssueStatusMapping := &models.JiraIssueStatusMapping{}
- err = mergeFieldsToJiraStatusMapping(jiraIssueStatusMapping, input.Body, map[string]interface{}{
- "ConnectionID": jiraIssueTypeMapping.ConnectionID,
- "UserType": jiraIssueTypeMapping.UserType,
- })
- if err != nil {
- return nil, err
- }
- // save
- err = wrapIssueStatusDuplicateErr(db.Create(jiraIssueStatusMapping).Error)
- if err != nil {
- return nil, err
- }
- return &core.ApiResourceOutput{Body: jiraIssueStatusMapping, Status: http.StatusOK}, nil
-}
-
-/*
-PUT /plugins/jira/connections/:connectionId/type-mappings/:userType/status-mappings/:userStatus
-{
- "standardStatus": "devlake standard status"
-}
-*/
-func PutIssueStatusMapping(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) {
- // load from db
- jiraIssueStatusMapping, err := findIssueStatusMappingFromInput(input)
- if err != nil {
- return nil, err
- }
- // update with request body
- err = mergeFieldsToJiraStatusMapping(jiraIssueStatusMapping, input.Body)
- if err != nil {
- return nil, err
- }
- // save
- err = wrapIssueStatusDuplicateErr(db.Save(jiraIssueStatusMapping).Error)
- if err != nil {
- return nil, err
- }
- return &core.ApiResourceOutput{Body: jiraIssueStatusMapping}, nil
-}
-
-/*
-DELETE /plugins/jira/connections/:connectionId/type-mappings/:userType/status-mappings/:userStatus
-*/
-func DeleteIssueStatusMapping(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) {
- jiraIssueStatusMapping, err := findIssueStatusMappingFromInput(input)
- if err != nil {
- return nil, err
- }
- err = db.Delete(jiraIssueStatusMapping).Error
- if err != nil {
- return nil, err
- }
- return &core.ApiResourceOutput{Body: jiraIssueStatusMapping}, nil
-}
-
-/*
-GET /plugins/jira/connections/:connectionId/type-mappings/:userType/status-mappings
-*/
-func ListIssueStatusMappings(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) {
- jiraIssueTypeMapping, err := findIssueTypeMappingByInputParam(input)
- if err != nil {
- return nil, err
- }
- jiraIssueStatusMappings, err := findIssueStatusMappingByConnectionIdAndUserType(
- jiraIssueTypeMapping.ConnectionID,
- jiraIssueTypeMapping.UserType,
- )
- if err != nil {
- return nil, err
- }
- return &core.ApiResourceOutput{Body: jiraIssueStatusMappings}, nil
-}
diff --git a/plugins/jira/api/issue_type_mapping.go b/plugins/jira/api/issue_type_mapping.go
deleted file mode 100644
index 7f135646..00000000
--- a/plugins/jira/api/issue_type_mapping.go
+++ /dev/null
@@ -1,214 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package api
-
-import (
- "fmt"
- "net/http"
-
- "github.com/apache/incubator-devlake/models/common"
-
- "github.com/apache/incubator-devlake/plugins/core"
- "github.com/apache/incubator-devlake/plugins/jira/models"
- "github.com/go-playground/validator/v10"
- "github.com/mitchellh/mapstructure"
- "gorm.io/gorm"
-)
-
-func findIssueTypeMappingByInputParam(input *core.ApiResourceInput) (*models.JiraIssueTypeMapping, error) {
- // load jira connection
- jiraConnection, err := findConnectionByInputParam(input)
- if err != nil {
- return nil, err
- }
- // load jira type mapping from db
- userType := input.Params["userType"]
- if userType == "" {
- return nil, fmt.Errorf("missing userType")
- }
- jiraIssueTypeMapping := &models.JiraIssueTypeMapping{}
- err = db.First(jiraIssueTypeMapping, jiraConnection.ID, userType).Error
- if err != nil {
- return nil, err
- }
-
- return jiraIssueTypeMapping, nil
-}
-
-func mergeFieldsToJiraTypeMapping(
- jiraIssueTypeMapping *models.JiraIssueTypeMapping,
- connections ...map[string]interface{},
-) error {
- // merge fields from connections to jiraIssueTypeMapping
- for _, connection := range connections {
- err := mapstructure.Decode(connection, jiraIssueTypeMapping)
- if err != nil {
- return err
- }
- }
- // validate
- vld := validator.New()
- err := vld.Struct(jiraIssueTypeMapping)
- if err != nil {
- return err
- }
- return nil
-}
-
-func wrapIssueTypeDuplicateErr(err error) error {
- if common.IsDuplicateError(err) {
- return fmt.Errorf("jira issue type mapping already exists")
- }
- return err
-}
-
-func saveTypeMappings(tx *gorm.DB, jiraConnectionId uint64, typeMappings interface{}) error {
- typeMappingsMap, ok := typeMappings.(map[string]interface{})
- if !ok {
- return fmt.Errorf("typeMappings is not a JSON object: %v", typeMappings)
- }
- err := tx.Where("connection_id = ?", jiraConnectionId).Delete(&models.JiraIssueTypeMapping{}).Error
- if err != nil {
- return err
- }
- for userType, typeMapping := range typeMappingsMap {
- typeMappingMap, ok := typeMapping.(map[string]interface{})
- if !ok {
- return fmt.Errorf("typeMapping is not a JSON object: %v", typeMapping)
- }
- jiraIssueTypeMapping := &models.JiraIssueTypeMapping{}
- err = mergeFieldsToJiraTypeMapping(jiraIssueTypeMapping, typeMappingMap, map[string]interface{}{
- "ConnectionID": jiraConnectionId,
- "UserType": userType,
- })
- if err != nil {
- return err
- }
- err = wrapIssueTypeDuplicateErr(tx.Create(jiraIssueTypeMapping).Error)
- if err != nil {
- return err
- }
-
- statusMappings := typeMappingMap["statusMappings"]
- if statusMappings != nil {
- err = saveStatusMappings(tx, jiraConnectionId, userType, statusMappings)
- if err != nil {
- return err
- }
- }
- }
- return nil
-}
-
-func findIssueTypeMappingByConnectionId(jiraConnectionId uint64) ([]*models.JiraIssueTypeMapping, error) {
- jiraIssueTypeMappings := make([]*models.JiraIssueTypeMapping, 0)
- err := db.Where("connection_id = ?", jiraConnectionId).Find(&jiraIssueTypeMappings).Error
- if err != nil {
- return nil, err
- }
- return jiraIssueTypeMappings, nil
-}
-
-/*
-POST /plugins/jira/connections/:connectionId/type-mappings
-{
- "userType": "user custom type",
- "standardType": "devlake standard type"
-}
-*/
-func PostIssueTypeMappings(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) {
- // create new
- jiraConnection, err := findConnectionByInputParam(input)
- if err != nil {
- return nil, err
- }
- jiraIssueTypeMapping := &models.JiraIssueTypeMapping{}
- err = mergeFieldsToJiraTypeMapping(jiraIssueTypeMapping, input.Body, map[string]interface{}{
- "ConnectionID": jiraConnection.ID,
- })
- if err != nil {
- return nil, err
- }
- // save
- err = wrapIssueTypeDuplicateErr(db.Create(jiraIssueTypeMapping).Error)
- if err != nil {
- return nil, err
- }
- return &core.ApiResourceOutput{Body: jiraIssueTypeMapping, Status: http.StatusOK}, nil
-}
-
-/*
-PUT /plugins/jira/connections/:connectionId/type-mappings/:userType
-{
- "standardType": "devlake standard type"
-}
-*/
-func PutIssueTypeMapping(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) {
- // load from db
- jiraIssueTypeMapping, err := findIssueTypeMappingByInputParam(input)
- if err != nil {
- return nil, err
- }
- // update with request body
- err = mergeFieldsToJiraTypeMapping(jiraIssueTypeMapping, input.Body)
- if err != nil {
- return nil, err
- }
- // save
- err = wrapIssueTypeDuplicateErr(db.Save(jiraIssueTypeMapping).Error)
- if err != nil {
- return nil, err
- }
- return &core.ApiResourceOutput{Body: jiraIssueTypeMapping}, nil
-}
-
-/*
-DELETE /plugins/jira/connections/:connectionId/type-mappings/:userType
-*/
-func DeleteIssueTypeMapping(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) {
- // load from db
- jiraIssueTypeMapping, err := findIssueTypeMappingByInputParam(input)
- if err != nil {
- return nil, err
- }
- err = db.Delete(jiraIssueTypeMapping).Error
- if err != nil {
- return nil, err
- }
- err = db.Where(
- "connection_id = ? AND user_type = ?",
- jiraIssueTypeMapping.ConnectionID,
- jiraIssueTypeMapping.UserType,
- ).Delete(&models.JiraIssueStatusMapping{}).Error
- if err != nil {
- return nil, err
- }
- return &core.ApiResourceOutput{Body: jiraIssueTypeMapping}, nil
-}
-
-/*
-GET /plugins/jira/connections/:connectionId/type-mappings
-*/
-func ListIssueTypeMappings(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) {
- jiraConnection, err := findConnectionByInputParam(input)
- if err != nil {
- return nil, err
- }
- jiraIssueTypeMappings, err := findIssueTypeMappingByConnectionId(jiraConnection.ID)
- return &core.ApiResourceOutput{Body: jiraIssueTypeMappings}, err
-}
diff --git a/plugins/jira/api/proxy.go b/plugins/jira/api/proxy.go
index 55331e01..e4a3eb90 100644
--- a/plugins/jira/api/proxy.go
+++ b/plugins/jira/api/proxy.go
@@ -20,8 +20,8 @@ package api
import (
"encoding/json"
"fmt"
+ "github.com/apache/incubator-devlake/utils"
"io/ioutil"
- "strconv"
"time"
"github.com/apache/incubator-devlake/plugins/core"
@@ -34,21 +34,13 @@ const (
)
func Proxy(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) {
- connectionId := input.Params["connectionId"]
- if connectionId == "" {
- return nil, fmt.Errorf("missing connectionid")
- }
- jiraConnectionId, err := strconv.ParseUint(connectionId, 10, 64)
- if err != nil {
- return nil, err
- }
jiraConnection := &models.JiraConnection{}
- err = db.First(jiraConnection, jiraConnectionId).Error
+ err := helper.FindConnectionByInput(input, jiraConnection, db)
if err != nil {
return nil, err
}
- encKey := cfg.GetString(core.EncodeKeyEnvStr)
- basicAuth, err := core.Decrypt(encKey, jiraConnection.BasicAuthEncoded)
+ basicAuth := utils.GetEncodedToken(jiraConnection.Username, jiraConnection.Password)
+
if err != nil {
return nil, err
}
diff --git a/plugins/jira/jira.go b/plugins/jira/jira.go
index 2b865ea6..2a40b901 100644
--- a/plugins/jira/jira.go
+++ b/plugins/jira/jira.go
@@ -150,6 +150,7 @@ func (plugin Jira) MigrationScripts() []migration.Script {
new(migrationscripts.UpdateSchemas20220505),
new(migrationscripts.UpdateSchemas20220507),
new(migrationscripts.UpdateSchemas20220518),
+ new(migrationscripts.UpdateSchemas20220524),
new(migrationscripts.UpdateSchemas20220525),
new(migrationscripts.UpdateSchemas20220526),
new(migrationscripts.UpdateSchemas20220527),
@@ -184,22 +185,6 @@ func (plugin Jira) ApiResources() map[string]map[string]core.ApiResourceHandler
"connections/:connectionId/boards": {
"GET": api.GetBoardsByConnectionId,
},
- "connections/:connectionId/type-mappings": {
- "POST": api.PostIssueTypeMappings,
- "GET": api.ListIssueTypeMappings,
- },
- "connections/:connectionId/type-mappings/:userType": {
- "PUT": api.PutIssueTypeMapping,
- "DELETE": api.DeleteIssueTypeMapping,
- },
- "connections/:connectionId/type-mappings/:userType/status-mappings": {
- "POST": api.PostIssueStatusMappings,
- "GET": api.ListIssueStatusMappings,
- },
- "connections/:connectionId/type-mappings/:userType/status-mappings/:userStatus": {
- "PUT": api.PutIssueStatusMapping,
- "DELETE": api.DeleteIssueStatusMapping,
- },
"connections/:connectionId/proxy/rest/*path": {
"GET": api.Proxy,
},
diff --git a/plugins/jira/models/connection.go b/plugins/jira/models/connection.go
index edb6b88f..f1ed2f8a 100644
--- a/plugins/jira/models/connection.go
+++ b/plugins/jira/models/connection.go
@@ -18,7 +18,7 @@ limitations under the License.
package models
import (
- "github.com/apache/incubator-devlake/models/common"
+ "github.com/apache/incubator-devlake/plugins/helper"
)
type EpicResponse struct {
@@ -28,9 +28,9 @@ type EpicResponse struct {
}
type TestConnectionRequest struct {
- Endpoint string `json:"endpoint"`
- Auth string `json:"auth"`
- Proxy string `json:"proxy"`
+ Endpoint string `json:"endpoint"`
+ Proxy string `json:"proxy"`
+ helper.BasicAuth `mapstructure:",squash"`
}
type BoardResponse struct {
@@ -40,15 +40,11 @@ type BoardResponse struct {
}
type JiraConnection struct {
- common.Model
- Name string `gorm:"type:varchar(100);uniqueIndex" json:"name" validate:"required"`
- Endpoint string `json:"endpoint" validate:"required"`
- BasicAuthEncoded string `json:"basicAuthEncoded" validate:"required"`
+ helper.RestConnection `mapstructure:",squash"`
+ helper.BasicAuth `mapstructure:",squash"`
EpicKeyField string `gorm:"type:varchar(50);" json:"epicKeyField"`
StoryPointField string `gorm:"type:varchar(50);" json:"storyPointField"`
RemotelinkCommitShaPattern string `gorm:"type:varchar(255);comment='golang regexp, the first group will be recognized as commit sha, ref https://github.com/google/re2/wiki/Syntax'" json:"remotelinkCommitShaPattern"`
- Proxy string `json:"proxy"`
- RateLimit int `comment:"api request rate limt per hour" json:"rateLimit"`
}
type JiraIssueTypeMapping struct {
diff --git a/plugins/jira/models/migrationscripts/updateSchemas20220505.go b/plugins/jira/models/migrationscripts/updateSchemas20220505.go
index 8e0e1610..ff4a6c81 100644
--- a/plugins/jira/models/migrationscripts/updateSchemas20220505.go
+++ b/plugins/jira/models/migrationscripts/updateSchemas20220505.go
@@ -19,16 +19,32 @@ package migrationscripts
import (
"context"
- "github.com/apache/incubator-devlake/plugins/jira/models"
+ "github.com/apache/incubator-devlake/models/common"
"github.com/apache/incubator-devlake/plugins/jira/models/migrationscripts/archived"
"gorm.io/gorm"
)
+type JiraConnection20220505 struct {
+ common.Model
+ Name string `gorm:"type:varchar(100);uniqueIndex" json:"name" validate:"required"`
+ Endpoint string `json:"endpoint" validate:"required"`
+ BasicAuthEncoded string `json:"basicAuthEncoded" validate:"required"`
+ EpicKeyField string `gorm:"type:varchar(50);" json:"epicKeyField"`
+ StoryPointField string `gorm:"type:varchar(50);" json:"storyPointField"`
+ RemotelinkCommitShaPattern string `gorm:"type:varchar(255);comment='golang regexp, the first group will be recognized as commit sha, ref https://github.com/google/re2/wiki/Syntax'" json:"remotelinkCommitShaPattern"`
+ Proxy string `json:"proxy"`
+ RateLimit int `comment:"api request rate limt per hour" json:"rateLimit"`
+}
+
+func (JiraConnection20220505) TableName() string {
+ return "_tool_jira_connections"
+}
+
type UpdateSchemas20220505 struct{}
func (*UpdateSchemas20220505) Up(ctx context.Context, db *gorm.DB) error {
- err := db.Migrator().RenameTable(archived.JiraSource{}, models.JiraConnection{})
+ err := db.Migrator().RenameTable(archived.JiraSource{}, JiraConnection20220505{})
if err != nil {
return err
}
diff --git a/plugins/jira/models/migrationscripts/updateSchemas20220524.go b/plugins/jira/models/migrationscripts/updateSchemas20220524.go
new file mode 100644
index 00000000..9c753610
--- /dev/null
+++ b/plugins/jira/models/migrationscripts/updateSchemas20220524.go
@@ -0,0 +1,72 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+ "context"
+ "github.com/apache/incubator-devlake/plugins/helper"
+ "gorm.io/gorm"
+)
+
+type JiraConnection20220524 struct {
+ helper.RestConnection
+ helper.BasicAuth
+ EpicKeyField string `gorm:"type:varchar(50);" json:"epicKeyField"`
+ StoryPointField string `gorm:"type:varchar(50);" json:"storyPointField"`
+ RemotelinkCommitShaPattern string `gorm:"type:varchar(255);comment='golang regexp, the first group will be recognized as commit sha, ref https://github.com/google/re2/wiki/Syntax'" json:"remotelinkCommitShaPattern"`
+}
+
+func (JiraConnection20220524) TableName() string {
+ return "_tool_jira_connections"
+}
+
+type UpdateSchemas20220524 struct{}
+
+func (*UpdateSchemas20220524) Up(ctx context.Context, db *gorm.DB) error {
+ var err error
+ if !db.Migrator().HasColumn(&JiraConnection20220505{}, "password") {
+ err = db.Migrator().AddColumn(&JiraConnection20220524{}, "password")
+ if err != nil {
+ return err
+ }
+ }
+
+ if !db.Migrator().HasColumn(&JiraConnection20220505{}, "username") {
+ err = db.Migrator().AddColumn(&JiraConnection20220524{}, "username")
+ if err != nil {
+ return err
+ }
+ }
+
+ if db.Migrator().HasColumn(&JiraConnection20220505{}, "basic_auth_encoded") {
+ err = db.Migrator().DropColumn(&JiraConnection20220505{}, "basic_auth_encoded")
+ if err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+func (*UpdateSchemas20220524) Version() uint64 {
+ return 20220507154646
+}
+
+func (*UpdateSchemas20220524) Name() string {
+ return "Add icon_url column to JiraIssue"
+}
diff --git a/plugins/jira/tasks/api_client.go b/plugins/jira/tasks/api_client.go
index 4896c05a..3673bee5 100644
--- a/plugins/jira/tasks/api_client.go
+++ b/plugins/jira/tasks/api_client.go
@@ -29,10 +29,12 @@ import (
func NewJiraApiClient(taskCtx core.TaskContext, connection *models.JiraConnection) (*helper.ApiAsyncClient, error) {
// load configuration
encKey := taskCtx.GetConfig(core.EncodeKeyEnvStr)
- auth, err := core.Decrypt(encKey, connection.BasicAuthEncoded)
+ decodedPassword, err := core.Decrypt(encKey, connection.Password)
if err != nil {
- return nil, fmt.Errorf("Failed to decrypt Auth Token: %w", err)
+ return nil, fmt.Errorf("Failed to decrypt Auth AccessToken: %w", err)
}
+ connection.Password = decodedPassword
+ auth := connection.GetEncodedToken()
// create synchronize api client so we can calculate api rate limit dynamically
headers := map[string]string{
@@ -44,7 +46,7 @@ func NewJiraApiClient(taskCtx core.TaskContext, connection *models.JiraConnectio
}
apiClient.SetAfterFunction(func(res *http.Response) error {
if res.StatusCode == http.StatusUnauthorized {
- return fmt.Errorf("authentication failed, please check your Basic Auth Token")
+ return fmt.Errorf("authentication failed, please check your AccessToken")
}
return nil
})
diff --git a/plugins/tapd/tasks/api_client.go b/plugins/tapd/tasks/api_client.go
index 406bc97a..977d076f 100644
--- a/plugins/tapd/tasks/api_client.go
+++ b/plugins/tapd/tasks/api_client.go
@@ -31,7 +31,7 @@ func NewTapdApiClient(taskCtx core.TaskContext, connection *models.TapdConnectio
encKey := taskCtx.GetConfig(core.EncodeKeyEnvStr)
auth, err := core.Decrypt(encKey, connection.BasicAuthEncoded)
if err != nil {
- return nil, fmt.Errorf("Failed to decrypt Auth Token: %w", err)
+ return nil, fmt.Errorf("Failed to decrypt Auth AccessToken: %w", err)
}
// create synchronize api client so we can calculate api rate limit dynamically
@@ -44,7 +44,7 @@ func NewTapdApiClient(taskCtx core.TaskContext, connection *models.TapdConnectio
}
apiClient.SetAfterFunction(func(res *http.Response) error {
if res.StatusCode == http.StatusUnprocessableEntity {
- return fmt.Errorf("authentication failed, please check your Basic Auth Token")
+ return fmt.Errorf("authentication failed, please check your AccessToken")
}
return nil
})
[incubator-devlake] 04/04: minor fixes
Posted by wa...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
warren pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit 1980cb39b70c92e117b84d84e1801b3a39dc543c
Author: Yingchu Chen <yi...@merico.dev>
AuthorDate: Thu Jun 2 16:57:24 2022 +0800
minor fixes
Signed-off-by: Yingchu Chen <yi...@merico.dev>
---
.../pages/configure/connections/ConnectionForm.jsx | 4 +-
config-ui/webpack.config.js | 18 ++-
config-ui/webpack.production.config.js | 18 ++-
plugins/gitlab/gitlab.go | 3 +
plugins/helper/connection.go | 118 +++++++++-------
plugins/helper/connection_test.go | 108 +++++++-------
plugins/helper/worker_scheduler_test.go | 2 +-
plugins/jira/api/connection.go | 155 ++-------------------
plugins/jira/api/init.go | 4 -
plugins/jira/api/proxy.go | 2 +-
plugins/jira/jira.go | 9 --
plugins/jira/models/connection.go | 15 --
.../migrationscripts/updateSchemas20220601.go | 7 +-
plugins/jira/tasks/api_client.go | 2 +-
plugins/jira/tasks/apiv2models/issue.go | 12 +-
plugins/jira/tasks/issue_extractor.go | 2 +-
plugins/tapd/api/connection.go | 27 ----
plugins/tapd/tapd.go | 8 +-
runner/directrun.go | 3 +-
19 files changed, 188 insertions(+), 329 deletions(-)
diff --git a/config-ui/src/pages/configure/connections/ConnectionForm.jsx b/config-ui/src/pages/configure/connections/ConnectionForm.jsx
index e5fe71d2..2e657c13 100644
--- a/config-ui/src/pages/configure/connections/ConnectionForm.jsx
+++ b/config-ui/src/pages/configure/connections/ConnectionForm.jsx
@@ -413,7 +413,7 @@ export default function ConnectionForm (props) {
? labels.username
: (
<>Username</>
- )}
+ )}
<span className='requiredStar'>*</span>
</Label>
<InputGroup
@@ -456,7 +456,7 @@ export default function ConnectionForm (props) {
? labels.password
: (
<>Password</>
- )}
+ )}
<span className='requiredStar'>*</span>
</Label>
<InputGroup
diff --git a/config-ui/webpack.config.js b/config-ui/webpack.config.js
index 5a253b25..51bd6a5f 100644
--- a/config-ui/webpack.config.js
+++ b/config-ui/webpack.config.js
@@ -1,4 +1,20 @@
-// DEVELOPMENT ONLY WEBPACK CONFIG
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
const path = require('path')
const webpack = require('webpack')
const BundleAnalyzerPlugin = require('webpack-bundle-analyzer').BundleAnalyzerPlugin
diff --git a/config-ui/webpack.production.config.js b/config-ui/webpack.production.config.js
index a8c34ae9..6989bd41 100644
--- a/config-ui/webpack.production.config.js
+++ b/config-ui/webpack.production.config.js
@@ -1,4 +1,20 @@
-/* eslint-disable import/no-extraneous-dependencies */
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
const path = require('path')
const webpack = require('webpack')
const BundleAnalyzerPlugin = require('webpack-bundle-analyzer').BundleAnalyzerPlugin
diff --git a/plugins/gitlab/gitlab.go b/plugins/gitlab/gitlab.go
index 7d2be2e5..e9a9f77d 100644
--- a/plugins/gitlab/gitlab.go
+++ b/plugins/gitlab/gitlab.go
@@ -41,6 +41,9 @@ func main() {
}
wsList := make([]*models.TapdWorkspace, 0)
err = db.Find(&wsList, "parent_id = ?", 59169984).Error
+ if err != nil {
+ panic(err)
+ }
projectList := []uint64{63281714,
34276182,
46319043,
diff --git a/plugins/helper/connection.go b/plugins/helper/connection.go
index 17933a5c..30e9ab0d 100644
--- a/plugins/helper/connection.go
+++ b/plugins/helper/connection.go
@@ -1,3 +1,20 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
package helper
import (
@@ -21,7 +38,7 @@ type BaseConnection struct {
type BasicAuth struct {
Username string `mapstructure:"username" validate:"required" json:"username"`
- Password string `mapstructure:"password" validate:"required" json:"password" encryptField:"yes"`
+ Password string `mapstructure:"password" validate:"required" json:"password" encrypt:"yes"`
}
func (ba BasicAuth) GetEncodedToken() string {
@@ -29,7 +46,7 @@ func (ba BasicAuth) GetEncodedToken() string {
}
type AccessToken struct {
- Token string `mapstructure:"token" validate:"required" json:"token" encryptField:"yes"`
+ Token string `mapstructure:"token" validate:"required" json:"token" encrypt:"yes"`
}
type RestConnection struct {
@@ -39,55 +56,50 @@ type RestConnection struct {
RateLimit int `comment:"api request rate limt per hour" json:"rateLimit"`
}
-// RefreshAndSaveConnection populate from request input into connection which come from REST functions to connection struct and save to DB
+// CreateConnection populate from request input into connection which come from REST functions to connection struct and save to DB
// and only change value which `data` has
// mergeFieldsToConnection merges fields from data
// `connection` is the pointer of a plugin connection
// `data` is http request input param
-func RefreshAndSaveConnection(connection interface{}, data map[string]interface{}, db *gorm.DB) error {
+func CreateConnection(data map[string]interface{}, connection interface{}, db *gorm.DB) error {
var err error
// update fields from request body
err = mergeFieldsToConnection(connection, data)
if err != nil {
return err
}
-
err = saveToDb(connection, db)
-
if err != nil {
return err
}
return nil
}
-func saveToDb(connection interface{}, db *gorm.DB) error {
- dataVal := reflect.ValueOf(connection)
- if dataVal.Kind() != reflect.Ptr {
- panic("entityPtr is not a pointer")
- }
- encKey, err := getEncKey()
+func PatchConnection(input *core.ApiResourceInput, connection interface{}, db *gorm.DB) error {
+ err := GetConnection(input.Params, connection, db)
if err != nil {
return err
}
- dataType := reflect.Indirect(dataVal).Type()
- fieldName := firstFieldNameWithTag(dataType, "encryptField")
- plainPwd := ""
- err = encryptField(dataVal, fieldName, encKey)
+
+ err = CreateConnection(input.Body, connection, db)
if err != nil {
return err
}
- err = db.Clauses(clause.OnConflict{UpdateAll: true}).Save(connection).Error
+
+ return nil
+}
+
+func saveToDb(connection interface{}, db *gorm.DB) error {
+ err := EncryptConnection(connection)
if err != nil {
return err
}
-
- err = decryptField(dataVal, fieldName, encKey)
+ err = db.Clauses(clause.OnConflict{UpdateAll: true}).Save(connection).Error
if err != nil {
return err
}
- dataVal.Elem().FieldByName(fieldName).Set(reflect.ValueOf(plainPwd))
- return err
+ return DecryptConnection(connection)
}
// mergeFieldsToConnection will populate all value in map to connection struct and validate the struct
@@ -110,7 +122,6 @@ func mergeFieldsToConnection(specificConnection interface{}, connections ...map[
}
func getEncKey() (string, error) {
- // encryptField
v := config.GetConfig()
encKey := v.GetString(core.EncodeKeyEnvStr)
if encKey == "" {
@@ -125,34 +136,44 @@ func getEncKey() (string, error) {
return encKey, nil
}
-// FindConnectionByInput finds connection from db by parsing request input and decrypt it
-func FindConnectionByInput(input *core.ApiResourceInput, connection interface{}, db *gorm.DB) error {
- dataVal := reflect.ValueOf(connection)
- if dataVal.Kind() != reflect.Ptr {
- return fmt.Errorf("connection is not a pointer")
- }
-
- id, err := GetConnectionIdByInputParam(input)
+// GetConnection finds connection from db by parsing request input and decrypt it
+func GetConnection(data map[string]string, connection interface{}, db *gorm.DB) error {
+ id, err := GetConnectionIdByInputParam(data)
if err != nil {
return fmt.Errorf("invalid connectionId")
}
err = db.First(connection, id).Error
if err != nil {
- fmt.Printf("--- %s", err.Error())
return err
}
- dataType := reflect.Indirect(dataVal).Type()
+ return DecryptConnection(connection)
- fieldName := firstFieldNameWithTag(dataType, "encryptField")
- return decryptField(dataVal, fieldName, "")
+}
+// ListConnections returns all connections with password/token decrypted
+func ListConnections(connections interface{}, db *gorm.DB) error {
+ err := db.Find(connections).Error
+ connPtr := reflect.ValueOf(connections)
+ connVal := reflect.Indirect(connPtr)
+ if err != nil {
+ return err
+ }
+ for i := 0; i < connVal.Len(); i++ {
+ //connVal.Index(i) returns value of ith elem in connections, .Elem() reutrns the original elem
+ tmp := connVal.Index(i).Elem()
+ err = DecryptConnection(tmp.Addr().Interface())
+ if err != nil {
+ return err
+ }
+ }
+ return nil
}
// GetConnectionIdByInputParam gets connectionId by parsing request input
-func GetConnectionIdByInputParam(input *core.ApiResourceInput) (uint64, error) {
- connectionId := input.Params["connectionId"]
+func GetConnectionIdByInputParam(data map[string]string) (uint64, error) {
+ connectionId := data["connectionId"]
if connectionId == "" {
return 0, fmt.Errorf("missing connectionId")
}
@@ -175,7 +196,7 @@ func firstFieldNameWithTag(t reflect.Type, tag string) string {
}
// DecryptConnection decrypts password/token field for connection
-func DecryptConnection(connection interface{}, fieldName string) error {
+func DecryptConnection(connection interface{}) error {
dataVal := reflect.ValueOf(connection)
if dataVal.Kind() != reflect.Ptr {
panic("connection is not a pointer")
@@ -184,23 +205,26 @@ func DecryptConnection(connection interface{}, fieldName string) error {
if err != nil {
return nil
}
- if len(fieldName) == 0 {
- dataType := reflect.Indirect(dataVal).Type()
- fieldName = firstFieldNameWithTag(dataType, "encryptField")
- }
- return decryptField(dataVal, fieldName, encKey)
-}
-
-func decryptField(dataVal reflect.Value, fieldName string, encKey string) error {
+ dataType := reflect.Indirect(dataVal).Type()
+ fieldName := firstFieldNameWithTag(dataType, "encrypt")
if len(fieldName) > 0 {
decryptStr, _ := core.Decrypt(encKey, dataVal.Elem().FieldByName(fieldName).String())
-
dataVal.Elem().FieldByName(fieldName).Set(reflect.ValueOf(decryptStr))
}
return nil
}
-func encryptField(dataVal reflect.Value, fieldName string, encKey string) error {
+func EncryptConnection(connection interface{}) error {
+ dataVal := reflect.ValueOf(connection)
+ if dataVal.Kind() != reflect.Ptr {
+ panic("connection is not a pointer")
+ }
+ encKey, err := getEncKey()
+ if err != nil {
+ return err
+ }
+ dataType := reflect.Indirect(dataVal).Type()
+ fieldName := firstFieldNameWithTag(dataType, "encrypt")
if len(fieldName) > 0 {
plainPwd := dataVal.Elem().FieldByName(fieldName).String()
encyptedStr, err := core.Encrypt(encKey, plainPwd)
diff --git a/plugins/helper/connection_test.go b/plugins/helper/connection_test.go
index 606730f3..bd1c7cc2 100644
--- a/plugins/helper/connection_test.go
+++ b/plugins/helper/connection_test.go
@@ -18,24 +18,24 @@ limitations under the License.
package helper
import (
- "github.com/apache/incubator-devlake/config"
- "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/stretchr/testify/assert"
"reflect"
"testing"
-
- "github.com/stretchr/testify/assert"
)
-type TestConnection struct {
+type MockConnection struct {
RestConnection `mapstructure:",squash"`
BasicAuth `mapstructure:",squash"`
- EpicKeyField string `gorm:"type:varchar(50);" json:"epicKeyField"`
StoryPointField string `gorm:"type:varchar(50);" json:"storyPointField"`
RemotelinkCommitShaPattern string `gorm:"type:varchar(255);comment='golang regexp, the first group will be recognized as commit sha, ref https://github.com/google/re2/wiki/Syntax'" json:"remotelinkCommitShaPattern"`
}
+func (MockConnection) TableName() string {
+ return "_tool_jira_connections"
+}
+
func TestMergeFieldsToConnection(t *testing.T) {
- v := &TestConnection{
+ v := &MockConnection{
RestConnection: RestConnection{
BaseConnection: BaseConnection{
Name: "1",
@@ -48,8 +48,6 @@ func TestMergeFieldsToConnection(t *testing.T) {
Username: "4",
Password: "5",
},
- EpicKeyField: "6",
- StoryPointField: "7",
RemotelinkCommitShaPattern: "8",
}
data := make(map[string]interface{})
@@ -58,9 +56,7 @@ func TestMergeFieldsToConnection(t *testing.T) {
data["Password"] = "5-5"
err := mergeFieldsToConnection(v, data)
- if err != nil {
- return
- }
+ assert.Nil(t, err)
assert.Equal(t, "4-4", v.Username)
assert.Equal(t, "2-2", v.Endpoint)
@@ -68,7 +64,7 @@ func TestMergeFieldsToConnection(t *testing.T) {
}
func TestDecryptAndEncrypt(t *testing.T) {
- v := &TestConnection{
+ v := &MockConnection{
RestConnection: RestConnection{
BaseConnection: BaseConnection{
Name: "1",
@@ -81,28 +77,21 @@ func TestDecryptAndEncrypt(t *testing.T) {
Username: "4",
Password: "5",
},
- EpicKeyField: "6",
- StoryPointField: "7",
RemotelinkCommitShaPattern: "8",
}
- dataVal := reflect.ValueOf(v)
- encKey := "test"
- err := encryptField(dataVal, "Password", encKey)
- if err != nil {
- return
- }
+ err := EncryptConnection(v)
+ assert.Nil(t, err)
+
assert.NotEqual(t, "5", v.Password)
- err = decryptField(dataVal, "Password", encKey)
- if err != nil {
- return
- }
+ err = DecryptConnection(v)
+ assert.Nil(t, err)
assert.Equal(t, "5", v.Password)
}
func TestDecryptConnection(t *testing.T) {
- v := &TestConnection{
+ v := &MockConnection{
RestConnection: RestConnection{
BaseConnection: BaseConnection{
Name: "1",
@@ -115,46 +104,20 @@ func TestDecryptConnection(t *testing.T) {
Username: "4",
Password: "5",
},
- EpicKeyField: "6",
- StoryPointField: "7",
RemotelinkCommitShaPattern: "8",
}
- encKey, err := getEncKey()
- if err != nil {
- return
- }
- dataVal := reflect.ValueOf(v)
- err = encryptField(dataVal, "Password", encKey)
- if err != nil {
- return
- }
+ err := EncryptConnection(v)
+ assert.Nil(t, err)
+
encryptedPwd := v.Password
- err = DecryptConnection(v, "Password")
- if err != nil {
- return
- }
+ err = DecryptConnection(v)
+ assert.Nil(t, err)
assert.NotEqual(t, encryptedPwd, v.Password)
assert.Equal(t, "5", v.Password)
}
-func TestGetEncKey(t *testing.T) {
- // encryptField
- v := config.GetConfig()
- encKey := v.GetString(core.EncodeKeyEnvStr)
- str, err := getEncKey()
- if err != nil {
- return
- }
- if len(encKey) > 0 {
- assert.Equal(t, encKey, str)
- } else {
- assert.NotEqual(t, 0, len(str))
- }
-
-}
-
func TestFirstFieldNameWithTag(t *testing.T) {
- v := &TestConnection{
+ v := &MockConnection{
RestConnection: RestConnection{
BaseConnection: BaseConnection{
Name: "1",
@@ -167,12 +130,37 @@ func TestFirstFieldNameWithTag(t *testing.T) {
Username: "4",
Password: "5",
},
- EpicKeyField: "6",
StoryPointField: "7",
RemotelinkCommitShaPattern: "8",
}
dataVal := reflect.ValueOf(v)
dataType := reflect.Indirect(dataVal).Type()
- fieldName := firstFieldNameWithTag(dataType, "encryptField")
+ fieldName := firstFieldNameWithTag(dataType, "encrypt")
assert.Equal(t, "Password", fieldName)
}
+
+//func TestListConnections(t *testing.T) {
+// jiraConnections := make([]*MockConnection, 0)
+// cfg := config.GetConfig()
+// dbUrl := cfg.GetString("DB_URL")
+// u, err := url.Parse(dbUrl)
+// dbUrl = fmt.Sprintf("%s@tcp(%s)%s?%s", u.User.String(), u.Host, u.Path, u.RawQuery)
+// dbConfig := &gorm.Config{
+// Logger: gormLogger.New(
+// log.Default(),
+// gormLogger.Config{
+// SlowThreshold: time.Second, // Slow SQL threshold
+// LogLevel: gormLogger.Error, // Log level
+// IgnoreRecordNotFoundError: true, // Ignore ErrRecordNotFound error for logger
+// Colorful: true, // Disable color
+// },
+// ),
+// // most of our operation are in batch, this can improve performance
+// PrepareStmt: true,
+// }
+// db, err := gorm.Open(mysql.Open(dbUrl), dbConfig)
+//
+// err = ListConnections(&jiraConnections, db)
+//
+// assert.Nil(t, err)
+//}
diff --git a/plugins/helper/worker_scheduler_test.go b/plugins/helper/worker_scheduler_test.go
index a3de7adb..4f65e7a0 100644
--- a/plugins/helper/worker_scheduler_test.go
+++ b/plugins/helper/worker_scheduler_test.go
@@ -88,7 +88,7 @@ func TestNewWorkerSchedulerWithoutSecond(t *testing.T) {
func TestNewWorkerSchedulerWithPanic(t *testing.T) {
testChannel := make(chan int, 100)
ctx, cancel := context.WithCancel(context.Background())
- s, _ := NewWorkerScheduler(1, 1, ctx)
+ s,_ := NewWorkerScheduler(1, 1, ctx)
defer s.Release()
_ = s.Submit(func() error {
testChannel <- 1
diff --git a/plugins/jira/api/connection.go b/plugins/jira/api/connection.go
index aa94537a..0c422bbe 100644
--- a/plugins/jira/api/connection.go
+++ b/plugins/jira/api/connection.go
@@ -21,7 +21,6 @@ import (
"fmt"
"net/http"
"net/url"
- "strconv"
"strings"
"time"
@@ -103,19 +102,8 @@ POST /plugins/jira/connections
{
"name": "jira data connection name",
"endpoint": "jira api endpoint, i.e. https://merico.atlassian.net/rest",
- "basicAuthEncoded": "generated by `echo -n <jira login email>:<jira token> | base64`",
- "epicKeyField": "name of customfield of epic key",
- "storyPointField": "name of customfield of story point",
- "typeMappings": { // optional, send empty object to delete all typeMappings of the data connection
- "userType": {
- "standardType": "devlake standard type",
- "statusMappings": { // optional, send empt object to delete all status mapping for the user type
- "userStatus": {
- "standardStatus": "devlake standard status"
- }
- }
- }
- }
+ "username": "username, usually should be email address",
+ "password": "jira api access token"
}
*/
func PostConnections(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) {
@@ -123,7 +111,7 @@ func PostConnections(input *core.ApiResourceInput) (*core.ApiResourceOutput, err
jiraConnection := &models.JiraConnection{}
// update from request and save to database
- err := helper.RefreshAndSaveConnection(jiraConnection, input.Body, db)
+ err := helper.CreateConnection(input.Body, jiraConnection, db)
if err != nil {
return nil, err
}
@@ -136,31 +124,13 @@ PATCH /plugins/jira/connections/:connectionId
{
"name": "jira data connection name",
"endpoint": "jira api endpoint, i.e. https://merico.atlassian.net/rest",
- "basicAuthEncoded": "generated by `echo -n <jira login email>:<jira token> | base64`",
- "epicKeyField": "name of customfield of epic key",
- "storyPointField": "name of customfield of story point",
- "typeMappings": { // optional, send empty object to delete all typeMappings of the data connection
- "userType": {
- "standardType": "devlake standard type",
- "statusMappings": { // optional, send empt object to delete all status mapping for the user type
- "userStatus": {
- "standardStatus": "devlake standard status"
- }
- }
- }
- }
+ "username": "username, usually should be email address",
+ "password": "jira api access token"
}
*/
func PatchConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) {
jiraConnection := &models.JiraConnection{}
- // load from db
- err := helper.FindConnectionByInput(input, jiraConnection, db)
- if err != nil {
- return nil, err
- }
-
- // update from request and save to database
- err = helper.RefreshAndSaveConnection(jiraConnection, input.Body, db)
+ err := helper.PatchConnection(input, jiraConnection, db)
if err != nil {
return nil, err
}
@@ -173,7 +143,7 @@ DELETE /plugins/jira/connections/:connectionId
*/
func DeleteConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) {
// load from db
- jiraConnectionID, err := helper.GetConnectionIdByInputParam(input)
+ jiraConnectionID, err := helper.GetConnectionIdByInputParam(input.Params)
if err != nil {
return nil, err
}
@@ -182,15 +152,6 @@ func DeleteConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput, er
if err != nil {
return nil, err
}
- err = db.Where("connection_id = ?", jiraConnectionID).Delete(&models.JiraIssueTypeMapping{}).Error
- if err != nil {
- return nil, err
- }
- err = db.Where("connection_id = ?", jiraConnectionID).Delete(&models.JiraIssueStatusMapping{}).Error
- if err != nil {
- return nil, err
- }
-
return &core.ApiResourceOutput{Body: jiraConnectionID}, nil
}
@@ -199,17 +160,11 @@ GET /plugins/jira/connections
*/
func ListConnections(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) {
jiraConnections := make([]*models.JiraConnection, 0)
- err := db.Find(&jiraConnections).Error
+
+ err := helper.ListConnections(&jiraConnections, db)
if err != nil {
return nil, err
}
- for i, _ := range jiraConnections {
- err = helper.DecryptConnection(jiraConnections[i], "Password")
- if err != nil {
- return nil, err
- }
- }
-
return &core.ApiResourceOutput{Body: jiraConnections, Status: http.StatusOK}, nil
}
@@ -220,100 +175,16 @@ GET /plugins/jira/connections/:connectionId
{
"name": "jira data connection name",
"endpoint": "jira api endpoint, i.e. https://merico.atlassian.net/rest",
- "basicAuthEncoded": "generated by `echo -n <jira login email>:<jira token> | base64`",
- "epicKeyField": "name of customfield of epic key",
- "storyPointField": "name of customfield of story point",
- "typeMappings": { // optional, send empty object to delete all typeMappings of the data connection
- "userType": {
- "standardType": "devlake standard type",
- "statusMappings": { // optional, send empt object to delete all status mapping for the user type
- "userStatus": {
- "standardStatus": "devlake standard status"
- }
- }
- }
- }
+ "username": "username, usually should be email address",
+ "password": "jira api access token"
}
*/
func GetConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) {
jiraConnection := &models.JiraConnection{}
- err := helper.FindConnectionByInput(input, jiraConnection, db)
+ err := helper.GetConnection(input.Params, jiraConnection, db)
if err != nil {
return nil, err
}
- detail := &models.JiraConnectionDetail{
- JiraConnection: *jiraConnection,
- }
-
- if err != nil {
- return nil, err
- }
-
- return &core.ApiResourceOutput{Body: detail}, nil
-}
-
-// GET /plugins/jira/connections/:connectionId/epics
-func GetEpicsByConnectionId(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) {
- jiraConnection := &models.JiraConnection{}
- err := helper.FindConnectionByInput(input, jiraConnection, db)
- if err != nil {
- return nil, err
- }
- return &core.ApiResourceOutput{Body: [1]models.EpicResponse{{
- Id: 1,
- Title: jiraConnection.EpicKeyField,
- Value: jiraConnection.EpicKeyField,
- }}}, nil
-}
-
-// GET /plugins/jira/connections/:connectionId/granularities
-type GranularitiesResponse struct {
- Id int
- Title string
- Value string
-}
-
-func GetGranularitiesByConnectionId(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) {
- jiraConnection := &models.JiraConnection{}
- err := helper.FindConnectionByInput(input, jiraConnection, db)
- if err != nil {
- return nil, err
- }
- if err != nil {
- return nil, err
- }
- return &core.ApiResourceOutput{Body: [1]GranularitiesResponse{
- {
- Id: 1,
- Title: "Story Point Field",
- Value: jiraConnection.StoryPointField,
- },
- }}, nil
-}
-
-// GET /plugins/jira/connections/:connectionId/boards
-func GetBoardsByConnectionId(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) {
- connectionId := input.Params["connectionId"]
- if connectionId == "" {
- return nil, fmt.Errorf("missing connectionid")
- }
- jiraConnectionId, err := strconv.ParseUint(connectionId, 10, 64)
- if err != nil {
- return nil, fmt.Errorf("invalid connectionId")
- }
- var jiraBoards []models.JiraBoard
- err = db.Where("connection_Id = ?", jiraConnectionId).Find(&jiraBoards).Error
- if err != nil {
- return nil, err
- }
- var boardResponses []models.BoardResponse
- for _, board := range jiraBoards {
- boardResponses = append(boardResponses, models.BoardResponse{
- Id: int(board.BoardId),
- Title: board.Name,
- Value: fmt.Sprintf("%v", board.BoardId),
- })
- }
- return &core.ApiResourceOutput{Body: boardResponses}, nil
+ return &core.ApiResourceOutput{Body: jiraConnection}, nil
}
diff --git a/plugins/jira/api/init.go b/plugins/jira/api/init.go
index 663d5033..acaa495e 100644
--- a/plugins/jira/api/init.go
+++ b/plugins/jira/api/init.go
@@ -24,11 +24,7 @@ import (
)
var db *gorm.DB
-var cfg *viper.Viper
-var log core.Logger
func Init(config *viper.Viper, logger core.Logger, database *gorm.DB) {
db = database
- cfg = config
- log = logger
}
diff --git a/plugins/jira/api/proxy.go b/plugins/jira/api/proxy.go
index e4a3eb90..398f4f1f 100644
--- a/plugins/jira/api/proxy.go
+++ b/plugins/jira/api/proxy.go
@@ -35,7 +35,7 @@ const (
func Proxy(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) {
jiraConnection := &models.JiraConnection{}
- err := helper.FindConnectionByInput(input, jiraConnection, db)
+ err := helper.GetConnection(input.Params, jiraConnection, db)
if err != nil {
return nil, err
}
diff --git a/plugins/jira/jira.go b/plugins/jira/jira.go
index a36177da..37313972 100644
--- a/plugins/jira/jira.go
+++ b/plugins/jira/jira.go
@@ -176,15 +176,6 @@ func (plugin Jira) ApiResources() map[string]map[string]core.ApiResourceHandler
"DELETE": api.DeleteConnection,
"GET": api.GetConnection,
},
- "connections/:connectionId/epics": {
- "GET": api.GetEpicsByConnectionId,
- },
- "connections/:connectionId/granularities": {
- "GET": api.GetGranularitiesByConnectionId,
- },
- "connections/:connectionId/boards": {
- "GET": api.GetBoardsByConnectionId,
- },
"connections/:connectionId/proxy/rest/*path": {
"GET": api.Proxy,
},
diff --git a/plugins/jira/models/connection.go b/plugins/jira/models/connection.go
index f1ed2f8a..fa0664f6 100644
--- a/plugins/jira/models/connection.go
+++ b/plugins/jira/models/connection.go
@@ -42,8 +42,6 @@ type BoardResponse struct {
type JiraConnection struct {
helper.RestConnection `mapstructure:",squash"`
helper.BasicAuth `mapstructure:",squash"`
- EpicKeyField string `gorm:"type:varchar(50);" json:"epicKeyField"`
- StoryPointField string `gorm:"type:varchar(50);" json:"storyPointField"`
RemotelinkCommitShaPattern string `gorm:"type:varchar(255);comment='golang regexp, the first group will be recognized as commit sha, ref https://github.com/google/re2/wiki/Syntax'" json:"remotelinkCommitShaPattern"`
}
@@ -60,19 +58,6 @@ type JiraIssueStatusMapping struct {
StandardStatus string `gorm:"type:varchar(50)" json:"standardStatus" validate:"required"`
}
-type JiraConnectionDetail struct {
- JiraConnection
- TypeMappings map[string]map[string]interface{} `json:"typeMappings"`
-}
-
func (JiraConnection) TableName() string {
return "_tool_jira_connections"
}
-
-func (JiraIssueTypeMapping) TableName() string {
- return "_tool_jira_issue_type_mappings"
-}
-
-func (JiraIssueStatusMapping) TableName() string {
- return "_tool_jira_issue_status_mappings"
-}
diff --git a/plugins/jira/models/migrationscripts/updateSchemas20220601.go b/plugins/jira/models/migrationscripts/updateSchemas20220601.go
index ec528dcb..24d3786c 100644
--- a/plugins/jira/models/migrationscripts/updateSchemas20220601.go
+++ b/plugins/jira/models/migrationscripts/updateSchemas20220601.go
@@ -58,9 +58,12 @@ func (*UpdateSchemas20220601) Up(ctx context.Context, db *gorm.DB) error {
if db.Migrator().HasColumn(&JiraConnection20220505{}, "basic_auth_encoded") {
connections := make([]*JiraConnection20220505, 0)
- db.Find(&connections)
+ err = db.Find(&connections).Error
+ if err != nil {
+ return err
+ }
for i, _ := range connections {
- err = helper.DecryptConnection(connections[i], "BasicAuthEncoded")
+ err = helper.DecryptConnection(connections[i])
if err != nil {
return err
}
diff --git a/plugins/jira/tasks/api_client.go b/plugins/jira/tasks/api_client.go
index 811c0b6a..b44b024b 100644
--- a/plugins/jira/tasks/api_client.go
+++ b/plugins/jira/tasks/api_client.go
@@ -28,7 +28,7 @@ import (
func NewJiraApiClient(taskCtx core.TaskContext, connection *models.JiraConnection) (*helper.ApiAsyncClient, error) {
// decrypt connection first
- err := helper.DecryptConnection(connection, "Password")
+ err := helper.DecryptConnection(connection)
if err != nil {
return nil, fmt.Errorf("Failed to decrypt Auth AccessToken: %w", err)
}
diff --git a/plugins/jira/tasks/apiv2models/issue.go b/plugins/jira/tasks/apiv2models/issue.go
index 3544150e..591ff7b7 100644
--- a/plugins/jira/tasks/apiv2models/issue.go
+++ b/plugins/jira/tasks/apiv2models/issue.go
@@ -151,11 +151,8 @@ type Issue struct {
} `json:"changelog"`
}
-func (i Issue) toToolLayer(connectionId uint64, epicField, storyPointField string) *models.JiraIssue {
+func (i Issue) toToolLayer(connectionId uint64) *models.JiraIssue {
var workload float64
- if storyPointField != "" {
- workload, _ = i.Fields.AllFields[storyPointField].(float64)
- }
result := &models.JiraIssue{
ConnectionId: connectionId,
IssueId: i.ID,
@@ -177,9 +174,6 @@ func (i Issue) toToolLayer(connectionId uint64, epicField, storyPointField strin
if i.Fields.Epic != nil {
result.EpicKey = i.Fields.Epic.Key
}
- if epicField != "" {
- result.EpicKey, _ = i.Fields.AllFields[epicField].(string)
- }
if i.Fields.Assignee != nil {
result.AssigneeAccountId = i.Fields.Assignee.getAccountId()
result.AssigneeDisplayName = i.Fields.Assignee.DisplayName
@@ -227,8 +221,8 @@ func (i *Issue) SetAllFields(raw datatypes.JSON) error {
return nil
}
-func (i Issue) ExtractEntities(connectionId uint64, epicField, storyPointField string) ([]uint64, *models.JiraIssue, bool, []*models.JiraWorklog, []*models.JiraChangelog, []*models.JiraChangelogItem, []*models.JiraUser) {
- issue := i.toToolLayer(connectionId, epicField, storyPointField)
+func (i Issue) ExtractEntities(connectionId uint64) ([]uint64, *models.JiraIssue, bool, []*models.JiraWorklog, []*models.JiraChangelog, []*models.JiraChangelogItem, []*models.JiraUser) {
+ issue := i.toToolLayer(connectionId)
var worklogs []*models.JiraWorklog
var changelogs []*models.JiraChangelog
var changelogItems []*models.JiraChangelogItem
diff --git a/plugins/jira/tasks/issue_extractor.go b/plugins/jira/tasks/issue_extractor.go
index 17882df8..266be502 100644
--- a/plugins/jira/tasks/issue_extractor.go
+++ b/plugins/jira/tasks/issue_extractor.go
@@ -106,7 +106,7 @@ func ExtractIssues(taskCtx core.SubTaskContext) error {
return nil, err
}
var results []interface{}
- sprints, issue, _, worklogs, changelogs, changelogItems, users := apiIssue.ExtractEntities(data.Connection.ID, data.Connection.EpicKeyField, data.Connection.StoryPointField)
+ sprints, issue, _, worklogs, changelogs, changelogItems, users := apiIssue.ExtractEntities(data.Connection.ID)
for _, sprintId := range sprints {
sprintIssue := &models.JiraSprintIssue{
ConnectionId: data.Connection.ID,
diff --git a/plugins/tapd/api/connection.go b/plugins/tapd/api/connection.go
index 5065f096..96c6bbba 100644
--- a/plugins/tapd/api/connection.go
+++ b/plugins/tapd/api/connection.go
@@ -281,30 +281,3 @@ func GetConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput, error
}
return &core.ApiResourceOutput{Body: detail}, nil
}
-
-// GET /plugins/tapd/connections/:connectionId/boards
-
-func GetBoardsByConnectionId(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) {
- connectionId := input.Params["connectionId"]
- if connectionId == "" {
- return nil, fmt.Errorf("missing connectionId")
- }
- tapdConnectionId, err := strconv.ParseUint(connectionId, 10, 64)
- if err != nil {
- return nil, fmt.Errorf("invalid connectionId")
- }
- var tapdWorkspaces []models.TapdWorkspace
- err = db.Where("connection_Id = ?", tapdConnectionId).Find(&tapdWorkspaces).Error
- if err != nil {
- return nil, err
- }
- var workSpaceResponses []models.WorkspaceResponse
- for _, workSpace := range tapdWorkspaces {
- workSpaceResponses = append(workSpaceResponses, models.WorkspaceResponse{
- Id: uint64(workSpace.ID),
- Title: workSpace.Name,
- Value: fmt.Sprintf("%v", workSpace.ID),
- })
- }
- return &core.ApiResourceOutput{Body: workSpaceResponses}, nil
-}
diff --git a/plugins/tapd/tapd.go b/plugins/tapd/tapd.go
index ea556438..c74d278d 100644
--- a/plugins/tapd/tapd.go
+++ b/plugins/tapd/tapd.go
@@ -180,9 +180,6 @@ func (plugin Tapd) ApiResources() map[string]map[string]core.ApiResourceHandler
"DELETE": api.DeleteConnection,
"GET": api.GetConnection,
},
- "connections/:connectionId/boards": {
- "GET": api.GetBoardsByConnectionId,
- },
"connections/:connectionId/proxy/rest/*path": {
"GET": api.Proxy,
},
@@ -220,7 +217,10 @@ func main() {
panic(err)
}
wsList := make([]*models.TapdWorkspace, 0)
- err = db.Find(&wsList, "parent_id = ?", 59169984).Error //nolint TODO: fix the unused err
+ err = db.Find(&wsList, "parent_id = ?", 59169984).Error
+ if err != nil {
+ panic(err)
+ }
for _, v := range wsList {
*workspaceId = v.ID
runner.DirectRun(c, args, PluginEntry, []string{}, map[string]interface{}{
diff --git a/runner/directrun.go b/runner/directrun.go
index 6ab5b83d..e2343cd4 100644
--- a/runner/directrun.go
+++ b/runner/directrun.go
@@ -47,7 +47,6 @@ func DirectRun(cmd *cobra.Command, args []string, pluginTask core.PluginTask, su
if err != nil {
panic(err)
}
- subtasks = tasks
cfg := config.GetConfig()
log := logger.Global.Nested(cmd.Use)
db, err := NewGormDb(cfg, log)
@@ -102,7 +101,7 @@ func DirectRun(cmd *cobra.Command, args []string, pluginTask core.PluginTask, su
db,
ctx,
cmd.Use,
- subtasks,
+ tasks,
options,
pluginTask,
nil,
[incubator-devlake] 03/04: unit test for connection
Posted by wa...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
warren pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit 7b485ed07d871150946de3390fd7e3e6fb9e757a
Author: Yingchu Chen <yi...@merico.dev>
AuthorDate: Wed Jun 1 19:07:30 2022 +0800
unit test for connection
Signed-off-by: Yingchu Chen <yi...@merico.dev>
---
plugins/helper/connection.go | 49 +++++------
plugins/helper/connection_test.go | 178 ++++++++++++++++++++++++++++++++++++++
plugins/jira/tasks/api_client.go | 9 +-
3 files changed, 205 insertions(+), 31 deletions(-)
diff --git a/plugins/helper/connection.go b/plugins/helper/connection.go
index 85163068..17933a5c 100644
--- a/plugins/helper/connection.go
+++ b/plugins/helper/connection.go
@@ -21,7 +21,7 @@ type BaseConnection struct {
type BasicAuth struct {
Username string `mapstructure:"username" validate:"required" json:"username"`
- Password string `mapstructure:"password" validate:"required" json:"password" encrypt:"yes"`
+ Password string `mapstructure:"password" validate:"required" json:"password" encryptField:"yes"`
}
func (ba BasicAuth) GetEncodedToken() string {
@@ -29,7 +29,7 @@ func (ba BasicAuth) GetEncodedToken() string {
}
type AccessToken struct {
- Token string `mapstructure:"token" validate:"required" json:"token" encrypt:"yes"`
+ Token string `mapstructure:"token" validate:"required" json:"token" encryptField:"yes"`
}
type RestConnection struct {
@@ -65,11 +65,14 @@ func saveToDb(connection interface{}, db *gorm.DB) error {
if dataVal.Kind() != reflect.Ptr {
panic("entityPtr is not a pointer")
}
-
+ encKey, err := getEncKey()
+ if err != nil {
+ return err
+ }
dataType := reflect.Indirect(dataVal).Type()
- fieldName := getEncryptField(dataType, "encrypt")
+ fieldName := firstFieldNameWithTag(dataType, "encryptField")
plainPwd := ""
- err := doEncrypt(dataVal, fieldName)
+ err = encryptField(dataVal, fieldName, encKey)
if err != nil {
return err
}
@@ -78,7 +81,7 @@ func saveToDb(connection interface{}, db *gorm.DB) error {
return err
}
- err = doDecrypt(dataVal, fieldName)
+ err = decryptField(dataVal, fieldName, encKey)
if err != nil {
return err
}
@@ -107,7 +110,7 @@ func mergeFieldsToConnection(specificConnection interface{}, connections ...map[
}
func getEncKey() (string, error) {
- // encrypt
+ // encryptField
v := config.GetConfig()
encKey := v.GetString(core.EncodeKeyEnvStr)
if encKey == "" {
@@ -142,8 +145,8 @@ func FindConnectionByInput(input *core.ApiResourceInput, connection interface{},
dataType := reflect.Indirect(dataVal).Type()
- fieldName := getEncryptField(dataType, "encrypt")
- return doDecrypt(dataVal, fieldName)
+ fieldName := firstFieldNameWithTag(dataType, "encryptField")
+ return decryptField(dataVal, fieldName, "")
}
@@ -156,12 +159,12 @@ func GetConnectionIdByInputParam(input *core.ApiResourceInput) (uint64, error) {
return strconv.ParseUint(connectionId, 10, 64)
}
-func getEncryptField(t reflect.Type, tag string) string {
+func firstFieldNameWithTag(t reflect.Type, tag string) string {
fieldName := ""
for i := 0; i < t.NumField(); i++ {
field := t.Field(i)
if field.Type.Kind() == reflect.Struct {
- fieldName = getEncryptField(field.Type, tag)
+ fieldName = firstFieldNameWithTag(field.Type, tag)
} else {
if field.Tag.Get(tag) == "yes" {
fieldName = field.Name
@@ -177,34 +180,30 @@ func DecryptConnection(connection interface{}, fieldName string) error {
if dataVal.Kind() != reflect.Ptr {
panic("connection is not a pointer")
}
+ encKey, err := getEncKey()
+ if err != nil {
+ return nil
+ }
if len(fieldName) == 0 {
dataType := reflect.Indirect(dataVal).Type()
- fieldName = getEncryptField(dataType, "encrypt")
+ fieldName = firstFieldNameWithTag(dataType, "encryptField")
}
- return doDecrypt(dataVal, fieldName)
+ return decryptField(dataVal, fieldName, encKey)
}
-func doDecrypt(dataVal reflect.Value, fieldName string) error {
- encryptCode, err := getEncKey()
- if err != nil {
- return err
- }
+func decryptField(dataVal reflect.Value, fieldName string, encKey string) error {
if len(fieldName) > 0 {
- decryptStr, _ := core.Decrypt(encryptCode, dataVal.Elem().FieldByName(fieldName).String())
+ decryptStr, _ := core.Decrypt(encKey, dataVal.Elem().FieldByName(fieldName).String())
dataVal.Elem().FieldByName(fieldName).Set(reflect.ValueOf(decryptStr))
}
return nil
}
-func doEncrypt(dataVal reflect.Value, fieldName string) error {
- encryptCode, err := getEncKey()
- if err != nil {
- return err
- }
+func encryptField(dataVal reflect.Value, fieldName string, encKey string) error {
if len(fieldName) > 0 {
plainPwd := dataVal.Elem().FieldByName(fieldName).String()
- encyptedStr, err := core.Encrypt(encryptCode, plainPwd)
+ encyptedStr, err := core.Encrypt(encKey, plainPwd)
if err != nil {
return err
diff --git a/plugins/helper/connection_test.go b/plugins/helper/connection_test.go
new file mode 100644
index 00000000..606730f3
--- /dev/null
+++ b/plugins/helper/connection_test.go
@@ -0,0 +1,178 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package helper
+
+import (
+ "github.com/apache/incubator-devlake/config"
+ "github.com/apache/incubator-devlake/plugins/core"
+ "reflect"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+)
+
+type TestConnection struct {
+ RestConnection `mapstructure:",squash"`
+ BasicAuth `mapstructure:",squash"`
+ EpicKeyField string `gorm:"type:varchar(50);" json:"epicKeyField"`
+ StoryPointField string `gorm:"type:varchar(50);" json:"storyPointField"`
+ RemotelinkCommitShaPattern string `gorm:"type:varchar(255);comment='golang regexp, the first group will be recognized as commit sha, ref https://github.com/google/re2/wiki/Syntax'" json:"remotelinkCommitShaPattern"`
+}
+
+func TestMergeFieldsToConnection(t *testing.T) {
+ v := &TestConnection{
+ RestConnection: RestConnection{
+ BaseConnection: BaseConnection{
+ Name: "1",
+ },
+ Endpoint: "2",
+ Proxy: "3",
+ RateLimit: 0,
+ },
+ BasicAuth: BasicAuth{
+ Username: "4",
+ Password: "5",
+ },
+ EpicKeyField: "6",
+ StoryPointField: "7",
+ RemotelinkCommitShaPattern: "8",
+ }
+ data := make(map[string]interface{})
+ data["Endpoint"] = "2-2"
+ data["Username"] = "4-4"
+ data["Password"] = "5-5"
+
+ err := mergeFieldsToConnection(v, data)
+ if err != nil {
+ return
+ }
+
+ assert.Equal(t, "4-4", v.Username)
+ assert.Equal(t, "2-2", v.Endpoint)
+ assert.Equal(t, "5-5", v.Password)
+}
+
+func TestDecryptAndEncrypt(t *testing.T) {
+ v := &TestConnection{
+ RestConnection: RestConnection{
+ BaseConnection: BaseConnection{
+ Name: "1",
+ },
+ Endpoint: "2",
+ Proxy: "3",
+ RateLimit: 0,
+ },
+ BasicAuth: BasicAuth{
+ Username: "4",
+ Password: "5",
+ },
+ EpicKeyField: "6",
+ StoryPointField: "7",
+ RemotelinkCommitShaPattern: "8",
+ }
+ dataVal := reflect.ValueOf(v)
+ encKey := "test"
+ err := encryptField(dataVal, "Password", encKey)
+ if err != nil {
+ return
+ }
+ assert.NotEqual(t, "5", v.Password)
+ err = decryptField(dataVal, "Password", encKey)
+ if err != nil {
+ return
+ }
+
+ assert.Equal(t, "5", v.Password)
+
+}
+
+func TestDecryptConnection(t *testing.T) {
+ v := &TestConnection{
+ RestConnection: RestConnection{
+ BaseConnection: BaseConnection{
+ Name: "1",
+ },
+ Endpoint: "2",
+ Proxy: "3",
+ RateLimit: 0,
+ },
+ BasicAuth: BasicAuth{
+ Username: "4",
+ Password: "5",
+ },
+ EpicKeyField: "6",
+ StoryPointField: "7",
+ RemotelinkCommitShaPattern: "8",
+ }
+ encKey, err := getEncKey()
+ if err != nil {
+ return
+ }
+ dataVal := reflect.ValueOf(v)
+ err = encryptField(dataVal, "Password", encKey)
+ if err != nil {
+ return
+ }
+ encryptedPwd := v.Password
+ err = DecryptConnection(v, "Password")
+ if err != nil {
+ return
+ }
+ assert.NotEqual(t, encryptedPwd, v.Password)
+ assert.Equal(t, "5", v.Password)
+}
+
+func TestGetEncKey(t *testing.T) {
+ // encryptField
+ v := config.GetConfig()
+ encKey := v.GetString(core.EncodeKeyEnvStr)
+ str, err := getEncKey()
+ if err != nil {
+ return
+ }
+ if len(encKey) > 0 {
+ assert.Equal(t, encKey, str)
+ } else {
+ assert.NotEqual(t, 0, len(str))
+ }
+
+}
+
+func TestFirstFieldNameWithTag(t *testing.T) {
+ v := &TestConnection{
+ RestConnection: RestConnection{
+ BaseConnection: BaseConnection{
+ Name: "1",
+ },
+ Endpoint: "2",
+ Proxy: "3",
+ RateLimit: 0,
+ },
+ BasicAuth: BasicAuth{
+ Username: "4",
+ Password: "5",
+ },
+ EpicKeyField: "6",
+ StoryPointField: "7",
+ RemotelinkCommitShaPattern: "8",
+ }
+ dataVal := reflect.ValueOf(v)
+ dataType := reflect.Indirect(dataVal).Type()
+ fieldName := firstFieldNameWithTag(dataType, "encryptField")
+ assert.Equal(t, "Password", fieldName)
+}
diff --git a/plugins/jira/tasks/api_client.go b/plugins/jira/tasks/api_client.go
index 3673bee5..811c0b6a 100644
--- a/plugins/jira/tasks/api_client.go
+++ b/plugins/jira/tasks/api_client.go
@@ -27,18 +27,15 @@ import (
)
func NewJiraApiClient(taskCtx core.TaskContext, connection *models.JiraConnection) (*helper.ApiAsyncClient, error) {
- // load configuration
- encKey := taskCtx.GetConfig(core.EncodeKeyEnvStr)
- decodedPassword, err := core.Decrypt(encKey, connection.Password)
+ // decrypt connection first
+ err := helper.DecryptConnection(connection, "Password")
if err != nil {
return nil, fmt.Errorf("Failed to decrypt Auth AccessToken: %w", err)
}
- connection.Password = decodedPassword
- auth := connection.GetEncodedToken()
// create synchronize api client so we can calculate api rate limit dynamically
headers := map[string]string{
- "Authorization": fmt.Sprintf("Basic %v", auth),
+ "Authorization": fmt.Sprintf("Basic %v", connection.GetEncodedToken()),
}
apiClient, err := helper.NewApiClient(connection.Endpoint, headers, 0, connection.Proxy, taskCtx.GetContext())
if err != nil {
[incubator-devlake] 02/04: fix(jira): modify migrationscripts to spread basicAuthEncoded to username/password
Posted by wa...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
warren pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit ab0dc9e8160c624d68fe5b33019f38b63768e5b3
Author: Yingchu Chen <yi...@merico.dev>
AuthorDate: Wed Jun 1 18:43:12 2022 +0800
fix(jira): modify migrationscripts to spread basicAuthEncoded to username/password
---
plugins/helper/connection.go | 6 +--
plugins/jira/jira.go | 2 +-
...Schemas20220524.go => updateSchemas20220601.go} | 59 ++++++++++++++++++----
3 files changed, 52 insertions(+), 15 deletions(-)
diff --git a/plugins/helper/connection.go b/plugins/helper/connection.go
index 93974f73..85163068 100644
--- a/plugins/helper/connection.go
+++ b/plugins/helper/connection.go
@@ -190,10 +190,8 @@ func doDecrypt(dataVal reflect.Value, fieldName string) error {
return err
}
if len(fieldName) > 0 {
- decryptStr, err := core.Decrypt(encryptCode, dataVal.Elem().FieldByName(fieldName).String())
- if err != nil {
- return err
- }
+ decryptStr, _ := core.Decrypt(encryptCode, dataVal.Elem().FieldByName(fieldName).String())
+
dataVal.Elem().FieldByName(fieldName).Set(reflect.ValueOf(decryptStr))
}
return nil
diff --git a/plugins/jira/jira.go b/plugins/jira/jira.go
index 2a40b901..a36177da 100644
--- a/plugins/jira/jira.go
+++ b/plugins/jira/jira.go
@@ -150,10 +150,10 @@ func (plugin Jira) MigrationScripts() []migration.Script {
new(migrationscripts.UpdateSchemas20220505),
new(migrationscripts.UpdateSchemas20220507),
new(migrationscripts.UpdateSchemas20220518),
- new(migrationscripts.UpdateSchemas20220524),
new(migrationscripts.UpdateSchemas20220525),
new(migrationscripts.UpdateSchemas20220526),
new(migrationscripts.UpdateSchemas20220527),
+ new(migrationscripts.UpdateSchemas20220601),
}
}
diff --git a/plugins/jira/models/migrationscripts/updateSchemas20220524.go b/plugins/jira/models/migrationscripts/updateSchemas20220601.go
similarity index 51%
rename from plugins/jira/models/migrationscripts/updateSchemas20220524.go
rename to plugins/jira/models/migrationscripts/updateSchemas20220601.go
index 9c753610..ec528dcb 100644
--- a/plugins/jira/models/migrationscripts/updateSchemas20220524.go
+++ b/plugins/jira/models/migrationscripts/updateSchemas20220601.go
@@ -19,11 +19,14 @@ package migrationscripts
import (
"context"
+ "encoding/base64"
"github.com/apache/incubator-devlake/plugins/helper"
"gorm.io/gorm"
+ "gorm.io/gorm/clause"
+ "strings"
)
-type JiraConnection20220524 struct {
+type JiraConnection20220601 struct {
helper.RestConnection
helper.BasicAuth
EpicKeyField string `gorm:"type:varchar(50);" json:"epicKeyField"`
@@ -31,29 +34,65 @@ type JiraConnection20220524 struct {
RemotelinkCommitShaPattern string `gorm:"type:varchar(255);comment='golang regexp, the first group will be recognized as commit sha, ref https://github.com/google/re2/wiki/Syntax'" json:"remotelinkCommitShaPattern"`
}
-func (JiraConnection20220524) TableName() string {
+func (JiraConnection20220601) TableName() string {
return "_tool_jira_connections"
}
-type UpdateSchemas20220524 struct{}
+type UpdateSchemas20220601 struct{}
-func (*UpdateSchemas20220524) Up(ctx context.Context, db *gorm.DB) error {
+func (*UpdateSchemas20220601) Up(ctx context.Context, db *gorm.DB) error {
var err error
if !db.Migrator().HasColumn(&JiraConnection20220505{}, "password") {
- err = db.Migrator().AddColumn(&JiraConnection20220524{}, "password")
+ err = db.Migrator().AddColumn(&JiraConnection20220601{}, "password")
if err != nil {
return err
}
}
if !db.Migrator().HasColumn(&JiraConnection20220505{}, "username") {
- err = db.Migrator().AddColumn(&JiraConnection20220524{}, "username")
+ err = db.Migrator().AddColumn(&JiraConnection20220601{}, "username")
if err != nil {
return err
}
}
if db.Migrator().HasColumn(&JiraConnection20220505{}, "basic_auth_encoded") {
+ connections := make([]*JiraConnection20220505, 0)
+ db.Find(&connections)
+ for i, _ := range connections {
+ err = helper.DecryptConnection(connections[i], "BasicAuthEncoded")
+ if err != nil {
+ return err
+ }
+ decodedStr, err := base64.StdEncoding.DecodeString(connections[i].BasicAuthEncoded)
+ if err != nil {
+ return err
+ }
+ strList := strings.Split(string(decodedStr), ":")
+ if len(strList) > 1 {
+ newConnection := JiraConnection20220601{
+ RestConnection: helper.RestConnection{
+ BaseConnection: helper.BaseConnection{
+ Name: connections[i].Name,
+ Model: connections[i].Model,
+ },
+ Endpoint: connections[i].Endpoint,
+ Proxy: connections[i].Proxy,
+ RateLimit: connections[i].RateLimit,
+ },
+ BasicAuth: helper.BasicAuth{
+ Username: strList[0],
+ Password: strList[1],
+ },
+ EpicKeyField: connections[i].EpicKeyField,
+ StoryPointField: connections[i].StoryPointField,
+ RemotelinkCommitShaPattern: connections[i].RemotelinkCommitShaPattern,
+ }
+ db.Clauses(clause.OnConflict{UpdateAll: true}).Create(newConnection)
+
+ }
+ connections[i].Name = strList[0]
+ }
err = db.Migrator().DropColumn(&JiraConnection20220505{}, "basic_auth_encoded")
if err != nil {
return err
@@ -63,10 +102,10 @@ func (*UpdateSchemas20220524) Up(ctx context.Context, db *gorm.DB) error {
return nil
}
-func (*UpdateSchemas20220524) Version() uint64 {
- return 20220507154646
+func (*UpdateSchemas20220601) Version() uint64 {
+ return 20220601154646
}
-func (*UpdateSchemas20220524) Name() string {
- return "Add icon_url column to JiraIssue"
+func (*UpdateSchemas20220601) Name() string {
+ return "change basic_auth to username/password"
}