You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by kl...@apache.org on 2023/12/08 07:06:39 UTC
(incubator-devlake) branch main updated: GitHub token (#6572)
This is an automated email from the ASF dual-hosted git repository.
klesh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 221259243 GitHub token (#6572)
221259243 is described below
commit 22125924321e8ced1d31970f479f44637c41f874
Author: Lynwee <li...@merico.dev>
AuthorDate: Fri Dec 8 15:06:35 2023 +0800
GitHub token (#6572)
* feat(github): update github token sanitizer
* feat(github): you can test multi tokens at once
* feat(github): update github tokens api
* fix(lint): fix lint errors
* fix(github): fix test when using AppKey to authenticate
* refactor(github): remove debug codes
* fix(swag): update github test connection api's doc
* refactor(github): remove duplicated codes
* refactor(github): reduce some codes
* fix(lint): fix lint errors
---
.../helpers/pluginhelper/api/connection_auths.go | 11 +-
.../helpers/pluginhelper/api/model_api_helper.go | 21 +-
backend/plugins/github/api/connection_api.go | 286 ++++++++++++++-------
backend/plugins/github/api/init.go | 3 +-
backend/plugins/github/models/connection.go | 221 +++++++++++++++-
backend/plugins/github/models/connection_test.go | 168 ++++++++++++
6 files changed, 607 insertions(+), 103 deletions(-)
diff --git a/backend/helpers/pluginhelper/api/connection_auths.go b/backend/helpers/pluginhelper/api/connection_auths.go
index 09116415b..d6170353e 100644
--- a/backend/helpers/pluginhelper/api/connection_auths.go
+++ b/backend/helpers/pluginhelper/api/connection_auths.go
@@ -150,8 +150,15 @@ func (ma *MultiAuth) CustomValidate(connection interface{}, v *validator.Validat
// TODO: deprecated, rename to CustomValidate instead
func (ma *MultiAuth) ValidateConnection(connection interface{}, v *validator.Validate) errors.Error {
- // the idea is to filtered out errors from unselected Authentication struct
- validationErrors := v.Struct(connection).(validator.ValidationErrors)
+ // the idea is to filter out errors from unselected Authentication struct
+ err := v.Struct(connection)
+ if err == nil {
+ return nil
+ }
+ validationErrors, ok := err.(validator.ValidationErrors)
+ if !ok {
+ return errors.Default.New("err: %s cannot be cast to validationErrors")
+ }
if validationErrors != nil {
filteredValidationErrors := make(validator.ValidationErrors, 0)
connType := reflect.TypeOf(connection).Elem()
diff --git a/backend/helpers/pluginhelper/api/model_api_helper.go b/backend/helpers/pluginhelper/api/model_api_helper.go
index f0f72bda3..6db832e2a 100644
--- a/backend/helpers/pluginhelper/api/model_api_helper.go
+++ b/backend/helpers/pluginhelper/api/model_api_helper.go
@@ -129,14 +129,29 @@ func (self *ModelApiHelper[M]) BatchSanitize(models []*M) []*M {
return models
}
+type CustomMerge[M dal.Tabler] interface {
+ Merge(target, src *M) error
+}
+
func (self *ModelApiHelper[M]) Patch(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
model, err := self.FindByPk(input)
if err != nil {
return nil, err
}
- err = utils.DecodeMapStruct(input.Body, model, true)
- if err != nil {
- return nil, errors.BadInput.Wrap(err, fmt.Sprintf("faled to patch %s", self.modelName))
+ if v, ok := (interface{}(model)).(CustomMerge[M]); ok {
+ modifiedModel := new(M)
+ err = utils.DecodeMapStruct(input.Body, modifiedModel, true)
+ if err != nil {
+ return nil, errors.BadInput.Wrap(err, fmt.Sprintf("faled to decode map struct %s", self.modelName))
+ }
+ if err := v.Merge(model, modifiedModel); err != nil {
+ return nil, errors.Convert(err)
+ }
+ } else {
+ err = utils.DecodeMapStruct(input.Body, model, true)
+ if err != nil {
+ return nil, errors.BadInput.Wrap(err, fmt.Sprintf("faled to patch %s", self.modelName))
+ }
}
err = self.dalHelper.Update(model)
if err != nil {
diff --git a/backend/plugins/github/api/connection_api.go b/backend/plugins/github/api/connection_api.go
index f5af1b32c..bbee00bcf 100644
--- a/backend/plugins/github/api/connection_api.go
+++ b/backend/plugins/github/api/connection_api.go
@@ -147,109 +147,209 @@ func testConnection(ctx context.Context, conn models.GithubConn) (*GithubTestCon
return nil, errors.Convert(err)
}
}
+ githubApiResponse := &GithubTestConnResponse{}
+ if conn.AuthMethod == models.AppKey {
+ if tokenTestResult, err := testGithubConnAppKeyAuth(ctx, conn); err != nil {
+ return nil, errors.Convert(err)
+ } else {
+ githubApiResponse.Success = tokenTestResult.Success
+ githubApiResponse.Message = tokenTestResult.Message
+ githubApiResponse.Login = tokenTestResult.Login
+ githubApiResponse.Installations = tokenTestResult.Installations
+ }
+ } else if conn.AuthMethod == models.AccessToken {
+ if tokenTestResult, err := testGithubConnAccessTokenAuth(ctx, conn); err != nil {
+ return nil, errors.Convert(err)
+ } else {
+ githubApiResponse.Success = tokenTestResult.Success
+ githubApiResponse.Warning = tokenTestResult.Warning
+ githubApiResponse.Message = tokenTestResult.Message
+ githubApiResponse.Login = tokenTestResult.Login
+ }
+ } else {
+ return nil, errors.BadInput.New("invalid authentication method")
+ }
+
+ return githubApiResponse, nil
+}
+
+type GitHubTestConnResult struct {
+ AuthMethod string `json:"auth_method"`
+
+ // AppKey
+ AppId string `json:"appId,omitempty"`
+ InstallationID int `json:"installationId,omitempty"`
+
+ // AccessToken
+ Token string `json:"token,omitempty"`
+
+ Success bool `json:"success"`
+ Message string `json:"message,omitempty"`
+ Login string `json:"login"`
+ Warning bool `json:"warning"`
+ Installations []models.GithubAppInstallation `json:"installations,omitempty"`
+}
+
+type GithubMultiTestConnResponse struct {
+ shared.ApiBody
+ Tokens []*GitHubTestConnResult `json:"tokens"`
+}
+
+// testGithubConnAccessTokenAuth only works when conn has one token
+func testGithubConnAccessTokenAuth(ctx context.Context, conn models.GithubConn) (*GitHubTestConnResult, error) {
apiClient, err := api.NewApiClientFromConnection(ctx, basicRes, &conn)
if err != nil {
return nil, err
}
- githubApiResponse := &GithubTestConnResponse{}
- if conn.AuthMethod == "AppKey" {
- jwt, err := conn.GithubAppKey.CreateJwt()
- if err != nil {
- return nil, err
- }
- res, err := apiClient.Get("app", nil, http.Header{
- "Authorization": []string{fmt.Sprintf("Bearer %s", jwt)},
- })
- if err != nil {
- return nil, errors.BadInput.Wrap(err, "verify token failed")
- }
- if res.StatusCode != http.StatusOK {
- return nil, errors.HttpStatus(res.StatusCode).New("unexpected status code while testing connection")
- }
- githubApp := &models.GithubApp{}
- err = api.UnmarshalResponse(res, githubApp)
- if err != nil {
- return nil, errors.BadInput.Wrap(err, "verify token failed")
- } else if githubApp.Slug == "" {
- return nil, errors.BadInput.Wrap(err, "invalid token")
- }
- res, err = apiClient.Get("app/installations", nil, http.Header{
- "Authorization": []string{fmt.Sprintf("Bearer %s", jwt)},
- })
- if err != nil {
- return nil, errors.BadInput.Wrap(err, "verify token failed")
+ res, err := apiClient.Get("user", nil, nil)
+ if err != nil {
+ return nil, errors.BadInput.Wrap(err, "verify token failed")
+ }
+ if res.StatusCode == http.StatusUnauthorized {
+ return nil, errors.HttpStatus(http.StatusBadRequest).New("StatusUnauthorized error when testing connection")
+ }
+ if res.StatusCode != http.StatusOK {
+ return nil, errors.HttpStatus(res.StatusCode).New("unexpected status code while testing connection")
+ }
+ githubUserOfToken := &models.GithubUserOfToken{}
+ err = api.UnmarshalResponse(res, githubUserOfToken)
+ if err != nil {
+ return nil, errors.BadInput.Wrap(err, "verify token failed")
+ } else if githubUserOfToken.Login == "" {
+ return nil, errors.BadInput.Wrap(err, "invalid token")
+ }
+ success := false
+ warning := false
+ var messages []string
+ // for github classic token, check permission
+ if strings.HasPrefix(conn.Token, "ghp_") {
+ scopes := res.Header.Get("X-OAuth-Scopes")
+ // convert "X-OAuth-Scopes" header to user permissions map
+ userPerms := map[string]bool{}
+ for _, userPerm := range strings.Split(scopes, ", ") {
+ userPerms[userPerm] = true
}
- if res.StatusCode != http.StatusOK {
- return nil, errors.HttpStatus(res.StatusCode).New("unexpected status code while testing connection")
+ // check public repo permission
+ missingPubPerms := findMissingPerms(userPerms, publicPermissions)
+ success = len(missingPubPerms) == 0
+ if !success {
+ messages = append(messages, fmt.Sprintf(
+ "Please check the field(s) %s",
+ strings.Join(missingPubPerms, ", "),
+ ))
}
- githubAppInstallations := &[]models.GithubAppInstallation{}
- err = api.UnmarshalResponse(res, githubAppInstallations)
- if err != nil {
- return nil, errors.BadInput.Wrap(err, "verify token failed")
+ // check private repo permission
+ missingPriPerms := findMissingPerms(userPerms, privatePermissions)
+ warning = len(missingPriPerms) > 0
+ if warning {
+ msgFmt := "If you want to collect private repositories, please check the field(s) %s"
+ if success {
+ // @Startrekzky and @yumengwang03 firmly believe that this is critical for users to understand the message
+ msgFmt = "This token is able to collect public repositories. " + msgFmt
+ }
+ messages = append(messages, fmt.Sprintf(
+ msgFmt,
+ strings.Join(missingPriPerms, ", "),
+ ))
}
- githubApiResponse.Success = true
- githubApiResponse.Message = "success"
- githubApiResponse.Login = githubApp.Slug
- githubApiResponse.Installations = *githubAppInstallations
+ }
- } else if conn.AuthMethod == "AccessToken" {
- res, err := apiClient.Get("user", nil, nil)
- if err != nil {
- return nil, errors.BadInput.Wrap(err, "verify token failed")
- }
- if res.StatusCode == http.StatusUnauthorized {
- return nil, errors.HttpStatus(http.StatusBadRequest).New("StatusUnauthorized error when testing connection")
- }
- if res.StatusCode != http.StatusOK {
- return nil, errors.HttpStatus(res.StatusCode).New("unexpected status code while testing connection")
+ sanitizeConn := conn.Sanitize()
+ tokenTestResult := GitHubTestConnResult{
+ AuthMethod: models.AccessToken,
+ Token: sanitizeConn.Token,
+ Success: success,
+ Message: strings.Join(messages, ";\n"),
+ Login: githubUserOfToken.Login,
+ Warning: warning,
+ }
+ return &tokenTestResult, nil
+}
+
+func getInstallationsWithGithubConnAppKeyAuth(ctx context.Context, conn models.GithubConn) (*GitHubTestConnResult, error) {
+ apiClient, err := api.NewApiClientFromConnection(ctx, basicRes, &conn)
+ if err != nil {
+ return nil, err
+ }
+ jwt, err := conn.GithubAppKey.CreateJwt()
+ if err != nil {
+ return nil, err
+ }
+ res, err := apiClient.Get("app", nil, http.Header{
+ "Authorization": []string{fmt.Sprintf("Bearer %s", jwt)},
+ })
+ if err != nil {
+ return nil, errors.BadInput.Wrap(err, "verify token(get app) failed")
+ }
+ if res.StatusCode != http.StatusOK {
+ return nil, errors.HttpStatus(res.StatusCode).New("unexpected status code while testing connection")
+ }
+ githubApp := &models.GithubApp{}
+ err = api.UnmarshalResponse(res, githubApp)
+ if err != nil {
+ return nil, errors.BadInput.Wrap(err, "verify token resp failed")
+ } else if githubApp.Slug == "" {
+ return nil, errors.BadInput.Wrap(err, "invalid token")
+ }
+ res, err = apiClient.Get("app/installations", nil, http.Header{
+ "Authorization": []string{fmt.Sprintf("Bearer %s", jwt)},
+ })
+ if err != nil {
+ return nil, errors.BadInput.Wrap(err, "verify token(get app installations) failed")
+ }
+ if res.StatusCode != http.StatusOK {
+ return nil, errors.HttpStatus(res.StatusCode).New("unexpected status code while testing connection")
+ }
+ githubAppInstallations := &[]models.GithubAppInstallation{}
+ err = api.UnmarshalResponse(res, githubAppInstallations)
+ if err != nil {
+ return nil, errors.BadInput.Wrap(err, "verify token failed")
+ }
+ tokenTestResult := GitHubTestConnResult{
+ AuthMethod: models.AppKey,
+ AppId: conn.AppId,
+ InstallationID: conn.InstallationID,
+ Success: true,
+ Message: "success",
+ Login: githubApp.Slug,
+ Installations: *githubAppInstallations,
+ }
+ return &tokenTestResult, nil
+}
+
+func testGithubConnAppKeyAuth(ctx context.Context, conn models.GithubConn) (*GitHubTestConnResult, error) {
+ // AppKey can only have one secretKey, can shouldn't have tokens.
+ conn.Token = ""
+ // I think connection with InstallationID needs another test
+ // But it's to be determined. So just ignore it temporarily.
+ conn.InstallationID = 0
+ return getInstallationsWithGithubConnAppKeyAuth(ctx, conn)
+}
+
+func testExistingConnection(ctx context.Context, conn models.GithubConn) (*GithubMultiTestConnResponse, errors.Error) {
+ if vld != nil {
+ if err := vld.StructExcept(conn, "GithubAppKey", "GithubAccessToken"); err != nil {
+ return nil, errors.Convert(err)
}
- githubUserOfToken := &models.GithubUserOfToken{}
- err = api.UnmarshalResponse(res, githubUserOfToken)
- if err != nil {
- return nil, errors.BadInput.Wrap(err, "verify token failed")
- } else if githubUserOfToken.Login == "" {
- return nil, errors.BadInput.Wrap(err, "invalid token")
+ }
+ githubApiResponse := &GithubMultiTestConnResponse{}
+ if conn.AuthMethod == models.AppKey {
+ if tokenTestResult, err := testGithubConnAppKeyAuth(ctx, conn); err != nil {
+ return nil, errors.Convert(err)
+ } else {
+ githubApiResponse.Tokens = append(githubApiResponse.Tokens, tokenTestResult)
}
- success := false
- warning := false
- messages := []string{}
- // for github classic token, check permission
- if strings.HasPrefix(conn.Token, "ghp_") {
- scopes := res.Header.Get("X-OAuth-Scopes")
- // convert "X-OAuth-Scopes" header to user permissions map
- userPerms := map[string]bool{}
- for _, userPerm := range strings.Split(scopes, ", ") {
- userPerms[userPerm] = true
- }
- // check public repo permission
- missingPubPerms := findMissingPerms(userPerms, publicPermissions)
- success = len(missingPubPerms) == 0
- if !success {
- messages = append(messages, fmt.Sprintf(
- "Please check the field(s) %s",
- strings.Join(missingPubPerms, ", "),
- ))
- }
- // check private repo permission
- missingPriPerms := findMissingPerms(userPerms, privatePermissions)
- warning = len(missingPriPerms) > 0
- if warning {
- msgFmt := "If you want to collect private repositories, please check the field(s) %s"
- if success {
- // @Startrekzky and @yumengwang03 firmly believe that this is critical for users to understand the message
- msgFmt = "This token is able to collect public repositories. " + msgFmt
- }
- messages = append(messages, fmt.Sprintf(
- msgFmt,
- strings.Join(missingPriPerms, ", "),
- ))
+ } else if conn.AuthMethod == models.AccessToken {
+ tokens := strings.Split(conn.Token, ",")
+ for _, token := range tokens {
+ testGithubConn := conn
+ testGithubConn.Token = token
+ if tokenTestResult, err := testGithubConnAccessTokenAuth(ctx, testGithubConn); err != nil {
+ return nil, errors.Convert(err)
+ } else {
+ githubApiResponse.Tokens = append(githubApiResponse.Tokens, tokenTestResult)
}
}
-
- githubApiResponse.Success = success
- githubApiResponse.Warning = warning
- githubApiResponse.Message = strings.Join(messages, ";\n")
- githubApiResponse.Login = githubUserOfToken.Login
} else {
return nil, errors.BadInput.New("invalid authentication method")
}
@@ -261,7 +361,7 @@ func testConnection(ctx context.Context, conn models.GithubConn) (*GithubTestCon
// @Summary test github connection
// @Description Test github Connection
// @Tags plugins/github
-// @Success 200 {object} GithubTestConnResponse
+// @Success 200 {object} GithubMultiTestConnResponse
// @Failure 400 {string} errcode.Error "Bad Request"
// @Failure 500 {string} errcode.Error "Internal Error"
// @Router /plugins/github/{connectionId}/test [POST]
@@ -270,7 +370,7 @@ func TestExistingConnection(input *plugin.ApiResourceInput) (*plugin.ApiResource
if err != nil {
return nil, err
}
- testConnectionResult, testConnectionErr := testConnection(context.TODO(), connection.GithubConn)
+ testConnectionResult, testConnectionErr := testExistingConnection(context.TODO(), connection.GithubConn)
if testConnectionErr != nil {
return nil, testConnectionErr
}
diff --git a/backend/plugins/github/api/init.go b/backend/plugins/github/api/init.go
index e072adf28..0951a7de5 100644
--- a/backend/plugins/github/api/init.go
+++ b/backend/plugins/github/api/init.go
@@ -18,9 +18,8 @@ limitations under the License.
package api
import (
- "github.com/apache/incubator-devlake/core/plugin"
-
"github.com/apache/incubator-devlake/core/context"
+ "github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/github/models"
"github.com/go-playground/validator/v10"
diff --git a/backend/plugins/github/models/connection.go b/backend/plugins/github/models/connection.go
index 2333e124b..9aec962ea 100644
--- a/backend/plugins/github/models/connection.go
+++ b/backend/plugins/github/models/connection.go
@@ -31,6 +31,11 @@ import (
"github.com/golang-jwt/jwt/v5"
)
+const (
+ AccessToken = "AccessToken"
+ AppKey = "AppKey"
+)
+
// GithubAccessToken supports fetching data with multiple tokens
type GithubAccessToken struct {
helper.AccessToken `mapstructure:",squash"`
@@ -54,11 +59,11 @@ type GithubConn struct {
// PrepareApiClient splits Token to tokens for SetupAuthentication to utilize
func (conn *GithubConn) PrepareApiClient(apiClient plugin.ApiClient) errors.Error {
- if conn.AuthMethod == "AccessToken" {
+ if conn.AuthMethod == AccessToken {
conn.tokens = strings.Split(conn.Token, ",")
}
- if conn.AuthMethod == "AppKey" && conn.InstallationID != 0 {
+ if conn.AuthMethod == AppKey && conn.InstallationID != 0 {
token, err := conn.getInstallationAccessToken(apiClient)
if err != nil {
return err
@@ -94,15 +99,225 @@ type GithubConnection struct {
EnableGraphql bool `mapstructure:"enableGraphql" json:"enableGraphql"`
}
+const (
+ GithubTokenTypeClassical = "classical"
+ GithubTokenTypeClassicalPrefixLen = 4
+ GithubTokenTypeClassicalShowPrefixLen = 8
+ GithubTokenTypeClassicalHiddenLen = 20
+ GithubTokenTypeFineGrained = "fine-grained"
+ GithubTokenTypeFineGrainedPrefixLen = 11
+ GithubTokenTypeFineGrainedShowPrefixLen = 8
+ GithubTokenTypeFineGrainedHiddenLen = 66
+ GithubTokenTypeUnknown = "unknown"
+)
+
func (connection GithubConnection) TableName() string {
return "_tool_github_connections"
}
+func (connection *GithubConnection) Merge(existed, modified *GithubConnection) error {
+ // There are many kinds of update, we just update all fields simply.
+ existedTokenStr := existed.Token
+ existSecretKey := existed.SecretKey
+
+ existed.AppId = modified.AppId
+ existed.SecretKey = modified.SecretKey
+ existed.InstallationID = modified.InstallationID
+ existed.AuthMethod = modified.AuthMethod
+ existed.Proxy = modified.Proxy
+ existed.Endpoint = modified.Endpoint
+ existed.RateLimitPerHour = modified.RateLimitPerHour
+
+ // handle secret
+ if existSecretKey == "" {
+ if modified.SecretKey != "" {
+ // add secret key, store it
+ existed.SecretKey = modified.SecretKey
+ }
+ // doesn't input secret key, pass
+ } else {
+ if modified.SecretKey == "" {
+ // delete secret key
+ existed.SecretKey = modified.SecretKey
+ } else {
+ // update secret key
+ sanitizeSecretKey := existed.SanitizeSecret().SecretKey
+ if sanitizeSecretKey == modified.SecretKey {
+ // change nothing, restore it
+ existed.SecretKey = existSecretKey
+ } else {
+ // has changed, replace it with the new secret key
+ existed.SecretKey = modified.SecretKey
+ }
+ }
+ }
+
+ // handle tokens
+ existedTokens := strings.Split(strings.TrimSpace(existedTokenStr), ",")
+ existedTokenMap := make(map[string]string) // {originalToken:sanitizedToken}
+ existedSanitizedTokenMap := make(map[string]string) // {sanitizedToken:originalToken}
+ for _, token := range existedTokens {
+ existedTokenMap[token] = existed.SanitizeToken(token)
+ existedSanitizedTokenMap[existed.SanitizeToken(token)] = token
+ }
+
+ modifiedTokens := strings.Split(strings.TrimSpace(modified.Token), ",")
+ modifiedTokenMap := make(map[string]string) // {originalToken:sanitizedToken}
+ for _, token := range modifiedTokens {
+ modifiedTokenMap[token] = existed.SanitizeToken(token)
+ }
+
+ var mergedToken []string
+ mergedTokenMap := make(map[string]struct{})
+
+ for token, sanitizeToken := range modifiedTokenMap {
+ // check token
+ if _, ok := existedTokenMap[token]; ok {
+ // find in db, modified but no update, ignore it
+ if _, ok := mergedTokenMap[token]; !ok {
+ mergedToken = append(mergedToken, token)
+ mergedTokenMap[token] = struct{}{}
+ }
+ } else {
+ // not found, a new token, we should keep it
+ // cannot be a sanitized token
+ if _, ok := existedSanitizedTokenMap[token]; !ok {
+ if token != sanitizeToken {
+ if _, ok := mergedTokenMap[token]; !ok {
+ mergedToken = append(mergedToken, token)
+ mergedTokenMap[token] = struct{}{}
+ }
+ }
+ }
+ }
+
+ // token may be a sanitized token
+ if v, ok := existedSanitizedTokenMap[token]; ok {
+ // find in db, modify nothing, just keep it
+ if _, ok := mergedTokenMap[v]; !ok {
+ mergedToken = append(mergedToken, v)
+ mergedTokenMap[v] = struct{}{}
+ }
+ } else {
+ // unexpected
+ fmt.Printf("unexpected token: %+v will be ignored\n", token)
+ }
+ // check sanitized token
+ if v, ok := existedSanitizedTokenMap[sanitizeToken]; ok {
+ // find in db, modify nothing, just keep it
+ if _, ok := mergedTokenMap[v]; !ok {
+ mergedToken = append(mergedToken, v)
+ mergedTokenMap[v] = struct{}{}
+ }
+ } else {
+ // a new token
+ // but we should check it
+ if sanitizeToken != token {
+ if _, ok := mergedTokenMap[token]; !ok {
+ mergedToken = append(mergedToken, token)
+ mergedTokenMap[token] = struct{}{}
+ }
+ }
+ }
+ }
+
+ existed.Token = strings.Join(mergedToken, ",")
+ return nil
+}
+
+func (conn *GithubConn) typeIs(token string) string {
+ // classical tokens:
+ // ghp_ for Personal Access Tokens
+ // gho_ for OAuth Access tokens
+ // ghu_ for GitHub App user-to-server tokens
+ // ghs_ for GitHub App server-to-server tokens
+ // ghr_ for GitHub App refresh tokens
+ // total len is 40, {prefix}{showPrefix}{secret}{showSuffix}
+ // fine-grained tokens
+ // github_pat_{82_characters}
+ classicalTokenClassicalPrefixes := []string{"ghp_", "gho_", "ghs_", "ghr_"}
+ classicalTokenFindGrainedPrefixes := []string{"github_pat_"}
+ for _, prefix := range classicalTokenClassicalPrefixes {
+ if strings.HasPrefix(token, prefix) {
+ return GithubTokenTypeClassical
+ }
+ }
+ for _, prefix := range classicalTokenFindGrainedPrefixes {
+ if strings.HasPrefix(token, prefix) {
+ return GithubTokenTypeFineGrained
+ }
+ }
+ return GithubTokenTypeUnknown
+}
+
func (connection GithubConnection) Sanitize() GithubConnection {
- connection.Token = ""
+ connection.GithubConn = connection.GithubConn.Sanitize()
return connection
}
+func (conn *GithubConn) Sanitize() GithubConn {
+ conn.SanitizeTokens()
+ conn.SanitizeSecret()
+ return *conn
+}
+
+func (conn *GithubConn) SanitizeSecret() GithubConn {
+ if conn.SecretKey == "" {
+ return *conn
+ }
+ secretKey := conn.SecretKey
+ showPrefixLen, showSuffixLen := 50, 50
+ hiddenLen := len(secretKey) - showSuffixLen - showSuffixLen
+ secret := strings.Repeat("*", hiddenLen)
+ conn.SecretKey = strings.Replace(secretKey, conn.SecretKey[showPrefixLen:showPrefixLen+hiddenLen], secret, -1)
+ return *conn
+}
+
+func (conn *GithubConn) SanitizeToken(token string) string {
+ if token == "" {
+ return token
+ }
+ sanitizedToken := token
+ var prefixLen, showPrefixLen, hiddenLen int
+
+ tokenType := conn.typeIs(token)
+ switch tokenType {
+ case GithubTokenTypeClassical:
+ prefixLen, showPrefixLen, hiddenLen = GithubTokenTypeClassicalPrefixLen, GithubTokenTypeClassicalShowPrefixLen, GithubTokenTypeClassicalHiddenLen
+ case GithubTokenTypeFineGrained:
+ prefixLen, showPrefixLen, hiddenLen = GithubTokenTypeFineGrainedPrefixLen, GithubTokenTypeFineGrainedShowPrefixLen, GithubTokenTypeFineGrainedHiddenLen
+ case GithubTokenTypeUnknown:
+ }
+ tokenLen := len(token)
+ if tokenLen >= prefixLen && prefixLen != 0 && hiddenLen != 0 {
+ secret := strings.Repeat("*", hiddenLen)
+ sanitizedToken = strings.Replace(token, token[prefixLen+showPrefixLen:prefixLen+showPrefixLen+hiddenLen], secret, -1)
+ }
+ return sanitizedToken
+}
+
+func (conn *GithubConn) SanitizeTokens() GithubConn {
+ if conn.Token == "" {
+ return *conn
+ }
+ tokens := strings.Split(conn.Token, ",")
+ var sanitizedTokens []string
+ for _, token := range tokens {
+ if token == "" {
+ continue
+ }
+ sanitizedToken := conn.SanitizeToken(token)
+ sanitizedTokens = append(sanitizedTokens, sanitizedToken)
+ }
+
+ if len(sanitizedTokens) > 0 {
+ conn.Token = strings.Join(sanitizedTokens, ",")
+ } else {
+ conn.Token = ""
+ }
+ return *conn
+}
+
// Using GithubUserOfToken because it requires authentication, and it is public information anyway.
type GithubUserOfToken struct {
Login string `json:"login"`
diff --git a/backend/plugins/github/models/connection_test.go b/backend/plugins/github/models/connection_test.go
index 3c04d0951..7b39cf0bd 100644
--- a/backend/plugins/github/models/connection_test.go
+++ b/backend/plugins/github/models/connection_test.go
@@ -18,6 +18,7 @@ limitations under the License.
package models
import (
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"testing"
"github.com/go-playground/validator/v10"
@@ -59,3 +60,170 @@ func TestAppKeyFail(t *testing.T) {
println()
}
}
+
+func TestGithubConnection_Sanitize(t *testing.T) {
+ type fields struct {
+ GithubConn GithubConn
+ }
+ tests := []struct {
+ name string
+ fields fields
+ want GithubConnection
+ }{
+ {
+ name: "test-empty",
+ fields: fields{
+ GithubConn: GithubConn{
+ GithubAccessToken: GithubAccessToken{
+ AccessToken: api.AccessToken{
+ Token: "",
+ },
+ },
+ },
+ },
+ want: GithubConnection{
+ GithubConn: GithubConn{
+ GithubAccessToken: GithubAccessToken{
+ AccessToken: api.AccessToken{
+ Token: "",
+ },
+ },
+ },
+ },
+ },
+ {
+ name: "test-empty-1",
+ fields: fields{
+ GithubConn: GithubConn{
+ GithubAccessToken: GithubAccessToken{
+ AccessToken: api.AccessToken{
+ Token: ",",
+ },
+ },
+ },
+ },
+ want: GithubConnection{
+ GithubConn: GithubConn{
+ GithubAccessToken: GithubAccessToken{
+ AccessToken: api.AccessToken{
+ Token: "",
+ },
+ },
+ },
+ },
+ },
+ {
+ name: "test-1",
+ fields: fields{
+ GithubConn: GithubConn{
+ GithubAccessToken: GithubAccessToken{
+ AccessToken: api.AccessToken{
+ Token: "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",
+ },
+ },
+ },
+ },
+ want: GithubConnection{
+ GithubConn: GithubConn{
+ GithubAccessToken: GithubAccessToken{
+ AccessToken: api.AccessToken{
+ Token: "ghp_wFxrXqjC********************bL2WGpTP",
+ },
+ },
+ },
+ },
+ },
+ {
+ name: "test-2",
+ fields: fields{
+ GithubConn: GithubConn{
+ GithubAccessToken: GithubAccessToken{
+ AccessToken: api.AccessToken{
+ Token: "ghp_wFxrXqjCiAf91234567tCQX8uombbL2WGpTP,ghp_abcdeqjCiAf91234567tCQX8uombbL2WGpTP",
+ },
+ },
+ },
+ },
+ want: GithubConnection{
+ GithubConn: GithubConn{
+ GithubAccessToken: GithubAccessToken{
+ AccessToken: api.AccessToken{
+ Token: "ghp_wFxrXqjC********************bL2WGpTP,ghp_abcdeqjC********************bL2WGpTP",
+ },
+ },
+ },
+ },
+ },
+ {
+ name: "test-3",
+ fields: fields{
+ GithubConn: GithubConn{
+ GithubAccessToken: GithubAccessToken{
+ AccessToken: api.AccessToken{
+ Token: "ghp_wFxrXqjCiAf91234567tCQX8uombbL2WGpTP,ghp_abcdeqjCiAf91234567tCQX8uombbL2WGpTP,",
+ },
+ },
+ },
+ },
+ want: GithubConnection{
+ GithubConn: GithubConn{
+ GithubAccessToken: GithubAccessToken{
+ AccessToken: api.AccessToken{
+ Token: "ghp_wFxrXqjC********************bL2WGpTP,ghp_abcdeqjC********************bL2WGpTP",
+ },
+ },
+ },
+ },
+ },
+ {
+ name: "test-4",
+ fields: fields{
+ GithubConn: GithubConn{
+ GithubAccessToken: GithubAccessToken{
+ AccessToken: api.AccessToken{
+ Token: "github_pat_11ABMS6RQ0kA7PzN5AThcs_jN8eJeu0BsMa8BDyC12345P9CI67891a2Pu6abcdefk3OEPAJLYMjkfT4U1,",
+ },
+ },
+ },
+ },
+ want: GithubConnection{
+ GithubConn: GithubConn{
+ GithubAccessToken: GithubAccessToken{
+ AccessToken: api.AccessToken{
+ Token: "github_pat_11ABMS6R******************************************************************MjkfT4U1",
+ },
+ },
+ },
+ },
+ },
+ {
+ name: "test-5",
+ fields: fields{
+ GithubConn: GithubConn{
+ GithubAppKey: GithubAppKey{
+ AppKey: api.AppKey{
+ SecretKey: "-----BEGIN RSA PRIVATE KEY-----\nMIIEpQIBAAKCAQEAuornPO7qIw8xasqbHTrAPO6/D1vNU0k00a/bguoUE3kGMMgp\ni2UOhb3JE5QC8y4mO++6arbqfnPFZVrZtg2W20yUpmsPKF5s2RR02+MwjOvZLYlF\nwdUQBqyjLOeuU0dioUxIQBk5jFKAAN5npe87crinruTH/wlQjYziVkXmlOZbMQ78\nrf/heW6WHwd5bRs/QZw+YY/W+7hVMDWh1/0X8d0b3lAT6VQahUIJRFRDZleWGVXW\nF9LMqGxhJd8N7LXX5Nddwdwdwdwdwdwdwddddddddddddddd7sFfEhyfdVKHlCnM\nwK2L6Jj5IJHO/AfGCuGfMRlhkH3ZhRa9Ii7VEa7EUXvzvxg48J83kv7mIWgSV/me\n6q7JZyxN+Z2+DYMabI2F1QFi0QM7eMAyaH6/Ri4qBPys [...]
+ },
+ },
+ },
+ },
+ want: GithubConnection{
+ GithubConn: GithubConn{
+ GithubAppKey: GithubAppKey{
+ AppKey: api.AppKey{
+ SecretKey: "-----BEGIN RSA PRIVATE KEY-----\nMIIEpQIBAAKCAQEAuo************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** [...]
+ },
+ },
+ },
+ },
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ connection := GithubConnection{
+ GithubConn: tt.fields.GithubConn,
+ }
+ assert.Equalf(t, tt.want, connection.Sanitize(), "Sanitize()")
+ })
+ }
+}