You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by kl...@apache.org on 2022/07/15 08:20:19 UTC
[incubator-devlake] branch main updated: force connect with flawed https cert (#2496)
This is an automated email from the ASF dual-hosted git repository.
klesh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new e35effd2 force connect with flawed https cert (#2496)
e35effd2 is described below
commit e35effd2f5258f77babac4f41b415b9d8aac25ac
Author: mappjzc <zh...@merico.dev>
AuthorDate: Fri Jul 15 16:20:14 2022 +0800
force connect with flawed https cert (#2496)
* fix: jenkins collect folder
Add queue.go
Add list.go
Add QueueIterator
Add some logic fix
Nddtfjiang <zh...@merico.dev>
* refactor: changed list and queue to helper temporary
move list.go and queue.go to helper temporary
Nddtfjiang <zh...@merico.dev>
* feat: force connect with flawed https cert
Add insecureSkipVerify in client Transport for api_client.
Add env IN_SECURE_SKIP_VERIFY to set if force to connect and skip the verify
Nddtfjiang <zh...@merico.dev>
* refactor: add basicres for apiclient
Add BasicRes to NewApiClient
Nddtfjiang <zh...@merico.dev>
---
.env.example | 5 +++++
plugins/ae/api/connection.go | 2 +-
plugins/ae/tasks/api_client.go | 2 +-
plugins/feishu/api/connection.go | 5 +++--
plugins/feishu/tasks/api_client.go | 5 +++--
plugins/gitee/api/connection.go | 1 +
plugins/gitee/tasks/api_client.go | 2 +-
plugins/github/api/blueprint.go | 1 +
plugins/github/api/connection.go | 1 +
plugins/github/tasks/api_client.go | 2 +-
plugins/gitlab/api/blueprint.go | 1 +
plugins/gitlab/api/connection.go | 2 +-
plugins/gitlab/tasks/api_client.go | 2 +-
plugins/helper/api_async_client.go | 1 +
plugins/helper/api_client.go | 17 ++++++++++++++++-
plugins/icla/tasks/api_client.go | 5 +++--
plugins/jenkins/api/connection.go | 2 ++
plugins/jenkins/tasks/client.go | 2 +-
plugins/jira/api/connection.go | 2 +-
plugins/jira/api/proxy.go | 1 +
plugins/jira/tasks/api_client.go | 3 ++-
plugins/tapd/api/connection.go | 2 ++
plugins/tapd/tasks/api_client.go | 3 ++-
utils/strconv.go | 8 ++++++++
24 files changed, 60 insertions(+), 17 deletions(-)
diff --git a/.env.example b/.env.example
index b8343150..3385e4a1 100644
--- a/.env.example
+++ b/.env.example
@@ -104,3 +104,8 @@ AE_PROXY=
# Sensitive information encryption key
##########################
ENCODE_KEY=
+
+##########################
+# Set if skip verify and connect with out trusted certificate when use https
+##########################
+IN_SECURE_SKIP_VERIFY=
\ No newline at end of file
diff --git a/plugins/ae/api/connection.go b/plugins/ae/api/connection.go
index b0bca088..541a7c3c 100644
--- a/plugins/ae/api/connection.go
+++ b/plugins/ae/api/connection.go
@@ -61,7 +61,7 @@ func TestConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput, erro
secretKey := connection.SecretKey
proxy := connection.Proxy
- apiClient, err := helper.NewApiClient(context.TODO(), endpoint, nil, 3*time.Second, proxy)
+ apiClient, err := helper.NewApiClient(context.TODO(), endpoint, nil, 3*time.Second, proxy, basicRes)
if err != nil {
return nil, err
}
diff --git a/plugins/ae/tasks/api_client.go b/plugins/ae/tasks/api_client.go
index 0ef1b6ae..8278e5ef 100644
--- a/plugins/ae/tasks/api_client.go
+++ b/plugins/ae/tasks/api_client.go
@@ -34,7 +34,7 @@ func CreateApiClient(taskCtx core.TaskContext, connection *models.AeConnection)
secretKey := connection.SecretKey
proxy := connection.Proxy
- apiClient, err := helper.NewApiClient(taskCtx.GetContext(), endpoint, nil, 0, proxy)
+ apiClient, err := helper.NewApiClient(taskCtx.GetContext(), endpoint, nil, 0, proxy, taskCtx)
if err != nil {
return nil, err
}
diff --git a/plugins/feishu/api/connection.go b/plugins/feishu/api/connection.go
index 97c2eb00..668c8fe1 100644
--- a/plugins/feishu/api/connection.go
+++ b/plugins/feishu/api/connection.go
@@ -20,11 +20,12 @@ package api
import (
"context"
"fmt"
+ "net/http"
+
"github.com/apache/incubator-devlake/plugins/feishu/apimodels"
"github.com/apache/incubator-devlake/plugins/feishu/models"
"github.com/apache/incubator-devlake/plugins/helper"
"github.com/mitchellh/mapstructure"
- "net/http"
"github.com/apache/incubator-devlake/plugins/core"
)
@@ -44,7 +45,7 @@ func TestConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput, erro
return nil, err
}
- authApiClient, err := helper.NewApiClient(context.TODO(), params.Endpoint, nil, 0, params.Proxy)
+ authApiClient, err := helper.NewApiClient(context.TODO(), params.Endpoint, nil, 0, params.Proxy, basicRes)
if err != nil {
return nil, err
}
diff --git a/plugins/feishu/tasks/api_client.go b/plugins/feishu/tasks/api_client.go
index 13684042..f726dc74 100644
--- a/plugins/feishu/tasks/api_client.go
+++ b/plugins/feishu/tasks/api_client.go
@@ -32,7 +32,8 @@ const AUTH_ENDPOINT = "https://open.feishu.cn"
const ENDPOINT = "https://open.feishu.cn/open-apis/vc/v1"
func NewFeishuApiClient(taskCtx core.TaskContext, connection *models.FeishuConnection) (*helper.ApiAsyncClient, error) {
- authApiClient, err := helper.NewApiClient(taskCtx.GetContext(), AUTH_ENDPOINT, nil, 0, connection.Proxy)
+
+ authApiClient, err := helper.NewApiClient(taskCtx.GetContext(), AUTH_ENDPOINT, nil, 0, connection.Proxy, taskCtx)
if err != nil {
return nil, err
}
@@ -55,7 +56,7 @@ func NewFeishuApiClient(taskCtx core.TaskContext, connection *models.FeishuConne
return nil, fmt.Errorf("failed to request access token")
}
// real request apiClient
- apiClient, err := helper.NewApiClient(taskCtx.GetContext(), ENDPOINT, nil, 0, connection.Proxy)
+ apiClient, err := helper.NewApiClient(taskCtx.GetContext(), ENDPOINT, nil, 0, connection.Proxy, taskCtx)
if err != nil {
return nil, err
}
diff --git a/plugins/gitee/api/connection.go b/plugins/gitee/api/connection.go
index 64c8c5d9..5fd6e5e7 100644
--- a/plugins/gitee/api/connection.go
+++ b/plugins/gitee/api/connection.go
@@ -52,6 +52,7 @@ func TestConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput, erro
nil,
3*time.Second,
connection.Proxy,
+ basicRes,
)
if err != nil {
return nil, err
diff --git a/plugins/gitee/tasks/api_client.go b/plugins/gitee/tasks/api_client.go
index f64d59cd..230f5d60 100644
--- a/plugins/gitee/tasks/api_client.go
+++ b/plugins/gitee/tasks/api_client.go
@@ -30,7 +30,7 @@ import (
)
func NewGiteeApiClient(taskCtx core.TaskContext, connection *models.GiteeConnection) (*helper.ApiAsyncClient, error) {
- apiClient, err := helper.NewApiClient(taskCtx.GetContext(), connection.Endpoint, nil, 0, connection.Proxy)
+ apiClient, err := helper.NewApiClient(taskCtx.GetContext(), connection.Endpoint, nil, 0, connection.Proxy, taskCtx)
if err != nil {
return nil, err
}
diff --git a/plugins/github/api/blueprint.go b/plugins/github/api/blueprint.go
index e882ce53..2db11803 100644
--- a/plugins/github/api/blueprint.go
+++ b/plugins/github/api/blueprint.go
@@ -107,6 +107,7 @@ func MakePipelinePlan(subtaskMetas []core.SubTaskMeta, connectionId uint64, scop
},
10*time.Second,
connection.Proxy,
+ basicRes,
)
if err != nil {
return nil, err
diff --git a/plugins/github/api/connection.go b/plugins/github/api/connection.go
index ebfec4c1..5890175d 100644
--- a/plugins/github/api/connection.go
+++ b/plugins/github/api/connection.go
@@ -69,6 +69,7 @@ func TestConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput, erro
},
3*time.Second,
params.Proxy,
+ basicRes,
)
if err != nil {
return nil, err
diff --git a/plugins/github/tasks/api_client.go b/plugins/github/tasks/api_client.go
index 6a37a893..326d52c4 100644
--- a/plugins/github/tasks/api_client.go
+++ b/plugins/github/tasks/api_client.go
@@ -35,7 +35,7 @@ func CreateApiClient(taskCtx core.TaskContext, connection *models.GithubConnecti
tokens := strings.Split(connection.Token, ",")
tokenIndex := 0
// create synchronize api client so we can calculate api rate limit dynamically
- apiClient, err := helper.NewApiClient(taskCtx.GetContext(), connection.Endpoint, nil, 0, connection.Proxy)
+ apiClient, err := helper.NewApiClient(taskCtx.GetContext(), connection.Endpoint, nil, 0, connection.Proxy, taskCtx)
if err != nil {
return nil, err
}
diff --git a/plugins/gitlab/api/blueprint.go b/plugins/gitlab/api/blueprint.go
index 3ad6aaa9..6a54f5eb 100644
--- a/plugins/gitlab/api/blueprint.go
+++ b/plugins/gitlab/api/blueprint.go
@@ -107,6 +107,7 @@ func MakePipelinePlan(subtaskMetas []core.SubTaskMeta, connectionId uint64, scop
},
10*time.Second,
connection.Proxy,
+ basicRes,
)
if err != nil {
return nil, err
diff --git a/plugins/gitlab/api/connection.go b/plugins/gitlab/api/connection.go
index 2261c5e0..0c4e2f9d 100644
--- a/plugins/gitlab/api/connection.go
+++ b/plugins/gitlab/api/connection.go
@@ -42,7 +42,6 @@ func TestConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput, erro
if err != nil {
return nil, err
}
-
// test connection
apiClient, err := helper.NewApiClient(
context.TODO(),
@@ -52,6 +51,7 @@ func TestConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput, erro
},
3*time.Second,
connection.Proxy,
+ basicRes,
)
if err != nil {
return nil, err
diff --git a/plugins/gitlab/tasks/api_client.go b/plugins/gitlab/tasks/api_client.go
index 8ba82555..fe40a4ab 100644
--- a/plugins/gitlab/tasks/api_client.go
+++ b/plugins/gitlab/tasks/api_client.go
@@ -34,7 +34,7 @@ func NewGitlabApiClient(taskCtx core.TaskContext, connection *models.GitlabConne
headers := map[string]string{
"Authorization": fmt.Sprintf("Bearer %v", connection.Token),
}
- apiClient, err := helper.NewApiClient(taskCtx.GetContext(), connection.Endpoint, headers, 0, connection.Proxy)
+ apiClient, err := helper.NewApiClient(taskCtx.GetContext(), connection.Endpoint, headers, 0, connection.Proxy, taskCtx)
if err != nil {
return nil, err
}
diff --git a/plugins/helper/api_async_client.go b/plugins/helper/api_async_client.go
index 49df187a..899bde74 100644
--- a/plugins/helper/api_async_client.go
+++ b/plugins/helper/api_async_client.go
@@ -62,6 +62,7 @@ func CreateAsyncApiClient(
}
apiClient.SetTimeout(timeout)
apiClient.SetLogger(taskCtx.GetLogger())
+
globalRateLimitPerHour, err := utils.StrToIntOr(taskCtx.GetConfig("API_REQUESTS_PER_HOUR"), 18000)
if err != nil {
return nil, fmt.Errorf("failed to parse API_REQUESTS_PER_HOUR: %w", err)
diff --git a/plugins/helper/api_client.go b/plugins/helper/api_client.go
index 8d0b2e28..6bebdb7b 100644
--- a/plugins/helper/api_client.go
+++ b/plugins/helper/api_client.go
@@ -20,6 +20,7 @@ package helper
import (
"bytes"
"context"
+ "crypto/tls"
"encoding/json"
"errors"
"fmt"
@@ -58,7 +59,9 @@ func NewApiClient(
headers map[string]string,
timeout time.Duration,
proxy string,
+ br core.BasicRes,
) (*ApiClient, error) {
+
parsedUrl, err := url.Parse(endpoint)
if err != nil {
return nil, fmt.Errorf("Invalid URL: %w", err)
@@ -84,6 +87,18 @@ func NewApiClient(
headers,
timeout,
)
+ // create the Transport
+ apiClient.client.Transport = &http.Transport{}
+
+ // set insecureSkipVerify
+ insecureSkipVerify, err := utils.StrToBoolOr(br.GetConfig("IN_SECURE_SKIP_VERIFY"), false)
+ if err != nil {
+ return nil, fmt.Errorf("failt to parse IN_SECURE_SKIP_VERIFY: %w", err)
+ }
+ if insecureSkipVerify {
+ apiClient.client.Transport.(*http.Transport).TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
+ }
+
if proxy != "" {
err = apiClient.SetProxy(proxy)
if err != nil {
@@ -153,7 +168,7 @@ func (apiClient *ApiClient) SetProxy(proxyUrl string) error {
return err
}
if pu.Scheme == "http" || pu.Scheme == "socks5" {
- apiClient.client.Transport = &http.Transport{Proxy: http.ProxyURL(pu)}
+ apiClient.client.Transport.(*http.Transport).Proxy = http.ProxyURL(pu)
}
return nil
}
diff --git a/plugins/icla/tasks/api_client.go b/plugins/icla/tasks/api_client.go
index d9d73d22..6e695e2b 100644
--- a/plugins/icla/tasks/api_client.go
+++ b/plugins/icla/tasks/api_client.go
@@ -19,10 +19,11 @@ package tasks
import (
"fmt"
+ "net/http"
+
"github.com/apache/incubator-devlake/plugins/core"
"github.com/apache/incubator-devlake/plugins/helper"
"github.com/apache/incubator-devlake/utils"
- "net/http"
)
const ENDPOINT = "https://people.apache.org/"
@@ -40,7 +41,7 @@ func NewIclaApiClient(taskCtx core.TaskContext) (*helper.ApiAsyncClient, error)
proxy := taskCtx.GetConfig("ICLA_PROXY")
// real request apiClient
- apiClient, err := helper.NewApiClient(taskCtx.GetContext(), ENDPOINT, nil, 0, proxy)
+ apiClient, err := helper.NewApiClient(taskCtx.GetContext(), ENDPOINT, nil, 0, proxy, taskCtx)
if err != nil {
return nil, err
}
diff --git a/plugins/jenkins/api/connection.go b/plugins/jenkins/api/connection.go
index 78472304..40b6b664 100644
--- a/plugins/jenkins/api/connection.go
+++ b/plugins/jenkins/api/connection.go
@@ -46,6 +46,7 @@ func TestConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput, erro
}
// test connection
encodedToken := utils.GetEncodedToken(connection.Username, connection.Password)
+
apiClient, err := helper.NewApiClient(
context.TODO(),
connection.Endpoint,
@@ -54,6 +55,7 @@ func TestConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput, erro
},
3*time.Second,
connection.Proxy,
+ basicRes,
)
if err != nil {
return nil, err
diff --git a/plugins/jenkins/tasks/client.go b/plugins/jenkins/tasks/client.go
index 42d9ecde..b226b321 100644
--- a/plugins/jenkins/tasks/client.go
+++ b/plugins/jenkins/tasks/client.go
@@ -32,7 +32,7 @@ func CreateApiClient(taskCtx core.TaskContext, connection *models.JenkinsConnect
"Authorization": fmt.Sprintf("Basic %v", connection.GetEncodedToken()),
}
- apiClient, err := helper.NewApiClient(taskCtx.GetContext(), connection.Endpoint, headers, 0, connection.Proxy)
+ apiClient, err := helper.NewApiClient(taskCtx.GetContext(), connection.Endpoint, headers, 0, connection.Proxy, taskCtx)
if err != nil {
return nil, err
}
diff --git a/plugins/jira/api/connection.go b/plugins/jira/api/connection.go
index 25296098..d3bf634b 100644
--- a/plugins/jira/api/connection.go
+++ b/plugins/jira/api/connection.go
@@ -45,7 +45,6 @@ func TestConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput, erro
if err != nil {
return nil, err
}
-
// test connection
apiClient, err := helper.NewApiClient(
context.TODO(),
@@ -55,6 +54,7 @@ func TestConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput, erro
},
3*time.Second,
connection.Proxy,
+ basicRes,
)
if err != nil {
return nil, err
diff --git a/plugins/jira/api/proxy.go b/plugins/jira/api/proxy.go
index 3511e73f..65ec132b 100644
--- a/plugins/jira/api/proxy.go
+++ b/plugins/jira/api/proxy.go
@@ -47,6 +47,7 @@ func Proxy(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) {
},
30*time.Second,
connection.Proxy,
+ basicRes,
)
if err != nil {
return nil, err
diff --git a/plugins/jira/tasks/api_client.go b/plugins/jira/tasks/api_client.go
index c31eb855..13f94aef 100644
--- a/plugins/jira/tasks/api_client.go
+++ b/plugins/jira/tasks/api_client.go
@@ -31,7 +31,8 @@ func NewJiraApiClient(taskCtx core.TaskContext, connection *models.JiraConnectio
headers := map[string]string{
"Authorization": fmt.Sprintf("Basic %v", connection.GetEncodedToken()),
}
- apiClient, err := helper.NewApiClient(taskCtx.GetContext(), connection.Endpoint, headers, 0, connection.Proxy)
+
+ apiClient, err := helper.NewApiClient(taskCtx.GetContext(), connection.Endpoint, headers, 0, connection.Proxy, taskCtx)
if err != nil {
return nil, err
}
diff --git a/plugins/tapd/api/connection.go b/plugins/tapd/api/connection.go
index 084b2985..ec1767c0 100644
--- a/plugins/tapd/api/connection.go
+++ b/plugins/tapd/api/connection.go
@@ -45,6 +45,7 @@ func TestConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput, erro
if err != nil {
return nil, err
}
+
// verify multiple token in parallel
// PLEASE NOTE: This works because GitHub API Client rotates tokens on each request
token := params.Auth
@@ -56,6 +57,7 @@ func TestConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput, erro
},
3*time.Second,
params.Proxy,
+ basicRes,
)
if err != nil {
return nil, fmt.Errorf("verify token failed for %s %w", token, err)
diff --git a/plugins/tapd/tasks/api_client.go b/plugins/tapd/tasks/api_client.go
index dc38b27d..c5b0c480 100644
--- a/plugins/tapd/tasks/api_client.go
+++ b/plugins/tapd/tasks/api_client.go
@@ -33,7 +33,8 @@ func NewTapdApiClient(taskCtx core.TaskContext, connection *models.TapdConnectio
headers := map[string]string{
"Authorization": fmt.Sprintf("Basic %v", auth),
}
- apiClient, err := helper.NewApiClient(taskCtx.GetContext(), connection.Endpoint, headers, 0, "")
+
+ apiClient, err := helper.NewApiClient(taskCtx.GetContext(), connection.Endpoint, headers, 0, "", taskCtx)
if err != nil {
return nil, err
}
diff --git a/utils/strconv.go b/utils/strconv.go
index 4327eeb7..18f244f6 100644
--- a/utils/strconv.go
+++ b/utils/strconv.go
@@ -37,3 +37,11 @@ func StrToDurationOr(text string, defaultValue time.Duration) (time.Duration, er
}
return time.ParseDuration(text)
}
+
+// StrToBoolOr Return defaultValue if text is empty, or try to convert it to bool
+func StrToBoolOr(text string, defaultValue bool) (bool, error) {
+ if text == "" {
+ return defaultValue, nil
+ }
+ return strconv.ParseBool(text)
+}