You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by kl...@apache.org on 2022/06/20 10:31:30 UTC
[incubator-devlake] branch main updated: fix: ignore http status code 404 (#2240)
This is an automated email from the ASF dual-hosted git repository.
klesh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new afd8723a fix: ignore http status code 404 (#2240)
afd8723a is described below
commit afd8723aba5d86eebb4b74de9c9b29922eab68d9
Author: mindlesscloud <li...@merico.dev>
AuthorDate: Mon Jun 20 18:31:25 2022 +0800
fix: ignore http status code 404 (#2240)
* fix: ignore http status code 404
* fix: register migrationscripts.UpdateSchemas20220615 to framework
* fix: dismiss 404 on collecting changelog and remotelink
* fix: cycle import
* fix: fix test TestFetchPageUndetermined
* refactor: move ApiClientBeforeRequest from to
---
plugins/helper/api_async_client.go | 6 ++++++
plugins/helper/api_client.go | 16 ++++++++++------
plugins/helper/api_collector.go | 21 +++++++++++++++++++--
plugins/helper/api_collector_test.go | 2 +-
plugins/helper/common/{callbacks.go => hooks.go} | 3 +++
plugins/jira/jira.go | 1 +
plugins/jira/tasks/api_client.go | 10 ++++++++++
plugins/jira/tasks/changelog_collector.go | 1 +
plugins/jira/tasks/remotelink_collector.go | 1 +
plugins/jira/tasks/worklog_collector.go | 1 +
10 files changed, 53 insertions(+), 9 deletions(-)
diff --git a/plugins/helper/api_async_client.go b/plugins/helper/api_async_client.go
index f29a7c50..4cf5b88e 100644
--- a/plugins/helper/api_async_client.go
+++ b/plugins/helper/api_async_client.go
@@ -154,6 +154,11 @@ func (apiClient *ApiAsyncClient) DoAsync(
res.Body = io.NopCloser(bytes.NewBuffer(body))
}
}
+ if err == ErrIgnoreAndContinue {
+ // make sure defer func got be executed
+ err = nil
+ return nil
+ }
// check
needRetry := false
@@ -220,6 +225,7 @@ type RateLimitedApiClient interface {
HasError() bool
NextTick(task func() error)
GetNumOfWorkers() int
+ SetAfterFunction(callback common.ApiClientAfterResponse)
}
var _ RateLimitedApiClient = (*ApiAsyncClient)(nil)
diff --git a/plugins/helper/api_client.go b/plugins/helper/api_client.go
index 001b4c66..bed0c8a2 100644
--- a/plugins/helper/api_client.go
+++ b/plugins/helper/api_client.go
@@ -21,6 +21,7 @@ import (
"bytes"
"context"
"encoding/json"
+ "errors"
"fmt"
"io"
"io/ioutil"
@@ -32,19 +33,19 @@ import (
"unicode/utf8"
"github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/helper/common"
"github.com/apache/incubator-devlake/utils"
)
-type ApiClientBeforeRequest func(req *http.Request) error
-type ApiClientAfterResponse func(res *http.Response) error
+var ErrIgnoreAndContinue = errors.New("ignore and continue")
// ApiClient is designed for simple api requests
type ApiClient struct {
client *http.Client
endpoint string
headers map[string]string
- beforeRequest ApiClientBeforeRequest
- afterReponse ApiClientAfterResponse
+ beforeRequest common.ApiClientBeforeRequest
+ afterReponse common.ApiClientAfterResponse
ctx context.Context
logger core.Logger
}
@@ -120,11 +121,11 @@ func (apiClient *ApiClient) GetHeaders() map[string]string {
return apiClient.headers
}
-func (apiClient *ApiClient) SetBeforeFunction(callback ApiClientBeforeRequest) {
+func (apiClient *ApiClient) SetBeforeFunction(callback common.ApiClientBeforeRequest) {
apiClient.beforeRequest = callback
}
-func (apiClient *ApiClient) SetAfterFunction(callback ApiClientAfterResponse) {
+func (apiClient *ApiClient) SetAfterFunction(callback common.ApiClientAfterResponse) {
apiClient.afterReponse = callback
}
@@ -220,6 +221,9 @@ func (apiClient *ApiClient) Do(
// after receive
if apiClient.afterReponse != nil {
err = apiClient.afterReponse(res)
+ if err == ErrIgnoreAndContinue {
+ return res, err
+ }
if err != nil {
res.Body.Close()
return nil, err
diff --git a/plugins/helper/api_collector.go b/plugins/helper/api_collector.go
index ed8a115c..1fe0a64b 100644
--- a/plugins/helper/api_collector.go
+++ b/plugins/helper/api_collector.go
@@ -28,6 +28,7 @@ import (
"github.com/apache/incubator-devlake/plugins/core"
"github.com/apache/incubator-devlake/plugins/core/dal"
+ "github.com/apache/incubator-devlake/plugins/helper/common"
)
// Pager contains pagination information for a api request
@@ -73,6 +74,7 @@ type ApiCollectorArgs struct {
// NORMALLY, DO NOT SPECIFY THIS PARAMETER, unless you know what it means
Concurrency int
ResponseParser func(res *http.Response) ([]json.RawMessage, error)
+ AfterResponse common.ApiClientAfterResponse
}
type ApiCollector struct {
@@ -104,11 +106,22 @@ func NewApiCollector(args ApiCollectorArgs) (*ApiCollector, error) {
if args.ResponseParser == nil {
return nil, fmt.Errorf("ResponseParser is required")
}
- return &ApiCollector{
+ apicllector := &ApiCollector{
RawDataSubTask: rawDataSubTask,
args: &args,
urlTemplate: tpl,
- }, nil
+ }
+ if args.AfterResponse != nil {
+ apicllector.SetAfterResponse(args.AfterResponse)
+ } else {
+ apicllector.SetAfterResponse(func(res *http.Response) error {
+ if res.StatusCode == http.StatusUnauthorized {
+ return fmt.Errorf("authentication failed, please check your AccessToken")
+ }
+ return nil
+ })
+ }
+ return apicllector, nil
}
// Start collection
@@ -273,6 +286,10 @@ func (collector *ApiCollector) generateUrl(pager *Pager, input interface{}) (str
return buf.String(), nil
}
+func (collector *ApiCollector) SetAfterResponse(f common.ApiClientAfterResponse) {
+ collector.args.ApiClient.SetAfterFunction(f)
+}
+
func (collector *ApiCollector) fetchAsync(reqData *RequestData, handler func(int, []byte, *http.Response) error) {
if reqData.Pager == nil {
reqData.Pager = &Pager{
diff --git a/plugins/helper/api_collector_test.go b/plugins/helper/api_collector_test.go
index 42613509..400d85f1 100644
--- a/plugins/helper/api_collector_test.go
+++ b/plugins/helper/api_collector_test.go
@@ -72,7 +72,7 @@ func TestFetchPageUndetermined(t *testing.T) {
}).Twice()
mockApi.On("HasError").Return(false)
mockApi.On("WaitAsync").Return(nil)
-
+ mockApi.On("SetAfterFunction", mock.Anything).Return()
params := struct {
Name string
}{Name: "testparams"}
diff --git a/plugins/helper/common/callbacks.go b/plugins/helper/common/hooks.go
similarity index 87%
rename from plugins/helper/common/callbacks.go
rename to plugins/helper/common/hooks.go
index e7dd5633..5454b07b 100644
--- a/plugins/helper/common/callbacks.go
+++ b/plugins/helper/common/hooks.go
@@ -20,3 +20,6 @@ package common
import "net/http"
type ApiAsyncCallback func(*http.Response) error
+
+type ApiClientBeforeRequest func(req *http.Request) error
+type ApiClientAfterResponse func(res *http.Response) error
diff --git a/plugins/jira/jira.go b/plugins/jira/jira.go
index 828c0baf..11f26281 100644
--- a/plugins/jira/jira.go
+++ b/plugins/jira/jira.go
@@ -168,6 +168,7 @@ func (plugin Jira) MigrationScripts() []migration.Script {
new(migrationscripts.UpdateSchemas20220527),
new(migrationscripts.UpdateSchemas20220601),
new(migrationscripts.UpdateSchemas20220614),
+ new(migrationscripts.UpdateSchemas20220615),
new(migrationscripts.UpdateSchemas20220616),
}
}
diff --git a/plugins/jira/tasks/api_client.go b/plugins/jira/tasks/api_client.go
index b6eddd8e..7138db81 100644
--- a/plugins/jira/tasks/api_client.go
+++ b/plugins/jira/tasks/api_client.go
@@ -79,3 +79,13 @@ func GetJiraServerInfo(client *helper.ApiAsyncClient) (*models.JiraServerInfo, i
}
return serverInfo, res.StatusCode, nil
}
+
+func ignoreHTTPStatus404(res *http.Response) error {
+ if res.StatusCode == http.StatusUnauthorized {
+ return fmt.Errorf("authentication failed, please check your AccessToken")
+ }
+ if res.StatusCode == http.StatusNotFound {
+ return helper.ErrIgnoreAndContinue
+ }
+ return nil
+}
diff --git a/plugins/jira/tasks/changelog_collector.go b/plugins/jira/tasks/changelog_collector.go
index ebbb9c44..fe5f0632 100644
--- a/plugins/jira/tasks/changelog_collector.go
+++ b/plugins/jira/tasks/changelog_collector.go
@@ -111,6 +111,7 @@ func CollectChangelogs(taskCtx core.SubTaskContext) error {
}
return data.Values, nil
},
+ AfterResponse: ignoreHTTPStatus404,
})
if err != nil {
diff --git a/plugins/jira/tasks/remotelink_collector.go b/plugins/jira/tasks/remotelink_collector.go
index ac8067e1..f1e2c7bf 100644
--- a/plugins/jira/tasks/remotelink_collector.go
+++ b/plugins/jira/tasks/remotelink_collector.go
@@ -94,6 +94,7 @@ func CollectRemotelinks(taskCtx core.SubTaskContext) error {
}
return result, nil
},
+ AfterResponse: ignoreHTTPStatus404,
})
if err != nil {
return err
diff --git a/plugins/jira/tasks/worklog_collector.go b/plugins/jira/tasks/worklog_collector.go
index 53282fb0..54f172fb 100644
--- a/plugins/jira/tasks/worklog_collector.go
+++ b/plugins/jira/tasks/worklog_collector.go
@@ -90,6 +90,7 @@ func CollectWorklogs(taskCtx core.SubTaskContext) error {
}
return data.Worklogs, nil
},
+ AfterResponse: ignoreHTTPStatus404,
})
if err != nil {
logger.Error("collect board error:", err)