You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by he...@apache.org on 2023/05/16 19:29:10 UTC
[incubator-devlake] branch main updated: fix: PagerDuty now limits to 10k latest incidents per API spec, and pulls based on service ids (#5196)
This is an automated email from the ASF dual-hosted git repository.
hez pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 5e94d2379 fix: PagerDuty now limits to 10k latest incidents per API spec, and pulls based on service ids (#5196)
5e94d2379 is described below
commit 5e94d237952801215b7126b9dfd20b265d4c4679
Author: Keon Amini <ke...@merico.dev>
AuthorDate: Tue May 16 12:29:05 2023 -0700
fix: PagerDuty now limits to 10k latest incidents per API spec, and pulls based on service ids (#5196)
---
backend/helpers/pluginhelper/api/api_async_client.go | 9 +++++----
backend/plugins/pagerduty/tasks/incidents_collector.go | 14 +++++++-------
2 files changed, 12 insertions(+), 11 deletions(-)
diff --git a/backend/helpers/pluginhelper/api/api_async_client.go b/backend/helpers/pluginhelper/api/api_async_client.go
index e45794c44..715245d64 100644
--- a/backend/helpers/pluginhelper/api/api_async_client.go
+++ b/backend/helpers/pluginhelper/api/api_async_client.go
@@ -181,13 +181,14 @@ func (apiClient *ApiAsyncClient) DoAsync(
// check
needRetry := false
+ errMessage := "unknown"
if err != nil {
needRetry = true
+ errMessage = err.Error()
} else if res.StatusCode >= HttpMinStatusRetryCode {
needRetry = true
- err = errors.HttpStatus(res.StatusCode).New(
- fmt.Sprintf("Http DoAsync error calling [%s %s]. Response: %s", method, path, string(respBody)),
- )
+ errMessage = fmt.Sprintf("Http DoAsync error calling [%s %s]. Response: %s", method, path, string(respBody))
+ err = errors.HttpStatus(res.StatusCode).New(errMessage)
}
// if it needs retry, check and retry
@@ -205,7 +206,7 @@ func (apiClient *ApiAsyncClient) DoAsync(
}
if err != nil {
- err = errors.Default.Wrap(err, fmt.Sprintf("retry exceeded %d times calling %s", retry, path))
+ err = errors.Default.Wrap(err, fmt.Sprintf("Retry exceeded %d times calling %s. The last error was: %s", retry, path, errMessage))
apiClient.logger.Error(err, "")
return errors.Convert(err)
}
diff --git a/backend/plugins/pagerduty/tasks/incidents_collector.go b/backend/plugins/pagerduty/tasks/incidents_collector.go
index 65b6b1cc8..146478f02 100644
--- a/backend/plugins/pagerduty/tasks/incidents_collector.go
+++ b/backend/plugins/pagerduty/tasks/incidents_collector.go
@@ -71,13 +71,12 @@ func CollectIncidents(taskCtx plugin.SubTaskContext) errors.Error {
TimeAfter: data.TimeAfter,
CollectNewRecordsByList: api.FinalizableApiCollectorListArgs{
PageSize: 100,
- GetTotalPages: func(res *http.Response, args *api.ApiCollectorArgs) (int, errors.Error) {
- paging := pagingInfo{}
- err := api.UnmarshalResponse(res, &paging)
- if err != nil {
- return 0, errors.BadInput.Wrap(err, "failed to determined paging count")
+ GetNextPageCustomData: func(prevReqData *api.RequestData, prevPageResponse *http.Response) (interface{}, errors.Error) {
+ pager := prevReqData.Pager
+ if pager.Skip+pager.Size >= 10_000 { // API limit. Can't exceed this or it'll error out
+ return nil, api.ErrFinishCollect
}
- return *paging.Total, nil
+ return nil, nil
},
FinalizableApiCollectorCommonArgs: api.FinalizableApiCollectorCommonArgs{
UrlTemplate: "incidents",
@@ -95,7 +94,8 @@ func CollectIncidents(taskCtx plugin.SubTaskContext) errors.Error {
} else {
query.Set("date_range", "all")
}
- query.Set("sort_by", "created_at:desc")
+ query.Set("service_ids[]", data.Options.ServiceId)
+ query.Set("sort_by", "created_at:desc") // the newest entries will be fetched first
query.Set("limit", fmt.Sprintf("%d", reqData.Pager.Size))
query.Set("offset", fmt.Sprintf("%d", reqData.Pager.Skip))
query.Set("total", "true")