You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by li...@apache.org on 2023/02/06 14:38:22 UTC
[incubator-devlake] branch main updated: [Bitbucket] Pagination params and merged date on PRs (#4292)
This is an automated email from the ASF dual-hosted git repository.
likyh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 9bdad0f8c [Bitbucket] Pagination params and merged date on PRs (#4292)
9bdad0f8c is described below
commit 9bdad0f8c5e4be8efdd0bd9f51d795de3002b9c9
Author: João Ricardo <jr...@gmail.com>
AuthorDate: Mon Feb 6 11:38:16 2023 -0300
[Bitbucket] Pagination params and merged date on PRs (#4292)
* fix(Bitbucket): change pagination params to `start` and `limit`
* feat(Bitbucket): Add merge commit datetime to the PR `MergedAt` property
* fix: change to set as merge commit date when err is nilt
* feat: add `GetNextPageCustomData` to `collectApiPullRequestCommits` task collector
* fix: fix for ci..
---------
Co-authored-by: linyh <ya...@meri.co>
---
backend/plugins/bitbucket/tasks/api_common.go | 59 ++++++++++++++++++----
.../plugins/bitbucket/tasks/pr_commit_collector.go | 17 ++++---
backend/plugins/bitbucket/tasks/pr_convertor.go | 9 ++++
3 files changed, 66 insertions(+), 19 deletions(-)
diff --git a/backend/plugins/bitbucket/tasks/api_common.go b/backend/plugins/bitbucket/tasks/api_common.go
index 0fa691007..4e456ac46 100644
--- a/backend/plugins/bitbucket/tasks/api_common.go
+++ b/backend/plugins/bitbucket/tasks/api_common.go
@@ -62,6 +62,23 @@ func CreateRawDataSubTaskArgs(taskCtx plugin.SubTaskContext, Table string) (*api
return RawDataSubTaskArgs, data
}
+func decodeResponse(res *http.Response, message interface{}) errors.Error {
+ if res == nil {
+ return errors.Default.New("res is nil")
+ }
+ defer res.Body.Close()
+ resBody, err := io.ReadAll(res.Body)
+ if err != nil {
+ return errors.Default.Wrap(err, fmt.Sprintf("error reading response from %s", res.Request.URL.String()))
+ }
+
+ err = errors.Convert(json.Unmarshal(resBody, &message))
+ if err != nil {
+ return errors.Default.Wrap(err, fmt.Sprintf("error decoding response from %s: raw response: %s", res.Request.URL.String(), string(resBody)))
+ }
+ return nil
+}
+
func GetQuery(reqData *api.RequestData) (url.Values, errors.Error) {
query := url.Values{}
query.Set("state", "all")
@@ -71,6 +88,35 @@ func GetQuery(reqData *api.RequestData) (url.Values, errors.Error) {
return query, nil
}
+func GetQueryForNext(reqData *api.RequestData) (url.Values, errors.Error) {
+ query := url.Values{}
+ query.Set("state", "all")
+ query.Set("pagelen", fmt.Sprintf("%v", reqData.Pager.Size))
+
+ if reqData.CustomData != nil {
+ query.Set("page", reqData.CustomData.(string))
+ }
+ return query, nil
+}
+
+func GetNextPageCustomData(_ *api.RequestData, prevPageResponse *http.Response) (interface{}, errors.Error) {
+ var rawMessages struct {
+ Next string `json:"next"`
+ }
+ err := decodeResponse(prevPageResponse, &rawMessages)
+ if err != nil {
+ return nil, err
+ }
+ if rawMessages.Next == `` {
+ return ``, api.ErrFinishCollect
+ }
+ u, err := errors.Convert01(url.Parse(rawMessages.Next))
+ if err != nil {
+ return nil, err
+ }
+ return u.Query()[`page`][0], nil
+}
+
func GetTotalPagesFromResponse(res *http.Response, args *api.ApiCollectorArgs) (int, errors.Error) {
body := &BitbucketPagination{}
err := api.UnmarshalResponse(res, body)
@@ -88,18 +134,9 @@ func GetRawMessageFromResponse(res *http.Response) ([]json.RawMessage, errors.Er
var rawMessages struct {
Values []json.RawMessage `json:"values"`
}
- if res == nil {
- return nil, errors.Default.New("res is nil")
- }
- defer res.Body.Close()
- resBody, err := io.ReadAll(res.Body)
+ err := decodeResponse(res, &rawMessages)
if err != nil {
- return nil, errors.Default.Wrap(err, fmt.Sprintf("error reading response from %s", res.Request.URL.String()))
- }
-
- err = errors.Convert(json.Unmarshal(resBody, &rawMessages))
- if err != nil {
- return nil, errors.Default.Wrap(err, fmt.Sprintf("error decoding response from %s: raw response: %s", res.Request.URL.String(), string(resBody)))
+ return nil, err
}
return rawMessages.Values, nil
diff --git a/backend/plugins/bitbucket/tasks/pr_commit_collector.go b/backend/plugins/bitbucket/tasks/pr_commit_collector.go
index c8eee20da..a0303275c 100644
--- a/backend/plugins/bitbucket/tasks/pr_commit_collector.go
+++ b/backend/plugins/bitbucket/tasks/pr_commit_collector.go
@@ -43,14 +43,15 @@ func CollectApiPullRequestCommits(taskCtx plugin.SubTaskContext) errors.Error {
defer iterator.Close()
collector, err := helper.NewApiCollector(helper.ApiCollectorArgs{
- RawDataSubTaskArgs: *rawDataSubTaskArgs,
- ApiClient: data.ApiClient,
- PageSize: 100,
- Incremental: false,
- Input: iterator,
- UrlTemplate: "repositories/{{ .Params.Owner }}/{{ .Params.Repo }}/pullrequests/{{ .Input.BitbucketId }}/commits",
- Query: GetQuery,
- ResponseParser: GetRawMessageFromResponse,
+ RawDataSubTaskArgs: *rawDataSubTaskArgs,
+ ApiClient: data.ApiClient,
+ PageSize: 100,
+ Incremental: false,
+ Input: iterator,
+ UrlTemplate: "repositories/{{ .Params.Owner }}/{{ .Params.Repo }}/pullrequests/{{ .Input.BitbucketId }}/commits",
+ GetNextPageCustomData: GetNextPageCustomData,
+ Query: GetQueryForNext,
+ ResponseParser: GetRawMessageFromResponse,
})
if err != nil {
diff --git a/backend/plugins/bitbucket/tasks/pr_convertor.go b/backend/plugins/bitbucket/tasks/pr_convertor.go
index b6ae47dc1..0f8794608 100644
--- a/backend/plugins/bitbucket/tasks/pr_convertor.go
+++ b/backend/plugins/bitbucket/tasks/pr_convertor.go
@@ -70,6 +70,15 @@ func ConvertPullRequests(taskCtx plugin.SubTaskContext) errors.Error {
},
Convert: func(inputRow interface{}) ([]interface{}, errors.Error) {
pr := inputRow.(*models.BitbucketPullRequest)
+
+ // Getting the merge reference commit
+ mergeCommit := &models.BitbucketCommit{}
+ err = db.First(mergeCommit, dal.Where("LEFT(sha, 12) = ?", pr.MergeCommitSha))
+ if err == nil {
+ // Setting the PR merged datetime to the commit commited datetime
+ pr.MergedAt = &mergeCommit.CommittedDate
+ }
+
domainPr := &code.PullRequest{
DomainEntity: domainlayer.DomainEntity{
Id: prIdGen.Generate(data.Options.ConnectionId, pr.BitbucketId),