You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by "warren830 (via GitHub)" <gi...@apache.org> on 2023/02/28 08:36:35 UTC
[GitHub] [incubator-devlake] warren830 opened a new pull request, #4542: fix(gitlab): fix bugs to resolve block
warren830 opened a new pull request, #4542:
URL: https://github.com/apache/incubator-devlake/pull/4542
### Summary
1. Add gitlab_updated_at to gitlab mr
2. Fix gitlab job api logic
3. modify mr test
### Does this close any open issues?
Closes #4541
### Screenshots
<img width="743" alt="image" src="https://user-images.githubusercontent.com/39366025/221798276-8af377a0-aef7-4936-94ef-2a9e73da8f11.png">
### Other Information
Any other information that is important to this PR.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@devlake.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [incubator-devlake] likyh merged pull request #4542: fix(gitlab): fix bugs to resolve block
Posted by "likyh (via GitHub)" <gi...@apache.org>.
likyh merged PR #4542:
URL: https://github.com/apache/incubator-devlake/pull/4542
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@devlake.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [incubator-devlake] likyh commented on a diff in pull request #4542: fix(gitlab): fix bugs to resolve block
Posted by "likyh (via GitHub)" <gi...@apache.org>.
likyh commented on code in PR #4542:
URL: https://github.com/apache/incubator-devlake/pull/4542#discussion_r1119798748
##########
backend/plugins/gitlab/tasks/job_collector.go:
##########
@@ -35,16 +49,72 @@ var CollectApiJobsMeta = plugin.SubTaskMeta{
func CollectApiJobs(taskCtx plugin.SubTaskContext) errors.Error {
Review Comment:
Does this collector support gitlab V11?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@devlake.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [incubator-devlake] likyh commented on a diff in pull request #4542: fix(gitlab): fix bugs to resolve block
Posted by "likyh (via GitHub)" <gi...@apache.org>.
likyh commented on code in PR #4542:
URL: https://github.com/apache/incubator-devlake/pull/4542#discussion_r1119797991
##########
backend/plugins/gitlab/tasks/job_collector.go:
##########
@@ -35,16 +49,72 @@ var CollectApiJobsMeta = plugin.SubTaskMeta{
func CollectApiJobs(taskCtx plugin.SubTaskContext) errors.Error {
rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_JOB_TABLE)
-
- collector, err := helper.NewApiCollector(helper.ApiCollectorArgs{
+ db := taskCtx.GetDal()
+ collector, err := helper.NewStatefulApiCollectorForFinalizableEntity(helper.FinalizableApiCollectorArgs{
RawDataSubTaskArgs: *rawDataSubTaskArgs,
ApiClient: data.ApiClient,
- PageSize: 100,
- Incremental: false,
- UrlTemplate: "projects/{{ .Params.ProjectId }}/jobs",
- Query: GetQuery,
- ResponseParser: GetRawMessageUpdatedAtAfter(data.TimeAfter),
- AfterResponse: ignoreHTTPStatus403, // ignore 403 for CI/CD disable
+ TimeAfter: data.TimeAfter, // set to nil to disable timeFilter
+ CollectNewRecordsByList: helper.FinalizableApiCollectorListArgs{
+ PageSize: 100,
+ Concurrency: 10,
+ FinalizableApiCollectorCommonArgs: helper.FinalizableApiCollectorCommonArgs{
+ UrlTemplate: "projects/{{ .Params.ProjectId }}/jobs",
+ Query: func(reqData *helper.RequestData, createdAfter *time.Time) (url.Values, errors.Error) {
+ query := url.Values{}
+ query.Set("with_stats", "true")
+ query.Set("sort", "asc")
+ query.Set("page", strconv.Itoa(reqData.Pager.Page))
+ query.Set("per_page", strconv.Itoa(reqData.Pager.Size))
+ return query, nil
+ },
+ ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) {
+ var items []json.RawMessage
+ err := helper.UnmarshalResponse(res, &items)
+ if err != nil {
+ return nil, err
+ }
+ return items, nil
+ },
+ AfterResponse: ignoreHTTPStatus403, // ignore 403 for CI/CD disable
+ },
+ GetCreated: func(item json.RawMessage) (time.Time, errors.Error) {
+ pr := &SimpleGitlabApiJob{}
+ err := json.Unmarshal(item, pr)
+ if err != nil {
+ return time.Time{}, errors.BadInput.Wrap(err, "failed to unmarshal github pull request")
Review Comment:
wrong message
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@devlake.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [incubator-devlake] likyh commented on a diff in pull request #4542: fix(gitlab): fix bugs to resolve block
Posted by "likyh (via GitHub)" <gi...@apache.org>.
likyh commented on code in PR #4542:
URL: https://github.com/apache/incubator-devlake/pull/4542#discussion_r1119797459
##########
backend/plugins/gitlab/tasks/job_collector.go:
##########
@@ -35,16 +49,72 @@ var CollectApiJobsMeta = plugin.SubTaskMeta{
func CollectApiJobs(taskCtx plugin.SubTaskContext) errors.Error {
rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_JOB_TABLE)
-
- collector, err := helper.NewApiCollector(helper.ApiCollectorArgs{
+ db := taskCtx.GetDal()
+ collector, err := helper.NewStatefulApiCollectorForFinalizableEntity(helper.FinalizableApiCollectorArgs{
RawDataSubTaskArgs: *rawDataSubTaskArgs,
ApiClient: data.ApiClient,
- PageSize: 100,
- Incremental: false,
- UrlTemplate: "projects/{{ .Params.ProjectId }}/jobs",
- Query: GetQuery,
- ResponseParser: GetRawMessageUpdatedAtAfter(data.TimeAfter),
- AfterResponse: ignoreHTTPStatus403, // ignore 403 for CI/CD disable
+ TimeAfter: data.TimeAfter, // set to nil to disable timeFilter
+ CollectNewRecordsByList: helper.FinalizableApiCollectorListArgs{
+ PageSize: 100,
+ Concurrency: 10,
+ FinalizableApiCollectorCommonArgs: helper.FinalizableApiCollectorCommonArgs{
+ UrlTemplate: "projects/{{ .Params.ProjectId }}/jobs",
+ Query: func(reqData *helper.RequestData, createdAfter *time.Time) (url.Values, errors.Error) {
+ query := url.Values{}
+ query.Set("with_stats", "true")
Review Comment:
there are no with_stats and sor query.
https://docs.gitlab.com/ee/api/jobs.html
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@devlake.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [incubator-devlake] warren830 commented on a diff in pull request #4542: fix(gitlab): fix bugs to resolve block
Posted by "warren830 (via GitHub)" <gi...@apache.org>.
warren830 commented on code in PR #4542:
URL: https://github.com/apache/incubator-devlake/pull/4542#discussion_r1119805834
##########
backend/plugins/gitlab/tasks/job_collector.go:
##########
@@ -35,16 +49,72 @@ var CollectApiJobsMeta = plugin.SubTaskMeta{
func CollectApiJobs(taskCtx plugin.SubTaskContext) errors.Error {
Review Comment:
Haven't checked
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@devlake.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [incubator-devlake] warren830 commented on a diff in pull request #4542: fix(gitlab): fix bugs to resolve block
Posted by "warren830 (via GitHub)" <gi...@apache.org>.
warren830 commented on code in PR #4542:
URL: https://github.com/apache/incubator-devlake/pull/4542#discussion_r1119805483
##########
backend/plugins/gitlab/tasks/job_collector.go:
##########
@@ -35,16 +49,72 @@ var CollectApiJobsMeta = plugin.SubTaskMeta{
func CollectApiJobs(taskCtx plugin.SubTaskContext) errors.Error {
rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_JOB_TABLE)
-
- collector, err := helper.NewApiCollector(helper.ApiCollectorArgs{
+ db := taskCtx.GetDal()
+ collector, err := helper.NewStatefulApiCollectorForFinalizableEntity(helper.FinalizableApiCollectorArgs{
RawDataSubTaskArgs: *rawDataSubTaskArgs,
ApiClient: data.ApiClient,
- PageSize: 100,
- Incremental: false,
- UrlTemplate: "projects/{{ .Params.ProjectId }}/jobs",
- Query: GetQuery,
- ResponseParser: GetRawMessageUpdatedAtAfter(data.TimeAfter),
- AfterResponse: ignoreHTTPStatus403, // ignore 403 for CI/CD disable
+ TimeAfter: data.TimeAfter, // set to nil to disable timeFilter
+ CollectNewRecordsByList: helper.FinalizableApiCollectorListArgs{
+ PageSize: 100,
+ Concurrency: 10,
+ FinalizableApiCollectorCommonArgs: helper.FinalizableApiCollectorCommonArgs{
+ UrlTemplate: "projects/{{ .Params.ProjectId }}/jobs",
+ Query: func(reqData *helper.RequestData, createdAfter *time.Time) (url.Values, errors.Error) {
+ query := url.Values{}
+ query.Set("with_stats", "true")
Review Comment:
This existed before I changed, so I just copied these params here
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@devlake.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [incubator-devlake] likyh commented on a diff in pull request #4542: fix(gitlab): fix bugs to resolve block
Posted by "likyh (via GitHub)" <gi...@apache.org>.
likyh commented on code in PR #4542:
URL: https://github.com/apache/incubator-devlake/pull/4542#discussion_r1119797459
##########
backend/plugins/gitlab/tasks/job_collector.go:
##########
@@ -35,16 +49,72 @@ var CollectApiJobsMeta = plugin.SubTaskMeta{
func CollectApiJobs(taskCtx plugin.SubTaskContext) errors.Error {
rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_JOB_TABLE)
-
- collector, err := helper.NewApiCollector(helper.ApiCollectorArgs{
+ db := taskCtx.GetDal()
+ collector, err := helper.NewStatefulApiCollectorForFinalizableEntity(helper.FinalizableApiCollectorArgs{
RawDataSubTaskArgs: *rawDataSubTaskArgs,
ApiClient: data.ApiClient,
- PageSize: 100,
- Incremental: false,
- UrlTemplate: "projects/{{ .Params.ProjectId }}/jobs",
- Query: GetQuery,
- ResponseParser: GetRawMessageUpdatedAtAfter(data.TimeAfter),
- AfterResponse: ignoreHTTPStatus403, // ignore 403 for CI/CD disable
+ TimeAfter: data.TimeAfter, // set to nil to disable timeFilter
+ CollectNewRecordsByList: helper.FinalizableApiCollectorListArgs{
+ PageSize: 100,
+ Concurrency: 10,
+ FinalizableApiCollectorCommonArgs: helper.FinalizableApiCollectorCommonArgs{
+ UrlTemplate: "projects/{{ .Params.ProjectId }}/jobs",
+ Query: func(reqData *helper.RequestData, createdAfter *time.Time) (url.Values, errors.Error) {
+ query := url.Values{}
+ query.Set("with_stats", "true")
Review Comment:
there are no query `with_stats` and `sort`.
https://docs.gitlab.com/ee/api/jobs.html
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@devlake.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org