You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by "magichair (via GitHub)" <gi...@apache.org> on 2025/03/18 13:54:36 UTC
[PR] feat(circleci): Cherry Pick #7820 #7986 [incubator-devlake]
magichair opened a new pull request, #8341:
URL: https://github.com/apache/incubator-devlake/pull/8341
### Summary
What does this PR do?
Re-introduces two PRs that fix various issues with the circleci plugin #7820 and #7986
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@devlake.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] feat(circleci): Cherry Pick #7820 #7986 [incubator-devlake]
Posted by "Nickcw6 (via GitHub)" <gi...@apache.org>.
Nickcw6 commented on code in PR #8341:
URL: https://github.com/apache/incubator-devlake/pull/8341#discussion_r2003915928
##########
backend/plugins/circleci/tasks/job_collector.go:
##########
@@ -43,30 +46,68 @@ func CollectJobs(taskCtx plugin.SubTaskContext) errors.Error {
logger := taskCtx.GetLogger()
logger.Info("collect jobs")
- clauses := []dal.Clause{
- dal.Select("id, pipeline_id"),
- dal.From(&models.CircleciWorkflow{}),
- dal.Where("_tool_circleci_workflows.connection_id = ? and _tool_circleci_workflows.project_slug = ? ", data.Options.ConnectionId, data.Options.ProjectSlug),
- }
+ collector, err := api.NewStatefulApiCollectorForFinalizableEntity(api.FinalizableApiCollectorArgs{
+ RawDataSubTaskArgs: *rawDataSubTaskArgs,
+ ApiClient: data.ApiClient,
+ CollectNewRecordsByList: api.FinalizableApiCollectorListArgs{
+ PageSize: int(data.Options.PageSize),
+ GetNextPageCustomData: ExtractNextPageToken,
+ BuildInputIterator: func(isIncremental bool, createdAfter *time.Time) (api.Iterator, errors.Error) {
+ clauses := []dal.Clause{
+ dal.Select("id, pipeline_id"), // pipeline_id not on individual job response but required for result
+ dal.From(&models.CircleciWorkflow{}),
+ dal.Where("connection_id = ? and project_slug = ?", data.Options.ConnectionId, data.Options.ProjectSlug),
+ }
- db := taskCtx.GetDal()
- cursor, err := db.Cursor(clauses...)
- if err != nil {
- return err
- }
- iterator, err := api.NewDalCursorIterator(db, cursor, reflect.TypeOf(models.CircleciWorkflow{}))
- if err != nil {
- return err
- }
+ if isIncremental {
+ clauses = append(clauses, dal.Where("created_date > ?", createdAfter))
+ }
Review Comment:
@jibsen-vh I think it's because the changes from this PR were not cherrypicked: https://github.com/apache/incubator-devlake/pull/7757
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@devlake.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] feat(circleci): Cherry Pick #7820 #7986 [incubator-devlake]
Posted by "klesh (via GitHub)" <gi...@apache.org>.
klesh commented on PR #8341:
URL: https://github.com/apache/incubator-devlake/pull/8341#issuecomment-2735083157
Thanks!
Will release a new `beta` this week.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@devlake.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] feat(circleci): Cherry Pick #7820 #7986 [incubator-devlake]
Posted by "Nickcw6 (via GitHub)" <gi...@apache.org>.
Nickcw6 commented on code in PR #8341:
URL: https://github.com/apache/incubator-devlake/pull/8341#discussion_r2003915928
##########
backend/plugins/circleci/tasks/job_collector.go:
##########
@@ -43,30 +46,68 @@ func CollectJobs(taskCtx plugin.SubTaskContext) errors.Error {
logger := taskCtx.GetLogger()
logger.Info("collect jobs")
- clauses := []dal.Clause{
- dal.Select("id, pipeline_id"),
- dal.From(&models.CircleciWorkflow{}),
- dal.Where("_tool_circleci_workflows.connection_id = ? and _tool_circleci_workflows.project_slug = ? ", data.Options.ConnectionId, data.Options.ProjectSlug),
- }
+ collector, err := api.NewStatefulApiCollectorForFinalizableEntity(api.FinalizableApiCollectorArgs{
+ RawDataSubTaskArgs: *rawDataSubTaskArgs,
+ ApiClient: data.ApiClient,
+ CollectNewRecordsByList: api.FinalizableApiCollectorListArgs{
+ PageSize: int(data.Options.PageSize),
+ GetNextPageCustomData: ExtractNextPageToken,
+ BuildInputIterator: func(isIncremental bool, createdAfter *time.Time) (api.Iterator, errors.Error) {
+ clauses := []dal.Clause{
+ dal.Select("id, pipeline_id"), // pipeline_id not on individual job response but required for result
+ dal.From(&models.CircleciWorkflow{}),
+ dal.Where("connection_id = ? and project_slug = ?", data.Options.ConnectionId, data.Options.ProjectSlug),
+ }
- db := taskCtx.GetDal()
- cursor, err := db.Cursor(clauses...)
- if err != nil {
- return err
- }
- iterator, err := api.NewDalCursorIterator(db, cursor, reflect.TypeOf(models.CircleciWorkflow{}))
- if err != nil {
- return err
- }
+ if isIncremental {
+ clauses = append(clauses, dal.Where("created_date > ?", createdAfter))
+ }
Review Comment:
@jibsen-vh I think it's because this PR has yet to be cherrypicked to apache:main: https://github.com/apache/incubator-devlake/pull/7757
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@devlake.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] feat(circleci): Cherry Pick #7820 #7986 [incubator-devlake]
Posted by "klesh (via GitHub)" <gi...@apache.org>.
klesh merged PR #8341:
URL: https://github.com/apache/incubator-devlake/pull/8341
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@devlake.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] feat(circleci): Cherry Pick #7820 #7986 [incubator-devlake]
Posted by "jibsen-vh (via GitHub)" <gi...@apache.org>.
jibsen-vh commented on code in PR #8341:
URL: https://github.com/apache/incubator-devlake/pull/8341#discussion_r2003371351
##########
backend/plugins/circleci/tasks/job_collector.go:
##########
@@ -43,30 +46,68 @@ func CollectJobs(taskCtx plugin.SubTaskContext) errors.Error {
logger := taskCtx.GetLogger()
logger.Info("collect jobs")
- clauses := []dal.Clause{
- dal.Select("id, pipeline_id"),
- dal.From(&models.CircleciWorkflow{}),
- dal.Where("_tool_circleci_workflows.connection_id = ? and _tool_circleci_workflows.project_slug = ? ", data.Options.ConnectionId, data.Options.ProjectSlug),
- }
+ collector, err := api.NewStatefulApiCollectorForFinalizableEntity(api.FinalizableApiCollectorArgs{
+ RawDataSubTaskArgs: *rawDataSubTaskArgs,
+ ApiClient: data.ApiClient,
+ CollectNewRecordsByList: api.FinalizableApiCollectorListArgs{
+ PageSize: int(data.Options.PageSize),
+ GetNextPageCustomData: ExtractNextPageToken,
+ BuildInputIterator: func(isIncremental bool, createdAfter *time.Time) (api.Iterator, errors.Error) {
+ clauses := []dal.Clause{
+ dal.Select("id, pipeline_id"), // pipeline_id not on individual job response but required for result
+ dal.From(&models.CircleciWorkflow{}),
+ dal.Where("connection_id = ? and project_slug = ?", data.Options.ConnectionId, data.Options.ProjectSlug),
+ }
- db := taskCtx.GetDal()
- cursor, err := db.Cursor(clauses...)
- if err != nil {
- return err
- }
- iterator, err := api.NewDalCursorIterator(db, cursor, reflect.TypeOf(models.CircleciWorkflow{}))
- if err != nil {
- return err
- }
+ if isIncremental {
+ clauses = append(clauses, dal.Where("created_date > ?", createdAfter))
+ }
Review Comment:
@klesh this brought in a regression it looks like. Something is missing with regards to migrations - I'm still investigating, but my local deploy of this didn't do well in the first incremental attempted job.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@devlake.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] feat(circleci): Cherry Pick #7820 #7986 [incubator-devlake]
Posted by "Nickcw6 (via GitHub)" <gi...@apache.org>.
Nickcw6 commented on code in PR #8341:
URL: https://github.com/apache/incubator-devlake/pull/8341#discussion_r2003915928
##########
backend/plugins/circleci/tasks/job_collector.go:
##########
@@ -43,30 +46,68 @@ func CollectJobs(taskCtx plugin.SubTaskContext) errors.Error {
logger := taskCtx.GetLogger()
logger.Info("collect jobs")
- clauses := []dal.Clause{
- dal.Select("id, pipeline_id"),
- dal.From(&models.CircleciWorkflow{}),
- dal.Where("_tool_circleci_workflows.connection_id = ? and _tool_circleci_workflows.project_slug = ? ", data.Options.ConnectionId, data.Options.ProjectSlug),
- }
+ collector, err := api.NewStatefulApiCollectorForFinalizableEntity(api.FinalizableApiCollectorArgs{
+ RawDataSubTaskArgs: *rawDataSubTaskArgs,
+ ApiClient: data.ApiClient,
+ CollectNewRecordsByList: api.FinalizableApiCollectorListArgs{
+ PageSize: int(data.Options.PageSize),
+ GetNextPageCustomData: ExtractNextPageToken,
+ BuildInputIterator: func(isIncremental bool, createdAfter *time.Time) (api.Iterator, errors.Error) {
+ clauses := []dal.Clause{
+ dal.Select("id, pipeline_id"), // pipeline_id not on individual job response but required for result
+ dal.From(&models.CircleciWorkflow{}),
+ dal.Where("connection_id = ? and project_slug = ?", data.Options.ConnectionId, data.Options.ProjectSlug),
+ }
- db := taskCtx.GetDal()
- cursor, err := db.Cursor(clauses...)
- if err != nil {
- return err
- }
- iterator, err := api.NewDalCursorIterator(db, cursor, reflect.TypeOf(models.CircleciWorkflow{}))
- if err != nil {
- return err
- }
+ if isIncremental {
+ clauses = append(clauses, dal.Where("created_date > ?", createdAfter))
+ }
Review Comment:
@jibsen-vh I think it's because this PR has yet to be cherrypicked to main: https://github.com/apache/incubator-devlake/pull/7757
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@devlake.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org