You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by kl...@apache.org on 2023/02/23 09:04:58 UTC
[incubator-devlake] branch main updated: refactor: bitbucket adopts timeAfter (#4495)
This is an automated email from the ASF dual-hosted git repository.
klesh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new b48d4293e refactor: bitbucket adopts timeAfter (#4495)
b48d4293e is described below
commit b48d4293eaa5581b4c73a5492b48736f28d2b272
Author: Likyh <ya...@meri.co>
AuthorDate: Thu Feb 23 17:04:54 2023 +0800
refactor: bitbucket adopts timeAfter (#4495)
---
backend/plugins/bitbucket/api/blueprint_v200.go | 4 ++--
backend/plugins/bitbucket/bitbucket.go | 8 ++++----
backend/plugins/bitbucket/impl/impl.go | 14 +++++++-------
backend/plugins/bitbucket/tasks/api_common.go | 21 ++++-----------------
backend/plugins/bitbucket/tasks/issue_collector.go | 2 +-
.../bitbucket/tasks/issue_comment_collector.go | 2 +-
.../plugins/bitbucket/tasks/pipeline_collector.go | 2 +-
.../bitbucket/tasks/pipeline_steps_collector.go | 2 +-
backend/plugins/bitbucket/tasks/pr_collector.go | 2 +-
.../plugins/bitbucket/tasks/pr_comment_collector.go | 2 +-
.../plugins/bitbucket/tasks/pr_commit_collector.go | 2 +-
backend/plugins/bitbucket/tasks/task_data.go | 8 ++++----
12 files changed, 28 insertions(+), 41 deletions(-)
diff --git a/backend/plugins/bitbucket/api/blueprint_v200.go b/backend/plugins/bitbucket/api/blueprint_v200.go
index 92a338fb7..cb1c74cbd 100644
--- a/backend/plugins/bitbucket/api/blueprint_v200.go
+++ b/backend/plugins/bitbucket/api/blueprint_v200.go
@@ -107,8 +107,8 @@ func makeDataSourcePipelinePlanV200(
ConnectionId: repo.ConnectionId,
FullName: repo.BitbucketId,
}
- if syncPolicy.CreatedDateAfter != nil {
- op.CreatedDateAfter = syncPolicy.CreatedDateAfter.Format(time.RFC3339)
+ if syncPolicy.TimeAfter != nil {
+ op.TimeAfter = syncPolicy.TimeAfter.Format(time.RFC3339)
}
options, err := tasks.EncodeTaskOptions(op)
if err != nil {
diff --git a/backend/plugins/bitbucket/bitbucket.go b/backend/plugins/bitbucket/bitbucket.go
index 91ee34941..8d6ba8b65 100644
--- a/backend/plugins/bitbucket/bitbucket.go
+++ b/backend/plugins/bitbucket/bitbucket.go
@@ -31,15 +31,15 @@ func main() {
cmd := &cobra.Command{Use: "bitbucket"}
connectionId := cmd.Flags().Uint64P("connectionId", "c", 0, "bitbucket connection id")
fullName := cmd.Flags().StringP("fullName", "n", "", "bitbucket id: owner/repo")
- createdDateAfter := cmd.Flags().StringP("createdDateAfter", "a", "", "collect data that are created after specified time, ie 2006-05-06T07:08:09Z")
+ timeAfter := cmd.Flags().StringP("timeAfter", "a", "", "collect data that are created after specified time, ie 2006-05-06T07:08:09Z")
_ = cmd.MarkFlagRequired("connectionId")
_ = cmd.MarkFlagRequired("fullName")
cmd.Run = func(cmd *cobra.Command, args []string) {
runner.DirectRun(cmd, args, PluginEntry, map[string]interface{}{
- "connectionId": *connectionId,
- "fullName": *fullName,
- "createdDateAfter": *createdDateAfter,
+ "connectionId": *connectionId,
+ "fullName": *fullName,
+ "timeAfter": *timeAfter,
})
}
runner.RunCmd(cmd)
diff --git a/backend/plugins/bitbucket/impl/impl.go b/backend/plugins/bitbucket/impl/impl.go
index 0a11007bd..927667108 100644
--- a/backend/plugins/bitbucket/impl/impl.go
+++ b/backend/plugins/bitbucket/impl/impl.go
@@ -148,20 +148,20 @@ func (p Bitbucket) PrepareTaskData(taskCtx plugin.TaskContext, options map[strin
return nil, err
}
- var createdDateAfter time.Time
- if op.CreatedDateAfter != "" {
- createdDateAfter, err = errors.Convert01(time.Parse(time.RFC3339, op.CreatedDateAfter))
+ var timeAfter time.Time
+ if op.TimeAfter != "" {
+ timeAfter, err = errors.Convert01(time.Parse(time.RFC3339, op.TimeAfter))
if err != nil {
- return nil, errors.BadInput.Wrap(err, "invalid value for `createdDateAfter`")
+ return nil, errors.BadInput.Wrap(err, "invalid value for `timeAfter`")
}
}
taskData := &tasks.BitbucketTaskData{
Options: op,
ApiClient: apiClient,
}
- if !createdDateAfter.IsZero() {
- taskData.CreatedDateAfter = &createdDateAfter
- logger.Debug("collect data updated createdDateAfter %s", createdDateAfter)
+ if !timeAfter.IsZero() {
+ taskData.TimeAfter = &timeAfter
+ logger.Debug("collect data updated timeAfter %s", timeAfter)
}
return taskData, nil
diff --git a/backend/plugins/bitbucket/tasks/api_common.go b/backend/plugins/bitbucket/tasks/api_common.go
index c3d994ce6..bb20a5100 100644
--- a/backend/plugins/bitbucket/tasks/api_common.go
+++ b/backend/plugins/bitbucket/tasks/api_common.go
@@ -99,16 +99,12 @@ func GetQueryCreatedAndUpdated(fields string, collectorWithState *api.ApiCollect
}
query.Set("fields", fields)
query.Set("sort", "created_on")
- if collectorWithState.IsIncremental() && collectorWithState.CreatedDateAfter != nil {
- latestSuccessStart := collectorWithState.LatestState.LatestSuccessStart.Format("2006-01-02")
- createdDateAfter := collectorWithState.CreatedDateAfter.Format("2006-01-02")
- query.Set("q", fmt.Sprintf("updated_on>=%s AND created_on>=%s", latestSuccessStart, createdDateAfter))
- } else if collectorWithState.IsIncremental() {
+ if collectorWithState.IsIncremental() {
latestSuccessStart := collectorWithState.LatestState.LatestSuccessStart.Format("2006-01-02")
query.Set("q", fmt.Sprintf("updated_on>=%s", latestSuccessStart))
- } else if collectorWithState.CreatedDateAfter != nil {
- createdDateAfter := collectorWithState.CreatedDateAfter.Format("2006-01-02")
- query.Set("q", fmt.Sprintf("created_on>=%s", createdDateAfter))
+ } else if collectorWithState.TimeAfter != nil {
+ timeAfter := collectorWithState.TimeAfter.Format("2006-01-02")
+ query.Set("q", fmt.Sprintf("updated_on>=%s", timeAfter))
}
return query, nil
@@ -181,9 +177,6 @@ func GetPullRequestsIterator(taskCtx plugin.SubTaskContext, collectorWithState *
data.Options.FullName, data.Options.ConnectionId,
),
}
- if collectorWithState.CreatedDateAfter != nil {
- clauses = append(clauses, dal.Where("bitbucket_created_at > ?", *collectorWithState.CreatedDateAfter))
- }
if collectorWithState.IsIncremental() {
clauses = append(clauses, dal.Where("bitbucket_updated_at > ?", *collectorWithState.LatestState.LatestSuccessStart))
}
@@ -207,9 +200,6 @@ func GetIssuesIterator(taskCtx plugin.SubTaskContext, collectorWithState *api.Ap
data.Options.FullName, data.Options.ConnectionId,
),
}
- if collectorWithState.CreatedDateAfter != nil {
- clauses = append(clauses, dal.Where("bitbucket_created_at > ?", *collectorWithState.CreatedDateAfter))
- }
if collectorWithState.IsIncremental() {
clauses = append(clauses, dal.Where("bitbucket_updated_at > ?", *collectorWithState.LatestState.LatestSuccessStart))
}
@@ -233,9 +223,6 @@ func GetPipelinesIterator(taskCtx plugin.SubTaskContext, collectorWithState *api
data.Options.FullName, data.Options.ConnectionId,
),
}
- if collectorWithState.CreatedDateAfter != nil {
- clauses = append(clauses, dal.Where("bitbucket_created_on > ?", *collectorWithState.CreatedDateAfter))
- }
if collectorWithState.IsIncremental() {
clauses = append(clauses, dal.Where("bitbucket_complete_on > ?", *collectorWithState.LatestState.LatestSuccessStart))
}
diff --git a/backend/plugins/bitbucket/tasks/issue_collector.go b/backend/plugins/bitbucket/tasks/issue_collector.go
index 9725ddc90..ef5f56fd5 100644
--- a/backend/plugins/bitbucket/tasks/issue_collector.go
+++ b/backend/plugins/bitbucket/tasks/issue_collector.go
@@ -35,7 +35,7 @@ var CollectApiIssuesMeta = plugin.SubTaskMeta{
func CollectApiIssues(taskCtx plugin.SubTaskContext) errors.Error {
rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_ISSUE_TABLE)
- collectorWithState, err := helper.NewApiCollectorWithState(*rawDataSubTaskArgs, data.CreatedDateAfter)
+ collectorWithState, err := helper.NewStatefulApiCollector(*rawDataSubTaskArgs, data.TimeAfter)
if err != nil {
return err
}
diff --git a/backend/plugins/bitbucket/tasks/issue_comment_collector.go b/backend/plugins/bitbucket/tasks/issue_comment_collector.go
index 8493f0844..9bc981125 100644
--- a/backend/plugins/bitbucket/tasks/issue_comment_collector.go
+++ b/backend/plugins/bitbucket/tasks/issue_comment_collector.go
@@ -36,7 +36,7 @@ var CollectApiIssueCommentsMeta = plugin.SubTaskMeta{
func CollectApiIssueComments(taskCtx plugin.SubTaskContext) errors.Error {
rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_ISSUE_COMMENTS_TABLE)
- collectorWithState, err := helper.NewApiCollectorWithState(*rawDataSubTaskArgs, data.CreatedDateAfter)
+ collectorWithState, err := helper.NewStatefulApiCollector(*rawDataSubTaskArgs, data.TimeAfter)
if err != nil {
return err
}
diff --git a/backend/plugins/bitbucket/tasks/pipeline_collector.go b/backend/plugins/bitbucket/tasks/pipeline_collector.go
index 86ab06d51..58a5f6a4c 100644
--- a/backend/plugins/bitbucket/tasks/pipeline_collector.go
+++ b/backend/plugins/bitbucket/tasks/pipeline_collector.go
@@ -35,7 +35,7 @@ var CollectApiPipelinesMeta = plugin.SubTaskMeta{
func CollectApiPipelines(taskCtx plugin.SubTaskContext) errors.Error {
rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_PIPELINE_TABLE)
- collectorWithState, err := helper.NewApiCollectorWithState(*rawDataSubTaskArgs, data.CreatedDateAfter)
+ collectorWithState, err := helper.NewStatefulApiCollector(*rawDataSubTaskArgs, data.TimeAfter)
if err != nil {
return err
}
diff --git a/backend/plugins/bitbucket/tasks/pipeline_steps_collector.go b/backend/plugins/bitbucket/tasks/pipeline_steps_collector.go
index 77299e013..6ceaa26a2 100644
--- a/backend/plugins/bitbucket/tasks/pipeline_steps_collector.go
+++ b/backend/plugins/bitbucket/tasks/pipeline_steps_collector.go
@@ -38,7 +38,7 @@ var CollectPipelineStepsMeta = plugin.SubTaskMeta{
func CollectPipelineSteps(taskCtx plugin.SubTaskContext) errors.Error {
rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_PIPELINE_STEPS_TABLE)
- collectorWithState, err := helper.NewApiCollectorWithState(*rawDataSubTaskArgs, data.CreatedDateAfter)
+ collectorWithState, err := helper.NewStatefulApiCollector(*rawDataSubTaskArgs, data.TimeAfter)
if err != nil {
return err
}
diff --git a/backend/plugins/bitbucket/tasks/pr_collector.go b/backend/plugins/bitbucket/tasks/pr_collector.go
index 5b84c0309..f804eb2af 100644
--- a/backend/plugins/bitbucket/tasks/pr_collector.go
+++ b/backend/plugins/bitbucket/tasks/pr_collector.go
@@ -38,7 +38,7 @@ var CollectApiPullRequestsMeta = plugin.SubTaskMeta{
func CollectApiPullRequests(taskCtx plugin.SubTaskContext) errors.Error {
rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_PULL_REQUEST_TABLE)
- collectorWithState, err := helper.NewApiCollectorWithState(*rawDataSubTaskArgs, data.CreatedDateAfter)
+ collectorWithState, err := helper.NewStatefulApiCollector(*rawDataSubTaskArgs, data.TimeAfter)
if err != nil {
return err
}
diff --git a/backend/plugins/bitbucket/tasks/pr_comment_collector.go b/backend/plugins/bitbucket/tasks/pr_comment_collector.go
index 1e5756778..f96a33c7b 100644
--- a/backend/plugins/bitbucket/tasks/pr_comment_collector.go
+++ b/backend/plugins/bitbucket/tasks/pr_comment_collector.go
@@ -36,7 +36,7 @@ var CollectApiPrCommentsMeta = plugin.SubTaskMeta{
func CollectApiPullRequestsComments(taskCtx plugin.SubTaskContext) errors.Error {
rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_PULL_REQUEST_COMMENTS_TABLE)
- collectorWithState, err := helper.NewApiCollectorWithState(*rawDataSubTaskArgs, data.CreatedDateAfter)
+ collectorWithState, err := helper.NewStatefulApiCollector(*rawDataSubTaskArgs, data.TimeAfter)
if err != nil {
return err
}
diff --git a/backend/plugins/bitbucket/tasks/pr_commit_collector.go b/backend/plugins/bitbucket/tasks/pr_commit_collector.go
index 91c33fe13..14c53d5d2 100644
--- a/backend/plugins/bitbucket/tasks/pr_commit_collector.go
+++ b/backend/plugins/bitbucket/tasks/pr_commit_collector.go
@@ -37,7 +37,7 @@ var CollectApiPrCommitsMeta = plugin.SubTaskMeta{
func CollectApiPullRequestCommits(taskCtx plugin.SubTaskContext) errors.Error {
rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_PULL_REQUEST_COMMITS_TABLE)
- collectorWithState, err := helper.NewApiCollectorWithState(*rawDataSubTaskArgs, data.CreatedDateAfter)
+ collectorWithState, err := helper.NewStatefulApiCollector(*rawDataSubTaskArgs, data.TimeAfter)
if err != nil {
return err
}
diff --git a/backend/plugins/bitbucket/tasks/task_data.go b/backend/plugins/bitbucket/tasks/task_data.go
index 4f447c899..02eb39874 100644
--- a/backend/plugins/bitbucket/tasks/task_data.go
+++ b/backend/plugins/bitbucket/tasks/task_data.go
@@ -29,15 +29,15 @@ type BitbucketOptions struct {
ConnectionId uint64 `json:"connectionId" mapstructure:"connectionId,omitempty"`
Tasks []string `json:"tasks,omitempty" mapstructure:",omitempty"`
FullName string `json:"fullName" mapstructure:"fullName"`
- CreatedDateAfter string `json:"createdDateAfter" mapstructure:"createdDateAfter,omitempty"`
+ TimeAfter string `json:"timeAfter" mapstructure:"timeAfter,omitempty"`
TransformationRuleId uint64 `json:"transformationRuleId" mapstructure:"transformationRuleId,omitempty"`
*models.BitbucketTransformationRule `mapstructure:"transformationRules,omitempty" json:"transformationRules"`
}
type BitbucketTaskData struct {
- Options *BitbucketOptions
- ApiClient *api.ApiAsyncClient
- CreatedDateAfter *time.Time
+ Options *BitbucketOptions
+ ApiClient *api.ApiAsyncClient
+ TimeAfter *time.Time
}
func DecodeAndValidateTaskOptions(options map[string]interface{}) (*BitbucketOptions, errors.Error) {