You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by "abeizn (via GitHub)" <gi...@apache.org> on 2023/09/12 09:46:41 UTC

[GitHub] [incubator-devlake] abeizn opened a new pull request, #6064: refactor: syncPolicy timeAfter

abeizn opened a new pull request, #6064:
URL: https://github.com/apache/incubator-devlake/pull/6064

   ### Summary
   adjust syncPolicy timeAfter
   
   ### Does this close any open issues?
   Closes #5841 
   
   ### Screenshots
   Include any relevant screenshots here.
   
   ### Other Information
   Any other information that is important to this PR.
   


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscribe@devlake.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


[GitHub] [incubator-devlake] abeizn commented on a diff in pull request #6064: refactor: syncPolicy timeAfter

Posted by "abeizn (via GitHub)" <gi...@apache.org>.
abeizn commented on code in PR #6064:
URL: https://github.com/apache/incubator-devlake/pull/6064#discussion_r1324062371


##########
backend/plugins/bitbucket/tasks/api_common.go:
##########
@@ -100,11 +101,12 @@ func GetQueryCreatedAndUpdated(fields string, collectorWithState *api.ApiCollect
 		}
 		query.Set("fields", fields)
 		query.Set("sort", "created_on")
+		syncPolicy := collectorWithState.Ctx.TaskContext().SyncPolicy()
 		if collectorWithState.IsIncremental() {
 			latestSuccessStart := collectorWithState.LatestState.LatestSuccessStart.Format(time.RFC3339)
 			query.Set("q", fmt.Sprintf("updated_on>=%s", latestSuccessStart))
-		} else if collectorWithState.TimeAfter != nil {
-			timeAfter := collectorWithState.TimeAfter.Format(time.RFC3339)
+		} else if syncPolicy != nil && syncPolicy.TimeAfter != nil {

Review Comment:
   fixed



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscribe@devlake.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


[GitHub] [incubator-devlake] abeizn commented on a diff in pull request #6064: refactor: syncPolicy timeAfter

Posted by "abeizn (via GitHub)" <gi...@apache.org>.
abeizn commented on code in PR #6064:
URL: https://github.com/apache/incubator-devlake/pull/6064#discussion_r1324062164


##########
backend/core/runner/run_task.go:
##########
@@ -54,8 +54,16 @@ func RunTask(
 		return err
 	}
 	syncPolicy := &models.SyncPolicy{}
-	syncPolicy.SkipOnFail = dbPipeline.SkipOnFail
-	syncPolicy.FullSync = dbPipeline.FullSync
+	blueprint := &models.Blueprint{}

Review Comment:
   fixed



##########
backend/plugins/ae/ae.go:
##########
@@ -30,12 +30,13 @@ func main() {
 	aeCmd := &cobra.Command{Use: "ae"}
 	connectionId := aeCmd.Flags().Uint64P("Connection-id", "c", 0, "ae connection id")
 	projectId := aeCmd.Flags().IntP("project-id", "p", 0, "ae project id")
+	timeAfter := aeCmd.Flags().StringP("timeAfter", "a", "", "collect data that are created after specified time, ie 2006-01-02T15:04:05Z")

Review Comment:
   fixed



##########
backend/plugins/bamboo/bamboo.go:
##########
@@ -31,6 +31,7 @@ func main() {
 	bambooCmd := &cobra.Command{Use: "bamboo"}
 	connectionId := bambooCmd.Flags().Uint64P("Connection-id", "c", 0, "bamboo connection id")
 	projectKey := bambooCmd.Flags().StringP("project-key", "p", "", "bamboo project key")
+	timeAfter := bambooCmd.Flags().StringP("timeAfter", "a", "", "collect data that are created after specified time, ie 2006-01-02T15:04:05Z")

Review Comment:
   fixed



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscribe@devlake.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


[GitHub] [incubator-devlake] klesh commented on a diff in pull request #6064: refactor: syncPolicy timeAfter

Posted by "klesh (via GitHub)" <gi...@apache.org>.
klesh commented on code in PR #6064:
URL: https://github.com/apache/incubator-devlake/pull/6064#discussion_r1323970447


##########
backend/core/runner/run_task.go:
##########
@@ -54,8 +54,16 @@ func RunTask(
 		return err
 	}
 	syncPolicy := &models.SyncPolicy{}
-	syncPolicy.SkipOnFail = dbPipeline.SkipOnFail
-	syncPolicy.FullSync = dbPipeline.FullSync
+	blueprint := &models.Blueprint{}

Review Comment:
   This doesn't seem right, Why do we need to access the bp when running a Task? Shouldn't we access the `dbPipeline.SyncPolicy` instead?



##########
backend/plugins/bamboo/bamboo.go:
##########
@@ -31,6 +31,7 @@ func main() {
 	bambooCmd := &cobra.Command{Use: "bamboo"}
 	connectionId := bambooCmd.Flags().Uint64P("Connection-id", "c", 0, "bamboo connection id")
 	projectKey := bambooCmd.Flags().StringP("project-key", "p", "", "bamboo project key")
+	timeAfter := bambooCmd.Flags().StringP("timeAfter", "a", "", "collect data that are created after specified time, ie 2006-01-02T15:04:05Z")

Review Comment:
   Same as the above



##########
backend/plugins/ae/ae.go:
##########
@@ -30,12 +30,13 @@ func main() {
 	aeCmd := &cobra.Command{Use: "ae"}
 	connectionId := aeCmd.Flags().Uint64P("Connection-id", "c", 0, "ae connection id")
 	projectId := aeCmd.Flags().IntP("project-id", "p", 0, "ae project id")
+	timeAfter := aeCmd.Flags().StringP("timeAfter", "a", "", "collect data that are created after specified time, ie 2006-01-02T15:04:05Z")

Review Comment:
   Should be `time-after` instead of `timeAfter` because the previous line is using `project-id`



##########
backend/plugins/bitbucket/tasks/api_common.go:
##########
@@ -100,11 +101,12 @@ func GetQueryCreatedAndUpdated(fields string, collectorWithState *api.ApiCollect
 		}
 		query.Set("fields", fields)
 		query.Set("sort", "created_on")
+		syncPolicy := collectorWithState.Ctx.TaskContext().SyncPolicy()
 		if collectorWithState.IsIncremental() {
 			latestSuccessStart := collectorWithState.LatestState.LatestSuccessStart.Format(time.RFC3339)
 			query.Set("q", fmt.Sprintf("updated_on>=%s", latestSuccessStart))
-		} else if collectorWithState.TimeAfter != nil {
-			timeAfter := collectorWithState.TimeAfter.Format(time.RFC3339)
+		} else if syncPolicy != nil && syncPolicy.TimeAfter != nil {

Review Comment:
   Doesn't seem correct.
   `collectorWithState.TimeAfter` may be later than `syncPolicy.TimeAfter` because of `incremental sync`, the `collectorWithState.TimeAfter` would be updated automatically after each data collection while `syncPolicy.TimeAfter` can only be updated by the user



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscribe@devlake.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


[GitHub] [incubator-devlake] klesh commented on a diff in pull request #6064: refactor: syncPolicy timeAfter

Posted by "klesh (via GitHub)" <gi...@apache.org>.
klesh commented on code in PR #6064:
URL: https://github.com/apache/incubator-devlake/pull/6064#discussion_r1324172220


##########
backend/helpers/pluginhelper/api/api_collector_with_state.go:
##########
@@ -154,18 +160,19 @@ func NewStatefulApiCollectorForFinalizableEntity(args FinalizableApiCollectorArg
 		Options: args.Options,
 		Params:  args.Params,
 		Table:   args.Table,
-	}, args.TimeAfter)
+	})
 	if err != nil {
 		return nil, err
 	}
 
 	// // prepare the basic variables
+	syncPolicy := manager.Ctx.TaskContext().SyncPolicy()
 	var isIncremental = manager.IsIncremental()
 	var createdAfter *time.Time
 	if isIncremental {
 		createdAfter = manager.LatestState.LatestSuccessStart
-	} else {
-		createdAfter = manager.TimeAfter
+	} else if syncPolicy != nil && syncPolicy.TimeAfter != nil {

Review Comment:
   the change for this file should not be needed since https://github.com/apache/incubator-devlake/pull/6064/files#diff-c0bf74c35ffacb49d0e0b6a70207324ed5ae01fb1f4c841e2ebeb9bf6b0627c2R67 already takes care the `manager.TimeAfter`



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscribe@devlake.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


[GitHub] [incubator-devlake] abeizn commented on a diff in pull request #6064: refactor: syncPolicy timeAfter

Posted by "abeizn (via GitHub)" <gi...@apache.org>.
abeizn commented on code in PR #6064:
URL: https://github.com/apache/incubator-devlake/pull/6064#discussion_r1324202411


##########
backend/helpers/pluginhelper/api/api_collector_with_state.go:
##########
@@ -154,18 +160,19 @@ func NewStatefulApiCollectorForFinalizableEntity(args FinalizableApiCollectorArg
 		Options: args.Options,
 		Params:  args.Params,
 		Table:   args.Table,
-	}, args.TimeAfter)
+	})
 	if err != nil {
 		return nil, err
 	}
 
 	// // prepare the basic variables
+	syncPolicy := manager.Ctx.TaskContext().SyncPolicy()
 	var isIncremental = manager.IsIncremental()
 	var createdAfter *time.Time
 	if isIncremental {
 		createdAfter = manager.LatestState.LatestSuccessStart
-	} else {
-		createdAfter = manager.TimeAfter
+	} else if syncPolicy != nil && syncPolicy.TimeAfter != nil {

Review Comment:
   fixed



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscribe@devlake.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


[GitHub] [incubator-devlake] klesh merged pull request #6064: refactor: syncPolicy timeAfter

Posted by "klesh (via GitHub)" <gi...@apache.org>.
klesh merged PR #6064:
URL: https://github.com/apache/incubator-devlake/pull/6064


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscribe@devlake.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org