You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by GitBox <gi...@apache.org> on 2022/11/10 06:16:24 UTC
[GitHub] [incubator-devlake] warren830 opened a new pull request, #3711: fix(tapd): resolve issues impacted incremental collection
warren830 opened a new pull request, #3711:
URL: https://github.com/apache/incubator-devlake/pull/3711
# Summary
1. use an more accurate `date` to update data by using since.In(CstZone).Format("2006-01-02") to present date
2. I found that some idGenerator which we used to generate `domain layer Id` is spreading everywhere, so make some changes to order them
3. another minor issue(never influence our previous data report): the entity task used a wrong tag name for the field title
4. I also found that bug/story/task are using an empty issue to generate their domain layer id, so I correct them in this pr
### Does this close any open issues?
Closes #3710
### Screenshots
Include any relevant screenshots here.
### Other Information
Any other information that is important to this PR.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@devlake.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [incubator-devlake] abeizn commented on a diff in pull request #3711: fix(tapd): resolve issues impacted incremental collection
Posted by GitBox <gi...@apache.org>.
abeizn commented on code in PR #3711:
URL: https://github.com/apache/incubator-devlake/pull/3711#discussion_r1018697206
##########
plugins/tapd/models/migrationscripts/20221109_add_init_tables.go:
##########
@@ -136,7 +136,7 @@ func (*addInitTables) Up(basicRes core.BasicRes) errors.Error {
}
func (*addInitTables) Version() uint64 {
- return 20221013201138
Review Comment:
need updated version?
##########
plugins/tapd/tasks/story_bug_collector.go:
##########
@@ -42,52 +38,33 @@ func CollectStoryBugs(taskCtx core.SubTaskContext) errors.Error {
logger := taskCtx.GetLogger()
logger.Info("collect storyBugs")
since := data.Since
- incremental := false
- if since == nil {
- // user didn't specify a time range to sync, try load from database
- var latestUpdated models.TapdStoryCommit
- clauses := []dal.Clause{
- dal.Where("connection_id = ? and workspace_id = ?", data.Options.ConnectionId, data.Options.WorkspaceId),
- dal.Orderby("created DESC"),
- }
- err := db.First(&latestUpdated, clauses...)
- if err != nil && !goerror.Is(err, gorm.ErrRecordNotFound) {
- return errors.NotFound.Wrap(err, "failed to get latest tapd changelog record")
- }
- if latestUpdated.Id > 0 {
- since = (*time.Time)(latestUpdated.Created)
- incremental = true
- }
- }
-
clauses := []dal.Clause{
- dal.Select("id"),
+ dal.Select("id as issue_id, modified as update_time"),
dal.From(&models.TapdStory{}),
- dal.Where("connection_id = ? and workspace_id = ?", data.Options.ConnectionId, data.Options.WorkspaceId),
+ dal.Where("_tool_tapd_stories.connection_id = ? and _tool_tapd_stories.workspace_id = ? ", data.Options.ConnectionId, data.Options.WorkspaceId),
}
if since != nil {
clauses = append(clauses, dal.Where("modified > ?", since))
}
-
cursor, err := db.Cursor(clauses...)
if err != nil {
return err
}
- iterator, err := helper.NewDalCursorIterator(db, cursor, reflect.TypeOf(SimpleStory{}))
+ iterator, err := helper.NewDalCursorIterator(db, cursor, reflect.TypeOf(models.Input{}))
if err != nil {
return err
}
collector, err := helper.NewApiCollector(helper.ApiCollectorArgs{
RawDataSubTaskArgs: *rawDataSubTaskArgs,
ApiClient: data.ApiClient,
- Incremental: incremental,
+ Incremental: since == nil,
Review Comment:
same as above
##########
helpers/migrationhelper/migrationhelper.go:
##########
@@ -183,7 +183,7 @@ func TransformTable[S any, D any](
) (err errors.Error) {
db := basicRes.GetDal()
tmpTableName := fmt.Sprintf("%s_%s", tableName, hashScript(script))
-
+ db.Dialect()
Review Comment:
What is the point of adding this line?
##########
plugins/tapd/tasks/task_commit_collector.go:
##########
@@ -78,23 +52,23 @@ func CollectTaskCommits(taskCtx core.SubTaskContext) errors.Error {
if err != nil {
return err
}
- iterator, err := helper.NewDalCursorIterator(db, cursor, reflect.TypeOf(SimpleTask{}))
+ iterator, err := helper.NewDalCursorIterator(db, cursor, reflect.TypeOf(models.Input{}))
if err != nil {
return err
}
collector, err := helper.NewApiCollector(helper.ApiCollectorArgs{
RawDataSubTaskArgs: *rawDataSubTaskArgs,
- Incremental: incremental,
+ Incremental: since == nil,
Review Comment:
same
##########
plugins/tapd/tasks/bug_commit_collector.go:
##########
@@ -77,24 +52,24 @@ func CollectBugCommits(taskCtx core.SubTaskContext) errors.Error {
if err != nil {
return err
}
- iterator, err := helper.NewDalCursorIterator(db, cursor, reflect.TypeOf(SimpleBug{}))
+ iterator, err := helper.NewDalCursorIterator(db, cursor, reflect.TypeOf(models.Input{}))
if err != nil {
return err
}
collector, err := helper.NewApiCollector(helper.ApiCollectorArgs{
RawDataSubTaskArgs: *rawDataSubTaskArgs,
- Incremental: incremental,
+ Incremental: since == nil,
Review Comment:
since != nil ?
##########
plugins/tapd/tasks/story_status_collector.go:
##########
@@ -38,7 +38,6 @@ func CollectStoryStatus(taskCtx core.SubTaskContext) errors.Error {
collector, err := helper.NewApiCollector(helper.ApiCollectorArgs{
RawDataSubTaskArgs: *rawDataSubTaskArgs,
ApiClient: data.ApiClient,
- PageSize: 100,
Review Comment:
why?
##########
plugins/tapd/tasks/iteration_collector.go:
##########
@@ -65,6 +65,7 @@ func CollectIterations(taskCtx core.SubTaskContext) errors.Error {
Incremental: incremental,
ApiClient: data.ApiClient,
PageSize: 100,
+ Concurrency: 3,
Review Comment:
why need to set Concurrency?
##########
plugins/tapd/tasks/story_commit_collector.go:
##########
@@ -78,23 +52,23 @@ func CollectStoryCommits(taskCtx core.SubTaskContext) errors.Error {
if err != nil {
return err
}
- iterator, err := helper.NewDalCursorIterator(db, cursor, reflect.TypeOf(SimpleStory{}))
+ iterator, err := helper.NewDalCursorIterator(db, cursor, reflect.TypeOf(models.Input{}))
if err != nil {
return err
}
collector, err := helper.NewApiCollector(helper.ApiCollectorArgs{
RawDataSubTaskArgs: *rawDataSubTaskArgs,
- Incremental: incremental,
+ Incremental: since == nil,
Review Comment:
same
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@devlake.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [incubator-devlake] abeizn merged pull request #3711: fix(tapd): resolve issues impacted incremental collection
Posted by GitBox <gi...@apache.org>.
abeizn merged PR #3711:
URL: https://github.com/apache/incubator-devlake/pull/3711
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@devlake.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [incubator-devlake] warren830 commented on a diff in pull request #3711: fix(tapd): resolve issues impacted incremental collection
Posted by GitBox <gi...@apache.org>.
warren830 commented on code in PR #3711:
URL: https://github.com/apache/incubator-devlake/pull/3711#discussion_r1018709670
##########
helpers/migrationhelper/migrationhelper.go:
##########
@@ -183,7 +183,7 @@ func TransformTable[S any, D any](
) (err errors.Error) {
db := basicRes.GetDal()
tmpTableName := fmt.Sprintf("%s_%s", tableName, hashScript(script))
-
+ db.Dialect()
Review Comment:
fixed
##########
plugins/tapd/tasks/task_commit_collector.go:
##########
@@ -78,23 +52,23 @@ func CollectTaskCommits(taskCtx core.SubTaskContext) errors.Error {
if err != nil {
return err
}
- iterator, err := helper.NewDalCursorIterator(db, cursor, reflect.TypeOf(SimpleTask{}))
+ iterator, err := helper.NewDalCursorIterator(db, cursor, reflect.TypeOf(models.Input{}))
if err != nil {
return err
}
collector, err := helper.NewApiCollector(helper.ApiCollectorArgs{
RawDataSubTaskArgs: *rawDataSubTaskArgs,
- Incremental: incremental,
+ Incremental: since == nil,
Review Comment:
fixed
##########
plugins/tapd/tasks/story_commit_collector.go:
##########
@@ -78,23 +52,23 @@ func CollectStoryCommits(taskCtx core.SubTaskContext) errors.Error {
if err != nil {
return err
}
- iterator, err := helper.NewDalCursorIterator(db, cursor, reflect.TypeOf(SimpleStory{}))
+ iterator, err := helper.NewDalCursorIterator(db, cursor, reflect.TypeOf(models.Input{}))
if err != nil {
return err
}
collector, err := helper.NewApiCollector(helper.ApiCollectorArgs{
RawDataSubTaskArgs: *rawDataSubTaskArgs,
- Incremental: incremental,
+ Incremental: since == nil,
Review Comment:
fixed
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@devlake.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [incubator-devlake] warren830 commented on a diff in pull request #3711: fix(tapd): resolve issues impacted incremental collection
Posted by GitBox <gi...@apache.org>.
warren830 commented on code in PR #3711:
URL: https://github.com/apache/incubator-devlake/pull/3711#discussion_r1018708537
##########
plugins/tapd/tasks/iteration_collector.go:
##########
@@ -65,6 +65,7 @@ func CollectIterations(taskCtx core.SubTaskContext) errors.Error {
Incremental: incremental,
ApiClient: data.ApiClient,
PageSize: 100,
+ Concurrency: 3,
Review Comment:
Because we dont have too many iterations, so 3 is good enough, otherwise, we will send too many requests that will get nothing
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@devlake.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [incubator-devlake] warren830 commented on a diff in pull request #3711: fix(tapd): resolve issues impacted incremental collection
Posted by GitBox <gi...@apache.org>.
warren830 commented on code in PR #3711:
URL: https://github.com/apache/incubator-devlake/pull/3711#discussion_r1018708762
##########
plugins/tapd/tasks/story_status_collector.go:
##########
@@ -38,7 +38,6 @@ func CollectStoryStatus(taskCtx core.SubTaskContext) errors.Error {
collector, err := helper.NewApiCollector(helper.ApiCollectorArgs{
RawDataSubTaskArgs: *rawDataSubTaskArgs,
ApiClient: data.ApiClient,
- PageSize: 100,
Review Comment:
Because this api does not support limit
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@devlake.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [incubator-devlake] warren830 commented on a diff in pull request #3711: fix(tapd): resolve issues impacted incremental collection
Posted by GitBox <gi...@apache.org>.
warren830 commented on code in PR #3711:
URL: https://github.com/apache/incubator-devlake/pull/3711#discussion_r1018706763
##########
plugins/tapd/models/migrationscripts/20221109_add_init_tables.go:
##########
@@ -136,7 +136,7 @@ func (*addInitTables) Up(basicRes core.BasicRes) errors.Error {
}
func (*addInitTables) Version() uint64 {
- return 20221013201138
Review Comment:
Yes, this will also run even this had been run before
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@devlake.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [incubator-devlake] warren830 commented on a diff in pull request #3711: fix(tapd): resolve issues impacted incremental collection
Posted by GitBox <gi...@apache.org>.
warren830 commented on code in PR #3711:
URL: https://github.com/apache/incubator-devlake/pull/3711#discussion_r1018709791
##########
plugins/tapd/tasks/story_bug_collector.go:
##########
@@ -42,52 +38,33 @@ func CollectStoryBugs(taskCtx core.SubTaskContext) errors.Error {
logger := taskCtx.GetLogger()
logger.Info("collect storyBugs")
since := data.Since
- incremental := false
- if since == nil {
- // user didn't specify a time range to sync, try load from database
- var latestUpdated models.TapdStoryCommit
- clauses := []dal.Clause{
- dal.Where("connection_id = ? and workspace_id = ?", data.Options.ConnectionId, data.Options.WorkspaceId),
- dal.Orderby("created DESC"),
- }
- err := db.First(&latestUpdated, clauses...)
- if err != nil && !goerror.Is(err, gorm.ErrRecordNotFound) {
- return errors.NotFound.Wrap(err, "failed to get latest tapd changelog record")
- }
- if latestUpdated.Id > 0 {
- since = (*time.Time)(latestUpdated.Created)
- incremental = true
- }
- }
-
clauses := []dal.Clause{
- dal.Select("id"),
+ dal.Select("id as issue_id, modified as update_time"),
dal.From(&models.TapdStory{}),
- dal.Where("connection_id = ? and workspace_id = ?", data.Options.ConnectionId, data.Options.WorkspaceId),
+ dal.Where("_tool_tapd_stories.connection_id = ? and _tool_tapd_stories.workspace_id = ? ", data.Options.ConnectionId, data.Options.WorkspaceId),
}
if since != nil {
clauses = append(clauses, dal.Where("modified > ?", since))
}
-
cursor, err := db.Cursor(clauses...)
if err != nil {
return err
}
- iterator, err := helper.NewDalCursorIterator(db, cursor, reflect.TypeOf(SimpleStory{}))
+ iterator, err := helper.NewDalCursorIterator(db, cursor, reflect.TypeOf(models.Input{}))
if err != nil {
return err
}
collector, err := helper.NewApiCollector(helper.ApiCollectorArgs{
RawDataSubTaskArgs: *rawDataSubTaskArgs,
ApiClient: data.ApiClient,
- Incremental: incremental,
+ Incremental: since == nil,
Review Comment:
fixed
##########
plugins/tapd/tasks/bug_commit_collector.go:
##########
@@ -77,24 +52,24 @@ func CollectBugCommits(taskCtx core.SubTaskContext) errors.Error {
if err != nil {
return err
}
- iterator, err := helper.NewDalCursorIterator(db, cursor, reflect.TypeOf(SimpleBug{}))
+ iterator, err := helper.NewDalCursorIterator(db, cursor, reflect.TypeOf(models.Input{}))
if err != nil {
return err
}
collector, err := helper.NewApiCollector(helper.ApiCollectorArgs{
RawDataSubTaskArgs: *rawDataSubTaskArgs,
- Incremental: incremental,
+ Incremental: since == nil,
Review Comment:
fixed
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@devlake.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org