You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by kl...@apache.org on 2023/02/17 08:33:05 UTC
[incubator-devlake] branch main updated: refactor: rename createdDateAfter to timeAfter for github (#4441)
This is an automated email from the ASF dual-hosted git repository.
klesh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 1cc51a2a4 refactor: rename createdDateAfter to timeAfter for github (#4441)
1cc51a2a4 is described below
commit 1cc51a2a4c47cbabbca5b8b51061457cfe9e523c
Author: Klesh Wong <zh...@merico.dev>
AuthorDate: Fri Feb 17 16:33:00 2023 +0800
refactor: rename createdDateAfter to timeAfter for github (#4441)
---
backend/plugins/github/api/blueprint_v200.go | 11 +++++----
backend/plugins/github/github.go | 10 ++++----
backend/plugins/github/impl/impl.go | 21 ++++++++---------
backend/plugins/github/tasks/cicd_run_collector.go | 7 +++---
backend/plugins/github/tasks/comment_collector.go | 6 ++---
backend/plugins/github/tasks/commit_collector.go | 2 +-
backend/plugins/github/tasks/issue_collector.go | 2 +-
backend/plugins/github/tasks/pr_collector.go | 2 +-
.../plugins/github/tasks/pr_commit_collector.go | 5 ++--
.../plugins/github/tasks/pr_review_collector.go | 2 +-
.../github/tasks/pr_review_comment_collector.go | 6 ++---
backend/plugins/github/tasks/task_data.go | 10 ++++----
backend/plugins/github_graphql/impl/impl.go | 27 +++++++++++-----------
backend/plugins/github_graphql/plugin_main.go | 10 ++++----
.../github_graphql/tasks/check_run_collector.go | 13 ++++++-----
.../github_graphql/tasks/issue_collector.go | 7 +++---
.../plugins/github_graphql/tasks/pr_collector.go | 9 ++++----
17 files changed, 77 insertions(+), 73 deletions(-)
diff --git a/backend/plugins/github/api/blueprint_v200.go b/backend/plugins/github/api/blueprint_v200.go
index 6462f207b..cd8def7c8 100644
--- a/backend/plugins/github/api/blueprint_v200.go
+++ b/backend/plugins/github/api/blueprint_v200.go
@@ -19,6 +19,10 @@ package api
import (
"fmt"
+ "net/url"
+ "strings"
+ "time"
+
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models/domainlayer"
@@ -32,9 +36,6 @@ import (
"github.com/apache/incubator-devlake/plugins/github/models"
"github.com/apache/incubator-devlake/plugins/github/tasks"
"github.com/go-playground/validator/v10"
- "net/url"
- "strings"
- "time"
)
func MakeDataSourcePipelinePlanV200(subtaskMetas []plugin.SubTaskMeta, connectionId uint64, bpScopes []*plugin.BlueprintScopeV200, syncPolicy *plugin.BlueprintSyncPolicy) (plugin.PipelinePlan, []plugin.Scope, errors.Error) {
@@ -109,8 +110,8 @@ func makeDataSourcePipelinePlanV200(
GithubId: githubRepo.GithubId,
Name: githubRepo.Name,
}
- if syncPolicy.CreatedDateAfter != nil {
- op.CreatedDateAfter = syncPolicy.CreatedDateAfter.Format(time.RFC3339)
+ if syncPolicy.TimeAfter != nil {
+ op.TimeAfter = syncPolicy.TimeAfter.Format(time.RFC3339)
}
options, err := tasks.EncodeTaskOptions(op)
if err != nil {
diff --git a/backend/plugins/github/github.go b/backend/plugins/github/github.go
index eed632320..947d303fb 100644
--- a/backend/plugins/github/github.go
+++ b/backend/plugins/github/github.go
@@ -32,7 +32,7 @@ func main() {
connectionId := cmd.Flags().Uint64P("connectionId", "c", 0, "github connection id")
owner := cmd.Flags().StringP("owner", "o", "", "github owner")
repo := cmd.Flags().StringP("repo", "r", "", "github repo")
- createdDateAfter := cmd.Flags().StringP("createdDateAfter", "a", "", "collect data that are created after specified time, ie 2006-05-06T07:08:09Z")
+ timeAfter := cmd.Flags().StringP("timeAfter", "a", "", "collect data that are created after specified time, ie 2006-05-06T07:08:09Z")
_ = cmd.MarkFlagRequired("connectionId")
_ = cmd.MarkFlagRequired("owner")
_ = cmd.MarkFlagRequired("repo")
@@ -50,10 +50,10 @@ func main() {
cmd.Run = func(cmd *cobra.Command, args []string) {
runner.DirectRun(cmd, args, PluginEntry, map[string]interface{}{
- "connectionId": *connectionId,
- "owner": *owner,
- "repo": *repo,
- "createdDateAfter": *createdDateAfter,
+ "connectionId": *connectionId,
+ "owner": *owner,
+ "repo": *repo,
+ "timeAfter": *timeAfter,
"transformationRules": map[string]interface{}{
"prType": *prType,
"prComponent": *prComponent,
diff --git a/backend/plugins/github/impl/impl.go b/backend/plugins/github/impl/impl.go
index 9c603c81b..756e73097 100644
--- a/backend/plugins/github/impl/impl.go
+++ b/backend/plugins/github/impl/impl.go
@@ -19,6 +19,8 @@ package impl
import (
"fmt"
+ "time"
+
"github.com/apache/incubator-devlake/core/context"
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
@@ -28,7 +30,6 @@ import (
"github.com/apache/incubator-devlake/plugins/github/models"
"github.com/apache/incubator-devlake/plugins/github/models/migrationscripts"
"github.com/apache/incubator-devlake/plugins/github/tasks"
- "time"
)
var _ plugin.PluginMeta = (*Github)(nil)
@@ -164,21 +165,19 @@ func (p Github) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]i
return nil, err
}
- var createdDateAfter time.Time
- if op.CreatedDateAfter != "" {
- createdDateAfter, err = errors.Convert01(time.Parse(time.RFC3339, op.CreatedDateAfter))
- if err != nil {
- return nil, errors.BadInput.Wrap(err, "invalid value for `createdDateAfter`")
- }
- }
taskData := &tasks.GithubTaskData{
Options: op,
ApiClient: apiClient,
}
- if !createdDateAfter.IsZero() {
- taskData.CreatedDateAfter = &createdDateAfter
- logger.Debug("collect data updated createdDateAfter %s", createdDateAfter)
+ if op.TimeAfter != "" {
+ var timeAfter time.Time
+ timeAfter, err = errors.Convert01(time.Parse(time.RFC3339, op.TimeAfter))
+ if err != nil {
+ return nil, errors.BadInput.Wrap(err, "invalid value for `timeAfter`")
+ }
+ taskData.TimeAfter = &timeAfter
+ logger.Debug("collect data updated timeAfter %s", timeAfter)
}
return taskData, nil
}
diff --git a/backend/plugins/github/tasks/cicd_run_collector.go b/backend/plugins/github/tasks/cicd_run_collector.go
index b3ccffe7f..5990f5231 100644
--- a/backend/plugins/github/tasks/cicd_run_collector.go
+++ b/backend/plugins/github/tasks/cicd_run_collector.go
@@ -20,11 +20,12 @@ package tasks
import (
"encoding/json"
"fmt"
+ "net/http"
+ "net/url"
+
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
- "net/http"
- "net/url"
)
const RAW_RUN_TABLE = "github_api_runs"
@@ -46,7 +47,7 @@ func CollectRuns(taskCtx plugin.SubTaskContext) errors.Error {
Name: data.Options.Name,
},
Table: RAW_RUN_TABLE,
- }, data.CreatedDateAfter)
+ }, data.TimeAfter)
if err != nil {
return err
}
diff --git a/backend/plugins/github/tasks/comment_collector.go b/backend/plugins/github/tasks/comment_collector.go
index 54f528299..35dd29042 100644
--- a/backend/plugins/github/tasks/comment_collector.go
+++ b/backend/plugins/github/tasks/comment_collector.go
@@ -39,7 +39,7 @@ func CollectApiComments(taskCtx plugin.SubTaskContext) errors.Error {
Name: data.Options.Name,
},
Table: RAW_COMMENTS_TABLE,
- }, data.CreatedDateAfter)
+ }, data.TimeAfter)
if err != nil {
return err
}
@@ -55,11 +55,11 @@ func CollectApiComments(taskCtx plugin.SubTaskContext) errors.Error {
query := url.Values{}
query.Set("state", "all")
// if data.CreatedDateAfter != nil, we set since once
- if data.CreatedDateAfter != nil {
+ if data.TimeAfter != nil {
// Note that `since` is for filtering records by the `updated` time
// which is not ideal for semantic reasons and would result in slightly more records than expected.
// But we have no choice since it is the only available field we could exploit from the API.
- query.Set("since", data.CreatedDateAfter.String())
+ query.Set("since", data.TimeAfter.String())
}
// if incremental == true, we overwrite it
if incremental {
diff --git a/backend/plugins/github/tasks/commit_collector.go b/backend/plugins/github/tasks/commit_collector.go
index 1bb40a5de..17be5f014 100644
--- a/backend/plugins/github/tasks/commit_collector.go
+++ b/backend/plugins/github/tasks/commit_collector.go
@@ -47,7 +47,7 @@ func CollectApiCommits(taskCtx plugin.SubTaskContext) errors.Error {
Name: data.Options.Name,
},
Table: RAW_COMMIT_TABLE,
- }, data.CreatedDateAfter)
+ }, data.TimeAfter)
if err != nil {
return err
}
diff --git a/backend/plugins/github/tasks/issue_collector.go b/backend/plugins/github/tasks/issue_collector.go
index 2c3f1e0d2..d70b11967 100644
--- a/backend/plugins/github/tasks/issue_collector.go
+++ b/backend/plugins/github/tasks/issue_collector.go
@@ -47,7 +47,7 @@ func CollectApiIssues(taskCtx plugin.SubTaskContext) errors.Error {
Name: data.Options.Name,
},
Table: RAW_ISSUE_TABLE,
- }, data.CreatedDateAfter)
+ }, data.TimeAfter)
if err != nil {
return err
}
diff --git a/backend/plugins/github/tasks/pr_collector.go b/backend/plugins/github/tasks/pr_collector.go
index 8064032b1..50cde330c 100644
--- a/backend/plugins/github/tasks/pr_collector.go
+++ b/backend/plugins/github/tasks/pr_collector.go
@@ -47,7 +47,7 @@ func CollectApiPullRequests(taskCtx plugin.SubTaskContext) errors.Error {
Name: data.Options.Name,
},
Table: RAW_PULL_REQUEST_TABLE,
- }, data.CreatedDateAfter)
+ }, data.TimeAfter)
if err != nil {
return err
}
diff --git a/backend/plugins/github/tasks/pr_commit_collector.go b/backend/plugins/github/tasks/pr_commit_collector.go
index 208e22a3c..29da846fb 100644
--- a/backend/plugins/github/tasks/pr_commit_collector.go
+++ b/backend/plugins/github/tasks/pr_commit_collector.go
@@ -20,11 +20,12 @@ package tasks
import (
"encoding/json"
"fmt"
- "github.com/apache/incubator-devlake/core/errors"
"net/http"
"net/url"
"reflect"
+ "github.com/apache/incubator-devlake/core/errors"
+
"github.com/apache/incubator-devlake/core/dal"
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
@@ -61,7 +62,7 @@ func CollectApiPullRequestCommits(taskCtx plugin.SubTaskContext) errors.Error {
Name: data.Options.Name,
},
Table: RAW_PR_COMMIT_TABLE,
- }, data.CreatedDateAfter)
+ }, data.TimeAfter)
if err != nil {
return err
}
diff --git a/backend/plugins/github/tasks/pr_review_collector.go b/backend/plugins/github/tasks/pr_review_collector.go
index f78118fa7..958ab4dfb 100644
--- a/backend/plugins/github/tasks/pr_review_collector.go
+++ b/backend/plugins/github/tasks/pr_review_collector.go
@@ -54,7 +54,7 @@ func CollectApiPullRequestReviews(taskCtx plugin.SubTaskContext) errors.Error {
Name: data.Options.Name,
},
Table: RAW_PR_REVIEW_TABLE,
- }, data.CreatedDateAfter)
+ }, data.TimeAfter)
if err != nil {
return err
}
diff --git a/backend/plugins/github/tasks/pr_review_comment_collector.go b/backend/plugins/github/tasks/pr_review_comment_collector.go
index da5c3bbf9..c8c02869c 100644
--- a/backend/plugins/github/tasks/pr_review_comment_collector.go
+++ b/backend/plugins/github/tasks/pr_review_comment_collector.go
@@ -42,7 +42,7 @@ func CollectPrReviewComments(taskCtx plugin.SubTaskContext) errors.Error {
Name: data.Options.Name,
},
Table: RAW_PR_REVIEW_COMMENTS_TABLE,
- }, data.CreatedDateAfter)
+ }, data.TimeAfter)
if err != nil {
return err
}
@@ -57,11 +57,11 @@ func CollectPrReviewComments(taskCtx plugin.SubTaskContext) errors.Error {
Query: func(reqData *helper.RequestData) (url.Values, errors.Error) {
query := url.Values{}
// if data.CreatedDateAfter != nil, we set since once
- if data.CreatedDateAfter != nil {
+ if data.TimeAfter != nil {
// Note that `since` is for filtering records by the `updated` time
// which is not ideal for semantic reasons and would result in slightly more records than expected.
// But we have no choice since it is the only available field we could exploit from the API.
- query.Set("since", data.CreatedDateAfter.String())
+ query.Set("since", data.TimeAfter.String())
}
// if incremental == true, we overwrite it
if incremental {
diff --git a/backend/plugins/github/tasks/task_data.go b/backend/plugins/github/tasks/task_data.go
index 5a7d612ac..419cf91c4 100644
--- a/backend/plugins/github/tasks/task_data.go
+++ b/backend/plugins/github/tasks/task_data.go
@@ -33,7 +33,7 @@ type GithubOptions struct {
TransformationRuleId uint64 `json:"transformationRuleId" mapstructure:"transformationRuleId,omitempty"`
GithubId int `json:"githubId" mapstructure:"githubId,omitempty"`
Tasks []string `json:"tasks,omitempty" mapstructure:",omitempty"`
- CreatedDateAfter string `json:"createdDateAfter" mapstructure:"createdDateAfter,omitempty"`
+ TimeAfter string `json:"timeAfter" mapstructure:"timeAfter,omitempty"`
Owner string `json:"owner" mapstructure:"owner,omitempty"`
Repo string `json:"repo" mapstructure:"repo,omitempty"`
Name string `json:"name" mapstructure:"name,omitempty"`
@@ -41,10 +41,10 @@ type GithubOptions struct {
}
type GithubTaskData struct {
- Options *GithubOptions
- ApiClient *helper.ApiAsyncClient
- GraphqlClient *helper.GraphqlAsyncClient
- CreatedDateAfter *time.Time
+ Options *GithubOptions
+ ApiClient *helper.ApiAsyncClient
+ GraphqlClient *helper.GraphqlAsyncClient
+ TimeAfter *time.Time
}
type GithubApiParams struct {
diff --git a/backend/plugins/github_graphql/impl/impl.go b/backend/plugins/github_graphql/impl/impl.go
index 0e3200f8a..abc675989 100644
--- a/backend/plugins/github_graphql/impl/impl.go
+++ b/backend/plugins/github_graphql/impl/impl.go
@@ -20,6 +20,11 @@ package impl
import (
"context"
"fmt"
+ "net/url"
+ "reflect"
+ "strings"
+ "time"
+
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/log"
@@ -31,10 +36,6 @@ import (
"github.com/apache/incubator-devlake/plugins/github_graphql/tasks"
"github.com/merico-dev/graphql"
"golang.org/x/oauth2"
- "net/url"
- "reflect"
- "strings"
- "time"
)
// make sure interface is implemented
@@ -150,13 +151,6 @@ func (p GithubGraphql) PrepareTaskData(taskCtx plugin.TaskContext, options map[s
if err != nil {
return nil, err
}
- var createdDateAfter time.Time
- if op.CreatedDateAfter != "" {
- createdDateAfter, err = errors.Convert01(time.Parse(time.RFC3339, op.CreatedDateAfter))
- if err != nil {
- return nil, errors.BadInput.Wrap(err, "invalid value for `createdDateAfter`")
- }
- }
tokens := strings.Split(connection.Token, ",")
src := oauth2.StaticTokenSource(
@@ -196,9 +190,14 @@ func (p GithubGraphql) PrepareTaskData(taskCtx plugin.TaskContext, options map[s
ApiClient: apiClient,
GraphqlClient: graphqlClient,
}
- if !createdDateAfter.IsZero() {
- taskData.CreatedDateAfter = &createdDateAfter
- logger.Debug("collect data updated createdDateAfter %s", createdDateAfter)
+ if op.TimeAfter != "" {
+ var timeAfter time.Time
+ timeAfter, err = errors.Convert01(time.Parse(time.RFC3339, op.TimeAfter))
+ if err != nil {
+ return nil, errors.BadInput.Wrap(err, "invalid value for `timeAfter`")
+ }
+ taskData.TimeAfter = &timeAfter
+ logger.Debug("collect data updated timeAfter %s", timeAfter)
}
return taskData, nil
diff --git a/backend/plugins/github_graphql/plugin_main.go b/backend/plugins/github_graphql/plugin_main.go
index 8f1bbaa67..e9edf721b 100644
--- a/backend/plugins/github_graphql/plugin_main.go
+++ b/backend/plugins/github_graphql/plugin_main.go
@@ -32,17 +32,17 @@ func main() {
connectionId := cmd.Flags().Uint64P("connectionId", "c", 0, "github connection id")
owner := cmd.Flags().StringP("owner", "o", "", "github owner")
repo := cmd.Flags().StringP("repo", "r", "", "github repo")
- CreatedDateAfter := cmd.Flags().StringP("createdDateAfter", "a", "", "collect data that are created after specified time, ie 2006-05-06T07:08:09Z")
+ timeAfter := cmd.Flags().StringP("timeAfter", "a", "", "collect data that are updated/created after specified time, ie 2006-05-06T07:08:09Z")
_ = cmd.MarkFlagRequired("connectionId")
_ = cmd.MarkFlagRequired("owner")
_ = cmd.MarkFlagRequired("repo")
cmd.Run = func(cmd *cobra.Command, args []string) {
runner.DirectRun(cmd, args, PluginEntry, map[string]interface{}{
- "connectionId": *connectionId,
- "owner": *owner,
- "repo": *repo,
- "createdDateAfter": *CreatedDateAfter,
+ "connectionId": *connectionId,
+ "owner": *owner,
+ "repo": *repo,
+ "timeAfter": *timeAfter,
})
}
runner.RunCmd(cmd)
diff --git a/backend/plugins/github_graphql/tasks/check_run_collector.go b/backend/plugins/github_graphql/tasks/check_run_collector.go
index b90496c9c..e21db4d63 100644
--- a/backend/plugins/github_graphql/tasks/check_run_collector.go
+++ b/backend/plugins/github_graphql/tasks/check_run_collector.go
@@ -19,6 +19,10 @@ package tasks
import (
"encoding/json"
+ "reflect"
+ "strings"
+ "time"
+
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
@@ -26,9 +30,6 @@ import (
"github.com/apache/incubator-devlake/plugins/github/models"
githubTasks "github.com/apache/incubator-devlake/plugins/github/tasks"
"github.com/merico-dev/graphql"
- "reflect"
- "strings"
- "time"
)
const RAW_CHECK_RUNS_TABLE = "github_graphql_check_runs"
@@ -107,7 +108,7 @@ func CollectCheckRun(taskCtx plugin.SubTaskContext) errors.Error {
Name: data.Options.Name,
},
Table: RAW_CHECK_RUNS_TABLE,
- }, data.CreatedDateAfter)
+ }, data.TimeAfter)
if err != nil {
return err
}
@@ -120,8 +121,8 @@ func CollectCheckRun(taskCtx plugin.SubTaskContext) errors.Error {
dal.Where("repo_id = ? and connection_id=?", data.Options.GithubId, data.Options.ConnectionId),
dal.Orderby("github_updated_at DESC"),
}
- if collectorWithState.CreatedDateAfter != nil {
- clauses = append(clauses, dal.Where("github_created_at > ?", *collectorWithState.CreatedDateAfter))
+ if collectorWithState.TimeAfter != nil {
+ clauses = append(clauses, dal.Where("github_created_at > ?", *collectorWithState.TimeAfter))
}
if incremental {
clauses = append(clauses, dal.Where("github_updated_at > ?", *collectorWithState.LatestState.LatestSuccessStart))
diff --git a/backend/plugins/github_graphql/tasks/issue_collector.go b/backend/plugins/github_graphql/tasks/issue_collector.go
index acf0b5071..7fac6690e 100644
--- a/backend/plugins/github_graphql/tasks/issue_collector.go
+++ b/backend/plugins/github_graphql/tasks/issue_collector.go
@@ -18,6 +18,9 @@ limitations under the License.
package tasks
import (
+ "strings"
+ "time"
+
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models/domainlayer/ticket"
@@ -26,8 +29,6 @@ import (
"github.com/apache/incubator-devlake/plugins/github/models"
githubTasks "github.com/apache/incubator-devlake/plugins/github/tasks"
"github.com/merico-dev/graphql"
- "strings"
- "time"
)
const RAW_ISSUES_TABLE = "github_graphql_issues"
@@ -129,7 +130,7 @@ func CollectIssue(taskCtx plugin.SubTaskContext) errors.Error {
results := make([]interface{}, 0, 1)
isFinish := false
for _, issue := range issues {
- if data.CreatedDateAfter != nil && !data.CreatedDateAfter.Before(issue.CreatedAt) {
+ if data.TimeAfter != nil && !data.TimeAfter.Before(issue.CreatedAt) {
isFinish = true
break
}
diff --git a/backend/plugins/github_graphql/tasks/pr_collector.go b/backend/plugins/github_graphql/tasks/pr_collector.go
index f18fd8aa5..b4b67d81d 100644
--- a/backend/plugins/github_graphql/tasks/pr_collector.go
+++ b/backend/plugins/github_graphql/tasks/pr_collector.go
@@ -18,15 +18,16 @@ limitations under the License.
package tasks
import (
+ "regexp"
+ "strings"
+ "time"
+
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/github/models"
"github.com/apache/incubator-devlake/plugins/github/tasks"
"github.com/merico-dev/graphql"
- "regexp"
- "strings"
- "time"
)
const RAW_PRS_TABLE = "github_graphql_prs"
@@ -179,7 +180,7 @@ func CollectPr(taskCtx plugin.SubTaskContext) errors.Error {
results := make([]interface{}, 0, 1)
isFinish := false
for _, rawL := range prs {
- if data.CreatedDateAfter != nil && !data.CreatedDateAfter.Before(rawL.CreatedAt) {
+ if data.TimeAfter != nil && !data.TimeAfter.Before(rawL.CreatedAt) {
isFinish = true
break
}