You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by GitBox <gi...@apache.org> on 2022/07/05 08:09:05 UTC
[GitHub] [incubator-devlake] likyh opened a new pull request, #2416: feat: [POC] add since in framework
likyh opened a new pull request, #2416:
URL: https://github.com/apache/incubator-devlake/pull/2416
# Summary
That's POC for simplifying plugin creating.
What's New:
1. manage `since` in framework
2.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@devlake.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [incubator-devlake] keon94 commented on a diff in pull request #2416: feat: [POC] add since in framework
Posted by GitBox <gi...@apache.org>.
keon94 commented on code in PR #2416:
URL: https://github.com/apache/incubator-devlake/pull/2416#discussion_r929259972
##########
plugins/github/tasks/issue_extractor.go:
##########
@@ -59,170 +45,6 @@ type IssuesResponse struct {
GithubUpdatedAt helper.Iso8601Time `json:"updated_at"`
}
-func ExtractApiIssues(taskCtx core.SubTaskContext) error {
- data := taskCtx.GetData().(*GithubTaskData)
- config := data.Options.TransformationRules
- var issueSeverityRegex *regexp.Regexp
- var issueComponentRegex *regexp.Regexp
- var issuePriorityRegex *regexp.Regexp
- var issueTypeBugRegex *regexp.Regexp
- var issueTypeRequirementRegex *regexp.Regexp
- var issueTypeIncidentRegex *regexp.Regexp
- var issueSeverity = config.IssueSeverity
- var err error
- if len(issueSeverity) > 0 {
- issueSeverityRegex, err = regexp.Compile(issueSeverity)
- if err != nil {
- return fmt.Errorf("regexp Compile issueSeverity failed:[%s] stack:[%s]", err.Error(), debug.Stack())
- }
- }
- var issueComponent = config.IssueComponent
- if len(issueComponent) > 0 {
- issueComponentRegex, err = regexp.Compile(issueComponent)
- if err != nil {
- return fmt.Errorf("regexp Compile issueComponent failed:[%s] stack:[%s]", err.Error(), debug.Stack())
- }
- }
- var issuePriority = config.IssuePriority
- if len(issuePriority) > 0 {
- issuePriorityRegex, err = regexp.Compile(issuePriority)
- if err != nil {
- return fmt.Errorf("regexp Compile issuePriority failed:[%s] stack:[%s]", err.Error(), debug.Stack())
- }
- }
- var issueTypeBug = config.IssueTypeBug
- if len(issueTypeBug) > 0 {
- issueTypeBugRegex, err = regexp.Compile(issueTypeBug)
- if err != nil {
- return fmt.Errorf("regexp Compile issueTypeBug failed:[%s] stack:[%s]", err.Error(), debug.Stack())
- }
- }
- var issueTypeRequirement = config.IssueTypeRequirement
- if len(issueTypeRequirement) > 0 {
- issueTypeRequirementRegex, err = regexp.Compile(issueTypeRequirement)
- if err != nil {
- return fmt.Errorf("regexp Compile issueTypeRequirement failed:[%s] stack:[%s]", err.Error(), debug.Stack())
- }
- }
- var issueTypeIncident = config.IssueTypeIncident
- if len(issueTypeIncident) > 0 {
- issueTypeIncidentRegex, err = regexp.Compile(issueTypeIncident)
- if err != nil {
- return fmt.Errorf("regexp Compile issueTypeIncident failed:[%s] stack:[%s]", err.Error(), debug.Stack())
- }
- }
-
- extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{
- RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
- Ctx: taskCtx,
- /*
- This struct will be JSONEncoded and stored into database along with raw data itself, to identity minimal
- set of data to be process, for example, we process JiraIssues by Board
- */
- Params: GithubApiParams{
- ConnectionId: data.Options.ConnectionId,
- Owner: data.Options.Owner,
- Repo: data.Options.Repo,
- },
- /*
- Table store raw data
- */
- Table: RAW_ISSUE_TABLE,
- },
- Extract: func(row *helper.RawData) ([]interface{}, error) {
- body := &IssuesResponse{}
- err := json.Unmarshal(row.Data, body)
- if err != nil {
- return nil, err
- }
- // need to extract 2 kinds of entities here
- if body.GithubId == 0 {
- return nil, nil
- }
- //If this is a pr, ignore
- if body.PullRequest.Url != "" {
- return nil, nil
- }
- results := make([]interface{}, 0, 2)
- githubIssue, err := convertGithubIssue(body, data.Options.ConnectionId, data.Repo.GithubId)
- if err != nil {
- return nil, err
- }
- if body.Assignee != nil {
- githubIssue.AssigneeId = body.Assignee.Id
- githubIssue.AssigneeName = body.Assignee.Login
- relatedUser, err := convertUser(body.Assignee, data.Options.ConnectionId)
- if err != nil {
- return nil, err
- }
- results = append(results, relatedUser)
- }
- if body.User != nil {
- githubIssue.AuthorId = body.User.Id
- githubIssue.AuthorName = body.User.Login
- relatedUser, err := convertUser(body.User, data.Options.ConnectionId)
- if err != nil {
- return nil, err
- }
- results = append(results, relatedUser)
- }
- for _, label := range body.Labels {
- results = append(results, &models.GithubIssueLabel{
- ConnectionId: data.Options.ConnectionId,
- IssueId: githubIssue.GithubId,
- LabelName: label.Name,
- })
- if issueSeverityRegex != nil {
- groups := issueSeverityRegex.FindStringSubmatch(label.Name)
- if len(groups) > 0 {
- githubIssue.Severity = groups[1]
- }
- }
-
- if issueComponentRegex != nil {
- groups := issueComponentRegex.FindStringSubmatch(label.Name)
- if len(groups) > 0 {
- githubIssue.Component = groups[1]
- }
- }
-
- if issuePriorityRegex != nil {
- groups := issuePriorityRegex.FindStringSubmatch(label.Name)
- if len(groups) > 0 {
- githubIssue.Priority = groups[1]
- }
- }
-
- if issueTypeBugRegex != nil {
- if ok := issueTypeBugRegex.MatchString(label.Name); ok {
- githubIssue.Type = ticket.BUG
- }
- }
-
- if issueTypeRequirementRegex != nil {
- if ok := issueTypeRequirementRegex.MatchString(label.Name); ok {
- githubIssue.Type = ticket.REQUIREMENT
- }
- }
-
- if issueTypeIncidentRegex != nil {
- if ok := issueTypeIncidentRegex.MatchString(label.Name); ok {
- githubIssue.Type = ticket.INCIDENT
- }
- }
- }
- results = append(results, githubIssue)
-
- return results, nil
- },
- })
-
- if err != nil {
- return err
- }
-
- return extractor.Execute()
-}
func convertGithubIssue(issue *IssuesResponse, connectionId uint64, repositoryId int) (*models.GithubIssue, error) {
Review Comment:
might as well move this helper function to issuer_collector.go too and delete this file
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@devlake.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [incubator-devlake] likyh commented on a diff in pull request #2416: feat: [POC] add since in framework
Posted by GitBox <gi...@apache.org>.
likyh commented on code in PR #2416:
URL: https://github.com/apache/incubator-devlake/pull/2416#discussion_r931876014
##########
plugins/helper/api_collector.go:
##########
@@ -75,6 +77,8 @@ type ApiCollectorArgs struct {
Concurrency int
ResponseParser func(res *http.Response) ([]json.RawMessage, error)
AfterResponse common.ApiClientAfterResponse
+
+ Extract func(row *RawData) (models []interface{}, latestUpdated *time.Time, err error)
Review Comment:
Good idea~
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@devlake.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [incubator-devlake] likyh commented on a diff in pull request #2416: feat: [POC] add since in framework
Posted by GitBox <gi...@apache.org>.
likyh commented on code in PR #2416:
URL: https://github.com/apache/incubator-devlake/pull/2416#discussion_r931875378
##########
plugins/github/tasks/issue_extractor.go:
##########
@@ -59,170 +45,6 @@ type IssuesResponse struct {
GithubUpdatedAt helper.Iso8601Time `json:"updated_at"`
}
-func ExtractApiIssues(taskCtx core.SubTaskContext) error {
- data := taskCtx.GetData().(*GithubTaskData)
- config := data.Options.TransformationRules
- var issueSeverityRegex *regexp.Regexp
- var issueComponentRegex *regexp.Regexp
- var issuePriorityRegex *regexp.Regexp
- var issueTypeBugRegex *regexp.Regexp
- var issueTypeRequirementRegex *regexp.Regexp
- var issueTypeIncidentRegex *regexp.Regexp
- var issueSeverity = config.IssueSeverity
- var err error
- if len(issueSeverity) > 0 {
- issueSeverityRegex, err = regexp.Compile(issueSeverity)
- if err != nil {
- return fmt.Errorf("regexp Compile issueSeverity failed:[%s] stack:[%s]", err.Error(), debug.Stack())
- }
- }
- var issueComponent = config.IssueComponent
- if len(issueComponent) > 0 {
- issueComponentRegex, err = regexp.Compile(issueComponent)
- if err != nil {
- return fmt.Errorf("regexp Compile issueComponent failed:[%s] stack:[%s]", err.Error(), debug.Stack())
- }
- }
- var issuePriority = config.IssuePriority
- if len(issuePriority) > 0 {
- issuePriorityRegex, err = regexp.Compile(issuePriority)
- if err != nil {
- return fmt.Errorf("regexp Compile issuePriority failed:[%s] stack:[%s]", err.Error(), debug.Stack())
- }
- }
- var issueTypeBug = config.IssueTypeBug
- if len(issueTypeBug) > 0 {
- issueTypeBugRegex, err = regexp.Compile(issueTypeBug)
- if err != nil {
- return fmt.Errorf("regexp Compile issueTypeBug failed:[%s] stack:[%s]", err.Error(), debug.Stack())
- }
- }
- var issueTypeRequirement = config.IssueTypeRequirement
- if len(issueTypeRequirement) > 0 {
- issueTypeRequirementRegex, err = regexp.Compile(issueTypeRequirement)
- if err != nil {
- return fmt.Errorf("regexp Compile issueTypeRequirement failed:[%s] stack:[%s]", err.Error(), debug.Stack())
- }
- }
- var issueTypeIncident = config.IssueTypeIncident
- if len(issueTypeIncident) > 0 {
- issueTypeIncidentRegex, err = regexp.Compile(issueTypeIncident)
- if err != nil {
- return fmt.Errorf("regexp Compile issueTypeIncident failed:[%s] stack:[%s]", err.Error(), debug.Stack())
- }
- }
-
- extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{
- RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
- Ctx: taskCtx,
- /*
- This struct will be JSONEncoded and stored into database along with raw data itself, to identity minimal
- set of data to be process, for example, we process JiraIssues by Board
- */
- Params: GithubApiParams{
- ConnectionId: data.Options.ConnectionId,
- Owner: data.Options.Owner,
- Repo: data.Options.Repo,
- },
- /*
- Table store raw data
- */
- Table: RAW_ISSUE_TABLE,
- },
- Extract: func(row *helper.RawData) ([]interface{}, error) {
- body := &IssuesResponse{}
- err := json.Unmarshal(row.Data, body)
- if err != nil {
- return nil, err
- }
- // need to extract 2 kinds of entities here
- if body.GithubId == 0 {
- return nil, nil
- }
- //If this is a pr, ignore
- if body.PullRequest.Url != "" {
- return nil, nil
- }
- results := make([]interface{}, 0, 2)
- githubIssue, err := convertGithubIssue(body, data.Options.ConnectionId, data.Repo.GithubId)
- if err != nil {
- return nil, err
- }
- if body.Assignee != nil {
- githubIssue.AssigneeId = body.Assignee.Id
- githubIssue.AssigneeName = body.Assignee.Login
- relatedUser, err := convertUser(body.Assignee, data.Options.ConnectionId)
- if err != nil {
- return nil, err
- }
- results = append(results, relatedUser)
- }
- if body.User != nil {
- githubIssue.AuthorId = body.User.Id
- githubIssue.AuthorName = body.User.Login
- relatedUser, err := convertUser(body.User, data.Options.ConnectionId)
- if err != nil {
- return nil, err
- }
- results = append(results, relatedUser)
- }
- for _, label := range body.Labels {
- results = append(results, &models.GithubIssueLabel{
- ConnectionId: data.Options.ConnectionId,
- IssueId: githubIssue.GithubId,
- LabelName: label.Name,
- })
- if issueSeverityRegex != nil {
- groups := issueSeverityRegex.FindStringSubmatch(label.Name)
- if len(groups) > 0 {
- githubIssue.Severity = groups[1]
- }
- }
-
- if issueComponentRegex != nil {
- groups := issueComponentRegex.FindStringSubmatch(label.Name)
- if len(groups) > 0 {
- githubIssue.Component = groups[1]
- }
- }
-
- if issuePriorityRegex != nil {
- groups := issuePriorityRegex.FindStringSubmatch(label.Name)
- if len(groups) > 0 {
- githubIssue.Priority = groups[1]
- }
- }
-
- if issueTypeBugRegex != nil {
- if ok := issueTypeBugRegex.MatchString(label.Name); ok {
- githubIssue.Type = ticket.BUG
- }
- }
-
- if issueTypeRequirementRegex != nil {
- if ok := issueTypeRequirementRegex.MatchString(label.Name); ok {
- githubIssue.Type = ticket.REQUIREMENT
- }
- }
-
- if issueTypeIncidentRegex != nil {
- if ok := issueTypeIncidentRegex.MatchString(label.Name); ok {
- githubIssue.Type = ticket.INCIDENT
- }
- }
- }
- results = append(results, githubIssue)
-
- return results, nil
- },
- })
-
- if err != nil {
- return err
- }
-
- return extractor.Execute()
-}
func convertGithubIssue(issue *IssuesResponse, connectionId uint64, repositoryId int) (*models.GithubIssue, error) {
Review Comment:
yeah~ you are right.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@devlake.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [incubator-devlake] likyh commented on a diff in pull request #2416: feat: [POC] add since in framework
Posted by GitBox <gi...@apache.org>.
likyh commented on code in PR #2416:
URL: https://github.com/apache/incubator-devlake/pull/2416#discussion_r913555653
##########
plugins/github/tasks/issue_extractor.go:
##########
@@ -213,7 +214,7 @@ func ExtractApiIssues(taskCtx core.SubTaskContext) error {
}
results = append(results, githubIssue)
- return results, nil
+ return results, &githubIssue.GithubUpdatedAt, nil
Review Comment:
Maybe we can create a new meta table to save it.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@devlake.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [incubator-devlake] likyh closed pull request #2416: feat: [POC] add since in framework
Posted by GitBox <gi...@apache.org>.
likyh closed pull request #2416: feat: [POC] add since in framework
URL: https://github.com/apache/incubator-devlake/pull/2416
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@devlake.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [incubator-devlake] warren830 commented on a diff in pull request #2416: feat: [POC] add since in framework
Posted by GitBox <gi...@apache.org>.
warren830 commented on code in PR #2416:
URL: https://github.com/apache/incubator-devlake/pull/2416#discussion_r913548286
##########
plugins/github/tasks/issue_extractor.go:
##########
@@ -213,7 +214,7 @@ func ExtractApiIssues(taskCtx core.SubTaskContext) error {
}
results = append(results, githubIssue)
- return results, nil
+ return results, &githubIssue.GithubUpdatedAt, nil
Review Comment:
What to do next after returning this `&githubIssue.GithubUpdatedAt`?
Do we save this to DB or just use a struct to store it in the memory?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@devlake.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [incubator-devlake] keon94 commented on pull request #2416: feat: [POC] add since in framework
Posted by GitBox <gi...@apache.org>.
keon94 commented on PR #2416:
URL: https://github.com/apache/incubator-devlake/pull/2416#issuecomment-1194570943
If we're going to merge collectors and extractors, is there a need for raw tables anymore?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@devlake.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [incubator-devlake] keon94 commented on a diff in pull request #2416: feat: [POC] add since in framework
Posted by GitBox <gi...@apache.org>.
keon94 commented on code in PR #2416:
URL: https://github.com/apache/incubator-devlake/pull/2416#discussion_r929267284
##########
plugins/helper/api_collector.go:
##########
@@ -75,6 +77,8 @@ type ApiCollectorArgs struct {
Concurrency int
ResponseParser func(res *http.Response) ([]json.RawMessage, error)
AfterResponse common.ApiClientAfterResponse
+
+ Extract func(row *RawData) (models []interface{}, latestUpdated *time.Time, err error)
Review Comment:
it's a matter of preference to some degree, but I suggest we wrap the returned _models_ and _latestUpdated_ in a struct to logically group the returned arguments. It also enables us to easily extend the # of params returned if needed later with minimal code change
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@devlake.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [incubator-devlake] likyh commented on pull request #2416: feat: [POC] add since in framework
Posted by GitBox <gi...@apache.org>.
likyh commented on PR #2416:
URL: https://github.com/apache/incubator-devlake/pull/2416#issuecomment-1197774979
> If we're going to merge collectors and extractors, is there a need for raw tables anymore?
I think it should be kept. It is very useful when developing and debugging.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@devlake.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org