You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by GitBox <gi...@apache.org> on 2022/05/23 03:59:33 UTC
[GitHub] [incubator-devlake] thenicetgp opened a new pull request, #1968: collector draft
thenicetgp opened a new pull request, #1968:
URL: https://github.com/apache/incubator-devlake/pull/1968
# draft
add two file
plugins/gitlab/models/issue.go 和 plugins/gitlab/task/issue_collector.go
issue.go里的GitlabIssue和GithubIssue的结构基本一致，repoId换成了ProjectId
issue_collector.go和github里的也基本一致
其中需要修改url
查询到
<img width="742" alt="截屏2022-05-23 11 51 20" src="https://user-images.githubusercontent.com/47466847/169740042-45441c4d-2552-4fbf-8d9d-9330c1f65a47.png">
故url修改为
UrlTemplate: "projects/{{ .Params.ProjectId }}/issues“
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@devlake.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [incubator-devlake] thenicetgp commented on a diff in pull request #1968: collector draft
Posted by GitBox <gi...@apache.org>.
thenicetgp commented on code in PR #1968:
URL: https://github.com/apache/incubator-devlake/pull/1968#discussion_r881486303
##########
plugins/gitlab/tasks/issue_collector.go:
##########
@@ -0,0 +1,126 @@
+package tasks
+
+import (
+ "encoding/json"
+ "fmt"
+ "net/http"
+ "net/url"
+
+ "github.com/merico-dev/lake/plugins/helper"
+
+ "github.com/merico-dev/lake/plugins/core"
+ "github.com/merico-dev/lake/plugins/gitlab/models"
+)
+
+const RAW_ISSUE_TABLE = "gitlab_api_issues"
+
+//This Struct was declared in shared.go
+//type GitlabApiParams struct {
+// ProjectId int
+//}
+
+var CollectApiIssuesMeta = core.SubTaskMeta{
+ Name: "collectApiIssues",
+ EntryPoint: CollectApiIssues,
+ EnabledByDefault: true,
+ Description: "Collect issues data from Gitlab api",
+}
+
+func CollectApiIssues(taskCtx core.SubTaskContext) error {
+ db := taskCtx.GetDb()
+ data := taskCtx.GetData().(*GitlabTaskData)
+
+ since := data.Since
+ incremental := false
+ // user didn't specify a time range to sync, try load from database
+ if since == nil {
+ var latestUpdated models.GitlabIssue
+ err := db.Model(&latestUpdated).
+ Where("project_id = ?", data.Options.ProjectId).
+ Order("gitlab_updated_at DESC").Limit(1).Find(&latestUpdated).Error
+
+ if err != nil {
+ return fmt.Errorf("failed to get latest gitlab issue record: %w", err)
+ }
+ if latestUpdated.GitlabId > 0 {
+ since = &latestUpdated.GitlabUpdatedAt
Review Comment:
maybe not GitlabUpdatedAt
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@devlake.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [incubator-devlake] warren830 commented on a diff in pull request #1968: collector draft
Posted by GitBox <gi...@apache.org>.
warren830 commented on code in PR #1968:
URL: https://github.com/apache/incubator-devlake/pull/1968#discussion_r880175908
##########
plugins/gitlab/tasks/issue_convertor.go:
##########
@@ -0,0 +1,89 @@
+package tasks
+
+import (
+ "reflect"
+ "strconv"
+
+ "github.com/merico-dev/lake/plugins/core"
+ "github.com/merico-dev/lake/plugins/helper"
+
+ "github.com/merico-dev/lake/models/domainlayer"
+ "github.com/merico-dev/lake/models/domainlayer/didgen"
+ "github.com/merico-dev/lake/models/domainlayer/ticket"
+ gitlabModels "github.com/merico-dev/lake/plugins/gitlab/models"
+)
+
+var ConvertIssuesMeta = core.SubTaskMeta{
+ Name: "convertIssues",
+ EntryPoint: ConvertIssues,
+ EnabledByDefault: true,
+ Description: "Convert tool layer table gitlab_issues into domain layer table issues",
+}
+
+func ConvertIssues(taskCtx core.SubTaskContext) error {
+ db := taskCtx.GetDb()
+ data := taskCtx.GetData().(*GitlabTaskData)
+ projectId := data.ProjectCommit.GitlabProjectId
+
+ issue := &gitlabModels.GitlabIssue{}
+ cursor, err := db.Model(issue).Where("project_id = ?", projectId).Rows()
+
+ if err != nil {
+ return err
+ }
+ defer cursor.Close()
+
+ issueIdGen := didgen.NewDomainIdGenerator(&gitlabModels.GitlabIssue{})
+ userIdGen := didgen.NewDomainIdGenerator(&gitlabModels.GitlabUser{})
+ boardIdGen := didgen.NewDomainIdGenerator(&gitlabModels.GitlabProject{})
+
+ converter, err := helper.NewDataConverter(helper.DataConverterArgs{
+ RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
+ Ctx: taskCtx,
+ Params: GitlabApiParams{
+ ProjectId: data.Options.ProjectId,
+ },
+ Table: RAW_ISSUE_TABLE,
+ },
+ InputRowType: reflect.TypeOf(gitlabModels.GitlabIssue{}),
+ Input: cursor,
+ Convert: func(inputRow interface{}) ([]interface{}, error) {
+ issue := inputRow.(*gitlabModels.GitlabIssue)
+ domainIssue := &ticket.Issue{
+ DomainEntity: domainlayer.DomainEntity{Id: issueIdGen.Generate(issue.GitlabId)},
+ Number: strconv.Itoa(issue.Number),
+ Title: issue.Title,
+ Description: issue.Body,
+ Priority: issue.Priority,
+ Type: issue.Type,
+ AssigneeId: userIdGen.Generate(issue.AssigneeId),
+ AssigneeName: issue.AssigneeName,
+ LeadTimeMinutes: issue.LeadTimeMinutes,
+ Url: issue.Url,
+ CreatedDate: &issue.GitlabCreatedAt,
+ UpdatedDate: &issue.GitlabUpdatedAt,
+ ResolutionDate: issue.ClosedAt,
+ Severity: issue.Severity,
+ Component: issue.Component,
+ }
+ if issue.State == "closed" {
Review Comment:
OriginalStatus should be issue.State
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@devlake.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [incubator-devlake] warren830 commented on a diff in pull request #1968: collector draft
Posted by GitBox <gi...@apache.org>.
warren830 commented on code in PR #1968:
URL: https://github.com/apache/incubator-devlake/pull/1968#discussion_r881710299
##########
plugins/gitlab/tasks/issue_extractor.go:
##########
@@ -0,0 +1,193 @@
+package tasks
+
+import (
+ "encoding/json"
+ "github.com/apache/incubator-devlake/plugins/core"
+
+ "github.com/apache/incubator-devlake/plugins/gitlab/models"
+ "github.com/apache/incubator-devlake/plugins/helper"
+)
+
+var ExtractApiIssuesMeta = core.SubTaskMeta{
+ Name: "extractApiIssues",
+ EntryPoint: ExtractApiIssues,
+ EnabledByDefault: true,
+ Description: "Extract raw Issues data into tool layer table gitlab_issues",
+}
+
+type IssuesResponse struct {
+ ProjectId int `json:"id"`
+ Milestone struct {
+ Due_date string
+ Project_id int
+ State string
+ Description string
+ Iid int
+ Id int
+ Title string
+ CreatedAt helper.Iso8601Time
+ UpdatedAt helper.Iso8601Time
+ }
+ Author struct{
+ State string
+ WebUrl string
+ AvatarUrl string
+ Username string
+ Id int
+ Name string
+ }
+ Description string
+ State string
+ Iid int
+ Assignees []struct {
+ AvatarUrl string
+ WebUrl string
+ State string
+ Username string
+ Id int
+ Name string
+ }
+ Assignee *struct {
+ AvatarUrl string
+ WebUrl string
+ State string
+ Username string
+ Id int
+ Name string
+ }
+ Type string
+ Labels []string `json:"labels"`
+ UpVotes int
+ DownVotes int
+ MergeRequestsCount int
+ Id int
+ Title string
+ GitlabUpdatedAt helper.Iso8601Time `json:"updated_at"`
+ GitlabCreatedAt helper.Iso8601Time `json:"created_at"`
+ GitlabClosedAt *helper.Iso8601Time `json:"closed_at"`
+ ClosedBy struct{
+ State string
+ WebUrl string
+ AvatarUrl string
+ Username string
+ Id int
+ Name string
+ }
+ UserNotesCount int
+ DueDate helper.Iso8601Time
+ WebUrl string `json:"web_url"`
+ References struct {
+ Short string
+ Relative string
+ Full string
+ }
+ TimeStats struct {
+ TimeEstimate int
Review Comment:
this one should be extracted
and convert to OriginalEstimateMinutes
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@devlake.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [incubator-devlake] thenicetgp commented on pull request #1968: add gitlab issues
Posted by GitBox <gi...@apache.org>.
thenicetgp commented on PR #1968:
URL: https://github.com/apache/incubator-devlake/pull/1968#issuecomment-1138236167
data in _raw_gitlab_api_issues(raw layer)
<img width="1053" alt="截屏2022-05-26 14 46 15" src="https://user-images.githubusercontent.com/47466847/170436549-1984f59d-ea33-43ce-a2b0-32a138daef7e.png">
data in _tool_gitlab_issues(tool layer)
<img width="1347" alt="截屏2022-05-26 14 51 51" src="https://user-images.githubusercontent.com/47466847/170436581-30534919-0e32-4286-b569-7857b4988ae9.png">
data in issues(domain layer)
<img width="1369" alt="截屏2022-05-26 14 52 15" src="https://user-images.githubusercontent.com/47466847/170436612-21a5c880-2bf9-43a4-b711-5cd7f42fd393.png">
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@devlake.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [incubator-devlake] warren830 commented on a diff in pull request #1968: collector draft
Posted by GitBox <gi...@apache.org>.
warren830 commented on code in PR #1968:
URL: https://github.com/apache/incubator-devlake/pull/1968#discussion_r879450677
##########
plugins/gitlab/tasks/issue_collector.go:
##########
@@ -0,0 +1,127 @@
+package tasks
+
+import (
+ "encoding/json"
+ "fmt"
+ "net/http"
+ "net/url"
+
+ "github.com/merico-dev/lake/plugins/helper"
+
+ "github.com/merico-dev/lake/plugins/core"
+ "github.com/merico-dev/lake/plugins/gitlab/models"
+)
+
+const RAW_ISSUE_TABLE = "gitlab_api_issues"
+
+//This Struct was declared in shared.go
+//type GitlabApiParams struct {
+// ProjectId int
+//}
+
+var CollectApiIssuesMeta = core.SubTaskMeta{
+ Name: "collectApiIssues",
+ EntryPoint: CollectApiIssues,
+ EnabledByDefault: true,
+ Description: "Collect issues data from Gitlab api",
+}
+
+func CollectApiIssues(taskCtx core.SubTaskContext) error {
+ db := taskCtx.GetDb()
+ data := taskCtx.GetData().(*GitlabTaskData)
+
+ since := data.Since
+ incremental := false
+ // user didn't specify a time range to sync, try load from database
+ if since == nil {
+ var latestUpdated models.GitlabIssue
+ err := db.Model(&latestUpdated).
+ Where("project_id = ?", data.ProjectCommit.GitlabProjectId).
+ Order("gitlab_updated_at DESC").Limit(1).Find(&latestUpdated).Error
+
+ if err != nil {
+ return fmt.Errorf("failed to get latest gitlab issue record: %w", err)
+ }
+ if latestUpdated.GitlabId > 0 {
+ since = &latestUpdated.GitlabUpdatedAt
+ incremental = true
+ }
+ }
+
+ collector, err := helper.NewApiCollector(helper.ApiCollectorArgs{
+ RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
+ Ctx: taskCtx,
+ /*
+ This struct will be JSONEncoded and stored into database along with raw data itself, to identity minimal
+ set of data to be process, for example, we process JiraIssues by Board
+ */
+ Params: GitlabApiParams{
+ ProjectId: data.Options.ProjectId,
+ },
+ /*
+ Table store raw data
+ */
+ Table: RAW_ISSUE_TABLE,
+ },
+ ApiClient: data.ApiClient,
+ PageSize: 100,
+ Incremental: incremental,
+ /*
+ url may use arbitrary variables from different source in any order, we need GoTemplate to allow more
+ flexible for all kinds of possibility.
+ Pager contains information for a particular page, calculated by ApiCollector, and will be passed into
+ GoTemplate to generate a url for that page.
+ We want to do page-fetching in ApiCollector, because the logic are highly similar, by doing so, we can
+ avoid duplicate logic for every tasks, and when we have a better idea like improving performance, we can
+ do it in one place
+ */
+ UrlTemplate: "projects/{{ .Params.ProjectId }}/issues",
+ /*
+ (Optional) Return query string for request, or you can plug them into UrlTemplate directly
+ */
+ Query: func(reqData *helper.RequestData) (url.Values, error) {
+ query := url.Values{}
+ query.Set("state", "all")
+ if since != nil {
+ query.Set("since", since.String())
+ }
+ query.Set("direction", "asc")
Review Comment:
please check if gitlab use `direction` and `since`, I think it might use other param names for this
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@devlake.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [incubator-devlake] warren830 commented on a diff in pull request #1968: collector draft
Posted by GitBox <gi...@apache.org>.
warren830 commented on code in PR #1968:
URL: https://github.com/apache/incubator-devlake/pull/1968#discussion_r882304201
##########
plugins/gitlab/models/migrationscripts/archived/issue.go:
##########
@@ -0,0 +1,33 @@
+package archived
+
+import (
+ "github.com/apache/incubator-devlake/models/migrationscripts/archived"
+ "time"
+)
+
+type GitlabIssue struct {
+ GitlabId int `gorm:"primaryKey"`
+ ProjectId int `gorm:"index"`
+ Number int `gorm:"index;comment:Used in API requests ex. api/repo/1/issue/<THIS_NUMBER>"`
+ State string `gorm:"type:varchar(255)"`
+ Title string
+ Body string
+ Priority string `gorm:"type:varchar(255)"`
+ Type string `gorm:"type:varchar(100)"`
+ Status string `gorm:"type:varchar(255)"`
+ AssigneeId int
+ AssigneeName string `gorm:"type:varchar(255)"`
+ LeadTimeMinutes uint
+ Url string `gorm:"type:varchar(255)"`
+ ClosedAt *time.Time
+ GitlabCreatedAt time.Time
+ GitlabUpdatedAt time.Time `gorm:"index"`
+ Severity string `gorm:"type:varchar(255)"`
+ Component string `gorm:"type:varchar(255)"`
+ TimeEstimate int64
+ TotalTimeSpent int64
+ archived.NoPKModel
+}
+func (GitlabIssue) TableName() string {
+ return "_tool_gitlab_issues"
+}
Review Comment:
move all code of this file to update_schemas20220525, and delete this file
##########
plugins/gitlab/models/migrationscripts/archived/issue_label.go:
##########
@@ -0,0 +1,15 @@
+package archived
+
+import (
+ "github.com/apache/incubator-devlake/models/migrationscripts/archived"
+)
+
+type GitlabIssueLabel struct {
+ IssueId int `gorm:"primaryKey;autoIncrement:false"`
+ LabelName string `gorm:"primaryKey;type:varchar(255)"`
+ archived.NoPKModel
+}
+
+func (GitlabIssueLabel) TableName() string{
+ return "_tool_gitlab_issue_labels"
+}
Review Comment:
move all code of this file to update_schemas20220525, and delete this file
##########
plugins/gitlab/tasks/issue_extractor.go:
##########
@@ -0,0 +1,195 @@
+package tasks
+
+import (
+ "encoding/json"
+ "github.com/apache/incubator-devlake/plugins/core"
+
+ "github.com/apache/incubator-devlake/plugins/gitlab/models"
+ "github.com/apache/incubator-devlake/plugins/helper"
+)
+
+var ExtractApiIssuesMeta = core.SubTaskMeta{
+ Name: "extractApiIssues",
+ EntryPoint: ExtractApiIssues,
+ EnabledByDefault: true,
+ Description: "Extract raw Issues data into tool layer table gitlab_issues",
+}
+
+type IssuesResponse struct {
+ ProjectId int `json:"id"`
Review Comment:
should be `json:project_id`
##########
plugins/gitlab/tasks/issue_extractor.go:
##########
@@ -0,0 +1,195 @@
+package tasks
+
+import (
+ "encoding/json"
+ "github.com/apache/incubator-devlake/plugins/core"
+
+ "github.com/apache/incubator-devlake/plugins/gitlab/models"
+ "github.com/apache/incubator-devlake/plugins/helper"
+)
+
+var ExtractApiIssuesMeta = core.SubTaskMeta{
+ Name: "extractApiIssues",
+ EntryPoint: ExtractApiIssues,
+ EnabledByDefault: true,
+ Description: "Extract raw Issues data into tool layer table gitlab_issues",
+}
+
+type IssuesResponse struct {
+ ProjectId int `json:"id"`
+ Milestone struct {
+ Due_date string
+ Project_id int
+ State string
+ Description string
+ Iid int
+ Id int
+ Title string
+ CreatedAt helper.Iso8601Time
+ UpdatedAt helper.Iso8601Time
+ }
+ Author struct{
+ State string
+ WebUrl string
+ AvatarUrl string
+ Username string
+ Id int
+ Name string
+ }
+ Description string
+ State string
+ Iid int
+ Assignees []struct {
+ AvatarUrl string
+ WebUrl string
+ State string
+ Username string
+ Id int
+ Name string
+ }
+ Assignee *struct {
+ AvatarUrl string
+ WebUrl string
+ State string
+ Username string
+ Id int
+ Name string
+ }
+ Type string
+ Labels []string `json:"labels"`
+ UpVotes int
+ DownVotes int
+ MergeRequestsCount int
+ Id int
+ Title string
+ GitlabUpdatedAt helper.Iso8601Time `json:"updated_at"`
+ GitlabCreatedAt helper.Iso8601Time `json:"created_at"`
+ GitlabClosedAt *helper.Iso8601Time `json:"closed_at"`
+ ClosedBy struct{
+ State string
+ WebUrl string
+ AvatarUrl string
+ Username string
+ Id int
+ Name string
+ }
+ UserNotesCount int
+ DueDate helper.Iso8601Time
+ WebUrl string `json:"web_url"`
+ References struct {
+ Short string
+ Relative string
+ Full string
+ }
+ TimeStats struct {
+ TimeEstimate int64
+ TotalTimeSpent int64
+ HumanTimeEstimate string
+ HumanTotalTimeSpent string
+ }
+ HasTasks bool
+ TaskStatus string
+ Confidential bool
+ DiscussionLocked bool
+ IssueType string
+ Serverity string
+ Links struct {
+ Self string `json:"url"`
+ Notes string
+ AwardEmoji string
+ Project string
+ }
+ TaskCompletionStatus struct {
+ Count int
+ CompletedCount int
+ }
+
+}
+
+func ExtractApiIssues(taskCtx core.SubTaskContext) error {
+ data := taskCtx.GetData().(*GitlabTaskData)
+
+ extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{
+ RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
+ Ctx: taskCtx,
+ /*
+ This struct will be JSONEncoded and stored into database along with raw data itself, to identity minimal
+ set of data to be process, for example, we process JiraIssues by Board
+ */
+ Params: GitlabApiParams{
+ ProjectId: data.Options.ProjectId,
+ },
+ /*
+ Table store raw data
+ */
+ Table: RAW_ISSUE_TABLE,
+ },
+ Extract: func(row *helper.RawData) ([]interface{}, error) {
+ body := &IssuesResponse{}
+ err := json.Unmarshal(row.Data, body)
+ if err != nil {
+ return nil, err
+ }
+ // need to extract 2 kinds of entities here
+ if body.ProjectId == 0 {
+ return nil, nil
+ }
+ ////If this is a pr, ignore
+ //if body.PullRequest.Url != "" {
+ // return nil, nil
+ //}
+ //If this is not Issue, ignore
+ if body.IssueType != "ISSUE" {
+ return nil, nil
+ }
+ results := make([]interface{}, 0, 2)
+ gitlabIssue, err := convertGitlabIssue(body, data.Options.ProjectId)
+ if err != nil {
+ return nil, err
+ }
+
+ for _, label := range body.Labels {
+ results = append(results, &models.GitlabIssueLabel{
+ IssueId: gitlabIssue.GitlabId,
+ LabelName: label,
+ })
+
+ }
+ results = append(results, gitlabIssue)
+
+ return results, nil
+ },
+ })
+
+ if err != nil {
+ return err
+ }
+
+ return extractor.Execute()
+}
+func convertGitlabIssue(issue *IssuesResponse, projectId int) (*models.GitlabIssue, error) {
+ gitlabIssue := &models.GitlabIssue{
+ GitlabId: issue.Id,
+ ProjectId: projectId,
+ Number: issue.Iid,
+ State: issue.State,
+ Title: issue.Title,
+ Body: issue.Description,
+ Url: issue.Links.Self,
+ ClosedAt: helper.Iso8601TimeToTime(issue.GitlabClosedAt),
Review Comment:
this seems not right
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@devlake.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [incubator-devlake] warren830 commented on a diff in pull request #1968: collector draft
Posted by GitBox <gi...@apache.org>.
warren830 commented on code in PR #1968:
URL: https://github.com/apache/incubator-devlake/pull/1968#discussion_r879451713
##########
plugins/gitlab/tasks/issue_convertor.go:
##########
@@ -0,0 +1,89 @@
+package tasks
+
+import (
+ "reflect"
+ "strconv"
+
+ "github.com/merico-dev/lake/plugins/core"
+ "github.com/merico-dev/lake/plugins/helper"
+
+ "github.com/merico-dev/lake/models/domainlayer"
+ "github.com/merico-dev/lake/models/domainlayer/didgen"
+ "github.com/merico-dev/lake/models/domainlayer/ticket"
+ gitlabModels "github.com/merico-dev/lake/plugins/gitlab/models"
+)
+
+var ConvertIssuesMeta = core.SubTaskMeta{
+ Name: "convertIssues",
+ EntryPoint: ConvertIssues,
+ EnabledByDefault: true,
+ Description: "Convert tool layer table gitlab_issues into domain layer table issues",
+}
+
+func ConvertIssues(taskCtx core.SubTaskContext) error {
+ db := taskCtx.GetDb()
+ data := taskCtx.GetData().(*GitlabTaskData)
+ projectId := data.ProjectCommit.GitlabProjectId
+
+ issue := &gitlabModels.GitlabIssue{}
+ cursor, err := db.Model(issue).Where("project_id = ?", projectId).Rows()
+
+ if err != nil {
+ return err
+ }
+ defer cursor.Close()
+
+ issueIdGen := didgen.NewDomainIdGenerator(&gitlabModels.GitlabIssue{})
+ userIdGen := didgen.NewDomainIdGenerator(&gitlabModels.GitlabUser{})
+ boardIdGen := didgen.NewDomainIdGenerator(&gitlabModels.GitlabProject{})
+
+ converter, err := helper.NewDataConverter(helper.DataConverterArgs{
+ RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
+ Ctx: taskCtx,
+ Params: GitlabApiParams{
+ ProjectId: data.Options.ProjectId,
+ },
+ Table: RAW_ISSUE_TABLE,
+ },
+ InputRowType: reflect.TypeOf(gitlabModels.GitlabIssue{}),
+ Input: cursor,
+ Convert: func(inputRow interface{}) ([]interface{}, error) {
+ issue := inputRow.(*gitlabModels.GitlabIssue)
+ domainIssue := &ticket.Issue{
+ DomainEntity: domainlayer.DomainEntity{Id: issueIdGen.Generate(issue.GitlabId)},
+ Number: strconv.Itoa(issue.Number),
+ Title: issue.Title,
+ Description: issue.Body,
+ Priority: issue.Priority,
+ Type: issue.Type,
+ AssigneeId: userIdGen.Generate(issue.AssigneeId),
+ AssigneeName: issue.AssigneeName,
+ LeadTimeMinutes: issue.LeadTimeMinutes,
+ Url: issue.Url,
+ CreatedDate: &issue.GitlabCreatedAt,
+ UpdatedDate: &issue.GitlabUpdatedAt,
+ ResolutionDate: issue.ClosedAt,
+ Severity: issue.Severity,
+ Component: issue.Component,
+ }
+ if issue.State == "closed" {
Review Comment:
please check the value of issue.State
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@devlake.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [incubator-devlake] thenicetgp commented on a diff in pull request #1968: collector draft
Posted by GitBox <gi...@apache.org>.
thenicetgp commented on code in PR #1968:
URL: https://github.com/apache/incubator-devlake/pull/1968#discussion_r880011171
##########
plugins/gitlab/tasks/issue_extractor.go:
##########
@@ -0,0 +1,192 @@
+package tasks
+
+import (
+ "encoding/json"
+
+ "github.com/merico-dev/lake/plugins/core"
+ "github.com/merico-dev/lake/plugins/gitlab/models"
+ "github.com/merico-dev/lake/plugins/helper"
+)
+
+var ExtractApiIssuesMeta = core.SubTaskMeta{
+ Name: "extractApiIssues",
+ EntryPoint: ExtractApiIssues,
+ EnabledByDefault: true,
+ Description: "Extract raw Issues data into tool layer table gitlab_issues",
+}
+
+type IssuesResponse struct {
+ ProjectId int `json:"id"`
+ Milestone struct {
+ Due_date string
+ Project_id int
+ State string
+ Description string
+ Iid int
+ Id int
+ Title string
+ CreatedAt *core.Iso8601Time
+ UpdatedAt *core.Iso8601Time
+ }
+ Author struct {
+ State string
+ WebUrl string
+ AvatarUrl string
+ Username string
+ Id int
+ Name string
+ }
+ Description string
+ State string
+ Iid int
+ Assignees []struct {
+ AvatarUrl string
+ WebUrl string
+ State string
+ Username string
+ Id int
+ Name string
+ }
+ Assignee *struct {
+ AvatarUrl string
+ WebUrl string
+ State string
+ Username string
+ Id int
+ Name string
+ }
+ Type string
+ Labels []string `json:"labels"`
+ UpVotes int
+ DownVotes int
+ MergeRequestsCount int
+ Id int
+ Title string
+ GitlabUpdatedAt core.Iso8601Time `json:"updated_at"`
+ GitlabCreatedAt core.Iso8601Time `json:"created_at"`
+ GitlabClosedAt *core.Iso8601Time `json:"closed_at"`
+ ClosedBy struct {
+ State string
+ WebUrl string
+ AvatarUrl string
+ Username string
+ Id int
+ Name string
+ }
+ UserNotesCount int
+ DueDate *core.Iso8601Time
+ WebUrl string `json:"web_url"`
+ References struct {
+ Short string
+ Relative string
+ Full string
+ }
+ TimeStats struct {
+ TimeEstimate int
+ TotalTimeSpent int
+ HumanTimeEstimate string
+ HumanTotalTimeSpent string
+ }
+ HasTasks bool
+ TaskStatus string
+ Confidential bool
+ DiscussionLocked bool
+ IssueType string
+ Serverity string
+ Links struct {
+ Self string `json:"url"`
+ Notes string
+ AwardEmoji string
+ Project string
+ }
+ TaskCompletionStatus struct {
+ Count int
+ CompletedCount int
+ }
+}
+
+func ExtractApiIssues(taskCtx core.SubTaskContext) error {
+ data := taskCtx.GetData().(*GitlabTaskData)
+
+ extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{
+ RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
+ Ctx: taskCtx,
+ /*
+ This struct will be JSONEncoded and stored into database along with raw data itself, to identity minimal
+ set of data to be process, for example, we process JiraIssues by Board
+ */
+ Params: GitlabApiParams{
+ ProjectId: data.Options.ProjectId,
+ },
+ /*
+ Table store raw data
+ */
+ Table: RAW_ISSUE_TABLE,
+ },
+ Extract: func(row *helper.RawData) ([]interface{}, error) {
+ body := &IssuesResponse{}
+ err := json.Unmarshal(row.Data, body)
+ if err != nil {
+ return nil, err
+ }
+ // need to extract 2 kinds of entities here
+ if body.ProjectId == 0 {
+ return nil, nil
+ }
+ ////If this is a pr, ignore
+ //if body.PullRequest.Url != "" {
+ // return nil, nil
+ //}
+ //If this is not Issue, ignore
+ if body.IssueType != "ISSUE" {
Review Comment:
I checked, the answer is yes
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@devlake.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [incubator-devlake] thenicetgp commented on a diff in pull request #1968: collector draft
Posted by GitBox <gi...@apache.org>.
thenicetgp commented on code in PR #1968:
URL: https://github.com/apache/incubator-devlake/pull/1968#discussion_r880021056
##########
plugins/gitlab/tasks/issue_convertor.go:
##########
@@ -0,0 +1,89 @@
+package tasks
+
+import (
+ "reflect"
+ "strconv"
+
+ "github.com/merico-dev/lake/plugins/core"
+ "github.com/merico-dev/lake/plugins/helper"
+
+ "github.com/merico-dev/lake/models/domainlayer"
+ "github.com/merico-dev/lake/models/domainlayer/didgen"
+ "github.com/merico-dev/lake/models/domainlayer/ticket"
+ gitlabModels "github.com/merico-dev/lake/plugins/gitlab/models"
+)
+
+var ConvertIssuesMeta = core.SubTaskMeta{
+ Name: "convertIssues",
+ EntryPoint: ConvertIssues,
+ EnabledByDefault: true,
+ Description: "Convert tool layer table gitlab_issues into domain layer table issues",
+}
+
+func ConvertIssues(taskCtx core.SubTaskContext) error {
+ db := taskCtx.GetDb()
+ data := taskCtx.GetData().(*GitlabTaskData)
+ projectId := data.ProjectCommit.GitlabProjectId
+
+ issue := &gitlabModels.GitlabIssue{}
+ cursor, err := db.Model(issue).Where("project_id = ?", projectId).Rows()
+
+ if err != nil {
+ return err
+ }
+ defer cursor.Close()
+
+ issueIdGen := didgen.NewDomainIdGenerator(&gitlabModels.GitlabIssue{})
+ userIdGen := didgen.NewDomainIdGenerator(&gitlabModels.GitlabUser{})
+ boardIdGen := didgen.NewDomainIdGenerator(&gitlabModels.GitlabProject{})
+
+ converter, err := helper.NewDataConverter(helper.DataConverterArgs{
+ RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
+ Ctx: taskCtx,
+ Params: GitlabApiParams{
+ ProjectId: data.Options.ProjectId,
+ },
+ Table: RAW_ISSUE_TABLE,
+ },
+ InputRowType: reflect.TypeOf(gitlabModels.GitlabIssue{}),
+ Input: cursor,
+ Convert: func(inputRow interface{}) ([]interface{}, error) {
+ issue := inputRow.(*gitlabModels.GitlabIssue)
+ domainIssue := &ticket.Issue{
+ DomainEntity: domainlayer.DomainEntity{Id: issueIdGen.Generate(issue.GitlabId)},
+ Number: strconv.Itoa(issue.Number),
+ Title: issue.Title,
+ Description: issue.Body,
+ Priority: issue.Priority,
+ Type: issue.Type,
+ AssigneeId: userIdGen.Generate(issue.AssigneeId),
+ AssigneeName: issue.AssigneeName,
+ LeadTimeMinutes: issue.LeadTimeMinutes,
+ Url: issue.Url,
+ CreatedDate: &issue.GitlabCreatedAt,
+ UpdatedDate: &issue.GitlabUpdatedAt,
+ ResolutionDate: issue.ClosedAt,
+ Severity: issue.Severity,
+ Component: issue.Component,
+ }
+ if issue.State == "closed" {
Review Comment:
How to understand “OriginalStatus need to be assigned value”, is there any other place that needs to be assigned a value?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@devlake.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [incubator-devlake] warren830 merged pull request #1968: add gitlab issues
Posted by GitBox <gi...@apache.org>.
warren830 merged PR #1968:
URL: https://github.com/apache/incubator-devlake/pull/1968
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@devlake.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [incubator-devlake] warren830 commented on a diff in pull request #1968: collector draft
Posted by GitBox <gi...@apache.org>.
warren830 commented on code in PR #1968:
URL: https://github.com/apache/incubator-devlake/pull/1968#discussion_r879451887
##########
plugins/gitlab/tasks/issue_convertor.go:
##########
@@ -0,0 +1,89 @@
+package tasks
+
+import (
+ "reflect"
+ "strconv"
+
+ "github.com/merico-dev/lake/plugins/core"
+ "github.com/merico-dev/lake/plugins/helper"
+
+ "github.com/merico-dev/lake/models/domainlayer"
+ "github.com/merico-dev/lake/models/domainlayer/didgen"
+ "github.com/merico-dev/lake/models/domainlayer/ticket"
+ gitlabModels "github.com/merico-dev/lake/plugins/gitlab/models"
+)
+
+var ConvertIssuesMeta = core.SubTaskMeta{
+ Name: "convertIssues",
+ EntryPoint: ConvertIssues,
+ EnabledByDefault: true,
+ Description: "Convert tool layer table gitlab_issues into domain layer table issues",
+}
+
+func ConvertIssues(taskCtx core.SubTaskContext) error {
+ db := taskCtx.GetDb()
+ data := taskCtx.GetData().(*GitlabTaskData)
+ projectId := data.ProjectCommit.GitlabProjectId
+
+ issue := &gitlabModels.GitlabIssue{}
+ cursor, err := db.Model(issue).Where("project_id = ?", projectId).Rows()
+
+ if err != nil {
+ return err
+ }
+ defer cursor.Close()
+
+ issueIdGen := didgen.NewDomainIdGenerator(&gitlabModels.GitlabIssue{})
+ userIdGen := didgen.NewDomainIdGenerator(&gitlabModels.GitlabUser{})
+ boardIdGen := didgen.NewDomainIdGenerator(&gitlabModels.GitlabProject{})
+
+ converter, err := helper.NewDataConverter(helper.DataConverterArgs{
+ RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
+ Ctx: taskCtx,
+ Params: GitlabApiParams{
+ ProjectId: data.Options.ProjectId,
+ },
+ Table: RAW_ISSUE_TABLE,
+ },
+ InputRowType: reflect.TypeOf(gitlabModels.GitlabIssue{}),
+ Input: cursor,
+ Convert: func(inputRow interface{}) ([]interface{}, error) {
+ issue := inputRow.(*gitlabModels.GitlabIssue)
+ domainIssue := &ticket.Issue{
+ DomainEntity: domainlayer.DomainEntity{Id: issueIdGen.Generate(issue.GitlabId)},
+ Number: strconv.Itoa(issue.Number),
+ Title: issue.Title,
+ Description: issue.Body,
+ Priority: issue.Priority,
+ Type: issue.Type,
+ AssigneeId: userIdGen.Generate(issue.AssigneeId),
+ AssigneeName: issue.AssigneeName,
+ LeadTimeMinutes: issue.LeadTimeMinutes,
+ Url: issue.Url,
+ CreatedDate: &issue.GitlabCreatedAt,
+ UpdatedDate: &issue.GitlabUpdatedAt,
+ ResolutionDate: issue.ClosedAt,
+ Severity: issue.Severity,
+ Component: issue.Component,
+ }
+ if issue.State == "closed" {
+ domainIssue.Status = ticket.DONE
Review Comment:
same above
##########
plugins/gitlab/tasks/issue_extractor.go:
##########
@@ -0,0 +1,192 @@
+package tasks
+
+import (
+ "encoding/json"
+
+ "github.com/merico-dev/lake/plugins/core"
+ "github.com/merico-dev/lake/plugins/gitlab/models"
+ "github.com/merico-dev/lake/plugins/helper"
+)
+
+var ExtractApiIssuesMeta = core.SubTaskMeta{
+ Name: "extractApiIssues",
+ EntryPoint: ExtractApiIssues,
+ EnabledByDefault: true,
+ Description: "Extract raw Issues data into tool layer table gitlab_issues",
+}
+
+type IssuesResponse struct {
+ ProjectId int `json:"id"`
+ Milestone struct {
+ Due_date string
+ Project_id int
+ State string
+ Description string
+ Iid int
+ Id int
+ Title string
+ CreatedAt *core.Iso8601Time
+ UpdatedAt *core.Iso8601Time
+ }
+ Author struct {
+ State string
+ WebUrl string
+ AvatarUrl string
+ Username string
+ Id int
+ Name string
+ }
+ Description string
+ State string
+ Iid int
+ Assignees []struct {
+ AvatarUrl string
+ WebUrl string
+ State string
+ Username string
+ Id int
+ Name string
+ }
+ Assignee *struct {
+ AvatarUrl string
+ WebUrl string
+ State string
+ Username string
+ Id int
+ Name string
+ }
+ Type string
+ Labels []string `json:"labels"`
+ UpVotes int
+ DownVotes int
+ MergeRequestsCount int
+ Id int
+ Title string
+ GitlabUpdatedAt core.Iso8601Time `json:"updated_at"`
+ GitlabCreatedAt core.Iso8601Time `json:"created_at"`
+ GitlabClosedAt *core.Iso8601Time `json:"closed_at"`
+ ClosedBy struct {
+ State string
+ WebUrl string
+ AvatarUrl string
+ Username string
+ Id int
+ Name string
+ }
+ UserNotesCount int
+ DueDate *core.Iso8601Time
+ WebUrl string `json:"web_url"`
+ References struct {
+ Short string
+ Relative string
+ Full string
+ }
+ TimeStats struct {
+ TimeEstimate int
+ TotalTimeSpent int
+ HumanTimeEstimate string
+ HumanTotalTimeSpent string
+ }
+ HasTasks bool
+ TaskStatus string
+ Confidential bool
+ DiscussionLocked bool
+ IssueType string
+ Serverity string
+ Links struct {
+ Self string `json:"url"`
+ Notes string
+ AwardEmoji string
+ Project string
+ }
+ TaskCompletionStatus struct {
+ Count int
+ CompletedCount int
+ }
+}
+
+func ExtractApiIssues(taskCtx core.SubTaskContext) error {
+ data := taskCtx.GetData().(*GitlabTaskData)
+
+ extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{
+ RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
+ Ctx: taskCtx,
+ /*
+ This struct will be JSONEncoded and stored into database along with raw data itself, to identity minimal
+ set of data to be process, for example, we process JiraIssues by Board
+ */
+ Params: GitlabApiParams{
+ ProjectId: data.Options.ProjectId,
+ },
+ /*
+ Table store raw data
+ */
+ Table: RAW_ISSUE_TABLE,
+ },
+ Extract: func(row *helper.RawData) ([]interface{}, error) {
+ body := &IssuesResponse{}
+ err := json.Unmarshal(row.Data, body)
+ if err != nil {
+ return nil, err
+ }
+ // need to extract 2 kinds of entities here
+ if body.ProjectId == 0 {
+ return nil, nil
+ }
+ ////If this is a pr, ignore
+ //if body.PullRequest.Url != "" {
+ // return nil, nil
+ //}
+ //If this is not Issue, ignore
+ if body.IssueType != "ISSUE" {
Review Comment:
please check if there is a IssueType
##########
plugins/gitlab/tasks/issue_convertor.go:
##########
@@ -0,0 +1,89 @@
+package tasks
+
+import (
+ "reflect"
+ "strconv"
+
+ "github.com/merico-dev/lake/plugins/core"
+ "github.com/merico-dev/lake/plugins/helper"
+
+ "github.com/merico-dev/lake/models/domainlayer"
+ "github.com/merico-dev/lake/models/domainlayer/didgen"
+ "github.com/merico-dev/lake/models/domainlayer/ticket"
+ gitlabModels "github.com/merico-dev/lake/plugins/gitlab/models"
+)
+
+var ConvertIssuesMeta = core.SubTaskMeta{
+ Name: "convertIssues",
+ EntryPoint: ConvertIssues,
+ EnabledByDefault: true,
+ Description: "Convert tool layer table gitlab_issues into domain layer table issues",
+}
+
+func ConvertIssues(taskCtx core.SubTaskContext) error {
+ db := taskCtx.GetDb()
+ data := taskCtx.GetData().(*GitlabTaskData)
+ projectId := data.ProjectCommit.GitlabProjectId
+
+ issue := &gitlabModels.GitlabIssue{}
+ cursor, err := db.Model(issue).Where("project_id = ?", projectId).Rows()
+
+ if err != nil {
+ return err
+ }
+ defer cursor.Close()
+
+ issueIdGen := didgen.NewDomainIdGenerator(&gitlabModels.GitlabIssue{})
+ userIdGen := didgen.NewDomainIdGenerator(&gitlabModels.GitlabUser{})
+ boardIdGen := didgen.NewDomainIdGenerator(&gitlabModels.GitlabProject{})
+
+ converter, err := helper.NewDataConverter(helper.DataConverterArgs{
+ RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
+ Ctx: taskCtx,
+ Params: GitlabApiParams{
+ ProjectId: data.Options.ProjectId,
+ },
+ Table: RAW_ISSUE_TABLE,
+ },
+ InputRowType: reflect.TypeOf(gitlabModels.GitlabIssue{}),
+ Input: cursor,
+ Convert: func(inputRow interface{}) ([]interface{}, error) {
+ issue := inputRow.(*gitlabModels.GitlabIssue)
+ domainIssue := &ticket.Issue{
+ DomainEntity: domainlayer.DomainEntity{Id: issueIdGen.Generate(issue.GitlabId)},
+ Number: strconv.Itoa(issue.Number),
+ Title: issue.Title,
+ Description: issue.Body,
+ Priority: issue.Priority,
+ Type: issue.Type,
+ AssigneeId: userIdGen.Generate(issue.AssigneeId),
+ AssigneeName: issue.AssigneeName,
+ LeadTimeMinutes: issue.LeadTimeMinutes,
+ Url: issue.Url,
+ CreatedDate: &issue.GitlabCreatedAt,
+ UpdatedDate: &issue.GitlabUpdatedAt,
+ ResolutionDate: issue.ClosedAt,
+ Severity: issue.Severity,
+ Component: issue.Component,
+ }
+ if issue.State == "closed" {
Review Comment:
and there is a OriginalStatus need to be assigned value
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@devlake.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [incubator-devlake] warren830 commented on a diff in pull request #1968: collector draft
Posted by GitBox <gi...@apache.org>.
warren830 commented on code in PR #1968:
URL: https://github.com/apache/incubator-devlake/pull/1968#discussion_r879451887
##########
plugins/gitlab/tasks/issue_convertor.go:
##########
@@ -0,0 +1,89 @@
+package tasks
+
+import (
+ "reflect"
+ "strconv"
+
+ "github.com/merico-dev/lake/plugins/core"
+ "github.com/merico-dev/lake/plugins/helper"
+
+ "github.com/merico-dev/lake/models/domainlayer"
+ "github.com/merico-dev/lake/models/domainlayer/didgen"
+ "github.com/merico-dev/lake/models/domainlayer/ticket"
+ gitlabModels "github.com/merico-dev/lake/plugins/gitlab/models"
+)
+
+var ConvertIssuesMeta = core.SubTaskMeta{
+ Name: "convertIssues",
+ EntryPoint: ConvertIssues,
+ EnabledByDefault: true,
+ Description: "Convert tool layer table gitlab_issues into domain layer table issues",
+}
+
+func ConvertIssues(taskCtx core.SubTaskContext) error {
+ db := taskCtx.GetDb()
+ data := taskCtx.GetData().(*GitlabTaskData)
+ projectId := data.ProjectCommit.GitlabProjectId
+
+ issue := &gitlabModels.GitlabIssue{}
+ cursor, err := db.Model(issue).Where("project_id = ?", projectId).Rows()
+
+ if err != nil {
+ return err
+ }
+ defer cursor.Close()
+
+ issueIdGen := didgen.NewDomainIdGenerator(&gitlabModels.GitlabIssue{})
+ userIdGen := didgen.NewDomainIdGenerator(&gitlabModels.GitlabUser{})
+ boardIdGen := didgen.NewDomainIdGenerator(&gitlabModels.GitlabProject{})
+
+ converter, err := helper.NewDataConverter(helper.DataConverterArgs{
+ RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
+ Ctx: taskCtx,
+ Params: GitlabApiParams{
+ ProjectId: data.Options.ProjectId,
+ },
+ Table: RAW_ISSUE_TABLE,
+ },
+ InputRowType: reflect.TypeOf(gitlabModels.GitlabIssue{}),
+ Input: cursor,
+ Convert: func(inputRow interface{}) ([]interface{}, error) {
+ issue := inputRow.(*gitlabModels.GitlabIssue)
+ domainIssue := &ticket.Issue{
+ DomainEntity: domainlayer.DomainEntity{Id: issueIdGen.Generate(issue.GitlabId)},
+ Number: strconv.Itoa(issue.Number),
+ Title: issue.Title,
+ Description: issue.Body,
+ Priority: issue.Priority,
+ Type: issue.Type,
+ AssigneeId: userIdGen.Generate(issue.AssigneeId),
+ AssigneeName: issue.AssigneeName,
+ LeadTimeMinutes: issue.LeadTimeMinutes,
+ Url: issue.Url,
+ CreatedDate: &issue.GitlabCreatedAt,
+ UpdatedDate: &issue.GitlabUpdatedAt,
+ ResolutionDate: issue.ClosedAt,
+ Severity: issue.Severity,
+ Component: issue.Component,
+ }
+ if issue.State == "closed" {
+ domainIssue.Status = ticket.DONE
Review Comment:
same above
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@devlake.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [incubator-devlake] warren830 commented on a diff in pull request #1968: collector draft
Posted by GitBox <gi...@apache.org>.
warren830 commented on code in PR #1968:
URL: https://github.com/apache/incubator-devlake/pull/1968#discussion_r881710799
##########
plugins/gitlab/tasks/issue_extractor.go:
##########
@@ -0,0 +1,193 @@
+package tasks
+
+import (
+ "encoding/json"
+ "github.com/apache/incubator-devlake/plugins/core"
+
+ "github.com/apache/incubator-devlake/plugins/gitlab/models"
+ "github.com/apache/incubator-devlake/plugins/helper"
+)
+
+var ExtractApiIssuesMeta = core.SubTaskMeta{
+ Name: "extractApiIssues",
+ EntryPoint: ExtractApiIssues,
+ EnabledByDefault: true,
+ Description: "Extract raw Issues data into tool layer table gitlab_issues",
+}
+
+type IssuesResponse struct {
+ ProjectId int `json:"id"`
+ Milestone struct {
+ Due_date string
+ Project_id int
+ State string
+ Description string
+ Iid int
+ Id int
+ Title string
+ CreatedAt helper.Iso8601Time
+ UpdatedAt helper.Iso8601Time
+ }
+ Author struct{
+ State string
+ WebUrl string
+ AvatarUrl string
+ Username string
+ Id int
+ Name string
+ }
+ Description string
+ State string
+ Iid int
+ Assignees []struct {
+ AvatarUrl string
+ WebUrl string
+ State string
+ Username string
+ Id int
+ Name string
+ }
+ Assignee *struct {
+ AvatarUrl string
+ WebUrl string
+ State string
+ Username string
+ Id int
+ Name string
+ }
+ Type string
+ Labels []string `json:"labels"`
+ UpVotes int
+ DownVotes int
+ MergeRequestsCount int
+ Id int
+ Title string
+ GitlabUpdatedAt helper.Iso8601Time `json:"updated_at"`
+ GitlabCreatedAt helper.Iso8601Time `json:"created_at"`
+ GitlabClosedAt *helper.Iso8601Time `json:"closed_at"`
+ ClosedBy struct{
+ State string
+ WebUrl string
+ AvatarUrl string
+ Username string
+ Id int
+ Name string
+ }
+ UserNotesCount int
+ DueDate helper.Iso8601Time
+ WebUrl string `json:"web_url"`
+ References struct {
+ Short string
+ Relative string
+ Full string
+ }
+ TimeStats struct {
+ TimeEstimate int
+ TotalTimeSpent int
Review Comment:
similar to above
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@devlake.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [incubator-devlake] warren830 commented on a diff in pull request #1968: collector draft
Posted by GitBox <gi...@apache.org>.
warren830 commented on code in PR #1968:
URL: https://github.com/apache/incubator-devlake/pull/1968#discussion_r882300851
##########
plugins/gitlab/tasks/issue_collector.go:
##########
@@ -0,0 +1,126 @@
+package tasks
+
+import (
+ "encoding/json"
+ "fmt"
+ "net/http"
+ "net/url"
+
+ "github.com/merico-dev/lake/plugins/helper"
+
+ "github.com/merico-dev/lake/plugins/core"
+ "github.com/merico-dev/lake/plugins/gitlab/models"
+)
+
+const RAW_ISSUE_TABLE = "gitlab_api_issues"
+
+//This Struct was declared in shared.go
+//type GitlabApiParams struct {
+// ProjectId int
+//}
+
+var CollectApiIssuesMeta = core.SubTaskMeta{
+ Name: "collectApiIssues",
+ EntryPoint: CollectApiIssues,
+ EnabledByDefault: true,
+ Description: "Collect issues data from Gitlab api",
+}
+
+func CollectApiIssues(taskCtx core.SubTaskContext) error {
+ db := taskCtx.GetDb()
+ data := taskCtx.GetData().(*GitlabTaskData)
+
+ since := data.Since
+ incremental := false
+ // user didn't specify a time range to sync, try load from database
+ if since == nil {
+ var latestUpdated models.GitlabIssue
+ err := db.Model(&latestUpdated).
+ Where("project_id = ?", data.Options.ProjectId).
+ Order("gitlab_updated_at DESC").Limit(1).Find(&latestUpdated).Error
+
+ if err != nil {
+ return fmt.Errorf("failed to get latest gitlab issue record: %w", err)
+ }
+ if latestUpdated.GitlabId > 0 {
+ since = &latestUpdated.GitlabUpdatedAt
Review Comment:
yes
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@devlake.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org