You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by na...@apache.org on 2025/03/04 06:31:43 UTC
(incubator-devlake) branch feat/due_date updated: feat: support customize due date field in jira plugin
This is an automated email from the ASF dual-hosted git repository.
narro pushed a commit to branch feat/due_date
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/feat/due_date by this push:
new d4e798a9f feat: support customize due date field in jira plugin
d4e798a9f is described below
commit d4e798a9f8cb450f06e7d82246751aa28afcbd2c
Author: narro wizard <co...@merico.dev>
AuthorDate: Tue Mar 4 14:31:30 2025 +0800
feat: support customize due date field in jira plugin
#8315
---
backend/core/models/common/iso8601time.go | 10 ++
backend/core/models/common/iso8601time_test.go | 82 ++++++++++++
backend/plugins/jira/e2e/issue_test.go | 139 +++++++++++++++++++++
.../_raw_jira_api_issues_for_due_date.csv | 2 +
.../_tool_jira_issues_for_due_date.csv | 2 +
.../e2e/snapshot_tables/issues_for_due_date.csv | 2 +
backend/plugins/jira/models/scope_config.go | 1 +
backend/plugins/jira/tasks/apiv2models/issue.go | 12 --
backend/plugins/jira/tasks/issue_extractor.go | 30 ++++-
9 files changed, 267 insertions(+), 13 deletions(-)
diff --git a/backend/core/models/common/iso8601time.go b/backend/core/models/common/iso8601time.go
index 512a22070..63f9084a2 100644
--- a/backend/core/models/common/iso8601time.go
+++ b/backend/core/models/common/iso8601time.go
@@ -138,6 +138,16 @@ func ConvertStringToTime(timeString string) (t time.Time, err error) {
return time.Parse(time.RFC3339, timeString)
}
+// ConvertStringToTimeInTz
+func ConvertStringToTimeInLoc(timeString string, loc *time.Location) (t time.Time, err error) {
+ for _, formatItem := range DateTimeFormats {
+ if formatItem.Matcher.MatchString(timeString) {
+ return time.ParseInLocation(formatItem.Format, timeString, loc)
+ }
+ }
+ return time.ParseInLocation(time.RFC3339, timeString, loc)
+}
+
// Iso8601TimeToTime FIXME ...
func Iso8601TimeToTime(iso8601Time *Iso8601Time) *time.Time {
if iso8601Time == nil {
diff --git a/backend/core/models/common/iso8601time_test.go b/backend/core/models/common/iso8601time_test.go
index 380c4b9f1..5a6448a76 100644
--- a/backend/core/models/common/iso8601time_test.go
+++ b/backend/core/models/common/iso8601time_test.go
@@ -124,3 +124,85 @@ func TestIso8601Time_Scan(t *testing.T) {
})
}
}
+
+func TestConvertStringToTime(t *testing.T) {
+ testCases := []struct {
+ name string
+ input string
+ output time.Time
+ err error
+ }{
+ {
+ name: "Valid time string",
+ input: "2023-03-01T12:30:00+0000",
+ output: time.Date(2023, 3, 1, 12, 30, 0, 0, time.UTC).Local(),
+ err: nil,
+ },
+ {
+ name: "Valid date string",
+ input: "2023-03-01",
+ output: time.Date(2023, 3, 1, 0, 0, 0, 0, time.UTC),
+ err: nil,
+ },
+ {
+ name: "Invalid time string",
+ input: "invalid",
+ output: time.Time{},
+ err: fmt.Errorf("parsing time \"invalid\" as \"2006-01-02T15:04:05Z07:00\": cannot parse \"invalid\" as \"2006\""),
+ },
+ }
+ for _, tc := range testCases {
+ t.Run(tc.name, func(t *testing.T) {
+ output, err := ConvertStringToTime(tc.input)
+ if !reflect.DeepEqual(tc.output, output) {
+ t.Errorf("Expected output to be %v, but got %v", tc.output, output)
+ }
+ assert.Equal(t, fmt.Sprintf("%v", err), fmt.Sprintf("%v", tc.err), "Expected error to be %v, but got %v", tc.err, err)
+ })
+ }
+}
+
+func TestConvertStringToTimeInLoc(t *testing.T) {
+ loc, err := time.LoadLocation("Asia/Shanghai")
+ if err != nil {
+ t.Fatalf("Failed to load location: %v", err)
+ }
+ testCases := []struct {
+ name string
+ input string
+ loc *time.Location
+ output time.Time
+ err error
+ }{
+ {
+ name: "Valid time string with location",
+ input: "2023-03-01T12:30:00+0800",
+ loc: loc,
+ output: time.Date(2023, 3, 1, 12, 30, 0, 0, loc),
+ err: nil,
+ },
+ {
+ name: "Valid date string with location",
+ input: "2023-03-01",
+ loc: loc,
+ output: time.Date(2023, 3, 1, 0, 0, 0, 0, loc),
+ err: nil,
+ },
+ {
+ name: "Invalid time string with location",
+ input: "invalid",
+ loc: loc,
+ output: time.Time{},
+ err: fmt.Errorf("parsing time \"invalid\" as \"2006-01-02T15:04:05Z07:00\": cannot parse \"invalid\" as \"2006\""),
+ },
+ }
+ for _, tc := range testCases {
+ t.Run(tc.name, func(t *testing.T) {
+ output, err := ConvertStringToTimeInLoc(tc.input, tc.loc)
+ if !reflect.DeepEqual(tc.output, output) {
+ t.Errorf("Expected output to be %v, but got %v", tc.output, output)
+ }
+ assert.Equal(t, fmt.Sprintf("%v", err), fmt.Sprintf("%v", tc.err), "Expected error to be %v, but got %v", tc.err, err)
+ })
+ }
+}
diff --git a/backend/plugins/jira/e2e/issue_test.go b/backend/plugins/jira/e2e/issue_test.go
index 8037c54de..5d50b6e04 100644
--- a/backend/plugins/jira/e2e/issue_test.go
+++ b/backend/plugins/jira/e2e/issue_test.go
@@ -250,3 +250,142 @@ func TestIssueDataFlow(t *testing.T) {
IgnoreTypes: []interface{}{common.NoPKModel{}},
})
}
+
+func TestIssueCustomizeDueDate(t *testing.T) {
+ var plugin impl.Jira
+ dataflowTester := e2ehelper.NewDataFlowTester(t, "jira", plugin)
+
+ taskData := &tasks.JiraTaskData{
+ Options: &tasks.JiraOptions{
+ ConnectionId: 2,
+ BoardId: 8,
+ ScopeConfig: &models.JiraScopeConfig{
+ StoryPointField: "customfield_10024",
+ DueDateField: "customfield_10003",
+ TypeMappings: map[string]models.TypeMapping{
+ "子任务": {
+ StandardType: "Sub-task",
+ StatusMappings: map[string]models.StatusMapping{
+ "done": {StandardStatus: "你好世界"},
+ "new": {StandardStatus: "\u6069\u5E95\u6EF4\u68AF\u6B38\u592B\u5178\u4EA2\u59C6"},
+ },
+ },
+ "任务": {
+ StandardType: "Task",
+ StatusMappings: map[string]models.StatusMapping{
+ "done": {StandardStatus: "hello world"},
+ "new": {StandardStatus: "110 100 100 116 102 46 99 111 109"},
+ },
+ },
+ // issueType "Test Execution" in raw_data and not fill here to test issueType not be defined
+ },
+ },
+ },
+ }
+ // import raw data table
+ dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_jira_api_issues_for_due_date.csv", "_raw_jira_api_issues")
+ dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_jira_api_issue_types.csv", "_raw_jira_api_issue_types")
+
+ // verify issue extraction
+ dataflowTester.FlushTabler(&models.JiraIssue{})
+ dataflowTester.FlushTabler(&models.JiraBoardIssue{})
+ dataflowTester.FlushTabler(&models.JiraSprintIssue{})
+ dataflowTester.FlushTabler(&models.JiraIssueComment{})
+ dataflowTester.FlushTabler(&models.JiraIssueChangelogs{})
+ dataflowTester.FlushTabler(&models.JiraIssueChangelogItems{})
+ dataflowTester.FlushTabler(&models.JiraWorklog{})
+ dataflowTester.FlushTabler(&models.JiraAccount{})
+ dataflowTester.FlushTabler(&models.JiraIssueType{})
+ dataflowTester.FlushTabler(&models.JiraIssueLabel{})
+ dataflowTester.FlushTabler(&models.JiraIssueField{})
+ dataflowTester.Subtask(tasks.ExtractIssueTypesMeta, taskData)
+ dataflowTester.Subtask(tasks.ExtractIssuesMeta, taskData)
+
+ dataflowTester.VerifyTable(
+ models.JiraIssue{},
+ "./snapshot_tables/_tool_jira_issues_for_due_date.csv",
+ e2ehelper.ColumnWithRawData(
+ "connection_id",
+ "issue_id",
+ "project_id",
+ "project_name",
+ "self",
+ "issue_key",
+ "summary",
+ "description",
+ "type",
+ "epic_key",
+ "status_name",
+ "status_key",
+ "story_point",
+ "original_estimate_minutes",
+ "aggregate_estimate_minutes",
+ "remaining_estimate_minutes",
+ "creator_account_id",
+ "creator_account_type",
+ "creator_display_name",
+ "assignee_account_id",
+ "assignee_account_type",
+ "assignee_display_name",
+ "priority_id",
+ "priority_name",
+ "parent_id",
+ "parent_key",
+ "sprint_id",
+ "sprint_name",
+ "resolution_date",
+ "created",
+ "updated",
+ "spent_minutes",
+ "lead_time_minutes",
+ "std_type",
+ "std_status",
+ "icon_url",
+ "changelog_total",
+ "comment_total",
+ "due_date",
+ ),
+ )
+
+ // verify issue conversion
+ dataflowTester.FlushTabler(&ticket.Issue{})
+ dataflowTester.FlushTabler(&ticket.BoardIssue{})
+ dataflowTester.FlushTabler(&ticket.IssueAssignee{})
+ dataflowTester.Subtask(tasks.ConvertIssuesMeta, taskData)
+ dataflowTester.VerifyTable(
+ ticket.Issue{},
+ "./snapshot_tables/issues_for_due_date.csv",
+ []string{
+ "id",
+ "url",
+ "icon_url",
+ "issue_key",
+ "title",
+ "description",
+ "epic_key",
+ "type",
+ "original_type",
+ "status",
+ "original_status",
+ "story_point",
+ "resolution_date",
+ "created_date",
+ "updated_date",
+ "lead_time_minutes",
+ "parent_issue_id",
+ "priority",
+ "original_estimate_minutes",
+ "time_spent_minutes",
+ "time_remaining_minutes",
+ "creator_id",
+ "creator_name",
+ "assignee_id",
+ "assignee_name",
+ "severity",
+ "component",
+ "original_project",
+ "due_date",
+ },
+ )
+
+}
diff --git a/backend/plugins/jira/e2e/raw_tables/_raw_jira_api_issues_for_due_date.csv b/backend/plugins/jira/e2e/raw_tables/_raw_jira_api_issues_for_due_date.csv
new file mode 100644
index 000000000..b8e9dd466
--- /dev/null
+++ b/backend/plugins/jira/e2e/raw_tables/_raw_jira_api_issues_for_due_date.csv
@@ -0,0 +1,2 @@
+"id","params","data","url","input","created_at"
+12441,"{""ConnectionId"":2,""BoardId"":8}","{""id"": ""10063"", ""key"": ""EE-1"", ""self"": ""https://merico.atlassian.net/rest/agile/1.0/issue/10063"", ""expand"": ""operations,versionedRepresentations,editmeta,changelog,renderedFields"", ""fields"": {""epic"": null, ""votes"": {""self"": ""https://merico.atlassian.net/rest/api/2/issue/EE-1/votes"", ""votes"": 0, ""hasVoted"": false}, ""labels"": [""frontEnd"",""Saas""], ""sprint"": null, ""status"": {""id"": ""10068"", ""name"": ""已完成 [...]
\ No newline at end of file
diff --git a/backend/plugins/jira/e2e/snapshot_tables/_tool_jira_issues_for_due_date.csv b/backend/plugins/jira/e2e/snapshot_tables/_tool_jira_issues_for_due_date.csv
new file mode 100644
index 000000000..e96bc0b2c
--- /dev/null
+++ b/backend/plugins/jira/e2e/snapshot_tables/_tool_jira_issues_for_due_date.csv
@@ -0,0 +1,2 @@
+connection_id,issue_id,project_id,project_name,self,issue_key,summary,description,type,epic_key,status_name,status_key,story_point,original_estimate_minutes,aggregate_estimate_minutes,remaining_estimate_minutes,creator_account_id,creator_account_type,creator_display_name,assignee_account_id,assignee_account_type,assignee_display_name,priority_id,priority_name,parent_id,parent_key,sprint_id,sprint_name,resolution_date,created,updated,spent_minutes,lead_time_minutes,std_type,std_status,ico [...]
+2,10063,10003,Enterprise Edition,https://merico.atlassian.net/rest/agile/1.0/issue/10063,EE-1,四个排序图：测试/注释覆盖度、复用度、模块性,,故事,,已完成,new,-1,,1260,0,5e9711ba34f7b90c0fbc37d3,,Rankin Zheng,5ecfbd0c730ec90c1999cadf,,Dingding Zhang,3,Medium,0,,0,,2020-06-19T06:31:18.495+00:00,2020-06-12T00:13:13.360+00:00,2021-03-28T08:06:08.713+00:00,,10458,故事,TODO,https://merico.atlassian.net/rest/api/2/universal_avatar/view/type/issuetype/avatar/10315?size=medium,25,1,2025-03-03T16:00:00.000+00:00,"{""Connectio [...]
diff --git a/backend/plugins/jira/e2e/snapshot_tables/issues_for_due_date.csv b/backend/plugins/jira/e2e/snapshot_tables/issues_for_due_date.csv
new file mode 100644
index 000000000..11e907816
--- /dev/null
+++ b/backend/plugins/jira/e2e/snapshot_tables/issues_for_due_date.csv
@@ -0,0 +1,2 @@
+id,url,icon_url,issue_key,title,description,epic_key,type,original_type,status,original_status,story_point,resolution_date,created_date,updated_date,lead_time_minutes,parent_issue_id,priority,original_estimate_minutes,time_spent_minutes,time_remaining_minutes,creator_id,creator_name,assignee_id,assignee_name,severity,component,original_project,due_date
+jira:JiraIssue:2:10063,https://merico.atlassian.net/browse/EE-1,https://merico.atlassian.net/rest/api/2/universal_avatar/view/type/issuetype/avatar/10315?size=medium,EE-1,四个排序图：测试/注释覆盖度、复用度、模块性,,,故事,故事,TODO,已完成,-1,2020-06-19T06:31:18.495+00:00,2020-06-12T00:13:13.360+00:00,2021-03-28T08:06:08.713+00:00,10458,,Medium,,,,jira:JiraAccount:2:5e9711ba34f7b90c0fbc37d3,Rankin Zheng,jira:JiraAccount:2:5ecfbd0c730ec90c1999cadf,Dingding Zhang,,,Enterprise Edition,2025-03-03T16:00:00.000+00:00
diff --git a/backend/plugins/jira/models/scope_config.go b/backend/plugins/jira/models/scope_config.go
index bb267a0f6..c64cd1eb0 100644
--- a/backend/plugins/jira/models/scope_config.go
+++ b/backend/plugins/jira/models/scope_config.go
@@ -44,6 +44,7 @@ type JiraScopeConfig struct {
common.ScopeConfig `mapstructure:",squash" json:",inline" gorm:"embedded"`
EpicKeyField string `mapstructure:"epicKeyField,omitempty" json:"epicKeyField" gorm:"type:varchar(255)"`
StoryPointField string `mapstructure:"storyPointField,omitempty" json:"storyPointField" gorm:"type:varchar(255)"`
+ DueDateField string `mapstructure:"dueDateField,omitempty" json:"dueDateField" gorm:"type:varchar(255)"`
RemotelinkCommitShaPattern string `mapstructure:"remotelinkCommitShaPattern,omitempty" json:"remotelinkCommitShaPattern" gorm:"type:varchar(255)"`
RemotelinkRepoPattern []CommitUrlPattern `mapstructure:"remotelinkRepoPattern,omitempty" json:"remotelinkRepoPattern" gorm:"type:json;serializer:json"`
TypeMappings map[string]TypeMapping `mapstructure:"typeMappings,omitempty" json:"typeMappings" gorm:"type:json;serializer:json"`
diff --git a/backend/plugins/jira/tasks/apiv2models/issue.go b/backend/plugins/jira/tasks/apiv2models/issue.go
index 1f33841fd..c62183d0f 100644
--- a/backend/plugins/jira/tasks/apiv2models/issue.go
+++ b/backend/plugins/jira/tasks/apiv2models/issue.go
@@ -19,7 +19,6 @@ package apiv2models
import (
"encoding/json"
- "strings"
"time"
"github.com/apache/incubator-devlake/core/errors"
@@ -279,17 +278,6 @@ func (i Issue) toToolLayer(connectionId uint64) *models.JiraIssue {
temp := *i.Fields.Timespent / 60
result.SpentMinutes = &temp
}
- if i.Fields.Duedate != "" && i.Fields.Duedate != "null" {
- // get timezone from i.Fields.Created
- duedateStr := strings.Trim(i.Fields.Duedate, "\"")
- // parse due date to time.Time
- if i.Fields.Created != nil {
- loc := i.Fields.Created.ToTime().Location()
- if t, err := time.ParseInLocation("2006-01-02", duedateStr, loc); err == nil {
- result.DueDate = &t
- }
- }
- }
return result
}
diff --git a/backend/plugins/jira/tasks/issue_extractor.go b/backend/plugins/jira/tasks/issue_extractor.go
index 11277a0a4..0fd71648a 100644
--- a/backend/plugins/jira/tasks/issue_extractor.go
+++ b/backend/plugins/jira/tasks/issue_extractor.go
@@ -18,12 +18,14 @@ limitations under the License.
package tasks
import (
+ "regexp"
"strconv"
"strings"
"time"
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/models/common"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/jira/models"
@@ -143,7 +145,33 @@ func extractIssues(data *JiraTaskData, mappings *typeMappings, apiIssue *apiv2mo
}
}
-
+ // default due date field is "duedate"
+ dueDateField := "duedate"
+ if data.Options.ScopeConfig != nil && data.Options.ScopeConfig.DueDateField != "" {
+ dueDateField = data.Options.ScopeConfig.DueDateField
+ }
+ unknownDueDate := apiIssue.Fields.AllFields[dueDateField]
+ switch dd := unknownDueDate.(type) {
+ case string:
+ // string, try to parse
+ if dd == "" || dd == "null" {
+ break
+ }
+ // use loc of issue.Created if dd is of type date (yyyy-MM-dd)
+ isDateStr, _ := regexp.Match(`\d{4}-\d{2}-\d{2}`, []byte(dd))
+ var temp time.Time
+ if isDateStr {
+ temp, _ = common.ConvertStringToTimeInLoc(dd, issue.Created.Location())
+ } else {
+ temp, _ = common.ConvertStringToTime(dd)
+ }
+ issue.DueDate = &temp
+ case nil:
+ default:
+ // not string, convert to time.Time, ignore it if failed
+ temp, _ := dd.(time.Time)
+ issue.DueDate = &temp
+ }
// code in next line will set issue.Type to issueType.Name
issue.Type = mappings.TypeIdMappings[issue.Type]
issue.StdType = mappings.StdTypeMappings[issue.Type]