You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by ab...@apache.org on 2023/06/13 08:11:44 UTC
[incubator-devlake] branch main updated: feat: Jira dev panel regex APIs (#5423)
This is an automated email from the ASF dual-hosted git repository.
abeizn pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new db1c42669 feat: Jira dev panel regex APIs (#5423)
db1c42669 is described below
commit db1c42669cc2f4306141574d869910af1dd3206d
Author: Liang Zhang <li...@merico.dev>
AuthorDate: Tue Jun 13 16:11:39 2023 +0800
feat: Jira dev panel regex APIs (#5423)
* feat: Jira dev panel regex APIs
* fix: unit test
---
backend/plugins/jira/api/scope_config.go | 130 +++++++++++++++++
backend/plugins/jira/api/scope_config_test.go | 161 +++++++++++++++++++++
backend/plugins/jira/e2e/issue_repo_commit_test.go | 8 +-
backend/plugins/jira/e2e/remotelink_test.go | 7 +-
backend/plugins/jira/impl/impl.go | 6 +
.../20230609_clear_repo_pattern.go | 49 +++++++
.../jira/models/migrationscripts/register.go | 1 +
.../jira/tasks/issue_repo_commit_convertor.go | 2 +-
backend/plugins/jira/tasks/remotelink_extractor.go | 2 +-
backend/plugins/jira/tasks/task_data.go | 25 ++--
backend/plugins/jira/tasks/task_data_test.go | 4 +-
11 files changed, 376 insertions(+), 19 deletions(-)
diff --git a/backend/plugins/jira/api/scope_config.go b/backend/plugins/jira/api/scope_config.go
index e2746e2df..178c94a32 100644
--- a/backend/plugins/jira/api/scope_config.go
+++ b/backend/plugins/jira/api/scope_config.go
@@ -22,8 +22,10 @@ import (
"fmt"
"net/http"
"net/url"
+ "regexp"
"sort"
"strconv"
+ "strings"
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
@@ -34,6 +36,25 @@ import (
"github.com/mitchellh/mapstructure"
)
+type genRegexReq struct {
+ Pattern string `json:"pattern"`
+}
+
+type genRegexResp struct {
+ Regex string `json:"regex"`
+}
+
+type applyRegexReq struct {
+ Regex string `json:"regex"`
+ Urls []string `json:"urls"`
+}
+
+type repo struct {
+ Namespace string `json:"namespace"`
+ RepoName string `json:"repo_name"`
+ CommitSha string `json:"commit_sha"`
+}
+
// CreateScopeConfig create scope config for Jira
// @Summary create scope config for Jira
// @Description create scope config for Jira
@@ -313,3 +334,112 @@ func GetCommitsURLs(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput,
}
return &plugin.ApiResourceOutput{Body: commitURLs, Status: http.StatusOK}, nil
}
+
+// GenRegex generate regex from url
+// @Summary generate regex from url
+// @Description generate regex from url
+// @Tags plugins/jira
+// @Param generate-regex body genRegexReq true "generate regex"
+// @Success 200 {object} genRegexResp
+// @Failure 400 {object} shared.ApiBody "Bad Request"
+// @Failure 500 {object} shared.ApiBody "Internal Error"
+// @Router /plugins/jira/generate-regex [POST]
+func GenRegex(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
+ var req genRegexReq
+ err := api.Decode(input.Body, &req, nil)
+ if err != nil {
+ return nil, err
+ }
+ err = checkInput(req.Pattern)
+ if err != nil {
+ return nil, err
+ }
+ reg := genRegex(req.Pattern)
+ _, e := regexp.Compile(reg)
+ if e != nil {
+ return nil, errors.BadInput.Wrap(e, "invalid url")
+ }
+
+ return &plugin.ApiResourceOutput{Body: genRegexResp{Regex: reg}, Status: http.StatusOK}, nil
+}
+
+func checkInput(input string) errors.Error {
+ input = strings.TrimSpace(input)
+ if input == "" {
+ return errors.BadInput.New("empty input")
+ }
+ if !strings.Contains(input, "{namespace}") {
+ return errors.BadInput.New("missing {namespace}")
+ }
+ if !strings.Contains(input, "{repo_name}") {
+ return errors.BadInput.New("missing {repo_name}")
+ }
+ if !strings.Contains(input, "{commit_sha}") {
+ return errors.BadInput.New("missing {commit_sha}")
+ }
+ return nil
+}
+
+func genRegex(s string) string {
+ s = strings.TrimSpace(s)
+ s = strings.Replace(s, "{namespace}", `(?P<namespace>\S+)`, -1)
+ s = strings.Replace(s, "{repo_name}", `(?P<repo_name>\S+)`, -1)
+ s = strings.Replace(s, "{commit_sha}", `(?P<commit_sha>\w{40})`, -1)
+ return s
+}
+
+// ApplyRegex return parsed commits URLs
+// @Summary return parsed commits URLs
+// @Description return parsed commits URLs
+// @Tags plugins/jira
+// @Param apply-regex body applyRegexReq true "apply regex"
+// @Success 200 {object} []string
+// @Failure 400 {object} shared.ApiBody "Bad Request"
+// @Failure 500 {object} shared.ApiBody "Internal Error"
+// @Router /plugins/jira/apply-regex [POST]
+func ApplyRegex(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
+ var req applyRegexReq
+ err := api.Decode(input.Body, &req, nil)
+ if err != nil {
+ return nil, err
+ }
+ var repos []*repo
+ for _, u := range req.Urls {
+ r, e1 := applyRegex(req.Regex, u)
+ if e1 != nil {
+ return nil, err
+ }
+ repos = append(repos, r)
+ }
+ return &plugin.ApiResourceOutput{Body: repos, Status: http.StatusOK}, nil
+}
+
+func applyRegex(regexStr, commitUrl string) (*repo, errors.Error) {
+ pattern, e := regexp.Compile(regexStr)
+ if e != nil {
+ return nil, errors.BadInput.Wrap(e, "invalid regex")
+ }
+ if !pattern.MatchString(commitUrl) {
+ return nil, errors.BadInput.New("invalid url")
+ }
+ group := pattern.FindStringSubmatch(commitUrl)
+ if len(group) != 4 {
+ return nil, errors.BadInput.New("invalid group count")
+ }
+ r := new(repo)
+ for i, name := range pattern.SubexpNames() {
+ if i != 0 && name != "" {
+ switch name {
+ case "namespace":
+ r.Namespace = group[i]
+ case "repo_name":
+ r.RepoName = group[i]
+ case "commit_sha":
+ r.CommitSha = group[i]
+ default:
+ return nil, errors.BadInput.New("invalid group name")
+ }
+ }
+ }
+ return r, nil
+}
diff --git a/backend/plugins/jira/api/scope_config_test.go b/backend/plugins/jira/api/scope_config_test.go
new file mode 100644
index 000000000..faca407f1
--- /dev/null
+++ b/backend/plugins/jira/api/scope_config_test.go
@@ -0,0 +1,161 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package api
+
+import (
+ "testing"
+
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/stretchr/testify/assert"
+)
+
+func generateThenApplyRegex(pattern, commitUrl string) (*repo, errors.Error) {
+ reg := genRegex(pattern)
+ return applyRegex(reg, commitUrl)
+}
+
+func Test_genRegex(t *testing.T) {
+ type args struct {
+ url string
+ }
+ tests := []struct {
+ name string
+ args args
+ want string
+ }{
+ {
+ "test1",
+ args{
+ "https://gitlab.com/{namespace}/{repo_name}/-/commit/{commit_sha}",
+ },
+ `https://gitlab.com/(?P<namespace>\S+)/(?P<repo_name>\S+)/-/commit/(?P<commit_sha>\w{40})`,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ assert.Equalf(t, tt.want, genRegex(tt.args.url), "genRegex(%v)", tt.args.url)
+ })
+ }
+}
+
+func Test_applyRegex(t *testing.T) {
+ type args struct {
+ regexStr string
+ commitUrl string
+ }
+ tests := []struct {
+ name string
+ args args
+ want *repo
+ want1 errors.Error
+ }{
+ {
+ "test1",
+ args{
+ `https://gitlab.com/(?P<namespace>[^/]+)/(?P<repo_name>[^/]+)/-/commit/(?P<commit_sha>\w{40})`,
+ "https://gitlab.com/apache/incubator-devlake/-/commit/1234567890123456789012345678901234567890",
+ },
+ &repo{
+ "apache",
+ "incubator-devlake",
+ "1234567890123456789012345678901234567890",
+ },
+ nil,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got, got1 := applyRegex(tt.args.regexStr, tt.args.commitUrl)
+ assert.Equalf(t, tt.want, got, "applyRegex(%v, %v)", tt.args.regexStr, tt.args.commitUrl)
+ assert.Equalf(t, tt.want1, got1, "applyRegex(%v, %v)", tt.args.regexStr, tt.args.commitUrl)
+ })
+ }
+}
+
+func Test_generateThenApplyRegex(t *testing.T) {
+ type args struct {
+ pattern string
+ commitUrl string
+ }
+ tests := []struct {
+ name string
+ args args
+ want *repo
+ want1 errors.Error
+ }{
+ {
+ "test1",
+ args{
+ "https://gitlab.com/{namespace}/{repo_name}/-/commit/{commit_sha}",
+ "https://gitlab.com/apache/incubator-devlake/-/commit/1234567890123456789012345678901234567890",
+ },
+ &repo{
+ "apache",
+ "incubator-devlake",
+ "1234567890123456789012345678901234567890",
+ },
+ nil,
+ },
+ {
+ "test2",
+ args{
+ "https://bitbucket.org/{namespace}/{repo_name}/commits/{commit_sha}",
+ "https://bitbucket.org/mynamespace/incubator-devlake/commits/fef8d697fbb9a2b336be6fa2e2848f585c86a622",
+ },
+ &repo{
+ "mynamespace",
+ "incubator-devlake",
+ "fef8d697fbb9a2b336be6fa2e2848f585c86a622",
+ },
+ nil,
+ },
+ {
+ "test3",
+ args{
+ "https://example.com/bitbucket/projects/{namespace}/repos/{repo_name}/commits/{commit_sha}",
+ "https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/1e23e7f1a0cb539c7408c38e5a37de3bc836bc94",
+ },
+ &repo{
+ "PROJECTNAME",
+ "ui_jira",
+ "1e23e7f1a0cb539c7408c38e5a37de3bc836bc94",
+ },
+ nil,
+ },
+ {
+ "test4",
+ args{
+ "https://gitlab.com/{namespace}/{repo_name}/commits/{commit_sha}",
+ "https://gitlab.com/namespace1/namespace2/myrepo/commits/050baf4575caf069275f5fa14db9ad4a21a79883",
+ },
+ &repo{
+ "namespace1/namespace2",
+ "myrepo",
+ "050baf4575caf069275f5fa14db9ad4a21a79883",
+ },
+ nil,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got, got1 := generateThenApplyRegex(tt.args.pattern, tt.args.commitUrl)
+ assert.Equalf(t, tt.want, got, "generateThenApplyRegex(%v, %v)", tt.args.pattern, tt.args.commitUrl)
+ assert.Equalf(t, tt.want1, got1, "generateThenApplyRegex(%v, %v)", tt.args.pattern, tt.args.commitUrl)
+ })
+ }
+}
diff --git a/backend/plugins/jira/e2e/issue_repo_commit_test.go b/backend/plugins/jira/e2e/issue_repo_commit_test.go
index 4a5f3e0df..c060fcf12 100644
--- a/backend/plugins/jira/e2e/issue_repo_commit_test.go
+++ b/backend/plugins/jira/e2e/issue_repo_commit_test.go
@@ -37,10 +37,10 @@ func TestConvertIssueRepoCommitsDataFlow(t *testing.T) {
BoardId: 8,
ScopeConfig: &tasks.JiraScopeConfig{
RemotelinkCommitShaPattern: `.*/commit/(.*)`,
- RemotelinkRepoPattern: []string{
- `https://bitbucket.org/(?P<namespace>[^/]+)/(?P<repo_name>[^/]+)/commits/(?P<commit_sha>\w{40})`,
- `https://gitlab.com/(?P<namespace>\S+)/(?P<repo_name>\S+)/-/commit/(?P<commit_sha>\w{40})`,
- `https://github.com/(?P<namespace>[^/]+)/(?P<repo_name>[^/]+)/commit/(?P<commit_sha>\w{40})`,
+ RemotelinkRepoPattern: []tasks.CommitUrlPattern{
+ {"", `https://bitbucket.org/(?P<namespace>[^/]+)/(?P<repo_name>[^/]+)/commits/(?P<commit_sha>\w{40})`},
+ {"", `https://gitlab.com/(?P<namespace>\S+)/(?P<repo_name>\S+)/-/commit/(?P<commit_sha>\w{40})`},
+ {"", `https://github.com/(?P<namespace>[^/]+)/(?P<repo_name>[^/]+)/commit/(?P<commit_sha>\w{40})`},
},
},
},
diff --git a/backend/plugins/jira/e2e/remotelink_test.go b/backend/plugins/jira/e2e/remotelink_test.go
index 5192e4b97..ee773f74f 100644
--- a/backend/plugins/jira/e2e/remotelink_test.go
+++ b/backend/plugins/jira/e2e/remotelink_test.go
@@ -36,7 +36,12 @@ func TestRemotelinkDataFlow(t *testing.T) {
BoardId: 8,
ScopeConfig: &tasks.JiraScopeConfig{
RemotelinkCommitShaPattern: ".*/commit/(.*)",
- RemotelinkRepoPattern: []string{`https://example.com/(?P<namespace>\S+)/(?P<repo_name>\S+)/-/commits/(?P<commit_sha>\w{40})`},
+ RemotelinkRepoPattern: []tasks.CommitUrlPattern{
+ {
+ Pattern: "",
+ Regex: `https://example.com/(?P<namespace>\S+)/(?P<repo_name>\S+)/-/commits/(?P<commit_sha>\w{40})`,
+ },
+ },
},
},
}
diff --git a/backend/plugins/jira/impl/impl.go b/backend/plugins/jira/impl/impl.go
index 933d0c0cf..4ced7222b 100644
--- a/backend/plugins/jira/impl/impl.go
+++ b/backend/plugins/jira/impl/impl.go
@@ -311,6 +311,12 @@ func (p Jira) ApiResources() map[string]map[string]plugin.ApiResourceHandler {
"connections/:connectionId/dev-panel-commits": {
"GET": api.GetCommitsURLs,
},
+ "generate-regex": {
+ "POST": api.GenRegex,
+ },
+ "apply-regex": {
+ "POST": api.ApplyRegex,
+ },
}
}
diff --git a/backend/plugins/jira/models/migrationscripts/20230609_clear_repo_pattern.go b/backend/plugins/jira/models/migrationscripts/20230609_clear_repo_pattern.go
new file mode 100644
index 000000000..c1d4a41c4
--- /dev/null
+++ b/backend/plugins/jira/models/migrationscripts/20230609_clear_repo_pattern.go
@@ -0,0 +1,49 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+ "encoding/json"
+
+ "github.com/apache/incubator-devlake/core/context"
+ "github.com/apache/incubator-devlake/core/dal"
+ "github.com/apache/incubator-devlake/core/errors"
+)
+
+type scopeConfig20230609 struct {
+ RemotelinkRepoPattern json.RawMessage `mapstructure:"remotelinkRepoPattern,omitempty" json:"remotelinkRepoPattern"`
+}
+
+func (r scopeConfig20230609) TableName() string {
+ return "_tool_jira_scope_configs"
+}
+
+type clearRepoPattern struct{}
+
+func (script *clearRepoPattern) Up(basicRes context.BasicRes) errors.Error {
+ db := basicRes.GetDal()
+ return db.UpdateColumn(&scopeConfig20230609{}, "remotelink_repo_pattern", []byte{}, dal.Where("1=1"))
+}
+
+func (*clearRepoPattern) Version() uint64 {
+ return 20230609093856
+}
+
+func (*clearRepoPattern) Name() string {
+ return "clear the value of _tool_jira_scope_configs.remotelink_repo_pattern"
+}
diff --git a/backend/plugins/jira/models/migrationscripts/register.go b/backend/plugins/jira/models/migrationscripts/register.go
index d7f3d095f..56a882e60 100644
--- a/backend/plugins/jira/models/migrationscripts/register.go
+++ b/backend/plugins/jira/models/migrationscripts/register.go
@@ -40,5 +40,6 @@ func All() []plugin.MigrationScript {
new(renameTr2ScopeConfig),
new(addRepoUrl),
new(addApplicationType),
+ new(clearRepoPattern),
}
}
diff --git a/backend/plugins/jira/tasks/issue_repo_commit_convertor.go b/backend/plugins/jira/tasks/issue_repo_commit_convertor.go
index c62892d55..afe150e96 100644
--- a/backend/plugins/jira/tasks/issue_repo_commit_convertor.go
+++ b/backend/plugins/jira/tasks/issue_repo_commit_convertor.go
@@ -56,7 +56,7 @@ func ConvertIssueRepoCommits(taskCtx plugin.SubTaskContext) errors.Error {
var commitRepoUrlRegexps []*regexp.Regexp
if sc := data.Options.ScopeConfig; sc != nil {
for _, s := range sc.RemotelinkRepoPattern {
- pattern, e := regexp.Compile(s)
+ pattern, e := regexp.Compile(s.Regex)
if e != nil {
return errors.Convert(e)
}
diff --git a/backend/plugins/jira/tasks/remotelink_extractor.go b/backend/plugins/jira/tasks/remotelink_extractor.go
index 2c2367409..d2585ac48 100644
--- a/backend/plugins/jira/tasks/remotelink_extractor.go
+++ b/backend/plugins/jira/tasks/remotelink_extractor.go
@@ -60,7 +60,7 @@ func ExtractRemotelinks(taskCtx plugin.SubTaskContext) errors.Error {
var commitRepoUrlRegexps []*regexp.Regexp
if sc := data.Options.ScopeConfig; sc != nil {
for _, s := range sc.RemotelinkRepoPattern {
- pattern, e := regexp.Compile(s)
+ pattern, e := regexp.Compile(s.Regex)
if e != nil {
return errors.Convert(e)
}
diff --git a/backend/plugins/jira/tasks/task_data.go b/backend/plugins/jira/tasks/task_data.go
index 68a3f67c5..5bb3eb79a 100644
--- a/backend/plugins/jira/tasks/task_data.go
+++ b/backend/plugins/jira/tasks/task_data.go
@@ -38,18 +38,23 @@ type TypeMapping struct {
StatusMappings StatusMappings `json:"statusMappings"`
}
+type CommitUrlPattern struct {
+ Pattern string `json:"pattern"`
+ Regex string `json:"regex"`
+}
+
type TypeMappings map[string]TypeMapping
type JiraScopeConfig struct {
- Entities []string `json:"entities"`
- ConnectionId uint64 `mapstructure:"connectionId" json:"connectionId"`
- Name string `gorm:"type:varchar(255)" validate:"required"`
- EpicKeyField string `json:"epicKeyField"`
- StoryPointField string `json:"storyPointField"`
- RemotelinkCommitShaPattern string `json:"remotelinkCommitShaPattern"`
- RemotelinkRepoPattern []string `json:"remotelinkRepoPattern"`
- TypeMappings TypeMappings `json:"typeMappings"`
- ApplicationType string `json:"applicationType"`
+ Entities []string `json:"entities"`
+ ConnectionId uint64 `mapstructure:"connectionId" json:"connectionId"`
+ Name string `gorm:"type:varchar(255)" validate:"required"`
+ EpicKeyField string `json:"epicKeyField"`
+ StoryPointField string `json:"storyPointField"`
+ RemotelinkCommitShaPattern string `json:"remotelinkCommitShaPattern"`
+ RemotelinkRepoPattern []CommitUrlPattern `json:"remotelinkRepoPattern"`
+ TypeMappings TypeMappings `json:"typeMappings"`
+ ApplicationType string `json:"applicationType"`
}
func (r *JiraScopeConfig) ToDb() (*models.JiraScopeConfig, errors.Error) {
@@ -90,7 +95,7 @@ func MakeScopeConfig(rule models.JiraScopeConfig) (*JiraScopeConfig, errors.Erro
return nil, errors.Default.Wrap(err, "unable to unmarshal the typeMapping")
}
}
- var remotelinkRepoPattern []string
+ var remotelinkRepoPattern []CommitUrlPattern
if len(rule.RemotelinkRepoPattern) > 0 {
err = json.Unmarshal(rule.RemotelinkRepoPattern, &remotelinkRepoPattern)
if err != nil {
diff --git a/backend/plugins/jira/tasks/task_data_test.go b/backend/plugins/jira/tasks/task_data_test.go
index 04088ac5b..3e546f566 100644
--- a/backend/plugins/jira/tasks/task_data_test.go
+++ b/backend/plugins/jira/tasks/task_data_test.go
@@ -41,7 +41,7 @@ func TestMakeScopeConfigs(t *testing.T) {
EpicKeyField: "epic",
StoryPointField: "story",
RemotelinkCommitShaPattern: "commit sha pattern",
- RemotelinkRepoPattern: []byte(`["abc","efg"]`),
+ RemotelinkRepoPattern: []byte(`[{"pattern":"","regex":"abc"},{"pattern":"","regex":"efg"}]`),
TypeMappings: []byte(`{"10040":{"standardType":"Incident","statusMappings":null}}`),
}},
&JiraScopeConfig{
@@ -49,7 +49,7 @@ func TestMakeScopeConfigs(t *testing.T) {
EpicKeyField: "epic",
StoryPointField: "story",
RemotelinkCommitShaPattern: "commit sha pattern",
- RemotelinkRepoPattern: []string{"abc", "efg"},
+ RemotelinkRepoPattern: []CommitUrlPattern{{"", "abc"}, {"", "efg"}},
TypeMappings: map[string]TypeMapping{"10040": {
StandardType: "Incident",
StatusMappings: nil,