You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by kl...@apache.org on 2022/11/29 09:44:25 UTC
[incubator-devlake] branch main updated: feat: github support scope and transformation rule (#3803)
This is an automated email from the ASF dual-hosted git repository.
klesh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new edd3510c7 feat: github support scope and transformation rule (#3803)
edd3510c7 is described below
commit edd3510c78216ef83dafe49e24b4b292221706cd
Author: mindlesscloud <li...@merico.dev>
AuthorDate: Tue Nov 29 17:44:20 2022 +0800
feat: github support scope and transformation rule (#3803)
* feat: github support scope and transformation rule
* fix: fix e2e test
* fix: implement core.PluginSource
* feat: pagenation support add refdiff_rule to _tool_github_transformation_rules
* fix: remove scopeId from _tool_jira_boards
---
plugins/github/api/scope.go | 164 +++++++++++++++++++++
plugins/github/api/transformation_rule.go | 128 ++++++++++++++++
.../migrationscripts/register.go => api/utils.go} | 37 +++--
plugins/github/e2e/cicd_test.go | 7 +-
plugins/github/e2e/comment_test.go | 7 +-
plugins/github/e2e/issue_test.go | 7 +-
plugins/github/e2e/milestone_test.go | 2 +-
plugins/github/e2e/pr_enrich_issue_test.go | 6 +-
plugins/github/e2e/pr_test.go | 7 +-
plugins/github/e2e/repo_test.go | 11 +-
plugins/github/impl/impl.go | 45 +++++-
plugins/github/models/connection.go | 13 --
.../20221124_add_trasformation_rule_table.go} | 23 ++-
.../archived/transformation_rules.go | 43 ++++++
plugins/github/models/migrationscripts/register.go | 1 +
plugins/github/models/repo.go | 25 ++--
plugins/github/models/transformation_rule.go | 43 ++++++
plugins/github/tasks/issue_extractor.go | 2 +-
plugins/github/tasks/repo_extractor.go | 1 -
plugins/github/tasks/task_data.go | 47 ++----
plugins/jira/api/scope.go | 3 -
plugins/jira/impl/impl.go | 3 +
plugins/jira/models/board.go | 1 -
.../20221116_add_trasformation_rule_table.go | 1 -
.../archived/transformation_rules.go | 4 +
plugins/jira/models/transformation_rules.go | 4 +
26 files changed, 516 insertions(+), 119 deletions(-)
diff --git a/plugins/github/api/scope.go b/plugins/github/api/scope.go
new file mode 100644
index 000000000..0bafb01a9
--- /dev/null
+++ b/plugins/github/api/scope.go
@@ -0,0 +1,164 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package api
+
+import (
+ "net/http"
+ "strconv"
+
+ "github.com/apache/incubator-devlake/errors"
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/core/dal"
+ "github.com/apache/incubator-devlake/plugins/github/models"
+ "github.com/apache/incubator-devlake/plugins/helper"
+ "github.com/mitchellh/mapstructure"
+)
+
+// PutScope create or update github repo
+// @Summary create or update github repo
+// @Description Create or update github repo
+// @Tags plugins/github
+// @Accept application/json
+// @Param connectionId path int true "connection ID"
+// @Param repoId path int true "repo ID"
+// @Param scope body models.GithubRepo true "json"
+// @Success 200 {object} models.GithubRepo
+// @Failure 400 {object} shared.ApiBody "Bad Request"
+// @Failure 500 {object} shared.ApiBody "Internal Error"
+// @Router /plugins/github/connections/{connectionId}/scopes/{repoId} [PUT]
+func PutScope(input *core.ApiResourceInput) (*core.ApiResourceOutput, errors.Error) {
+ connectionId, repoId := extractParam(input.Params)
+ if connectionId*repoId == 0 {
+ return nil, errors.BadInput.New("invalid connectionId or repoId")
+ }
+ var repo models.GithubRepo
+ err := errors.Convert(mapstructure.Decode(input.Body, &repo))
+ if err != nil {
+ return nil, errors.BadInput.Wrap(err, "decoding Github repo error")
+ }
+ err = verifyRepo(&repo)
+ if err != nil {
+ return nil, err
+ }
+ err = basicRes.GetDal().CreateOrUpdate(repo)
+ if err != nil {
+ return nil, errors.Default.Wrap(err, "error on saving GithubRepo")
+ }
+ return &core.ApiResourceOutput{Body: repo, Status: http.StatusOK}, nil
+}
+
+// UpdateScope patch to github repo
+// @Summary patch to github repo
+// @Description patch to github repo
+// @Tags plugins/github
+// @Accept application/json
+// @Param connectionId path int true "connection ID"
+// @Param repoId path int true "repo ID"
+// @Param scope body models.GithubRepo true "json"
+// @Success 200 {object} models.GithubRepo
+// @Failure 400 {object} shared.ApiBody "Bad Request"
+// @Failure 500 {object} shared.ApiBody "Internal Error"
+// @Router /plugins/github/connections/{connectionId}/scopes/{repoId} [PATCH]
+func UpdateScope(input *core.ApiResourceInput) (*core.ApiResourceOutput, errors.Error) {
+ connectionId, repoId := extractParam(input.Params)
+ if connectionId*repoId == 0 {
+ return nil, errors.BadInput.New("invalid connectionId or repoId")
+ }
+ var repo models.GithubRepo
+ err := basicRes.GetDal().First(&repo, dal.Where("connection_id = ? AND github_id = ?", connectionId, repoId))
+ if err != nil {
+ return nil, errors.Default.Wrap(err, "getting GithubRepo error")
+ }
+ err = helper.DecodeMapStruct(input.Body, &repo)
+ if err != nil {
+ return nil, errors.Default.Wrap(err, "patch github repo error")
+ }
+ err = verifyRepo(&repo)
+ if err != nil {
+ return nil, err
+ }
+ err = basicRes.GetDal().Update(repo)
+ if err != nil {
+ return nil, errors.Default.Wrap(err, "error on saving GithubRepo")
+ }
+ return &core.ApiResourceOutput{Body: repo, Status: http.StatusOK}, nil
+}
+
+// GetScopeList get Github repos
+// @Summary get Github repos
+// @Description get Github repos
+// @Tags plugins/github
+// @Param connectionId path int true "connection ID"
+// @Param pageSize query int false "page size, default 50"
+// @Param page query int false "page size, default 1"
+// @Success 200 {object} []models.GithubRepo
+// @Failure 400 {object} shared.ApiBody "Bad Request"
+// @Failure 500 {object} shared.ApiBody "Internal Error"
+// @Router /plugins/github/connections/{connectionId}/scopes/ [GET]
+func GetScopeList(input *core.ApiResourceInput) (*core.ApiResourceOutput, errors.Error) {
+ var repos []models.GithubRepo
+ connectionId, _ := extractParam(input.Params)
+ if connectionId == 0 {
+ return nil, errors.BadInput.New("invalid path params")
+ }
+ limit, offset := getLimitOffset(input.Query)
+ err := basicRes.GetDal().All(&repos, dal.Where("connection_id = ?", connectionId), dal.Limit(limit), dal.Offset(offset))
+ if err != nil {
+ return nil, err
+ }
+ return &core.ApiResourceOutput{Body: repos, Status: http.StatusOK}, nil
+}
+
+// GetScope get one Github repo
+// @Summary get one Github repo
+// @Description get one Github repo
+// @Tags plugins/github
+// @Param connectionId path int true "connection ID"
+// @Param repoId path int true "repo ID"
+// @Success 200 {object} models.GithubRepo
+// @Failure 400 {object} shared.ApiBody "Bad Request"
+// @Failure 500 {object} shared.ApiBody "Internal Error"
+// @Router /plugins/github/connections/{connectionId}/scopes/{repoId} [GET]
+func GetScope(input *core.ApiResourceInput) (*core.ApiResourceOutput, errors.Error) {
+ var repo models.GithubRepo
+ connectionId, repoId := extractParam(input.Params)
+ if connectionId*repoId == 0 {
+ return nil, errors.BadInput.New("invalid path params")
+ }
+ err := basicRes.GetDal().First(&repo, dal.Where("connection_id = ? AND github_id = ?", connectionId, repoId))
+ if err != nil {
+ return nil, err
+ }
+ return &core.ApiResourceOutput{Body: repo, Status: http.StatusOK}, nil
+}
+
+func extractParam(params map[string]string) (uint64, uint64) {
+ connectionId, _ := strconv.ParseUint(params["connectionId"], 10, 64)
+ repoId, _ := strconv.ParseUint(params["repoId"], 10, 64)
+ return connectionId, repoId
+}
+
+func verifyRepo(repo *models.GithubRepo) errors.Error {
+ if repo.ConnectionId == 0 {
+ return errors.BadInput.New("invalid connectionId")
+ }
+ if repo.GithubId <= 0 {
+ return errors.BadInput.New("invalid github ID")
+ }
+ return nil
+}
diff --git a/plugins/github/api/transformation_rule.go b/plugins/github/api/transformation_rule.go
new file mode 100644
index 000000000..f027be902
--- /dev/null
+++ b/plugins/github/api/transformation_rule.go
@@ -0,0 +1,128 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package api
+
+import (
+ "net/http"
+ "strconv"
+
+ "github.com/apache/incubator-devlake/errors"
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/core/dal"
+ "github.com/apache/incubator-devlake/plugins/github/models"
+ "github.com/apache/incubator-devlake/plugins/helper"
+ "github.com/mitchellh/mapstructure"
+)
+
+// CreateTransformationRule create transformation rule for Github
+// @Summary create transformation rule for Github
+// @Description create transformation rule for Github
+// @Tags plugins/github
+// @Accept application/json
+// @Param transformationRule body models.TransformationRules true "transformation rule"
+// @Success 200 {object} models.TransformationRules
+// @Failure 400 {object} shared.ApiBody "Bad Request"
+// @Failure 500 {object} shared.ApiBody "Internal Error"
+// @Router /plugins/github/transformation_rules [POST]
+func CreateTransformationRule(input *core.ApiResourceInput) (*core.ApiResourceOutput, errors.Error) {
+ var rule models.TransformationRules
+ err := mapstructure.Decode(input.Body, &rule)
+ if err != nil {
+ return nil, errors.Default.Wrap(err, "error in decoding transformation rule")
+ }
+ err = basicRes.GetDal().Create(&rule)
+ if err != nil {
+ return nil, errors.Default.Wrap(err, "error on saving TransformationRule")
+ }
+ return &core.ApiResourceOutput{Body: rule, Status: http.StatusOK}, nil
+}
+
+// UpdateTransformationRule update transformation rule for Github
+// @Summary update transformation rule for Github
+// @Description update transformation rule for Github
+// @Tags plugins/github
+// @Accept application/json
+// @Param id path int true "id"
+// @Param transformationRule body models.TransformationRules true "transformation rule"
+// @Success 200 {object} models.TransformationRules
+// @Failure 400 {object} shared.ApiBody "Bad Request"
+// @Failure 500 {object} shared.ApiBody "Internal Error"
+// @Router /plugins/github/transformation_rules/{id} [PATCH]
+func UpdateTransformationRule(input *core.ApiResourceInput) (*core.ApiResourceOutput, errors.Error) {
+ transformationRuleId, err := strconv.ParseUint(input.Params["id"], 10, 64)
+ if err != nil {
+ return nil, errors.Default.Wrap(err, "the transformation rule ID should be an integer")
+ }
+ var old models.TransformationRules
+ err = basicRes.GetDal().First(&old, dal.Where("id = ?", transformationRuleId))
+ if err != nil {
+ return nil, errors.Default.Wrap(err, "error on saving TransformationRule")
+ }
+ err = helper.DecodeMapStruct(input.Body, &old)
+ if err != nil {
+ return nil, errors.Default.Wrap(err, "error decoding map into transformationRule")
+ }
+ old.ID = transformationRuleId
+ err = basicRes.GetDal().Update(&old, dal.Where("id = ?", transformationRuleId))
+ if err != nil {
+ return nil, errors.Default.Wrap(err, "error on saving TransformationRule")
+ }
+ return &core.ApiResourceOutput{Body: old, Status: http.StatusOK}, nil
+}
+
+// GetTransformationRule return one transformation rule
+// @Summary return one transformation rule
+// @Description return one transformation rule
+// @Tags plugins/github
+// @Param id path int true "id"
+// @Success 200 {object} models.TransformationRules
+// @Failure 400 {object} shared.ApiBody "Bad Request"
+// @Failure 500 {object} shared.ApiBody "Internal Error"
+// @Router /plugins/github/transformation_rules/{id} [GET]
+func GetTransformationRule(input *core.ApiResourceInput) (*core.ApiResourceOutput, errors.Error) {
+ transformationRuleId, err := strconv.ParseUint(input.Params["id"], 10, 64)
+ if err != nil {
+ return nil, errors.Default.Wrap(err, "the transformation rule ID should be an integer")
+ }
+ var rule models.TransformationRules
+ err = basicRes.GetDal().First(&rule, dal.Where("id = ?", transformationRuleId))
+ if err != nil {
+ return nil, errors.Default.Wrap(err, "error on get TransformationRule")
+ }
+ return &core.ApiResourceOutput{Body: rule, Status: http.StatusOK}, nil
+}
+
+// GetTransformationRuleList return all transformation rules
+// @Summary return all transformation rules
+// @Description return all transformation rules
+// @Tags plugins/github
+// @Param pageSize query int false "page size, default 50"
+// @Param page query int false "page size, default 1"
+// @Success 200 {object} []models.TransformationRules
+// @Failure 400 {object} shared.ApiBody "Bad Request"
+// @Failure 500 {object} shared.ApiBody "Internal Error"
+// @Router /plugins/github/transformation_rules [GET]
+func GetTransformationRuleList(input *core.ApiResourceInput) (*core.ApiResourceOutput, errors.Error) {
+ var rules []models.TransformationRules
+ limit, offset := getLimitOffset(input.Query)
+ err := basicRes.GetDal().All(&rules, dal.Limit(limit), dal.Offset(offset))
+ if err != nil {
+ return nil, errors.Default.Wrap(err, "error on get TransformationRule list")
+ }
+ return &core.ApiResourceOutput{Body: rules, Status: http.StatusOK}, nil
+}
diff --git a/plugins/github/models/migrationscripts/register.go b/plugins/github/api/utils.go
similarity index 62%
copy from plugins/github/models/migrationscripts/register.go
copy to plugins/github/api/utils.go
index 9378bf9dd..5a4682b8e 100644
--- a/plugins/github/models/migrationscripts/register.go
+++ b/plugins/github/api/utils.go
@@ -15,21 +15,34 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
-package migrationscripts
+package api
import (
- "github.com/apache/incubator-devlake/plugins/core"
+ "net/url"
+ "strconv"
)
-// All return all the migration scripts
-func All() []core.MigrationScript {
- return []core.MigrationScript{
- new(addInitTables),
- new(addGithubRunsTable),
- new(addGithubJobsTable),
- new(addGithubPipelineTable),
- new(deleteGithubPipelineTable),
- new(addHeadRepoIdFieldInGithubPr),
- new(addEnableGraphqlForConnection),
+const pageSize = 50
+
+func getPageParam(q url.Values) (int, int) {
+ var size, page int
+ if ps := q["pageSize"]; len(ps) > 0 {
+ size, _ = strconv.Atoi(ps[0])
+ }
+ if p := q["page"]; len(p) > 0 {
+ page, _ = strconv.Atoi(p[0])
+ }
+ if size < 1 {
+ size = pageSize
}
+ if page < 1 {
+ page = 1
+ }
+ return size, page
+}
+
+func getLimitOffset(q url.Values) (int, int) {
+ limit, page := getPageParam(q)
+ offset := (page - 1) * limit
+ return limit, offset
}
diff --git a/plugins/github/e2e/cicd_test.go b/plugins/github/e2e/cicd_test.go
index 4890aed00..2ac4748fe 100644
--- a/plugins/github/e2e/cicd_test.go
+++ b/plugins/github/e2e/cicd_test.go
@@ -34,9 +34,10 @@ func TestGithubCICDDataFlow(t *testing.T) {
taskData := &tasks.GithubTaskData{
Options: &tasks.GithubOptions{
- ConnectionId: 1,
- Owner: "panjf2000",
- Repo: "ants",
+ ConnectionId: 1,
+ Owner: "panjf2000",
+ Repo: "ants",
+ TransformationRules: new(models.TransformationRules),
},
Repo: &models.GithubRepo{
GithubId: 134018330,
diff --git a/plugins/github/e2e/comment_test.go b/plugins/github/e2e/comment_test.go
index de87f90b2..3d7d8d029 100644
--- a/plugins/github/e2e/comment_test.go
+++ b/plugins/github/e2e/comment_test.go
@@ -20,13 +20,12 @@ package e2e
import (
"testing"
+ "github.com/apache/incubator-devlake/helpers/e2ehelper"
"github.com/apache/incubator-devlake/models/domainlayer/code"
"github.com/apache/incubator-devlake/models/domainlayer/ticket"
+ "github.com/apache/incubator-devlake/plugins/github/impl"
"github.com/apache/incubator-devlake/plugins/github/models"
"github.com/apache/incubator-devlake/plugins/github/tasks"
-
- "github.com/apache/incubator-devlake/helpers/e2ehelper"
- "github.com/apache/incubator-devlake/plugins/github/impl"
)
func TestCommentDataFlow(t *testing.T) {
@@ -41,7 +40,7 @@ func TestCommentDataFlow(t *testing.T) {
ConnectionId: 1,
Owner: "panjf2000",
Repo: "ants",
- TransformationRules: models.TransformationRules{
+ TransformationRules: &models.TransformationRules{
PrType: "type/(.*)$",
PrComponent: "component/(.*)$",
PrBodyClosePattern: "(?mi)(fix|close|resolve|fixes|closes|resolves|fixed|closed|resolved)[\\s]*.*(((and )?(#|https:\\/\\/github.com\\/%s\\/%s\\/issues\\/)\\d+[ ]*)+)",
diff --git a/plugins/github/e2e/issue_test.go b/plugins/github/e2e/issue_test.go
index 794ec946b..7706dfb66 100644
--- a/plugins/github/e2e/issue_test.go
+++ b/plugins/github/e2e/issue_test.go
@@ -20,11 +20,10 @@ package e2e
import (
"testing"
- "github.com/apache/incubator-devlake/models/domainlayer/ticket"
- "github.com/apache/incubator-devlake/plugins/github/models"
-
"github.com/apache/incubator-devlake/helpers/e2ehelper"
+ "github.com/apache/incubator-devlake/models/domainlayer/ticket"
"github.com/apache/incubator-devlake/plugins/github/impl"
+ "github.com/apache/incubator-devlake/plugins/github/models"
"github.com/apache/incubator-devlake/plugins/github/tasks"
)
@@ -40,7 +39,7 @@ func TestIssueDataFlow(t *testing.T) {
ConnectionId: 1,
Owner: "panjf2000",
Repo: "ants",
- TransformationRules: models.TransformationRules{
+ TransformationRules: &models.TransformationRules{
PrType: "type/(.*)$",
PrComponent: "component/(.*)$",
PrBodyClosePattern: "(?mi)(fix|close|resolve|fixes|closes|resolves|fixed|closed|resolved)[\\s]*.*(((and )?(#|https:\\/\\/github.com\\/%s\\/%s\\/issues\\/)\\d+[ ]*)+)",
diff --git a/plugins/github/e2e/milestone_test.go b/plugins/github/e2e/milestone_test.go
index 4384c0ce6..b6b6153f6 100644
--- a/plugins/github/e2e/milestone_test.go
+++ b/plugins/github/e2e/milestone_test.go
@@ -44,7 +44,7 @@ func TestMilestoneDataFlow(t *testing.T) {
ConnectionId: 1,
Owner: "panjf2000",
Repo: "ants",
- TransformationRules: models.TransformationRules{
+ TransformationRules: &models.TransformationRules{
PrType: "type/(.*)$",
PrComponent: "component/(.*)$",
PrBodyClosePattern: "(?mi)(fix|close|resolve|fixes|closes|resolves|fixed|closed|resolved)[\\s]*.*(((and )?(#|https:\\/\\/github.com\\/%s\\/%s\\/issues\\/)\\d+[ ]*)+)",
diff --git a/plugins/github/e2e/pr_enrich_issue_test.go b/plugins/github/e2e/pr_enrich_issue_test.go
index b5b6458cf..7a06e3ec4 100644
--- a/plugins/github/e2e/pr_enrich_issue_test.go
+++ b/plugins/github/e2e/pr_enrich_issue_test.go
@@ -18,12 +18,12 @@ limitations under the License.
package e2e
import (
- "github.com/apache/incubator-devlake/models/domainlayer/crossdomain"
- "github.com/apache/incubator-devlake/plugins/github/models"
"testing"
"github.com/apache/incubator-devlake/helpers/e2ehelper"
+ "github.com/apache/incubator-devlake/models/domainlayer/crossdomain"
"github.com/apache/incubator-devlake/plugins/github/impl"
+ "github.com/apache/incubator-devlake/plugins/github/models"
"github.com/apache/incubator-devlake/plugins/github/tasks"
)
@@ -39,7 +39,7 @@ func TestPrEnrichIssueDataFlow(t *testing.T) {
ConnectionId: 1,
Owner: "panjf2000",
Repo: "ants",
- TransformationRules: models.TransformationRules{
+ TransformationRules: &models.TransformationRules{
PrType: "type/(.*)$",
PrComponent: "component/(.*)$",
PrBodyClosePattern: "(?mi)(fix|close|resolve|fixes|closes|resolves|fixed|closed|resolved)[\\s]*.*(((and )?(#|https:\\/\\/github.com\\/%s\\/%s\\/issues\\/)\\d+[ ]*)+)",
diff --git a/plugins/github/e2e/pr_test.go b/plugins/github/e2e/pr_test.go
index 2c7bd82df..c06dce8f2 100644
--- a/plugins/github/e2e/pr_test.go
+++ b/plugins/github/e2e/pr_test.go
@@ -20,11 +20,10 @@ package e2e
import (
"testing"
- "github.com/apache/incubator-devlake/models/domainlayer/code"
- "github.com/apache/incubator-devlake/plugins/github/models"
-
"github.com/apache/incubator-devlake/helpers/e2ehelper"
+ "github.com/apache/incubator-devlake/models/domainlayer/code"
"github.com/apache/incubator-devlake/plugins/github/impl"
+ "github.com/apache/incubator-devlake/plugins/github/models"
"github.com/apache/incubator-devlake/plugins/github/tasks"
)
@@ -40,7 +39,7 @@ func TestPrDataFlow(t *testing.T) {
ConnectionId: 1,
Owner: "panjf2000",
Repo: "ants",
- TransformationRules: models.TransformationRules{
+ TransformationRules: &models.TransformationRules{
PrType: "type/(.*)$",
PrComponent: "component/(.*)$",
PrBodyClosePattern: "(?mi)(fix|close|resolve|fixes|closes|resolves|fixed|closed|resolved)[\\s]*.*(((and )?(#|https:\\/\\/github.com\\/%s\\/%s\\/issues\\/)\\d+[ ]*)+)",
diff --git a/plugins/github/e2e/repo_test.go b/plugins/github/e2e/repo_test.go
index ee35282fd..1e33e1912 100644
--- a/plugins/github/e2e/repo_test.go
+++ b/plugins/github/e2e/repo_test.go
@@ -18,17 +18,16 @@ limitations under the License.
package e2e
import (
- "github.com/apache/incubator-devlake/models/common"
- "github.com/apache/incubator-devlake/models/domainlayer/devops"
"testing"
+ "github.com/apache/incubator-devlake/helpers/e2ehelper"
+ "github.com/apache/incubator-devlake/models/common"
"github.com/apache/incubator-devlake/models/domainlayer/code"
"github.com/apache/incubator-devlake/models/domainlayer/crossdomain"
+ "github.com/apache/incubator-devlake/models/domainlayer/devops"
"github.com/apache/incubator-devlake/models/domainlayer/ticket"
- "github.com/apache/incubator-devlake/plugins/github/models"
-
- "github.com/apache/incubator-devlake/helpers/e2ehelper"
"github.com/apache/incubator-devlake/plugins/github/impl"
+ "github.com/apache/incubator-devlake/plugins/github/models"
"github.com/apache/incubator-devlake/plugins/github/tasks"
)
@@ -44,7 +43,7 @@ func TestRepoDataFlow(t *testing.T) {
ConnectionId: 1,
Owner: "panjf2000",
Repo: "ants",
- TransformationRules: models.TransformationRules{
+ TransformationRules: &models.TransformationRules{
PrType: "type/(.*)$",
PrComponent: "component/(.*)$",
},
diff --git a/plugins/github/impl/impl.go b/plugins/github/impl/impl.go
index 84ef2151b..e8281794d 100644
--- a/plugins/github/impl/impl.go
+++ b/plugins/github/impl/impl.go
@@ -23,6 +23,7 @@ import (
"github.com/apache/incubator-devlake/errors"
"github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/core/dal"
"github.com/apache/incubator-devlake/plugins/github/api"
"github.com/apache/incubator-devlake/plugins/github/models"
"github.com/apache/incubator-devlake/plugins/github/models/migrationscripts"
@@ -39,9 +40,22 @@ var _ core.PluginApi = (*Github)(nil)
var _ core.PluginModel = (*Github)(nil)
var _ core.PluginBlueprintV100 = (*Github)(nil)
var _ core.CloseablePluginTask = (*Github)(nil)
+var _ core.PluginSource = (*Github)(nil)
type Github struct{}
+func (plugin Github) Connection() interface{} {
+ return &models.GithubConnection{}
+}
+
+func (plugin Github) Scope() interface{} {
+ return &models.GithubRepo{}
+}
+
+func (plugin Github) TransformationRule() interface{} {
+ return &models.TransformationRules{}
+}
+
func (plugin Github) Init(config *viper.Viper, logger core.Logger, db *gorm.DB) errors.Error {
api.Init(config, logger, db)
return nil
@@ -153,7 +167,19 @@ func (plugin Github) PrepareTaskData(taskCtx core.TaskContext, options map[strin
return nil, errors.BadInput.Wrap(err, "invalid value for `since`")
}
}
-
+ if op.TransformationRules == nil && op.TransformationRuleId != 0 {
+ var transformationRule models.TransformationRules
+ err = taskCtx.GetDal().First(&transformationRule, dal.Where("id = ?", op.TransformationRuleId))
+ if err != nil {
+ return nil, errors.BadInput.Wrap(err, "fail to get transformationRule")
+ }
+ op.TransformationRules = &transformationRule
+ }
+ var repo models.GithubRepo
+ err = taskCtx.GetDal().First(&repo, dal.Where("name = ? AND owner_login = ", op.Repo, op.Owner))
+ if err != nil {
+ return nil, errors.BadInput.Wrap(err, "fail to get repo")
+ }
apiClient, err := tasks.CreateApiClient(taskCtx, connection)
if err != nil {
return nil, errors.Default.Wrap(err, "unable to get github API client instance")
@@ -161,6 +187,7 @@ func (plugin Github) PrepareTaskData(taskCtx core.TaskContext, options map[strin
taskData := &tasks.GithubTaskData{
Options: op,
ApiClient: apiClient,
+ Repo: &repo,
}
if !since.IsZero() {
@@ -192,6 +219,22 @@ func (plugin Github) ApiResources() map[string]map[string]core.ApiResourceHandle
"PATCH": api.PatchConnection,
"DELETE": api.DeleteConnection,
},
+ "connections/:connectionId/scopes/:repoId": {
+ "GET": api.GetScope,
+ "PUT": api.PutScope,
+ "PATCH": api.UpdateScope,
+ },
+ "connections/:connectionId/scopes": {
+ "GET": api.GetScopeList,
+ },
+ "transformation_rules": {
+ "POST": api.CreateTransformationRule,
+ "GET": api.GetTransformationRuleList,
+ },
+ "transformation_rules/:id": {
+ "PATCH": api.UpdateTransformationRule,
+ "GET": api.GetTransformationRule,
+ },
"connections/:connectionId/proxy/rest/*path": {
"GET": api.Proxy,
},
diff --git a/plugins/github/models/connection.go b/plugins/github/models/connection.go
index da522e774..474bd5a6d 100644
--- a/plugins/github/models/connection.go
+++ b/plugins/github/models/connection.go
@@ -33,19 +33,6 @@ type GithubConnection struct {
EnableGraphql bool `mapstructure:"enableGraphql" json:"enableGraphql"`
}
-type TransformationRules struct {
- PrType string `mapstructure:"prType" json:"prType"`
- PrComponent string `mapstructure:"prComponent" json:"prComponent"`
- PrBodyClosePattern string `mapstructure:"prBodyClosePattern" json:"prBodyClosePattern"`
- IssueSeverity string `mapstructure:"issueSeverity" json:"issueSeverity"`
- IssuePriority string `mapstructure:"issuePriority" json:"issuePriority"`
- IssueComponent string `mapstructure:"issueComponent" json:"issueComponent"`
- IssueTypeBug string `mapstructure:"issueTypeBug" json:"issueTypeBug"`
- IssueTypeIncident string `mapstructure:"issueTypeIncident" json:"issueTypeIncident"`
- IssueTypeRequirement string `mapstructure:"issueTypeRequirement" json:"issueTypeRequirement"`
- DeploymentPattern string `mapstructure:"deploymentPattern" json:"deploymentPattern"`
-}
-
func (GithubConnection) TableName() string {
return "_tool_github_connections"
}
diff --git a/plugins/jira/models/migrationscripts/20221116_add_trasformation_rule_table.go b/plugins/github/models/migrationscripts/20221124_add_trasformation_rule_table.go
similarity index 58%
copy from plugins/jira/models/migrationscripts/20221116_add_trasformation_rule_table.go
copy to plugins/github/models/migrationscripts/20221124_add_trasformation_rule_table.go
index e5d2f863c..d6305ae23 100644
--- a/plugins/jira/models/migrationscripts/20221116_add_trasformation_rule_table.go
+++ b/plugins/github/models/migrationscripts/20221124_add_trasformation_rule_table.go
@@ -21,28 +21,27 @@ import (
"github.com/apache/incubator-devlake/errors"
"github.com/apache/incubator-devlake/helpers/migrationhelper"
"github.com/apache/incubator-devlake/plugins/core"
- "github.com/apache/incubator-devlake/plugins/jira/models/migrationscripts/archived"
+ "github.com/apache/incubator-devlake/plugins/github/models/migrationscripts/archived"
)
-type jiraBoard20221116 struct {
+type githubRepo20221124 struct {
TransformationRuleId uint64
- ScopeId string `gorm:"type:varchar(255)"`
}
-func (jiraBoard20221116) TableName() string {
- return "_tool_jira_boards"
+func (githubRepo20221124) TableName() string {
+ return "_tool_github_repos"
}
-type addTransformationRule20221116 struct{}
+type addTransformationRule20221124 struct{}
-func (script *addTransformationRule20221116) Up(basicRes core.BasicRes) errors.Error {
- return migrationhelper.AutoMigrateTables(basicRes, &jiraBoard20221116{}, &archived.JiraTransformationRule{})
+func (*addTransformationRule20221124) Up(basicRes core.BasicRes) errors.Error {
+ return migrationhelper.AutoMigrateTables(basicRes, &githubRepo20221124{}, &archived.TransformationRules{})
}
-func (*addTransformationRule20221116) Version() uint64 {
- return 20221117122532
+func (*addTransformationRule20221124) Version() uint64 {
+ return 20221124095900
}
-func (*addTransformationRule20221116) Name() string {
- return "add table _tool_jira_transformation_rules, add transformation_rule_id to _tool_jira_boards"
+func (*addTransformationRule20221124) Name() string {
+ return "add table _tool_github_transformation_rules, add transformation_rule_id to _tool_github_repos"
}
diff --git a/plugins/github/models/migrationscripts/archived/transformation_rules.go b/plugins/github/models/migrationscripts/archived/transformation_rules.go
new file mode 100644
index 000000000..52a6a0a1b
--- /dev/null
+++ b/plugins/github/models/migrationscripts/archived/transformation_rules.go
@@ -0,0 +1,43 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package archived
+
+import (
+ "encoding/json"
+
+ "github.com/apache/incubator-devlake/models/migrationscripts/archived"
+)
+
+type TransformationRules struct {
+ archived.Model
+ PrType string `mapstructure:"prType" json:"prType" gorm:"type:varchar(255)"`
+ PrComponent string `mapstructure:"prComponent" json:"prComponent" gorm:"type:varchar(255)"`
+ PrBodyClosePattern string `mapstructure:"prBodyClosePattern" json:"prBodyClosePattern" gorm:"type:varchar(255)"`
+ IssueSeverity string `mapstructure:"issueSeverity" json:"issueSeverity" gorm:"type:varchar(255)"`
+ IssuePriority string `mapstructure:"issuePriority" json:"issuePriority" gorm:"type:varchar(255)"`
+ IssueComponent string `mapstructure:"issueComponent" json:"issueComponent" gorm:"type:varchar(255)"`
+ IssueTypeBug string `mapstructure:"issueTypeBug" json:"issueTypeBug" gorm:"type:varchar(255)"`
+ IssueTypeIncident string `mapstructure:"issueTypeIncident" json:"issueTypeIncident" gorm:"type:varchar(255)"`
+ IssueTypeRequirement string `mapstructure:"issueTypeRequirement" json:"issueTypeRequirement" gorm:"type:varchar(255)"`
+ DeploymentPattern string `mapstructure:"deploymentPattern" json:"deploymentPattern" gorm:"type:varchar(255)"`
+ RefdiffRule json.RawMessage `mapstructure:"refdiffRule" json:"refdiffRule"`
+}
+
+func (TransformationRules) TableName() string {
+ return "_tool_github_transformation_rules"
+}
diff --git a/plugins/github/models/migrationscripts/register.go b/plugins/github/models/migrationscripts/register.go
index 9378bf9dd..137406707 100644
--- a/plugins/github/models/migrationscripts/register.go
+++ b/plugins/github/models/migrationscripts/register.go
@@ -31,5 +31,6 @@ func All() []core.MigrationScript {
new(deleteGithubPipelineTable),
new(addHeadRepoIdFieldInGithubPr),
new(addEnableGraphqlForConnection),
+ new(addTransformationRule20221124),
}
}
diff --git a/plugins/github/models/repo.go b/plugins/github/models/repo.go
index a60dbc36d..5ae5516ba 100644
--- a/plugins/github/models/repo.go
+++ b/plugins/github/models/repo.go
@@ -23,18 +23,19 @@ import (
)
type GithubRepo struct {
- ConnectionId uint64 `gorm:"primaryKey"`
- GithubId int `gorm:"primaryKey"`
- Name string `gorm:"type:varchar(255)"`
- HTMLUrl string `gorm:"type:varchar(255)"`
- Description string
- OwnerId int `json:"ownerId"`
- OwnerLogin string `json:"ownerLogin" gorm:"type:varchar(255)"`
- Language string `json:"language" gorm:"type:varchar(255)"`
- ParentGithubId int `json:"parentId"`
- ParentHTMLUrl string `json:"parentHtmlUrl"`
- CreatedDate time.Time `json:"createdDate"`
- UpdatedDate *time.Time `json:"updatedDate"`
+ ConnectionId uint64 `gorm:"primaryKey"`
+ GithubId int `gorm:"primaryKey"`
+ TransformationRuleId uint64
+ Name string `gorm:"type:varchar(255)"`
+ HTMLUrl string `gorm:"type:varchar(255)"`
+ Description string
+ OwnerId int `json:"ownerId"`
+ OwnerLogin string `json:"ownerLogin" gorm:"type:varchar(255)"`
+ Language string `json:"language" gorm:"type:varchar(255)"`
+ ParentGithubId int `json:"parentId"`
+ ParentHTMLUrl string `json:"parentHtmlUrl"`
+ CreatedDate time.Time `json:"createdDate"`
+ UpdatedDate *time.Time `json:"updatedDate"`
common.NoPKModel
}
diff --git a/plugins/github/models/transformation_rule.go b/plugins/github/models/transformation_rule.go
new file mode 100644
index 000000000..f53eece58
--- /dev/null
+++ b/plugins/github/models/transformation_rule.go
@@ -0,0 +1,43 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package models
+
+import (
+ "encoding/json"
+
+ "github.com/apache/incubator-devlake/models/common"
+)
+
+type TransformationRules struct {
+ common.Model
+ PrType string `mapstructure:"prType" json:"prType" gorm:"type:varchar(255)"`
+ PrComponent string `mapstructure:"prComponent" json:"prComponent" gorm:"type:varchar(255)"`
+ PrBodyClosePattern string `mapstructure:"prBodyClosePattern" json:"prBodyClosePattern" gorm:"type:varchar(255)"`
+ IssueSeverity string `mapstructure:"issueSeverity" json:"issueSeverity" gorm:"type:varchar(255)"`
+ IssuePriority string `mapstructure:"issuePriority" json:"issuePriority" gorm:"type:varchar(255)"`
+ IssueComponent string `mapstructure:"issueComponent" json:"issueComponent" gorm:"type:varchar(255)"`
+ IssueTypeBug string `mapstructure:"issueTypeBug" json:"issueTypeBug" gorm:"type:varchar(255)"`
+ IssueTypeIncident string `mapstructure:"issueTypeIncident" json:"issueTypeIncident" gorm:"type:varchar(255)"`
+ IssueTypeRequirement string `mapstructure:"issueTypeRequirement" json:"issueTypeRequirement" gorm:"type:varchar(255)"`
+ DeploymentPattern string `mapstructure:"deploymentPattern" json:"deploymentPattern" gorm:"type:varchar(255)"`
+ RefdiffRule json.RawMessage `mapstructure:"refdiffRule" json:"refdiffRule"`
+}
+
+func (TransformationRules) TableName() string {
+ return "_tool_github_transformation_rules"
+}
diff --git a/plugins/github/tasks/issue_extractor.go b/plugins/github/tasks/issue_extractor.go
index eb290b772..e95bbc474 100644
--- a/plugins/github/tasks/issue_extractor.go
+++ b/plugins/github/tasks/issue_extractor.go
@@ -223,7 +223,7 @@ func convertGithubLabels(issueRegexes *IssueRegexes, issue *IssuesResponse, gith
return results, nil
}
-func NewIssueRegexes(config models.TransformationRules) (*IssueRegexes, errors.Error) {
+func NewIssueRegexes(config *models.TransformationRules) (*IssueRegexes, errors.Error) {
var issueRegexes IssueRegexes
var issueSeverity = config.IssueSeverity
var err error
diff --git a/plugins/github/tasks/repo_extractor.go b/plugins/github/tasks/repo_extractor.go
index faa4a6e22..59ed2d362 100644
--- a/plugins/github/tasks/repo_extractor.go
+++ b/plugins/github/tasks/repo_extractor.go
@@ -21,7 +21,6 @@ import (
"encoding/json"
"fmt"
"github.com/apache/incubator-devlake/errors"
-
"github.com/apache/incubator-devlake/plugins/core"
"github.com/apache/incubator-devlake/plugins/github/models"
"github.com/apache/incubator-devlake/plugins/helper"
diff --git a/plugins/github/tasks/task_data.go b/plugins/github/tasks/task_data.go
index 728f22833..0516b807b 100644
--- a/plugins/github/tasks/task_data.go
+++ b/plugins/github/tasks/task_data.go
@@ -27,12 +27,13 @@ import (
)
type GithubOptions struct {
- ConnectionId uint64 `json:"connectionId"`
- Tasks []string `json:"tasks,omitempty"`
- Since string
- Owner string
- Repo string
- models.TransformationRules `mapstructure:"transformationRules" json:"transformationRules"`
+ ConnectionId uint64 `json:"connectionId"`
+ TransformationRuleId uint64 `json:"transformationRuleId"`
+ Tasks []string `json:"tasks,omitempty"`
+ Since string
+ Owner string
+ Repo string
+ *models.TransformationRules `mapstructure:"transformationRules" json:"transformationRules"`
}
type GithubTaskData struct {
@@ -55,40 +56,12 @@ func DecodeAndValidateTaskOptions(options map[string]interface{}) (*GithubOption
if op.Repo == "" {
return nil, errors.BadInput.New("repo is required for GitHub execution")
}
- if op.PrType == "" {
- op.PrType = "type/(.*)$"
- }
- if op.PrComponent == "" {
- op.PrComponent = "component/(.*)$"
- }
- if op.PrBodyClosePattern == "" {
- op.PrBodyClosePattern = "(?mi)(fix|close|resolve|fixes|closes|resolves|fixed|closed|resolved)[\\s]*.*(((and )?(#|https:\\/\\/github.com\\/%s\\/%s\\/issues\\/)\\d+[ ]*)+)"
- }
- if op.IssueSeverity == "" {
- op.IssueSeverity = "severity/(.*)$"
- }
- if op.IssuePriority == "" {
- op.IssuePriority = "^(highest|high|medium|low)$"
- }
- if op.IssueComponent == "" {
- op.IssueComponent = "component/(.*)$"
- }
- if op.IssueTypeBug == "" {
- op.IssueTypeBug = "^(bug|failure|error)$"
- }
- if op.IssueTypeIncident == "" {
- op.IssueTypeIncident = ""
- }
- if op.IssueTypeRequirement == "" {
- op.IssueTypeRequirement = "^(feat|feature|proposal|requirement)$"
- }
- if op.DeploymentPattern == "" {
- op.DeploymentPattern = "(?i)deploy"
- }
-
// find the needed GitHub now
if op.ConnectionId == 0 {
return nil, errors.BadInput.New("connectionId is invalid")
}
+ if op.TransformationRules == nil && op.TransformationRuleId == 0 {
+ op.TransformationRules = new(models.TransformationRules)
+ }
return &op, nil
}
diff --git a/plugins/jira/api/scope.go b/plugins/jira/api/scope.go
index d6ce8c33c..2a241455c 100644
--- a/plugins/jira/api/scope.go
+++ b/plugins/jira/api/scope.go
@@ -157,8 +157,5 @@ func verifyBoard(board *models.JiraBoard) errors.Error {
if board.BoardId == 0 {
return errors.BadInput.New("invalid boardId")
}
- if board.ScopeId != strconv.FormatUint(board.BoardId, 10) {
- return errors.BadInput.New("the scope_id does not match the board_id")
- }
return nil
}
diff --git a/plugins/jira/impl/impl.go b/plugins/jira/impl/impl.go
index 8c01cb351..330ceebfa 100644
--- a/plugins/jira/impl/impl.go
+++ b/plugins/jira/impl/impl.go
@@ -95,6 +95,9 @@ func (plugin Jira) Description() string {
func (plugin Jira) SubTaskMetas() []core.SubTaskMeta {
return []core.SubTaskMeta{
+ tasks.CollectBoardMeta,
+ tasks.ExtractBoardMeta,
+
tasks.CollectStatusMeta,
tasks.ExtractStatusMeta,
diff --git a/plugins/jira/models/board.go b/plugins/jira/models/board.go
index a091ebca0..17568b142 100644
--- a/plugins/jira/models/board.go
+++ b/plugins/jira/models/board.go
@@ -25,7 +25,6 @@ type JiraBoard struct {
common.NoPKModel
ConnectionId uint64 `gorm:"primaryKey"`
BoardId uint64 `gorm:"primaryKey"`
- ScopeId string
TransformationRuleId uint64
ProjectId uint
Name string `gorm:"type:varchar(255)"`
diff --git a/plugins/jira/models/migrationscripts/20221116_add_trasformation_rule_table.go b/plugins/jira/models/migrationscripts/20221116_add_trasformation_rule_table.go
index e5d2f863c..475388c88 100644
--- a/plugins/jira/models/migrationscripts/20221116_add_trasformation_rule_table.go
+++ b/plugins/jira/models/migrationscripts/20221116_add_trasformation_rule_table.go
@@ -26,7 +26,6 @@ import (
type jiraBoard20221116 struct {
TransformationRuleId uint64
- ScopeId string `gorm:"type:varchar(255)"`
}
func (jiraBoard20221116) TableName() string {
diff --git a/plugins/jira/models/migrationscripts/archived/transformation_rules.go b/plugins/jira/models/migrationscripts/archived/transformation_rules.go
index a13420baf..c0d59d75a 100644
--- a/plugins/jira/models/migrationscripts/archived/transformation_rules.go
+++ b/plugins/jira/models/migrationscripts/archived/transformation_rules.go
@@ -30,3 +30,7 @@ type JiraTransformationRule struct {
RemotelinkCommitShaPattern string `json:"remotelinkCommitShaPattern" gorm:"type:varchar(255)"`
TypeMappings json.RawMessage `json:"typeMappings"`
}
+
+func (JiraTransformationRule) TableName() string {
+ return "_tool_jira_transformation_rules"
+}
diff --git a/plugins/jira/models/transformation_rules.go b/plugins/jira/models/transformation_rules.go
index 1aa3aa055..998290e82 100644
--- a/plugins/jira/models/transformation_rules.go
+++ b/plugins/jira/models/transformation_rules.go
@@ -30,3 +30,7 @@ type JiraTransformationRule struct {
RemotelinkCommitShaPattern string `json:"remotelinkCommitShaPattern" gorm:"type:varchar(255)"`
TypeMappings json.RawMessage `json:"typeMappings"`
}
+
+func (JiraTransformationRule) TableName() string {
+ return "_tool_jira_transformation_rules"
+}