You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by kl...@apache.org on 2022/06/23 08:34:03 UTC
[incubator-devlake] branch main updated: Github blueprint normal mode support (#2331)
This is an automated email from the ASF dual-hosted git repository.
klesh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 6ad98a4a Github blueprint normal mode support (#2331)
6ad98a4a is described below
commit 6ad98a4a5a8b772879310651ebbea4b9acf793a9
Author: Klesh Wong <zh...@merico.dev>
AuthorDate: Thu Jun 23 16:33:59 2022 +0800
Github blueprint normal mode support (#2331)
* fix: existing blueprint records should be updated
* feat: github blueprint normal mode support
* feat: generate once at creation
* fix: missing ASF header
---
Makefile | 2 +-
models/migrationscripts/updateSchemas20220616.go | 2 +
plugins/github/api/blueprint.go | 129 +++++++++++++++++++++
plugins/github/api/connection.go | 3 +-
plugins/github/impl/impl.go | 50 ++------
plugins/github/tasks/comment_collector.go | 4 +-
plugins/github/tasks/comment_extractor.go | 2 +
plugins/github/tasks/commit_collector.go | 4 +-
plugins/github/tasks/commit_convertor.go | 4 +-
plugins/github/tasks/commit_extractor.go | 1 +
plugins/github/tasks/commit_stats_collector.go | 6 +-
plugins/github/tasks/commit_stats_extractor.go | 2 +
plugins/github/tasks/event_collector.go | 4 +-
plugins/github/tasks/event_extractor.go | 1 +
plugins/github/tasks/issue_collector.go | 4 +-
plugins/github/tasks/issue_comment_convertor.go | 4 +-
plugins/github/tasks/issue_convertor.go | 4 +-
plugins/github/tasks/issue_extractor.go | 1 +
plugins/github/tasks/issue_label_convertor.go | 4 +-
plugins/github/tasks/pr_collector.go | 4 +-
plugins/github/tasks/pr_comment_convertor.go | 4 +-
plugins/github/tasks/pr_commit_collector.go | 4 +-
plugins/github/tasks/pr_commit_convertor.go | 4 +-
plugins/github/tasks/pr_commit_extractor.go | 1 +
plugins/github/tasks/pr_convertor.go | 4 +-
plugins/github/tasks/pr_extractor.go | 1 +
plugins/github/tasks/pr_issue_convertor.go | 4 +-
plugins/github/tasks/pr_issue_enricher.go | 10 +-
plugins/github/tasks/pr_label_convertor.go | 4 +-
plugins/github/tasks/pr_review_collector.go | 6 +-
plugins/github/tasks/pr_review_extractor.go | 1 +
plugins/github/tasks/repo_collector.go | 1 +
plugins/github/tasks/repo_convertor.go | 4 +-
...{repositorie_extractor.go => repo_extractor.go} | 2 +
plugins/github/tasks/task_data.go | 49 ++++++++
plugins/github/tasks/user_convertor.go | 4 +-
plugins/helper/pipeline_plan.go | 49 ++++++++
plugins/helper/pipeline_plan_test.go | 76 ++++++++++++
services/blueprint.go | 22 ++--
.../github/tasks/task_data.go => utils/strings.go | 40 +++----
40 files changed, 425 insertions(+), 100 deletions(-)
diff --git a/Makefile b/Makefile
index 50eb5404..c8abaaf0 100644
--- a/Makefile
+++ b/Makefile
@@ -66,7 +66,7 @@ mock:
test: unit-test e2e-test
unit-test: mock build
- set -e; for m in $$(go list ./... | egrep -v 'test|models|e2e'); do echo $$m; go test -timeout 60s -gcflags=all=-l -v $$m; done
+ set -e; for m in $$(go list ./... | egrep -v 'test|models|e2e'); do echo $$m; go test -timeout 60s -v $$m; done
e2e-test: build
PLUGIN_DIR=$(shell readlink -f bin/plugins) go test -timeout 300s -v ./test/...
diff --git a/models/migrationscripts/updateSchemas20220616.go b/models/migrationscripts/updateSchemas20220616.go
index dcb84c32..9729677c 100644
--- a/models/migrationscripts/updateSchemas20220616.go
+++ b/models/migrationscripts/updateSchemas20220616.go
@@ -39,6 +39,8 @@ func (*updateSchemas20220616) Up(ctx context.Context, db *gorm.DB) error {
if err != nil {
return err
}
+ db.Model(&Blueprint20220616{}).Where("mode is null").Update("mode", "ADVANCED")
+ db.Model(&Blueprint20220616{}).Where("is_manual is null").Update("is_manual", false)
return nil
}
diff --git a/plugins/github/api/blueprint.go b/plugins/github/api/blueprint.go
new file mode 100644
index 00000000..f68c7168
--- /dev/null
+++ b/plugins/github/api/blueprint.go
@@ -0,0 +1,129 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package api
+
+import (
+ "encoding/json"
+ "fmt"
+ "io/ioutil"
+ "net/http"
+ "net/url"
+ "strings"
+ "time"
+
+ "github.com/apache/incubator-devlake/models/domainlayer/didgen"
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/github/models"
+ "github.com/apache/incubator-devlake/plugins/github/tasks"
+ "github.com/apache/incubator-devlake/plugins/helper"
+ "github.com/apache/incubator-devlake/utils"
+)
+
+func MakePipelinePlan(subtaskMetas []core.SubTaskMeta, connectionId uint64, scope []*core.BlueprintScopeV100) (core.PipelinePlan, error) {
+ var err error
+ plan := make(core.PipelinePlan, len(scope))
+ for i, scopeElem := range scope {
+ // handle taskOptions and transformationRules, by dumping them to taskOptions
+ taskOptions := make(map[string]interface{})
+ err = json.Unmarshal(scopeElem.Options, &taskOptions)
+ if err != nil {
+ return nil, err
+ }
+ err = json.Unmarshal(scopeElem.Transformation, &taskOptions)
+ if err != nil {
+ return nil, err
+ }
+ taskOptions["connectionId"] = connectionId
+ op, err := tasks.DecodeAndValidateTaskOptions(taskOptions)
+ if err != nil {
+ return nil, err
+ }
+ // subtasks
+ subtasks, err := helper.MakePipelinePlanSubtasks(subtaskMetas, scopeElem.Entities)
+ if err != nil {
+ return nil, err
+ }
+ stage := core.PipelineStage{
+ {
+ Plugin: "github",
+ Subtasks: subtasks,
+ Options: taskOptions,
+ },
+ }
+ // collect git data by gitextractor if CODE was requested
+ if utils.StringsContains(scopeElem.Entities, core.DOMAIN_TYPE_CODE) {
+ // here is the tricky part, we have to obtain the repo id beforehand
+ connection := new(models.GithubConnection)
+ err = connectionHelper.FirstById(connection, connectionId)
+ if err != nil {
+ return nil, err
+ }
+ token := strings.Split(connection.Token, ",")[0]
+ apiClient, err := helper.NewApiClient(
+ connection.Endpoint,
+ map[string]string{
+ "Authorization": fmt.Sprintf("Bearer %s", token),
+ },
+ 10*time.Second,
+ connection.Proxy,
+ nil,
+ )
+ if err != nil {
+ return nil, err
+ }
+ res, err := apiClient.Get(fmt.Sprintf("repos/%s/%s", op.Owner, op.Repo), nil, nil)
+ if err != nil {
+ return nil, err
+ }
+ if res.StatusCode != http.StatusOK {
+ return nil, fmt.Errorf(
+ "unexpected status code when requesting repo detail %d %s",
+ res.StatusCode, res.Request.URL.String(),
+ )
+ }
+ defer res.Body.Close()
+ body, err := ioutil.ReadAll(res.Body)
+ if err != nil {
+ return nil, err
+ }
+ apiRepo := new(tasks.GithubApiRepo)
+ err = json.Unmarshal(body, apiRepo)
+ if err != nil {
+ return nil, err
+ }
+ cloneUrl, err := url.Parse(apiRepo.CloneUrl)
+ if err != nil {
+ return nil, err
+ }
+ cloneUrl.User = url.UserPassword("git", token)
+ stage = append(stage, &core.PipelineTask{
+ Plugin: "gitextractor",
+ Options: map[string]interface{}{
+ // TODO: url should be configuration
+ // TODO: to support private repo: username is needed for repo cloning, and we have to take
+ // multi-token support into consideration, this is hairy
+ "url": cloneUrl.String(),
+ "repoId": didgen.NewDomainIdGenerator(&models.GithubRepo{}).Generate(connectionId, apiRepo.GithubId),
+ },
+ })
+ // TODO, add refdiff in the future
+ }
+ plan[i] = stage
+ }
+ return plan, nil
+}
diff --git a/plugins/github/api/connection.go b/plugins/github/api/connection.go
index db439ef5..bf2e0895 100644
--- a/plugins/github/api/connection.go
+++ b/plugins/github/api/connection.go
@@ -19,11 +19,12 @@ package api
import (
"fmt"
- "github.com/apache/incubator-devlake/plugins/github/models"
"net/http"
"strings"
"time"
+ "github.com/apache/incubator-devlake/plugins/github/models"
+
"github.com/apache/incubator-devlake/plugins/helper"
"github.com/mitchellh/mapstructure"
diff --git a/plugins/github/impl/impl.go b/plugins/github/impl/impl.go
index acb2236e..142d49c2 100644
--- a/plugins/github/impl/impl.go
+++ b/plugins/github/impl/impl.go
@@ -18,7 +18,6 @@ limitations under the License.
package impl
import (
- "fmt"
"github.com/apache/incubator-devlake/migration"
"github.com/apache/incubator-devlake/plugins/core"
"github.com/apache/incubator-devlake/plugins/github/api"
@@ -26,7 +25,6 @@ import (
"github.com/apache/incubator-devlake/plugins/github/models/migrationscripts"
"github.com/apache/incubator-devlake/plugins/github/tasks"
"github.com/apache/incubator-devlake/plugins/helper"
- "github.com/mitchellh/mapstructure"
"github.com/spf13/viper"
"gorm.io/gorm"
)
@@ -36,6 +34,7 @@ var _ core.PluginInit = (*Github)(nil)
var _ core.PluginTask = (*Github)(nil)
var _ core.PluginApi = (*Github)(nil)
var _ core.Migratable = (*Github)(nil)
+var _ core.PluginBlueprintV100 = (*Github)(nil)
type Github struct{}
@@ -84,49 +83,10 @@ func (plugin Github) SubTaskMetas() []core.SubTaskMeta {
}
func (plugin Github) PrepareTaskData(taskCtx core.TaskContext, options map[string]interface{}) (interface{}, error) {
- var op tasks.GithubOptions
- err := mapstructure.Decode(options, &op)
+ op, err := tasks.DecodeAndValidateTaskOptions(options)
if err != nil {
return nil, err
}
- if op.Owner == "" {
- return nil, fmt.Errorf("owner is required for GitHub execution")
- }
- if op.Repo == "" {
- return nil, fmt.Errorf("repo is required for GitHub execution")
- }
- if op.PrType == "" {
- op.PrType = "type/(.*)$"
- }
- if op.PrComponent == "" {
- op.PrComponent = "component/(.*)$"
- }
- if op.PrBodyClosePattern == "" {
- op.PrBodyClosePattern = "(?mi)(fix|close|resolve|fixes|closes|resolves|fixed|closed|resolved)[\\s]*.*(((and )?(#|https:\\/\\/github.com\\/%s\\/%s\\/issues\\/)\\d+[ ]*)+)"
- }
- if op.IssueSeverity == "" {
- op.IssueSeverity = "severity/(.*)$"
- }
- if op.IssuePriority == "" {
- op.IssuePriority = "^(highest|high|medium|low)$"
- }
- if op.IssueComponent == "" {
- op.IssueComponent = "component/(.*)$"
- }
- if op.IssueTypeBug == "" {
- op.IssueTypeBug = "^(bug|failure|error)$"
- }
- if op.IssueTypeIncident == "" {
- op.IssueTypeIncident = ""
- }
- if op.IssueTypeRequirement == "" {
- op.IssueTypeRequirement = "^(feat|feature|proposal|requirement)$"
- }
-
- // find the needed GitHub now
- if op.ConnectionId == 0 {
- return nil, fmt.Errorf("connectionId is invalid")
- }
connectionHelper := helper.NewConnectionHelper(
taskCtx,
nil,
@@ -143,7 +103,7 @@ func (plugin Github) PrepareTaskData(taskCtx core.TaskContext, options map[strin
}
return &tasks.GithubTaskData{
- Options: &op,
+ Options: op,
ApiClient: apiClient,
}, nil
}
@@ -174,3 +134,7 @@ func (plugin Github) ApiResources() map[string]map[string]core.ApiResourceHandle
},
}
}
+
+func (plugin Github) MakePipelinePlan(connectionId uint64, scope []*core.BlueprintScopeV100) (core.PipelinePlan, error) {
+ return api.MakePipelinePlan(plugin.SubTaskMetas(), connectionId, scope)
+}
diff --git a/plugins/github/tasks/comment_collector.go b/plugins/github/tasks/comment_collector.go
index b9b42a91..2f6b49d0 100644
--- a/plugins/github/tasks/comment_collector.go
+++ b/plugins/github/tasks/comment_collector.go
@@ -20,10 +20,11 @@ package tasks
import (
"encoding/json"
"fmt"
- . "github.com/apache/incubator-devlake/plugins/core/dal"
"net/http"
"net/url"
+ . "github.com/apache/incubator-devlake/plugins/core/dal"
+
"github.com/apache/incubator-devlake/plugins/helper"
"github.com/apache/incubator-devlake/plugins/core"
@@ -130,4 +131,5 @@ var CollectApiCommentsMeta = core.SubTaskMeta{
EntryPoint: CollectApiComments,
EnabledByDefault: true,
Description: "Collect comments data from Github api",
+ DomainTypes: []string{core.DOMAIN_TYPE_CODE, core.DOMAIN_TYPE_TICKET},
}
diff --git a/plugins/github/tasks/comment_extractor.go b/plugins/github/tasks/comment_extractor.go
index c932ab4f..6946d349 100644
--- a/plugins/github/tasks/comment_extractor.go
+++ b/plugins/github/tasks/comment_extractor.go
@@ -19,6 +19,7 @@ package tasks
import (
"encoding/json"
+
"github.com/apache/incubator-devlake/plugins/core/dal"
"github.com/apache/incubator-devlake/plugins/core"
@@ -33,6 +34,7 @@ var ExtractApiCommentsMeta = core.SubTaskMeta{
EnabledByDefault: true,
Description: "Extract raw comment data into tool layer table github_pull_request_comments" +
"and github_issue_comments",
+ DomainTypes: []string{core.DOMAIN_TYPE_CODE, core.DOMAIN_TYPE_TICKET},
}
type IssueComment struct {
diff --git a/plugins/github/tasks/commit_collector.go b/plugins/github/tasks/commit_collector.go
index 425bcfac..7bd4e99c 100644
--- a/plugins/github/tasks/commit_collector.go
+++ b/plugins/github/tasks/commit_collector.go
@@ -20,10 +20,11 @@ package tasks
import (
"encoding/json"
"fmt"
- "github.com/apache/incubator-devlake/plugins/core/dal"
"net/http"
"net/url"
+ "github.com/apache/incubator-devlake/plugins/core/dal"
+
"github.com/apache/incubator-devlake/plugins/helper"
"github.com/apache/incubator-devlake/plugins/core"
@@ -37,6 +38,7 @@ var CollectApiCommitsMeta = core.SubTaskMeta{
EntryPoint: CollectApiCommits,
EnabledByDefault: false,
Description: "Collect commits data from Github api",
+ DomainTypes: []string{core.DOMAIN_TYPE_CODE},
}
func CollectApiCommits(taskCtx core.SubTaskContext) error {
diff --git a/plugins/github/tasks/commit_convertor.go b/plugins/github/tasks/commit_convertor.go
index d6d75d82..a7aae72c 100644
--- a/plugins/github/tasks/commit_convertor.go
+++ b/plugins/github/tasks/commit_convertor.go
@@ -18,9 +18,10 @@ limitations under the License.
package tasks
import (
- "github.com/apache/incubator-devlake/plugins/core/dal"
"reflect"
+ "github.com/apache/incubator-devlake/plugins/core/dal"
+
"github.com/apache/incubator-devlake/models/domainlayer/code"
"github.com/apache/incubator-devlake/models/domainlayer/didgen"
"github.com/apache/incubator-devlake/plugins/core"
@@ -33,6 +34,7 @@ var ConvertCommitsMeta = core.SubTaskMeta{
EntryPoint: ConvertCommits,
EnabledByDefault: false,
Description: "Convert tool layer table github_commits into domain layer table commits",
+ DomainTypes: []string{core.DOMAIN_TYPE_CODE},
}
func ConvertCommits(taskCtx core.SubTaskContext) error {
diff --git a/plugins/github/tasks/commit_extractor.go b/plugins/github/tasks/commit_extractor.go
index 0ff36890..bebafa99 100644
--- a/plugins/github/tasks/commit_extractor.go
+++ b/plugins/github/tasks/commit_extractor.go
@@ -30,6 +30,7 @@ var ExtractApiCommitsMeta = core.SubTaskMeta{
EntryPoint: ExtractApiCommits,
EnabledByDefault: false,
Description: "Extract raw commit data into tool layer table github_commits",
+ DomainTypes: []string{core.DOMAIN_TYPE_CODE},
}
type CommitsResponse struct {
diff --git a/plugins/github/tasks/commit_stats_collector.go b/plugins/github/tasks/commit_stats_collector.go
index fa5fbba1..b5b46b9d 100644
--- a/plugins/github/tasks/commit_stats_collector.go
+++ b/plugins/github/tasks/commit_stats_collector.go
@@ -20,13 +20,14 @@ package tasks
import (
"encoding/json"
"fmt"
- "github.com/apache/incubator-devlake/plugins/core/dal"
- "github.com/apache/incubator-devlake/plugins/helper"
"io/ioutil"
"net/http"
"net/url"
"reflect"
+ "github.com/apache/incubator-devlake/plugins/core/dal"
+ "github.com/apache/incubator-devlake/plugins/helper"
+
"github.com/apache/incubator-devlake/plugins/core"
"github.com/apache/incubator-devlake/plugins/github/models"
)
@@ -38,6 +39,7 @@ var CollectApiCommitStatsMeta = core.SubTaskMeta{
EntryPoint: CollectApiCommitStats,
EnabledByDefault: false,
Description: "Collect commitStats data from Github api",
+ DomainTypes: []string{core.DOMAIN_TYPE_CODE},
}
func CollectApiCommitStats(taskCtx core.SubTaskContext) error {
diff --git a/plugins/github/tasks/commit_stats_extractor.go b/plugins/github/tasks/commit_stats_extractor.go
index f2f65e4d..48a20c5a 100644
--- a/plugins/github/tasks/commit_stats_extractor.go
+++ b/plugins/github/tasks/commit_stats_extractor.go
@@ -19,6 +19,7 @@ package tasks
import (
"encoding/json"
+
"github.com/apache/incubator-devlake/plugins/core/dal"
"github.com/apache/incubator-devlake/plugins/core"
@@ -31,6 +32,7 @@ var ExtractApiCommitStatsMeta = core.SubTaskMeta{
EntryPoint: ExtractApiCommitStats,
EnabledByDefault: false,
Description: "Extract raw commit stats data into tool layer table github_commit_stats",
+ DomainTypes: []string{core.DOMAIN_TYPE_CODE},
}
type ApiSingleCommitResponse struct {
diff --git a/plugins/github/tasks/event_collector.go b/plugins/github/tasks/event_collector.go
index f02fcc42..67156dad 100644
--- a/plugins/github/tasks/event_collector.go
+++ b/plugins/github/tasks/event_collector.go
@@ -20,10 +20,11 @@ package tasks
import (
"encoding/json"
"fmt"
- "github.com/apache/incubator-devlake/plugins/core/dal"
"net/http"
"net/url"
+ "github.com/apache/incubator-devlake/plugins/core/dal"
+
"github.com/apache/incubator-devlake/plugins/helper"
"github.com/apache/incubator-devlake/plugins/core"
@@ -39,6 +40,7 @@ var CollectApiEventsMeta = core.SubTaskMeta{
EntryPoint: CollectApiEvents,
EnabledByDefault: true,
Description: "Collect Events data from Github api",
+ DomainTypes: []string{core.DOMAIN_TYPE_TICKET},
}
func CollectApiEvents(taskCtx core.SubTaskContext) error {
diff --git a/plugins/github/tasks/event_extractor.go b/plugins/github/tasks/event_extractor.go
index d55283de..9081bc88 100644
--- a/plugins/github/tasks/event_extractor.go
+++ b/plugins/github/tasks/event_extractor.go
@@ -30,6 +30,7 @@ var ExtractApiEventsMeta = core.SubTaskMeta{
EntryPoint: ExtractApiEvents,
EnabledByDefault: true,
Description: "Extract raw Events data into tool layer table github_issue_events",
+ DomainTypes: []string{core.DOMAIN_TYPE_TICKET},
}
type IssueEvent struct {
diff --git a/plugins/github/tasks/issue_collector.go b/plugins/github/tasks/issue_collector.go
index fe378c0b..35f618d1 100644
--- a/plugins/github/tasks/issue_collector.go
+++ b/plugins/github/tasks/issue_collector.go
@@ -20,10 +20,11 @@ package tasks
import (
"encoding/json"
"fmt"
- "github.com/apache/incubator-devlake/plugins/core/dal"
"net/http"
"net/url"
+ "github.com/apache/incubator-devlake/plugins/core/dal"
+
"github.com/apache/incubator-devlake/plugins/helper"
"github.com/apache/incubator-devlake/plugins/core"
@@ -44,6 +45,7 @@ var CollectApiIssuesMeta = core.SubTaskMeta{
EntryPoint: CollectApiIssues,
EnabledByDefault: true,
Description: "Collect issues data from Github api",
+ DomainTypes: []string{core.DOMAIN_TYPE_TICKET},
}
func CollectApiIssues(taskCtx core.SubTaskContext) error {
diff --git a/plugins/github/tasks/issue_comment_convertor.go b/plugins/github/tasks/issue_comment_convertor.go
index 3099f916..27d57efa 100644
--- a/plugins/github/tasks/issue_comment_convertor.go
+++ b/plugins/github/tasks/issue_comment_convertor.go
@@ -18,6 +18,8 @@ limitations under the License.
package tasks
import (
+ "reflect"
+
"github.com/apache/incubator-devlake/models/domainlayer"
"github.com/apache/incubator-devlake/models/domainlayer/didgen"
"github.com/apache/incubator-devlake/models/domainlayer/ticket"
@@ -25,7 +27,6 @@ import (
"github.com/apache/incubator-devlake/plugins/core/dal"
githubModels "github.com/apache/incubator-devlake/plugins/github/models"
"github.com/apache/incubator-devlake/plugins/helper"
- "reflect"
)
var ConvertIssueCommentsMeta = core.SubTaskMeta{
@@ -33,6 +34,7 @@ var ConvertIssueCommentsMeta = core.SubTaskMeta{
EntryPoint: ConvertIssueComments,
EnabledByDefault: true,
Description: "ConvertIssueComments data from Github api",
+ DomainTypes: []string{core.DOMAIN_TYPE_TICKET},
}
func ConvertIssueComments(taskCtx core.SubTaskContext) error {
diff --git a/plugins/github/tasks/issue_convertor.go b/plugins/github/tasks/issue_convertor.go
index b5071cf9..b5ebdcaf 100644
--- a/plugins/github/tasks/issue_convertor.go
+++ b/plugins/github/tasks/issue_convertor.go
@@ -18,10 +18,11 @@ limitations under the License.
package tasks
import (
- "github.com/apache/incubator-devlake/plugins/core/dal"
"reflect"
"strconv"
+ "github.com/apache/incubator-devlake/plugins/core/dal"
+
"github.com/apache/incubator-devlake/plugins/core"
"github.com/apache/incubator-devlake/plugins/helper"
@@ -36,6 +37,7 @@ var ConvertIssuesMeta = core.SubTaskMeta{
EntryPoint: ConvertIssues,
EnabledByDefault: true,
Description: "Convert tool layer table github_issues into domain layer table issues",
+ DomainTypes: []string{core.DOMAIN_TYPE_TICKET},
}
func ConvertIssues(taskCtx core.SubTaskContext) error {
diff --git a/plugins/github/tasks/issue_extractor.go b/plugins/github/tasks/issue_extractor.go
index 700e8e76..4643501b 100644
--- a/plugins/github/tasks/issue_extractor.go
+++ b/plugins/github/tasks/issue_extractor.go
@@ -32,6 +32,7 @@ var ExtractApiIssuesMeta = core.SubTaskMeta{
EntryPoint: ExtractApiIssues,
EnabledByDefault: true,
Description: "Extract raw Issues data into tool layer table github_issues",
+ DomainTypes: []string{core.DOMAIN_TYPE_TICKET},
}
type IssuesResponse struct {
diff --git a/plugins/github/tasks/issue_label_convertor.go b/plugins/github/tasks/issue_label_convertor.go
index 6ba44d77..b5b7f105 100644
--- a/plugins/github/tasks/issue_label_convertor.go
+++ b/plugins/github/tasks/issue_label_convertor.go
@@ -18,9 +18,10 @@ limitations under the License.
package tasks
import (
- "github.com/apache/incubator-devlake/plugins/core/dal"
"reflect"
+ "github.com/apache/incubator-devlake/plugins/core/dal"
+
"github.com/apache/incubator-devlake/models/domainlayer/didgen"
"github.com/apache/incubator-devlake/models/domainlayer/ticket"
"github.com/apache/incubator-devlake/plugins/core"
@@ -33,6 +34,7 @@ var ConvertIssueLabelsMeta = core.SubTaskMeta{
EntryPoint: ConvertIssueLabels,
EnabledByDefault: true,
Description: "Convert tool layer table github_issue_labels into domain layer table issue_labels",
+ DomainTypes: []string{core.DOMAIN_TYPE_TICKET},
}
func ConvertIssueLabels(taskCtx core.SubTaskContext) error {
diff --git a/plugins/github/tasks/pr_collector.go b/plugins/github/tasks/pr_collector.go
index afdc1db5..9a6905f2 100644
--- a/plugins/github/tasks/pr_collector.go
+++ b/plugins/github/tasks/pr_collector.go
@@ -20,10 +20,11 @@ package tasks
import (
"encoding/json"
"fmt"
- "github.com/apache/incubator-devlake/plugins/core/dal"
"net/http"
"net/url"
+ "github.com/apache/incubator-devlake/plugins/core/dal"
+
"github.com/apache/incubator-devlake/plugins/helper"
"github.com/apache/incubator-devlake/plugins/core"
@@ -39,6 +40,7 @@ var CollectApiPullRequestsMeta = core.SubTaskMeta{
EntryPoint: CollectApiPullRequests,
EnabledByDefault: true,
Description: "Collect PullRequests data from Github api",
+ DomainTypes: []string{core.DOMAIN_TYPE_CODE},
}
func CollectApiPullRequests(taskCtx core.SubTaskContext) error {
diff --git a/plugins/github/tasks/pr_comment_convertor.go b/plugins/github/tasks/pr_comment_convertor.go
index 4538d92e..9bc13b6e 100644
--- a/plugins/github/tasks/pr_comment_convertor.go
+++ b/plugins/github/tasks/pr_comment_convertor.go
@@ -18,6 +18,8 @@ limitations under the License.
package tasks
import (
+ "reflect"
+
"github.com/apache/incubator-devlake/models/domainlayer"
"github.com/apache/incubator-devlake/models/domainlayer/code"
"github.com/apache/incubator-devlake/models/domainlayer/didgen"
@@ -25,7 +27,6 @@ import (
"github.com/apache/incubator-devlake/plugins/core/dal"
githubModels "github.com/apache/incubator-devlake/plugins/github/models"
"github.com/apache/incubator-devlake/plugins/helper"
- "reflect"
)
var ConvertPullRequestCommentsMeta = core.SubTaskMeta{
@@ -33,6 +34,7 @@ var ConvertPullRequestCommentsMeta = core.SubTaskMeta{
EntryPoint: ConvertPullRequestComments,
EnabledByDefault: true,
Description: "ConvertPullRequestComments data from Github api",
+ DomainTypes: []string{core.DOMAIN_TYPE_CODE},
}
func ConvertPullRequestComments(taskCtx core.SubTaskContext) error {
diff --git a/plugins/github/tasks/pr_commit_collector.go b/plugins/github/tasks/pr_commit_collector.go
index 53ec2b7f..faf39137 100644
--- a/plugins/github/tasks/pr_commit_collector.go
+++ b/plugins/github/tasks/pr_commit_collector.go
@@ -20,11 +20,12 @@ package tasks
import (
"encoding/json"
"fmt"
- "github.com/apache/incubator-devlake/plugins/core/dal"
"net/http"
"net/url"
"reflect"
+ "github.com/apache/incubator-devlake/plugins/core/dal"
+
"github.com/apache/incubator-devlake/plugins/helper"
"github.com/apache/incubator-devlake/plugins/core"
@@ -40,6 +41,7 @@ var CollectApiPullRequestCommitsMeta = core.SubTaskMeta{
EntryPoint: CollectApiPullRequestCommits,
EnabledByDefault: true,
Description: "Collect PullRequestCommits data from Github api",
+ DomainTypes: []string{core.DOMAIN_TYPE_CODE},
}
type SimplePr struct {
diff --git a/plugins/github/tasks/pr_commit_convertor.go b/plugins/github/tasks/pr_commit_convertor.go
index d4a5d564..c306e4e9 100644
--- a/plugins/github/tasks/pr_commit_convertor.go
+++ b/plugins/github/tasks/pr_commit_convertor.go
@@ -18,9 +18,10 @@ limitations under the License.
package tasks
import (
- "github.com/apache/incubator-devlake/plugins/core/dal"
"reflect"
+ "github.com/apache/incubator-devlake/plugins/core/dal"
+
"github.com/apache/incubator-devlake/models/domainlayer/code"
"github.com/apache/incubator-devlake/models/domainlayer/didgen"
"github.com/apache/incubator-devlake/plugins/core"
@@ -33,6 +34,7 @@ var ConvertPullRequestCommitsMeta = core.SubTaskMeta{
EntryPoint: ConvertPullRequestCommits,
EnabledByDefault: true,
Description: "Convert tool layer table github_pull_request_commits into domain layer table pull_request_commits",
+ DomainTypes: []string{core.DOMAIN_TYPE_CODE},
}
func ConvertPullRequestCommits(taskCtx core.SubTaskContext) (err error) {
diff --git a/plugins/github/tasks/pr_commit_extractor.go b/plugins/github/tasks/pr_commit_extractor.go
index 20c86090..74cb4dc5 100644
--- a/plugins/github/tasks/pr_commit_extractor.go
+++ b/plugins/github/tasks/pr_commit_extractor.go
@@ -31,6 +31,7 @@ var ExtractApiPullRequestCommitsMeta = core.SubTaskMeta{
EntryPoint: ExtractApiPullRequestCommits,
EnabledByDefault: true,
Description: "Extract raw PullRequestCommits data into tool layer table github_commits",
+ DomainTypes: []string{core.DOMAIN_TYPE_CODE},
}
type PrCommitsResponse struct {
diff --git a/plugins/github/tasks/pr_convertor.go b/plugins/github/tasks/pr_convertor.go
index 07159fd8..b4a8e973 100644
--- a/plugins/github/tasks/pr_convertor.go
+++ b/plugins/github/tasks/pr_convertor.go
@@ -18,9 +18,10 @@ limitations under the License.
package tasks
import (
- "github.com/apache/incubator-devlake/plugins/core/dal"
"reflect"
+ "github.com/apache/incubator-devlake/plugins/core/dal"
+
"github.com/apache/incubator-devlake/models/domainlayer"
"github.com/apache/incubator-devlake/models/domainlayer/code"
"github.com/apache/incubator-devlake/models/domainlayer/didgen"
@@ -34,6 +35,7 @@ var ConvertPullRequestsMeta = core.SubTaskMeta{
EntryPoint: ConvertPullRequests,
EnabledByDefault: true,
Description: "ConvertPullRequests data from Github api",
+ DomainTypes: []string{core.DOMAIN_TYPE_CODE},
}
func ConvertPullRequests(taskCtx core.SubTaskContext) error {
diff --git a/plugins/github/tasks/pr_extractor.go b/plugins/github/tasks/pr_extractor.go
index 00a58525..1790d658 100644
--- a/plugins/github/tasks/pr_extractor.go
+++ b/plugins/github/tasks/pr_extractor.go
@@ -31,6 +31,7 @@ var ExtractApiPullRequestsMeta = core.SubTaskMeta{
EntryPoint: ExtractApiPullRequests,
EnabledByDefault: true,
Description: "Extract raw PullRequests data into tool layer table github_pull_requests",
+ DomainTypes: []string{core.DOMAIN_TYPE_CODE},
}
type GithubApiPullRequest struct {
diff --git a/plugins/github/tasks/pr_issue_convertor.go b/plugins/github/tasks/pr_issue_convertor.go
index 84b6ed1e..10b9cac8 100644
--- a/plugins/github/tasks/pr_issue_convertor.go
+++ b/plugins/github/tasks/pr_issue_convertor.go
@@ -18,9 +18,10 @@ limitations under the License.
package tasks
import (
- "github.com/apache/incubator-devlake/plugins/core/dal"
"reflect"
+ "github.com/apache/incubator-devlake/plugins/core/dal"
+
"github.com/apache/incubator-devlake/models/domainlayer/crossdomain"
"github.com/apache/incubator-devlake/models/domainlayer/didgen"
"github.com/apache/incubator-devlake/plugins/core"
@@ -33,6 +34,7 @@ var ConvertPullRequestIssuesMeta = core.SubTaskMeta{
EntryPoint: ConvertPullRequestIssues,
EnabledByDefault: true,
Description: "Convert tool layer table github_pull_request_issues into domain layer table pull_request_issues",
+ DomainTypes: []string{core.DOMAIN_TYPE_CODE},
}
func ConvertPullRequestIssues(taskCtx core.SubTaskContext) error {
diff --git a/plugins/github/tasks/pr_issue_enricher.go b/plugins/github/tasks/pr_issue_enricher.go
index 9ce7f25f..510c0363 100644
--- a/plugins/github/tasks/pr_issue_enricher.go
+++ b/plugins/github/tasks/pr_issue_enricher.go
@@ -18,14 +18,15 @@ limitations under the License.
package tasks
import (
- "github.com/apache/incubator-devlake/plugins/core"
- "github.com/apache/incubator-devlake/plugins/core/dal"
- githubModels "github.com/apache/incubator-devlake/plugins/github/models"
- "github.com/apache/incubator-devlake/plugins/helper"
"reflect"
"regexp"
"strconv"
"strings"
+
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/core/dal"
+ githubModels "github.com/apache/incubator-devlake/plugins/github/models"
+ "github.com/apache/incubator-devlake/plugins/helper"
)
var EnrichPullRequestIssuesMeta = core.SubTaskMeta{
@@ -33,6 +34,7 @@ var EnrichPullRequestIssuesMeta = core.SubTaskMeta{
EntryPoint: EnrichPullRequestIssues,
EnabledByDefault: true,
Description: "Create tool layer table github_pull_request_issues from github_pull_reqeusts",
+ DomainTypes: []string{core.DOMAIN_TYPE_CODE, core.DOMAIN_TYPE_TICKET},
}
func EnrichPullRequestIssues(taskCtx core.SubTaskContext) (err error) {
diff --git a/plugins/github/tasks/pr_label_convertor.go b/plugins/github/tasks/pr_label_convertor.go
index ad64c925..bfa6f109 100644
--- a/plugins/github/tasks/pr_label_convertor.go
+++ b/plugins/github/tasks/pr_label_convertor.go
@@ -18,9 +18,10 @@ limitations under the License.
package tasks
import (
- "github.com/apache/incubator-devlake/plugins/core/dal"
"reflect"
+ "github.com/apache/incubator-devlake/plugins/core/dal"
+
"github.com/apache/incubator-devlake/models/domainlayer/code"
"github.com/apache/incubator-devlake/models/domainlayer/didgen"
"github.com/apache/incubator-devlake/plugins/core"
@@ -33,6 +34,7 @@ var ConvertPullRequestLabelsMeta = core.SubTaskMeta{
EntryPoint: ConvertPullRequestLabels,
EnabledByDefault: true,
Description: "Convert tool layer table github_pull_request_labels into domain layer table pull_request_labels",
+ DomainTypes: []string{core.DOMAIN_TYPE_CODE},
}
func ConvertPullRequestLabels(taskCtx core.SubTaskContext) error {
diff --git a/plugins/github/tasks/pr_review_collector.go b/plugins/github/tasks/pr_review_collector.go
index 47ba8f4e..3364b251 100644
--- a/plugins/github/tasks/pr_review_collector.go
+++ b/plugins/github/tasks/pr_review_collector.go
@@ -20,12 +20,13 @@ package tasks
import (
"encoding/json"
"fmt"
- "github.com/apache/incubator-devlake/plugins/core/dal"
- "github.com/apache/incubator-devlake/plugins/github/models"
"net/http"
"net/url"
"reflect"
+ "github.com/apache/incubator-devlake/plugins/core/dal"
+ "github.com/apache/incubator-devlake/plugins/github/models"
+
"github.com/apache/incubator-devlake/plugins/helper"
"github.com/apache/incubator-devlake/plugins/core"
@@ -40,6 +41,7 @@ var CollectApiPullRequestReviewsMeta = core.SubTaskMeta{
EntryPoint: CollectApiPullRequestReviews,
EnabledByDefault: true,
Description: "Collect PullRequestReviews data from Github api",
+ DomainTypes: []string{core.DOMAIN_TYPE_CODE},
}
func CollectApiPullRequestReviews(taskCtx core.SubTaskContext) error {
diff --git a/plugins/github/tasks/pr_review_extractor.go b/plugins/github/tasks/pr_review_extractor.go
index 70eaad1a..1136ed0d 100644
--- a/plugins/github/tasks/pr_review_extractor.go
+++ b/plugins/github/tasks/pr_review_extractor.go
@@ -31,6 +31,7 @@ var ExtractApiPullRequestReviewersMeta = core.SubTaskMeta{
EntryPoint: ExtractApiPullRequestReviewers,
EnabledByDefault: true,
Description: "Extract raw PullRequestReviewers data into tool layer table github_reviewers",
+ DomainTypes: []string{core.DOMAIN_TYPE_CODE},
}
type PullRequestReview struct {
diff --git a/plugins/github/tasks/repo_collector.go b/plugins/github/tasks/repo_collector.go
index 29544354..ae41431a 100644
--- a/plugins/github/tasks/repo_collector.go
+++ b/plugins/github/tasks/repo_collector.go
@@ -36,6 +36,7 @@ var CollectApiRepoMeta = core.SubTaskMeta{
EntryPoint: CollectApiRepositories,
Required: true,
Description: "Collect repositories data from Github api",
+ DomainTypes: core.DOMAIN_TYPES,
}
func CollectApiRepositories(taskCtx core.SubTaskContext) error {
diff --git a/plugins/github/tasks/repo_convertor.go b/plugins/github/tasks/repo_convertor.go
index 4ad0d6bc..7809df04 100644
--- a/plugins/github/tasks/repo_convertor.go
+++ b/plugins/github/tasks/repo_convertor.go
@@ -19,9 +19,10 @@ package tasks
import (
"fmt"
- "github.com/apache/incubator-devlake/plugins/core/dal"
"reflect"
+ "github.com/apache/incubator-devlake/plugins/core/dal"
+
"github.com/apache/incubator-devlake/models/domainlayer"
"github.com/apache/incubator-devlake/models/domainlayer/code"
"github.com/apache/incubator-devlake/models/domainlayer/didgen"
@@ -36,6 +37,7 @@ var ConvertRepoMeta = core.SubTaskMeta{
EntryPoint: ConvertRepo,
EnabledByDefault: true,
Description: "Convert tool layer table github_repos into domain layer table repos and boards",
+ DomainTypes: core.DOMAIN_TYPES,
}
func ConvertRepo(taskCtx core.SubTaskContext) error {
diff --git a/plugins/github/tasks/repositorie_extractor.go b/plugins/github/tasks/repo_extractor.go
similarity index 97%
rename from plugins/github/tasks/repositorie_extractor.go
rename to plugins/github/tasks/repo_extractor.go
index 27102cca..4446ff8a 100644
--- a/plugins/github/tasks/repositorie_extractor.go
+++ b/plugins/github/tasks/repo_extractor.go
@@ -31,6 +31,7 @@ var ExtractApiRepoMeta = core.SubTaskMeta{
EntryPoint: ExtractApiRepositories,
Required: true,
Description: "Extract raw Repositories data into tool layer table github_repos",
+ DomainTypes: core.DOMAIN_TYPES,
}
type ApiRepoResponse GithubApiRepo
@@ -45,6 +46,7 @@ type GithubApiRepo struct {
Parent *GithubApiRepo `json:"parent"`
CreatedAt helper.Iso8601Time `json:"created_at"`
UpdatedAt *helper.Iso8601Time `json:"updated_at"`
+ CloneUrl string `json:"clone_url"`
}
func ExtractApiRepositories(taskCtx core.SubTaskContext) error {
diff --git a/plugins/github/tasks/task_data.go b/plugins/github/tasks/task_data.go
index 6a7d6ad4..1866cdca 100644
--- a/plugins/github/tasks/task_data.go
+++ b/plugins/github/tasks/task_data.go
@@ -18,10 +18,12 @@ limitations under the License.
package tasks
import (
+ "fmt"
"time"
"github.com/apache/incubator-devlake/plugins/github/models"
"github.com/apache/incubator-devlake/plugins/helper"
+ "github.com/mitchellh/mapstructure"
)
type GithubOptions struct {
@@ -39,3 +41,50 @@ type GithubTaskData struct {
Since *time.Time
Repo *models.GithubRepo
}
+
+func DecodeAndValidateTaskOptions(options map[string]interface{}) (*GithubOptions, error) {
+ var op GithubOptions
+ err := mapstructure.Decode(options, &op)
+ if err != nil {
+ return nil, err
+ }
+ if op.Owner == "" {
+ return nil, fmt.Errorf("owner is required for GitHub execution")
+ }
+ if op.Repo == "" {
+ return nil, fmt.Errorf("repo is required for GitHub execution")
+ }
+ if op.PrType == "" {
+ op.PrType = "type/(.*)$"
+ }
+ if op.PrComponent == "" {
+ op.PrComponent = "component/(.*)$"
+ }
+ if op.PrBodyClosePattern == "" {
+ op.PrBodyClosePattern = "(?mi)(fix|close|resolve|fixes|closes|resolves|fixed|closed|resolved)[\\s]*.*(((and )?(#|https:\\/\\/github.com\\/%s\\/%s\\/issues\\/)\\d+[ ]*)+)"
+ }
+ if op.IssueSeverity == "" {
+ op.IssueSeverity = "severity/(.*)$"
+ }
+ if op.IssuePriority == "" {
+ op.IssuePriority = "^(highest|high|medium|low)$"
+ }
+ if op.IssueComponent == "" {
+ op.IssueComponent = "component/(.*)$"
+ }
+ if op.IssueTypeBug == "" {
+ op.IssueTypeBug = "^(bug|failure|error)$"
+ }
+ if op.IssueTypeIncident == "" {
+ op.IssueTypeIncident = ""
+ }
+ if op.IssueTypeRequirement == "" {
+ op.IssueTypeRequirement = "^(feat|feature|proposal|requirement)$"
+ }
+
+ // find the needed GitHub now
+ if op.ConnectionId == 0 {
+ return nil, fmt.Errorf("connectionId is invalid")
+ }
+ return &op, nil
+}
diff --git a/plugins/github/tasks/user_convertor.go b/plugins/github/tasks/user_convertor.go
index b35ea21f..284746ea 100644
--- a/plugins/github/tasks/user_convertor.go
+++ b/plugins/github/tasks/user_convertor.go
@@ -18,9 +18,10 @@ limitations under the License.
package tasks
import (
- "github.com/apache/incubator-devlake/plugins/core/dal"
"reflect"
+ "github.com/apache/incubator-devlake/plugins/core/dal"
+
"github.com/apache/incubator-devlake/models/domainlayer"
"github.com/apache/incubator-devlake/models/domainlayer/didgen"
"github.com/apache/incubator-devlake/models/domainlayer/user"
@@ -34,6 +35,7 @@ var ConvertUsersMeta = core.SubTaskMeta{
EntryPoint: ConvertUsers,
EnabledByDefault: false,
Description: "Convert tool layer table github_users into domain layer table users",
+ DomainTypes: []string{core.DOMAIN_TYPE_CROSS},
}
func ConvertUsers(taskCtx core.SubTaskContext) error {
diff --git a/plugins/helper/pipeline_plan.go b/plugins/helper/pipeline_plan.go
new file mode 100644
index 00000000..f8f1d6d3
--- /dev/null
+++ b/plugins/helper/pipeline_plan.go
@@ -0,0 +1,49 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package helper
+
+import (
+ "fmt"
+
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/utils"
+)
+
+// MakePipelinePlanSubtasks generates subtasks list based on sub-task meta information and entities wanted by user
+func MakePipelinePlanSubtasks(subtaskMetas []core.SubTaskMeta, entities []string) ([]string, error) {
+ subtasks := make([]string, 0)
+ if len(entities) == 0 {
+ return subtasks, nil
+ }
+ wanted := make(map[string]bool, len(entities))
+ for _, entity := range entities {
+ if !utils.StringsContains(core.DOMAIN_TYPES, entity) {
+ return nil, fmt.Errorf("invalid entity(domain type): %s", entity)
+ }
+ wanted[entity] = true
+ }
+ for _, subtaskMeta := range subtaskMetas {
+ for _, neededBy := range subtaskMeta.DomainTypes {
+ if wanted[neededBy] {
+ subtasks = append(subtasks, subtaskMeta.Name)
+ break
+ }
+ }
+ }
+ return subtasks, nil
+}
diff --git a/plugins/helper/pipeline_plan_test.go b/plugins/helper/pipeline_plan_test.go
new file mode 100644
index 00000000..f39e86d6
--- /dev/null
+++ b/plugins/helper/pipeline_plan_test.go
@@ -0,0 +1,76 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package helper
+
+import (
+ "testing"
+
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/stretchr/testify/assert"
+)
+
+func TestMakePipelinePlanSubtasks(t *testing.T) {
+
+ subtasks1, err := MakePipelinePlanSubtasks(
+ []core.SubTaskMeta{
+ {
+ Name: "collectApiIssues",
+ DomainTypes: []string{core.DOMAIN_TYPE_TICKET},
+ },
+ {
+ Name: "extractApiIssues",
+ DomainTypes: []string{core.DOMAIN_TYPE_TICKET},
+ },
+ {
+ Name: "collectApiPullRequests",
+ DomainTypes: []string{core.DOMAIN_TYPE_CODE},
+ },
+ },
+ []string{core.DOMAIN_TYPE_TICKET},
+ )
+ assert.Nil(t, err)
+ assert.Equal(
+ t,
+ subtasks1,
+ []string{"collectApiIssues", "extractApiIssues"},
+ )
+
+ subtasks2, err := MakePipelinePlanSubtasks(
+ []core.SubTaskMeta{
+ {
+ Name: "collectApiRepo",
+ DomainTypes: []string{core.DOMAIN_TYPE_TICKET, core.DOMAIN_TYPE_CODE},
+ },
+ {
+ Name: "collectApiIssues",
+ DomainTypes: []string{core.DOMAIN_TYPE_TICKET},
+ },
+ {
+ Name: "collectApiPullRequests",
+ DomainTypes: []string{core.DOMAIN_TYPE_CODE},
+ },
+ },
+ []string{core.DOMAIN_TYPE_TICKET},
+ )
+ assert.Nil(t, err)
+ assert.Equal(
+ t,
+ subtasks2,
+ []string{"collectApiRepo", "collectApiIssues"},
+ )
+}
diff --git a/services/blueprint.go b/services/blueprint.go
index f6b6eb8c..080c6aae 100644
--- a/services/blueprint.go
+++ b/services/blueprint.go
@@ -184,17 +184,19 @@ func ReloadBlueprints(c *cron.Cron) error {
}
c.Stop()
for _, pp := range blueprints {
- if pp.Mode == models.BLUEPRINT_MODE_NORMAL {
- // for NORMAL mode, we have to generate the actual pipeline plan beforehand
- pp.Plan, err = GeneratePlanJson(pp.Settings)
- if err != nil {
- return err
- }
- err = db.Save(pp).Error
- if err != nil {
- return err
+ /*
+ if pp.Mode == models.BLUEPRINT_MODE_NORMAL {
+ // for NORMAL mode, we have to generate the actual pipeline plan beforehand
+ pp.Plan, err = GeneratePlanJson(pp.Settings)
+ if err != nil {
+ return err
+ }
+ err = db.Save(pp).Error
+ if err != nil {
+ return err
+ }
}
- }
+ */
var plan core.PipelinePlan
err = json.Unmarshal(pp.Plan, &plan)
if err != nil {
diff --git a/plugins/github/tasks/task_data.go b/utils/strings.go
similarity index 54%
copy from plugins/github/tasks/task_data.go
copy to utils/strings.go
index 6a7d6ad4..b4a57bed 100644
--- a/plugins/github/tasks/task_data.go
+++ b/utils/strings.go
@@ -15,27 +15,27 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
-package tasks
+package utils
-import (
- "time"
-
- "github.com/apache/incubator-devlake/plugins/github/models"
- "github.com/apache/incubator-devlake/plugins/helper"
-)
-
-type GithubOptions struct {
- ConnectionId uint64 `json:"connectionId"`
- Tasks []string `json:"tasks,omitempty"`
- Since string
- Owner string
- Repo string
- models.TransformationRules `mapstructure:"transformationRules" json:"transformationRules"`
+// StringsUniq returns a new String Slice contains deduped elements from `source`
+func StringsUniq(source []string) []string {
+ book := make(map[string]bool, len(source))
+ target := make([]string, 0, len(source))
+ for _, str := range source {
+ if !book[str] {
+ book[str] = true
+ target = append(target, str)
+ }
+ }
+ return target
}
-type GithubTaskData struct {
- Options *GithubOptions
- ApiClient *helper.ApiAsyncClient
- Since *time.Time
- Repo *models.GithubRepo
+// StringsContains checks if `source` String Slice contains `target` string
+func StringsContains(slice []string, target string) bool {
+ for _, str := range slice {
+ if str == target {
+ return true
+ }
+ }
+ return false
}