You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by kl...@apache.org on 2023/07/20 07:29:07 UTC
[incubator-devlake] branch main updated: refactor: add subtask register for github plugin (#5411)
This is an automated email from the ASF dual-hosted git repository.
klesh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new cc39ef25a refactor: add subtask register for github plugin (#5411)
cc39ef25a is described below
commit cc39ef25a5931be9479e742c12299a9c30f5e400
Author: chenggui53 <ch...@gmail.com>
AuthorDate: Thu Jul 20 15:29:02 2023 +0800
refactor: add subtask register for github plugin (#5411)
* refactor: add subtask registert for github
* refactor: add subtask register
* refactor: add subtask register
* refactor: add subtask register
* refactor: add subtask register
* refactor: add subtask register
* refactor: add subtask register
* refactor: add subtask register
* refactor: add subtask register
* refactor: add subtask register
* refactor: add subtask register
* refactor: add subtask register
* refactor: add subtask register
* refactor: add subtask register
* refactor: add subtask register
* refactor: add subtask register
* refactor: add subtask meta register
* refactor: add subtask register
* refactor: add subtask register
* refactor: add subtask register
* refactor: Update remote.go
* refactor: add subtask register
* refactor: add subtask register
* refactor: add subtask register
* refactor: add subtask register
* refactor: add subtask register
* refactor: update subtask register
* refactor: update subtask register
* refactor: add subtask register
* refactor: add subtask register
* refactor: add subtask register
---
backend/core/plugin/plugin_task.go | 3 +
.../pluginhelper/api/api_collector_with_state.go | 2 +-
backend/helpers/pluginhelper/api/batch_save.go | 2 +-
.../helpers/pluginhelper/api/connection_auths.go | 3 +-
.../sorter}/interface.go | 9 ++-
.../sorter/subtask.go} | 76 ++++++--------------
.../sorter/subtask_test.go} | 11 +--
.../pluginhelper/subtaskmeta/sorter/table.go | 80 +++++++++++++++++++++
.../pluginhelper/subtaskmeta/sorter/table_test.go | 81 ++++++++++++++++++++++
.../pluginhelper/subtaskmeta/sorter/utils.go | 72 +++++++++++++++++++
.../pluginhelper/subtaskmeta/sorter/utils_test.go | 73 +++++++++++++++++++
backend/plugins/github/impl/impl.go | 63 ++++-------------
.../github/impl/impl_test.go} | 24 +++++--
backend/plugins/github/tasks/account_collector.go | 24 ++++---
backend/plugins/github/tasks/account_convertor.go | 14 +++-
backend/plugins/github/tasks/account_extractor.go | 9 ++-
.../plugins/github/tasks/account_org_collector.go | 32 ++++++---
.../plugins/github/tasks/account_org_extractor.go | 9 ++-
backend/plugins/github/tasks/api_client.go | 7 --
backend/plugins/github/tasks/cicd_job_collector.go | 6 ++
backend/plugins/github/tasks/cicd_job_convertor.go | 16 ++++-
backend/plugins/github/tasks/cicd_job_extractor.go | 6 ++
backend/plugins/github/tasks/cicd_run_collector.go | 6 ++
backend/plugins/github/tasks/cicd_run_convertor.go | 13 ++++
backend/plugins/github/tasks/cicd_run_extractor.go | 6 ++
backend/plugins/github/tasks/comment_collector.go | 22 +++---
backend/plugins/github/tasks/comment_extractor.go | 12 +++-
backend/plugins/github/tasks/commit_collector.go | 6 ++
backend/plugins/github/tasks/commit_convertor.go | 17 ++++-
backend/plugins/github/tasks/commit_extractor.go | 9 +++
.../plugins/github/tasks/commit_stats_collector.go | 9 +++
.../plugins/github/tasks/commit_stats_extractor.go | 9 +++
backend/plugins/github/tasks/event_collector.go | 6 ++
backend/plugins/github/tasks/event_extractor.go | 7 ++
.../github/tasks/issue_assignee_convertor.go | 11 +++
backend/plugins/github/tasks/issue_collector.go | 6 ++
.../github/tasks/issue_comment_convertor.go | 10 +++
backend/plugins/github/tasks/issue_convertor.go | 19 ++++-
backend/plugins/github/tasks/issue_extractor.go | 10 +++
.../plugins/github/tasks/issue_label_convertor.go | 12 +++-
.../plugins/github/tasks/milestone_collector.go | 6 ++
.../plugins/github/tasks/milestone_converter.go | 16 ++++-
.../plugins/github/tasks/milestone_extractor.go | 7 ++
backend/plugins/github/tasks/pr_collector.go | 6 ++
.../plugins/github/tasks/pr_comment_convertor.go | 14 +++-
.../plugins/github/tasks/pr_commit_collector.go | 6 ++
.../plugins/github/tasks/pr_commit_convertor.go | 10 +++
.../plugins/github/tasks/pr_commit_extractor.go | 9 +++
backend/plugins/github/tasks/pr_convertor.go | 10 +++
backend/plugins/github/tasks/pr_extractor.go | 9 +++
backend/plugins/github/tasks/pr_issue_convertor.go | 14 +++-
backend/plugins/github/tasks/pr_issue_enricher.go | 16 ++++-
backend/plugins/github/tasks/pr_label_convertor.go | 12 +++-
.../plugins/github/tasks/pr_review_collector.go | 6 ++
.../github/tasks/pr_review_comment_collector.go | 22 +++---
.../github/tasks/pr_review_comment_extractor.go | 10 ++-
.../plugins/github/tasks/pr_review_convertor.go | 13 +++-
.../plugins/github/tasks/pr_review_extractor.go | 12 +++-
.../github/tasks/register.go} | 8 ++-
backend/plugins/github/tasks/repo_convertor.go | 26 +++++--
backend/plugins/github/tasks/shared.go | 7 ++
backend/plugins/github/tasks/task_data.go | 1 -
backend/plugins/gitlab/impl/impl.go | 6 +-
63 files changed, 857 insertions(+), 201 deletions(-)
diff --git a/backend/core/plugin/plugin_task.go b/backend/core/plugin/plugin_task.go
index 9baf274e3..d7dbc14bc 100644
--- a/backend/core/plugin/plugin_task.go
+++ b/backend/core/plugin/plugin_task.go
@@ -19,6 +19,7 @@ package plugin
import (
"context"
+
corecontext "github.com/apache/incubator-devlake/core/context"
"github.com/apache/incubator-devlake/core/errors"
)
@@ -98,6 +99,8 @@ type SubTaskMeta struct {
Description string
DomainTypes []string
Dependencies []*SubTaskMeta
+ DependencyTables []string
+ ProductTables []string
}
// PluginTask Implement this interface to let framework run tasks for you
diff --git a/backend/helpers/pluginhelper/api/api_collector_with_state.go b/backend/helpers/pluginhelper/api/api_collector_with_state.go
index 65d4e2176..b3282459b 100644
--- a/backend/helpers/pluginhelper/api/api_collector_with_state.go
+++ b/backend/helpers/pluginhelper/api/api_collector_with_state.go
@@ -41,7 +41,7 @@ type ApiCollectorStateManager struct {
ExecuteStart time.Time
}
-// NewApiCollectorWithState create a new ApiCollectorStateManager
+// NewStatefulApiCollector create a new ApiCollectorStateManager
func NewStatefulApiCollector(args RawDataSubTaskArgs, timeAfter *time.Time) (*ApiCollectorStateManager, errors.Error) {
db := args.Ctx.GetDal()
diff --git a/backend/helpers/pluginhelper/api/batch_save.go b/backend/helpers/pluginhelper/api/batch_save.go
index bf4cf06bb..f89b93106 100644
--- a/backend/helpers/pluginhelper/api/batch_save.go
+++ b/backend/helpers/pluginhelper/api/batch_save.go
@@ -29,7 +29,7 @@ import (
"github.com/apache/incubator-devlake/core/log"
)
-// BatchSave performs mulitple records persistence of a specific type in one sql query to improve the performance
+// BatchSave performs multiple records persistence of a specific type in one sql query to improve the performance
type BatchSave struct {
basicRes context.BasicRes
log log.Logger
diff --git a/backend/helpers/pluginhelper/api/connection_auths.go b/backend/helpers/pluginhelper/api/connection_auths.go
index acb4da04d..6219b9a5d 100644
--- a/backend/helpers/pluginhelper/api/connection_auths.go
+++ b/backend/helpers/pluginhelper/api/connection_auths.go
@@ -24,9 +24,8 @@ import (
"reflect"
"strings"
- "github.com/apache/incubator-devlake/core/plugin"
-
"github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
"github.com/go-playground/validator/v10"
)
diff --git a/backend/helpers/pluginhelper/subtaskmeta_sorter/interface.go b/backend/helpers/pluginhelper/subtaskmeta/sorter/interface.go
similarity index 82%
copy from backend/helpers/pluginhelper/subtaskmeta_sorter/interface.go
copy to backend/helpers/pluginhelper/subtaskmeta/sorter/interface.go
index fb4e30b5e..61a51f5b7 100644
--- a/backend/helpers/pluginhelper/subtaskmeta_sorter/interface.go
+++ b/backend/helpers/pluginhelper/subtaskmeta/sorter/interface.go
@@ -15,10 +15,13 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
-package subtaskmeta_sorter
+package sorter
-import "github.com/apache/incubator-devlake/core/plugin"
+import (
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
+)
type SubTaskMetaSorter interface {
- Sort() ([]plugin.SubTaskMeta, error)
+ Sort() ([]plugin.SubTaskMeta, errors.Error)
}
diff --git a/backend/helpers/pluginhelper/subtaskmeta_sorter/dependency.go b/backend/helpers/pluginhelper/subtaskmeta/sorter/subtask.go
similarity index 53%
rename from backend/helpers/pluginhelper/subtaskmeta_sorter/dependency.go
rename to backend/helpers/pluginhelper/subtaskmeta/sorter/subtask.go
index d310c1fe5..a3873b8d4 100644
--- a/backend/helpers/pluginhelper/subtaskmeta_sorter/dependency.go
+++ b/backend/helpers/pluginhelper/subtaskmeta/sorter/subtask.go
@@ -15,12 +15,14 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
-package subtaskmeta_sorter
+package sorter
import (
"fmt"
+
+ "github.com/apache/incubator-devlake/core/errors"
+
"github.com/apache/incubator-devlake/core/plugin"
- "sort"
)
type DependencySorter struct {
@@ -31,12 +33,12 @@ func NewDependencySorter(metas []*plugin.SubTaskMeta) SubTaskMetaSorter {
return &DependencySorter{metas: metas}
}
-func (d *DependencySorter) Sort() ([]plugin.SubTaskMeta, error) {
- return topologicalSort(d.metas)
+func (d *DependencySorter) Sort() ([]plugin.SubTaskMeta, errors.Error) {
+ return dependenciesTopologicalSort(d.metas)
}
// stable topological sort
-func topologicalSort(metas []*plugin.SubTaskMeta) ([]plugin.SubTaskMeta, error) {
+func dependenciesTopologicalSort(metas []*plugin.SubTaskMeta) ([]plugin.SubTaskMeta, errors.Error) {
// which state will make a cycle
dependenciesMap := make(map[string][]string)
nameMetaMap := make(map[string]*plugin.SubTaskMeta)
@@ -56,63 +58,25 @@ func topologicalSort(metas []*plugin.SubTaskMeta) ([]plugin.SubTaskMeta, error)
dependenciesMap[item.Name] = make([]string, 0)
}
} else {
- return nil, fmt.Errorf("duplicate subtaskmetas detected in list: %s", item.Name)
+ return nil, errors.Convert(fmt.Errorf("duplicate subtaskmetas detected in list: %s", item.Name))
}
}
- orderedSubtaskList := make([]plugin.SubTaskMeta, 0)
- for {
- if len(dependenciesMap) == 0 {
- break
- }
-
- tmpList := make([]string, 0)
- for key, item := range dependenciesMap {
- if len(item) == 0 {
- tmpList = append(tmpList, key)
- }
- }
- if len(tmpList) == 0 {
- return nil, fmt.Errorf("cyclic dependency detected: %v", dependenciesMap)
- }
-
- // remove item in dependencies map
- for key, value := range dependenciesMap {
- if contains(tmpList, key) {
- delete(dependenciesMap, key)
- } else {
- dependenciesMap[key] = removeElements(value, tmpList)
- }
- }
-
- sort.Strings(tmpList)
- // convert item to subtaskmeta by name, and append to orderedSubtaskList
- for _, item := range tmpList {
- value, ok := nameMetaMap[item]
- if !ok {
- return nil, fmt.Errorf("illeagal subtaskmeta detected %s", item)
- }
- orderedSubtaskList = append(orderedSubtaskList, *value)
- }
+ // sort
+ orderStrList, err := topologicalSortSameElements(dependenciesMap)
+ if err != nil {
+ return nil, errors.Convert(err)
}
- return orderedSubtaskList, nil
-}
-func contains[T comparable](itemList []T, item T) bool {
- for _, newItem := range itemList {
- if item == newItem {
- return true
+ // gen list by sorted name list and return
+ orderedSubtaskList := make([]plugin.SubTaskMeta, 0)
+ for _, item := range orderStrList {
+ value, ok := nameMetaMap[item]
+ if !ok {
+ return nil, errors.Convert(fmt.Errorf("illeagal subtaskmeta detected %s", item))
}
+ orderedSubtaskList = append(orderedSubtaskList, *value)
}
- return false
-}
-func removeElements[T comparable](raw, toRemove []T) []T {
- newList := make([]T, 0)
- for _, item := range raw {
- if !contains(toRemove, item) {
- newList = append(newList, item)
- }
- }
- return newList
+ return orderedSubtaskList, nil
}
diff --git a/backend/helpers/pluginhelper/subtaskmeta_sorter/dependency_test.go b/backend/helpers/pluginhelper/subtaskmeta/sorter/subtask_test.go
similarity index 90%
rename from backend/helpers/pluginhelper/subtaskmeta_sorter/dependency_test.go
rename to backend/helpers/pluginhelper/subtaskmeta/sorter/subtask_test.go
index 451837ff2..0bf43bcff 100644
--- a/backend/helpers/pluginhelper/subtaskmeta_sorter/dependency_test.go
+++ b/backend/helpers/pluginhelper/subtaskmeta/sorter/subtask_test.go
@@ -15,12 +15,13 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
-package subtaskmeta_sorter
+package sorter
import (
- "github.com/apache/incubator-devlake/core/plugin"
"reflect"
"testing"
+
+ "github.com/apache/incubator-devlake/core/plugin"
)
func Test_topologicalSort(t *testing.T) {
@@ -85,13 +86,13 @@ func Test_topologicalSort(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
- got, err := topologicalSort(tt.args.metas)
+ got, err := dependenciesTopologicalSort(tt.args.metas)
if (err != nil) != tt.wantErr {
- t.Errorf("topologicalSort() error = %v, wantErr %v", err, tt.wantErr)
+ t.Errorf("dependenciesTopologicalSort() error = %v, wantErr %v", err, tt.wantErr)
return
}
if !reflect.DeepEqual(got, tt.want) {
- t.Errorf("topologicalSort() got = %v, want %v", got, tt.want)
+ t.Errorf("dependenciesTopologicalSort() got = %v, want %v", got, tt.want)
}
})
}
diff --git a/backend/helpers/pluginhelper/subtaskmeta/sorter/table.go b/backend/helpers/pluginhelper/subtaskmeta/sorter/table.go
new file mode 100644
index 000000000..9468b5c95
--- /dev/null
+++ b/backend/helpers/pluginhelper/subtaskmeta/sorter/table.go
@@ -0,0 +1,80 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package sorter
+
+import (
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
+)
+
+type TableSorter struct {
+ metas []*plugin.SubTaskMeta
+}
+
+func NewTableSorter(metas []*plugin.SubTaskMeta) SubTaskMetaSorter {
+ return &TableSorter{metas: metas}
+}
+
+func (d *TableSorter) Sort() ([]plugin.SubTaskMeta, errors.Error) {
+ return tableTopologicalSort(d.metas)
+}
+
+func tableTopologicalSort(metas []*plugin.SubTaskMeta) ([]plugin.SubTaskMeta, errors.Error) {
+ constructedMetas := constructDependenciesByTable(metas)
+ return dependenciesTopologicalSort(constructedMetas)
+}
+
+func constructDependenciesByTable(metas []*plugin.SubTaskMeta) []*plugin.SubTaskMeta {
+ // construct map by metas and their produced tables, the key is table, and value is metas
+ tableMetasMap := make(map[string][]*plugin.SubTaskMeta)
+ for _, item := range metas {
+ for _, tableItem := range item.ProductTables {
+ if value, ok := tableMetasMap[tableItem]; ok {
+ tableMetasMap[tableItem] = append(value, item)
+ } else {
+ tableMetasMap[tableItem] = []*plugin.SubTaskMeta{item}
+ }
+ }
+ }
+ // construct meta dependencies by meta.TableDependencies
+ // use noDupMap to deduplicate dependencies of meta
+ noDupMap := make(map[*plugin.SubTaskMeta]map[*plugin.SubTaskMeta]any)
+ for _, metaItem := range metas {
+ // convert dependency tables to dependency metas
+ dependenciesMap, ok := noDupMap[metaItem]
+ if !ok {
+ noDupMap[metaItem] = make(map[*plugin.SubTaskMeta]any)
+ dependenciesMap = noDupMap[metaItem]
+ }
+ for _, tableItem := range metaItem.DependencyTables {
+ for _, item := range tableMetasMap[tableItem] {
+ dependenciesMap[item] = ""
+ }
+ }
+ metaItem.Dependencies = keys(dependenciesMap)
+ }
+ return metas
+}
+
+func keys[T comparable](raw map[T]any) []T {
+ list := make([]T, 0)
+ for key := range raw {
+ list = append(list, key)
+ }
+ return list
+}
diff --git a/backend/helpers/pluginhelper/subtaskmeta/sorter/table_test.go b/backend/helpers/pluginhelper/subtaskmeta/sorter/table_test.go
new file mode 100644
index 000000000..4503d1eb0
--- /dev/null
+++ b/backend/helpers/pluginhelper/subtaskmeta/sorter/table_test.go
@@ -0,0 +1,81 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package sorter
+
+import (
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
+ "reflect"
+ "testing"
+)
+
+func Test_tableTopologicalSort(t *testing.T) {
+ pluginA := plugin.SubTaskMeta{
+ Name: "A",
+ DependencyTables: []string{},
+ ProductTables: []string{"_TOOL_TEST_TABLE", "_TOOL_TEST_TABLE2"},
+ }
+ pluginB := plugin.SubTaskMeta{
+ Name: "B",
+ DependencyTables: []string{"_TOOL_TEST_TABLE"},
+ ProductTables: []string{"_TOOL_TEST_TABLE2", "_TOOL_TEST_TABLE3"},
+ }
+ pluginC := plugin.SubTaskMeta{
+ Name: "C",
+ DependencyTables: []string{"_TOOL_TEST_TABLE"},
+ ProductTables: []string{"_TOOL_TEST_TABLE3"},
+ }
+ pluginD := plugin.SubTaskMeta{
+ Name: "D",
+ DependencyTables: []string{"_TOOL_TEST_TABLE2", "_TOOL_TEST_TABLE3"},
+ ProductTables: []string{"_TOOL_TEST_TABLE4"},
+ }
+
+ type args struct {
+ metas []*plugin.SubTaskMeta
+ }
+ tests := []struct {
+ name string
+ args args
+ want []plugin.SubTaskMeta
+ want1 errors.Error
+ }{
+ {
+ name: "test sorter",
+ args: args{[]*plugin.SubTaskMeta{&pluginA, &pluginB, &pluginC, &pluginD}},
+ want: []plugin.SubTaskMeta{pluginA, pluginB, pluginC, pluginD},
+ want1: nil,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got, got1 := tableTopologicalSort(tt.args.metas)
+ if len(got) != len(tt.want) {
+ t.Errorf("tableTopologicalSort() got = %v, want %v", got, tt.want)
+ }
+ for index, item := range got {
+ if item.Name != tt.want[index].Name {
+ t.Errorf("tableTopologicalSort() got = %v, want %v, not equal with index = %d", got, tt.want, index)
+ }
+ }
+ if !reflect.DeepEqual(got1, tt.want1) {
+ t.Errorf("tableTopologicalSort() got1 = %v, want %v", got1, tt.want1)
+ }
+ })
+ }
+}
diff --git a/backend/helpers/pluginhelper/subtaskmeta/sorter/utils.go b/backend/helpers/pluginhelper/subtaskmeta/sorter/utils.go
new file mode 100644
index 000000000..d2d6c9fea
--- /dev/null
+++ b/backend/helpers/pluginhelper/subtaskmeta/sorter/utils.go
@@ -0,0 +1,72 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package sorter
+
+import (
+ "fmt"
+ "sort"
+)
+
+// topologicalSortSameElements
+func topologicalSortSameElements(sameElementsDependencyMap map[string][]string) ([]string, error) {
+ sortedKeyList := make([]string, 0)
+ for {
+ if len(sameElementsDependencyMap) == 0 {
+ break
+ }
+ tmpList := make([]string, 0)
+ for key, item := range sameElementsDependencyMap {
+ if len(item) == 0 {
+ tmpList = append(tmpList, key)
+ }
+ }
+ if len(tmpList) == 0 {
+ return nil, fmt.Errorf("cyclic dependency detected: %v", sameElementsDependencyMap)
+ }
+ // remove item in dependencies map
+ for key, value := range sameElementsDependencyMap {
+ if contains(tmpList, key) {
+ delete(sameElementsDependencyMap, key)
+ } else {
+ sameElementsDependencyMap[key] = removeElements(value, tmpList)
+ }
+ }
+ sort.Strings(tmpList)
+ sortedKeyList = append(sortedKeyList, tmpList...)
+ }
+ return sortedKeyList, nil
+}
+
+func contains[T comparable](itemList []T, item T) bool {
+ for _, newItem := range itemList {
+ if item == newItem {
+ return true
+ }
+ }
+ return false
+}
+
+func removeElements[T comparable](raw, toRemove []T) []T {
+ newList := make([]T, 0)
+ for _, item := range raw {
+ if !contains(toRemove, item) {
+ newList = append(newList, item)
+ }
+ }
+ return newList
+}
diff --git a/backend/helpers/pluginhelper/subtaskmeta/sorter/utils_test.go b/backend/helpers/pluginhelper/subtaskmeta/sorter/utils_test.go
new file mode 100644
index 000000000..175d143ad
--- /dev/null
+++ b/backend/helpers/pluginhelper/subtaskmeta/sorter/utils_test.go
@@ -0,0 +1,73 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package sorter
+
+import (
+ "reflect"
+ "testing"
+)
+
+func Test_topologicalSortSameElements(t *testing.T) {
+ type args struct {
+ dependenciesMap map[string][]string
+ }
+ tests := []struct {
+ name string
+ args args
+ want []string
+ wantErr bool
+ }{
+ {
+ name: "correct stable sort",
+ args: args{
+ map[string][]string{
+ "Aa": {"B", "C"},
+ "Ac": {"B", "C"},
+ "B": {"C"},
+ "C": {},
+ },
+ },
+ want: []string{"C", "B", "Aa", "Ac"},
+ wantErr: false,
+ },
+ {
+ name: "cyclic error",
+ args: args{
+ map[string][]string{
+ "A": {"B", "C"},
+ "B": {"C"},
+ "C": {"A"},
+ },
+ },
+ want: nil,
+ wantErr: true,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got, err := topologicalSortSameElements(tt.args.dependenciesMap)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("topologicalSortSameElements() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ if !reflect.DeepEqual(got, tt.want) {
+ t.Errorf("topologicalSortSameElements() got = %v, want %v", got, tt.want)
+ }
+ })
+ }
+}
diff --git a/backend/plugins/github/impl/impl.go b/backend/plugins/github/impl/impl.go
index 08f2ef986..0151d81de 100644
--- a/backend/plugins/github/impl/impl.go
+++ b/backend/plugins/github/impl/impl.go
@@ -19,13 +19,13 @@ package impl
import (
"fmt"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/subtaskmeta/sorter"
"time"
- "github.com/apache/incubator-devlake/core/models/domainlayer/devops"
-
"github.com/apache/incubator-devlake/core/context"
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/models/domainlayer/devops"
"github.com/apache/incubator-devlake/core/plugin"
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/github/api"
@@ -45,8 +45,19 @@ var _ interface {
plugin.CloseablePluginTask
} = (*Github)(nil)
+var sortedSubtaskMetas []plugin.SubTaskMeta
+
type Github struct{}
+func init() {
+ var err error
+ // check subtask meta loop and gen subtask list when plugin init
+ sortedSubtaskMetas, err = sorter.NewTableSorter(tasks.SubTaskMetaList).Sort()
+ if err != nil {
+ panic(err)
+ }
+}
+
func (p Github) Connection() dal.Tabler {
return &models.GithubConnection{}
}
@@ -103,53 +114,7 @@ func (p Github) Name() string {
}
func (p Github) SubTaskMetas() []plugin.SubTaskMeta {
- return []plugin.SubTaskMeta{
- tasks.CollectApiIssuesMeta,
- tasks.ExtractApiIssuesMeta,
- tasks.CollectApiPullRequestsMeta,
- tasks.ExtractApiPullRequestsMeta,
- tasks.CollectApiCommentsMeta,
- tasks.ExtractApiCommentsMeta,
- tasks.CollectApiEventsMeta,
- tasks.ExtractApiEventsMeta,
- tasks.CollectApiPullRequestCommitsMeta,
- tasks.ExtractApiPullRequestCommitsMeta,
- tasks.CollectApiPullRequestReviewsMeta,
- tasks.ExtractApiPullRequestReviewsMeta,
- tasks.CollectApiPrReviewCommentsMeta,
- tasks.ExtractApiPrReviewCommentsMeta,
- tasks.CollectApiCommitsMeta,
- tasks.ExtractApiCommitsMeta,
- tasks.CollectApiCommitStatsMeta,
- tasks.ExtractApiCommitStatsMeta,
- tasks.CollectMilestonesMeta,
- tasks.ExtractMilestonesMeta,
- tasks.CollectAccountsMeta,
- tasks.ExtractAccountsMeta,
- tasks.CollectAccountOrgMeta,
- tasks.ExtractAccountOrgMeta,
- tasks.CollectRunsMeta,
- tasks.ExtractRunsMeta,
- tasks.ConvertRunsMeta,
- tasks.CollectJobsMeta,
- tasks.ExtractJobsMeta,
- tasks.ConvertJobsMeta,
- tasks.EnrichPullRequestIssuesMeta,
- tasks.ConvertRepoMeta,
- tasks.ConvertIssuesMeta,
- tasks.ConvertIssueAssigneeMeta,
- tasks.ConvertCommitsMeta,
- tasks.ConvertIssueLabelsMeta,
- tasks.ConvertPullRequestCommitsMeta,
- tasks.ConvertPullRequestsMeta,
- tasks.ConvertPullRequestReviewsMeta,
- tasks.ConvertPullRequestLabelsMeta,
- tasks.ConvertPullRequestIssuesMeta,
- tasks.ConvertIssueCommentsMeta,
- tasks.ConvertPullRequestCommentsMeta,
- tasks.ConvertMilestonesMeta,
- tasks.ConvertAccountsMeta,
- }
+ return sortedSubtaskMetas
}
func (p Github) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]interface{}) (interface{}, errors.Error) {
diff --git a/backend/helpers/pluginhelper/subtaskmeta_sorter/interface.go b/backend/plugins/github/impl/impl_test.go
similarity index 55%
copy from backend/helpers/pluginhelper/subtaskmeta_sorter/interface.go
copy to backend/plugins/github/impl/impl_test.go
index fb4e30b5e..63af43b09 100644
--- a/backend/helpers/pluginhelper/subtaskmeta_sorter/interface.go
+++ b/backend/plugins/github/impl/impl_test.go
@@ -6,7 +6,7 @@ The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
+ http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
@@ -15,10 +15,24 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
-package subtaskmeta_sorter
+package impl
-import "github.com/apache/incubator-devlake/core/plugin"
+import (
+ "testing"
-type SubTaskMetaSorter interface {
- Sort() ([]plugin.SubTaskMeta, error)
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/subtaskmeta/sorter"
+ "github.com/apache/incubator-devlake/plugins/github/tasks"
+)
+
+func Test_genSubtaskList(t *testing.T) {
+ testSortedSubtaskList, err := sorter.NewTableSorter(tasks.SubTaskMetaList).Sort()
+ if err != nil {
+ t.Error(err)
+ return
+ }
+ subtaskNameList := make([]string, len(testSortedSubtaskList))
+ for index, item := range testSortedSubtaskList {
+ subtaskNameList[index] = item.Name
+ }
+ t.Logf("got subtask list %s", subtaskNameList)
}
diff --git a/backend/plugins/github/tasks/account_collector.go b/backend/plugins/github/tasks/account_collector.go
index 2aa8beada..931a7ae7c 100644
--- a/backend/plugins/github/tasks/account_collector.go
+++ b/backend/plugins/github/tasks/account_collector.go
@@ -30,12 +30,28 @@ import (
"github.com/apache/incubator-devlake/plugins/github/models"
)
+func init() {
+ RegisterSubtaskMeta(&CollectAccountsMeta)
+}
+
const RAW_ACCOUNT_TABLE = "github_api_accounts"
type SimpleAccount struct {
Login string
}
+var CollectAccountsMeta = plugin.SubTaskMeta{
+ Name: "collectAccounts",
+ EntryPoint: CollectAccounts,
+ EnabledByDefault: true,
+ Description: "Collect accounts data from Github api, does not support either timeFilter or diffSync.",
+ DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS},
+ DependencyTables: []string{
+ //models.GithubRepoAccount{}.TableName() // cursor, config will not regard as dependency
+ },
+ ProductTables: []string{RAW_ACCOUNT_TABLE},
+}
+
func CollectAccounts(taskCtx plugin.SubTaskContext) errors.Error {
db := taskCtx.GetDal()
data := taskCtx.GetData().(*GithubTaskData)
@@ -85,11 +101,3 @@ func CollectAccounts(taskCtx plugin.SubTaskContext) errors.Error {
}
return collector.Execute()
}
-
-var CollectAccountsMeta = plugin.SubTaskMeta{
- Name: "collectAccounts",
- EntryPoint: CollectAccounts,
- EnabledByDefault: true,
- Description: "Collect accounts data from Github api, does not support either timeFilter or diffSync.",
- DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS},
-}
diff --git a/backend/plugins/github/tasks/account_convertor.go b/backend/plugins/github/tasks/account_convertor.go
index 53ad1563d..c93c80407 100644
--- a/backend/plugins/github/tasks/account_convertor.go
+++ b/backend/plugins/github/tasks/account_convertor.go
@@ -18,6 +18,9 @@ limitations under the License.
package tasks
import (
+ "reflect"
+ "strings"
+
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models/common"
@@ -27,16 +30,23 @@ import (
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/github/models"
- "reflect"
- "strings"
)
+func init() {
+ RegisterSubtaskMeta(&ConvertAccountsMeta)
+}
+
var ConvertAccountsMeta = plugin.SubTaskMeta{
Name: "convertAccounts",
EntryPoint: ConvertAccounts,
EnabledByDefault: true,
Description: "Convert tool layer table github_accounts into domain layer table accounts",
DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS},
+ DependencyTables: []string{
+ models.GithubAccount{}.TableName(), // cursor
+ models.GithubRepoAccount{}.TableName(), // cursor
+ models.GithubAccountOrg{}.TableName()}, // account id gen
+ ProductTables: []string{crossdomain.Account{}.TableName()},
}
type GithubAccountWithOrg struct {
diff --git a/backend/plugins/github/tasks/account_extractor.go b/backend/plugins/github/tasks/account_extractor.go
index 3c2f6896b..7fa2c0f13 100644
--- a/backend/plugins/github/tasks/account_extractor.go
+++ b/backend/plugins/github/tasks/account_extractor.go
@@ -19,19 +19,26 @@ package tasks
import (
"encoding/json"
+ "time"
+
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/github/models"
- "time"
)
+func init() {
+ RegisterSubtaskMeta(&ExtractAccountsMeta)
+}
+
var ExtractAccountsMeta = plugin.SubTaskMeta{
Name: "extractAccounts",
EntryPoint: ExtractAccounts,
EnabledByDefault: true,
Description: "Extract raw account data into tool layer table github_accounts",
DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS},
+ DependencyTables: []string{RAW_ACCOUNT_TABLE},
+ ProductTables: []string{models.GithubAccount{}.TableName()},
}
type DetailGithubAccountResponse struct {
diff --git a/backend/plugins/github/tasks/account_org_collector.go b/backend/plugins/github/tasks/account_org_collector.go
index ae8c569ad..86889fdb4 100644
--- a/backend/plugins/github/tasks/account_org_collector.go
+++ b/backend/plugins/github/tasks/account_org_collector.go
@@ -19,16 +19,21 @@ package tasks
import (
"encoding/json"
+ "io"
+ "net/http"
+ "reflect"
+
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/github/models"
- "io"
- "net/http"
- "reflect"
)
+func init() {
+ RegisterSubtaskMeta(&CollectAccountOrgMeta)
+}
+
const RAW_ACCOUNT_ORG_TABLE = "github_api_account_orgs"
type SimpleAccountWithId struct {
@@ -36,6 +41,19 @@ type SimpleAccountWithId struct {
AccountId int
}
+var CollectAccountOrgMeta = plugin.SubTaskMeta{
+ Name: "collectAccountOrg",
+ EntryPoint: CollectAccountOrg,
+ EnabledByDefault: true,
+ Description: "Collect accounts org data from Github api, does not support either timeFilter or diffSync.",
+ DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS},
+ DependencyTables: []string{
+ models.GithubRepoAccount{}.TableName(), // cursor
+ models.GithubAccount{}.TableName(), // cursor
+ },
+ ProductTables: []string{RAW_ACCOUNT_ORG_TABLE},
+}
+
func CollectAccountOrg(taskCtx plugin.SubTaskContext) errors.Error {
db := taskCtx.GetDal()
data := taskCtx.GetData().(*GithubTaskData)
@@ -85,11 +103,3 @@ func CollectAccountOrg(taskCtx plugin.SubTaskContext) errors.Error {
}
return collector.Execute()
}
-
-var CollectAccountOrgMeta = plugin.SubTaskMeta{
- Name: "collectAccountOrg",
- EntryPoint: CollectAccountOrg,
- EnabledByDefault: true,
- Description: "Collect accounts org data from Github api, does not support either timeFilter or diffSync.",
- DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS},
-}
diff --git a/backend/plugins/github/tasks/account_org_extractor.go b/backend/plugins/github/tasks/account_org_extractor.go
index 1f37c1041..60d93ce7f 100644
--- a/backend/plugins/github/tasks/account_org_extractor.go
+++ b/backend/plugins/github/tasks/account_org_extractor.go
@@ -19,18 +19,25 @@ package tasks
import (
"encoding/json"
+
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/github/models"
)
+func init() {
+ RegisterSubtaskMeta(&ExtractAccountOrgMeta)
+}
+
var ExtractAccountOrgMeta = plugin.SubTaskMeta{
- Name: "ExtractAccountOrg",
+ Name: "extractAccountOrg",
EntryPoint: ExtractAccountOrg,
EnabledByDefault: true,
Description: "Extract raw account org data into tool layer table github_account_orgs",
DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS},
+ DependencyTables: []string{RAW_ACCOUNT_ORG_TABLE},
+ ProductTables: []string{models.GithubAccountOrg{}.TableName()},
}
type GithubAccountOrgsResponse struct {
diff --git a/backend/plugins/github/tasks/api_client.go b/backend/plugins/github/tasks/api_client.go
index 3e1440e71..c9bfa852c 100644
--- a/backend/plugins/github/tasks/api_client.go
+++ b/backend/plugins/github/tasks/api_client.go
@@ -83,10 +83,3 @@ func CreateApiClient(taskCtx plugin.TaskContext, connection *models.GithubConnec
}
return asyncApiClient, nil
}
-
-func ignoreHTTPStatus422(res *http.Response) errors.Error {
- if res.StatusCode == http.StatusUnprocessableEntity {
- return api.ErrIgnoreAndContinue
- }
- return nil
-}
diff --git a/backend/plugins/github/tasks/cicd_job_collector.go b/backend/plugins/github/tasks/cicd_job_collector.go
index 4dd95c5c5..787476c31 100644
--- a/backend/plugins/github/tasks/cicd_job_collector.go
+++ b/backend/plugins/github/tasks/cicd_job_collector.go
@@ -31,6 +31,10 @@ import (
"github.com/apache/incubator-devlake/plugins/github/models"
)
+func init() {
+ RegisterSubtaskMeta(&CollectJobsMeta)
+}
+
const RAW_JOB_TABLE = "github_api_jobs"
var CollectJobsMeta = plugin.SubTaskMeta{
@@ -39,6 +43,8 @@ var CollectJobsMeta = plugin.SubTaskMeta{
EnabledByDefault: true,
Description: "Collect Jobs data from Github action api, supports both timeFilter and diffSync.",
DomainTypes: []string{plugin.DOMAIN_TYPE_CICD},
+ DependencyTables: []string{models.GithubRun{}.TableName()},
+ ProductTables: []string{RAW_JOB_TABLE},
}
func CollectJobs(taskCtx plugin.SubTaskContext) errors.Error {
diff --git a/backend/plugins/github/tasks/cicd_job_convertor.go b/backend/plugins/github/tasks/cicd_job_convertor.go
index b59ae50da..b1fd214f3 100644
--- a/backend/plugins/github/tasks/cicd_job_convertor.go
+++ b/backend/plugins/github/tasks/cicd_job_convertor.go
@@ -18,6 +18,9 @@ limitations under the License.
package tasks
import (
+ "reflect"
+ "strings"
+
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models/domainlayer"
@@ -26,16 +29,25 @@ import (
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/github/models"
- "reflect"
- "strings"
)
+func init() {
+ RegisterSubtaskMeta(&ConvertJobsMeta)
+}
+
var ConvertJobsMeta = plugin.SubTaskMeta{
Name: "convertJobs",
EntryPoint: ConvertJobs,
EnabledByDefault: true,
Description: "Convert tool layer table github_jobs into domain layer table cicd_tasks",
DomainTypes: []string{plugin.DOMAIN_TYPE_CICD},
+ DependencyTables: []string{
+ RAW_JOB_TABLE,
+ models.GithubJob{}.TableName(), // cursor and generator
+ models.GithubRun{}.TableName(), // id generator
+ //models.GithubRepo{}.TableName(), // id generator, but config will not regard as dependency
+ },
+ ProductTables: []string{devops.CICDTask{}.TableName()},
}
type SimpleBranch struct {
diff --git a/backend/plugins/github/tasks/cicd_job_extractor.go b/backend/plugins/github/tasks/cicd_job_extractor.go
index edb96cccf..9193b0342 100644
--- a/backend/plugins/github/tasks/cicd_job_extractor.go
+++ b/backend/plugins/github/tasks/cicd_job_extractor.go
@@ -28,12 +28,18 @@ import (
"github.com/apache/incubator-devlake/plugins/github/models"
)
+func init() {
+ RegisterSubtaskMeta(&ExtractJobsMeta)
+}
+
var ExtractJobsMeta = plugin.SubTaskMeta{
Name: "extractJobs",
EntryPoint: ExtractJobs,
EnabledByDefault: true,
Description: "Extract raw run data into tool layer table github_jobs",
DomainTypes: []string{plugin.DOMAIN_TYPE_CICD},
+ DependencyTables: []string{RAW_JOB_TABLE},
+ ProductTables: []string{models.GithubJob{}.TableName()},
}
func ExtractJobs(taskCtx plugin.SubTaskContext) errors.Error {
diff --git a/backend/plugins/github/tasks/cicd_run_collector.go b/backend/plugins/github/tasks/cicd_run_collector.go
index 3af4a66ce..78f9da18c 100644
--- a/backend/plugins/github/tasks/cicd_run_collector.go
+++ b/backend/plugins/github/tasks/cicd_run_collector.go
@@ -33,6 +33,10 @@ import (
"github.com/apache/incubator-devlake/plugins/github/models"
)
+func init() {
+ RegisterSubtaskMeta(&CollectRunsMeta)
+}
+
const RAW_RUN_TABLE = "github_api_runs"
// Although the API accepts a maximum of 100 entries per page, sometimes
@@ -56,6 +60,8 @@ var CollectRunsMeta = plugin.SubTaskMeta{
EnabledByDefault: true,
Description: "Collect Runs data from Github action api, supports both timeFilter and diffSync.",
DomainTypes: []string{plugin.DOMAIN_TYPE_CICD},
+ DependencyTables: []string{},
+ ProductTables: []string{RAW_RUN_TABLE},
}
func CollectRuns(taskCtx plugin.SubTaskContext) errors.Error {
diff --git a/backend/plugins/github/tasks/cicd_run_convertor.go b/backend/plugins/github/tasks/cicd_run_convertor.go
index 09bccc84f..151ebb4f2 100644
--- a/backend/plugins/github/tasks/cicd_run_convertor.go
+++ b/backend/plugins/github/tasks/cicd_run_convertor.go
@@ -31,12 +31,25 @@ import (
"github.com/apache/incubator-devlake/plugins/github/models"
)
+func init() {
+ RegisterSubtaskMeta(&ConvertRunsMeta)
+}
+
var ConvertRunsMeta = plugin.SubTaskMeta{
Name: "convertRuns",
EntryPoint: ConvertRuns,
EnabledByDefault: true,
Description: "Convert tool layer table github_runs into domain layer table cicd_pipeline",
DomainTypes: []string{plugin.DOMAIN_TYPE_CICD},
+ DependencyTables: []string{
+ //models.GithubRepo{}.TableName(), // config will not regard as dependency
+ models.GithubRun{}.TableName(),
+ RAW_RUN_TABLE,
+ },
+ ProductTables: []string{
+ devops.CICDPipeline{}.TableName(),
+ devops.CiCDPipelineCommit{}.TableName(),
+ },
}
func ConvertRuns(taskCtx plugin.SubTaskContext) errors.Error {
diff --git a/backend/plugins/github/tasks/cicd_run_extractor.go b/backend/plugins/github/tasks/cicd_run_extractor.go
index de3326f09..a62edef91 100644
--- a/backend/plugins/github/tasks/cicd_run_extractor.go
+++ b/backend/plugins/github/tasks/cicd_run_extractor.go
@@ -27,12 +27,18 @@ import (
"github.com/apache/incubator-devlake/plugins/github/models"
)
+func init() {
+ RegisterSubtaskMeta(&ExtractRunsMeta)
+}
+
var ExtractRunsMeta = plugin.SubTaskMeta{
Name: "extractRuns",
EntryPoint: ExtractRuns,
EnabledByDefault: true,
Description: "Extract raw run data into tool layer table github_runs",
DomainTypes: []string{plugin.DOMAIN_TYPE_CICD},
+ DependencyTables: []string{RAW_RUN_TABLE},
+ ProductTables: []string{models.GithubRun{}.TableName()},
}
func ExtractRuns(taskCtx plugin.SubTaskContext) errors.Error {
diff --git a/backend/plugins/github/tasks/comment_collector.go b/backend/plugins/github/tasks/comment_collector.go
index 709a7dc35..402bfa4e8 100644
--- a/backend/plugins/github/tasks/comment_collector.go
+++ b/backend/plugins/github/tasks/comment_collector.go
@@ -28,8 +28,22 @@ import (
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
)
+func init() {
+ RegisterSubtaskMeta(&CollectApiCommentsMeta)
+}
+
const RAW_COMMENTS_TABLE = "github_api_comments"
+var CollectApiCommentsMeta = plugin.SubTaskMeta{
+ Name: "collectApiComments",
+ EntryPoint: CollectApiComments,
+ EnabledByDefault: true,
+ Description: "Collect comments data from Github api, supports both timeFilter and diffSync.",
+ DomainTypes: []string{plugin.DOMAIN_TYPE_CODE_REVIEW, plugin.DOMAIN_TYPE_TICKET},
+ DependencyTables: []string{},
+ ProductTables: []string{RAW_COMMENTS_TABLE},
+}
+
func CollectApiComments(taskCtx plugin.SubTaskContext) errors.Error {
data := taskCtx.GetData().(*GithubTaskData)
collectorWithState, err := helper.NewStatefulApiCollector(helper.RawDataSubTaskArgs{
@@ -87,11 +101,3 @@ func CollectApiComments(taskCtx plugin.SubTaskContext) errors.Error {
return collectorWithState.Execute()
}
-
-var CollectApiCommentsMeta = plugin.SubTaskMeta{
- Name: "collectApiComments",
- EntryPoint: CollectApiComments,
- EnabledByDefault: true,
- Description: "Collect comments data from Github api, supports both timeFilter and diffSync.",
- DomainTypes: []string{plugin.DOMAIN_TYPE_CODE_REVIEW, plugin.DOMAIN_TYPE_TICKET},
-}
diff --git a/backend/plugins/github/tasks/comment_extractor.go b/backend/plugins/github/tasks/comment_extractor.go
index d0bc66a3e..9f18b0091 100644
--- a/backend/plugins/github/tasks/comment_extractor.go
+++ b/backend/plugins/github/tasks/comment_extractor.go
@@ -19,6 +19,7 @@ package tasks
import (
"encoding/json"
+
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
@@ -27,13 +28,22 @@ import (
githubUtils "github.com/apache/incubator-devlake/plugins/github/utils"
)
+func init() {
+ RegisterSubtaskMeta(&ExtractApiCommentsMeta)
+}
+
var ExtractApiCommentsMeta = plugin.SubTaskMeta{
Name: "extractApiComments",
EntryPoint: ExtractApiComments,
EnabledByDefault: true,
Description: "Extract raw comment data into tool layer table github_pull_request_comments" +
"and github_issue_comments",
- DomainTypes: []string{plugin.DOMAIN_TYPE_CODE_REVIEW, plugin.DOMAIN_TYPE_TICKET},
+ DomainTypes: []string{plugin.DOMAIN_TYPE_CODE_REVIEW, plugin.DOMAIN_TYPE_TICKET},
+ DependencyTables: []string{RAW_COMMENTS_TABLE},
+ ProductTables: []string{
+ models.GithubPrComment{}.TableName(),
+ models.GithubIssueComment{}.TableName(),
+ models.GithubRepoAccount{}.TableName()},
}
type IssueComment struct {
diff --git a/backend/plugins/github/tasks/commit_collector.go b/backend/plugins/github/tasks/commit_collector.go
index 35f7e95da..33e005331 100644
--- a/backend/plugins/github/tasks/commit_collector.go
+++ b/backend/plugins/github/tasks/commit_collector.go
@@ -28,6 +28,10 @@ import (
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
)
+func init() {
+ RegisterSubtaskMeta(&CollectApiCommitsMeta)
+}
+
const RAW_COMMIT_TABLE = "github_api_commits"
var CollectApiCommitsMeta = plugin.SubTaskMeta{
@@ -36,6 +40,8 @@ var CollectApiCommitsMeta = plugin.SubTaskMeta{
EnabledByDefault: false,
Description: "Collect commits data from Github api, supports both timeFilter and diffSync.",
DomainTypes: []string{plugin.DOMAIN_TYPE_CODE},
+ DependencyTables: []string{},
+ ProductTables: []string{RAW_COMMIT_TABLE},
}
func CollectApiCommits(taskCtx plugin.SubTaskContext) errors.Error {
diff --git a/backend/plugins/github/tasks/commit_convertor.go b/backend/plugins/github/tasks/commit_convertor.go
index 5b8589374..e8dcfa5e9 100644
--- a/backend/plugins/github/tasks/commit_convertor.go
+++ b/backend/plugins/github/tasks/commit_convertor.go
@@ -18,6 +18,8 @@ limitations under the License.
package tasks
import (
+ "reflect"
+
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models/domainlayer/code"
@@ -25,15 +27,26 @@ import (
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/github/models"
- "reflect"
)
+func init() {
+ RegisterSubtaskMeta(&ConvertCommitsMeta)
+}
+
var ConvertCommitsMeta = plugin.SubTaskMeta{
Name: "convertCommits",
EntryPoint: ConvertCommits,
EnabledByDefault: false,
Description: "Convert tool layer table github_commits into domain layer table commits",
DomainTypes: []string{plugin.DOMAIN_TYPE_CODE},
+ DependencyTables: []string{
+ models.GithubCommit{}.TableName(), // cursor
+ models.GithubRepoCommit{}.TableName(), // cursor
+ //models.GithubRepo{}.TableName(), // id generator, but config not regard as dependency
+ RAW_COMMIT_TABLE},
+ ProductTables: []string{
+ code.Commit{}.TableName(),
+ code.RepoCommit{}.TableName()},
}
func ConvertCommits(taskCtx plugin.SubTaskContext) errors.Error {
@@ -64,7 +77,7 @@ func ConvertCommits(taskCtx plugin.SubTaskContext) errors.Error {
ConnectionId: data.Options.ConnectionId,
Name: data.Options.Name,
},
- Table: RAW_COMMENTS_TABLE,
+ Table: RAW_COMMIT_TABLE,
},
InputRowType: reflect.TypeOf(models.GithubCommit{}),
Input: cursor,
diff --git a/backend/plugins/github/tasks/commit_extractor.go b/backend/plugins/github/tasks/commit_extractor.go
index b1d486105..51a71640e 100644
--- a/backend/plugins/github/tasks/commit_extractor.go
+++ b/backend/plugins/github/tasks/commit_extractor.go
@@ -19,18 +19,27 @@ package tasks
import (
"encoding/json"
+
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/github/models"
)
+func init() {
+ RegisterSubtaskMeta(&ExtractApiCommitsMeta)
+}
+
var ExtractApiCommitsMeta = plugin.SubTaskMeta{
Name: "extractApiCommits",
EntryPoint: ExtractApiCommits,
EnabledByDefault: false,
Description: "Extract raw commit data into tool layer table github_commits",
DomainTypes: []string{plugin.DOMAIN_TYPE_CODE},
+ DependencyTables: []string{RAW_COMMIT_TABLE},
+ ProductTables: []string{
+ models.GithubCommit{}.TableName(),
+ models.GithubRepoCommit{}.TableName()},
}
type CommitsResponse struct {
diff --git a/backend/plugins/github/tasks/commit_stats_collector.go b/backend/plugins/github/tasks/commit_stats_collector.go
index 6d2cbc410..3aa71635f 100644
--- a/backend/plugins/github/tasks/commit_stats_collector.go
+++ b/backend/plugins/github/tasks/commit_stats_collector.go
@@ -32,6 +32,10 @@ import (
"github.com/apache/incubator-devlake/plugins/github/models"
)
+func init() {
+ RegisterSubtaskMeta(&CollectApiCommitStatsMeta)
+}
+
const RAW_COMMIT_STATS_TABLE = "github_api_commit_stats"
var CollectApiCommitStatsMeta = plugin.SubTaskMeta{
@@ -40,6 +44,11 @@ var CollectApiCommitStatsMeta = plugin.SubTaskMeta{
EnabledByDefault: false,
Description: "Collect commitStats data from Github api, does not support either timeFilter or diffSync.",
DomainTypes: []string{plugin.DOMAIN_TYPE_CODE},
+ DependencyTables: []string{
+ //models.GithubRepoCommit{}.TableName(), // cursor, config will not regard as dependency
+ //models.GithubCommit{}.TableName()}, // cursor
+ },
+ ProductTables: []string{RAW_COMMIT_STATS_TABLE},
}
func CollectApiCommitStats(taskCtx plugin.SubTaskContext) errors.Error {
diff --git a/backend/plugins/github/tasks/commit_stats_extractor.go b/backend/plugins/github/tasks/commit_stats_extractor.go
index 0cbe30993..e6011f5c0 100644
--- a/backend/plugins/github/tasks/commit_stats_extractor.go
+++ b/backend/plugins/github/tasks/commit_stats_extractor.go
@@ -19,6 +19,7 @@ package tasks
import (
"encoding/json"
+
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
@@ -26,12 +27,20 @@ import (
"github.com/apache/incubator-devlake/plugins/github/models"
)
+func init() {
+ RegisterSubtaskMeta(&ExtractApiCommitStatsMeta)
+}
+
var ExtractApiCommitStatsMeta = plugin.SubTaskMeta{
Name: "extractApiCommitStats",
EntryPoint: ExtractApiCommitStats,
EnabledByDefault: false,
Description: "Extract raw commit stats data into tool layer table github_commit_stats",
DomainTypes: []string{plugin.DOMAIN_TYPE_CODE},
+ DependencyTables: []string{RAW_COMMIT_STATS_TABLE},
+ ProductTables: []string{
+ models.GithubCommit{}.TableName(),
+ models.GithubCommitStat{}.TableName()},
}
type ApiSingleCommitResponse struct {
diff --git a/backend/plugins/github/tasks/event_collector.go b/backend/plugins/github/tasks/event_collector.go
index da15a98de..a26d5ef64 100644
--- a/backend/plugins/github/tasks/event_collector.go
+++ b/backend/plugins/github/tasks/event_collector.go
@@ -33,6 +33,10 @@ import (
"github.com/apache/incubator-devlake/plugins/github/models"
)
+func init() {
+ RegisterSubtaskMeta(&CollectApiEventsMeta)
+}
+
const RAW_EVENTS_TABLE = "github_api_events"
type SimpleGithubApiEvents struct {
@@ -46,6 +50,8 @@ var CollectApiEventsMeta = plugin.SubTaskMeta{
EnabledByDefault: true,
Description: "Collect Events data from Github api, supports both timeFilter and diffSync.",
DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET},
+ DependencyTables: []string{},
+ ProductTables: []string{RAW_EVENTS_TABLE},
}
func CollectApiEvents(taskCtx plugin.SubTaskContext) errors.Error {
diff --git a/backend/plugins/github/tasks/event_extractor.go b/backend/plugins/github/tasks/event_extractor.go
index 94e3ddeb7..7868874d2 100644
--- a/backend/plugins/github/tasks/event_extractor.go
+++ b/backend/plugins/github/tasks/event_extractor.go
@@ -19,18 +19,25 @@ package tasks
import (
"encoding/json"
+
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/github/models"
)
+func init() {
+ RegisterSubtaskMeta(&ExtractApiEventsMeta)
+}
+
var ExtractApiEventsMeta = plugin.SubTaskMeta{
Name: "extractApiEvents",
EntryPoint: ExtractApiEvents,
EnabledByDefault: true,
Description: "Extract raw Events data into tool layer table github_issue_events",
DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET},
+ DependencyTables: []string{RAW_EVENTS_TABLE},
+ ProductTables: []string{models.GithubIssueEvent{}.TableName()},
}
type IssueEvent struct {
diff --git a/backend/plugins/github/tasks/issue_assignee_convertor.go b/backend/plugins/github/tasks/issue_assignee_convertor.go
index 373bec2b5..24c3c0f16 100644
--- a/backend/plugins/github/tasks/issue_assignee_convertor.go
+++ b/backend/plugins/github/tasks/issue_assignee_convertor.go
@@ -29,12 +29,23 @@ import (
"github.com/apache/incubator-devlake/plugins/github/models"
)
+func init() {
+ RegisterSubtaskMeta(&ConvertIssueAssigneeMeta)
+}
+
var ConvertIssueAssigneeMeta = plugin.SubTaskMeta{
Name: "convertIssueAssignee",
EntryPoint: ConvertIssueAssignee,
EnabledByDefault: true,
Description: "Convert tool layer table _tool_github_issue_assignees into domain layer table issue_assignees",
DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET},
+ DependencyTables: []string{
+ //models.GithubIssueAssignee{}.TableName(), // cursor, not regard as dependency
+ models.GithubIssue{}.TableName(), // id generator
+ RAW_ISSUE_TABLE,
+ //models.GithubAccount{}.TableName(), // id generator, config will not regard as dependency
+ },
+ ProductTables: []string{models.GithubIssueAssignee{}.TableName()},
}
func ConvertIssueAssignee(taskCtx plugin.SubTaskContext) errors.Error {
diff --git a/backend/plugins/github/tasks/issue_collector.go b/backend/plugins/github/tasks/issue_collector.go
index e646f7e08..f88942319 100644
--- a/backend/plugins/github/tasks/issue_collector.go
+++ b/backend/plugins/github/tasks/issue_collector.go
@@ -30,12 +30,18 @@ import (
const RAW_ISSUE_TABLE = "github_api_issues"
+func init() {
+ RegisterSubtaskMeta(&CollectApiIssuesMeta)
+}
+
var CollectApiIssuesMeta = plugin.SubTaskMeta{
Name: "collectApiIssues",
EntryPoint: CollectApiIssues,
EnabledByDefault: true,
Description: "Collect issues data from Github api, supports both timeFilter and diffSync.",
DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET},
+ DependencyTables: []string{},
+ ProductTables: []string{RAW_ISSUE_TABLE},
}
func CollectApiIssues(taskCtx plugin.SubTaskContext) errors.Error {
diff --git a/backend/plugins/github/tasks/issue_comment_convertor.go b/backend/plugins/github/tasks/issue_comment_convertor.go
index d1d7f70e4..860a5fc14 100644
--- a/backend/plugins/github/tasks/issue_comment_convertor.go
+++ b/backend/plugins/github/tasks/issue_comment_convertor.go
@@ -30,12 +30,22 @@ import (
"github.com/apache/incubator-devlake/plugins/github/models"
)
+func init() {
+ RegisterSubtaskMeta(&ConvertIssueCommentsMeta)
+}
+
var ConvertIssueCommentsMeta = plugin.SubTaskMeta{
Name: "convertIssueComments",
EntryPoint: ConvertIssueComments,
EnabledByDefault: true,
Description: "ConvertIssueComments data from Github api",
DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET},
+ DependencyTables: []string{
+ models.GithubIssueComment{}.TableName(), // cursor
+ models.GithubIssue{}.TableName(), // cursor and id generator
+ models.GithubAccount{}.TableName(), // id generator
+ RAW_COMMENTS_TABLE},
+ ProductTables: []string{ticket.IssueComment{}.TableName()},
}
func ConvertIssueComments(taskCtx plugin.SubTaskContext) errors.Error {
diff --git a/backend/plugins/github/tasks/issue_convertor.go b/backend/plugins/github/tasks/issue_convertor.go
index 9aa6cccf4..165f21835 100644
--- a/backend/plugins/github/tasks/issue_convertor.go
+++ b/backend/plugins/github/tasks/issue_convertor.go
@@ -18,6 +18,10 @@ limitations under the License.
package tasks
import (
+ "reflect"
+ "strconv"
+ "strings"
+
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models/domainlayer"
@@ -26,17 +30,26 @@ import (
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/github/models"
- "reflect"
- "strconv"
- "strings"
)
+func init() {
+ RegisterSubtaskMeta(&ConvertIssuesMeta)
+}
+
var ConvertIssuesMeta = plugin.SubTaskMeta{
Name: "convertIssues",
EntryPoint: ConvertIssues,
EnabledByDefault: true,
Description: "Convert tool layer table github_issues into domain layer table issues",
DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET},
+ DependencyTables: []string{
+ models.GithubIssue{}.TableName(), // cursor
+ models.GithubAccount{}.TableName(), // id generator
+ //models.GithubRepo{}.TableName(), // id generator, but config not regard as dependency
+ RAW_ISSUE_TABLE},
+ ProductTables: []string{
+ ticket.Issue{}.TableName(),
+ ticket.BoardIssue{}.TableName()},
}
func ConvertIssues(taskCtx plugin.SubTaskContext) errors.Error {
diff --git a/backend/plugins/github/tasks/issue_extractor.go b/backend/plugins/github/tasks/issue_extractor.go
index 29f6c43e2..c48eeab18 100644
--- a/backend/plugins/github/tasks/issue_extractor.go
+++ b/backend/plugins/github/tasks/issue_extractor.go
@@ -29,12 +29,22 @@ import (
"github.com/apache/incubator-devlake/plugins/github/models"
)
+func init() {
+ RegisterSubtaskMeta(&ExtractApiIssuesMeta)
+}
+
var ExtractApiIssuesMeta = plugin.SubTaskMeta{
Name: "extractApiIssues",
EntryPoint: ExtractApiIssues,
EnabledByDefault: true,
Description: "Extract raw Issues data into tool layer table github_issues",
DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET},
+ DependencyTables: []string{RAW_ISSUE_TABLE},
+ ProductTables: []string{
+ models.GithubIssue{}.TableName(),
+ models.GithubIssueLabel{}.TableName(),
+ models.GithubRepoAccount{}.TableName(),
+ models.GithubIssueAssignee{}.TableName()},
}
type IssuesResponse struct {
diff --git a/backend/plugins/github/tasks/issue_label_convertor.go b/backend/plugins/github/tasks/issue_label_convertor.go
index 87ac3ba8b..8643d48cf 100644
--- a/backend/plugins/github/tasks/issue_label_convertor.go
+++ b/backend/plugins/github/tasks/issue_label_convertor.go
@@ -18,6 +18,8 @@ limitations under the License.
package tasks
import (
+ "reflect"
+
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models/domainlayer/didgen"
@@ -25,15 +27,23 @@ import (
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/github/models"
- "reflect"
)
+func init() {
+ RegisterSubtaskMeta(&ConvertIssueLabelsMeta)
+}
+
var ConvertIssueLabelsMeta = plugin.SubTaskMeta{
Name: "convertIssueLabels",
EntryPoint: ConvertIssueLabels,
EnabledByDefault: true,
Description: "Convert tool layer table github_issue_labels into domain layer table issue_labels",
DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET},
+ DependencyTables: []string{
+ models.GithubIssueLabel{}.TableName(), // cursor
+ models.GithubIssue{}.TableName(), // cursor
+ RAW_ISSUE_TABLE},
+ ProductTables: []string{ticket.IssueLabel{}.TableName()},
}
func ConvertIssueLabels(taskCtx plugin.SubTaskContext) errors.Error {
diff --git a/backend/plugins/github/tasks/milestone_collector.go b/backend/plugins/github/tasks/milestone_collector.go
index 51d4d3439..03d4ff586 100644
--- a/backend/plugins/github/tasks/milestone_collector.go
+++ b/backend/plugins/github/tasks/milestone_collector.go
@@ -28,6 +28,10 @@ import (
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
)
+func init() {
+ RegisterSubtaskMeta(&CollectMilestonesMeta)
+}
+
const RAW_MILESTONE_TABLE = "github_milestones"
var CollectMilestonesMeta = plugin.SubTaskMeta{
@@ -36,6 +40,8 @@ var CollectMilestonesMeta = plugin.SubTaskMeta{
EnabledByDefault: true,
Description: "Collect milestone data from Github api, does not support either timeFilter or diffSync.",
DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET},
+ DependencyTables: []string{},
+ ProductTables: []string{RAW_MILESTONE_TABLE},
}
func CollectApiMilestones(taskCtx plugin.SubTaskContext) errors.Error {
diff --git a/backend/plugins/github/tasks/milestone_converter.go b/backend/plugins/github/tasks/milestone_converter.go
index 57adddd73..8b97beee5 100644
--- a/backend/plugins/github/tasks/milestone_converter.go
+++ b/backend/plugins/github/tasks/milestone_converter.go
@@ -18,6 +18,8 @@ limitations under the License.
package tasks
import (
+ "reflect"
+
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models/common"
@@ -27,15 +29,27 @@ import (
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/github/models"
- "reflect"
)
+func init() {
+ RegisterSubtaskMeta(&ConvertMilestonesMeta)
+}
+
var ConvertMilestonesMeta = plugin.SubTaskMeta{
Name: "convertMilestones",
EntryPoint: ConvertMilestones,
EnabledByDefault: true,
Description: "Convert tool layer table github_milestones into domain layer table milestones",
DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET},
+ DependencyTables: []string{
+ models.GithubIssue{}.TableName(), // cursor
+ models.GithubMilestone{}.TableName(), // cursor
+ //models.GithubRepo{}.TableName(), // id generator, but config not regard as dependency
+ RAW_MILESTONE_TABLE},
+ ProductTables: []string{
+ ticket.Sprint{}.TableName(),
+ ticket.BoardSprint{}.TableName(),
+ ticket.SprintIssue{}.TableName()},
}
type MilestoneConverterModel struct {
diff --git a/backend/plugins/github/tasks/milestone_extractor.go b/backend/plugins/github/tasks/milestone_extractor.go
index f5dd8f16d..b4ca218b4 100644
--- a/backend/plugins/github/tasks/milestone_extractor.go
+++ b/backend/plugins/github/tasks/milestone_extractor.go
@@ -19,18 +19,25 @@ package tasks
import (
"encoding/json"
+
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/github/models"
)
+func init() {
+ RegisterSubtaskMeta(&ExtractMilestonesMeta)
+}
+
var ExtractMilestonesMeta = plugin.SubTaskMeta{
Name: "extractMilestones",
EntryPoint: ExtractMilestones,
EnabledByDefault: true,
Description: "Extract raw milestone data into tool layer table github_milestones",
DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET},
+ DependencyTables: []string{RAW_MILESTONE_TABLE},
+ ProductTables: []string{models.GithubMilestone{}.TableName()},
}
type MilestonesResponse struct {
diff --git a/backend/plugins/github/tasks/pr_collector.go b/backend/plugins/github/tasks/pr_collector.go
index d15bf3db1..02ab884d9 100644
--- a/backend/plugins/github/tasks/pr_collector.go
+++ b/backend/plugins/github/tasks/pr_collector.go
@@ -33,6 +33,10 @@ import (
"github.com/apache/incubator-devlake/plugins/github/models"
)
+func init() {
+ RegisterSubtaskMeta(&CollectApiPullRequestsMeta)
+}
+
const RAW_PULL_REQUEST_TABLE = "github_api_pull_requests"
var CollectApiPullRequestsMeta = plugin.SubTaskMeta{
@@ -41,6 +45,8 @@ var CollectApiPullRequestsMeta = plugin.SubTaskMeta{
EnabledByDefault: true,
Description: "Collect PullRequests data from Github api, supports both timeFilter and diffSync.",
DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS, plugin.DOMAIN_TYPE_CODE_REVIEW},
+ DependencyTables: []string{},
+ ProductTables: []string{RAW_PULL_REQUEST_TABLE},
}
type SimpleGithubPr struct {
diff --git a/backend/plugins/github/tasks/pr_comment_convertor.go b/backend/plugins/github/tasks/pr_comment_convertor.go
index 0b7b77bca..7df730c92 100644
--- a/backend/plugins/github/tasks/pr_comment_convertor.go
+++ b/backend/plugins/github/tasks/pr_comment_convertor.go
@@ -18,6 +18,8 @@ limitations under the License.
package tasks
import (
+ "reflect"
+
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models/domainlayer"
@@ -26,15 +28,25 @@ import (
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/github/models"
- "reflect"
)
+func init() {
+ RegisterSubtaskMeta(&ConvertPullRequestCommentsMeta)
+}
+
var ConvertPullRequestCommentsMeta = plugin.SubTaskMeta{
Name: "convertPullRequestComments",
EntryPoint: ConvertPullRequestComments,
EnabledByDefault: true,
Description: "ConvertPullRequestComments data from Github api",
DomainTypes: []string{plugin.DOMAIN_TYPE_CODE_REVIEW},
+ DependencyTables: []string{
+ models.GithubPrComment{}.TableName(), // cursor
+ models.GithubPullRequest{}.TableName(), // cursor
+ models.GithubAccount{}.TableName(), // id generator
+ models.GithubPrReview{}.TableName(), // id generator
+ RAW_COMMENTS_TABLE},
+ ProductTables: []string{code.PullRequestComment{}.TableName()},
}
func ConvertPullRequestComments(taskCtx plugin.SubTaskContext) errors.Error {
diff --git a/backend/plugins/github/tasks/pr_commit_collector.go b/backend/plugins/github/tasks/pr_commit_collector.go
index 2ee08cf76..30ac310e8 100644
--- a/backend/plugins/github/tasks/pr_commit_collector.go
+++ b/backend/plugins/github/tasks/pr_commit_collector.go
@@ -34,6 +34,10 @@ import (
"github.com/apache/incubator-devlake/plugins/github/models"
)
+func init() {
+ RegisterSubtaskMeta(&CollectApiPullRequestCommitsMeta)
+}
+
const RAW_PR_COMMIT_TABLE = "github_api_pull_request_commits"
// this struct should be moved to `gitub_api_common.go`
@@ -44,6 +48,8 @@ var CollectApiPullRequestCommitsMeta = plugin.SubTaskMeta{
EnabledByDefault: true,
Description: "Collect PullRequestCommits data from Github api, supports both timeFilter and diffSync.",
DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS, plugin.DOMAIN_TYPE_CODE_REVIEW},
+ DependencyTables: []string{models.GithubPullRequest{}.TableName()},
+ ProductTables: []string{RAW_PR_COMMIT_TABLE},
}
type SimplePr struct {
diff --git a/backend/plugins/github/tasks/pr_commit_convertor.go b/backend/plugins/github/tasks/pr_commit_convertor.go
index 267d343e7..5b6ab7042 100644
--- a/backend/plugins/github/tasks/pr_commit_convertor.go
+++ b/backend/plugins/github/tasks/pr_commit_convertor.go
@@ -29,12 +29,22 @@ import (
"github.com/apache/incubator-devlake/plugins/github/models"
)
+func init() {
+ RegisterSubtaskMeta(&ConvertPullRequestCommitsMeta)
+}
+
var ConvertPullRequestCommitsMeta = plugin.SubTaskMeta{
Name: "convertPullRequestCommits",
EntryPoint: ConvertPullRequestCommits,
EnabledByDefault: true,
Description: "Convert tool layer table github_pull_request_commits into domain layer table pull_request_commits",
DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS, plugin.DOMAIN_TYPE_CODE_REVIEW},
+ DependencyTables: []string{
+ models.GithubPrCommit{}.TableName(), // cursor
+ models.GithubPullRequest{}.TableName(), // cursor
+ models.GithubPullRequest{}.TableName(), // id generator
+ RAW_PR_COMMIT_TABLE},
+ ProductTables: []string{code.PullRequestCommit{}.TableName()},
}
func ConvertPullRequestCommits(taskCtx plugin.SubTaskContext) (err errors.Error) {
diff --git a/backend/plugins/github/tasks/pr_commit_extractor.go b/backend/plugins/github/tasks/pr_commit_extractor.go
index 8418bbc8e..6bf025c8d 100644
--- a/backend/plugins/github/tasks/pr_commit_extractor.go
+++ b/backend/plugins/github/tasks/pr_commit_extractor.go
@@ -28,12 +28,21 @@ import (
"github.com/apache/incubator-devlake/plugins/github/models"
)
+func init() {
+ RegisterSubtaskMeta(&ExtractApiPullRequestCommitsMeta)
+}
+
var ExtractApiPullRequestCommitsMeta = plugin.SubTaskMeta{
Name: "extractApiPullRequestCommits",
EntryPoint: ExtractApiPullRequestCommits,
EnabledByDefault: true,
Description: "Extract raw PullRequestCommits data into tool layer table github_commits",
DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS, plugin.DOMAIN_TYPE_CODE_REVIEW},
+ DependencyTables: []string{RAW_PR_COMMIT_TABLE},
+ ProductTables: []string{
+ models.GithubRepoCommit{}.TableName(),
+ models.GithubCommit{}.TableName(),
+ models.GithubPrCommit{}.TableName()},
}
type PrCommitsResponse struct {
diff --git a/backend/plugins/github/tasks/pr_convertor.go b/backend/plugins/github/tasks/pr_convertor.go
index 4bee55443..672484de0 100644
--- a/backend/plugins/github/tasks/pr_convertor.go
+++ b/backend/plugins/github/tasks/pr_convertor.go
@@ -30,12 +30,22 @@ import (
"github.com/apache/incubator-devlake/plugins/github/models"
)
+func init() {
+ RegisterSubtaskMeta(&ConvertPullRequestsMeta)
+}
+
var ConvertPullRequestsMeta = plugin.SubTaskMeta{
Name: "convertPullRequests",
EntryPoint: ConvertPullRequests,
EnabledByDefault: true,
Description: "ConvertPullRequests data from Github api",
DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS, plugin.DOMAIN_TYPE_CODE_REVIEW},
+ DependencyTables: []string{
+ models.GithubPullRequest{}.TableName(), // cursor
+ //models.GithubRepo{}.TableName(), // id generator, but not regard as dependency
+ models.GithubAccount{}.TableName(), // cursor
+ RAW_PULL_REQUEST_TABLE},
+ ProductTables: []string{code.PullRequest{}.TableName()},
}
func ConvertPullRequests(taskCtx plugin.SubTaskContext) errors.Error {
diff --git a/backend/plugins/github/tasks/pr_extractor.go b/backend/plugins/github/tasks/pr_extractor.go
index 8a03cdabc..8b2d49347 100644
--- a/backend/plugins/github/tasks/pr_extractor.go
+++ b/backend/plugins/github/tasks/pr_extractor.go
@@ -27,12 +27,21 @@ import (
"github.com/apache/incubator-devlake/plugins/github/models"
)
+func init() {
+ RegisterSubtaskMeta(&ExtractApiPullRequestsMeta)
+}
+
var ExtractApiPullRequestsMeta = plugin.SubTaskMeta{
Name: "extractApiPullRequests",
EntryPoint: ExtractApiPullRequests,
EnabledByDefault: true,
Description: "Extract raw PullRequests data into tool layer table github_pull_requests",
DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS, plugin.DOMAIN_TYPE_CODE_REVIEW},
+ DependencyTables: []string{RAW_PULL_REQUEST_TABLE},
+ ProductTables: []string{
+ models.GithubRepoAccount{}.TableName(),
+ models.GithubPrLabel{}.TableName(),
+ models.GithubPullRequest{}.TableName()},
}
type GithubApiPullRequest struct {
diff --git a/backend/plugins/github/tasks/pr_issue_convertor.go b/backend/plugins/github/tasks/pr_issue_convertor.go
index e8611c88b..1c5906847 100644
--- a/backend/plugins/github/tasks/pr_issue_convertor.go
+++ b/backend/plugins/github/tasks/pr_issue_convertor.go
@@ -18,6 +18,8 @@ limitations under the License.
package tasks
import (
+ "reflect"
+
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models/domainlayer/crossdomain"
@@ -25,15 +27,23 @@ import (
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/github/models"
- "reflect"
)
+func init() {
+ RegisterSubtaskMeta(&ConvertPullRequestIssuesMeta)
+}
+
var ConvertPullRequestIssuesMeta = plugin.SubTaskMeta{
Name: "convertPullRequestIssues",
EntryPoint: ConvertPullRequestIssues,
EnabledByDefault: true,
- Description: "Convert tool layer table github_pull_request_issues into domain layer table pull_request_issues",
+ Description: "Convert tool layer table github_pull_request_issues into domain layer table pull_request_issues",
DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS},
+ DependencyTables: []string{
+ models.GithubPrIssue{}.TableName(), // cursor and id generator
+ models.GithubPullRequest{}.TableName(), // cursor and id generator
+ RAW_PULL_REQUEST_TABLE},
+ ProductTables: []string{crossdomain.PullRequestIssue{}.TableName()},
}
func ConvertPullRequestIssues(taskCtx plugin.SubTaskContext) errors.Error {
diff --git a/backend/plugins/github/tasks/pr_issue_enricher.go b/backend/plugins/github/tasks/pr_issue_enricher.go
index ed0cbacae..f2b8c13fe 100644
--- a/backend/plugins/github/tasks/pr_issue_enricher.go
+++ b/backend/plugins/github/tasks/pr_issue_enricher.go
@@ -30,12 +30,20 @@ import (
"github.com/apache/incubator-devlake/plugins/github/models"
)
+func init() {
+ RegisterSubtaskMeta(&EnrichPullRequestIssuesMeta)
+}
+
var EnrichPullRequestIssuesMeta = plugin.SubTaskMeta{
Name: "enrichPullRequestIssues",
EntryPoint: EnrichPullRequestIssues,
EnabledByDefault: true,
- Description: "Create tool layer table github_pull_request_issues from github_pull_reqeusts",
+ Description: "Create tool layer table github_pull_request_issues from github_pull_requests",
DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS},
+ DependencyTables: []string{
+ models.GithubPullRequest{}.TableName(), // cursor
+ RAW_PULL_REQUEST_TABLE},
+ ProductTables: []string{models.GithubPrIssue{}.TableName()},
}
func EnrichPullRequestIssues(taskCtx plugin.SubTaskContext) (err errors.Error) {
@@ -54,7 +62,8 @@ func EnrichPullRequestIssues(taskCtx plugin.SubTaskContext) (err errors.Error) {
}
}
charPattern := regexp.MustCompile(`[\/a-zA-Z\s,]+`)
- cursor, err := db.Cursor(dal.From(&models.GithubPullRequest{}), dal.Where("repo_id = ? and connection_id = ?", repoId, data.Options.ConnectionId))
+ cursor, err := db.Cursor(dal.From(&models.GithubPullRequest{}),
+ dal.Where("repo_id = ? and connection_id = ?", repoId, data.Options.ConnectionId))
if err != nil {
return err
}
@@ -101,7 +110,8 @@ func EnrichPullRequestIssues(taskCtx plugin.SubTaskContext) (err errors.Error) {
}
err = db.All(
issue,
- dal.Where("number = ? and repo_id = ? and connection_id = ?", issueNumber, repoId, data.Options.ConnectionId),
+ dal.Where("number = ? and repo_id = ? and connection_id = ?",
+ issueNumber, repoId, data.Options.ConnectionId),
dal.Limit(1),
)
if err != nil {
diff --git a/backend/plugins/github/tasks/pr_label_convertor.go b/backend/plugins/github/tasks/pr_label_convertor.go
index 2c8b443cc..9053dcd98 100644
--- a/backend/plugins/github/tasks/pr_label_convertor.go
+++ b/backend/plugins/github/tasks/pr_label_convertor.go
@@ -18,6 +18,8 @@ limitations under the License.
package tasks
import (
+ "reflect"
+
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models/domainlayer/code"
@@ -25,15 +27,23 @@ import (
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/github/models"
- "reflect"
)
+func init() {
+ RegisterSubtaskMeta(&ConvertPullRequestLabelsMeta)
+}
+
var ConvertPullRequestLabelsMeta = plugin.SubTaskMeta{
Name: "convertPullRequestLabels",
EntryPoint: ConvertPullRequestLabels,
EnabledByDefault: true,
Description: "Convert tool layer table github_pull_request_labels into domain layer table pull_request_labels",
DomainTypes: []string{plugin.DOMAIN_TYPE_CODE_REVIEW},
+ DependencyTables: []string{
+ models.GithubPrLabel{}.TableName(), // cursor
+ models.GithubPullRequest{}.TableName(), // cursor and id generator
+ RAW_PULL_REQUEST_TABLE},
+ ProductTables: []string{code.PullRequestLabel{}.TableName()},
}
func ConvertPullRequestLabels(taskCtx plugin.SubTaskContext) errors.Error {
diff --git a/backend/plugins/github/tasks/pr_review_collector.go b/backend/plugins/github/tasks/pr_review_collector.go
index 888186cea..268f565ce 100644
--- a/backend/plugins/github/tasks/pr_review_collector.go
+++ b/backend/plugins/github/tasks/pr_review_collector.go
@@ -31,6 +31,10 @@ import (
"github.com/apache/incubator-devlake/plugins/github/models"
)
+func init() {
+ RegisterSubtaskMeta(&CollectApiPullRequestReviewsMeta)
+}
+
const RAW_PR_REVIEW_TABLE = "github_api_pull_request_reviews"
// this struct should be moved to `gitub_api_common.go`
@@ -41,6 +45,8 @@ var CollectApiPullRequestReviewsMeta = plugin.SubTaskMeta{
EnabledByDefault: true,
Description: "Collect PullRequestReviews data from Github api, supports both timeFilter and diffSync.",
DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS, plugin.DOMAIN_TYPE_CODE_REVIEW},
+ DependencyTables: []string{models.GithubPullRequest{}.TableName()},
+ ProductTables: []string{RAW_PR_REVIEW_TABLE},
}
func CollectApiPullRequestReviews(taskCtx plugin.SubTaskContext) errors.Error {
diff --git a/backend/plugins/github/tasks/pr_review_comment_collector.go b/backend/plugins/github/tasks/pr_review_comment_collector.go
index 2468b1b93..a2c817221 100644
--- a/backend/plugins/github/tasks/pr_review_comment_collector.go
+++ b/backend/plugins/github/tasks/pr_review_comment_collector.go
@@ -28,10 +28,24 @@ import (
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
)
+func init() {
+ RegisterSubtaskMeta(&CollectApiPrReviewCommentsMeta)
+}
+
const RAW_PR_REVIEW_COMMENTS_TABLE = "github_api_pull_request_review_comments"
// this struct should be moved to `github_api_common.go`
+var CollectApiPrReviewCommentsMeta = plugin.SubTaskMeta{
+ Name: "collectApiPrReviewCommentsMeta",
+ EntryPoint: CollectPrReviewComments,
+ EnabledByDefault: true,
+ Description: "Collect pr review comments data from Github api, supports both timeFilter and diffSync.",
+ DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS, plugin.DOMAIN_TYPE_CODE_REVIEW},
+ DependencyTables: []string{},
+ ProductTables: []string{RAW_PR_REVIEW_COMMENTS_TABLE},
+}
+
func CollectPrReviewComments(taskCtx plugin.SubTaskContext) errors.Error {
data := taskCtx.GetData().(*GithubTaskData)
@@ -88,11 +102,3 @@ func CollectPrReviewComments(taskCtx plugin.SubTaskContext) errors.Error {
return collectorWithState.Execute()
}
-
-var CollectApiPrReviewCommentsMeta = plugin.SubTaskMeta{
- Name: "collectApiPrReviewCommentsMeta",
- EntryPoint: CollectPrReviewComments,
- EnabledByDefault: true,
- Description: "Collect pr review comments data from Github api, supports both timeFilter and diffSync.",
- DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS, plugin.DOMAIN_TYPE_CODE_REVIEW},
-}
diff --git a/backend/plugins/github/tasks/pr_review_comment_extractor.go b/backend/plugins/github/tasks/pr_review_comment_extractor.go
index fa7181e7c..575d8a8b7 100644
--- a/backend/plugins/github/tasks/pr_review_comment_extractor.go
+++ b/backend/plugins/github/tasks/pr_review_comment_extractor.go
@@ -30,13 +30,21 @@ import (
"github.com/apache/incubator-devlake/plugins/github/models"
)
+func init() {
+ RegisterSubtaskMeta(&ExtractApiPrReviewCommentsMeta)
+}
+
var ExtractApiPrReviewCommentsMeta = plugin.SubTaskMeta{
Name: "extractApiPrReviewComments",
EntryPoint: ExtractApiPrReviewComments,
EnabledByDefault: true,
Description: "Extract raw comment data into tool layer table github_pull_request_comments" +
"and github_issue_comments",
- DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS, plugin.DOMAIN_TYPE_CODE_REVIEW},
+ DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS, plugin.DOMAIN_TYPE_CODE_REVIEW},
+ DependencyTables: []string{RAW_PR_REVIEW_COMMENTS_TABLE},
+ ProductTables: []string{
+ models.GithubRepoAccount{}.TableName(),
+ models.GithubPrCommit{}.TableName()},
}
func ExtractApiPrReviewComments(taskCtx plugin.SubTaskContext) errors.Error {
diff --git a/backend/plugins/github/tasks/pr_review_convertor.go b/backend/plugins/github/tasks/pr_review_convertor.go
index f526c1592..280f204a6 100644
--- a/backend/plugins/github/tasks/pr_review_convertor.go
+++ b/backend/plugins/github/tasks/pr_review_convertor.go
@@ -18,6 +18,8 @@ limitations under the License.
package tasks
import (
+ "reflect"
+
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models/domainlayer"
@@ -26,15 +28,24 @@ import (
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/github/models"
- "reflect"
)
+func init() {
+ RegisterSubtaskMeta(&ConvertPullRequestReviewsMeta)
+}
+
var ConvertPullRequestReviewsMeta = plugin.SubTaskMeta{
Name: "convertPullRequestReviews",
EntryPoint: ConvertPullRequestReviews,
EnabledByDefault: true,
Description: "ConvertPullRequestReviews data from Github api",
DomainTypes: []string{plugin.DOMAIN_TYPE_CODE_REVIEW},
+ DependencyTables: []string{
+ models.GithubPrReview{}.TableName(), // cursor and id generator
+ models.GithubPullRequest{}.TableName(), // cursor and id generator
+ models.GithubAccount{}.TableName(), // id generator
+ RAW_PR_REVIEW_TABLE},
+ ProductTables: []string{code.PullRequestComment{}.TableName()},
}
func ConvertPullRequestReviews(taskCtx plugin.SubTaskContext) errors.Error {
diff --git a/backend/plugins/github/tasks/pr_review_extractor.go b/backend/plugins/github/tasks/pr_review_extractor.go
index 665ed3a5d..44be34040 100644
--- a/backend/plugins/github/tasks/pr_review_extractor.go
+++ b/backend/plugins/github/tasks/pr_review_extractor.go
@@ -19,19 +19,29 @@ package tasks
import (
"encoding/json"
+ "strings"
+
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/github/models"
- "strings"
)
+func init() {
+ RegisterSubtaskMeta(&ExtractApiPullRequestReviewsMeta)
+}
+
var ExtractApiPullRequestReviewsMeta = plugin.SubTaskMeta{
Name: "extractApiPullRequestReviews",
EntryPoint: ExtractApiPullRequestReviews,
EnabledByDefault: true,
Description: "Extract raw PullRequestReviewers data into tool layer table github_reviewers",
DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS, plugin.DOMAIN_TYPE_CODE_REVIEW},
+ DependencyTables: []string{RAW_PR_REVIEW_TABLE},
+ ProductTables: []string{
+ models.GithubRepoAccount{}.TableName(),
+ models.GithubReviewer{}.TableName(),
+ models.GithubPrReview{}.TableName()},
}
type PullRequestReview struct {
diff --git a/backend/helpers/pluginhelper/subtaskmeta_sorter/interface.go b/backend/plugins/github/tasks/register.go
similarity index 83%
rename from backend/helpers/pluginhelper/subtaskmeta_sorter/interface.go
rename to backend/plugins/github/tasks/register.go
index fb4e30b5e..51b12a76c 100644
--- a/backend/helpers/pluginhelper/subtaskmeta_sorter/interface.go
+++ b/backend/plugins/github/tasks/register.go
@@ -15,10 +15,12 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
-package subtaskmeta_sorter
+package tasks
import "github.com/apache/incubator-devlake/core/plugin"
-type SubTaskMetaSorter interface {
- Sort() ([]plugin.SubTaskMeta, error)
+var SubTaskMetaList []*plugin.SubTaskMeta
+
+func RegisterSubtaskMeta(meta *plugin.SubTaskMeta) {
+ SubTaskMetaList = append(SubTaskMetaList, meta)
}
diff --git a/backend/plugins/github/tasks/repo_convertor.go b/backend/plugins/github/tasks/repo_convertor.go
index 418ef7a00..9afe4665a 100644
--- a/backend/plugins/github/tasks/repo_convertor.go
+++ b/backend/plugins/github/tasks/repo_convertor.go
@@ -34,9 +34,9 @@ import (
"github.com/apache/incubator-devlake/plugins/github/models"
)
-const RAW_REPOSITORIES_TABLE = "github_api_repositories"
-
-type ApiRepoResponse GithubApiRepo
+func init() {
+ RegisterSubtaskMeta(&ConvertRepoMeta)
+}
type GithubApiRepo struct {
Name string `json:"name"`
@@ -56,8 +56,22 @@ var ConvertRepoMeta = plugin.SubTaskMeta{
Name: "convertRepo",
EntryPoint: ConvertRepo,
EnabledByDefault: true,
- Description: "Convert tool layer table github_repos into domain layer table repos and boards",
- DomainTypes: []string{plugin.DOMAIN_TYPE_CODE, plugin.DOMAIN_TYPE_TICKET, plugin.DOMAIN_TYPE_CICD, plugin.DOMAIN_TYPE_CODE_REVIEW, plugin.DOMAIN_TYPE_CROSS},
+ Description: "Convert tool layer table github_repos into domain layer table repos and boards",
+ DomainTypes: []string{
+ plugin.DOMAIN_TYPE_CODE,
+ plugin.DOMAIN_TYPE_TICKET,
+ plugin.DOMAIN_TYPE_CICD,
+ plugin.DOMAIN_TYPE_CODE_REVIEW,
+ plugin.DOMAIN_TYPE_CROSS},
+ DependencyTables: []string{
+ //models.GithubRepo{}.TableName(), // config will not regard as dependency
+ //RAW_REPOSITORIES_TABLE,
+ },
+ ProductTables: []string{
+ code.Repo{}.TableName(),
+ ticket.Board{}.TableName(),
+ crossdomain.BoardRepo{}.TableName(),
+ devops.CicdScope{}.TableName()},
}
func ConvertRepo(taskCtx plugin.SubTaskContext) errors.Error {
@@ -85,7 +99,7 @@ func ConvertRepo(taskCtx plugin.SubTaskContext) errors.Error {
ConnectionId: data.Options.ConnectionId,
Name: data.Options.Name,
},
- Table: RAW_REPOSITORIES_TABLE,
+ Table: models.GithubRepo{}.TableName(),
},
Convert: func(inputRow interface{}) ([]interface{}, errors.Error) {
repository := inputRow.(*models.GithubRepo)
diff --git a/backend/plugins/github/tasks/shared.go b/backend/plugins/github/tasks/shared.go
index dbde93a93..823b8b2dd 100644
--- a/backend/plugins/github/tasks/shared.go
+++ b/backend/plugins/github/tasks/shared.go
@@ -42,3 +42,10 @@ func ignoreHTTPStatus404(res *http.Response) errors.Error {
}
return nil
}
+
+func ignoreHTTPStatus422(res *http.Response) errors.Error {
+ if res.StatusCode == http.StatusUnprocessableEntity {
+ return api.ErrIgnoreAndContinue
+ }
+ return nil
+}
diff --git a/backend/plugins/github/tasks/task_data.go b/backend/plugins/github/tasks/task_data.go
index ac8d24a90..8dcc777f8 100644
--- a/backend/plugins/github/tasks/task_data.go
+++ b/backend/plugins/github/tasks/task_data.go
@@ -23,7 +23,6 @@ import (
"time"
"github.com/apache/incubator-devlake/core/errors"
-
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/github/models"
)
diff --git a/backend/plugins/gitlab/impl/impl.go b/backend/plugins/gitlab/impl/impl.go
index 835020c24..7053f7b19 100644
--- a/backend/plugins/gitlab/impl/impl.go
+++ b/backend/plugins/gitlab/impl/impl.go
@@ -19,7 +19,7 @@ package impl
import (
"fmt"
- "github.com/apache/incubator-devlake/helpers/pluginhelper/subtaskmeta_sorter"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/subtaskmeta/sorter"
"time"
"github.com/apache/incubator-devlake/core/context"
@@ -49,7 +49,7 @@ type Gitlab string
func init() {
// check subtask meta loop when init subtask meta
- if _, err := subtaskmeta_sorter.NewDependencySorter(tasks.SubTaskMetaList).Sort(); err != nil {
+ if _, err := sorter.NewDependencySorter(tasks.SubTaskMetaList).Sort(); err != nil {
panic(err)
}
}
@@ -109,7 +109,7 @@ func (p Gitlab) Name() string {
}
func (p Gitlab) SubTaskMetas() []plugin.SubTaskMeta {
- list, err := subtaskmeta_sorter.NewDependencySorter(tasks.SubTaskMetaList).Sort()
+ list, err := sorter.NewDependencySorter(tasks.SubTaskMetaList).Sort()
if err != nil {
panic(err)
}