You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by kl...@apache.org on 2022/08/11 14:17:53 UTC
[incubator-devlake] branch main updated: Table info (#2720)
This is an automated email from the ASF dual-hosted git repository.
klesh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 1088df44 Table info (#2720)
1088df44 is described below
commit 1088df447cec98fa4092010fc0fbd143db8283b3
Author: mappjzc <zh...@merico.dev>
AuthorDate: Thu Aug 11 22:17:50 2022 +0800
Table info (#2720)
* feat: table info regiest and get
Add some struct for table info.
Add RegisterTablesInfo for regist the table info.
Add GetTotalInfo,TraversalTable,Traversal for get the table info.
Nddtfjiang <zh...@merico.dev>
* feat: gitlab regist table models
Add RegisterTablesInfo for gitlab.
Add GetTableInfo api to get the table info for gitlab.
Nddtfjiang <zh...@merico.dev>
* refactor: remove table data saved
remove the table data saved.
Nddtfjiang <zh...@merico.dev>
* feat: add tables info for plugin
remote api to show tables info at gitlab.
add GetTablesInfo for all plugin
Nddtfjiang <zh...@merico.dev>
---
plugins/ae/impl/impl.go | 8 +++++++
plugins/core/plugin_model.go | 26 ++++++++++++++++++++++
plugins/dbt/dbt.go | 4 ++++
plugins/feishu/impl/impl.go | 7 ++++++
plugins/gitee/impl/impl.go | 21 ++++++++++++++++++
plugins/gitextractor/gitextractor.go | 7 +++++-
plugins/github/impl/impl.go | 28 ++++++++++++++++++++++++
plugins/gitlab/impl/impl.go | 22 +++++++++++++++++++
plugins/icla/plugin_main.go | 6 ++++++
plugins/jenkins/impl/impl.go | 14 ++++++++++++
plugins/jira/impl/impl.go | 23 ++++++++++++++++++++
plugins/org/impl/impl.go | 4 ++++
plugins/refdiff/refdiff.go | 4 ++++
plugins/starrocks/starrocks.go | 4 ++++
plugins/tapd/impl/impl.go | 42 ++++++++++++++++++++++++++++++++++++
runner/directrun.go | 12 ++++++-----
runner/loader.go | 1 +
17 files changed, 227 insertions(+), 6 deletions(-)
diff --git a/plugins/ae/impl/impl.go b/plugins/ae/impl/impl.go
index cfc747a4..54bbee8f 100644
--- a/plugins/ae/impl/impl.go
+++ b/plugins/ae/impl/impl.go
@@ -45,6 +45,14 @@ func (plugin AE) Init(config *viper.Viper, logger core.Logger, db *gorm.DB) erro
api.Init(config, logger, db)
return nil
}
+func (plugin AE) GetTablesInfo() []core.Tabler {
+ return []core.Tabler{
+ &models.AECommit{},
+ &models.AEProject{},
+ &models.AeConnection{},
+ &models.AeResponse{},
+ }
+}
func (plugin AE) Description() string {
return "To collect and enrich data from AE"
diff --git a/plugins/core/plugin_model.go b/plugins/core/plugin_model.go
new file mode 100644
index 00000000..5ddec7f6
--- /dev/null
+++ b/plugins/core/plugin_model.go
@@ -0,0 +1,26 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package core
+
+type Tabler interface {
+ TableName() string
+}
+
+type PluginModel interface {
+ GetTablesInfo() []Tabler
+}
diff --git a/plugins/dbt/dbt.go b/plugins/dbt/dbt.go
index c3ce3d3e..5fdbabd4 100644
--- a/plugins/dbt/dbt.go
+++ b/plugins/dbt/dbt.go
@@ -42,6 +42,10 @@ func (plugin Dbt) SubTaskMetas() []core.SubTaskMeta {
}
}
+func (plugin Dbt) GetTablesInfo() []core.Tabler {
+ return []core.Tabler{}
+}
+
func (plugin Dbt) PrepareTaskData(taskCtx core.TaskContext, options map[string]interface{}) (interface{}, error) {
var op tasks.DbtOptions
err := mapstructure.Decode(options, &op)
diff --git a/plugins/feishu/impl/impl.go b/plugins/feishu/impl/impl.go
index e987077d..0b748cd4 100644
--- a/plugins/feishu/impl/impl.go
+++ b/plugins/feishu/impl/impl.go
@@ -76,6 +76,13 @@ func (plugin Feishu) Init(config *viper.Viper, logger core.Logger, db *gorm.DB)
return nil
}
+func (plugin Feishu) GetTablesInfo() []core.Tabler {
+ return []core.Tabler{
+ &models.FeishuConnection{},
+ &models.FeishuMeetingTopUserItem{},
+ }
+}
+
func (plugin Feishu) Description() string {
return "To collect and enrich data from Feishu"
}
diff --git a/plugins/gitee/impl/impl.go b/plugins/gitee/impl/impl.go
index 44f8662a..57bb4bae 100644
--- a/plugins/gitee/impl/impl.go
+++ b/plugins/gitee/impl/impl.go
@@ -46,6 +46,27 @@ func (plugin Gitee) Init(config *viper.Viper, logger core.Logger, db *gorm.DB) e
return nil
}
+func (plugin Gitee) GetTablesInfo() []core.Tabler {
+ return []core.Tabler{
+ &models.GiteeConnection{},
+ &models.GiteeAccount{},
+ &models.GiteeCommit{},
+ &models.GiteeCommitStat{},
+ &models.GiteeIssue{},
+ &models.GiteeIssueComment{},
+ &models.GiteeIssueLabel{},
+ &models.GiteePullRequest{},
+ &models.GiteePullRequestComment{},
+ &models.GiteePullRequestCommit{},
+ &models.GiteePullRequestIssue{},
+ &models.GiteePullRequestLabel{},
+ &models.GiteeRepo{},
+ &models.GiteeRepoCommit{},
+ &models.GiteeResponse{},
+ &models.GiteeReviewer{},
+ }
+}
+
func (plugin Gitee) Description() string {
return "To collect and enrich data from Gitee"
}
diff --git a/plugins/gitextractor/gitextractor.go b/plugins/gitextractor/gitextractor.go
index 146e869d..81686d99 100644
--- a/plugins/gitextractor/gitextractor.go
+++ b/plugins/gitextractor/gitextractor.go
@@ -18,13 +18,14 @@ limitations under the License.
package main
import (
+ "strings"
+
"github.com/apache/incubator-devlake/plugins/core"
"github.com/apache/incubator-devlake/plugins/gitextractor/models"
"github.com/apache/incubator-devlake/plugins/gitextractor/parser"
"github.com/apache/incubator-devlake/plugins/gitextractor/store"
"github.com/apache/incubator-devlake/plugins/gitextractor/tasks"
"github.com/mitchellh/mapstructure"
- "strings"
)
var _ core.PluginMeta = (*GitExtractor)(nil)
@@ -32,6 +33,10 @@ var _ core.PluginTask = (*GitExtractor)(nil)
type GitExtractor struct{}
+func (plugin GitExtractor) GetTablesInfo() []core.Tabler {
+ return []core.Tabler{}
+}
+
func (plugin GitExtractor) Description() string {
return "extract infos from git repository"
}
diff --git a/plugins/github/impl/impl.go b/plugins/github/impl/impl.go
index 94dc18de..92f88214 100644
--- a/plugins/github/impl/impl.go
+++ b/plugins/github/impl/impl.go
@@ -46,6 +46,34 @@ func (plugin Github) Init(config *viper.Viper, logger core.Logger, db *gorm.DB)
return nil
}
+func (plugin Github) GetTablesInfo() []core.Tabler {
+ return []core.Tabler{
+ &models.GithubConnection{},
+ &models.GithubAccount{},
+ &models.GithubAccountOrg{},
+ &models.GithubCommit{},
+ &models.GithubCommitStat{},
+ &models.GithubIssue{},
+ &models.GithubIssueComment{},
+ &models.GithubIssueEvent{},
+ &models.GithubIssueLabel{},
+ &models.GithubJob{},
+ &models.GithubMilestone{},
+ &models.GithubPipeline{},
+ &models.GithubPrComment{},
+ &models.GithubPrCommit{},
+ &models.GithubPrIssue{},
+ &models.GithubPrLabel{},
+ &models.GithubPrReview{},
+ &models.GithubPullRequest{},
+ &models.GithubRepo{},
+ &models.GithubRepoAccount{},
+ &models.GithubRepoCommit{},
+ &models.GithubReviewer{},
+ &models.GithubRun{},
+ }
+}
+
func (plugin Github) Description() string {
return "To collect and enrich data from GitHub"
}
diff --git a/plugins/gitlab/impl/impl.go b/plugins/gitlab/impl/impl.go
index c3c67fe2..514d47bb 100644
--- a/plugins/gitlab/impl/impl.go
+++ b/plugins/gitlab/impl/impl.go
@@ -33,6 +33,7 @@ import (
var _ core.PluginMeta = (*Gitlab)(nil)
var _ core.PluginInit = (*Gitlab)(nil)
+var _ core.PluginModel = (*Gitlab)(nil)
var _ core.PluginTask = (*Gitlab)(nil)
var _ core.PluginApi = (*Gitlab)(nil)
var _ core.Migratable = (*Gitlab)(nil)
@@ -46,6 +47,27 @@ func (plugin Gitlab) Init(config *viper.Viper, logger core.Logger, db *gorm.DB)
return nil
}
+func (plugin Gitlab) GetTablesInfo() []core.Tabler {
+ return []core.Tabler{
+ &models.GitlabConnection{},
+ &models.GitlabAccount{},
+ &models.GitlabCommit{},
+ &models.GitlabIssue{},
+ &models.GitlabIssueLabel{},
+ &models.GitlabJob{},
+ &models.GitlabMergeRequest{},
+ &models.GitlabMrComment{},
+ &models.GitlabMrCommit{},
+ &models.GitlabMrLabel{},
+ &models.GitlabMrNote{},
+ &models.GitlabPipeline{},
+ &models.GitlabProject{},
+ &models.GitlabProjectCommit{},
+ &models.GitlabReviewer{},
+ &models.GitlabTag{},
+ }
+}
+
func (plugin Gitlab) Description() string {
return "To collect and enrich data from Gitlab"
}
diff --git a/plugins/icla/plugin_main.go b/plugins/icla/plugin_main.go
index 57972adf..5bfcaa76 100644
--- a/plugins/icla/plugin_main.go
+++ b/plugins/icla/plugin_main.go
@@ -42,6 +42,12 @@ var PluginEntry Icla //nolint
type Icla struct{}
+func (plugin Icla) GetTablesInfo() []core.Tabler {
+ return []core.Tabler{
+ &models.IclaCommitter{},
+ }
+}
+
func (plugin Icla) Description() string {
return "collect some Icla data"
}
diff --git a/plugins/jenkins/impl/impl.go b/plugins/jenkins/impl/impl.go
index 299a43a1..677a8d07 100644
--- a/plugins/jenkins/impl/impl.go
+++ b/plugins/jenkins/impl/impl.go
@@ -45,6 +45,20 @@ func (plugin Jenkins) Init(config *viper.Viper, logger core.Logger, db *gorm.DB)
return nil
}
+func (plugin Jenkins) GetTablesInfo() []core.Tabler {
+ return []core.Tabler{
+ &models.JenkinsBuild{},
+ &models.JenkinsBuildRepo{},
+ &models.JenkinsConnection{},
+ &models.JenkinsJob{},
+ &models.JenkinsJobDag{},
+ &models.JenkinsPipeline{},
+ &models.JenkinsResponse{},
+ &models.JenkinsStage{},
+ &models.JenkinsTask{},
+ }
+}
+
func (plugin Jenkins) Description() string {
return "To collect and enrich data from Jenkins"
}
diff --git a/plugins/jira/impl/impl.go b/plugins/jira/impl/impl.go
index cedd9f29..2c38afc9 100644
--- a/plugins/jira/impl/impl.go
+++ b/plugins/jira/impl/impl.go
@@ -48,6 +48,29 @@ func (plugin Jira) Init(config *viper.Viper, logger core.Logger, db *gorm.DB) er
api.Init(config, logger, db)
return nil
}
+func (plugin Jira) GetTablesInfo() []core.Tabler {
+ return []core.Tabler{
+ &models.ApiMyselfResponse{},
+ &models.JiraAccount{},
+ &models.JiraBoard{},
+ &models.JiraBoardIssue{},
+ &models.JiraBoardSprint{},
+ &models.JiraConnection{},
+ &models.JiraIssue{},
+ &models.JiraIssueChangelogItems{},
+ &models.JiraIssueChangelogs{},
+ &models.JiraIssueCommit{},
+ &models.JiraIssueLabel{},
+ &models.JiraIssueType{},
+ &models.JiraProject{},
+ &models.JiraRemotelink{},
+ &models.JiraServerInfo{},
+ &models.JiraSprint{},
+ &models.JiraSprintIssue{},
+ &models.JiraStatus{},
+ &models.JiraWorklog{},
+ }
+}
func (plugin Jira) Description() string {
return "To collect and enrich data from JIRA"
diff --git a/plugins/org/impl/impl.go b/plugins/org/impl/impl.go
index f99ce1af..daa27223 100644
--- a/plugins/org/impl/impl.go
+++ b/plugins/org/impl/impl.go
@@ -42,6 +42,10 @@ func (plugin *Org) Init(config *viper.Viper, logger core.Logger, db *gorm.DB) er
return nil
}
+func (plugin Org) GetTablesInfo() []core.Tabler {
+ return []core.Tabler{}
+}
+
func (plugin Org) Description() string {
return "collect data related to team and organization"
}
diff --git a/plugins/refdiff/refdiff.go b/plugins/refdiff/refdiff.go
index 34098182..4e57b48d 100644
--- a/plugins/refdiff/refdiff.go
+++ b/plugins/refdiff/refdiff.go
@@ -43,6 +43,10 @@ func (plugin RefDiff) Description() string {
return "Calculate commits diff for specified ref pairs based on `commits` and `commit_parents` tables"
}
+func (plugin RefDiff) GetTablesInfo() []core.Tabler {
+ return []core.Tabler{}
+}
+
func (plugin RefDiff) Init(config *viper.Viper, logger core.Logger, db *gorm.DB) error {
return nil
}
diff --git a/plugins/starrocks/starrocks.go b/plugins/starrocks/starrocks.go
index 99f062b7..f22903de 100644
--- a/plugins/starrocks/starrocks.go
+++ b/plugins/starrocks/starrocks.go
@@ -40,6 +40,10 @@ func (s StarRocks) PrepareTaskData(taskCtx core.TaskContext, options map[string]
return &op, nil
}
+func (plugin StarRocks) GetTablesInfo() []core.Tabler {
+ return []core.Tabler{}
+}
+
func (s StarRocks) Description() string {
return "Sync data from database to StarRocks"
}
diff --git a/plugins/tapd/impl/impl.go b/plugins/tapd/impl/impl.go
index 43df5152..fc26f21d 100644
--- a/plugins/tapd/impl/impl.go
+++ b/plugins/tapd/impl/impl.go
@@ -49,6 +49,48 @@ func (plugin Tapd) Init(config *viper.Viper, logger core.Logger, db *gorm.DB) er
return nil
}
+func (plugin Tapd) GetTablesInfo() []core.Tabler {
+ return []core.Tabler{
+ &models.TapdAccount{},
+ &models.TapdBug{},
+ &models.TapdBugChangelog{},
+ &models.TapdBugChangelogItem{},
+ &models.TapdBugCommit{},
+ &models.TapdBugCustomFields{},
+ &models.TapdBugLabel{},
+ &models.TapdBugStatus{},
+ &models.TapdConnection{},
+ &models.TapdConnectionDetail{},
+ &models.TapdIssue{},
+ &models.TapdIteration{},
+ &models.TapdIterationBug{},
+ &models.TapdIterationStory{},
+ &models.TapdIterationTask{},
+ &models.TapdStory{},
+ &models.TapdStoryBug{},
+ &models.TapdStoryCategory{},
+ &models.TapdStoryChangelog{},
+ &models.TapdStoryChangelogItem{},
+ &models.TapdStoryCommit{},
+ &models.TapdStoryCustomFields{},
+ &models.TapdStoryLabel{},
+ &models.TapdStoryStatus{},
+ &models.TapdSubWorkspace{},
+ &models.TapdTask{},
+ &models.TapdTaskChangelog{},
+ &models.TapdTaskChangelogItem{},
+ &models.TapdTaskCommit{},
+ &models.TapdTaskCustomFields{},
+ &models.TapdTaskLabel{},
+ &models.TapdWorkSpaceBug{},
+ &models.TapdWorkSpaceStory{},
+ &models.TapdWorkSpaceTask{},
+ &models.TapdWorklog{},
+ &models.TapdWorkspace{},
+ &models.TapdWorkspaceIteration{},
+ }
+}
+
func (plugin Tapd) Description() string {
return "To collect and enrich data from Tapd"
}
diff --git a/runner/directrun.go b/runner/directrun.go
index b93dd643..df88530d 100644
--- a/runner/directrun.go
+++ b/runner/directrun.go
@@ -21,16 +21,17 @@ import (
"context"
"errors"
"fmt"
- "github.com/apache/incubator-devlake/config"
- "github.com/apache/incubator-devlake/logger"
- "github.com/apache/incubator-devlake/migration"
- "github.com/apache/incubator-devlake/plugins/core"
- "github.com/spf13/cobra"
"io"
"os"
"os/signal"
"runtime"
"syscall"
+
+ "github.com/apache/incubator-devlake/config"
+ "github.com/apache/incubator-devlake/logger"
+ "github.com/apache/incubator-devlake/migration"
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/spf13/cobra"
)
// RunCmd FIXME ...
@@ -64,6 +65,7 @@ func DirectRun(cmd *cobra.Command, args []string, pluginTask core.PluginTask, op
panic(err)
}
}
+
err = core.RegisterPlugin(cmd.Use, pluginTask.(core.PluginMeta))
if err != nil {
panic(err)
diff --git a/runner/loader.go b/runner/loader.go
index 4952aa7b..bcc6eb5b 100644
--- a/runner/loader.go
+++ b/runner/loader.go
@@ -63,6 +63,7 @@ func LoadPlugins(pluginsDir string, config *viper.Viper, logger core.Logger, db
if err != nil {
return nil
}
+
logger.Info(`plugin loaded %s`, pluginName)
}
return nil