You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by ab...@apache.org on 2023/11/02 03:17:15 UTC
(incubator-devlake) branch main updated: refactor(github): move deployment convertor from github_ql to github plugin (#6382)
This is an automated email from the ASF dual-hosted git repository.
abeizn pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new e24efd732 refactor(github): move deployment convertor from github_ql to github plugin (#6382)
e24efd732 is described below
commit e24efd732edc2c84415088b59654d32ef9d40b62
Author: Lynwee <li...@merico.dev>
AuthorDate: Thu Nov 2 11:17:11 2023 +0800
refactor(github): move deployment convertor from github_ql to github plugin (#6382)
* refactor(github): move deployment convertor from github_ql to github plugin
* fix(github): enable ConvertDeployment by default
---
.../tasks/deployment_convertor.go | 33 +++++++++++++---------
backend/plugins/github/tasks/shared.go | 15 ++++++++++
backend/plugins/github_graphql/impl/impl.go | 4 +--
.../tasks/deployment_collector_extractor.go | 16 +++++------
4 files changed, 45 insertions(+), 23 deletions(-)
diff --git a/backend/plugins/github_graphql/tasks/deployment_convertor.go b/backend/plugins/github/tasks/deployment_convertor.go
similarity index 82%
rename from backend/plugins/github_graphql/tasks/deployment_convertor.go
rename to backend/plugins/github/tasks/deployment_convertor.go
index 051d08ac3..17b3befb4 100644
--- a/backend/plugins/github_graphql/tasks/deployment_convertor.go
+++ b/backend/plugins/github/tasks/deployment_convertor.go
@@ -18,8 +18,6 @@ limitations under the License.
package tasks
import (
- "reflect"
-
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models/domainlayer"
@@ -27,24 +25,33 @@ import (
"github.com/apache/incubator-devlake/core/models/domainlayer/didgen"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
- githubModels "github.com/apache/incubator-devlake/plugins/github/models"
+ "github.com/apache/incubator-devlake/plugins/github/models"
+ "reflect"
)
-var _ plugin.SubTaskEntryPoint = ConvertDeployment
+func init() {
+ RegisterSubtaskMeta(&ConvertDeploymentsMeta)
+}
+
+const (
+ RAW_DEPLOYMENT_TABLE = "github_deployment"
+)
-var ConvertDeploymentMeta = plugin.SubTaskMeta{
- Name: "ConvertDeployment",
+var ConvertDeploymentsMeta = plugin.SubTaskMeta{
+ Name: "ConvertDeployments",
EntryPoint: ConvertDeployment,
EnabledByDefault: true,
- Description: "Convert github deployment from tool layer to domain layer",
+ Description: "Convert tool layer table github_deployments into domain layer table deployment",
DomainTypes: []string{plugin.DOMAIN_TYPE_CICD},
+ DependencyTables: []string{models.GithubDeployment{}.TableName()},
+ ProductTables: []string{devops.CicdDeploymentCommit{}.TableName(), devops.CICDDeployment{}.TableName()},
}
func ConvertDeployment(taskCtx plugin.SubTaskContext) errors.Error {
db := taskCtx.GetDal()
- rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_DEPLOYMENT)
+ rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_DEPLOYMENT_TABLE)
cursor, err := db.Cursor(
- dal.From(&githubModels.GithubDeployment{}),
+ dal.From(&models.GithubDeployment{}),
dal.Where("connection_id = ? and github_id = ?", data.Options.ConnectionId, data.Options.GithubId),
)
if err != nil {
@@ -52,15 +59,15 @@ func ConvertDeployment(taskCtx plugin.SubTaskContext) errors.Error {
}
defer cursor.Close()
- deploymentIdGen := didgen.NewDomainIdGenerator(&githubModels.GithubDeployment{})
- deploymentScopeIdGen := didgen.NewDomainIdGenerator(&githubModels.GithubRepo{})
+ deploymentIdGen := didgen.NewDomainIdGenerator(&models.GithubDeployment{})
+ deploymentScopeIdGen := didgen.NewDomainIdGenerator(&models.GithubRepo{})
converter, err := api.NewDataConverter(api.DataConverterArgs{
- InputRowType: reflect.TypeOf(githubModels.GithubDeployment{}),
+ InputRowType: reflect.TypeOf(models.GithubDeployment{}),
Input: cursor,
RawDataSubTaskArgs: *rawDataSubTaskArgs,
Convert: func(inputRow interface{}) ([]interface{}, errors.Error) {
- githubDeployment := inputRow.(*githubModels.GithubDeployment)
+ githubDeployment := inputRow.(*models.GithubDeployment)
deploymentCommit := &devops.CicdDeploymentCommit{
DomainEntity: domainlayer.DomainEntity{
Id: deploymentIdGen.Generate(githubDeployment.ConnectionId, githubDeployment.Id),
diff --git a/backend/plugins/github/tasks/shared.go b/backend/plugins/github/tasks/shared.go
index 823b8b2dd..e92e58fca 100644
--- a/backend/plugins/github/tasks/shared.go
+++ b/backend/plugins/github/tasks/shared.go
@@ -19,7 +19,9 @@ package tasks
import (
"github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "github.com/apache/incubator-devlake/plugins/github/models"
"github.com/apache/incubator-devlake/plugins/github/utils"
"net/http"
)
@@ -49,3 +51,16 @@ func ignoreHTTPStatus422(res *http.Response) errors.Error {
}
return nil
}
+
+func CreateRawDataSubTaskArgs(taskCtx plugin.SubTaskContext, table string) (*api.RawDataSubTaskArgs, *GithubTaskData) {
+ data := taskCtx.GetData().(*GithubTaskData)
+ RawDataSubTaskArgs := &api.RawDataSubTaskArgs{
+ Ctx: taskCtx,
+ Params: models.GithubApiParams{
+ Name: data.Options.Name,
+ ConnectionId: data.Options.ConnectionId,
+ },
+ Table: table,
+ }
+ return RawDataSubTaskArgs, data
+}
diff --git a/backend/plugins/github_graphql/impl/impl.go b/backend/plugins/github_graphql/impl/impl.go
index 0b1fef529..bf1808663 100644
--- a/backend/plugins/github_graphql/impl/impl.go
+++ b/backend/plugins/github_graphql/impl/impl.go
@@ -127,8 +127,8 @@ func (p GithubGraphql) SubTaskMetas() []plugin.SubTaskMeta {
githubTasks.ConvertAccountsMeta,
// deployment
- tasks.CollectAndExtractDeploymentMeta,
- tasks.ConvertDeploymentMeta,
+ tasks.CollectAndExtractDeploymentsMeta,
+ githubTasks.ConvertDeploymentsMeta,
}
}
diff --git a/backend/plugins/github_graphql/tasks/deployment_collector_extractor.go b/backend/plugins/github_graphql/tasks/deployment_collector_extractor.go
index 4d1bd0dec..f4f318240 100644
--- a/backend/plugins/github_graphql/tasks/deployment_collector_extractor.go
+++ b/backend/plugins/github_graphql/tasks/deployment_collector_extractor.go
@@ -29,17 +29,17 @@ import (
"time"
)
-var _ plugin.SubTaskEntryPoint = CollectAndExtractDeployment
+var _ plugin.SubTaskEntryPoint = CollectAndExtractDeployments
const (
- RAW_DEPLOYMENT = "github_deployment"
+ RAW_DEPLOYMENT = "github_graphql_deployment"
)
-var CollectAndExtractDeploymentMeta = plugin.SubTaskMeta{
- Name: "CollectAndExtractDeployment",
- EntryPoint: CollectAndExtractDeployment,
+var CollectAndExtractDeploymentsMeta = plugin.SubTaskMeta{
+ Name: "CollectAndExtractDeployments",
+ EntryPoint: CollectAndExtractDeployments,
EnabledByDefault: true,
- Description: "collect and extract github deployments to raw and tool layer",
+ Description: "collect and extract github deployments to raw and tool layer from GithubGraphql api",
DomainTypes: []string{plugin.DOMAIN_TYPE_CICD},
}
@@ -84,9 +84,9 @@ type GraphqlQueryDeploymentDeployment struct {
UpdatedAt time.Time
}
-// CollectAndExtractDeployment will request github api via graphql and store the result into raw layer by default
+// CollectAndExtractDeployments will request github api via graphql and store the result into raw layer by default
// ResponseParser's return will be stored to tool layer. So it's called CollectorAndExtractor.
-func CollectAndExtractDeployment(taskCtx plugin.SubTaskContext) errors.Error {
+func CollectAndExtractDeployments(taskCtx plugin.SubTaskContext) errors.Error {
data := taskCtx.GetData().(*githubTasks.GithubTaskData)