You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by kl...@apache.org on 2023/05/30 02:59:54 UTC
[incubator-devlake] branch main updated: refactor: rename transformation rule to scope config - github (#5306)
This is an automated email from the ASF dual-hosted git repository.
klesh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 3847f035d refactor: rename transformation rule to scope config - github (#5306)
3847f035d is described below
commit 3847f035d4573bb2509ecd17e4b6318d1a14194f
Author: Klesh Wong <zh...@merico.dev>
AuthorDate: Tue May 30 10:59:50 2023 +0800
refactor: rename transformation rule to scope config - github (#5306)
* refactor: rename transformation rule to scope config - github
* refactor: no squash for taskOptions.scopeConfig
* fix: json serializer fails on pg
---
backend/core/plugin/plugin_blueprint.go | 27 ---
backend/core/plugin/plugin_pipeline.go | 49 ++++
backend/go.mod | 27 ++-
backend/go.sum | 25 ++
backend/plugins/github/api/blueprint.go | 259 ---------------------
backend/plugins/github/api/blueprint_V200_test.go | 46 ++--
backend/plugins/github/api/blueprint_test.go | 199 ----------------
backend/plugins/github/api/blueprint_v200.go | 113 +++++++--
backend/plugins/github/api/init.go | 13 +-
backend/plugins/github/api/scope.go | 2 +-
backend/plugins/github/api/swagger.go | 59 +----
backend/plugins/github/api/transformation_rule.go | 60 ++---
backend/plugins/github/e2e/comment_test.go | 2 +-
backend/plugins/github/e2e/issue_test.go | 2 +-
backend/plugins/github/e2e/milestone_test.go | 5 +-
backend/plugins/github/e2e/pr_enrich_issue_test.go | 2 +-
backend/plugins/github/e2e/pr_test.go | 2 +-
backend/plugins/github/e2e/repo_test.go | 2 +-
backend/plugins/github/github.go | 2 +-
backend/plugins/github/impl/impl.go | 49 ++--
...22_add_connection_id_to_transformation_rules.go | 23 +-
.../migrationscripts/20230526_scope_config.go | 56 +++++
.../github/models/migrationscripts/register.go | 2 +
backend/plugins/github/models/repo.go | 33 +--
.../{transformation_rule.go => scope_config.go} | 8 +-
backend/plugins/github/tasks/issue_extractor.go | 4 +-
backend/plugins/github/tasks/pr_extractor.go | 2 +-
backend/plugins/github/tasks/pr_issue_enricher.go | 11 +-
backend/plugins/github/tasks/task_data.go | 17 +-
backend/plugins/github_graphql/impl/impl.go | 7 +-
.../github_graphql/tasks/issue_collector.go | 2 +-
.../plugins/github_graphql/tasks/pr_collector.go | 2 +-
32 files changed, 413 insertions(+), 699 deletions(-)
diff --git a/backend/core/plugin/plugin_blueprint.go b/backend/core/plugin/plugin_blueprint.go
index 30671af50..928dd5096 100644
--- a/backend/core/plugin/plugin_blueprint.go
+++ b/backend/core/plugin/plugin_blueprint.go
@@ -24,33 +24,6 @@ import (
"github.com/apache/incubator-devlake/core/errors"
)
-// PipelineTask represents a smallest unit of execution inside a PipelinePlan
-type PipelineTask struct {
- // Plugin name
- Plugin string `json:"plugin" binding:"required"`
- Subtasks []string `json:"subtasks"`
- Options map[string]interface{} `json:"options"`
-}
-
-// PipelineStage consist of multiple PipelineTasks, they will be executed in parallel
-type PipelineStage []*PipelineTask
-
-// PipelinePlan consist of multiple PipelineStages, they will be executed in sequential order
-type PipelinePlan []PipelineStage
-
-// IsEmpty checks if a PipelinePlan is empty
-func (plan PipelinePlan) IsEmpty() bool {
- if len(plan) == 0 {
- return true
- }
- for _, stage := range plan {
- if len(stage) > 0 {
- return false
- }
- }
- return true
-}
-
// PluginBlueprintV100 is used to support Blueprint Normal model, for Plugin and Blueprint to
// collaboarte and generate a sophisticated Pipeline Plan based on User Settings.
// V100 doesn't support Project, and being deprecated, please use PluginBlueprintV200 instead
diff --git a/backend/core/plugin/plugin_pipeline.go b/backend/core/plugin/plugin_pipeline.go
new file mode 100644
index 000000000..750a0b858
--- /dev/null
+++ b/backend/core/plugin/plugin_pipeline.go
@@ -0,0 +1,49 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package plugin
+
+type GenericPipelineTask[T any] struct {
+ Plugin string `json:"plugin" binding:"required"`
+ Subtasks []string `json:"subtasks"`
+ Options T `json:"options"`
+}
+
+type GenericPipelineStage[T any] []*GenericPipelineTask[T]
+type GenericPipelinePlan[T any] []GenericPipelineStage[T]
+
+// PipelineTask represents a smallest unit of execution inside a PipelinePlan
+type PipelineTask GenericPipelineTask[map[string]interface{}]
+
+// PipelineStage consist of multiple PipelineTasks, they will be executed in parallel
+type PipelineStage []*PipelineTask
+
+// PipelinePlan consist of multiple PipelineStages, they will be executed in sequential order
+type PipelinePlan []PipelineStage
+
+// IsEmpty checks if a PipelinePlan is empty
+func (plan PipelinePlan) IsEmpty() bool {
+ if len(plan) == 0 {
+ return true
+ }
+ for _, stage := range plan {
+ if len(stage) > 0 {
+ return false
+ }
+ }
+ return true
+}
diff --git a/backend/go.mod b/backend/go.mod
index 416f8fe91..0d560cd7a 100644
--- a/backend/go.mod
+++ b/backend/go.mod
@@ -29,7 +29,7 @@ require (
github.com/spf13/cast v1.4.1
github.com/spf13/cobra v1.5.0
github.com/spf13/viper v1.8.1
- github.com/stretchr/testify v1.8.0
+ github.com/stretchr/testify v1.8.1
github.com/swaggo/gin-swagger v1.4.3
github.com/swaggo/swag v1.8.3
github.com/tidwall/gjson v1.14.3
@@ -37,17 +37,20 @@ require (
github.com/x-cray/logrus-prefixed-formatter v0.5.2
go.temporal.io/api v1.7.1-0.20220223032354-6e6fe738916a
go.temporal.io/sdk v1.14.0
- golang.org/x/crypto v0.8.0
+ golang.org/x/crypto v0.9.0
golang.org/x/exp v0.0.0-20221028150844-83b7d23a625f
golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602
golang.org/x/sync v0.1.0
gorm.io/datatypes v1.0.1
- gorm.io/driver/mysql v1.3.3
- gorm.io/driver/postgres v1.4.5
- gorm.io/gorm v1.24.1-0.20221019064659-5dd2bb482755
+ gorm.io/driver/mysql v1.5.1
+ gorm.io/driver/postgres v1.5.2
+ gorm.io/gorm v1.25.1
)
-require github.com/jmespath/go-jmespath v0.4.0 // indirect
+require (
+ github.com/jackc/pgx/v5 v5.3.1 // indirect
+ github.com/jmespath/go-jmespath v0.4.0 // indirect
+)
require (
github.com/KyleBanks/depth v1.2.1 // indirect
@@ -72,7 +75,7 @@ require (
github.com/go-openapi/swag v0.21.1 // indirect
github.com/go-playground/locales v0.14.0 // indirect
github.com/go-playground/universal-translator v0.18.0 // indirect
- github.com/go-sql-driver/mysql v1.6.0 // indirect
+ github.com/go-sql-driver/mysql v1.7.1 // indirect
github.com/gogo/googleapis v1.4.1 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/gogo/status v1.1.0 // indirect
@@ -88,7 +91,7 @@ require (
github.com/jackc/pgio v1.0.0 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgproto3/v2 v2.3.1 // indirect
- github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect
+ github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
github.com/jackc/pgtype v1.12.0 // indirect
github.com/jackc/pgx/v4 v4.17.2 // indirect
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
@@ -117,7 +120,7 @@ require (
github.com/sergi/go-diff v1.1.0 // indirect
github.com/spf13/jwalterweatherman v1.1.0 // indirect
github.com/spf13/pflag v1.0.6-0.20200504143853-81378bbcd8a1 // indirect
- github.com/stretchr/objx v0.4.0 // indirect
+ github.com/stretchr/objx v0.5.0 // indirect
github.com/subosito/gotenv v1.2.0 // indirect
github.com/tidwall/match v1.1.1 // indirect
github.com/tidwall/pretty v1.2.0 // indirect
@@ -125,9 +128,9 @@ require (
github.com/xanzy/ssh-agent v0.3.0 // indirect
go.uber.org/atomic v1.9.0 // indirect
golang.org/x/mod v0.8.0
- golang.org/x/net v0.9.0 // indirect
- golang.org/x/sys v0.7.0 // indirect
- golang.org/x/term v0.7.0 // indirect
+ golang.org/x/net v0.10.0 // indirect
+ golang.org/x/sys v0.8.0 // indirect
+ golang.org/x/term v0.8.0 // indirect
golang.org/x/text v0.9.0 // indirect
golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect
golang.org/x/tools v0.6.0 // indirect
diff --git a/backend/go.sum b/backend/go.sum
index 0b792721f..e093f42f3 100644
--- a/backend/go.sum
+++ b/backend/go.sum
@@ -226,6 +226,9 @@ github.com/go-playground/validator/v10 v10.9.0/go.mod h1:74x4gJWsvQexRdW8Pn3dXSG
github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE=
github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
+github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI=
+github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI=
+github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo=
github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=
@@ -406,6 +409,8 @@ github.com/jackc/pgproto3/v2 v2.3.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwX
github.com/jackc/pgservicefile v0.0.0-20200307190119-3430c5407db8/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E=
github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b h1:C8S2+VttkHFdOOCXJe+YGfa4vHYwlt4Zx+IVXQ97jYg=
github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E=
+github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk=
+github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg=
github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc=
github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw=
@@ -426,6 +431,8 @@ github.com/jackc/pgx/v4 v4.10.1/go.mod h1:QlrWebbs3kqEZPHCTGyxecvzG6tvIsYu+A5b1r
github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c/go.mod h1:1QD0+tgSXP7iUjYm9C1NxKhny7lq6ee99u/z+IHFcgs=
github.com/jackc/pgx/v4 v4.17.2 h1:0Ut0rpeKwvIVbMQ1KbMBU4h6wxehBI535LK6Flheh8E=
github.com/jackc/pgx/v4 v4.17.2/go.mod h1:lcxIZN44yMIrWI78a5CpucdD14hX0SBDbNRvjDBItsw=
+github.com/jackc/pgx/v5 v5.3.1 h1:Fcr8QJ1ZeLi5zsPZqQeUZhNhxfkkKBOgJuYkJHoBOtU=
+github.com/jackc/pgx/v5 v5.3.1/go.mod h1:t3JDKnCBlYIc0ewLF0Q7B8MXmoIaBOZj/ic7iHozM/8=
github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
github.com/jackc/puddle v1.1.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
@@ -676,6 +683,8 @@ github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoH
github.com/stretchr/objx v0.3.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
github.com/stretchr/objx v0.4.0 h1:M2gUjqZET1qApGOWNSnZ49BAIMX4F/1plDv3+l31EJ4=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
+github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
@@ -685,6 +694,8 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
+github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s=
github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
github.com/swaggo/files v0.0.0-20210815190702-a29dd2bc99b2/go.mod h1:lKJPbtWzJ9JhsTN1k1gZgleJWY/cqq0psdoMmaThG3w=
@@ -790,6 +801,8 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y
golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.8.0 h1:pd9TJtTueMTVQXzk8E2XESSMQDj/U7OUu0PqJqPXQjQ=
golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE=
+golang.org/x/crypto v0.9.0 h1:LF6fAI+IutBocDJ2OT0Q1g8plpYljMZ4+lty+dsqw3g=
+golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -883,6 +896,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug
golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM=
golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
+golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M=
+golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -987,12 +1002,16 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU=
golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU=
+golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.7.0 h1:BEvjmm5fURWqcfbSKTdpkDXYBrUS1c0m8agp14W48vQ=
golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
+golang.org/x/term v0.8.0 h1:n5xxQn2i3PC0yLAbjTpNT85q/Kgzcr2gIoX9OrJUols=
+golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -1235,9 +1254,13 @@ gorm.io/datatypes v1.0.1/go.mod h1:HEHoUU3/PO5ZXfAJcVWl11+zWlE16+O0X2DgJEb4Ixs=
gorm.io/driver/mysql v1.0.5/go.mod h1:N1OIhHAIhx5SunkMGqWbGFVeh4yTNWKmMo1GOAsohLI=
gorm.io/driver/mysql v1.3.3 h1:jXG9ANrwBc4+bMvBcSl8zCfPBaVoPyBEBshA8dA93X8=
gorm.io/driver/mysql v1.3.3/go.mod h1:ChK6AHbHgDCFZyJp0F+BmVGb06PSIoh9uVYKAlRbb2U=
+gorm.io/driver/mysql v1.5.1 h1:WUEH5VF9obL/lTtzjmML/5e6VfFR/788coz2uaVCAZw=
+gorm.io/driver/mysql v1.5.1/go.mod h1:Jo3Xu7mMhCyj8dlrb3WoCaRd1FhsVh+yMXb1jUInf5o=
gorm.io/driver/postgres v1.0.8/go.mod h1:4eOzrI1MUfm6ObJU/UcmbXyiHSs8jSwH95G5P5dxcAg=
gorm.io/driver/postgres v1.4.5 h1:mTeXTTtHAgnS9PgmhN2YeUbazYpLhUI1doLnw42XUZc=
gorm.io/driver/postgres v1.4.5/go.mod h1:GKNQYSJ14qvWkvPwXljMGehpKrhlDNsqYRr5HnYGncg=
+gorm.io/driver/postgres v1.5.2 h1:ytTDxxEv+MplXOfFe3Lzm7SjG09fcdb3Z/c056DTBx0=
+gorm.io/driver/postgres v1.5.2/go.mod h1:fmpX0m2I1PKuR7mKZiEluwrP3hbs+ps7JIGMUBpCgl8=
gorm.io/driver/sqlite v1.1.4 h1:PDzwYE+sI6De2+mxAneV9Xs11+ZyKV6oxD3wDGkaNvM=
gorm.io/driver/sqlite v1.1.4/go.mod h1:mJCeTFr7+crvS+TRnWc5Z3UvwxUN1BGBLMrf5LA9DYw=
gorm.io/driver/sqlserver v1.0.7 h1:uwUtb0kdFwW5PkRbd2KJ2h4wlsqvLSjox1XVg/RnzRE=
@@ -1250,6 +1273,8 @@ gorm.io/gorm v1.21.6/go.mod h1:F+OptMscr0P2F2qU97WT1WimdH9GaQPoDW7AYd5i2Y0=
gorm.io/gorm v1.23.1/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk=
gorm.io/gorm v1.24.1-0.20221019064659-5dd2bb482755 h1:7AdrbfcvKnzejfqP5g37fdSZOXH/JvaPIzBIHTOqXKk=
gorm.io/gorm v1.24.1-0.20221019064659-5dd2bb482755/go.mod h1:DVrVomtaYTbqs7gB/x2uVvqnXzv0nqjB396B8cG4dBA=
+gorm.io/gorm v1.25.1 h1:nsSALe5Pr+cM3V1qwwQ7rOkw+6UeLrX5O4v3llhHa64=
+gorm.io/gorm v1.25.1/go.mod h1:L4uxeKpfBml98NYqVqwAdmV1a2nBtAec/cf3fpucW/k=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
diff --git a/backend/plugins/github/api/blueprint.go b/backend/plugins/github/api/blueprint.go
deleted file mode 100644
index 4f3ab62d0..000000000
--- a/backend/plugins/github/api/blueprint.go
+++ /dev/null
@@ -1,259 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package api
-
-import (
- "context"
- "encoding/json"
- "fmt"
- "io"
- "net/http"
- "net/url"
- "strings"
-
- "github.com/apache/incubator-devlake/core/errors"
-
- "github.com/apache/incubator-devlake/core/models/domainlayer/didgen"
- "github.com/apache/incubator-devlake/core/plugin"
- "github.com/apache/incubator-devlake/core/utils"
- helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
- aha "github.com/apache/incubator-devlake/helpers/pluginhelper/api/apihelperabstract"
- "github.com/apache/incubator-devlake/plugins/github/models"
- "github.com/apache/incubator-devlake/plugins/github/tasks"
-)
-
-func MakePipelinePlan(
- subtaskMetas []plugin.SubTaskMeta,
- connectionId uint64,
- scope []*plugin.BlueprintScopeV100,
-) (plugin.PipelinePlan, errors.Error) {
- var err errors.Error
- connection := new(models.GithubConnection)
- err = connectionHelper.FirstById(connection, connectionId)
- if err != nil {
- return nil, err
- }
- apiClient, err := helper.NewApiClientFromConnection(context.TODO(), basicRes, connection)
- if err != nil {
- return nil, err
- }
- plan, err := makePipelinePlan(subtaskMetas, scope, apiClient, connection)
- if err != nil {
- return nil, err
- }
- return plan, nil
-}
-
-func makePipelinePlan(
- subtaskMetas []plugin.SubTaskMeta,
- scopeV100s []*plugin.BlueprintScopeV100,
- apiClient aha.ApiClientAbstract,
- connection *models.GithubConnection,
-) (plugin.PipelinePlan, errors.Error) {
- var err errors.Error
- var repo *tasks.GithubApiRepo
- plan := make(plugin.PipelinePlan, len(scopeV100s))
- for i, scopeElem := range scopeV100s {
- // handle taskOptions and transformationRules, by dumping them to taskOptions
- transformationRules := make(map[string]interface{})
- if len(scopeElem.Transformation) > 0 {
- err = errors.Convert(json.Unmarshal(scopeElem.Transformation, &transformationRules))
- if err != nil {
- return nil, err
- }
- }
- // construct task options for github
- options := make(map[string]interface{})
- err = errors.Convert(json.Unmarshal(scopeElem.Options, &options))
- if err != nil {
- return nil, err
- }
- options["connectionId"] = connection.ID
- options["transformationRules"] = transformationRules
- // make sure task options is valid
- op, err := tasks.DecodeAndValidateTaskOptions(options)
- if err != nil {
- return nil, err
- }
-
- // refdiff
- if refdiffRules, ok := transformationRules["refdiff"]; ok && refdiffRules != nil {
- // add a new task to next stage
- j := i + 1
- if j == len(plan) {
- plan = append(plan, nil)
- }
- repo, err = MemorizedGetApiRepo(repo, op, apiClient)
- if err != nil {
- return nil, err
- }
- ops := refdiffRules.(map[string]interface{})
- ops["repoId"] = didgen.NewDomainIdGenerator(&models.GithubRepo{}).Generate(connection.ID, repo.GithubId)
- plan[j] = plugin.PipelineStage{{
- Plugin: "refdiff",
- Options: ops,
- },
- }
- // remove it from github transformationRules
- delete(transformationRules, "refdiff")
- }
-
- stage := plan[i]
- if stage == nil {
- stage = plugin.PipelineStage{}
- }
- stage, err = addGithub(subtaskMetas, connection, scopeElem.Entities, stage, options)
- if err != nil {
- return nil, err
- }
- // collect git data by gitextractor if CODE was requested
- repo, err = MemorizedGetApiRepo(repo, op, apiClient)
- if err != nil {
- return nil, err
- }
- stage, err = addGitex(scopeElem.Entities, connection, repo, stage)
- if err != nil {
- return nil, err
- }
- // This is just to add a dora subtask, then we can add another two subtasks at the end of plans
- // The only purpose is to adapt old blueprints
- // DEPRECATED, will be removed in v0.17
- // dora
- if productionPattern, ok := transformationRules["productionPattern"]; ok && productionPattern != nil {
- j := i + 1
- if j == len(plan) {
- plan = append(plan, nil)
- }
- // add a new task to next stage
- if plan[j] != nil {
- j++
- }
- if j == len(plan) {
- plan = append(plan, nil)
- }
- plan[j] = plugin.PipelineStage{
- {
- Plugin: "dora",
- Subtasks: []string{"EnrichTaskEnv"},
- Options: map[string]interface{}{},
- },
- }
- }
- plan[i] = stage
- repo = nil
- }
- return plan, nil
-}
-
-func addGitex(entities []string,
- connection *models.GithubConnection,
- repo *tasks.GithubApiRepo,
- stage plugin.PipelineStage,
-) (plugin.PipelineStage, errors.Error) {
- if utils.StringsContains(entities, plugin.DOMAIN_TYPE_CODE) {
- // here is the tricky part, we have to obtain the repo id beforehand
- token := strings.Split(connection.Token, ",")[0]
- cloneUrl, err := errors.Convert01(url.Parse(repo.CloneUrl))
- if err != nil {
- return nil, err
- }
- cloneUrl.User = url.UserPassword("git", token)
- stage = append(stage, &plugin.PipelineTask{
- Plugin: "gitextractor",
- Options: map[string]interface{}{
- "url": cloneUrl.String(),
- "repoId": didgen.NewDomainIdGenerator(&models.GithubRepo{}).Generate(connection.ID, repo.GithubId),
- "proxy": connection.Proxy,
- },
- })
- }
- return stage, nil
-}
-
-func addGithub(subtaskMetas []plugin.SubTaskMeta, connection *models.GithubConnection, entities []string, stage plugin.PipelineStage, options map[string]interface{}) (plugin.PipelineStage, errors.Error) {
- // construct github(graphql) task
- if connection.EnableGraphql {
- // FIXME this need fix when 2 plugins merged
- p, err := plugin.GetPlugin(`github_graphql`)
- if err != nil {
- return nil, err
- }
- if pluginGq, ok := p.(plugin.PluginTask); ok {
- subtasks, err := helper.MakePipelinePlanSubtasks(pluginGq.SubTaskMetas(), entities)
- if err != nil {
- return nil, err
- }
- stage = append(stage, &plugin.PipelineTask{
- Plugin: "github_graphql",
- Subtasks: subtasks,
- Options: options,
- })
- } else {
- return nil, errors.BadInput.New("plugin github_graphql does not support SubTaskMetas")
- }
- } else {
- subtasks, err := helper.MakePipelinePlanSubtasks(subtaskMetas, entities)
- if err != nil {
- return nil, err
- }
- stage = append(stage, &plugin.PipelineTask{
- Plugin: "github",
- Subtasks: subtasks,
- Options: options,
- })
- }
- return stage, nil
-}
-
-func getApiRepo(
- op *tasks.GithubOptions,
- apiClient aha.ApiClientAbstract,
-) (*tasks.GithubApiRepo, errors.Error) {
- repoRes := &tasks.GithubApiRepo{}
- res, err := apiClient.Get(fmt.Sprintf("repos/%s", op.Name), nil, nil)
- if err != nil {
- return nil, err
- }
- defer res.Body.Close()
- if res.StatusCode != http.StatusOK {
- return nil, errors.HttpStatus(res.StatusCode).New(fmt.Sprintf("unexpected status code when requesting repo detail from %s", res.Request.URL.String()))
- }
- body, err := errors.Convert01(io.ReadAll(res.Body))
- if err != nil {
- return nil, err
- }
- err = errors.Convert(json.Unmarshal(body, repoRes))
- if err != nil {
- return nil, err
- }
- return repoRes, nil
-}
-
-func MemorizedGetApiRepo(
- repo *tasks.GithubApiRepo,
- op *tasks.GithubOptions, apiClient aha.ApiClientAbstract,
-) (*tasks.GithubApiRepo, errors.Error) {
- if repo == nil {
- var err errors.Error
- repo, err = getApiRepo(op, apiClient)
- if err != nil {
- return nil, err
- }
- }
- return repo, nil
-}
diff --git a/backend/plugins/github/api/blueprint_V200_test.go b/backend/plugins/github/api/blueprint_V200_test.go
index 5df199f2d..8ad53adf0 100644
--- a/backend/plugins/github/api/blueprint_V200_test.go
+++ b/backend/plugins/github/api/blueprint_V200_test.go
@@ -62,8 +62,7 @@ func TestMakeDataSourcePipelinePlanV200(t *testing.T) {
// Refresh Global Variables and set the sql mock
basicRes = NewMockBasicRes()
bs := &plugin.BlueprintScopeV200{
- Entities: []string{"CODE", "TICKET"},
- Id: "1",
+ Id: "1",
}
bpScopes := make([]*plugin.BlueprintScopeV200, 0)
bpScopes = append(bpScopes, bs)
@@ -135,18 +134,21 @@ func TestMakeDataSourcePipelinePlanV200(t *testing.T) {
// NewMockBasicRes FIXME ...
func NewMockBasicRes() *mockcontext.BasicRes {
testGithubRepo := &models.GithubRepo{
- ConnectionId: 1,
- GithubId: 12345,
- Name: "test/testRepo",
- CloneUrl: "https://this_is_cloneUrl",
- TransformationRuleId: 1,
+ ConnectionId: 1,
+ GithubId: 12345,
+ Name: "test/testRepo",
+ CloneUrl: "https://this_is_cloneUrl",
+ ScopeConfigId: 1,
}
- testTransformationRule := &models.GithubTransformationRule{
- Model: common.Model{
- ID: 1,
+ testScopeConfig := &models.GithubScopeConfig{
+ ScopeConfig: common.ScopeConfig{
+ Model: common.Model{
+ ID: 1,
+ },
+ Entities: []string{"CODE", "TICKET"},
},
- Name: "github transformation rule",
+ Name: "github scope config",
PrType: "hey,man,wasup",
Refdiff: map[string]interface{}{
"tagsPattern": "pattern",
@@ -157,15 +159,23 @@ func NewMockBasicRes() *mockcontext.BasicRes {
mockRes := new(mockcontext.BasicRes)
mockDal := new(mockdal.Dal)
- mockDal.On("First", mock.Anything, mock.Anything).Run(func(args mock.Arguments) {
+ mockDal.On("First", mock.AnythingOfType("*models.GithubRepo"), mock.Anything).Run(func(args mock.Arguments) {
dst := args.Get(0).(*models.GithubRepo)
*dst = *testGithubRepo
- }).Return(nil).Once()
-
- mockDal.On("First", mock.Anything, mock.Anything).Run(func(args mock.Arguments) {
- dst := args.Get(0).(*models.GithubTransformationRule)
- *dst = *testTransformationRule
- }).Return(nil).Once()
+ }).Return(nil)
+
+ mockDal.On("First", mock.AnythingOfType("*models.GithubScopeConfig"), mock.Anything).Run(func(args mock.Arguments) {
+ dst := args.Get(0).(*models.GithubScopeConfig)
+ *dst = *testScopeConfig
+ }).Return(nil)
+ // mockDal.On("First", mock.Anything, mock.Anything).Run(func(args mock.Arguments) {
+ // switch dst := args.Get(0).(type) {
+ // case *models.GithubRepo:
+ // *dst = *testGithubRepo
+ // case *models.GithubScopeConfig:
+ // *dst = *testScopeConfig
+ // }
+ // }).Return(nil)
mockRes.On("GetDal").Return(mockDal)
mockRes.On("GetConfig", mock.Anything).Return("")
diff --git a/backend/plugins/github/api/blueprint_test.go b/backend/plugins/github/api/blueprint_test.go
deleted file mode 100644
index 58170b75b..000000000
--- a/backend/plugins/github/api/blueprint_test.go
+++ /dev/null
@@ -1,199 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package api
-
-import (
- "bytes"
- "encoding/json"
- "io"
- "net/http"
- "testing"
-
- "github.com/apache/incubator-devlake/core/errors"
- "github.com/apache/incubator-devlake/core/models/common"
- "github.com/apache/incubator-devlake/core/plugin"
- helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
- aha "github.com/apache/incubator-devlake/helpers/pluginhelper/api/apihelperabstract"
- mockplugin "github.com/apache/incubator-devlake/mocks/core/plugin"
- mockaha "github.com/apache/incubator-devlake/mocks/helpers/pluginhelper/api/apihelperabstract"
- "github.com/apache/incubator-devlake/plugins/github/models"
- "github.com/apache/incubator-devlake/plugins/github/tasks"
- "github.com/stretchr/testify/assert"
- "github.com/stretchr/testify/mock"
-)
-
-var repo = &tasks.GithubApiRepo{
- GithubId: 12345,
- CloneUrl: "https://this_is_cloneUrl",
- CreatedAt: helper.Iso8601Time{},
-}
-
-func TestMakePipelinePlan(t *testing.T) {
- var bs = &plugin.BlueprintScopeV100{
- Entities: []string{"CODE"},
- Options: json.RawMessage(`{
- "owner": "test",
- "repo": "testRepo"
- }`),
- Transformation: json.RawMessage(`{
- "prType": "hey,man,wasup",
- "refdiff": {
- "tagsPattern": "pattern",
- "tagsLimit": 10,
- "tagsOrder": "reverse semver"
- },
- "productionPattern": "xxxx"
- }`),
- }
- prepareMockMeta(t)
- mockApiClient := prepareMockClient(t, repo)
- connection := &models.GithubConnection{
- BaseConnection: helper.BaseConnection{
- Name: "github-test",
- Model: common.Model{
- ID: 1,
- },
- },
- GithubConn: models.GithubConn{
- RestConnection: helper.RestConnection{
- Endpoint: "https://api.github.com/",
- Proxy: "",
- RateLimitPerHour: 0,
- },
- GithubAccessToken: models.GithubAccessToken{
- AccessToken: helper.AccessToken{
- Token: "123",
- },
- },
- },
- }
- scopes := make([]*plugin.BlueprintScopeV100, 0)
- scopes = append(scopes, bs)
- plan, err := makePipelinePlan(nil, scopes, mockApiClient, connection)
- assert.Nil(t, err)
-
- expectPlan := plugin.PipelinePlan{
- plugin.PipelineStage{
- {
- Plugin: "github",
- Subtasks: []string{},
- Options: map[string]interface{}{
- "connectionId": uint64(1),
- "owner": "test",
- "repo": "testRepo",
- "transformationRules": map[string]interface{}{
- "prType": "hey,man,wasup",
- "productionPattern": "xxxx",
- },
- },
- },
- {
- Plugin: "gitextractor",
- Options: map[string]interface{}{
- "proxy": "",
- "repoId": "github:GithubRepo:1:12345",
- "url": "https://git:123@this_is_cloneUrl",
- },
- },
- },
- plugin.PipelineStage{
- {
- Plugin: "refdiff",
- Options: map[string]interface{}{
- "repoId": "github:GithubRepo:1:12345",
- "tagsLimit": float64(10),
- "tagsOrder": "reverse semver",
- "tagsPattern": "pattern",
- },
- },
- },
- plugin.PipelineStage{
- {
- Plugin: "dora",
- Subtasks: []string{"EnrichTaskEnv"},
- Options: map[string]interface{}{},
- },
- },
- }
- assert.Equal(t, expectPlan, plan)
-}
-
-func TestMemorizedGetApiRepo(t *testing.T) {
- op := prepareOptions(t)
- expect := repo
- repo1, err := MemorizedGetApiRepo(repo, op, nil)
- assert.Nil(t, err)
- assert.Equal(t, expect, repo1)
- mockApiClient := prepareMockClient(t, repo)
- repo2, err := MemorizedGetApiRepo(nil, op, mockApiClient)
- assert.Nil(t, err)
- assert.NotEqual(t, expect, repo2)
-}
-
-func TestGetApiRepo(t *testing.T) {
- op := prepareOptions(t)
- mockClient := prepareMockClient(t, repo)
- repo1, err := getApiRepo(op, mockClient)
- assert.Nil(t, err)
- assert.Equal(t, repo.GithubId, repo1.GithubId)
-}
-
-func prepareMockMeta(t *testing.T) {
- mockMeta := mockplugin.NewPluginMeta(t)
- mockMeta.On("RootPkgPath").Return("github.com/apache/incubator-devlake/plugins/github")
- err := plugin.RegisterPlugin("github", mockMeta)
- assert.Nil(t, err)
-}
-
-func prepareMockClient(t *testing.T, repo *tasks.GithubApiRepo) aha.ApiClientAbstract {
- mockApiCLient := mockaha.NewApiClientAbstract(t)
- js, err := json.Marshal(repo)
- assert.Nil(t, err)
- res := &http.Response{}
- res.Body = io.NopCloser(bytes.NewBuffer(js))
- res.StatusCode = http.StatusOK
- mockApiCLient.On("Get", "repos/test/testRepo", mock.Anything, mock.Anything).Return(res, nil)
- return mockApiCLient
-}
-
-func prepareOptions(t *testing.T) *tasks.GithubOptions {
- var bs = &plugin.BlueprintScopeV100{
- Entities: []string{"CODE"},
- Options: json.RawMessage(`{
- "owner": "test",
- "repo": "testRepo"
- }`),
- Transformation: json.RawMessage(`{
- "prType": "hey,man,wasup",
- "refdiff": {
- "tagsPattern": "pattern",
- "tagsLimit": 10,
- "tagsOrder": "reverse semver"
- },
- "productionPattern": "xxxx"
- }`),
- }
- options := make(map[string]interface{})
- err := errors.Convert(json.Unmarshal(bs.Options, &options))
- assert.Nil(t, err)
- options["connectionId"] = 1
- // make sure task options is valid
- op, err := tasks.DecodeAndValidateTaskOptions(options)
- assert.Nil(t, err)
- return op
-}
diff --git a/backend/plugins/github/api/blueprint_v200.go b/backend/plugins/github/api/blueprint_v200.go
index fac202d45..e151bd844 100644
--- a/backend/plugins/github/api/blueprint_v200.go
+++ b/backend/plugins/github/api/blueprint_v200.go
@@ -19,7 +19,10 @@ package api
import (
"context"
+ "encoding/json"
"fmt"
+ "io"
+ "net/http"
"net/url"
"strings"
"time"
@@ -34,6 +37,7 @@ import (
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/core/utils"
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ aha "github.com/apache/incubator-devlake/helpers/pluginhelper/api/apihelperabstract"
"github.com/apache/incubator-devlake/plugins/github/models"
"github.com/apache/incubator-devlake/plugins/github/tasks"
"github.com/go-playground/validator/v10"
@@ -87,21 +91,21 @@ func makeDataSourcePipelinePlanV200(
if err != nil {
return nil, errors.Default.Wrap(err, fmt.Sprintf("fail to find repo %s", bpScope.Id))
}
- transformationRule := &models.GithubTransformationRule{}
- // get transformation rules from db
+ scopeConfig := &models.GithubScopeConfig{}
+ // get scope configs from db
db := basicRes.GetDal()
- err = db.First(transformationRule, dal.Where(`id = ?`, githubRepo.TransformationRuleId))
+ err = db.First(scopeConfig, dal.Where(`id = ?`, githubRepo.ScopeConfigId))
if err != nil && !db.IsErrorNotFound(err) {
return nil, err
}
// refdiff
- if transformationRule != nil && transformationRule.Refdiff != nil {
+ if scopeConfig != nil && scopeConfig.Refdiff != nil {
// add a new task to next stage
j := i + 1
if j == len(plan) {
plan = append(plan, nil)
}
- refdiffOp := transformationRule.Refdiff
+ refdiffOp := scopeConfig.Refdiff
refdiffOp["repoId"] = didgen.NewDomainIdGenerator(&models.GithubRepo{}).Generate(connection.ID, githubRepo.GithubId)
plan[j] = plugin.PipelineStage{
{
@@ -109,7 +113,7 @@ func makeDataSourcePipelinePlanV200(
Options: refdiffOp,
},
}
- transformationRule.Refdiff = nil
+ scopeConfig.Refdiff = nil
}
// construct task options for github
@@ -125,13 +129,13 @@ func makeDataSourcePipelinePlanV200(
if err != nil {
return nil, err
}
- stage, err = addGithub(subtaskMetas, connection, bpScope.Entities, stage, options)
+ stage, err = addGithub(subtaskMetas, connection, scopeConfig.Entities, stage, options)
if err != nil {
return nil, err
}
// add gitex stage
- if utils.StringsContains(bpScope.Entities, plugin.DOMAIN_TYPE_CODE) {
+ if utils.StringsContains(scopeConfig.Entities, plugin.DOMAIN_TYPE_CODE) {
cloneUrl, err := errors.Convert01(url.Parse(githubRepo.CloneUrl))
if err != nil {
return nil, err
@@ -162,9 +166,19 @@ func makeScopesV200(bpScopes []*plugin.BlueprintScopeV200, connection *models.Gi
if err != nil {
return nil, errors.Default.Wrap(err, fmt.Sprintf("fail to find repo%s", bpScope.Id))
}
- if utils.StringsContains(bpScope.Entities, plugin.DOMAIN_TYPE_CODE_REVIEW) ||
- utils.StringsContains(bpScope.Entities, plugin.DOMAIN_TYPE_CODE) ||
- utils.StringsContains(bpScope.Entities, plugin.DOMAIN_TYPE_CROSS) {
+
+ scopeConfig := &models.GithubScopeConfig{}
+ // get scope configs from db
+ db := basicRes.GetDal()
+ err = db.First(scopeConfig, dal.Where(`id = ?`, githubRepo.ScopeConfigId))
+ if err != nil && !db.IsErrorNotFound(err) {
+ return nil, err
+ }
+
+ basicRes.GetDal().First(scopeConfig, dal.Where(`id = ?`, githubRepo.ScopeConfigId))
+ if utils.StringsContains(scopeConfig.Entities, plugin.DOMAIN_TYPE_CODE_REVIEW) ||
+ utils.StringsContains(scopeConfig.Entities, plugin.DOMAIN_TYPE_CODE) ||
+ utils.StringsContains(scopeConfig.Entities, plugin.DOMAIN_TYPE_CROSS) {
// if we don't need to collect gitex, we need to add repo to scopes here
scopeRepo := &code.Repo{
DomainEntity: domainlayer.DomainEntity{
@@ -178,7 +192,7 @@ func makeScopesV200(bpScopes []*plugin.BlueprintScopeV200, connection *models.Gi
scopes = append(scopes, scopeRepo)
}
// add cicd_scope to scopes
- if utils.StringsContains(bpScope.Entities, plugin.DOMAIN_TYPE_CICD) {
+ if utils.StringsContains(scopeConfig.Entities, plugin.DOMAIN_TYPE_CICD) {
scopeCICD := &devops.CicdScope{
DomainEntity: domainlayer.DomainEntity{
Id: didgen.NewDomainIdGenerator(&models.GithubRepo{}).Generate(connection.ID, githubRepo.GithubId),
@@ -188,7 +202,7 @@ func makeScopesV200(bpScopes []*plugin.BlueprintScopeV200, connection *models.Gi
scopes = append(scopes, scopeCICD)
}
// add board to scopes
- if utils.StringsContains(bpScope.Entities, plugin.DOMAIN_TYPE_TICKET) {
+ if utils.StringsContains(scopeConfig.Entities, plugin.DOMAIN_TYPE_TICKET) {
scopeTicket := &ticket.Board{
DomainEntity: domainlayer.DomainEntity{
Id: didgen.NewDomainIdGenerator(&models.GithubRepo{}).Generate(connection.ID, githubRepo.GithubId),
@@ -200,3 +214,76 @@ func makeScopesV200(bpScopes []*plugin.BlueprintScopeV200, connection *models.Gi
}
return scopes, nil
}
+
+func addGithub(subtaskMetas []plugin.SubTaskMeta, connection *models.GithubConnection, entities []string, stage plugin.PipelineStage, options map[string]interface{}) (plugin.PipelineStage, errors.Error) {
+ // construct github(graphql) task
+ if connection.EnableGraphql {
+ // FIXME this need fix when 2 plugins merged
+ p, err := plugin.GetPlugin(`github_graphql`)
+ if err != nil {
+ return nil, err
+ }
+ if pluginGq, ok := p.(plugin.PluginTask); ok {
+ subtasks, err := helper.MakePipelinePlanSubtasks(pluginGq.SubTaskMetas(), entities)
+ if err != nil {
+ return nil, err
+ }
+ stage = append(stage, &plugin.PipelineTask{
+ Plugin: "github_graphql",
+ Subtasks: subtasks,
+ Options: options,
+ })
+ } else {
+ return nil, errors.BadInput.New("plugin github_graphql does not support SubTaskMetas")
+ }
+ } else {
+ subtasks, err := helper.MakePipelinePlanSubtasks(subtaskMetas, entities)
+ if err != nil {
+ return nil, err
+ }
+ stage = append(stage, &plugin.PipelineTask{
+ Plugin: "github",
+ Subtasks: subtasks,
+ Options: options,
+ })
+ }
+ return stage, nil
+}
+
+func getApiRepo(
+ op *tasks.GithubOptions,
+ apiClient aha.ApiClientAbstract,
+) (*tasks.GithubApiRepo, errors.Error) {
+ repoRes := &tasks.GithubApiRepo{}
+ res, err := apiClient.Get(fmt.Sprintf("repos/%s", op.Name), nil, nil)
+ if err != nil {
+ return nil, err
+ }
+ defer res.Body.Close()
+ if res.StatusCode != http.StatusOK {
+ return nil, errors.HttpStatus(res.StatusCode).New(fmt.Sprintf("unexpected status code when requesting repo detail from %s", res.Request.URL.String()))
+ }
+ body, err := errors.Convert01(io.ReadAll(res.Body))
+ if err != nil {
+ return nil, err
+ }
+ err = errors.Convert(json.Unmarshal(body, repoRes))
+ if err != nil {
+ return nil, err
+ }
+ return repoRes, nil
+}
+
+func MemorizedGetApiRepo(
+ repo *tasks.GithubApiRepo,
+ op *tasks.GithubOptions, apiClient aha.ApiClientAbstract,
+) (*tasks.GithubApiRepo, errors.Error) {
+ if repo == nil {
+ var err errors.Error
+ repo, err = getApiRepo(op, apiClient)
+ if err != nil {
+ return nil, err
+ }
+ }
+ return repo, nil
+}
diff --git a/backend/plugins/github/api/init.go b/backend/plugins/github/api/init.go
index f7e1ab7b2..9657f5918 100644
--- a/backend/plugins/github/api/init.go
+++ b/backend/plugins/github/api/init.go
@@ -18,20 +18,21 @@ limitations under the License.
package api
import (
+ "strconv"
+
"github.com/apache/incubator-devlake/core/context"
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/github/models"
"github.com/go-playground/validator/v10"
- "strconv"
)
var vld *validator.Validate
var connectionHelper *api.ConnectionApiHelper
-var scopeHelper *api.ScopeApiHelper[models.GithubConnection, models.GithubRepo, models.GithubTransformationRule]
+var scopeHelper *api.ScopeApiHelper[models.GithubConnection, models.GithubRepo, models.GithubScopeConfig]
var basicRes context.BasicRes
-var trHelper *api.TransformationRuleHelper[models.GithubTransformationRule]
+var scHelper *api.ScopeConfigHelper[models.GithubScopeConfig]
func Init(br context.BasicRes) {
basicRes = br
@@ -45,11 +46,11 @@ func Init(br context.BasicRes) {
ScopeIdColumnName: "github_id",
RawScopeParamName: "Name",
}
- scopeHelper = api.NewScopeHelper[models.GithubConnection, models.GithubRepo, models.GithubTransformationRule](
+ scopeHelper = api.NewScopeHelper[models.GithubConnection, models.GithubRepo, models.GithubScopeConfig](
basicRes,
vld,
connectionHelper,
- api.NewScopeDatabaseHelperImpl[models.GithubConnection, models.GithubRepo, models.GithubTransformationRule](
+ api.NewScopeDatabaseHelperImpl[models.GithubConnection, models.GithubRepo, models.GithubScopeConfig](
basicRes, connectionHelper, params),
params,
&api.ScopeHelperOptions{
@@ -69,7 +70,7 @@ func Init(br context.BasicRes) {
},
},
)
- trHelper = api.NewTransformationRuleHelper[models.GithubTransformationRule](
+ scHelper = api.NewScopeConfigHelper[models.GithubScopeConfig](
basicRes,
vld,
)
diff --git a/backend/plugins/github/api/scope.go b/backend/plugins/github/api/scope.go
index 3346b113c..4deba7649 100644
--- a/backend/plugins/github/api/scope.go
+++ b/backend/plugins/github/api/scope.go
@@ -26,7 +26,7 @@ import (
type ScopeRes struct {
models.GithubRepo
- TransformationRuleName string `json:"transformationRuleName,omitempty"`
+ ScopeConfigName string `json:"scopeConfigName,omitempty"`
}
type ScopeReq api.ScopeReq[models.GithubRepo]
diff --git a/backend/plugins/github/api/swagger.go b/backend/plugins/github/api/swagger.go
index 90e9ef929..bf93d67ac 100644
--- a/backend/plugins/github/api/swagger.go
+++ b/backend/plugins/github/api/swagger.go
@@ -17,59 +17,16 @@ limitations under the License.
package api
-// @Summary pipelines plan for github
-// @Description pipelines plan for github
-// @Tags plugins/github
-// @Accept application/json
-// @Param pipeline body GithubPipelinePlan true "json"
-// @Router /pipelines/github/pipeline-plan [post]
-func _() {}
+import (
+ "github.com/apache/incubator-devlake/plugins/github/tasks"
+)
-type GithubPipelinePlan [][]struct {
- Plugin string `json:"plugin"`
- Subtasks []string `json:"subtasks"`
- Options struct {
- ConnectionID int `json:"connectionId"`
- Owner string `json:"owner"`
- Repo string `json:"repo"`
- Since string
- Transformation CodeTransformationRules `json:"transformation"`
- } `json:"options"`
-}
+type GithubTaskOptions tasks.GithubOptions
-type CodeTransformationRules struct {
- PrType string `mapstructure:"prType" json:"prType"`
- PrComponent string `mapstructure:"prComponent" json:"prComponent"`
- PrBodyClosePattern string `mapstructure:"prBodyClosePattern" json:"prBodyClosePattern"`
- IssueSeverity string `mapstructure:"issueSeverity" json:"issueSeverity"`
- IssuePriority string `mapstructure:"issuePriority" json:"issuePriority"`
- IssueComponent string `mapstructure:"issueComponent" json:"issueComponent"`
- IssueTypeBug string `mapstructure:"issueTypeBug" json:"issueTypeBug"`
- IssueTypeIncident string `mapstructure:"issueTypeIncident" json:"issueTypeIncident"`
- IssueTypeRequirement string `mapstructure:"issueTypeRequirement" json:"issueTypeRequirement"`
-}
-
-// @Summary blueprints setting for github
-// @Description blueprint setting for github
+// @Summary github task options for pipelines
+// @Description This is a dummy API to demonstrate the available task options for github pipelines
// @Tags plugins/github
// @Accept application/json
-// @Param blueprint body GithubBlueprintSetting true "json"
-// @Router /blueprints/github/blueprint-setting [post]
+// @Param pipeline body GithubTaskOptions true "json"
+// @Router /pipelines/github/pipeline-task [post]
func _() {}
-
-type GithubBlueprintSetting []struct {
- Version string `json:"version"`
- Connections []struct {
- Plugin string `json:"plugin"`
- ConnectionID int `json:"connectionId"`
- Scope []struct {
- Transformation CodeTransformationRules `json:"transformation"`
- Options struct {
- Owner string `json:"owner"`
- Repo string `json:"repo"`
- Since string
- } `json:"options"`
- Entities []string `json:"entities"`
- } `json:"scopes"`
- } `json:"connections"`
-}
diff --git a/backend/plugins/github/api/transformation_rule.go b/backend/plugins/github/api/transformation_rule.go
index 89a86bd12..d802f8c9c 100644
--- a/backend/plugins/github/api/transformation_rule.go
+++ b/backend/plugins/github/api/transformation_rule.go
@@ -22,62 +22,62 @@ import (
"github.com/apache/incubator-devlake/core/plugin"
)
-// CreateTransformationRule create transformation rule for Github
-// @Summary create transformation rule for Github
-// @Description create transformation rule for Github
+// CreateScopeConfig create scope config for Github
+// @Summary create scope config for Github
+// @Description create scope config for Github
// @Tags plugins/github
// @Accept application/json
// @Param connectionId path int true "connectionId"
-// @Param transformationRule body models.GithubTransformationRule true "transformation rule"
-// @Success 200 {object} models.GithubTransformationRule
+// @Param scopeConfig body models.GithubScopeConfig true "scope config"
+// @Success 200 {object} models.GithubScopeConfig
// @Failure 400 {object} shared.ApiBody "Bad Request"
// @Failure 500 {object} shared.ApiBody "Internal Error"
-// @Router /plugins/github/connections/{connectionId}/transformation_rules [POST]
-func CreateTransformationRule(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
- return trHelper.Create(input)
+// @Router /plugins/github/connections/{connectionId}/scope_configs [POST]
+func CreateScopeConfig(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
+ return scHelper.Create(input)
}
-// UpdateTransformationRule update transformation rule for Github
-// @Summary update transformation rule for Github
-// @Description update transformation rule for Github
+// UpdateScopeConfig update scope config for Github
+// @Summary update scope config for Github
+// @Description update scope config for Github
// @Tags plugins/github
// @Accept application/json
// @Param id path int true "id"
// @Param connectionId path int true "connectionId"
-// @Param transformationRule body models.GithubTransformationRule true "transformation rule"
-// @Success 200 {object} models.GithubTransformationRule
+// @Param scopeConfig body models.GithubScopeConfig true "scope config"
+// @Success 200 {object} models.GithubScopeConfig
// @Failure 400 {object} shared.ApiBody "Bad Request"
// @Failure 500 {object} shared.ApiBody "Internal Error"
-// @Router /plugins/github/connections/{connectionId}/transformation_rules/{id} [PATCH]
-func UpdateTransformationRule(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
- return trHelper.Update(input)
+// @Router /plugins/github/connections/{connectionId}/scope_configs/{id} [PATCH]
+func UpdateScopeConfig(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
+ return scHelper.Update(input)
}
-// GetTransformationRule return one transformation rule
-// @Summary return one transformation rule
-// @Description return one transformation rule
+// GetScopeConfig return one scope config
+// @Summary return one scope config
+// @Description return one scope config
// @Tags plugins/github
// @Param id path int true "id"
// @Param connectionId path int true "connectionId"
-// @Success 200 {object} models.GithubTransformationRule
+// @Success 200 {object} models.GithubScopeConfig
// @Failure 400 {object} shared.ApiBody "Bad Request"
// @Failure 500 {object} shared.ApiBody "Internal Error"
-// @Router /plugins/github/connections/{connectionId}/transformation_rules/{id} [GET]
-func GetTransformationRule(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
- return trHelper.Get(input)
+// @Router /plugins/github/connections/{connectionId}/scope_configs/{id} [GET]
+func GetScopeConfig(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
+ return scHelper.Get(input)
}
-// GetTransformationRuleList return all transformation rules
-// @Summary return all transformation rules
-// @Description return all transformation rules
+// GetScopeConfigList return all scope configs
+// @Summary return all scope configs
+// @Description return all scope configs
// @Tags plugins/github
// @Param pageSize query int false "page size, default 50"
// @Param page query int false "page size, default 1"
// @Param connectionId path int true "connectionId"
-// @Success 200 {object} []models.GithubTransformationRule
+// @Success 200 {object} []models.GithubScopeConfig
// @Failure 400 {object} shared.ApiBody "Bad Request"
// @Failure 500 {object} shared.ApiBody "Internal Error"
-// @Router /plugins/github/connections/{connectionId}/transformation_rules [GET]
-func GetTransformationRuleList(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
- return trHelper.List(input)
+// @Router /plugins/github/connections/{connectionId}/scope_configs [GET]
+func GetScopeConfigList(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
+ return scHelper.List(input)
}
diff --git a/backend/plugins/github/e2e/comment_test.go b/backend/plugins/github/e2e/comment_test.go
index 17147bdab..36fd6f340 100644
--- a/backend/plugins/github/e2e/comment_test.go
+++ b/backend/plugins/github/e2e/comment_test.go
@@ -37,7 +37,7 @@ func TestCommentDataFlow(t *testing.T) {
ConnectionId: 1,
Name: "panjf2000/ants",
GithubId: 134018330,
- GithubTransformationRule: &models.GithubTransformationRule{
+ ScopeConfig: &models.GithubScopeConfig{
PrType: "type/(.*)$",
PrComponent: "component/(.*)$",
PrBodyClosePattern: "(?mi)(fix|close|resolve|fixes|closes|resolves|fixed|closed|resolved)[\\s]*.*(((and )?(#|https:\\/\\/github.com\\/%s\\/issues\\/)\\d+[ ]*)+)",
diff --git a/backend/plugins/github/e2e/issue_test.go b/backend/plugins/github/e2e/issue_test.go
index a5c015cc4..d3d838c2a 100644
--- a/backend/plugins/github/e2e/issue_test.go
+++ b/backend/plugins/github/e2e/issue_test.go
@@ -36,7 +36,7 @@ func TestIssueDataFlow(t *testing.T) {
ConnectionId: 1,
Name: "panjf2000/ants",
GithubId: 134018330,
- GithubTransformationRule: &models.GithubTransformationRule{
+ ScopeConfig: &models.GithubScopeConfig{
PrType: "type/(.*)$",
PrComponent: "component/(.*)$",
PrBodyClosePattern: "(?mi)(fix|close|resolve|fixes|closes|resolves|fixed|closed|resolved)[\\s]*.*(((and )?(#|https:\\/\\/github.com\\/%s\\/issues\\/)\\d+[ ]*)+)",
diff --git a/backend/plugins/github/e2e/milestone_test.go b/backend/plugins/github/e2e/milestone_test.go
index 461fd5a1f..9e4672b60 100644
--- a/backend/plugins/github/e2e/milestone_test.go
+++ b/backend/plugins/github/e2e/milestone_test.go
@@ -18,12 +18,13 @@ limitations under the License.
package e2e
import (
+ "testing"
+
"github.com/apache/incubator-devlake/core/models/domainlayer/ticket"
"github.com/apache/incubator-devlake/helpers/e2ehelper"
"github.com/apache/incubator-devlake/plugins/github/impl"
"github.com/apache/incubator-devlake/plugins/github/models"
"github.com/apache/incubator-devlake/plugins/github/tasks"
- "testing"
)
func TestMilestoneDataFlow(t *testing.T) {
@@ -35,7 +36,7 @@ func TestMilestoneDataFlow(t *testing.T) {
ConnectionId: 1,
Name: "panjf2000/ants",
GithubId: 134018330,
- GithubTransformationRule: &models.GithubTransformationRule{
+ ScopeConfig: &models.GithubScopeConfig{
PrType: "type/(.*)$",
PrComponent: "component/(.*)$",
PrBodyClosePattern: "(?mi)(fix|close|resolve|fixes|closes|resolves|fixed|closed|resolved)[\\s]*.*(((and )?(#|https:\\/\\/github.com\\/%s\\/issues\\/)\\d+[ ]*)+)",
diff --git a/backend/plugins/github/e2e/pr_enrich_issue_test.go b/backend/plugins/github/e2e/pr_enrich_issue_test.go
index 86989494d..720391075 100644
--- a/backend/plugins/github/e2e/pr_enrich_issue_test.go
+++ b/backend/plugins/github/e2e/pr_enrich_issue_test.go
@@ -36,7 +36,7 @@ func TestPrEnrichIssueDataFlow(t *testing.T) {
ConnectionId: 1,
Name: "panjf2000/ants",
GithubId: 134018330,
- GithubTransformationRule: &models.GithubTransformationRule{
+ ScopeConfig: &models.GithubScopeConfig{
PrType: "type/(.*)$",
PrComponent: "component/(.*)$",
PrBodyClosePattern: "(?mi)(fix|close|resolve|fixes|closes|resolves|fixed|closed|resolved)[\\s]*.*(((and )?(#|https:\\/\\/github.com\\/%s\\/issues\\/)\\d+[ ]*)+)",
diff --git a/backend/plugins/github/e2e/pr_test.go b/backend/plugins/github/e2e/pr_test.go
index 440a5c3cb..82a5b4bec 100644
--- a/backend/plugins/github/e2e/pr_test.go
+++ b/backend/plugins/github/e2e/pr_test.go
@@ -36,7 +36,7 @@ func TestPrDataFlow(t *testing.T) {
ConnectionId: 1,
Name: "panjf2000/ants",
GithubId: 134018330,
- GithubTransformationRule: &models.GithubTransformationRule{
+ ScopeConfig: &models.GithubScopeConfig{
PrType: "type/(.*)$",
PrComponent: "component/(.*)$",
PrBodyClosePattern: "(?mi)(fix|close|resolve|fixes|closes|resolves|fixed|closed|resolved)[\\s]*.*(((and )?(#|https:\\/\\/github.com\\/%s\\/issues\\/)\\d+[ ]*)+)",
diff --git a/backend/plugins/github/e2e/repo_test.go b/backend/plugins/github/e2e/repo_test.go
index 6cef18f14..5ceccec09 100644
--- a/backend/plugins/github/e2e/repo_test.go
+++ b/backend/plugins/github/e2e/repo_test.go
@@ -40,7 +40,7 @@ func TestRepoDataFlow(t *testing.T) {
ConnectionId: 1,
Name: "panjf2000/ants",
GithubId: 134018330,
- GithubTransformationRule: &models.GithubTransformationRule{
+ ScopeConfig: &models.GithubScopeConfig{
PrType: "type/(.*)$",
PrComponent: "component/(.*)$",
},
diff --git a/backend/plugins/github/github.go b/backend/plugins/github/github.go
index ef6169da5..5dd7f8c03 100644
--- a/backend/plugins/github/github.go
+++ b/backend/plugins/github/github.go
@@ -55,7 +55,7 @@ func main() {
"owner": *owner,
"repo": *repo,
"timeAfter": *timeAfter,
- "transformationRules": map[string]interface{}{
+ "scopeConfig": map[string]interface{}{
"prType": *prType,
"prComponent": *prComponent,
"prBodyClosePattern": *prBodyClosePattern,
diff --git a/backend/plugins/github/impl/impl.go b/backend/plugins/github/impl/impl.go
index c5ebaae5d..606ffc8c3 100644
--- a/backend/plugins/github/impl/impl.go
+++ b/backend/plugins/github/impl/impl.go
@@ -19,9 +19,10 @@ package impl
import (
"fmt"
- "github.com/apache/incubator-devlake/core/models/domainlayer/devops"
"time"
+ "github.com/apache/incubator-devlake/core/models/domainlayer/devops"
+
"github.com/apache/incubator-devlake/core/context"
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
@@ -38,7 +39,7 @@ var _ plugin.PluginInit = (*Github)(nil)
var _ plugin.PluginTask = (*Github)(nil)
var _ plugin.PluginApi = (*Github)(nil)
var _ plugin.PluginModel = (*Github)(nil)
-var _ plugin.PluginBlueprintV100 = (*Github)(nil)
+var _ plugin.DataSourcePluginBlueprintV200 = (*Github)(nil)
var _ plugin.CloseablePluginTask = (*Github)(nil)
var _ plugin.PluginSource = (*Github)(nil)
@@ -52,8 +53,8 @@ func (p Github) Scope() interface{} {
return &models.GithubRepo{}
}
-func (p Github) TransformationRule() interface{} {
- return &models.GithubTransformationRule{}
+func (p Github) ScopeConfig() interface{} {
+ return &models.GithubScopeConfig{}
}
func (p Github) Init(basicRes context.BasicRes) errors.Error {
@@ -167,10 +168,10 @@ func (p Github) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]i
}
regexEnricher := helper.NewRegexEnricher()
- if err = regexEnricher.TryAdd(devops.DEPLOYMENT, op.DeploymentPattern); err != nil {
+ if err = regexEnricher.TryAdd(devops.DEPLOYMENT, op.ScopeConfig.DeploymentPattern); err != nil {
return nil, errors.BadInput.Wrap(err, "invalid value for `deploymentPattern`")
}
- if err = regexEnricher.TryAdd(devops.PRODUCTION, op.ProductionPattern); err != nil {
+ if err = regexEnricher.TryAdd(devops.PRODUCTION, op.ScopeConfig.ProductionPattern); err != nil {
return nil, errors.BadInput.Wrap(err, "invalid value for `productionPattern`")
}
@@ -223,13 +224,13 @@ func (p Github) ApiResources() map[string]map[string]plugin.ApiResourceHandler {
"GET": api.GetScopeList,
"PUT": api.PutScope,
},
- "connections/:connectionId/transformation_rules": {
- "POST": api.CreateTransformationRule,
- "GET": api.GetTransformationRuleList,
+ "connections/:connectionId/scope_configs": {
+ "POST": api.CreateScopeConfig,
+ "GET": api.GetScopeConfigList,
},
- "connections/:connectionId/transformation_rules/:id": {
- "PATCH": api.UpdateTransformationRule,
- "GET": api.GetTransformationRule,
+ "connections/:connectionId/scope_configs/:id": {
+ "PATCH": api.UpdateScopeConfig,
+ "GET": api.GetScopeConfig,
},
"connections/:connectionId/proxy/rest/*path": {
"GET": api.Proxy,
@@ -237,10 +238,6 @@ func (p Github) ApiResources() map[string]map[string]plugin.ApiResourceHandler {
}
}
-func (p Github) MakePipelinePlan(connectionId uint64, scope []*plugin.BlueprintScopeV100) (plugin.PipelinePlan, errors.Error) {
- return api.MakePipelinePlan(p.SubTaskMetas(), connectionId, scope)
-}
-
func (p Github) MakeDataSourcePipelinePlanV200(connectionId uint64, scopes []*plugin.BlueprintScopeV200, syncPolicy plugin.BlueprintSyncPolicy) (pp plugin.PipelinePlan, sc []plugin.Scope, err errors.Error) {
return api.MakeDataSourcePipelinePlanV200(p.SubTaskMetas(), connectionId, scopes, &syncPolicy)
}
@@ -271,8 +268,8 @@ func EnrichOptions(taskCtx plugin.TaskContext,
if err == nil {
op.Name = githubRepo.Name
op.GithubId = githubRepo.GithubId
- if op.TransformationRuleId == 0 {
- op.TransformationRuleId = githubRepo.TransformationRuleId
+ if op.ScopeConfigId == 0 {
+ op.ScopeConfigId = githubRepo.ScopeConfigId
}
} else {
if taskCtx.GetDal().IsErrorNotFound(err) && op.Name != "" {
@@ -292,18 +289,18 @@ func EnrichOptions(taskCtx plugin.TaskContext,
return errors.Default.Wrap(err, fmt.Sprintf("fail to find repo %s", op.Name))
}
}
- // Set GithubTransformationRule if it's nil, this has lower priority
- if op.GithubTransformationRule == nil && op.TransformationRuleId != 0 {
- var transformationRule models.GithubTransformationRule
+ // Set GithubScopeConfig if it's nil, this has lower priority
+ if op.ScopeConfig == nil && op.ScopeConfigId != 0 {
+ var scopeConfig models.GithubScopeConfig
db := taskCtx.GetDal()
- err = db.First(&transformationRule, dal.Where("id = ?", githubRepo.TransformationRuleId))
+ err = db.First(&scopeConfig, dal.Where("id = ?", githubRepo.ScopeConfigId))
if err != nil && !db.IsErrorNotFound(err) {
- return errors.BadInput.Wrap(err, "fail to get transformationRule")
+ return errors.BadInput.Wrap(err, "fail to get scopeConfig")
}
- op.GithubTransformationRule = &transformationRule
+ op.ScopeConfig = &scopeConfig
}
- if op.GithubTransformationRule == nil && op.TransformationRuleId == 0 {
- op.GithubTransformationRule = new(models.GithubTransformationRule)
+ if op.ScopeConfig == nil && op.ScopeConfigId == 0 {
+ op.ScopeConfig = new(models.GithubScopeConfig)
}
return err
}
diff --git a/backend/plugins/github/models/migrationscripts/20230322_add_connection_id_to_transformation_rules.go b/backend/plugins/github/models/migrationscripts/20230322_add_connection_id_to_transformation_rules.go
index 61db54ce7..f82afff0b 100644
--- a/backend/plugins/github/models/migrationscripts/20230322_add_connection_id_to_transformation_rules.go
+++ b/backend/plugins/github/models/migrationscripts/20230322_add_connection_id_to_transformation_rules.go
@@ -22,25 +22,34 @@ import (
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/helpers/migrationhelper"
- "github.com/apache/incubator-devlake/plugins/github/models"
)
type addConnectionIdToTransformationRule struct{}
-type transformationRule20220322 struct {
+type repo20230322 struct {
+ ConnectionId uint64 `gorm:"primaryKey"`
+ GithubId int `gorm:"primaryKey"`
+ TransformationRuleId uint64
+}
+
+func (repo20230322) TableName() string {
+ return "_tool_github_repos"
+}
+
+type transformationRule20230322 struct {
ConnectionId uint64
}
-func (transformationRule20220322) TableName() string {
+func (transformationRule20230322) TableName() string {
return "_tool_github_transformation_rules"
}
func (u *addConnectionIdToTransformationRule) Up(baseRes context.BasicRes) errors.Error {
- err := migrationhelper.AutoMigrateTables(baseRes, &transformationRule20220322{})
+ err := migrationhelper.AutoMigrateTables(baseRes, &transformationRule20230322{})
if err != nil {
return err
}
- var scopes []models.GithubRepo
+ var scopes []repo20230322
err = baseRes.GetDal().All(&scopes)
if err != nil {
return err
@@ -55,14 +64,14 @@ func (u *addConnectionIdToTransformationRule) Up(baseRes context.BasicRes) error
// set connection_id for rules
for trId, cId := range idMap {
err = baseRes.GetDal().UpdateColumn(
- &models.GithubTransformationRule{}, "connection_id", cId,
+ &transformationRule20230322{}, "connection_id", cId,
dal.Where("id = ?", trId))
if err != nil {
return err
}
}
// delete all rules that are not referenced.
- return baseRes.GetDal().Delete(&models.GithubTransformationRule{}, dal.Where("connection_id IS NULL OR connection_id = 0"))
+ return baseRes.GetDal().Delete(&transformationRule20230322{}, dal.Where("connection_id IS NULL OR connection_id = 0"))
}
func (*addConnectionIdToTransformationRule) Version() uint64 {
diff --git a/backend/plugins/github/models/migrationscripts/20230526_scope_config.go b/backend/plugins/github/models/migrationscripts/20230526_scope_config.go
new file mode 100644
index 000000000..42abee3fc
--- /dev/null
+++ b/backend/plugins/github/models/migrationscripts/20230526_scope_config.go
@@ -0,0 +1,56 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+ "github.com/apache/incubator-devlake/core/context"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/helpers/migrationhelper"
+)
+
+type renameTr2ScopeConfig struct {
+}
+
+type scopeConfig20230526 struct {
+ Entities []string `gorm:"type:json" json:"entities"`
+}
+
+func (scopeConfig20230526) TableName() string {
+ return "_tool_github_scope_configs"
+}
+
+func (u *renameTr2ScopeConfig) Up(baseRes context.BasicRes) errors.Error {
+ db := baseRes.GetDal()
+ err := db.RenameColumn("_tool_github_repos", "transformation_rule_id", "scope_config_id")
+ if err != nil {
+ return err
+ }
+ err = db.RenameTable("_tool_github_transformation_rules", "_tool_github_scope_configs")
+ if err != nil {
+ return err
+ }
+ return migrationhelper.AutoMigrateTables(baseRes, &scopeConfig20230526{})
+}
+
+func (*renameTr2ScopeConfig) Version() uint64 {
+ return 20230526161352
+}
+
+func (*renameTr2ScopeConfig) Name() string {
+ return "rename transformation rule to scope config for github"
+}
diff --git a/backend/plugins/github/models/migrationscripts/register.go b/backend/plugins/github/models/migrationscripts/register.go
index f11587f34..bf800e468 100644
--- a/backend/plugins/github/models/migrationscripts/register.go
+++ b/backend/plugins/github/models/migrationscripts/register.go
@@ -23,6 +23,7 @@ import (
// All return all the migration scripts
func All() []plugin.MigrationScript {
+ println("github register.go")
return []plugin.MigrationScript{
new(addInitTables),
new(addGithubRunsTable),
@@ -39,5 +40,6 @@ func All() []plugin.MigrationScript {
new(addGithubCommitAuthorInfo),
new(fixRunNameToText),
new(addGithubMultiAuth),
+ new(renameTr2ScopeConfig),
}
}
diff --git a/backend/plugins/github/models/repo.go b/backend/plugins/github/models/repo.go
index 4b4cf6a30..b320a5bb2 100644
--- a/backend/plugins/github/models/repo.go
+++ b/backend/plugins/github/models/repo.go
@@ -18,29 +18,30 @@ limitations under the License.
package models
import (
- "github.com/apache/incubator-devlake/core/models/common"
- "github.com/apache/incubator-devlake/core/plugin"
"strconv"
"time"
+
+ "github.com/apache/incubator-devlake/core/models/common"
+ "github.com/apache/incubator-devlake/core/plugin"
)
var _ plugin.ToolLayerScope = (*GithubRepo)(nil)
type GithubRepo struct {
- ConnectionId uint64 `json:"connectionId" gorm:"primaryKey" validate:"required" mapstructure:"connectionId,omitempty"`
- GithubId int `json:"githubId" gorm:"primaryKey" validate:"required" mapstructure:"githubId"`
- Name string `json:"name" gorm:"type:varchar(255)" mapstructure:"name,omitempty"`
- HTMLUrl string `json:"HTMLUrl" gorm:"type:varchar(255)" mapstructure:"HTMLUrl,omitempty"`
- Description string `json:"description" mapstructure:"description,omitempty"`
- TransformationRuleId uint64 `json:"transformationRuleId,omitempty" mapstructure:"transformationRuleId,omitempty"`
- OwnerId int `json:"ownerId" mapstructure:"ownerId,omitempty"`
- Language string `json:"language" gorm:"type:varchar(255)" mapstructure:"language,omitempty"`
- ParentGithubId int `json:"parentId" mapstructure:"parentGithubId,omitempty"`
- ParentHTMLUrl string `json:"parentHtmlUrl" mapstructure:"parentHtmlUrl,omitempty"`
- CloneUrl string `json:"cloneUrl" gorm:"type:varchar(255)" mapstructure:"cloneUrl,omitempty"`
- CreatedDate *time.Time `json:"createdDate" mapstructure:"-"`
- UpdatedDate *time.Time `json:"updatedDate" mapstructure:"-"`
- common.NoPKModel `json:"-" mapstructure:"-"`
+ ConnectionId uint64 `json:"connectionId" gorm:"primaryKey" validate:"required" mapstructure:"connectionId,omitempty"`
+ GithubId int `json:"githubId" gorm:"primaryKey" validate:"required" mapstructure:"githubId"`
+ Name string `json:"name" gorm:"type:varchar(255)" mapstructure:"name,omitempty"`
+ HTMLUrl string `json:"HTMLUrl" gorm:"type:varchar(255)" mapstructure:"HTMLUrl,omitempty"`
+ Description string `json:"description" mapstructure:"description,omitempty"`
+ ScopeConfigId uint64 `json:"scopeConfigId,omitempty" mapstructure:"scopeConfigId,omitempty"`
+ OwnerId int `json:"ownerId" mapstructure:"ownerId,omitempty"`
+ Language string `json:"language" gorm:"type:varchar(255)" mapstructure:"language,omitempty"`
+ ParentGithubId int `json:"parentId" mapstructure:"parentGithubId,omitempty"`
+ ParentHTMLUrl string `json:"parentHtmlUrl" mapstructure:"parentHtmlUrl,omitempty"`
+ CloneUrl string `json:"cloneUrl" gorm:"type:varchar(255)" mapstructure:"cloneUrl,omitempty"`
+ CreatedDate *time.Time `json:"createdDate" mapstructure:"-"`
+ UpdatedDate *time.Time `json:"updatedDate" mapstructure:"-"`
+ common.NoPKModel `json:"-" mapstructure:"-"`
}
func (r GithubRepo) ScopeId() string {
diff --git a/backend/plugins/github/models/transformation_rule.go b/backend/plugins/github/models/scope_config.go
similarity index 93%
rename from backend/plugins/github/models/transformation_rule.go
rename to backend/plugins/github/models/scope_config.go
index d7ecb35df..7abaf831f 100644
--- a/backend/plugins/github/models/transformation_rule.go
+++ b/backend/plugins/github/models/scope_config.go
@@ -22,8 +22,8 @@ import (
"gorm.io/datatypes"
)
-type GithubTransformationRule struct {
- common.Model `mapstructure:"-"`
+type GithubScopeConfig struct {
+ common.ScopeConfig `mapstructure:",squash" json:",inline" gorm:"embedded"`
ConnectionId uint64 `mapstructure:"connectionId" json:"connectionId"`
Name string `mapstructure:"name" json:"name" gorm:"type:varchar(255);index:idx_name_github,unique" validate:"required"`
PrType string `mapstructure:"prType,omitempty" json:"prType" gorm:"type:varchar(255)"`
@@ -40,6 +40,6 @@ type GithubTransformationRule struct {
Refdiff datatypes.JSONMap `mapstructure:"refdiff,omitempty" json:"refdiff" swaggertype:"object" format:"json"`
}
-func (GithubTransformationRule) TableName() string {
- return "_tool_github_transformation_rules"
+func (GithubScopeConfig) TableName() string {
+ return "_tool_github_scope_configs"
}
diff --git a/backend/plugins/github/tasks/issue_extractor.go b/backend/plugins/github/tasks/issue_extractor.go
index 45963d52c..7bc2a6f5e 100644
--- a/backend/plugins/github/tasks/issue_extractor.go
+++ b/backend/plugins/github/tasks/issue_extractor.go
@@ -73,7 +73,7 @@ type IssueRegexes struct {
func ExtractApiIssues(taskCtx plugin.SubTaskContext) errors.Error {
data := taskCtx.GetData().(*GithubTaskData)
- config := data.Options.GithubTransformationRule
+ config := data.Options.ScopeConfig
issueRegexes, err := NewIssueRegexes(config)
if err != nil {
return nil
@@ -222,7 +222,7 @@ func convertGithubLabels(issueRegexes *IssueRegexes, issue *IssuesResponse, gith
return results, nil
}
-func NewIssueRegexes(config *models.GithubTransformationRule) (*IssueRegexes, errors.Error) {
+func NewIssueRegexes(config *models.GithubScopeConfig) (*IssueRegexes, errors.Error) {
var issueRegexes IssueRegexes
if config == nil {
return &issueRegexes, nil
diff --git a/backend/plugins/github/tasks/pr_extractor.go b/backend/plugins/github/tasks/pr_extractor.go
index 7734306eb..8a03cdabc 100644
--- a/backend/plugins/github/tasks/pr_extractor.go
+++ b/backend/plugins/github/tasks/pr_extractor.go
@@ -66,7 +66,7 @@ type GithubApiPullRequest struct {
func ExtractApiPullRequests(taskCtx plugin.SubTaskContext) errors.Error {
data := taskCtx.GetData().(*GithubTaskData)
- config := data.Options.GithubTransformationRule
+ config := data.Options.ScopeConfig
var labelTypeRegex *regexp.Regexp
var labelComponentRegex *regexp.Regexp
var prType = config.PrType
diff --git a/backend/plugins/github/tasks/pr_issue_enricher.go b/backend/plugins/github/tasks/pr_issue_enricher.go
index 0e784e400..ed0cbacae 100644
--- a/backend/plugins/github/tasks/pr_issue_enricher.go
+++ b/backend/plugins/github/tasks/pr_issue_enricher.go
@@ -18,15 +18,16 @@ limitations under the License.
package tasks
import (
+ "reflect"
+ "regexp"
+ "strconv"
+ "strings"
+
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/github/models"
- "reflect"
- "regexp"
- "strconv"
- "strings"
)
var EnrichPullRequestIssuesMeta = plugin.SubTaskMeta{
@@ -43,7 +44,7 @@ func EnrichPullRequestIssues(taskCtx plugin.SubTaskContext) (err errors.Error) {
repoId := data.Options.GithubId
var prBodyCloseRegex *regexp.Regexp
- prBodyClosePattern := data.Options.PrBodyClosePattern
+ prBodyClosePattern := data.Options.ScopeConfig.PrBodyClosePattern
//the pattern before the issue number, sometimes, the issue number is #1098, sometimes it is https://xxx/#1098
prBodyClosePattern = strings.Replace(prBodyClosePattern, "%s", data.Options.Name, 1)
if len(prBodyClosePattern) > 0 {
diff --git a/backend/plugins/github/tasks/task_data.go b/backend/plugins/github/tasks/task_data.go
index ec0b367e2..0f507342d 100644
--- a/backend/plugins/github/tasks/task_data.go
+++ b/backend/plugins/github/tasks/task_data.go
@@ -29,15 +29,14 @@ import (
)
type GithubOptions struct {
- ConnectionId uint64 `json:"connectionId" mapstructure:"connectionId,omitempty"`
- TransformationRuleId uint64 `json:"transformationRuleId" mapstructure:"transformationRuleId,omitempty"`
- GithubId int `json:"githubId" mapstructure:"githubId,omitempty"`
- Tasks []string `json:"tasks,omitempty" mapstructure:",omitempty"`
- TimeAfter string `json:"timeAfter" mapstructure:"timeAfter,omitempty"`
- Owner string `json:"owner" mapstructure:"owner,omitempty"`
- Repo string `json:"repo" mapstructure:"repo,omitempty"`
- Name string `json:"name" mapstructure:"name,omitempty"`
- *models.GithubTransformationRule `mapstructure:"transformationRules,omitempty" json:"transformationRules"`
+ ConnectionId uint64 `json:"connectionId" mapstructure:"connectionId,omitempty"`
+ ScopeConfigId uint64 `json:"scopeConfigId" mapstructure:"scopeConfigId,omitempty"`
+ GithubId int `json:"githubId" mapstructure:"githubId,omitempty"`
+ TimeAfter string `json:"timeAfter" mapstructure:"timeAfter,omitempty"`
+ Owner string `json:"owner" mapstructure:"owner,omitempty"`
+ Repo string `json:"repo" mapstructure:"repo,omitempty"`
+ Name string `json:"name" mapstructure:"name,omitempty"`
+ ScopeConfig *models.GithubScopeConfig `mapstructure:"scopeConfig,omitempty" json:"scopeConfig"`
}
type GithubTaskData struct {
diff --git a/backend/plugins/github_graphql/impl/impl.go b/backend/plugins/github_graphql/impl/impl.go
index 5d4466378..68008164b 100644
--- a/backend/plugins/github_graphql/impl/impl.go
+++ b/backend/plugins/github_graphql/impl/impl.go
@@ -20,12 +20,13 @@ package impl
import (
"context"
"fmt"
- "github.com/apache/incubator-devlake/core/models/domainlayer/devops"
"net/url"
"reflect"
"strings"
"time"
+ "github.com/apache/incubator-devlake/core/models/domainlayer/devops"
+
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/log"
@@ -57,8 +58,8 @@ func (p GithubGraphql) Scope() interface{} {
return &models.GithubRepo{}
}
-func (p GithubGraphql) TransformationRule() interface{} {
- return &models.GithubTransformationRule{}
+func (p GithubGraphql) ScopeConfig() interface{} {
+ return &models.GithubScopeConfig{}
}
func (p GithubGraphql) Description() string {
diff --git a/backend/plugins/github_graphql/tasks/issue_collector.go b/backend/plugins/github_graphql/tasks/issue_collector.go
index 468a6b0e3..535dc016e 100644
--- a/backend/plugins/github_graphql/tasks/issue_collector.go
+++ b/backend/plugins/github_graphql/tasks/issue_collector.go
@@ -86,7 +86,7 @@ var _ plugin.SubTaskEntryPoint = CollectIssue
func CollectIssue(taskCtx plugin.SubTaskContext) errors.Error {
db := taskCtx.GetDal()
data := taskCtx.GetData().(*githubTasks.GithubTaskData)
- config := data.Options.GithubTransformationRule
+ config := data.Options.ScopeConfig
issueRegexes, err := githubTasks.NewIssueRegexes(config)
if err != nil {
return nil
diff --git a/backend/plugins/github_graphql/tasks/pr_collector.go b/backend/plugins/github_graphql/tasks/pr_collector.go
index 825ff6ae9..cb24dabf4 100644
--- a/backend/plugins/github_graphql/tasks/pr_collector.go
+++ b/backend/plugins/github_graphql/tasks/pr_collector.go
@@ -131,7 +131,7 @@ var _ plugin.SubTaskEntryPoint = CollectPr
func CollectPr(taskCtx plugin.SubTaskContext) errors.Error {
data := taskCtx.GetData().(*tasks.GithubTaskData)
- config := data.Options.GithubTransformationRule
+ config := data.Options.ScopeConfig
var labelTypeRegex *regexp.Regexp
var labelComponentRegex *regexp.Regexp
var err errors.Error