You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by ly...@apache.org on 2024/09/06 08:37:43 UTC
(incubator-devlake) branch main updated: feat(github\gitlab): doesn't init api client when re-transformating (#7966)
This is an automated email from the ASF dual-hosted git repository.
lynwee pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new fde073acd feat(github\gitlab): doesn't init api client when re-transformating (#7966)
fde073acd is described below
commit fde073acd95a19e171133d1158f2e3b474b0d7dc
Author: Lynwee <15...@qq.com>
AuthorDate: Fri Sep 6 16:37:39 2024 +0800
feat(github\gitlab): doesn't init api client when re-transformating (#7966)
* feat(github): doesn't init api client when re-transformating
* fix(framework): fix wrong sub task number
* fix: weekly bug retro panels support time range selected (#7972)
* fix: weekly bug retro panels support time range selected
* fix: lint
* Revert "add cloud api proxy (#7948)" (#7975)
This reverts commit 566f61bcebd262c1ddd8aef304da32f268c64da8.
* fix: file level metrics dashboard (#7974)
Co-authored-by: Startrekzky <ka...@merico.dev>
* fix: work logs no code data (#7977)
* fix: work logs no code data
* fix: oss migration logs
* fix: revert worklogs code (#7984)
* chore(mod): update github.com/gin-contrib/cors
* chore(golang): keep version 1.20
* feat(framework): add skipCollector parameter
* fix(github): remove debug codes
* chore(test): fix errors
---------
Co-authored-by: abeizn <zi...@merico.dev>
Co-authored-by: 青湛 <0x...@gmail.com>
Co-authored-by: Louis.z <lo...@gmail.com>
Co-authored-by: Startrekzky <ka...@merico.dev>
---
backend/core/plugin/plugin_blueprint.go | 1 +
backend/core/runner/run_task.go | 3 +-
backend/go.mod | 4 +-
backend/go.sum | 19 ++-
.../plugins/azuredevops_go/api/blueprint_v200.go | 1 +
backend/plugins/azuredevops_go/impl/impl.go | 3 +-
backend/plugins/bamboo/api/blueprint_v200.go | 1 +
backend/plugins/bamboo/impl/impl.go | 3 +-
backend/plugins/bitbucket/api/blueprint_v200.go | 1 +
backend/plugins/bitbucket/impl/impl.go | 6 +-
.../plugins/bitbucket_server/api/blueprint_v200.go | 1 +
backend/plugins/bitbucket_server/impl/impl.go | 6 +-
backend/plugins/circleci/api/blueprint_v200.go | 1 +
backend/plugins/circleci/impl/impl.go | 3 +-
backend/plugins/github/api/blueprint_v200.go | 19 +--
backend/plugins/github/impl/impl.go | 55 +++++---
backend/plugins/github_graphql/impl/impl.go | 147 +++++++++++----------
backend/plugins/gitlab/api/blueprint_v200.go | 1 +
backend/plugins/gitlab/impl/impl.go | 3 +-
backend/plugins/jenkins/api/blueprint_v200.go | 1 +
backend/plugins/jenkins/impl/impl.go | 3 +-
backend/plugins/jira/api/blueprint_v200.go | 1 +
backend/plugins/jira/impl/impl.go | 3 +-
backend/plugins/opsgenie/api/blueprint_v200.go | 1 +
backend/plugins/opsgenie/impl/impl.go | 3 +-
backend/plugins/pagerduty/api/blueprint_v200.go | 1 +
backend/plugins/pagerduty/impl/impl.go | 3 +-
backend/plugins/sonarqube/api/blueprint_v200.go | 1 +
backend/plugins/sonarqube/impl/impl.go | 3 +-
backend/plugins/tapd/api/blueprint_v200.go | 1 +
backend/plugins/tapd/impl/impl.go | 3 +-
backend/plugins/teambition/api/blueprint_v200.go | 1 +
backend/plugins/teambition/impl/impl.go | 3 +-
backend/plugins/trello/api/blueprint_v200.go | 1 +
backend/plugins/trello/impl/impl.go | 3 +-
backend/plugins/webhook/impl/impl.go | 1 +
backend/plugins/zentao/api/blueprint_v200.go | 1 +
backend/plugins/zentao/impl/impl.go | 3 +-
backend/server/services/blueprint_makeplan_v200.go | 11 +-
.../services/blueprint_makeplan_v200_test.go | 2 +-
.../services/remote/plugin/plugin_extensions.go | 1 +
41 files changed, 199 insertions(+), 130 deletions(-)
diff --git a/backend/core/plugin/plugin_blueprint.go b/backend/core/plugin/plugin_blueprint.go
index 492e54d43..b722bb9e2 100644
--- a/backend/core/plugin/plugin_blueprint.go
+++ b/backend/core/plugin/plugin_blueprint.go
@@ -72,6 +72,7 @@ type DataSourcePluginBlueprintV200 interface {
MakeDataSourcePipelinePlanV200(
connectionId uint64,
scopes []*models.BlueprintScope,
+ skipCollectors bool,
) (models.PipelinePlan, []Scope, errors.Error)
}
diff --git a/backend/core/runner/run_task.go b/backend/core/runner/run_task.go
index 02d92c34c..6abc7a760 100644
--- a/backend/core/runner/run_task.go
+++ b/backend/core/runner/run_task.go
@@ -244,11 +244,11 @@ func RunPluginSubTasks(
defer closeablePlugin.Close(taskCtx)
}
options := task.Options
+ taskCtx.SetSyncPolicy(syncPolicy)
taskData, err := pluginTask.PrepareTaskData(taskCtx, options)
if err != nil {
return errors.Default.Wrap(err, fmt.Sprintf("error preparing task data for %s", task.Plugin))
}
- taskCtx.SetSyncPolicy(syncPolicy)
taskCtx.SetData(taskData)
// record subtasks sequence to DB
@@ -303,6 +303,7 @@ func RunPluginSubTasks(
// subtask was disabled
continue
}
+ subtaskNumber++
// run subtask
if progress != nil {
progress <- plugin.RunningProgress{
diff --git a/backend/go.mod b/backend/go.mod
index 819fcfb6d..6a438214b 100644
--- a/backend/go.mod
+++ b/backend/go.mod
@@ -67,7 +67,7 @@ require (
github.com/go-openapi/swag v0.22.3 // indirect
github.com/go-playground/locales v0.14.1 // indirect
github.com/go-playground/universal-translator v0.18.1 // indirect
- github.com/goccy/go-json v0.10.2 // indirect
+ github.com/goccy/go-json v0.10.3 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.2 // indirect
@@ -108,7 +108,7 @@ require (
github.com/xanzy/ssh-agent v0.3.3 // indirect
golang.org/x/arch v0.7.0 // indirect
golang.org/x/net v0.28.0 // indirect
- golang.org/x/sys v0.23.0 // indirect
+ golang.org/x/sys v0.24.0 // indirect
golang.org/x/text v0.17.0 // indirect
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect
google.golang.org/appengine v1.6.7 // indirect
diff --git a/backend/go.sum b/backend/go.sum
index 70c2daa08..dc22ad75a 100644
--- a/backend/go.sum
+++ b/backend/go.sum
@@ -49,11 +49,13 @@ github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5
github.com/ProtonMail/go-crypto v1.0.0 h1:LRuvITjQWX+WIfr930YHG2HNfjR1uOfyf5vE0kC2U78=
github.com/ProtonMail/go-crypto v1.0.0/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0=
github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8=
+github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4=
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
+github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM=
github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0=
@@ -116,6 +118,7 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.m
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY=
+github.com/frankban/quicktest v1.14.4/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI=
github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU=
@@ -127,11 +130,13 @@ github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeME
github.com/gin-contrib/cors v1.6.0 h1:0Z7D/bVhE6ja07lI8CTjTonp6SB07o8bNuFyRbsBUQg=
github.com/gin-contrib/cors v1.6.0/go.mod h1:cI+h6iOAyxKRtUtC6iF/Si1KSFvGm/gK+kshxlCi8ro=
github.com/gin-contrib/gzip v0.0.6 h1:NjcunTcGAj5CO1gn4N8jHOSIeRFHIbn51z6K+xaN4d4=
+github.com/gin-contrib/gzip v0.0.6/go.mod h1:QOJlmV2xmayAjkNS2Y8NQsMneuRShOU/kjovCXNuzzk=
github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg=
github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU=
github.com/gliderlabs/ssh v0.3.7 h1:iV3Bqi942d9huXnzEF2Mt+CY9gLu8DNM4Obd+8bODRE=
+github.com/gliderlabs/ssh v0.3.7/go.mod h1:zpHEXBstFnQYtGnB8k8kQLol82umzn/2/snG7alWVD8=
github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA=
github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og=
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI=
@@ -139,6 +144,7 @@ github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmS
github.com/go-git/go-billy/v5 v5.5.0 h1:yEY4yhzCDuMGSv83oGxiBotRzhwhNr8VZyphhiu+mTU=
github.com/go-git/go-billy/v5 v5.5.0/go.mod h1:hmexnoNsr2SJU1Ju67OaNz5ASJY3+sHgFRpCtpDCKow=
github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMje31YglSBqCdIqdhKBW8lokaMrL3uTkpGYlE2OOT4=
+github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII=
github.com/go-git/go-git/v5 v5.12.0 h1:7Md+ndsjrzZxbddRDZjF14qK+NN56sy6wkqaVrjZtys=
github.com/go-git/go-git/v5 v5.12.0/go.mod h1:FTM9VKtnI2m65hNI/TenDDDnUf2Q9FHnXYjuz9i5OEY=
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
@@ -158,6 +164,7 @@ github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/
github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g=
github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14=
github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s=
+github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA=
github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY=
github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY=
@@ -171,8 +178,8 @@ github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/gocarina/gocsv v0.0.0-20220707092902-b9da1f06c77e h1:GMIV+S6grz+vlIaUsP+fedQ6L+FovyMPMY26WO8dwQE=
github.com/gocarina/gocsv v0.0.0-20220707092902-b9da1f06c77e/go.mod h1:5YoVOkjYAQumqlV356Hj3xeYh4BdZuLE0/nRkf2NKkI=
-github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
-github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
+github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA=
+github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
@@ -227,6 +234,7 @@ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
@@ -414,6 +422,7 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI=
+github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M=
github.com/panjf2000/ants/v2 v2.4.6 h1:drmj9mcygn2gawZ155dRbo+NfXEfAssjZNU1qoIb4gQ=
github.com/panjf2000/ants/v2 v2.4.6/go.mod h1:f6F0NZVFsGCp5A7QW/Zj/m92atWwOkY0OIhFxRNFr4A=
github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
@@ -422,6 +431,7 @@ github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCko
github.com/pelletier/go-toml/v2 v2.1.1 h1:LWAJwfNvjQZCFIDKWYQaM62NcYeYViCmWIwmOStowAI=
github.com/pelletier/go-toml/v2 v2.1.1/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc=
github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4=
+github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
github.com/pjbgf/sha1cd v0.3.0 h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4=
github.com/pjbgf/sha1cd v0.3.0/go.mod h1:nZ1rrWOcGJ5uZgEEVL1VUM9iRQiZvWdbZjkKyFzPPsI=
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
@@ -741,8 +751,8 @@ golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.23.0 h1:YfKFowiIMvtgl1UERQoTPPToxltDeZfbj4H7dVUCwmM=
-golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg=
+golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
@@ -750,6 +760,7 @@ golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
golang.org/x/term v0.23.0 h1:F6D4vR+EHoL9/sWAWgAR1H2DcHr4PareCbAaCo1RpuU=
+golang.org/x/term v0.23.0/go.mod h1:DgV24QBUrK6jhZXl+20l6UWznPlwAHm1Q1mGHtydmSk=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
diff --git a/backend/plugins/azuredevops_go/api/blueprint_v200.go b/backend/plugins/azuredevops_go/api/blueprint_v200.go
index 698da41a3..febe5b777 100644
--- a/backend/plugins/azuredevops_go/api/blueprint_v200.go
+++ b/backend/plugins/azuredevops_go/api/blueprint_v200.go
@@ -38,6 +38,7 @@ func MakePipelinePlanV200(
subtaskMetas []plugin.SubTaskMeta,
connectionId uint64,
bpScopes []*coreModels.BlueprintScope,
+ skipCollectors bool,
) (coreModels.PipelinePlan, []plugin.Scope, errors.Error) {
// load connection, scope and scopeConfig from the db
connection, err := dsHelper.ConnSrv.FindByPk(connectionId)
diff --git a/backend/plugins/azuredevops_go/impl/impl.go b/backend/plugins/azuredevops_go/impl/impl.go
index 44cc01e52..b86349065 100644
--- a/backend/plugins/azuredevops_go/impl/impl.go
+++ b/backend/plugins/azuredevops_go/impl/impl.go
@@ -253,8 +253,9 @@ func (p Azuredevops) ApiResources() map[string]map[string]plugin.ApiResourceHand
func (p Azuredevops) MakeDataSourcePipelinePlanV200(
connectionId uint64,
scopes []*coreModels.BlueprintScope,
+ skipCollectors bool,
) (pp coreModels.PipelinePlan, sc []plugin.Scope, err errors.Error) {
- return api.MakePipelinePlanV200(p.SubTaskMetas(), connectionId, scopes)
+ return api.MakePipelinePlanV200(p.SubTaskMetas(), connectionId, scopes, skipCollectors)
}
func (p Azuredevops) Close(taskCtx plugin.TaskContext) errors.Error {
diff --git a/backend/plugins/bamboo/api/blueprint_v200.go b/backend/plugins/bamboo/api/blueprint_v200.go
index dd6ce9135..355bf38d7 100644
--- a/backend/plugins/bamboo/api/blueprint_v200.go
+++ b/backend/plugins/bamboo/api/blueprint_v200.go
@@ -35,6 +35,7 @@ func MakePipelinePlanV200(
subtaskMetas []plugin.SubTaskMeta,
connectionId uint64,
bpScopes []*coreModels.BlueprintScope,
+ skipCollectors bool,
) (coreModels.PipelinePlan, []plugin.Scope, errors.Error) {
// load connection, scope and scopeConfig from the db
connection, err := dsHelper.ConnSrv.FindByPk(connectionId)
diff --git a/backend/plugins/bamboo/impl/impl.go b/backend/plugins/bamboo/impl/impl.go
index 96999a742..f3ffc2650 100644
--- a/backend/plugins/bamboo/impl/impl.go
+++ b/backend/plugins/bamboo/impl/impl.go
@@ -67,8 +67,9 @@ func (p Bamboo) ScopeConfig() dal.Tabler {
func (p Bamboo) MakeDataSourcePipelinePlanV200(
connectionId uint64,
scopes []*coreModels.BlueprintScope,
+ skipCollectors bool,
) (coreModels.PipelinePlan, []plugin.Scope, errors.Error) {
- return api.MakePipelinePlanV200(p.SubTaskMetas(), connectionId, scopes)
+ return api.MakePipelinePlanV200(p.SubTaskMetas(), connectionId, scopes, skipCollectors)
}
func (p Bamboo) GetTablesInfo() []dal.Tabler {
diff --git a/backend/plugins/bitbucket/api/blueprint_v200.go b/backend/plugins/bitbucket/api/blueprint_v200.go
index 5c0509bec..e9ce9f333 100644
--- a/backend/plugins/bitbucket/api/blueprint_v200.go
+++ b/backend/plugins/bitbucket/api/blueprint_v200.go
@@ -40,6 +40,7 @@ func MakeDataSourcePipelinePlanV200(
subtaskMetas []plugin.SubTaskMeta,
connectionId uint64,
bpScopes []*coreModels.BlueprintScope,
+ skipCollectors bool,
) (coreModels.PipelinePlan, []plugin.Scope, errors.Error) {
// load connection, scope and scopeConfig from the db
connection, err := dsHelper.ConnSrv.FindByPk(connectionId)
diff --git a/backend/plugins/bitbucket/impl/impl.go b/backend/plugins/bitbucket/impl/impl.go
index 3fd1ec6f8..90ed91712 100644
--- a/backend/plugins/bitbucket/impl/impl.go
+++ b/backend/plugins/bitbucket/impl/impl.go
@@ -189,8 +189,10 @@ func (p Bitbucket) MigrationScripts() []plugin.MigrationScript {
func (p Bitbucket) MakeDataSourcePipelinePlanV200(
connectionId uint64,
- scopes []*coreModels.BlueprintScope) (pp coreModels.PipelinePlan, sc []plugin.Scope, err errors.Error) {
- return api.MakeDataSourcePipelinePlanV200(p.SubTaskMetas(), connectionId, scopes)
+ scopes []*coreModels.BlueprintScope,
+ skipCollectors bool,
+) (pp coreModels.PipelinePlan, sc []plugin.Scope, err errors.Error) {
+ return api.MakeDataSourcePipelinePlanV200(p.SubTaskMetas(), connectionId, scopes, skipCollectors)
}
func (p Bitbucket) ApiResources() map[string]map[string]plugin.ApiResourceHandler {
diff --git a/backend/plugins/bitbucket_server/api/blueprint_v200.go b/backend/plugins/bitbucket_server/api/blueprint_v200.go
index 8a3a73b37..21ce03ff6 100644
--- a/backend/plugins/bitbucket_server/api/blueprint_v200.go
+++ b/backend/plugins/bitbucket_server/api/blueprint_v200.go
@@ -40,6 +40,7 @@ func MakeDataSourcePipelinePlanV200(
subtaskMetas []plugin.SubTaskMeta,
connectionId uint64,
bpScopes []*coreModels.BlueprintScope,
+ skipCollectors bool,
) (coreModels.PipelinePlan, []plugin.Scope, errors.Error) {
// get the connection info for url
connection, err := dsHelper.ConnSrv.FindByPk(connectionId)
diff --git a/backend/plugins/bitbucket_server/impl/impl.go b/backend/plugins/bitbucket_server/impl/impl.go
index bc2366d18..9b041f4bd 100644
--- a/backend/plugins/bitbucket_server/impl/impl.go
+++ b/backend/plugins/bitbucket_server/impl/impl.go
@@ -152,8 +152,10 @@ func (p BitbucketServer) MigrationScripts() []plugin.MigrationScript {
func (p BitbucketServer) MakeDataSourcePipelinePlanV200(
connectionId uint64,
- scopes []*coreModels.BlueprintScope) (pp coreModels.PipelinePlan, sc []plugin.Scope, err errors.Error) {
- return api.MakeDataSourcePipelinePlanV200(p.SubTaskMetas(), connectionId, scopes)
+ scopes []*coreModels.BlueprintScope,
+ skipCollectors bool,
+) (pp coreModels.PipelinePlan, sc []plugin.Scope, err errors.Error) {
+ return api.MakeDataSourcePipelinePlanV200(p.SubTaskMetas(), connectionId, scopes, skipCollectors)
}
func (p BitbucketServer) ApiResources() map[string]map[string]plugin.ApiResourceHandler {
diff --git a/backend/plugins/circleci/api/blueprint_v200.go b/backend/plugins/circleci/api/blueprint_v200.go
index c2b9e6618..ddafa98ff 100644
--- a/backend/plugins/circleci/api/blueprint_v200.go
+++ b/backend/plugins/circleci/api/blueprint_v200.go
@@ -35,6 +35,7 @@ func MakeDataSourcePipelinePlanV200(
subtaskMetas []plugin.SubTaskMeta,
connectionId uint64,
bpScopes []*coreModels.BlueprintScope,
+ skipCollectors bool,
) (coreModels.PipelinePlan, []plugin.Scope, errors.Error) {
connection, err := dsHelper.ConnSrv.FindByPk(connectionId)
if err != nil {
diff --git a/backend/plugins/circleci/impl/impl.go b/backend/plugins/circleci/impl/impl.go
index 4610d887d..e3728aece 100644
--- a/backend/plugins/circleci/impl/impl.go
+++ b/backend/plugins/circleci/impl/impl.go
@@ -89,8 +89,9 @@ func (p Circleci) SubTaskMetas() []plugin.SubTaskMeta {
func (p Circleci) MakeDataSourcePipelinePlanV200(
connectionId uint64,
scopes []*coreModels.BlueprintScope,
+ skipCollectors bool,
) (pp coreModels.PipelinePlan, sc []plugin.Scope, err errors.Error) {
- return api.MakeDataSourcePipelinePlanV200(p.SubTaskMetas(), connectionId, scopes)
+ return api.MakeDataSourcePipelinePlanV200(p.SubTaskMetas(), connectionId, scopes, skipCollectors)
}
func (p Circleci) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]interface{}) (interface{}, errors.Error) {
diff --git a/backend/plugins/github/api/blueprint_v200.go b/backend/plugins/github/api/blueprint_v200.go
index 909323918..3016db8cb 100644
--- a/backend/plugins/github/api/blueprint_v200.go
+++ b/backend/plugins/github/api/blueprint_v200.go
@@ -45,22 +45,25 @@ func MakeDataSourcePipelinePlanV200(
subtaskMetas []plugin.SubTaskMeta,
connectionId uint64,
bpScopes []*coreModels.BlueprintScope,
+ skipCollectors bool,
) (coreModels.PipelinePlan, []plugin.Scope, errors.Error) {
// load connection, scope and scopeConfig from the db
connection, err := dsHelper.ConnSrv.FindByPk(connectionId)
if err != nil {
- return nil, nil, err
+ return nil, nil, errors.Default.Wrap(err, "find by pk")
}
scopeDetails, err := dsHelper.ScopeSrv.MapScopeDetails(connectionId, bpScopes)
if err != nil {
- return nil, nil, err
+ return nil, nil, errors.Default.Wrap(err, "map scope detail")
}
- // needed for the connection to populate its access tokens
- // if AppKey authentication method is selected
- _, err = helper.NewApiClientFromConnection(context.TODO(), basicRes, connection)
- if err != nil {
- return nil, nil, err
+ if !skipCollectors {
+ // needed for the connection to populate its access tokens
+ // if AppKey authentication method is selected
+ _, err = helper.NewApiClientFromConnection(context.TODO(), basicRes, connection)
+ if err != nil {
+ return nil, nil, errors.Default.Wrap(err, "create api client")
+ }
}
plan, err := makeDataSourcePipelinePlanV200(subtaskMetas, scopeDetails, connection)
@@ -69,7 +72,7 @@ func MakeDataSourcePipelinePlanV200(
}
scopes, err := makeScopesV200(scopeDetails, connection)
if err != nil {
- return nil, nil, err
+ return nil, nil, errors.Default.Wrap(err, "make scopes v200")
}
return plan, scopes, nil
diff --git a/backend/plugins/github/impl/impl.go b/backend/plugins/github/impl/impl.go
index f73f33120..07342b786 100644
--- a/backend/plugins/github/impl/impl.go
+++ b/backend/plugins/github/impl/impl.go
@@ -137,11 +137,22 @@ func (p Github) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]i
if err != nil {
return nil, errors.Default.Wrap(err, "unable to get github connection by the given connection ID")
}
- apiClient, err := tasks.CreateApiClient(taskCtx, connection)
- if err != nil {
- return nil, errors.Default.Wrap(err, "unable to get github API client instance")
+
+ taskData := &tasks.GithubTaskData{
+ Options: op,
+ }
+
+ syncPolicy := taskCtx.SyncPolicy()
+ var apiClient *helper.ApiClient
+ if !syncPolicy.SkipCollectors {
+ newApiClient, err := tasks.CreateApiClient(taskCtx, connection)
+ if err != nil {
+ return nil, errors.Default.Wrap(err, "unable to get github API client instance")
+ }
+ apiClient = newApiClient.ApiClient
+ taskData.ApiClient = newApiClient
}
- err = EnrichOptions(taskCtx, op, apiClient.ApiClient)
+ err = EnrichOptions(taskCtx, op, apiClient)
if err != nil {
return nil, err
}
@@ -156,12 +167,7 @@ func (p Github) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]i
if err = regexEnricher.TryAdd(devops.ENV_NAME_PATTERN, op.ScopeConfig.EnvNamePattern); err != nil {
return nil, errors.BadInput.Wrap(err, "invalid value for `envNamePattern`")
}
-
- taskData := &tasks.GithubTaskData{
- Options: op,
- ApiClient: apiClient,
- RegexEnricher: regexEnricher,
- }
+ taskData.RegexEnricher = regexEnricher
return taskData, nil
}
@@ -230,8 +236,9 @@ func (p Github) ApiResources() map[string]map[string]plugin.ApiResourceHandler {
func (p Github) MakeDataSourcePipelinePlanV200(
connectionId uint64,
scopes []*coreModels.BlueprintScope,
+ skipCollectors bool,
) (pp coreModels.PipelinePlan, sc []plugin.Scope, err errors.Error) {
- return api.MakeDataSourcePipelinePlanV200(p.SubTaskMetas(), connectionId, scopes)
+ return api.MakeDataSourcePipelinePlanV200(p.SubTaskMetas(), connectionId, scopes, skipCollectors)
}
func (p Github) Close(taskCtx plugin.TaskContext) errors.Error {
@@ -239,7 +246,9 @@ func (p Github) Close(taskCtx plugin.TaskContext) errors.Error {
if !ok {
return errors.Default.New(fmt.Sprintf("GetData failed when try to close %+v", taskCtx))
}
- data.ApiClient.Release()
+ if data.ApiClient != nil {
+ data.ApiClient.Release()
+ }
return nil
}
@@ -266,17 +275,19 @@ func EnrichOptions(taskCtx plugin.TaskContext,
} else {
if taskCtx.GetDal().IsErrorNotFound(err) && op.Name != "" {
var repo *tasks.GithubApiRepo
- repo, err = api.MemorizedGetApiRepo(repo, op, apiClient)
- if err != nil {
- return err
- }
- logger.Debug(fmt.Sprintf("Current repo: %s", repo.FullName))
- scope := convertApiRepoToScope(repo, op.ConnectionId)
- err = taskCtx.GetDal().CreateIfNotExist(scope)
- if err != nil {
- return err
+ if apiClient != nil {
+ repo, err = api.MemorizedGetApiRepo(repo, op, apiClient)
+ if err != nil {
+ return err
+ }
+ logger.Debug(fmt.Sprintf("Current repo: %s", repo.FullName))
+ scope := convertApiRepoToScope(repo, op.ConnectionId)
+ err = taskCtx.GetDal().CreateIfNotExist(scope)
+ if err != nil {
+ return err
+ }
+ op.GithubId = repo.GithubId
}
- op.GithubId = repo.GithubId
} else {
return errors.Default.Wrap(err, fmt.Sprintf("fail to find repo %s", op.Name))
}
diff --git a/backend/plugins/github_graphql/impl/impl.go b/backend/plugins/github_graphql/impl/impl.go
index a1d77d176..d4953a18f 100644
--- a/backend/plugins/github_graphql/impl/impl.go
+++ b/backend/plugins/github_graphql/impl/impl.go
@@ -20,14 +20,13 @@ package impl
import (
"context"
"fmt"
+ "github.com/apache/incubator-devlake/core/models/domainlayer/devops"
"net/http"
"net/url"
"reflect"
"strings"
"time"
- "github.com/apache/incubator-devlake/core/models/domainlayer/devops"
-
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/log"
@@ -170,78 +169,88 @@ func (p GithubGraphql) PrepareTaskData(taskCtx plugin.TaskContext, options map[s
return nil, errors.Default.Wrap(err, "unable to get github connection by the given connection ID: %v")
}
- apiClient, err := githubTasks.CreateApiClient(taskCtx, connection)
- if err != nil {
- return nil, errors.Default.Wrap(err, "unable to get github API client instance")
- }
-
- err = githubImpl.EnrichOptions(taskCtx, &op, apiClient.ApiClient)
- if err != nil {
- return nil, err
+ taskData := &githubTasks.GithubTaskData{
+ Options: &op,
}
- tokens := strings.Split(connection.Token, ",")
- src := oauth2.StaticTokenSource(
- &oauth2.Token{AccessToken: tokens[0]},
- )
- oauthContext := taskCtx.GetContext()
- proxy := connection.GetProxy()
- if proxy != "" {
- pu, err := url.Parse(proxy)
+ syncPolicy := taskCtx.SyncPolicy()
+ var enricherApiClient *helper.ApiClient
+ if !syncPolicy.SkipCollectors {
+ apiClient, err := githubTasks.CreateApiClient(taskCtx, connection)
if err != nil {
- return nil, errors.Convert(err)
+ return nil, errors.Default.Wrap(err, "unable to get github API client instance")
}
- if pu.Scheme == "http" || pu.Scheme == "socks5" {
- proxyClient := &http.Client{
- Transport: &http.Transport{Proxy: http.ProxyURL(pu)},
+ enricherApiClient = apiClient.ApiClient
+
+ tokens := strings.Split(connection.Token, ",")
+ src := oauth2.StaticTokenSource(
+ &oauth2.Token{AccessToken: tokens[0]},
+ )
+ oauthContext := taskCtx.GetContext()
+ proxy := connection.GetProxy()
+ if proxy != "" {
+ pu, err := url.Parse(proxy)
+ if err != nil {
+ return nil, errors.Convert(err)
+ }
+ if pu.Scheme == "http" || pu.Scheme == "socks5" {
+ proxyClient := &http.Client{
+ Transport: &http.Transport{Proxy: http.ProxyURL(pu)},
+ }
+ oauthContext = context.WithValue(
+ taskCtx.GetContext(),
+ oauth2.HTTPClient,
+ proxyClient,
+ )
+ logger.Debug("Proxy set in oauthContext to %s", proxy)
+ } else {
+ return nil, errors.BadInput.New("Unsupported scheme set in proxy")
}
- oauthContext = context.WithValue(
- taskCtx.GetContext(),
- oauth2.HTTPClient,
- proxyClient,
- )
- logger.Debug("Proxy set in oauthContext to %s", proxy)
- } else {
- return nil, errors.BadInput.New("Unsupported scheme set in proxy")
}
- }
- httpClient := oauth2.NewClient(oauthContext, src)
- endpoint, err := errors.Convert01(url.Parse(connection.Endpoint))
- if err != nil {
- return nil, errors.BadInput.Wrap(err, fmt.Sprintf("malformed connection endpoint supplied: %s", connection.Endpoint))
- }
+ httpClient := oauth2.NewClient(oauthContext, src)
+ endpoint, err := errors.Convert01(url.Parse(connection.Endpoint))
+ if err != nil {
+ return nil, errors.BadInput.Wrap(err, fmt.Sprintf("malformed connection endpoint supplied: %s", connection.Endpoint))
+ }
- // github.com and github enterprise have different graphql endpoints
- endpoint.Path = "/graphql" // see https://docs.github.com/en/graphql/guides/forming-calls-with-graphql
- if endpoint.Hostname() != "api.github.com" {
- // see https://docs.github.com/en/enterprise-server@3.11/graphql/guides/forming-calls-with-graphql
- endpoint.Path = "/api/graphql"
- }
- client := graphql.NewClient(endpoint.String(), httpClient)
- graphqlClient, err := helper.CreateAsyncGraphqlClient(taskCtx, client, taskCtx.GetLogger(),
- func(ctx context.Context, client *graphql.Client, logger log.Logger) (rateRemaining int, resetAt *time.Time, err errors.Error) {
- var query GraphQueryRateLimit
- dataErrors, err := errors.Convert01(client.Query(taskCtx.GetContext(), &query, nil))
- if err != nil {
- return 0, nil, err
- }
- if len(dataErrors) > 0 {
- return 0, nil, errors.Default.Wrap(dataErrors[0], `query rate limit fail`)
- }
- logger.Info(`github graphql init success with remaining %d/%d and will reset at %s`,
- query.RateLimit.Remaining, query.RateLimit.Limit, query.RateLimit.ResetAt)
- return int(query.RateLimit.Remaining), &query.RateLimit.ResetAt, nil
+ // github.com and github enterprise have different graphql endpoints
+ endpoint.Path = "/graphql" // see https://docs.github.com/en/graphql/guides/forming-calls-with-graphql
+ if endpoint.Hostname() != "api.github.com" {
+ // see https://docs.github.com/en/enterprise-server@3.11/graphql/guides/forming-calls-with-graphql
+ endpoint.Path = "/api/graphql"
+ }
+ client := graphql.NewClient(endpoint.String(), httpClient)
+ graphqlClient, err := helper.CreateAsyncGraphqlClient(taskCtx, client, taskCtx.GetLogger(),
+ func(ctx context.Context, client *graphql.Client, logger log.Logger) (rateRemaining int, resetAt *time.Time, err errors.Error) {
+ var query GraphQueryRateLimit
+ dataErrors, err := errors.Convert01(client.Query(taskCtx.GetContext(), &query, nil))
+ if err != nil {
+ return 0, nil, err
+ }
+ if len(dataErrors) > 0 {
+ return 0, nil, errors.Default.Wrap(dataErrors[0], `query rate limit fail`)
+ }
+ logger.Info(`github graphql init success with remaining %d/%d and will reset at %s`,
+ query.RateLimit.Remaining, query.RateLimit.Limit, query.RateLimit.ResetAt)
+ return int(query.RateLimit.Remaining), &query.RateLimit.ResetAt, nil
+ })
+ if err != nil {
+ return nil, err
+ }
+
+ graphqlClient.SetGetRateCost(func(q interface{}) int {
+ v := reflect.ValueOf(q)
+ return int(v.Elem().FieldByName(`RateLimit`).FieldByName(`Cost`).Int())
})
+ taskData.ApiClient = apiClient
+ taskData.GraphqlClient = graphqlClient
+ }
+ err = githubImpl.EnrichOptions(taskCtx, &op, enricherApiClient)
if err != nil {
return nil, err
}
- graphqlClient.SetGetRateCost(func(q interface{}) int {
- v := reflect.ValueOf(q)
- return int(v.Elem().FieldByName(`RateLimit`).FieldByName(`Cost`).Int())
- })
-
regexEnricher := helper.NewRegexEnricher()
if err = regexEnricher.TryAdd(devops.DEPLOYMENT, op.ScopeConfig.DeploymentPattern); err != nil {
return nil, errors.BadInput.Wrap(err, "invalid value for `deploymentPattern`")
@@ -252,13 +261,7 @@ func (p GithubGraphql) PrepareTaskData(taskCtx plugin.TaskContext, options map[s
if err = regexEnricher.TryAdd(devops.ENV_NAME_PATTERN, op.ScopeConfig.EnvNamePattern); err != nil {
return nil, errors.BadInput.Wrap(err, "invalid value for `envNamePattern`")
}
-
- taskData := &githubTasks.GithubTaskData{
- Options: &op,
- ApiClient: apiClient,
- GraphqlClient: graphqlClient,
- RegexEnricher: regexEnricher,
- }
+ taskData.RegexEnricher = regexEnricher
return taskData, nil
}
@@ -277,8 +280,12 @@ func (p GithubGraphql) Close(taskCtx plugin.TaskContext) errors.Error {
if !ok {
return errors.Default.New(fmt.Sprintf("GetData failed when try to close %+v", taskCtx))
}
- data.ApiClient.Release()
- data.GraphqlClient.Release()
+ if data.ApiClient != nil {
+ data.ApiClient.Release()
+ }
+ if data.GraphqlClient != nil {
+ data.GraphqlClient.Release()
+ }
return nil
}
diff --git a/backend/plugins/gitlab/api/blueprint_v200.go b/backend/plugins/gitlab/api/blueprint_v200.go
index 458d57bb3..e8d431841 100644
--- a/backend/plugins/gitlab/api/blueprint_v200.go
+++ b/backend/plugins/gitlab/api/blueprint_v200.go
@@ -44,6 +44,7 @@ func MakePipelinePlanV200(
subtaskMetas []plugin.SubTaskMeta,
connectionId uint64,
bpScopes []*coreModels.BlueprintScope,
+ skipCollectors bool,
) (coreModels.PipelinePlan, []plugin.Scope, errors.Error) {
// load connection, scope and scopeConfig from the db
connection, err := dsHelper.ConnSrv.FindByPk(connectionId)
diff --git a/backend/plugins/gitlab/impl/impl.go b/backend/plugins/gitlab/impl/impl.go
index 1d72bf052..3fdb86220 100644
--- a/backend/plugins/gitlab/impl/impl.go
+++ b/backend/plugins/gitlab/impl/impl.go
@@ -76,8 +76,9 @@ func (p Gitlab) ScopeConfig() dal.Tabler {
func (p Gitlab) MakeDataSourcePipelinePlanV200(
connectionId uint64,
scopes []*coreModels.BlueprintScope,
+ skipCollectors bool,
) (coreModels.PipelinePlan, []plugin.Scope, errors.Error) {
- return api.MakePipelinePlanV200(p.SubTaskMetas(), connectionId, scopes)
+ return api.MakePipelinePlanV200(p.SubTaskMetas(), connectionId, scopes, skipCollectors)
}
func (p Gitlab) GetTablesInfo() []dal.Tabler {
diff --git a/backend/plugins/jenkins/api/blueprint_v200.go b/backend/plugins/jenkins/api/blueprint_v200.go
index cd51141e2..cce0be37d 100644
--- a/backend/plugins/jenkins/api/blueprint_v200.go
+++ b/backend/plugins/jenkins/api/blueprint_v200.go
@@ -37,6 +37,7 @@ func MakeDataSourcePipelinePlanV200(
subtaskMetas []plugin.SubTaskMeta,
connectionId uint64,
bpScopes []*coreModels.BlueprintScope,
+ skipCollectors bool,
) (coreModels.PipelinePlan, []plugin.Scope, errors.Error) {
// load connection, scope and scopeConfig from the db
connection, err := dsHelper.ConnSrv.FindByPk(connectionId)
diff --git a/backend/plugins/jenkins/impl/impl.go b/backend/plugins/jenkins/impl/impl.go
index c33e50daa..0b6ff4b8c 100644
--- a/backend/plugins/jenkins/impl/impl.go
+++ b/backend/plugins/jenkins/impl/impl.go
@@ -166,8 +166,9 @@ func (p Jenkins) MigrationScripts() []plugin.MigrationScript {
func (p Jenkins) MakeDataSourcePipelinePlanV200(
connectionId uint64,
scopes []*coreModels.BlueprintScope,
+ skipCollectors bool,
) (pp coreModels.PipelinePlan, sc []plugin.Scope, err errors.Error) {
- return api.MakeDataSourcePipelinePlanV200(p.SubTaskMetas(), connectionId, scopes)
+ return api.MakeDataSourcePipelinePlanV200(p.SubTaskMetas(), connectionId, scopes, skipCollectors)
}
func (p Jenkins) ApiResources() map[string]map[string]plugin.ApiResourceHandler {
diff --git a/backend/plugins/jira/api/blueprint_v200.go b/backend/plugins/jira/api/blueprint_v200.go
index e3859286b..f60f5cf13 100644
--- a/backend/plugins/jira/api/blueprint_v200.go
+++ b/backend/plugins/jira/api/blueprint_v200.go
@@ -36,6 +36,7 @@ func MakeDataSourcePipelinePlanV200(
subtaskMetas []plugin.SubTaskMeta,
connectionId uint64,
bpScopes []*coreModels.BlueprintScope,
+ skipCollectors bool,
) (coreModels.PipelinePlan, []plugin.Scope, errors.Error) {
// load connection, scope and scopeConfig from the db
connection, err := dsHelper.ConnSrv.FindByPk(connectionId)
diff --git a/backend/plugins/jira/impl/impl.go b/backend/plugins/jira/impl/impl.go
index 126e513e2..f86c73ff3 100644
--- a/backend/plugins/jira/impl/impl.go
+++ b/backend/plugins/jira/impl/impl.go
@@ -257,8 +257,9 @@ func (p Jira) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]int
func (p Jira) MakeDataSourcePipelinePlanV200(
connectionId uint64,
scopes []*coreModels.BlueprintScope,
+ skipCollectors bool,
) (pp coreModels.PipelinePlan, sc []plugin.Scope, err errors.Error) {
- return api.MakeDataSourcePipelinePlanV200(p.SubTaskMetas(), connectionId, scopes)
+ return api.MakeDataSourcePipelinePlanV200(p.SubTaskMetas(), connectionId, scopes, skipCollectors)
}
func (p Jira) RootPkgPath() string {
diff --git a/backend/plugins/opsgenie/api/blueprint_v200.go b/backend/plugins/opsgenie/api/blueprint_v200.go
index 3b34e7fef..ddc5acd7b 100644
--- a/backend/plugins/opsgenie/api/blueprint_v200.go
+++ b/backend/plugins/opsgenie/api/blueprint_v200.go
@@ -34,6 +34,7 @@ func MakeDataSourcePipelinePlanV200(
subtaskMetas []plugin.SubTaskMeta,
connectionId uint64,
bpScopes []*coreModels.BlueprintScope,
+ skipCollectors bool,
) (coreModels.PipelinePlan, []plugin.Scope, errors.Error) {
connection, err := dsHelper.ConnSrv.FindByPk(connectionId)
if err != nil {
diff --git a/backend/plugins/opsgenie/impl/impl.go b/backend/plugins/opsgenie/impl/impl.go
index a3470da04..fc234e7aa 100644
--- a/backend/plugins/opsgenie/impl/impl.go
+++ b/backend/plugins/opsgenie/impl/impl.go
@@ -191,6 +191,7 @@ func (p Opsgenie) ApiResources() map[string]map[string]plugin.ApiResourceHandler
func (p Opsgenie) MakeDataSourcePipelinePlanV200(
connectionId uint64,
scopes []*coreModels.BlueprintScope,
+ skipCollectors bool,
) (coreModels.PipelinePlan, []plugin.Scope, errors.Error) {
- return api.MakeDataSourcePipelinePlanV200(p.SubTaskMetas(), connectionId, scopes)
+ return api.MakeDataSourcePipelinePlanV200(p.SubTaskMetas(), connectionId, scopes, skipCollectors)
}
diff --git a/backend/plugins/pagerduty/api/blueprint_v200.go b/backend/plugins/pagerduty/api/blueprint_v200.go
index 2e088b4fd..de5f0adcb 100644
--- a/backend/plugins/pagerduty/api/blueprint_v200.go
+++ b/backend/plugins/pagerduty/api/blueprint_v200.go
@@ -34,6 +34,7 @@ func MakeDataSourcePipelinePlanV200(
subtaskMetas []plugin.SubTaskMeta,
connectionId uint64,
bpScopes []*coreModels.BlueprintScope,
+ skipCollectors bool,
) (coreModels.PipelinePlan, []plugin.Scope, errors.Error) {
connection, err := dsHelper.ConnSrv.FindByPk(connectionId)
if err != nil {
diff --git a/backend/plugins/pagerduty/impl/impl.go b/backend/plugins/pagerduty/impl/impl.go
index 417156b72..8ed6b21b9 100644
--- a/backend/plugins/pagerduty/impl/impl.go
+++ b/backend/plugins/pagerduty/impl/impl.go
@@ -174,8 +174,9 @@ func (p PagerDuty) ApiResources() map[string]map[string]plugin.ApiResourceHandle
func (p PagerDuty) MakeDataSourcePipelinePlanV200(
connectionId uint64,
scopes []*coreModels.BlueprintScope,
+ skipCollectors bool,
) (coreModels.PipelinePlan, []plugin.Scope, errors.Error) {
- return api.MakeDataSourcePipelinePlanV200(p.SubTaskMetas(), connectionId, scopes)
+ return api.MakeDataSourcePipelinePlanV200(p.SubTaskMetas(), connectionId, scopes, skipCollectors)
}
func (p PagerDuty) Close(taskCtx plugin.TaskContext) errors.Error {
diff --git a/backend/plugins/sonarqube/api/blueprint_v200.go b/backend/plugins/sonarqube/api/blueprint_v200.go
index e4804534b..e2995aa65 100644
--- a/backend/plugins/sonarqube/api/blueprint_v200.go
+++ b/backend/plugins/sonarqube/api/blueprint_v200.go
@@ -39,6 +39,7 @@ func MakeDataSourcePipelinePlanV200(
subtaskMetas []plugin.SubTaskMeta,
connectionId uint64,
bpScopes []*coreModels.BlueprintScope,
+ skipCollectors bool,
) (coreModels.PipelinePlan, []plugin.Scope, errors.Error) {
// load connection, scope and scopeConfig from the db
connection, err := dsHelper.ConnSrv.FindByPk(connectionId)
diff --git a/backend/plugins/sonarqube/impl/impl.go b/backend/plugins/sonarqube/impl/impl.go
index 9fbef9213..a6c8dbbab 100644
--- a/backend/plugins/sonarqube/impl/impl.go
+++ b/backend/plugins/sonarqube/impl/impl.go
@@ -207,8 +207,9 @@ func (p Sonarqube) ApiResources() map[string]map[string]plugin.ApiResourceHandle
func (p Sonarqube) MakeDataSourcePipelinePlanV200(
connectionId uint64,
scopes []*coreModels.BlueprintScope,
+ skipCollectors bool,
) (pp coreModels.PipelinePlan, sc []plugin.Scope, err errors.Error) {
- return api.MakeDataSourcePipelinePlanV200(p.SubTaskMetas(), connectionId, scopes)
+ return api.MakeDataSourcePipelinePlanV200(p.SubTaskMetas(), connectionId, scopes, skipCollectors)
}
func (p Sonarqube) Close(taskCtx plugin.TaskContext) errors.Error {
diff --git a/backend/plugins/tapd/api/blueprint_v200.go b/backend/plugins/tapd/api/blueprint_v200.go
index 676ed30cf..c9304721c 100644
--- a/backend/plugins/tapd/api/blueprint_v200.go
+++ b/backend/plugins/tapd/api/blueprint_v200.go
@@ -34,6 +34,7 @@ func MakeDataSourcePipelinePlanV200(
subtaskMetas []plugin.SubTaskMeta,
connectionId uint64,
bpScopes []*coreModels.BlueprintScope,
+ skipCollectors bool,
) (coreModels.PipelinePlan, []plugin.Scope, errors.Error) {
connection, err := dsHelper.ConnSrv.FindByPk(connectionId)
if err != nil {
diff --git a/backend/plugins/tapd/impl/impl.go b/backend/plugins/tapd/impl/impl.go
index 9f3e932ea..231e2fd4a 100644
--- a/backend/plugins/tapd/impl/impl.go
+++ b/backend/plugins/tapd/impl/impl.go
@@ -247,8 +247,9 @@ func (p Tapd) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]int
func (p Tapd) MakeDataSourcePipelinePlanV200(
connectionId uint64,
scopes []*coreModels.BlueprintScope,
+ skipCollectors bool,
) (pp coreModels.PipelinePlan, sc []plugin.Scope, err errors.Error) {
- return api.MakeDataSourcePipelinePlanV200(p.SubTaskMetas(), connectionId, scopes)
+ return api.MakeDataSourcePipelinePlanV200(p.SubTaskMetas(), connectionId, scopes, skipCollectors)
}
func (p Tapd) RootPkgPath() string {
diff --git a/backend/plugins/teambition/api/blueprint_v200.go b/backend/plugins/teambition/api/blueprint_v200.go
index c71d5b644..f9704a54e 100644
--- a/backend/plugins/teambition/api/blueprint_v200.go
+++ b/backend/plugins/teambition/api/blueprint_v200.go
@@ -34,6 +34,7 @@ func MakeDataSourcePipelinePlanV200(
subtaskMetas []plugin.SubTaskMeta,
connectionId uint64,
bpScopes []*coreModels.BlueprintScope,
+ skipCollectors bool,
) (coreModels.PipelinePlan, []plugin.Scope, errors.Error) {
connection, err := dsHelper.ConnSrv.FindByPk(connectionId)
if err != nil {
diff --git a/backend/plugins/teambition/impl/impl.go b/backend/plugins/teambition/impl/impl.go
index 2140dfe6d..b203d3765 100644
--- a/backend/plugins/teambition/impl/impl.go
+++ b/backend/plugins/teambition/impl/impl.go
@@ -121,8 +121,9 @@ func (p Teambition) SubTaskMetas() []plugin.SubTaskMeta {
func (p Teambition) MakeDataSourcePipelinePlanV200(
connectionId uint64,
scopes []*coreModels.BlueprintScope,
+ skipCollectors bool,
) (pp coreModels.PipelinePlan, sc []plugin.Scope, err errors.Error) {
- return api.MakeDataSourcePipelinePlanV200(p.SubTaskMetas(), connectionId, scopes)
+ return api.MakeDataSourcePipelinePlanV200(p.SubTaskMetas(), connectionId, scopes, skipCollectors)
}
func (p Teambition) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]interface{}) (interface{}, errors.Error) {
diff --git a/backend/plugins/trello/api/blueprint_v200.go b/backend/plugins/trello/api/blueprint_v200.go
index 18fd5132a..72ac25e4e 100644
--- a/backend/plugins/trello/api/blueprint_v200.go
+++ b/backend/plugins/trello/api/blueprint_v200.go
@@ -37,6 +37,7 @@ func MakePipelinePlanV200(
subtaskMetas []plugin.SubTaskMeta,
connectionId uint64,
bpScopes []*coreModels.BlueprintScope,
+ skipCollectors bool,
) (coreModels.PipelinePlan, []plugin.Scope, errors.Error) {
connection, err := dsHelper.ConnSrv.FindByPk(connectionId)
if err != nil {
diff --git a/backend/plugins/trello/impl/impl.go b/backend/plugins/trello/impl/impl.go
index f6b3ee457..96d166a98 100644
--- a/backend/plugins/trello/impl/impl.go
+++ b/backend/plugins/trello/impl/impl.go
@@ -198,8 +198,9 @@ func (p Trello) ApiResources() map[string]map[string]plugin.ApiResourceHandler {
func (p Trello) MakeDataSourcePipelinePlanV200(
connectionId uint64,
scopes []*coreModels.BlueprintScope,
+ skipCollectors bool,
) (coreModels.PipelinePlan, []plugin.Scope, errors.Error) {
- return api.MakePipelinePlanV200(p.SubTaskMetas(), connectionId, scopes)
+ return api.MakePipelinePlanV200(p.SubTaskMetas(), connectionId, scopes, skipCollectors)
}
func (p Trello) Close(taskCtx plugin.TaskContext) errors.Error {
diff --git a/backend/plugins/webhook/impl/impl.go b/backend/plugins/webhook/impl/impl.go
index 6b1bac0af..ada304783 100644
--- a/backend/plugins/webhook/impl/impl.go
+++ b/backend/plugins/webhook/impl/impl.go
@@ -63,6 +63,7 @@ func (p Webhook) GetTablesInfo() []dal.Tabler {
func (p Webhook) MakeDataSourcePipelinePlanV200(
connectionId uint64,
_ []*coreModels.BlueprintScope,
+ skipCollectors bool,
) (pp coreModels.PipelinePlan, sc []plugin.Scope, err errors.Error) {
return api.MakeDataSourcePipelinePlanV200(connectionId)
}
diff --git a/backend/plugins/zentao/api/blueprint_v200.go b/backend/plugins/zentao/api/blueprint_v200.go
index 1da9355d4..83005094c 100644
--- a/backend/plugins/zentao/api/blueprint_v200.go
+++ b/backend/plugins/zentao/api/blueprint_v200.go
@@ -34,6 +34,7 @@ func MakeDataSourcePipelinePlanV200(
subtaskMetas []plugin.SubTaskMeta,
connectionId uint64,
bpScopes []*coreModels.BlueprintScope,
+ skipCollectors bool,
) (coreModels.PipelinePlan, []plugin.Scope, errors.Error) {
connection, err := dsHelper.ConnSrv.FindByPk(connectionId)
if err != nil {
diff --git a/backend/plugins/zentao/impl/impl.go b/backend/plugins/zentao/impl/impl.go
index e46f8a107..fc674112e 100644
--- a/backend/plugins/zentao/impl/impl.go
+++ b/backend/plugins/zentao/impl/impl.go
@@ -295,8 +295,9 @@ func (p Zentao) ApiResources() map[string]map[string]plugin.ApiResourceHandler {
func (p Zentao) MakeDataSourcePipelinePlanV200(
connectionId uint64,
scopes []*coreModels.BlueprintScope,
+ skipCollectors bool,
) (pp coreModels.PipelinePlan, sc []plugin.Scope, err errors.Error) {
- return api.MakeDataSourcePipelinePlanV200(p.SubTaskMetas(), connectionId, scopes)
+ return api.MakeDataSourcePipelinePlanV200(p.SubTaskMetas(), connectionId, scopes, skipCollectors)
}
func (p Zentao) Close(taskCtx plugin.TaskContext) errors.Error {
diff --git a/backend/server/services/blueprint_makeplan_v200.go b/backend/server/services/blueprint_makeplan_v200.go
index bd89d7de3..727e843de 100644
--- a/backend/server/services/blueprint_makeplan_v200.go
+++ b/backend/server/services/blueprint_makeplan_v200.go
@@ -56,9 +56,10 @@ func GeneratePlanJsonV200(
sourcePlans[i], pluginScopes, err = pluginBp.MakeDataSourcePipelinePlanV200(
connection.ConnectionId,
connection.Scopes,
+ skipCollectors,
)
if err != nil {
- return nil, err
+ return nil, errors.Default.Wrap(err, "make data source pipeline plan v200")
}
// collect scopes for the project. a github repository may produce
// 2 scopes, 1 repo and 1 board
@@ -101,7 +102,7 @@ func GeneratePlanJsonV200(
for metricPluginName, metricPluginOptJson := range metrics {
p, err := plugin.GetPlugin(metricPluginName)
if err != nil {
- return nil, err
+ return nil, errors.Default.Wrap(err, "get plugin")
}
if pluginBp, ok := p.(plugin.MetricPluginBlueprintV200); ok {
// If we enable one metric plugin, even if it has nil option, we still process it
@@ -110,7 +111,7 @@ func GeneratePlanJsonV200(
}
metricPlans[i], err = pluginBp.MakeMetricPluginPipelinePlanV200(projectName, metricPluginOptJson)
if err != nil {
- return nil, err
+ return nil, errors.Default.Wrap(err, "make metric plugin pipeline plan v200")
}
i++
} else {
@@ -123,12 +124,12 @@ func GeneratePlanJsonV200(
if projectName != "" {
p, err := plugin.GetPlugin("org")
if err != nil {
- return nil, err
+ return nil, errors.Default.Wrap(err, "get plugin org")
}
if pluginBp, ok := p.(plugin.ProjectMapper); ok {
planForProjectMapping, err = pluginBp.MapProject(projectName, scopes)
if err != nil {
- return nil, err
+ return nil, errors.Default.Wrap(err, "make project")
}
}
}
diff --git a/backend/server/services/blueprint_makeplan_v200_test.go b/backend/server/services/blueprint_makeplan_v200_test.go
index 22b53b532..25b586596 100644
--- a/backend/server/services/blueprint_makeplan_v200_test.go
+++ b/backend/server/services/blueprint_makeplan_v200_test.go
@@ -55,7 +55,7 @@ func TestMakePlanV200(t *testing.T) {
&ticket.Board{DomainEntity: domainlayer.DomainEntity{Id: "github:GithubRepo:1:123"}, Name: "apache/incubator-devlake"},
}
github := new(mockplugin.CompositeDataSourcePluginBlueprintV200)
- github.On("MakeDataSourcePipelinePlanV200", githubConnId, githubScopes).Return(githubOutputPlan, githubOutputScopes, nil)
+ github.On("MakeDataSourcePipelinePlanV200", githubConnId, githubScopes, false).Return(githubOutputPlan, githubOutputScopes, nil)
// mock dora plugin as a metric plugin
doraName := "TestMakePlanV200-dora"
diff --git a/backend/server/services/remote/plugin/plugin_extensions.go b/backend/server/services/remote/plugin/plugin_extensions.go
index 5f1f4612c..13abba195 100644
--- a/backend/server/services/remote/plugin/plugin_extensions.go
+++ b/backend/server/services/remote/plugin/plugin_extensions.go
@@ -43,6 +43,7 @@ func (p remoteMetricPlugin) MakeMetricPluginPipelinePlanV200(projectName string,
func (p remoteDatasourcePlugin) MakeDataSourcePipelinePlanV200(
connectionId uint64,
bpScopes []*coreModels.BlueprintScope,
+ skipCollectors bool,
) (coreModels.PipelinePlan, []plugin.Scope, errors.Error) {
connection := p.connectionTabler.New()
err := p.connHelper.FirstById(connection, connectionId)