You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by mi...@apache.org on 2024/09/26 09:36:33 UTC
(incubator-devlake) branch feat-8017 updated (d7b1760cb -> 74e0baf54)
This is an automated email from the ASF dual-hosted git repository.
mintsweet pushed a change to branch feat-8017
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
omit d7b1760cb fix: adjust the api for connections token check
omit 66c7cd2b4 Merge branch 'main' into feat-8017
omit 13011ea45 feat(blueprint): add new api to check connection tokens (#8102)
omit d52d4c72b feat: add connection token check before collect data
omit 31d5bc9f5 Merge branch 'dev-1' into feat-8017
omit 564b163c8 fix(project): update check project api doc
omit ed483e750 Dev 1 (#8101)
omit 2e8a20fae fix(project): update project check api doc
omit 8fd313448 fix(project): fix api doc
omit 7358588ec Dev 1 (#8100)
omit c446aa372 Merge remote-tracking branch 'origin/feat-8017' into dev-1
omit 3c3f2f266 feat(project): update token check API response
omit fa6c764d9 feat(project): add token check result in project check API (#8099)
omit 86e2cbce1 feat(project): add token check result in project check API
omit 0570677f1 Merge branch 'main' into feat-8017
omit 49caa2af1 feat: check pipelines' tokens before executing/creating pipeline (#8055)
add f3b134bee Add missing URL (#8097)
add 323b98e66 feat: improve performance of the listing-scope-config-projects-and-scopes (#8104)
add fb0dd2e87 feat: dora config (#8103)
new 6e45e9d7d feat: check pipelines' tokens before executing/creating pipeline (#8055)
new 168809cbc feat(project): add token check result in project check API (#8099)
new 031db5b2f Dev 1 (#8100)
new 2c0b3f5cd Dev 1 (#8101)
new c8296a8c0 fix(project): update check project api doc
new f8b4efc4d feat: add connection token check before collect data
new 626db747c feat(blueprint): add new api to check connection tokens (#8102)
new 288be5004 fix: adjust the api for connections token check
new 74e0baf54 fix: data scope search cannot select
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (d7b1760cb)
\
N -- N -- N refs/heads/feat-8017 (74e0baf54)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 9 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../core/models/domainlayer/devops/cicd_task.go | 10 +
.../helpers/pluginhelper/api/enrich_with_regex.go | 53 +-
backend/helpers/pluginhelper/api/pagenation.go | 24 +
.../srvhelper/scope_config_service_helper.go | 102 ++--
backend/plugins/bamboo/api/connection_api.go | 127 +++++
backend/plugins/bamboo/e2e/deploy_build_test.go | 12 +-
backend/plugins/bamboo/e2e/job_build_test.go | 11 +-
backend/plugins/bamboo/e2e/job_test.go | 7 +-
.../plugins/bamboo/e2e/plan_build_commits_test.go | 8 +-
backend/plugins/bamboo/e2e/plan_build_test.go | 12 +-
.../e2e/snapshot_tables/cicd_deployments.csv | 28 +-
backend/plugins/bamboo/impl/impl.go | 33 +-
.../20240906_add_env_name_list_to_scope_config.go} | 24 +-
.../bamboo/models/migrationscripts/register.go | 1 +
backend/plugins/bamboo/models/scope_config.go | 5 +-
.../deploy_build_to_deployment_commit_convertor.go | 6 +-
.../tasks/deploy_build_to_deployment_convertor.go | 2 +-
.../plugins/bamboo/tasks/job_build_extractor.go | 6 +-
.../plugins/bamboo/tasks/plan_build_convertor.go | 2 +-
.../plugins/bamboo/tasks/plan_build_extractor.go | 9 +-
backend/plugins/bamboo/tasks/plan_convertor.go | 5 +-
backend/plugins/bamboo/tasks/shared.go | 2 +-
backend/plugins/bamboo/tasks/shared_test.go | 2 +-
backend/plugins/bitbucket/api/connection_api.go | 98 ++++
backend/plugins/bitbucket/e2e/deployments_test.go | 7 +-
.../plugins/bitbucket/e2e/pipeline_steps_test.go | 8 +-
backend/plugins/bitbucket/e2e/pipelines_test.go | 8 +-
backend/plugins/bitbucket/impl/impl.go | 15 +-
backend/plugins/bitbucket/models/scope_config.go | 4 +-
.../plugins/bitbucket/tasks/pipeline_extractor.go | 6 +-
.../bitbucket/tasks/pipeline_steps_extractor.go | 6 +-
backend/plugins/circleci/api/connection_api.go | 96 ++++
backend/plugins/circleci/e2e/job_test.go | 7 +-
backend/plugins/circleci/e2e/workflow_test.go | 7 +-
backend/plugins/circleci/impl/impl.go | 15 +-
backend/plugins/circleci/models/scope_config.go | 4 +-
backend/plugins/circleci/tasks/job_converter.go | 6 +-
.../plugins/circleci/tasks/workflow_converter.go | 6 +-
backend/plugins/github/api/connection_api.go | 123 +++++
backend/plugins/github/e2e/cicd_job_test.go | 6 +
backend/plugins/github/e2e/cicd_run_test.go | 6 +
backend/plugins/github/impl/impl.go | 29 +-
.../20240906_add_env_name_list_to_scope_config.go} | 24 +-
.../github/models/migrationscripts/register.go | 1 +
backend/plugins/github/models/scope_config.go | 5 +-
backend/plugins/github/tasks/cicd_job_extractor.go | 7 +-
backend/plugins/github/tasks/cicd_run_extractor.go | 6 +-
backend/plugins/github_graphql/impl/impl.go | 25 +-
.../github_graphql/tasks/deployment_convertor.go | 3 +-
.../plugins/github_graphql/tasks/job_extractor.go | 6 +-
backend/plugins/gitlab/api/connection_api.go | 124 +++++
backend/plugins/gitlab/e2e/job_test.go | 7 +-
backend/plugins/gitlab/impl/impl.go | 29 +-
.../20240906_add_env_name_list_to_scope_config.go} | 24 +-
.../gitlab/models/migrationscripts/register.go | 1 +
backend/plugins/gitlab/models/scope_config.go | 5 +-
.../plugins/gitlab/tasks/deployment_convertor.go | 2 +-
backend/plugins/gitlab/tasks/job_convertor.go | 7 +-
.../gitlab/tasks/pipeline_detail_extractor.go | 6 +-
backend/plugins/jenkins/api/connection_api.go | 96 ++++
backend/plugins/jenkins/e2e/builds_test.go | 2 +-
.../cicd_pipeline_commits_multibranch.csv | 26 +-
.../jenkins/e2e/snapshot_tables/cicd_pipelines.csv | 26 +-
.../snapshot_tables/cicd_pipelines_multibranch.csv | 32 +-
.../jenkins/e2e/snapshot_tables/cicd_tasks.csv | 24 +-
.../snapshot_tables/cicd_tasks_after_stages.csv | 4 +-
.../cicd_tasks_after_stages_no_prod_regex.csv | 28 +-
.../e2e/snapshot_tables/cicd_tasks_multibranch.csv | 6 +-
.../snapshot_tables/cicd_tasks_no_prod_regex.csv | 24 +-
backend/plugins/jenkins/e2e/stages_test.go | 1 +
backend/plugins/jenkins/impl/impl.go | 22 +-
backend/plugins/jenkins/models/build.go | 20 +-
.../20240911_add_url_for_builds.go} | 24 +-
.../jenkins/models/migrationscripts/register.go | 1 +
backend/plugins/jenkins/models/response.go | 1 +
backend/plugins/jenkins/models/scope_config.go | 4 +-
.../plugins/jenkins/tasks/build_cicd_convertor.go | 12 +-
backend/plugins/jenkins/tasks/build_collector.go | 2 +-
backend/plugins/jenkins/tasks/build_extractor.go | 1 +
backend/plugins/jenkins/tasks/stage_convertor.go | 6 +-
.../services/remote/plugin/connection_api.go | 89 +++-
.../server/services/remote/plugin/default_api.go | 3 +
config-ui/package.json | 6 +-
config-ui/src/api/scope-config/index.ts | 18 +-
config-ui/src/api/scope-config/types.ts | 5 +
config-ui/src/components/index.ts | 1 +
.../components/{tip-layout => show-more}/index.tsx | 31 +-
.../components/check-matched-items/index.tsx | 115 +++++
.../components/data-scope-remote/search-local.tsx | 141 +++---
.../components/data-scope-remote/search-remote.tsx | 8 +-
.../plugins/components/data-scope-select/index.tsx | 26 +-
.../src/plugins/components/deployments/index.tsx | 109 ++++
config-ui/src/plugins/components/index.ts | 2 +
.../plugins/components/scope-config-form/index.tsx | 28 +-
.../register/azure/assets/workflow-run.jpeg | Bin 0 -> 149316 bytes
config-ui/src/plugins/register/azure/config.tsx | 2 -
.../src/plugins/register/azure/transformation.tsx | 30 +-
.../src/plugins/register/azure/workflow-run.tsx | 98 ++++
.../register/bamboo/assets/workflow-run.jpeg | Bin 0 -> 113157 bytes
config-ui/src/plugins/register/bamboo/config.tsx | 6 +-
.../src/plugins/register/bamboo/transformation.tsx | 120 +++--
.../src/plugins/register/bamboo/workflow-run.tsx | 94 ++++
.../register/bitbucket/assets/workflow-run-1.jpeg | Bin 0 -> 71873 bytes
.../register/bitbucket/assets/workflow-run-2.jpeg | Bin 0 -> 52643 bytes
.../src/plugins/register/bitbucket/config.tsx | 2 -
.../plugins/register/bitbucket/transformation.tsx | 107 ++--
.../plugins/register/bitbucket/workflow-run.tsx | 104 ++++
.../register/circleci/assets/workflow-run.jpeg | Bin 0 -> 197685 bytes
config-ui/src/plugins/register/circleci/config.tsx | 5 +-
.../plugins/register/circleci/transformation.tsx | 41 +-
.../src/plugins/register/circleci/workflow-run.tsx | 97 ++++
.../register/github/assets/workflow-run.jpeg | Bin 0 -> 189484 bytes
config-ui/src/plugins/register/github/config.tsx | 3 -
.../src/plugins/register/github/transformation.tsx | 129 +++--
.../src/plugins/register/github/workflow-run.tsx | 103 ++++
.../register/gitlab/assets/workflow-run.jpeg | Bin 0 -> 193279 bytes
config-ui/src/plugins/register/gitlab/config.tsx | 6 +-
.../src/plugins/register/gitlab/transformation.tsx | 131 +++--
.../src/plugins/register/gitlab/workflow-run.tsx | 100 ++++
.../register/jenkins/assets/workflow-run.jpeg | Bin 0 -> 238201 bytes
config-ui/src/plugins/register/jenkins/config.tsx | 5 +-
.../plugins/register/jenkins/transformation.tsx | 35 +-
.../src/plugins/register/jenkins/workflow-run.tsx | 97 ++++
config-ui/src/release/stable.ts | 1 +
config-ui/yarn.lock | 560 ++++++++++++---------
125 files changed, 3139 insertions(+), 911 deletions(-)
copy backend/{core/models/migrationscripts/20231208_add_commit_msg_to_cicd_pipeline_commits.go => plugins/bamboo/models/migrationscripts/20240906_add_env_name_list_to_scope_config.go} (60%)
copy backend/{core/models/migrationscripts/20231208_add_commit_msg_to_cicd_pipeline_commits.go => plugins/github/models/migrationscripts/20240906_add_env_name_list_to_scope_config.go} (60%)
copy backend/{core/models/migrationscripts/20231208_add_commit_msg_to_cicd_pipeline_commits.go => plugins/gitlab/models/migrationscripts/20240906_add_env_name_list_to_scope_config.go} (60%)
copy backend/plugins/{tapd/models/migrationscripts/20230330_delete_issue.go => jenkins/models/migrationscripts/20240911_add_url_for_builds.go} (69%)
copy config-ui/src/components/{tip-layout => show-more}/index.tsx (57%)
create mode 100644 config-ui/src/plugins/components/check-matched-items/index.tsx
create mode 100644 config-ui/src/plugins/components/deployments/index.tsx
create mode 100644 config-ui/src/plugins/register/azure/assets/workflow-run.jpeg
create mode 100644 config-ui/src/plugins/register/azure/workflow-run.tsx
create mode 100644 config-ui/src/plugins/register/bamboo/assets/workflow-run.jpeg
create mode 100644 config-ui/src/plugins/register/bamboo/workflow-run.tsx
create mode 100644 config-ui/src/plugins/register/bitbucket/assets/workflow-run-1.jpeg
create mode 100644 config-ui/src/plugins/register/bitbucket/assets/workflow-run-2.jpeg
create mode 100644 config-ui/src/plugins/register/bitbucket/workflow-run.tsx
create mode 100644 config-ui/src/plugins/register/circleci/assets/workflow-run.jpeg
create mode 100644 config-ui/src/plugins/register/circleci/workflow-run.tsx
create mode 100644 config-ui/src/plugins/register/github/assets/workflow-run.jpeg
create mode 100644 config-ui/src/plugins/register/github/workflow-run.tsx
create mode 100644 config-ui/src/plugins/register/gitlab/assets/workflow-run.jpeg
create mode 100644 config-ui/src/plugins/register/gitlab/workflow-run.tsx
create mode 100644 config-ui/src/plugins/register/jenkins/assets/workflow-run.jpeg
create mode 100644 config-ui/src/plugins/register/jenkins/workflow-run.tsx
(incubator-devlake) 09/09: fix: data scope search cannot select
Posted by mi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mintsweet pushed a commit to branch feat-8017
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit 74e0baf540c84a4c747185af789b9af9a466fa7b
Author: mintsweet <0x...@gmail.com>
AuthorDate: Thu Sep 26 21:36:18 2024 +1200
fix: data scope search cannot select
---
.../src/plugins/components/data-scope-remote/search-local.tsx | 10 ++++++----
1 file changed, 6 insertions(+), 4 deletions(-)
diff --git a/config-ui/src/plugins/components/data-scope-remote/search-local.tsx b/config-ui/src/plugins/components/data-scope-remote/search-local.tsx
index 67b8f4a7f..31c126db3 100644
--- a/config-ui/src/plugins/components/data-scope-remote/search-local.tsx
+++ b/config-ui/src/plugins/components/data-scope-remote/search-local.tsx
@@ -104,9 +104,7 @@ export const SearchLocal = ({ mode, plugin, connectionId, config, disabledScope,
async (groupId?: string | number) => {
return {
data: searchDebounce
- ? scope
- .filter((it) => it.title.includes(searchDebounce) && !it.canExpand)
- .map((it) => ({ ...it, parentId: null }))
+ ? scope.filter((it) => it.title.includes(searchDebounce) && !it.canExpand)
: scope.filter((it) => it.parentId === (groupId ?? null)),
hasMore: false,
};
@@ -238,7 +236,11 @@ export const SearchLocal = ({ mode, plugin, connectionId, config, disabledScope,
selectable
disabledIds={(disabledScope ?? []).map((it) => it.id)}
selectedIds={selectedScope.map((it) => it.id)}
- onSelectedIds={(_, data) => onChange(data ?? [])}
+ onSelectedIds={(_, data) => {
+ console.log(_);
+ console.log(data);
+ onChange(data ?? []);
+ }}
/>
</>
)}
(incubator-devlake) 08/09: fix: adjust the api for connections token check
Posted by mi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mintsweet pushed a commit to branch feat-8017
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit 288be500457eb72d322fc84fb1a480b11a056e02
Author: mintsweet <0x...@gmail.com>
AuthorDate: Wed Sep 25 18:29:09 2024 +1200
fix: adjust the api for connections token check
---
config-ui/src/api/blueprint/index.ts | 2 ++
config-ui/src/api/project/index.ts | 11 +----------
config-ui/src/routes/blueprint/detail/status-panel.tsx | 10 +++++-----
3 files changed, 8 insertions(+), 15 deletions(-)
diff --git a/config-ui/src/api/blueprint/index.ts b/config-ui/src/api/blueprint/index.ts
index 3876d3e20..99460c088 100644
--- a/config-ui/src/api/blueprint/index.ts
+++ b/config-ui/src/api/blueprint/index.ts
@@ -43,3 +43,5 @@ type TriggerQuery = {
export const trigger = (id: ID, data: TriggerQuery = { skipCollectors: false, fullSync: false }) =>
request(`/blueprints/${id}/trigger`, { method: 'post', data });
+
+export const connectionsTokenCheck = (id: ID) => request(`/blueprints/${id}/connections-token-check`);
diff --git a/config-ui/src/api/project/index.ts b/config-ui/src/api/project/index.ts
index bc27b5e00..8e79ee5fa 100644
--- a/config-ui/src/api/project/index.ts
+++ b/config-ui/src/api/project/index.ts
@@ -24,16 +24,7 @@ export const list = (data: Pagination & { keyword?: string }): Promise<{ count:
export const get = (name: string): Promise<IProject> => request(`/projects/${encodeURIComponent(name)}`);
-export const check = (
- name: string,
- data?: { check_token: 1 },
-): Promise<{
- exist: boolean;
- tokens: Array<{ pluginName: string; connectionId: ID; success: boolean }>;
-}> =>
- request(`/projects/${encodeURIComponent(name)}/check`, {
- data,
- });
+export const checkName = (name: string) => request(`/projects/${encodeURIComponent(name)}/check`);
export const create = (data: Pick<IProject, 'name' | 'description' | 'metrics'>) =>
request('/projects', {
diff --git a/config-ui/src/routes/blueprint/detail/status-panel.tsx b/config-ui/src/routes/blueprint/detail/status-panel.tsx
index f9aaccc38..9e48d9968 100644
--- a/config-ui/src/routes/blueprint/detail/status-panel.tsx
+++ b/config-ui/src/routes/blueprint/detail/status-panel.tsx
@@ -74,15 +74,15 @@ export const StatusPanel = ({ from, blueprint, pipelineId, onRefresh }: Props) =
skipCollectors?: boolean;
fullSync?: boolean;
}) => {
- if (!skipCollectors && from === FromEnum.project) {
- const [success, res] = await operator(() => API.project.check(blueprint.projectName, { check_token: 1 }), {
+ if (!skipCollectors) {
+ const [success, res] = await operator(() => API.blueprint.connectionsTokenCheck(blueprint.id), {
hideToast: true,
setOperating,
});
- if (success && res.tokens.length) {
- const connectionFailed = res.tokens
- .filter((token: any) => !token.success)
+ if (success && res.length) {
+ const connectionFailed = res
+ .filter((it: any) => !it.success)
.map((it: any) => {
const unique = `${it.pluginName}-${it.connectionId}`;
const connection = connections.find((c) => c.unique === unique);
(incubator-devlake) 02/09: feat(project): add token check result in project check API (#8099)
Posted by mi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mintsweet pushed a commit to branch feat-8017
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit 168809cbcec92700cd933a635dcedefb0ac59fbf
Author: Lynwee <15...@qq.com>
AuthorDate: Wed Sep 25 10:29:38 2024 +0800
feat(project): add token check result in project check API (#8099)
---
backend/core/models/project.go | 22 +++++++++++++++++++++-
backend/server/api/project/project.go | 9 +++++++++
backend/server/services/project.go | 27 +++++++++++++++++++++++++++
3 files changed, 57 insertions(+), 1 deletion(-)
diff --git a/backend/core/models/project.go b/backend/core/models/project.go
index 99c61785d..5227d2a1c 100644
--- a/backend/core/models/project.go
+++ b/backend/core/models/project.go
@@ -73,9 +73,29 @@ type ApiOutputProject struct {
}
type ApiProjectCheck struct {
- Exist bool `json:"exist" mapstructure:"exist"`
+ Exist bool `json:"exist" mapstructure:"exist"`
+ Tokens *ApiProjectCheckToken `json:"tokens,omitempty" mapstructure:"tokens"`
}
+type SuccessAndMessage struct {
+ Success bool `json:"success" mapstructure:"success"`
+ Message string `json:"message" mapstructure:"message"`
+}
+
+// ApiProjectCheckToken
+//
+// {
+// "plugin_name":
+// {
+// "connection_id":
+// {
+// "success": true,
+// "message": ""
+// }
+// }
+// }
+type ApiProjectCheckToken = map[string]map[int]SuccessAndMessage
+
type Store struct {
StoreKey string `gorm:"primaryKey;type:varchar(255)"`
StoreValue json.RawMessage `gorm:"type:json;serializer:json"`
diff --git a/backend/server/api/project/project.go b/backend/server/api/project/project.go
index e826c71d1..b4a14fff5 100644
--- a/backend/server/api/project/project.go
+++ b/backend/server/api/project/project.go
@@ -73,6 +73,15 @@ func GetProjectCheck(c *gin.Context) {
projectOutputCheck.Exist = true
}
+ if c.Query("check_token") == "1" {
+ checkTokenResult, err := services.CheckProjectTokens(projectName)
+ if err != nil {
+ shared.ApiOutputError(c, errors.Default.Wrap(err, "error check project tokens"))
+ return
+ }
+ projectOutputCheck.Tokens = checkTokenResult
+ }
+
shared.ApiOutputSuccess(c, projectOutputCheck, http.StatusOK) // //shared.ApiOutputSuccess(c, projectOutputCheck, http.StatusOK)
}
diff --git a/backend/server/services/project.go b/backend/server/services/project.go
index 614f10680..2e5c28aa8 100644
--- a/backend/server/services/project.go
+++ b/backend/server/services/project.go
@@ -193,6 +193,33 @@ func GetProject(name string) (*models.ApiOutputProject, errors.Error) {
return makeProjectOutput(project, false)
}
+func CheckProjectTokens(name string) (*models.ApiProjectCheckToken, errors.Error) {
+ blueprint, err := GetBlueprintByProjectName(name)
+ if err != nil {
+ return nil, err
+ }
+ ret := make(map[string]map[int]models.SuccessAndMessage)
+ for _, connection := range blueprint.Connections {
+ pluginName := connection.PluginName
+ connectionId := int(connection.ConnectionId)
+ if _, ok := ret[pluginName]; !ok {
+ ret[pluginName] = make(map[int]models.SuccessAndMessage)
+ }
+ connectionTokenResult := models.SuccessAndMessage{
+ Success: true,
+ Message: "success",
+ }
+ if err := checkConnectionToken(logger, *connection); err != nil {
+ ret[pluginName][connectionId] = models.SuccessAndMessage{
+ Success: false,
+ Message: err.Error(),
+ }
+ }
+ ret[pluginName][connectionId] = connectionTokenResult
+ }
+ return &ret, nil
+}
+
// PatchProject FIXME ...
func PatchProject(name string, body map[string]interface{}) (*models.ApiOutputProject, errors.Error) {
projectInput := &models.ApiInputProject{}
(incubator-devlake) 06/09: feat: add connection token check before collect data
Posted by mi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mintsweet pushed a commit to branch feat-8017
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit f8b4efc4d97704aa42df9c897f1f5937bf7c2ed7
Author: mintsweet <0x...@gmail.com>
AuthorDate: Wed Sep 25 15:54:48 2024 +1200
feat: add connection token check before collect data
---
config-ui/src/api/project/index.ts | 11 +++-
.../src/routes/blueprint/detail/status-panel.tsx | 75 ++++++++++++++++++++--
2 files changed, 80 insertions(+), 6 deletions(-)
diff --git a/config-ui/src/api/project/index.ts b/config-ui/src/api/project/index.ts
index 8e79ee5fa..bc27b5e00 100644
--- a/config-ui/src/api/project/index.ts
+++ b/config-ui/src/api/project/index.ts
@@ -24,7 +24,16 @@ export const list = (data: Pagination & { keyword?: string }): Promise<{ count:
export const get = (name: string): Promise<IProject> => request(`/projects/${encodeURIComponent(name)}`);
-export const checkName = (name: string) => request(`/projects/${encodeURIComponent(name)}/check`);
+export const check = (
+ name: string,
+ data?: { check_token: 1 },
+): Promise<{
+ exist: boolean;
+ tokens: Array<{ pluginName: string; connectionId: ID; success: boolean }>;
+}> =>
+ request(`/projects/${encodeURIComponent(name)}/check`, {
+ data,
+ });
export const create = (data: Pick<IProject, 'name' | 'description' | 'metrics'>) =>
request('/projects', {
diff --git a/config-ui/src/routes/blueprint/detail/status-panel.tsx b/config-ui/src/routes/blueprint/detail/status-panel.tsx
index 09d2fdf25..f9aaccc38 100644
--- a/config-ui/src/routes/blueprint/detail/status-panel.tsx
+++ b/config-ui/src/routes/blueprint/detail/status-panel.tsx
@@ -17,14 +17,15 @@
*/
import { useState, useMemo } from 'react';
-import { useNavigate } from 'react-router-dom';
-import { MoreOutlined, DeleteOutlined } from '@ant-design/icons';
-import { Card, Modal, Switch, Button, Tooltip, Dropdown, Flex, Space } from 'antd';
+import { useNavigate, Link } from 'react-router-dom';
+import { MoreOutlined, DeleteOutlined, WarningOutlined } from '@ant-design/icons';
+import { theme, Card, Modal, Switch, Button, Tooltip, Dropdown, Flex, Space } from 'antd';
import API from '@/api';
import { Message } from '@/components';
import { getCron } from '@/config';
-import { useRefreshData } from '@/hooks';
+import { selectAllConnections } from '@/features/connections';
+import { useAppSelector, useRefreshData } from '@/hooks';
import { PipelineInfo, PipelineTasks, PipelineTable } from '@/routes/pipeline';
import { IBlueprint } from '@/types';
import { formatTime, operator } from '@/utils';
@@ -39,13 +40,22 @@ interface Props {
}
export const StatusPanel = ({ from, blueprint, pipelineId, onRefresh }: Props) => {
- const [type, setType] = useState<'delete' | 'fullSync'>();
+ const [type, setType] = useState<'delete' | 'fullSync' | 'checkTokenFailed'>();
const [page, setPage] = useState(1);
const [pageSize] = useState(10);
const [operating, setOperating] = useState(false);
+ const [connectionFailed, setConnectionFailed] = useState<
+ Array<{ unique: string; name: string; plugin: string; connectionId: ID }>
+ >([]);
const navigate = useNavigate();
+ const {
+ token: { orange5 },
+ } = theme.useToken();
+
+ const connections = useAppSelector(selectAllConnections);
+
const cron = useMemo(() => getCron(blueprint.isManual, blueprint.cronConfig), [blueprint]);
const { ready, data } = useRefreshData(
@@ -64,6 +74,32 @@ export const StatusPanel = ({ from, blueprint, pipelineId, onRefresh }: Props) =
skipCollectors?: boolean;
fullSync?: boolean;
}) => {
+ if (!skipCollectors && from === FromEnum.project) {
+ const [success, res] = await operator(() => API.project.check(blueprint.projectName, { check_token: 1 }), {
+ hideToast: true,
+ setOperating,
+ });
+
+ if (success && res.tokens.length) {
+ const connectionFailed = res.tokens
+ .filter((token: any) => !token.success)
+ .map((it: any) => {
+ const unique = `${it.pluginName}-${it.connectionId}`;
+ const connection = connections.find((c) => c.unique === unique);
+ return {
+ unique,
+ name: connection?.name ?? '',
+ plugin: it.pluginName,
+ connectionId: it.connectionId,
+ };
+ });
+
+ setType('checkTokenFailed');
+ setConnectionFailed(connectionFailed);
+ return;
+ }
+ }
+
const [success] = await operator(() => API.blueprint.trigger(blueprint.id, { skipCollectors, fullSync }), {
setOperating,
formatMessage: () => 'Trigger blueprint successful.',
@@ -245,6 +281,35 @@ export const StatusPanel = ({ from, blueprint, pipelineId, onRefresh }: Props) =
<Message content="This operation may take a long time as it will empty all of your existing data and re-collect it." />
</Modal>
)}
+
+ {type === 'checkTokenFailed' && (
+ <Modal
+ open
+ title={
+ <>
+ <WarningOutlined style={{ marginRight: 8, fontSize: 20, color: orange5 }} />
+ <span>Invalid Token(s) Detected</span>
+ </>
+ }
+ width={820}
+ footer={null}
+ onCancel={() => {
+ handleResetType();
+ setConnectionFailed([]);
+ }}
+ >
+ <p>There are invalid tokens in the following connections. Please update them before re-syncing the data.</p>
+ <ul style={{ paddingLeft: 20 }}>
+ {connectionFailed.map((it) => (
+ <li key={it.unique} style={{ listStyle: 'initial' }}>
+ <Link to={`/connections/${it.plugin}/${it.connectionId}`} target="_blank">
+ {it.name}
+ </Link>
+ </li>
+ ))}
+ </ul>
+ </Modal>
+ )}
</Flex>
);
};
(incubator-devlake) 05/09: fix(project): update check project api doc
Posted by mi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mintsweet pushed a commit to branch feat-8017
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit c8296a8c04294f57d88c40c94c735e00859c7c8f
Author: d4x1 <15...@qq.com>
AuthorDate: Wed Sep 25 11:28:39 2024 +0800
fix(project): update check project api doc
---
backend/server/api/project/project.go | 1 +
1 file changed, 1 insertion(+)
diff --git a/backend/server/api/project/project.go b/backend/server/api/project/project.go
index 08e70c626..43687da73 100644
--- a/backend/server/api/project/project.go
+++ b/backend/server/api/project/project.go
@@ -58,6 +58,7 @@ func GetProject(c *gin.Context) {
// @Tags framework/projects
// @Accept application/json
// @Param projectName path string true "project name"
+// @Param check_token query int false "need to check token validity or not"
// @Success 200 {object} models.ApiProjectCheck
// @Failure 400 {string} errcode.Error "Bad Request"
// @Failure 500 {string} errcode.Error "Internal Error"
(incubator-devlake) 01/09: feat: check pipelines' tokens before executing/creating pipeline (#8055)
Posted by mi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mintsweet pushed a commit to branch feat-8017
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit 6e45e9d7de8d692309f761c4b7e7827eca98feff
Author: Lynwee <15...@qq.com>
AuthorDate: Thu Sep 19 12:13:26 2024 +0800
feat: check pipelines' tokens before executing/creating pipeline (#8055)
* feat(pipeline): check pipelines' tokens before executing/creating pipeline
* fix(framework): fix lint errors
* fix(framework): fix lint errors
* feat(github): check multi tokens when creating pipelines
---
backend/core/plugin/plugin_api.go | 4 ++
backend/core/plugin/plugin_blueprint.go | 10 +++
backend/helpers/pluginhelper/api/misc_helpers.go | 10 +++
backend/plugins/ae/impl/impl.go | 5 ++
backend/plugins/azuredevops_go/impl/impl.go | 5 ++
backend/plugins/bamboo/impl/impl.go | 5 ++
backend/plugins/bitbucket/impl/impl.go | 5 ++
backend/plugins/bitbucket_server/impl/impl.go | 5 ++
backend/plugins/circleci/impl/impl.go | 5 ++
backend/plugins/customize/impl/impl.go | 4 ++
backend/plugins/dbt/impl/impl.go | 4 ++
backend/plugins/dora/impl/impl.go | 5 +-
backend/plugins/feishu/impl/impl.go | 5 ++
backend/plugins/gitee/impl/impl.go | 5 ++
backend/plugins/gitextractor/impl/impl.go | 4 ++
.../github/api/shared.go} | 25 +++++---
backend/plugins/github/impl/impl.go | 4 ++
backend/plugins/gitlab/impl/impl.go | 5 ++
backend/plugins/icla/impl/impl.go | 4 ++
backend/plugins/issue_trace/impl/enricher.go | 5 +-
backend/plugins/jenkins/impl/impl.go | 5 ++
backend/plugins/jira/impl/impl.go | 5 ++
backend/plugins/linker/impl/impl.go | 4 ++
backend/plugins/opsgenie/impl/impl.go | 5 ++
backend/plugins/org/impl/impl.go | 22 +++++++
backend/plugins/org/tasks/check_token.go | 75 ++++++++++++++++++++++
backend/plugins/org/tasks/task_data.go | 10 ++-
backend/plugins/pagerduty/impl/impl.go | 5 ++
backend/plugins/refdiff/impl/impl.go | 4 ++
backend/plugins/slack/impl/impl.go | 5 ++
backend/plugins/sonarqube/impl/impl.go | 5 ++
backend/plugins/starrocks/impl/impl.go | 4 ++
backend/plugins/tapd/impl/impl.go | 5 ++
backend/plugins/teambition/impl/impl.go | 5 ++
backend/plugins/trello/impl/impl.go | 5 ++
backend/plugins/webhook/impl/impl.go | 4 ++
backend/plugins/zentao/impl/impl.go | 5 ++
backend/server/services/blueprint.go | 69 ++++++++++++++++++++
backend/server/services/blueprint_makeplan_v200.go | 15 +++++
backend/server/services/project.go | 2 +-
.../server/services/remote/models/plugin_remote.go | 1 +
.../server/services/remote/plugin/plugin_impl.go | 5 ++
42 files changed, 376 insertions(+), 13 deletions(-)
diff --git a/backend/core/plugin/plugin_api.go b/backend/core/plugin/plugin_api.go
index f9195d2ee..ea41a4895 100644
--- a/backend/core/plugin/plugin_api.go
+++ b/backend/core/plugin/plugin_api.go
@@ -73,6 +73,10 @@ type PluginApi interface {
ApiResources() map[string]map[string]ApiResourceHandler
}
+type PluginTestConnectionAPI interface {
+ TestConnection(id uint64) errors.Error
+}
+
const wrapResponseError = "WRAP_RESPONSE_ERROR"
func WrapTestConnectionErrResp(basicRes context.BasicRes, err errors.Error) errors.Error {
diff --git a/backend/core/plugin/plugin_blueprint.go b/backend/core/plugin/plugin_blueprint.go
index b722bb9e2..a24513564 100644
--- a/backend/core/plugin/plugin_blueprint.go
+++ b/backend/core/plugin/plugin_blueprint.go
@@ -93,6 +93,16 @@ type ProjectMapper interface {
MapProject(projectName string, scopes []Scope) (models.PipelinePlan, errors.Error)
}
+type ProjectTokenCheckerConnection struct {
+ PluginName string
+ ConnectionId uint64
+}
+
+// ProjectTokenChecker is implemented by the plugin org, which generate a task tp check all connection's tokens
+type ProjectTokenChecker interface {
+ MakePipeline(skipCollectors bool, projectName string, scopes []ProjectTokenCheckerConnection) (models.PipelinePlan, errors.Error)
+}
+
// CompositeDataSourcePluginBlueprintV200 is for unit test
type CompositeDataSourcePluginBlueprintV200 interface {
PluginMeta
diff --git a/backend/helpers/pluginhelper/api/misc_helpers.go b/backend/helpers/pluginhelper/api/misc_helpers.go
index e7315ceab..3dcf8778c 100644
--- a/backend/helpers/pluginhelper/api/misc_helpers.go
+++ b/backend/helpers/pluginhelper/api/misc_helpers.go
@@ -18,9 +18,11 @@ limitations under the License.
package api
import (
+ "fmt"
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models"
+ "github.com/apache/incubator-devlake/core/plugin"
)
// CallDB wraps DB calls with this signature, and handles the case if the struct is wrapped in a models.DynamicTabler.
@@ -31,3 +33,11 @@ func CallDB(f func(any, ...dal.Clause) errors.Error, x any, clauses ...dal.Claus
}
return f(x, clauses...)
}
+
+func GenerateTestingConnectionApiResourceInput(connectionID uint64) *plugin.ApiResourceInput {
+ return &plugin.ApiResourceInput{
+ Params: map[string]string{
+ "connectionId": fmt.Sprintf("%d", connectionID),
+ },
+ }
+}
diff --git a/backend/plugins/ae/impl/impl.go b/backend/plugins/ae/impl/impl.go
index 389857e1f..7356d9e0e 100644
--- a/backend/plugins/ae/impl/impl.go
+++ b/backend/plugins/ae/impl/impl.go
@@ -130,6 +130,11 @@ func (p AE) MigrationScripts() []plugin.MigrationScript {
return migrationscripts.All()
}
+func (p AE) TestConnection(id uint64) errors.Error {
+ _, err := api.TestExistingConnection(helper.GenerateTestingConnectionApiResourceInput(id))
+ return err
+}
+
func (p AE) ApiResources() map[string]map[string]plugin.ApiResourceHandler {
return map[string]map[string]plugin.ApiResourceHandler{
"test": {
diff --git a/backend/plugins/azuredevops_go/impl/impl.go b/backend/plugins/azuredevops_go/impl/impl.go
index c567e49f5..2aecfde51 100644
--- a/backend/plugins/azuredevops_go/impl/impl.go
+++ b/backend/plugins/azuredevops_go/impl/impl.go
@@ -255,6 +255,11 @@ func (p Azuredevops) ApiResources() map[string]map[string]plugin.ApiResourceHand
}
}
+func (p Azuredevops) TestConnection(id uint64) errors.Error {
+ _, err := api.TestExistingConnection(helper.GenerateTestingConnectionApiResourceInput(id))
+ return err
+}
+
func (p Azuredevops) MakeDataSourcePipelinePlanV200(
connectionId uint64,
scopes []*coreModels.BlueprintScope,
diff --git a/backend/plugins/bamboo/impl/impl.go b/backend/plugins/bamboo/impl/impl.go
index b90331df1..b7737770a 100644
--- a/backend/plugins/bamboo/impl/impl.go
+++ b/backend/plugins/bamboo/impl/impl.go
@@ -216,6 +216,11 @@ func (p Bamboo) MigrationScripts() []plugin.MigrationScript {
return migrationscripts.All()
}
+func (p Bamboo) TestConnection(id uint64) errors.Error {
+ _, err := api.TestExistingConnection(helper.GenerateTestingConnectionApiResourceInput(id))
+ return err
+}
+
func (p Bamboo) ApiResources() map[string]map[string]plugin.ApiResourceHandler {
return map[string]map[string]plugin.ApiResourceHandler{
"test": {
diff --git a/backend/plugins/bitbucket/impl/impl.go b/backend/plugins/bitbucket/impl/impl.go
index dd917790c..4ff5bc5e9 100644
--- a/backend/plugins/bitbucket/impl/impl.go
+++ b/backend/plugins/bitbucket/impl/impl.go
@@ -211,6 +211,11 @@ func (p Bitbucket) MakeDataSourcePipelinePlanV200(
return api.MakeDataSourcePipelinePlanV200(p.SubTaskMetas(), connectionId, scopes, skipCollectors)
}
+func (p Bitbucket) TestConnection(id uint64) errors.Error {
+ _, err := api.TestExistingConnection(helper.GenerateTestingConnectionApiResourceInput(id))
+ return err
+}
+
func (p Bitbucket) ApiResources() map[string]map[string]plugin.ApiResourceHandler {
return map[string]map[string]plugin.ApiResourceHandler{
"test": {
diff --git a/backend/plugins/bitbucket_server/impl/impl.go b/backend/plugins/bitbucket_server/impl/impl.go
index 18b4a4679..fe446d76f 100644
--- a/backend/plugins/bitbucket_server/impl/impl.go
+++ b/backend/plugins/bitbucket_server/impl/impl.go
@@ -170,6 +170,11 @@ func (p BitbucketServer) MakeDataSourcePipelinePlanV200(
return api.MakeDataSourcePipelinePlanV200(p.SubTaskMetas(), connectionId, scopes, skipCollectors)
}
+func (p BitbucketServer) TestConnection(id uint64) errors.Error {
+ _, err := api.TestExistingConnection(helper.GenerateTestingConnectionApiResourceInput(id))
+ return err
+}
+
func (p BitbucketServer) ApiResources() map[string]map[string]plugin.ApiResourceHandler {
return map[string]map[string]plugin.ApiResourceHandler{
"connections/:connectionId/test": {
diff --git a/backend/plugins/circleci/impl/impl.go b/backend/plugins/circleci/impl/impl.go
index 8cca5895d..c08766a6d 100644
--- a/backend/plugins/circleci/impl/impl.go
+++ b/backend/plugins/circleci/impl/impl.go
@@ -176,6 +176,11 @@ func (p Circleci) MigrationScripts() []plugin.MigrationScript {
return migrationscripts.All()
}
+func (p Circleci) TestConnection(id uint64) errors.Error {
+ _, err := api.TestExistingConnection(helper.GenerateTestingConnectionApiResourceInput(id))
+ return err
+}
+
func (p Circleci) ApiResources() map[string]map[string]plugin.ApiResourceHandler {
return map[string]map[string]plugin.ApiResourceHandler{
"test": {
diff --git a/backend/plugins/customize/impl/impl.go b/backend/plugins/customize/impl/impl.go
index facb0515d..59b967aa5 100644
--- a/backend/plugins/customize/impl/impl.go
+++ b/backend/plugins/customize/impl/impl.go
@@ -110,3 +110,7 @@ func (p Customize) ApiResources() map[string]map[string]plugin.ApiResourceHandle
},
}
}
+
+func (p Customize) TestConnection(id uint64) errors.Error {
+ return nil
+}
diff --git a/backend/plugins/dbt/impl/impl.go b/backend/plugins/dbt/impl/impl.go
index 88eafa9f6..55b7c2918 100644
--- a/backend/plugins/dbt/impl/impl.go
+++ b/backend/plugins/dbt/impl/impl.go
@@ -74,3 +74,7 @@ func (p Dbt) RootPkgPath() string {
func (p Dbt) Name() string {
return "dbt"
}
+
+func (p Dbt) TestConnection(id uint64) errors.Error {
+ return nil
+}
diff --git a/backend/plugins/dora/impl/impl.go b/backend/plugins/dora/impl/impl.go
index 246097398..0d41a94eb 100644
--- a/backend/plugins/dora/impl/impl.go
+++ b/backend/plugins/dora/impl/impl.go
@@ -19,7 +19,6 @@ package impl
import (
"encoding/json"
-
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
coreModels "github.com/apache/incubator-devlake/core/models"
@@ -168,3 +167,7 @@ func (p Dora) MakeMetricPluginPipelinePlanV200(projectName string, options json.
}
return plan, nil
}
+
+func (p Dora) TestConnection(id uint64) errors.Error {
+ return nil
+}
diff --git a/backend/plugins/feishu/impl/impl.go b/backend/plugins/feishu/impl/impl.go
index 92e0f1571..3158b5f1f 100644
--- a/backend/plugins/feishu/impl/impl.go
+++ b/backend/plugins/feishu/impl/impl.go
@@ -132,6 +132,11 @@ func (p Feishu) MigrationScripts() []plugin.MigrationScript {
return migrationscripts.All()
}
+func (p Feishu) TestConnection(id uint64) errors.Error {
+ _, err := api.TestExistingConnection(helper.GenerateTestingConnectionApiResourceInput(id))
+ return err
+}
+
func (p Feishu) ApiResources() map[string]map[string]plugin.ApiResourceHandler {
return map[string]map[string]plugin.ApiResourceHandler{
"test": {
diff --git a/backend/plugins/gitee/impl/impl.go b/backend/plugins/gitee/impl/impl.go
index 35be389e7..b422fbafc 100644
--- a/backend/plugins/gitee/impl/impl.go
+++ b/backend/plugins/gitee/impl/impl.go
@@ -187,6 +187,11 @@ func (p Gitee) MigrationScripts() []plugin.MigrationScript {
return migrationscripts.All()
}
+func (p Gitee) TestConnection(id uint64) errors.Error {
+ _, err := api.TestExistingConnection(helper.GenerateTestingConnectionApiResourceInput(id))
+ return err
+}
+
func (p Gitee) ApiResources() map[string]map[string]plugin.ApiResourceHandler {
return map[string]map[string]plugin.ApiResourceHandler{
"test": {
diff --git a/backend/plugins/gitextractor/impl/impl.go b/backend/plugins/gitextractor/impl/impl.go
index 468e51734..45c84ab6e 100644
--- a/backend/plugins/gitextractor/impl/impl.go
+++ b/backend/plugins/gitextractor/impl/impl.go
@@ -122,3 +122,7 @@ func (p GitExtractor) Close(taskCtx plugin.TaskContext) errors.Error {
func (p GitExtractor) RootPkgPath() string {
return "github.com/apache/incubator-devlake/plugins/gitextractor"
}
+
+func (p GitExtractor) TestConnection(id uint64) errors.Error {
+ return nil
+}
diff --git a/backend/helpers/pluginhelper/api/misc_helpers.go b/backend/plugins/github/api/shared.go
similarity index 57%
copy from backend/helpers/pluginhelper/api/misc_helpers.go
copy to backend/plugins/github/api/shared.go
index e7315ceab..61bb39801 100644
--- a/backend/helpers/pluginhelper/api/misc_helpers.go
+++ b/backend/plugins/github/api/shared.go
@@ -18,16 +18,25 @@ limitations under the License.
package api
import (
- "github.com/apache/incubator-devlake/core/dal"
+ "context"
+ "fmt"
"github.com/apache/incubator-devlake/core/errors"
- "github.com/apache/incubator-devlake/core/models"
+ "github.com/apache/incubator-devlake/core/plugin"
)
-// CallDB wraps DB calls with this signature, and handles the case if the struct is wrapped in a models.DynamicTabler.
-func CallDB(f func(any, ...dal.Clause) errors.Error, x any, clauses ...dal.Clause) errors.Error {
- if dynamic, ok := x.(models.DynamicTabler); ok {
- clauses = append(clauses, dal.From(dynamic.TableName()))
- x = dynamic.Unwrap()
+func TestExistingConnectionForTokenCheck(input *plugin.ApiResourceInput) errors.Error {
+ connection, err := dsHelper.ConnApi.GetMergedConnection(input)
+ if err != nil {
+ return err
}
- return f(x, clauses...)
+ testConnectionResult, testConnectionErr := testExistingConnection(context.TODO(), connection.GithubConn)
+ if testConnectionErr != nil {
+ return testConnectionErr
+ }
+ for _, token := range testConnectionResult.Tokens {
+ if !token.Success {
+ return errors.Default.New(fmt.Sprintf("token %s failed with msg: %s", token.Token, token.Message))
+ }
+ }
+ return nil
}
diff --git a/backend/plugins/github/impl/impl.go b/backend/plugins/github/impl/impl.go
index 52a772cf6..e03092370 100644
--- a/backend/plugins/github/impl/impl.go
+++ b/backend/plugins/github/impl/impl.go
@@ -191,6 +191,10 @@ func (p Github) MigrationScripts() []plugin.MigrationScript {
return migrationscripts.All()
}
+func (p Github) TestConnection(id uint64) errors.Error {
+ return api.TestExistingConnectionForTokenCheck(helper.GenerateTestingConnectionApiResourceInput(id))
+}
+
func (p Github) ApiResources() map[string]map[string]plugin.ApiResourceHandler {
return map[string]map[string]plugin.ApiResourceHandler{
"test": {
diff --git a/backend/plugins/gitlab/impl/impl.go b/backend/plugins/gitlab/impl/impl.go
index d292db691..46f63b52f 100644
--- a/backend/plugins/gitlab/impl/impl.go
+++ b/backend/plugins/gitlab/impl/impl.go
@@ -244,6 +244,11 @@ func (p Gitlab) MigrationScripts() []plugin.MigrationScript {
return migrationscripts.All()
}
+func (p Gitlab) TestConnection(id uint64) errors.Error {
+ _, err := api.TestExistingConnection(helper.GenerateTestingConnectionApiResourceInput(id))
+ return err
+}
+
func (p Gitlab) ApiResources() map[string]map[string]plugin.ApiResourceHandler {
return map[string]map[string]plugin.ApiResourceHandler{
"test": {
diff --git a/backend/plugins/icla/impl/impl.go b/backend/plugins/icla/impl/impl.go
index 1f327f313..243074126 100644
--- a/backend/plugins/icla/impl/impl.go
+++ b/backend/plugins/icla/impl/impl.go
@@ -104,6 +104,10 @@ func (p Icla) ApiResources() map[string]map[string]plugin.ApiResourceHandler {
return nil
}
+func (p Icla) TestConnection(id uint64) errors.Error {
+ return nil
+}
+
func (p Icla) Close(taskCtx plugin.TaskContext) errors.Error {
data, ok := taskCtx.GetData().(*tasks.IclaTaskData)
if !ok {
diff --git a/backend/plugins/issue_trace/impl/enricher.go b/backend/plugins/issue_trace/impl/enricher.go
index 1506f2563..aeca45ade 100644
--- a/backend/plugins/issue_trace/impl/enricher.go
+++ b/backend/plugins/issue_trace/impl/enricher.go
@@ -19,7 +19,6 @@ package impl
import (
"encoding/json"
-
"github.com/apache/incubator-devlake/core/context"
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
@@ -151,6 +150,10 @@ func (p IssueTrace) GetTablesInfo() []dal.Tabler {
}
}
+func (p IssueTrace) TestConnection(id uint64) errors.Error {
+ return nil
+}
+
func (p IssueTrace) MakeMetricPluginPipelinePlanV200(projectName string, options json.RawMessage) (coreModels.PipelinePlan, errors.Error) {
op := &tasks.Options{}
if options != nil && string(options) != "\"\"" {
diff --git a/backend/plugins/jenkins/impl/impl.go b/backend/plugins/jenkins/impl/impl.go
index 2a686037f..5a7f478b6 100644
--- a/backend/plugins/jenkins/impl/impl.go
+++ b/backend/plugins/jenkins/impl/impl.go
@@ -177,6 +177,11 @@ func (p Jenkins) MakeDataSourcePipelinePlanV200(
return api.MakeDataSourcePipelinePlanV200(p.SubTaskMetas(), connectionId, scopes, skipCollectors)
}
+func (p Jenkins) TestConnection(id uint64) errors.Error {
+ _, err := api.TestExistingConnection(helper.GenerateTestingConnectionApiResourceInput(id))
+ return err
+}
+
func (p Jenkins) ApiResources() map[string]map[string]plugin.ApiResourceHandler {
return map[string]map[string]plugin.ApiResourceHandler{
"test": {
diff --git a/backend/plugins/jira/impl/impl.go b/backend/plugins/jira/impl/impl.go
index b7a62da20..51e623f32 100644
--- a/backend/plugins/jira/impl/impl.go
+++ b/backend/plugins/jira/impl/impl.go
@@ -285,6 +285,11 @@ func (p Jira) MigrationScripts() []plugin.MigrationScript {
return migrationscripts.All()
}
+func (p Jira) TestConnection(id uint64) errors.Error {
+ _, err := api.TestExistingConnection(helper.GenerateTestingConnectionApiResourceInput(id))
+ return err
+}
+
func (p Jira) ApiResources() map[string]map[string]plugin.ApiResourceHandler {
return map[string]map[string]plugin.ApiResourceHandler{
"test": {
diff --git a/backend/plugins/linker/impl/impl.go b/backend/plugins/linker/impl/impl.go
index 917b22df9..27b9a6cef 100644
--- a/backend/plugins/linker/impl/impl.go
+++ b/backend/plugins/linker/impl/impl.go
@@ -125,3 +125,7 @@ func (p Linker) MakeMetricPluginPipelinePlanV200(projectName string, options jso
}
return plan, nil
}
+
+func (p Linker) TestConnection(id uint64) errors.Error {
+ return nil
+}
diff --git a/backend/plugins/opsgenie/impl/impl.go b/backend/plugins/opsgenie/impl/impl.go
index 5785d53ff..7da8552e6 100644
--- a/backend/plugins/opsgenie/impl/impl.go
+++ b/backend/plugins/opsgenie/impl/impl.go
@@ -155,6 +155,11 @@ func (p Opsgenie) Close(taskCtx plugin.TaskContext) errors.Error {
return nil
}
+func (p Opsgenie) TestConnection(id uint64) errors.Error {
+ _, err := api.TestExistingConnection(helper.GenerateTestingConnectionApiResourceInput(id))
+ return err
+}
+
func (p Opsgenie) ApiResources() map[string]map[string]plugin.ApiResourceHandler {
return map[string]map[string]plugin.ApiResourceHandler{
"test": {
diff --git a/backend/plugins/org/impl/impl.go b/backend/plugins/org/impl/impl.go
index cd4ffa7fd..f0e6c4a0a 100644
--- a/backend/plugins/org/impl/impl.go
+++ b/backend/plugins/org/impl/impl.go
@@ -59,11 +59,33 @@ func (p Org) Name() string {
func (p Org) SubTaskMetas() []plugin.SubTaskMeta {
return []plugin.SubTaskMeta{
+ tasks.TaskCheckTokenMeta,
tasks.ConnectUserAccountsExactMeta,
tasks.SetProjectMappingMeta,
}
}
+func (p Org) MakePipeline(skipCollectors bool, projectName string, connections []plugin.ProjectTokenCheckerConnection) (coreModels.PipelinePlan, errors.Error) {
+ var plan coreModels.PipelinePlan
+ var stage coreModels.PipelineStage
+
+ // construct task options for Org
+ options := make(map[string]interface{})
+ if !skipCollectors {
+ options["projectConnections"] = connections
+ }
+
+ stage = append(stage, &coreModels.PipelineTask{
+ Plugin: p.Name(),
+ Subtasks: []string{
+ tasks.TaskCheckTokenMeta.Name,
+ },
+ Options: options,
+ })
+ plan = append(plan, stage)
+ return plan, nil
+}
+
func (p Org) MapProject(projectName string, scopes []plugin.Scope) (coreModels.PipelinePlan, errors.Error) {
var plan coreModels.PipelinePlan
var stage coreModels.PipelineStage
diff --git a/backend/plugins/org/tasks/check_token.go b/backend/plugins/org/tasks/check_token.go
new file mode 100644
index 000000000..f147fe8e6
--- /dev/null
+++ b/backend/plugins/org/tasks/check_token.go
@@ -0,0 +1,75 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "fmt"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
+ "golang.org/x/sync/errgroup"
+)
+
+var TaskCheckTokenMeta = plugin.SubTaskMeta{
+ Name: "checkTokens",
+ EntryPoint: checkProjectTokens,
+ EnabledByDefault: true,
+ Description: "set project mapping",
+ DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS},
+}
+
+func checkProjectTokens(taskCtx plugin.SubTaskContext) errors.Error {
+ logger := taskCtx.GetLogger()
+ taskData := taskCtx.GetData().(*TaskData)
+ connections := taskData.Options.ProjectConnections
+ logger.Debug("connections %+v", connections)
+ if len(connections) == 0 {
+ return nil
+ }
+
+ g := new(errgroup.Group)
+ for _, connection := range connections {
+ conn := connection
+ logger.Debug("check conn: %+v", conn)
+ g.Go(func() error {
+ if err := checkConnectionToken(conn.ConnectionId, conn.PluginName); err != nil {
+ return err
+ }
+ return nil
+ })
+ }
+ if err := g.Wait(); err != nil {
+ return errors.Convert(err)
+ }
+ return nil
+}
+
+func checkConnectionToken(connectionID uint64, pluginName string) errors.Error {
+ pluginEntry, err := plugin.GetPlugin(pluginName)
+ if err != nil {
+ return err
+ }
+ if v, ok := pluginEntry.(plugin.PluginTestConnectionAPI); ok {
+ if err := v.TestConnection(connectionID); err != nil {
+ return err
+ }
+ return nil
+ } else {
+ msg := fmt.Sprintf("plugin: %s doesn't impl test connection api", pluginName)
+ return errors.Default.New(msg)
+ }
+}
diff --git a/backend/plugins/org/tasks/task_data.go b/backend/plugins/org/tasks/task_data.go
index ca2a9b300..c92954740 100644
--- a/backend/plugins/org/tasks/task_data.go
+++ b/backend/plugins/org/tasks/task_data.go
@@ -20,8 +20,14 @@ package tasks
import "github.com/apache/incubator-devlake/core/plugin"
type Options struct {
- ConnectionId uint64 `json:"connectionId"`
- ProjectMappings []ProjectMapping `json:"projectMappings"`
+ ConnectionId uint64 `json:"connectionId"`
+ ProjectMappings []ProjectMapping `json:"projectMappings"`
+ ProjectConnections []ProjectConnection `json:"projectConnections"`
+}
+
+type ProjectConnection struct {
+ PluginName string
+ ConnectionId uint64
}
// ProjectMapping represents the relations between project and scopes
diff --git a/backend/plugins/pagerduty/impl/impl.go b/backend/plugins/pagerduty/impl/impl.go
index b237b30e4..827df8701 100644
--- a/backend/plugins/pagerduty/impl/impl.go
+++ b/backend/plugins/pagerduty/impl/impl.go
@@ -138,6 +138,11 @@ func (p PagerDuty) MigrationScripts() []plugin.MigrationScript {
return migrationscripts.All()
}
+func (p PagerDuty) TestConnection(id uint64) errors.Error {
+ _, err := api.TestExistingConnection(helper.GenerateTestingConnectionApiResourceInput(id))
+ return err
+}
+
func (p PagerDuty) ApiResources() map[string]map[string]plugin.ApiResourceHandler {
return map[string]map[string]plugin.ApiResourceHandler{
"test": {
diff --git a/backend/plugins/refdiff/impl/impl.go b/backend/plugins/refdiff/impl/impl.go
index b5572f643..e660b1424 100644
--- a/backend/plugins/refdiff/impl/impl.go
+++ b/backend/plugins/refdiff/impl/impl.go
@@ -110,3 +110,7 @@ func (p RefDiff) RootPkgPath() string {
func (p RefDiff) ApiResources() map[string]map[string]plugin.ApiResourceHandler {
return nil
}
+
+func (p RefDiff) TestConnection(id uint64) errors.Error {
+ return nil
+}
diff --git a/backend/plugins/slack/impl/impl.go b/backend/plugins/slack/impl/impl.go
index 8b3de9142..9ac894373 100644
--- a/backend/plugins/slack/impl/impl.go
+++ b/backend/plugins/slack/impl/impl.go
@@ -131,6 +131,11 @@ func (p Slack) MigrationScripts() []plugin.MigrationScript {
return migrationscripts.All()
}
+func (p Slack) TestConnection(id uint64) errors.Error {
+ _, err := api.TestExistingConnection(helper.GenerateTestingConnectionApiResourceInput(id))
+ return err
+}
+
func (p Slack) ApiResources() map[string]map[string]plugin.ApiResourceHandler {
return map[string]map[string]plugin.ApiResourceHandler{
"test": {
diff --git a/backend/plugins/sonarqube/impl/impl.go b/backend/plugins/sonarqube/impl/impl.go
index b6d214032..f0da89812 100644
--- a/backend/plugins/sonarqube/impl/impl.go
+++ b/backend/plugins/sonarqube/impl/impl.go
@@ -170,6 +170,11 @@ func (p Sonarqube) MigrationScripts() []plugin.MigrationScript {
return migrationscripts.All()
}
+func (p Sonarqube) TestConnection(id uint64) errors.Error {
+ _, err := api.TestExistingConnection(helper.GenerateTestingConnectionApiResourceInput(id))
+ return err
+}
+
func (p Sonarqube) ApiResources() map[string]map[string]plugin.ApiResourceHandler {
return map[string]map[string]plugin.ApiResourceHandler{
"test": {
diff --git a/backend/plugins/starrocks/impl/impl.go b/backend/plugins/starrocks/impl/impl.go
index 511b8c7ec..de036bace 100644
--- a/backend/plugins/starrocks/impl/impl.go
+++ b/backend/plugins/starrocks/impl/impl.go
@@ -67,3 +67,7 @@ func (s StarRocks) Name() string {
func (s StarRocks) RootPkgPath() string {
return "github.com/merico-dev/lake/plugins/starrocks"
}
+
+func (s StarRocks) TestConnection(id uint64) errors.Error {
+ return nil
+}
diff --git a/backend/plugins/tapd/impl/impl.go b/backend/plugins/tapd/impl/impl.go
index 2885a4337..2e133d821 100644
--- a/backend/plugins/tapd/impl/impl.go
+++ b/backend/plugins/tapd/impl/impl.go
@@ -265,6 +265,11 @@ func (p Tapd) MigrationScripts() []plugin.MigrationScript {
return migrationscripts.All()
}
+func (p Tapd) TestConnection(id uint64) errors.Error {
+ _, err := api.TestExistingConnection(helper.GenerateTestingConnectionApiResourceInput(id))
+ return err
+}
+
func (p Tapd) ApiResources() map[string]map[string]plugin.ApiResourceHandler {
return map[string]map[string]plugin.ApiResourceHandler{
"test": {
diff --git a/backend/plugins/teambition/impl/impl.go b/backend/plugins/teambition/impl/impl.go
index 662cace0f..728602d86 100644
--- a/backend/plugins/teambition/impl/impl.go
+++ b/backend/plugins/teambition/impl/impl.go
@@ -169,6 +169,11 @@ func (p Teambition) MigrationScripts() []plugin.MigrationScript {
return migrationscripts.All()
}
+func (p Teambition) TestConnection(id uint64) errors.Error {
+ _, err := api.TestExistingConnection(helper.GenerateTestingConnectionApiResourceInput(id))
+ return err
+}
+
func (p Teambition) ApiResources() map[string]map[string]plugin.ApiResourceHandler {
return map[string]map[string]plugin.ApiResourceHandler{
"test": {
diff --git a/backend/plugins/trello/impl/impl.go b/backend/plugins/trello/impl/impl.go
index b3a0f088d..fdcb10fdc 100644
--- a/backend/plugins/trello/impl/impl.go
+++ b/backend/plugins/trello/impl/impl.go
@@ -151,6 +151,11 @@ func (p Trello) ScopeConfig() dal.Tabler {
return &models.TrelloScopeConfig{}
}
+func (p Trello) TestConnection(id uint64) errors.Error {
+ _, err := api.TestExistingConnection(helper.GenerateTestingConnectionApiResourceInput(id))
+ return err
+}
+
func (p Trello) ApiResources() map[string]map[string]plugin.ApiResourceHandler {
return map[string]map[string]plugin.ApiResourceHandler{
"test": {
diff --git a/backend/plugins/webhook/impl/impl.go b/backend/plugins/webhook/impl/impl.go
index ada304783..90e1f0198 100644
--- a/backend/plugins/webhook/impl/impl.go
+++ b/backend/plugins/webhook/impl/impl.go
@@ -77,6 +77,10 @@ func (p Webhook) MigrationScripts() []plugin.MigrationScript {
return migrationscripts.All()
}
+func (p Webhook) TestConnection(id uint64) errors.Error {
+ return nil
+}
+
func (p Webhook) ApiResources() map[string]map[string]plugin.ApiResourceHandler {
return map[string]map[string]plugin.ApiResourceHandler{
"connections": {
diff --git a/backend/plugins/zentao/impl/impl.go b/backend/plugins/zentao/impl/impl.go
index 5aa74c966..55a98165d 100644
--- a/backend/plugins/zentao/impl/impl.go
+++ b/backend/plugins/zentao/impl/impl.go
@@ -277,6 +277,11 @@ func (p Zentao) MigrationScripts() []plugin.MigrationScript {
return migrationscripts.All()
}
+func (p Zentao) TestConnection(id uint64) errors.Error {
+ _, err := api.TestExistingConnection(helper.GenerateTestingConnectionApiResourceInput(id))
+ return err
+}
+
func (p Zentao) ApiResources() map[string]map[string]plugin.ApiResourceHandler {
return map[string]map[string]plugin.ApiResourceHandler{
"test": {
diff --git a/backend/server/services/blueprint.go b/backend/server/services/blueprint.go
index 4d106674d..ff88211dd 100644
--- a/backend/server/services/blueprint.go
+++ b/backend/server/services/blueprint.go
@@ -20,6 +20,9 @@ package services
import (
"encoding/json"
"fmt"
+ "github.com/apache/incubator-devlake/core/log"
+ "github.com/apache/incubator-devlake/core/plugin"
+ "golang.org/x/sync/errgroup"
"strings"
"sync"
@@ -419,6 +422,11 @@ func TriggerBlueprint(id uint64, triggerSyncPolicy *models.TriggerSyncPolicy, sh
}
blueprint.SkipCollectors = triggerSyncPolicy.SkipCollectors
blueprint.FullSync = triggerSyncPolicy.FullSync
+
+ if err := checkBlueprintTokens(blueprint, triggerSyncPolicy); err != nil {
+ return nil, errors.Default.Wrap(err, "check blue print tokens")
+ }
+
pipeline, err := createPipelineByBlueprint(blueprint, &models.SyncPolicy{
SkipOnFail: false,
TimeAfter: nil,
@@ -434,3 +442,64 @@ func TriggerBlueprint(id uint64, triggerSyncPolicy *models.TriggerSyncPolicy, sh
}
return pipeline, nil
}
+
+func needToCheckToken(triggerSyncPolicy *models.TriggerSyncPolicy) bool {
+ // case1: retransform
+ if triggerSyncPolicy.SkipCollectors && !triggerSyncPolicy.FullSync {
+ return false
+ }
+ // case2: collect data: triggerSyncPolicy.SkipCollectors == false && triggerSyncPolicy.FullSync == false
+ // case3: collect data with fullsync: triggerSyncPolicy.SkipCollectors == false && triggerSyncPolicy.FullSync == true
+ // case4: others
+ return true
+}
+
+func checkBlueprintTokens(blueprint *models.Blueprint, triggerSyncPolicy *models.TriggerSyncPolicy) errors.Error {
+ if blueprint == nil {
+ return errors.Default.New("blueprint is nil")
+ }
+ if triggerSyncPolicy == nil {
+ return errors.Default.New("triggerSyncPolicy is nil")
+ }
+
+ if !needToCheckToken(triggerSyncPolicy) {
+ return nil
+ }
+
+ if len(blueprint.Connections) == 0 {
+ return nil
+ }
+
+ g := new(errgroup.Group)
+ for _, connection := range blueprint.Connections {
+ conn := *connection
+ g.Go(func() error {
+ if err := checkConnectionToken(logger, conn); err != nil {
+ return err
+ }
+ return nil
+ })
+ }
+ if err := g.Wait(); err != nil {
+ return errors.Convert(err)
+ }
+
+ return nil
+}
+
+func checkConnectionToken(logger log.Logger, connection models.BlueprintConnection) errors.Error {
+ pluginEntry, err := plugin.GetPlugin(connection.PluginName)
+ if err != nil {
+ return err
+ }
+ if v, ok := pluginEntry.(plugin.PluginTestConnectionAPI); ok {
+ if err := v.TestConnection(connection.ConnectionId); err != nil {
+ logger.Error(err, "plugin: %s, id: %d", connection.PluginName, connection.ConnectionId)
+ return err
+ }
+ return nil
+ } else {
+ msg := fmt.Sprintf("plugin: %s doesn't impl test connection api", connection.PluginName)
+ return errors.Default.New(msg)
+ }
+}
diff --git a/backend/server/services/blueprint_makeplan_v200.go b/backend/server/services/blueprint_makeplan_v200.go
index 727e843de..960465069 100644
--- a/backend/server/services/blueprint_makeplan_v200.go
+++ b/backend/server/services/blueprint_makeplan_v200.go
@@ -121,11 +121,25 @@ func GeneratePlanJsonV200(
}
}
var planForProjectMapping coreModels.PipelinePlan
+ var planForProjectTokenChecker coreModels.PipelinePlan
if projectName != "" {
p, err := plugin.GetPlugin("org")
if err != nil {
return nil, errors.Default.Wrap(err, "get plugin org")
}
+ if pluginBp, ok := p.(plugin.ProjectTokenChecker); ok {
+ var simpleConns []plugin.ProjectTokenCheckerConnection
+ for _, connection := range connections {
+ simpleConns = append(simpleConns, plugin.ProjectTokenCheckerConnection{
+ PluginName: connection.PluginName,
+ ConnectionId: connection.ConnectionId,
+ })
+ }
+ planForProjectTokenChecker, err = pluginBp.MakePipeline(skipCollectors, projectName, simpleConns)
+ if err != nil {
+ return nil, errors.Default.Wrap(err, "org token checker make pipeline")
+ }
+ }
if pluginBp, ok := p.(plugin.ProjectMapper); ok {
planForProjectMapping, err = pluginBp.MapProject(projectName, scopes)
if err != nil {
@@ -134,6 +148,7 @@ func GeneratePlanJsonV200(
}
}
plan := SequentializePipelinePlans(
+ planForProjectTokenChecker,
planForProjectMapping,
ParallelizePipelinePlans(sourcePlans...),
ParallelizePipelinePlans(metricPlans...),
diff --git a/backend/server/services/project.go b/backend/server/services/project.go
index 98c9600a0..614f10680 100644
--- a/backend/server/services/project.go
+++ b/backend/server/services/project.go
@@ -219,7 +219,7 @@ func PatchProject(name string, body map[string]interface{}) (*models.ApiOutputPr
return nil, err
}
- // allowed to changed the name
+ // allowed to change the name
if projectInput.Name == "" {
projectInput.Name = name
}
diff --git a/backend/server/services/remote/models/plugin_remote.go b/backend/server/services/remote/models/plugin_remote.go
index 988283630..3c97d8021 100644
--- a/backend/server/services/remote/models/plugin_remote.go
+++ b/backend/server/services/remote/models/plugin_remote.go
@@ -30,4 +30,5 @@ type RemotePlugin interface {
plugin.PluginModel
plugin.PluginMigration
plugin.PluginSource
+ plugin.PluginTestConnectionAPI
}
diff --git a/backend/server/services/remote/plugin/plugin_impl.go b/backend/server/services/remote/plugin/plugin_impl.go
index 6242b1600..ea7e912c5 100644
--- a/backend/server/services/remote/plugin/plugin_impl.go
+++ b/backend/server/services/remote/plugin/plugin_impl.go
@@ -230,6 +230,11 @@ func (p *remotePluginImpl) ApiResources() map[string]map[string]plugin.ApiResour
return p.resources
}
+func (p *remotePluginImpl) TestConnection(id uint64) errors.Error {
+ _, err := p.resources["connections/:connectionId/test"]["POST"](api.GenerateTestingConnectionApiResourceInput(id))
+ return err
+}
+
func (p *remotePluginImpl) OpenApiSpec() string {
return p.openApiSpec
}
(incubator-devlake) 03/09: Dev 1 (#8100)
Posted by mi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mintsweet pushed a commit to branch feat-8017
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit 031db5b2ff64aa5a6969bcf1f9d9ca7d25b828f1
Author: Lynwee <15...@qq.com>
AuthorDate: Wed Sep 25 11:11:54 2024 +0800
Dev 1 (#8100)
* feat(project): add token check result in project check API
* feat(project): update token check API response
---
backend/core/models/project.go | 22 ++++++----------------
backend/server/services/project.go | 25 ++++++++++++-------------
2 files changed, 18 insertions(+), 29 deletions(-)
diff --git a/backend/core/models/project.go b/backend/core/models/project.go
index 5227d2a1c..6acafe27f 100644
--- a/backend/core/models/project.go
+++ b/backend/core/models/project.go
@@ -77,24 +77,14 @@ type ApiProjectCheck struct {
Tokens *ApiProjectCheckToken `json:"tokens,omitempty" mapstructure:"tokens"`
}
-type SuccessAndMessage struct {
- Success bool `json:"success" mapstructure:"success"`
- Message string `json:"message" mapstructure:"message"`
+type TokenResultSuccessAndMessage struct {
+ PluginName string `json:"pluginName" mapstructure:"pluginName"`
+ ConnectionID uint64 `json:"connectionId" mapstructure:"connectionId"`
+ Success bool `json:"success" mapstructure:"success"`
+ Message string `json:"message" mapstructure:"message"`
}
-// ApiProjectCheckToken
-//
-// {
-// "plugin_name":
-// {
-// "connection_id":
-// {
-// "success": true,
-// "message": ""
-// }
-// }
-// }
-type ApiProjectCheckToken = map[string]map[int]SuccessAndMessage
+type ApiProjectCheckToken []TokenResultSuccessAndMessage
type Store struct {
StoreKey string `gorm:"primaryKey;type:varchar(255)"`
diff --git a/backend/server/services/project.go b/backend/server/services/project.go
index 2e5c28aa8..96afb6bf4 100644
--- a/backend/server/services/project.go
+++ b/backend/server/services/project.go
@@ -198,25 +198,24 @@ func CheckProjectTokens(name string) (*models.ApiProjectCheckToken, errors.Error
if err != nil {
return nil, err
}
- ret := make(map[string]map[int]models.SuccessAndMessage)
+ var ret models.ApiProjectCheckToken
for _, connection := range blueprint.Connections {
pluginName := connection.PluginName
- connectionId := int(connection.ConnectionId)
- if _, ok := ret[pluginName]; !ok {
- ret[pluginName] = make(map[int]models.SuccessAndMessage)
- }
- connectionTokenResult := models.SuccessAndMessage{
- Success: true,
- Message: "success",
+ connectionId := connection.ConnectionId
+ connectionTokenResult := models.TokenResultSuccessAndMessage{
+ PluginName: pluginName,
+ ConnectionID: connectionId,
+ Success: true,
+ Message: "success",
}
if err := checkConnectionToken(logger, *connection); err != nil {
- ret[pluginName][connectionId] = models.SuccessAndMessage{
- Success: false,
- Message: err.Error(),
- }
+ connectionTokenResult.Success = false
+ connectionTokenResult.Message = err.Error()
+
}
- ret[pluginName][connectionId] = connectionTokenResult
+ ret = append(ret, connectionTokenResult)
}
+
return &ret, nil
}
(incubator-devlake) 07/09: feat(blueprint): add new api to check connection tokens (#8102)
Posted by mi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mintsweet pushed a commit to branch feat-8017
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit 626db747cc3d58ae0ab4e9c09ce52472b6e2c60f
Author: Lynwee <15...@qq.com>
AuthorDate: Wed Sep 25 13:22:41 2024 +0800
feat(blueprint): add new api to check connection tokens (#8102)
---
backend/core/models/blueprint.go | 9 +++++++++
backend/core/models/project.go | 12 +-----------
backend/server/api/blueprints/blueprints.go | 24 ++++++++++++++++++++++++
backend/server/api/project/project.go | 12 ------------
backend/server/api/router.go | 1 +
backend/server/services/blueprint.go | 24 ++++++++++++++++++++++++
backend/server/services/project.go | 26 --------------------------
7 files changed, 59 insertions(+), 49 deletions(-)
diff --git a/backend/core/models/blueprint.go b/backend/core/models/blueprint.go
index f30adec9f..9dc468822 100644
--- a/backend/core/models/blueprint.go
+++ b/backend/core/models/blueprint.go
@@ -92,3 +92,12 @@ type SyncPolicy struct {
TimeAfter *time.Time `json:"timeAfter"`
TriggerSyncPolicy
}
+
+type ConnectionTokenCheckResult struct {
+ PluginName string `json:"pluginName" mapstructure:"pluginName"`
+ ConnectionID uint64 `json:"connectionId" mapstructure:"connectionId"`
+ Success bool `json:"success" mapstructure:"success"`
+ Message string `json:"message" mapstructure:"message"`
+}
+
+type ApiBlueprintConnectionTokenCheck []ConnectionTokenCheckResult
diff --git a/backend/core/models/project.go b/backend/core/models/project.go
index 6acafe27f..99c61785d 100644
--- a/backend/core/models/project.go
+++ b/backend/core/models/project.go
@@ -73,19 +73,9 @@ type ApiOutputProject struct {
}
type ApiProjectCheck struct {
- Exist bool `json:"exist" mapstructure:"exist"`
- Tokens *ApiProjectCheckToken `json:"tokens,omitempty" mapstructure:"tokens"`
+ Exist bool `json:"exist" mapstructure:"exist"`
}
-type TokenResultSuccessAndMessage struct {
- PluginName string `json:"pluginName" mapstructure:"pluginName"`
- ConnectionID uint64 `json:"connectionId" mapstructure:"connectionId"`
- Success bool `json:"success" mapstructure:"success"`
- Message string `json:"message" mapstructure:"message"`
-}
-
-type ApiProjectCheckToken []TokenResultSuccessAndMessage
-
type Store struct {
StoreKey string `gorm:"primaryKey;type:varchar(255)"`
StoreValue json.RawMessage `gorm:"type:json;serializer:json"`
diff --git a/backend/server/api/blueprints/blueprints.go b/backend/server/api/blueprints/blueprints.go
index 23383d468..17376d2c8 100644
--- a/backend/server/api/blueprints/blueprints.go
+++ b/backend/server/api/blueprints/blueprints.go
@@ -18,6 +18,7 @@ limitations under the License.
package blueprints
import (
+ "github.com/spf13/cast"
"net/http"
"strconv"
@@ -226,6 +227,29 @@ func GetBlueprintPipelines(c *gin.Context) {
shared.ApiOutputSuccess(c, shared.ResponsePipelines{Pipelines: pipelines, Count: count}, http.StatusOK)
}
+// @Summary get connection token check by blueprint id
+// @Description get connection token check by blueprint id
+// @Tags framework/blueprints
+// @Accept application/json
+// @Param blueprintId path int true "blueprint id"
+// @Success 200 {object} models.ApiBlueprintConnectionTokenCheck
+// @Failure 400 {object} shared.ApiBody "Bad Request"
+// @Failure 500 {object} shared.ApiBody "Internal Error"
+// @Router /blueprints/{blueprintId}/connections-token-check [get]
+func GetBlueprintConnectionTokenCheck(c *gin.Context) {
+ blueprintId, err := cast.ToUint64E(c.Param("blueprintId"))
+ if err != nil {
+ shared.ApiOutputError(c, errors.BadInput.Wrap(err, "bad blueprintID format supplied"))
+ return
+ }
+ resp, err := services.CheckBlueprintConnectionTokens(blueprintId)
+ if err != nil {
+ shared.ApiOutputError(c, errors.Default.Wrap(err, "error getting blue print connection token check"))
+ return
+ }
+ shared.ApiOutputSuccess(c, resp, http.StatusOK)
+}
+
// @Summary delete blueprint by id
// @Description delete blueprint by id
// @Tags framework/blueprints
diff --git a/backend/server/api/project/project.go b/backend/server/api/project/project.go
index 43687da73..fd5c14ebb 100644
--- a/backend/server/api/project/project.go
+++ b/backend/server/api/project/project.go
@@ -58,14 +58,12 @@ func GetProject(c *gin.Context) {
// @Tags framework/projects
// @Accept application/json
// @Param projectName path string true "project name"
-// @Param check_token query int false "need to check token validity or not"
// @Success 200 {object} models.ApiProjectCheck
// @Failure 400 {string} errcode.Error "Bad Request"
// @Failure 500 {string} errcode.Error "Internal Error"
// @Router /projects/{projectName}/check [get]
func GetProjectCheck(c *gin.Context) {
projectName := c.Param("projectName")
-
projectOutputCheck := &models.ApiProjectCheck{}
_, err := services.GetProject(projectName)
if err != nil {
@@ -73,16 +71,6 @@ func GetProjectCheck(c *gin.Context) {
} else {
projectOutputCheck.Exist = true
}
-
- if c.Query("check_token") == "1" {
- checkTokenResult, err := services.CheckProjectTokens(projectName)
- if err != nil {
- shared.ApiOutputError(c, errors.Default.Wrap(err, "error check project tokens"))
- return
- }
- projectOutputCheck.Tokens = checkTokenResult
- }
-
shared.ApiOutputSuccess(c, projectOutputCheck, http.StatusOK) // //shared.ApiOutputSuccess(c, projectOutputCheck, http.StatusOK)
}
diff --git a/backend/server/api/router.go b/backend/server/api/router.go
index 721517cf5..226eb3035 100644
--- a/backend/server/api/router.go
+++ b/backend/server/api/router.go
@@ -59,6 +59,7 @@ func RegisterRouter(r *gin.Engine, basicRes context.BasicRes) {
r.GET("/blueprints/:blueprintId", blueprints.Get)
r.POST("/blueprints/:blueprintId/trigger", blueprints.Trigger)
r.GET("/blueprints/:blueprintId/pipelines", blueprints.GetBlueprintPipelines)
+ r.GET("/blueprints/:blueprintId/connections-token-check", blueprints.GetBlueprintConnectionTokenCheck)
r.POST("/tasks/:taskId/rerun", task.PostRerun)
diff --git a/backend/server/services/blueprint.go b/backend/server/services/blueprint.go
index ff88211dd..aced9eeea 100644
--- a/backend/server/services/blueprint.go
+++ b/backend/server/services/blueprint.go
@@ -117,6 +117,30 @@ func GetBlueprint(blueprintId uint64, shouldSanitize bool) (*models.Blueprint, e
return blueprint, nil
}
+func CheckBlueprintConnectionTokens(blueprintId uint64) (*models.ApiBlueprintConnectionTokenCheck, errors.Error) {
+ blueprint, err := GetBlueprint(blueprintId, false)
+ if err != nil {
+ return nil, err
+ }
+ var ret models.ApiBlueprintConnectionTokenCheck
+ for _, connection := range blueprint.Connections {
+ pluginName := connection.PluginName
+ connectionId := connection.ConnectionId
+ connectionTokenResult := models.ConnectionTokenCheckResult{
+ PluginName: pluginName,
+ ConnectionID: connectionId,
+ Success: true,
+ Message: "success",
+ }
+ if err := checkConnectionToken(logger, *connection); err != nil {
+ connectionTokenResult.Success = false
+ connectionTokenResult.Message = err.Error()
+ }
+ ret = append(ret, connectionTokenResult)
+ }
+ return &ret, nil
+}
+
// GetBlueprintByProjectName returns the detail of a given ProjectName
func GetBlueprintByProjectName(projectName string) (*models.Blueprint, errors.Error) {
if projectName == "" {
diff --git a/backend/server/services/project.go b/backend/server/services/project.go
index 96afb6bf4..614f10680 100644
--- a/backend/server/services/project.go
+++ b/backend/server/services/project.go
@@ -193,32 +193,6 @@ func GetProject(name string) (*models.ApiOutputProject, errors.Error) {
return makeProjectOutput(project, false)
}
-func CheckProjectTokens(name string) (*models.ApiProjectCheckToken, errors.Error) {
- blueprint, err := GetBlueprintByProjectName(name)
- if err != nil {
- return nil, err
- }
- var ret models.ApiProjectCheckToken
- for _, connection := range blueprint.Connections {
- pluginName := connection.PluginName
- connectionId := connection.ConnectionId
- connectionTokenResult := models.TokenResultSuccessAndMessage{
- PluginName: pluginName,
- ConnectionID: connectionId,
- Success: true,
- Message: "success",
- }
- if err := checkConnectionToken(logger, *connection); err != nil {
- connectionTokenResult.Success = false
- connectionTokenResult.Message = err.Error()
-
- }
- ret = append(ret, connectionTokenResult)
- }
-
- return &ret, nil
-}
-
// PatchProject FIXME ...
func PatchProject(name string, body map[string]interface{}) (*models.ApiOutputProject, errors.Error) {
projectInput := &models.ApiInputProject{}
(incubator-devlake) 04/09: Dev 1 (#8101)
Posted by mi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mintsweet pushed a commit to branch feat-8017
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit 2c0b3f5cd6531997e8aa934bb61efe541d0d9049
Author: Lynwee <15...@qq.com>
AuthorDate: Wed Sep 25 11:22:24 2024 +0800
Dev 1 (#8101)
* feat(project): add token check result in project check API
* feat(project): update token check API response
* fix(project): fix api doc
* fix(project): update project check api doc
---
backend/server/api/project/project.go | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/backend/server/api/project/project.go b/backend/server/api/project/project.go
index b4a14fff5..08e70c626 100644
--- a/backend/server/api/project/project.go
+++ b/backend/server/api/project/project.go
@@ -53,12 +53,12 @@ func GetProject(c *gin.Context) {
shared.ApiOutputSuccess(c, projectOutput, http.StatusOK)
}
-// @Summary Get project exist check
-// @Description Get project exist check
+// @Summary Get project related check
+// @Description Get project related check info, such existence, token validity
// @Tags framework/projects
// @Accept application/json
// @Param projectName path string true "project name"
-// @Success 200 {object} models.ApiOutputProject
+// @Success 200 {object} models.ApiProjectCheck
// @Failure 400 {string} errcode.Error "Bad Request"
// @Failure 500 {string} errcode.Error "Internal Error"
// @Router /projects/{projectName}/check [get]