You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by ma...@apache.org on 2023/03/02 07:03:36 UTC
[incubator-devlake] branch main updated: fix(gitlab): delete account detail collector and extractor (#4565)
This is an automated email from the ASF dual-hosted git repository.
mappjzc pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 6a29440b4 fix(gitlab): delete account detail collector and extractor (#4565)
6a29440b4 is described below
commit 6a29440b42d9d6d52e3d6fac185005864dfff8d7
Author: Warren Chen <yi...@merico.dev>
AuthorDate: Thu Mar 2 15:03:32 2023 +0800
fix(gitlab): delete account detail collector and extractor (#4565)
---
backend/plugins/gitlab/impl/impl.go | 2 -
.../gitlab/tasks/account_detail_collector.go | 117 ---------------------
.../gitlab/tasks/account_detail_extractor.go | 78 --------------
3 files changed, 197 deletions(-)
diff --git a/backend/plugins/gitlab/impl/impl.go b/backend/plugins/gitlab/impl/impl.go
index 9bcd8abf2..a5949aa49 100644
--- a/backend/plugins/gitlab/impl/impl.go
+++ b/backend/plugins/gitlab/impl/impl.go
@@ -114,8 +114,6 @@ func (p Gitlab) SubTaskMetas() []plugin.SubTaskMeta {
tasks.EnrichMergeRequestsMeta,
tasks.CollectAccountsMeta,
tasks.ExtractAccountsMeta,
- tasks.CollectAccountDetailsMeta,
- tasks.ExtractAccountDetailsMeta,
tasks.ConvertAccountsMeta,
tasks.ConvertProjectMeta,
tasks.ConvertApiMergeRequestsMeta,
diff --git a/backend/plugins/gitlab/tasks/account_detail_collector.go b/backend/plugins/gitlab/tasks/account_detail_collector.go
deleted file mode 100644
index c10d96450..000000000
--- a/backend/plugins/gitlab/tasks/account_detail_collector.go
+++ /dev/null
@@ -1,117 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package tasks
-
-import (
- "net/url"
- "reflect"
-
- "github.com/apache/incubator-devlake/core/dal"
- "github.com/apache/incubator-devlake/core/errors"
- "github.com/apache/incubator-devlake/core/plugin"
- "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
- "github.com/apache/incubator-devlake/plugins/gitlab/models"
- "golang.org/x/mod/semver"
-)
-
-const RAW_USER_DETAIL_TABLE = "gitlab_api_user_details"
-
-var CollectAccountDetailsMeta = plugin.SubTaskMeta{
- Name: "collectAccountDetails",
- EntryPoint: CollectAccountDetails,
- EnabledByDefault: true,
- Description: "collect gitlab user details",
- DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS},
-}
-
-func CollectAccountDetails(taskCtx plugin.SubTaskContext) errors.Error {
-
- rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_USER_DETAIL_TABLE)
- logger := taskCtx.GetLogger()
- logger.Info("collect gitlab user details")
-
- if !NeedAccountDetails(data.ApiClient) {
- logger.Info("Don't need collect gitlab user details,skip")
- return nil
- }
-
- iterator, err := GetAccountsIterator(taskCtx)
- if err != nil {
- return err
- }
- defer iterator.Close()
-
- collector, err := api.NewApiCollector(api.ApiCollectorArgs{
- RawDataSubTaskArgs: *rawDataSubTaskArgs,
- ApiClient: data.ApiClient,
- UrlTemplate: "/projects/{{ .Params.ProjectId }}/members/{{ .Input.GitlabId }}",
- Input: iterator,
- //PageSize: 100,
- Query: func(reqData *api.RequestData) (url.Values, errors.Error) {
- query := url.Values{}
- // query.Set("sort", "asc")
- // query.Set("page", fmt.Sprintf("%v", reqData.Pager.Page))
- // query.Set("per_page", fmt.Sprintf("%v", reqData.Pager.Size))
- return query, nil
- },
-
- ResponseParser: GetOneRawMessageFromResponse,
- })
-
- if err != nil {
- logger.Error(err, "collect user error")
- return err
- }
-
- return collector.Execute()
-}
-
-// checking if we need detail data
-func NeedAccountDetails(apiClient *api.ApiAsyncClient) bool {
- if apiClient == nil {
- return false
- }
-
- if version, ok := apiClient.GetData(models.GitlabApiClientData_ApiVersion).(string); ok {
- if semver.Compare(version, "v13.11") < 0 && version != "" {
- return true
- }
- }
-
- return false
-}
-
-func GetAccountsIterator(taskCtx plugin.SubTaskContext) (*api.DalCursorIterator, errors.Error) {
- db := taskCtx.GetDal()
- data := taskCtx.GetData().(*GitlabTaskData)
- clauses := []dal.Clause{
- dal.Select("ga.gitlab_id,ga.gitlab_id as iid"),
- dal.From("_tool_gitlab_accounts ga"),
- dal.Where(
- `ga.connection_id = ?`,
- data.Options.ConnectionId,
- ),
- }
- // construct the input iterator
- cursor, err := db.Cursor(clauses...)
- if err != nil {
- return nil, err
- }
-
- return api.NewDalCursorIterator(db, cursor, reflect.TypeOf(GitlabInput{}))
-}
diff --git a/backend/plugins/gitlab/tasks/account_detail_extractor.go b/backend/plugins/gitlab/tasks/account_detail_extractor.go
deleted file mode 100644
index e16616232..000000000
--- a/backend/plugins/gitlab/tasks/account_detail_extractor.go
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package tasks
-
-import (
- "encoding/json"
-
- "github.com/apache/incubator-devlake/core/errors"
- "github.com/apache/incubator-devlake/core/plugin"
- "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
- "github.com/apache/incubator-devlake/plugins/gitlab/models"
-)
-
-var ExtractAccountDetailsMeta = plugin.SubTaskMeta{
- Name: "extractAccountDetails",
- EntryPoint: ExtractAccountDetails,
- EnabledByDefault: true,
- Description: "Extract detail raw workspace data into tool layer table _tool_gitlab_accounts",
- DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS},
-}
-
-func ExtractAccountDetails(taskCtx plugin.SubTaskContext) errors.Error {
- rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_USER_DETAIL_TABLE)
-
- logger := taskCtx.GetLogger()
- logger.Info("Extract gitlab user details")
-
- if !NeedAccountDetails(data.ApiClient) {
- logger.Info("Don't need Extract gitlab user details,skip")
- return nil
- }
-
- extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{
- RawDataSubTaskArgs: *rawDataSubTaskArgs,
- Extract: func(row *api.RawData) ([]interface{}, errors.Error) {
- var userRes models.GitlabAccount
- err := errors.Convert(json.Unmarshal(row.Data, &userRes))
- if err != nil {
- return nil, err
- }
-
- results := make([]interface{}, 0)
- GitlabAccount := &models.GitlabAccount{
- ConnectionId: data.Options.ConnectionId,
- GitlabId: userRes.GitlabId,
- Username: userRes.Username,
- Name: userRes.Name,
- State: userRes.State,
- MembershipState: userRes.MembershipState,
- AvatarUrl: userRes.AvatarUrl,
- WebUrl: userRes.WebUrl,
- }
- results = append(results, GitlabAccount)
- return results, nil
- },
- })
-
- if err != nil {
- return err
- }
-
- return extractor.Execute()
-}