You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by "likyh (via GitHub)" <gi...@apache.org> on 2023/02/09 16:00:29 UTC
[GitHub] [incubator-devlake] likyh opened a new pull request, #4373: feat: add increment & timeFilter for bitbucket
likyh opened a new pull request, #4373:
URL: https://github.com/apache/incubator-devlake/pull/4373
### Summary
What does this PR do?
### Does this close any open issues?
Closes xx
### Screenshots
CURD for transformationRule:
### Other Information
Any other information that is important to this PR.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@devlake.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [incubator-devlake] warren830 commented on a diff in pull request #4373: feat: add increment & timeFilter for bitbucket
Posted by "warren830 (via GitHub)" <gi...@apache.org>.
warren830 commented on code in PR #4373:
URL: https://github.com/apache/incubator-devlake/pull/4373#discussion_r1106882781
##########
backend/plugins/bitbucket/api/blueprint_v200.go:
##########
@@ -0,0 +1,198 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package api
+
+import (
+ "fmt"
+ "github.com/apache/incubator-devlake/core/dal"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/models/domainlayer"
+ "github.com/apache/incubator-devlake/core/models/domainlayer/code"
+ "github.com/apache/incubator-devlake/core/models/domainlayer/devops"
+ "github.com/apache/incubator-devlake/core/models/domainlayer/didgen"
+ "github.com/apache/incubator-devlake/core/models/domainlayer/ticket"
+ "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/core/utils"
+ helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "github.com/apache/incubator-devlake/plugins/bitbucket/models"
+ "github.com/apache/incubator-devlake/plugins/bitbucket/tasks"
+ "github.com/go-playground/validator/v10"
+ "net/url"
+ "strings"
+ "time"
+)
+
+func MakeDataSourcePipelinePlanV200(subtaskMetas []plugin.SubTaskMeta, connectionId uint64, bpScopes []*plugin.BlueprintScopeV200, syncPolicy *plugin.BlueprintSyncPolicy) (plugin.PipelinePlan, []plugin.Scope, errors.Error) {
+ connectionHelper := helper.NewConnectionHelper(basicRes, validator.New())
+ // get the connection info for url
+ connection := &models.BitbucketConnection{}
+ err := connectionHelper.FirstById(connection, connectionId)
+ if err != nil {
+ return nil, nil, err
+ }
+
+ plan := make(plugin.PipelinePlan, len(bpScopes))
+ plan, err = makeDataSourcePipelinePlanV200(subtaskMetas, plan, bpScopes, connection, syncPolicy)
+ if err != nil {
+ return nil, nil, err
+ }
+ scopes, err := makeScopesV200(bpScopes, connection)
+ if err != nil {
+ return nil, nil, err
+ }
+
+ return plan, scopes, nil
+}
+
+func makeDataSourcePipelinePlanV200(
+ subtaskMetas []plugin.SubTaskMeta,
+ plan plugin.PipelinePlan,
+ bpScopes []*plugin.BlueprintScopeV200,
+ connection *models.BitbucketConnection,
+ syncPolicy *plugin.BlueprintSyncPolicy,
+) (plugin.PipelinePlan, errors.Error) {
+ var err errors.Error
+ for i, bpScope := range bpScopes {
+ stage := plan[i]
+ if stage == nil {
+ stage = plugin.PipelineStage{}
+ }
+ repo := &models.BitbucketRepo{}
+ // get repo from db
+ err = basicRes.GetDal().First(repo, dal.Where(`connection_id = ? AND bitbucket_id = ?`, connection.ID, bpScope.Id))
+ if err != nil {
+ return nil, errors.Default.Wrap(err, fmt.Sprintf("fail to find repo %s", bpScope.Id))
+ }
+ transformationRule := &models.BitbucketTransformationRule{}
+ // get transformation rules from db
+ db := basicRes.GetDal()
+ err = db.First(transformationRule, dal.Where(`id = ?`, repo.TransformationRuleId))
+ if err != nil && !db.IsErrorNotFound(err) {
+ return nil, err
+ }
+ // refdiff
+ if transformationRule != nil && transformationRule.Refdiff != nil {
+ // add a new task to next stage
+ j := i + 1
+ if j == len(plan) {
+ plan = append(plan, nil)
+ }
+ refdiffOp := transformationRule.Refdiff
+ refdiffOp["repoId"] = didgen.NewDomainIdGenerator(&models.BitbucketRepo{}).Generate(connection.ID, repo.BitbucketId)
+ plan[j] = plugin.PipelineStage{
+ {
+ Plugin: "refdiff",
+ Options: refdiffOp,
+ },
+ }
+ transformationRule.Refdiff = nil
+ }
+
+ // construct task options for bitbucket
+ op := &tasks.BitbucketOptions{
+ ConnectionId: repo.ConnectionId,
+ FullName: repo.BitbucketId,
+ }
+ if syncPolicy.CreatedDateAfter != nil {
+ op.CreatedDateAfter = syncPolicy.CreatedDateAfter.Format(time.RFC3339)
+ }
+ options, err := tasks.EncodeTaskOptions(op)
+ if err != nil {
+ return nil, err
+ }
+
+ subtasks, err := helper.MakePipelinePlanSubtasks(subtaskMetas, bpScope.Entities)
+ if err != nil {
+ return nil, err
+ }
+ stage = append(stage, &plugin.PipelineTask{
+ Plugin: "bitbucket",
+ Subtasks: subtasks,
+ Options: options,
+ })
+ if err != nil {
+ return nil, err
+ }
+
+ // add gitex stage
+ if utils.StringsContains(bpScope.Entities, plugin.DOMAIN_TYPE_CODE) {
+ cloneUrl, err := errors.Convert01(url.Parse(repo.CloneUrl))
+ if err != nil {
+ return nil, err
+ }
+ token := strings.Split(connection.Password, ",")[0]
Review Comment:
Can we use multiple tokens in Bitbucket?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@devlake.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [incubator-devlake] likyh commented on a diff in pull request #4373: feat: add increment & timeFilter for bitbucket
Posted by "likyh (via GitHub)" <gi...@apache.org>.
likyh commented on code in PR #4373:
URL: https://github.com/apache/incubator-devlake/pull/4373#discussion_r1106608383
##########
backend/plugins/bitbucket/bitbucket.go:
##########
@@ -30,17 +30,17 @@ var PluginEntry impl.Bitbucket //nolint
func main() {
cmd := &cobra.Command{Use: "bitbucket"}
connectionId := cmd.Flags().Uint64P("connectionId", "c", 0, "bitbucket connection id")
- owner := cmd.Flags().StringP("owner", "o", "", "bitbucket owner")
- repo := cmd.Flags().StringP("repo", "r", "", "bitbucket repo")
+ fullName := cmd.Flags().StringP("fullName", "n", "", "bitbucket id: owner/repo")
Review Comment:
use one column for options.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@devlake.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [incubator-devlake] likyh commented on a diff in pull request #4373: feat: add increment & timeFilter for bitbucket
Posted by "likyh (via GitHub)" <gi...@apache.org>.
likyh commented on code in PR #4373:
URL: https://github.com/apache/incubator-devlake/pull/4373#discussion_r1106609648
##########
backend/plugins/bitbucket/tasks/account_convertor.go:
##########
@@ -53,17 +53,9 @@ func ConvertAccounts(taskCtx plugin.SubTaskContext) errors.Error {
accountIdGen := didgen.NewDomainIdGenerator(&bitbucketModels.BitbucketAccount{})
converter, err := api.NewDataConverter(api.DataConverterArgs{
- InputRowType: reflect.TypeOf(bitbucketModels.BitbucketAccount{}),
- Input: cursor,
- RawDataSubTaskArgs: api.RawDataSubTaskArgs{
Review Comment:
replace with CreateRawDataSubTaskArgs
##########
backend/plugins/bitbucket/models/migrationscripts/20230206_add_scope_and_transformation.go:
##########
@@ -0,0 +1,68 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+ "github.com/apache/incubator-devlake/core/context"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/helpers/migrationhelper"
+ "github.com/apache/incubator-devlake/plugins/bitbucket/models/migrationscripts/archived"
+)
+
+type BitbucketRepo20230206 struct {
+ TransformationRuleId uint64 `json:"transformationRuleId,omitempty" mapstructure:"transformationRuleId,omitempty"`
+ CloneUrl string `json:"cloneUrl" gorm:"type:varchar(255)" mapstructure:"cloneUrl,omitempty"`
+ Owner string `json:"owner" mapstructure:"owner,omitempty"`
Review Comment:
rename owner_id to owner. because other `XXX_id` are uuid.
##########
backend/plugins/bitbucket/tasks/deployment_extractor.go:
##########
@@ -27,19 +27,18 @@ import (
)
type bitbucketApiDeploymentsResponse struct {
- Type string `json:"type"`
- Number int `json:"number"`
- UUID string `json:"uuid"`
- Key string `json:"key"`
- Step struct {
- UUID string `json:"uuid"`
- } `json:"step"`
- Environment struct {
- UUID string `json:"uuid"`
- } `json:"environment"`
+ Type string `json:"type"`
+ UUID string `json:"uuid"`
+ //Key string `json:"key"`
Review Comment:
delete because the fields column in collector
##########
backend/plugins/bitbucket/tasks/deployment_collector.go:
##########
@@ -41,12 +41,14 @@ func CollectApiDeployments(taskCtx plugin.SubTaskContext) errors.Error {
ApiClient: data.ApiClient,
PageSize: 50,
Incremental: false,
- UrlTemplate: "repositories/{{ .Params.Owner }}/{{ .Params.Repo }}/deployments/",
- Query: GetQuery,
- ResponseParser: GetRawMessageFromResponse,
- GetTotalPages: GetTotalPagesFromResponse,
+ UrlTemplate: "repositories/{{ .Params.FullName }}/deployments/",
+ Query: GetQueryFields(`values.type,values.uuid,` +
+ `values.release.pipeline,values.release.key,values.release.name,values.release.url,values.release.created_on,` +
+ `values.release.commit.hash,values.release.commit.links.html,` +
+ `values.state.name,values.state.url,values.state.started_on,values.state.completed_on,values.last_update_time`),
Review Comment:
add fields to reduce response body to 30%. But it's no use in collect speed.
##########
backend/plugins/gitlab/api/remote.go:
##########
@@ -250,7 +250,7 @@ func RemoteScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, er
// @Tags plugins/gitlab
// @Accept application/json
// @Param connectionId path int false "connection ID"
-// @Param search query string false "group ID"
+// @Param search query string false "search"
Review Comment:
typo
##########
backend/plugins/bitbucket/models/migrationscripts/20230206_add_scope_and_transformation.go:
##########
@@ -0,0 +1,68 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+ "github.com/apache/incubator-devlake/core/context"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/helpers/migrationhelper"
+ "github.com/apache/incubator-devlake/plugins/bitbucket/models/migrationscripts/archived"
+)
+
+type BitbucketRepo20230206 struct {
+ TransformationRuleId uint64 `json:"transformationRuleId,omitempty" mapstructure:"transformationRuleId,omitempty"`
+ CloneUrl string `json:"cloneUrl" gorm:"type:varchar(255)" mapstructure:"cloneUrl,omitempty"`
Review Comment:
record clone url to add gitextractor easily.
##########
backend/plugins/bitbucket/api/remote.go:
##########
@@ -0,0 +1,371 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package api
+
+import (
+ "context"
+ "encoding/base64"
+ "encoding/json"
+ "fmt"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "github.com/apache/incubator-devlake/plugins/bitbucket/models"
+ "github.com/apache/incubator-devlake/plugins/bitbucket/tasks"
+ "net/http"
+ "net/url"
+ "strconv"
+ "strings"
+)
+
+type RemoteScopesChild struct {
+ Type string `json:"type"`
+ ParentId *string `json:"parentId"`
+ Id string `json:"id"`
+ Name string `json:"name"`
+ Data interface{} `json:"data"`
+}
+
+type RemoteScopesOutput struct {
+ Children []RemoteScopesChild `json:"children"`
+ NextPageToken string `json:"nextPageToken"`
+}
+
+type SearchRemoteScopesOutput struct {
+ Children []RemoteScopesChild `json:"children"`
+ Page int `json:"page"`
+ PageSize int `json:"pageSize"`
+}
+
+type PageData struct {
+ Page int `json:"page"`
+ PerPage int `json:"per_page"`
+}
+
+type WorkspaceResponse struct {
+ Pagelen int `json:"pagelen"`
+ Page int `json:"page"`
+ Size int `json:"size"`
+ Values []struct {
+ //Type string `json:"type"`
+ //Permission string `json:"permission"`
+ //LastAccessed time.Time `json:"last_accessed"`
+ //AddedOn time.Time `json:"added_on"`
+ Workspace WorkspaceItem `json:"workspace"`
+ } `json:"values"`
+}
+
+type WorkspaceItem struct {
+ //Type string `json:"type"`
+ //Uuid string `json:"uuid"`
+ Slug string `json:"slug"`
+ Name string `json:"name"`
+}
+
+type ReposResponse struct {
+ Pagelen int `json:"pagelen"`
+ Page int `json:"page"`
+ Size int `json:"size"`
+ Values []tasks.BitbucketApiRepo `json:"values"`
+}
+
+const RemoteScopesPerPage int = 100
+const TypeScope string = "scope"
+const TypeGroup string = "group"
+
+// RemoteScopes list all available scope for users
+// @Summary list all available scope for users
+// @Description list all available scope for users
+// @Tags plugins/bitbucket
+// @Accept application/json
+// @Param connectionId path int false "connection ID"
+// @Param groupId query string false "group ID"
+// @Param pageToken query string false "page Token"
+// @Success 200 {object} RemoteScopesOutput
+// @Failure 400 {object} shared.ApiBody "Bad Request"
+// @Failure 500 {object} shared.ApiBody "Internal Error"
+// @Router /plugins/bitbucket/connections/{connectionId}/remote-scopes [GET]
+func RemoteScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
+ connectionId, _ := extractParam(input.Params)
+ if connectionId == 0 {
+ return nil, errors.BadInput.New("invalid connectionId")
+ }
+
+ connection := &models.BitbucketConnection{}
+ err := connectionHelper.First(connection, input.Params)
+ if err != nil {
+ return nil, err
+ }
+
+ groupId, ok := input.Query["groupId"]
+ if !ok || len(groupId) == 0 {
+ groupId = []string{""}
+ }
+
+ pageToken, ok := input.Query["pageToken"]
+ if !ok || len(pageToken) == 0 {
+ pageToken = []string{""}
+ }
+
+ // get gid and pageData
+ gid := groupId[0]
+ pageData, err := GetPageDataFromPageToken(pageToken[0])
+ if err != nil {
+ return nil, errors.BadInput.New("failed to get page token")
+ }
+
+ // create api client
+ apiClient, err := api.NewApiClientFromConnection(context.TODO(), basicRes, connection)
+ if err != nil {
+ return nil, err
+ }
+
+ query, err := GetQueryFromPageData(pageData)
+ if err != nil {
+ return nil, err
+ }
+
+ var res *http.Response
+ outputBody := &RemoteScopesOutput{}
+
+ // list groups part
+ if gid == "" {
+ query.Set("sort", "workspace.slug")
+ query.Set("fields", "values.workspace.slug,values.workspace.name,pagelen,page,size")
+ res, err = apiClient.Get("/user/permissions/workspaces", query, nil)
+ if err != nil {
+ return nil, err
+ }
+
+ resBody := &WorkspaceResponse{}
+ err = api.UnmarshalResponse(res, resBody)
+ if err != nil {
+ return nil, err
+ }
+
+ // append group to output
+ for _, group := range resBody.Values {
+ child := RemoteScopesChild{
+ Type: TypeGroup,
+ Id: group.Workspace.Slug,
+ Name: group.Workspace.Name,
+ // don't need to save group into data
+ Data: nil,
+ }
+ if gid != "" {
Review Comment:
fixed
##########
backend/plugins/bitbucket/api/remote.go:
##########
@@ -0,0 +1,371 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package api
+
+import (
+ "context"
+ "encoding/base64"
+ "encoding/json"
+ "fmt"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "github.com/apache/incubator-devlake/plugins/bitbucket/models"
+ "github.com/apache/incubator-devlake/plugins/bitbucket/tasks"
+ "net/http"
+ "net/url"
+ "strconv"
+ "strings"
+)
+
+type RemoteScopesChild struct {
+ Type string `json:"type"`
+ ParentId *string `json:"parentId"`
+ Id string `json:"id"`
+ Name string `json:"name"`
+ Data interface{} `json:"data"`
+}
+
+type RemoteScopesOutput struct {
+ Children []RemoteScopesChild `json:"children"`
+ NextPageToken string `json:"nextPageToken"`
+}
+
+type SearchRemoteScopesOutput struct {
+ Children []RemoteScopesChild `json:"children"`
+ Page int `json:"page"`
+ PageSize int `json:"pageSize"`
+}
+
+type PageData struct {
+ Page int `json:"page"`
+ PerPage int `json:"per_page"`
+}
+
+type WorkspaceResponse struct {
+ Pagelen int `json:"pagelen"`
+ Page int `json:"page"`
+ Size int `json:"size"`
+ Values []struct {
+ //Type string `json:"type"`
+ //Permission string `json:"permission"`
+ //LastAccessed time.Time `json:"last_accessed"`
+ //AddedOn time.Time `json:"added_on"`
+ Workspace WorkspaceItem `json:"workspace"`
+ } `json:"values"`
+}
+
+type WorkspaceItem struct {
+ //Type string `json:"type"`
+ //Uuid string `json:"uuid"`
+ Slug string `json:"slug"`
+ Name string `json:"name"`
+}
+
+type ReposResponse struct {
+ Pagelen int `json:"pagelen"`
+ Page int `json:"page"`
+ Size int `json:"size"`
+ Values []tasks.BitbucketApiRepo `json:"values"`
+}
+
+const RemoteScopesPerPage int = 100
+const TypeScope string = "scope"
+const TypeGroup string = "group"
+
+// RemoteScopes list all available scope for users
+// @Summary list all available scope for users
+// @Description list all available scope for users
+// @Tags plugins/bitbucket
+// @Accept application/json
+// @Param connectionId path int false "connection ID"
+// @Param groupId query string false "group ID"
+// @Param pageToken query string false "page Token"
+// @Success 200 {object} RemoteScopesOutput
+// @Failure 400 {object} shared.ApiBody "Bad Request"
+// @Failure 500 {object} shared.ApiBody "Internal Error"
+// @Router /plugins/bitbucket/connections/{connectionId}/remote-scopes [GET]
+func RemoteScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
+ connectionId, _ := extractParam(input.Params)
+ if connectionId == 0 {
+ return nil, errors.BadInput.New("invalid connectionId")
+ }
+
+ connection := &models.BitbucketConnection{}
+ err := connectionHelper.First(connection, input.Params)
+ if err != nil {
+ return nil, err
+ }
+
+ groupId, ok := input.Query["groupId"]
+ if !ok || len(groupId) == 0 {
+ groupId = []string{""}
+ }
+
+ pageToken, ok := input.Query["pageToken"]
+ if !ok || len(pageToken) == 0 {
+ pageToken = []string{""}
+ }
+
+ // get gid and pageData
+ gid := groupId[0]
+ pageData, err := GetPageDataFromPageToken(pageToken[0])
+ if err != nil {
+ return nil, errors.BadInput.New("failed to get page token")
+ }
+
+ // create api client
+ apiClient, err := api.NewApiClientFromConnection(context.TODO(), basicRes, connection)
+ if err != nil {
+ return nil, err
+ }
+
+ query, err := GetQueryFromPageData(pageData)
+ if err != nil {
+ return nil, err
+ }
+
+ var res *http.Response
+ outputBody := &RemoteScopesOutput{}
+
+ // list groups part
+ if gid == "" {
+ query.Set("sort", "workspace.slug")
+ query.Set("fields", "values.workspace.slug,values.workspace.name,pagelen,page,size")
+ res, err = apiClient.Get("/user/permissions/workspaces", query, nil)
+ if err != nil {
+ return nil, err
+ }
+
+ resBody := &WorkspaceResponse{}
+ err = api.UnmarshalResponse(res, resBody)
+ if err != nil {
+ return nil, err
+ }
+
+ // append group to output
+ for _, group := range resBody.Values {
+ child := RemoteScopesChild{
+ Type: TypeGroup,
+ Id: group.Workspace.Slug,
+ Name: group.Workspace.Name,
+ // don't need to save group into data
+ Data: nil,
+ }
+ if gid != "" {
+ child.ParentId = &gid
+ }
+
+ outputBody.Children = append(outputBody.Children, child)
+ }
+
+ // check groups count
+ if resBody.Size < pageData.PerPage {
Review Comment:
resBody are not the same type
##########
backend/plugins/bitbucket/api/remote.go:
##########
@@ -0,0 +1,371 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package api
+
+import (
+ "context"
+ "encoding/base64"
+ "encoding/json"
+ "fmt"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "github.com/apache/incubator-devlake/plugins/bitbucket/models"
+ "github.com/apache/incubator-devlake/plugins/bitbucket/tasks"
+ "net/http"
+ "net/url"
+ "strconv"
+ "strings"
+)
+
+type RemoteScopesChild struct {
+ Type string `json:"type"`
+ ParentId *string `json:"parentId"`
+ Id string `json:"id"`
+ Name string `json:"name"`
+ Data interface{} `json:"data"`
+}
+
+type RemoteScopesOutput struct {
+ Children []RemoteScopesChild `json:"children"`
+ NextPageToken string `json:"nextPageToken"`
+}
+
+type SearchRemoteScopesOutput struct {
+ Children []RemoteScopesChild `json:"children"`
+ Page int `json:"page"`
+ PageSize int `json:"pageSize"`
+}
+
+type PageData struct {
+ Page int `json:"page"`
+ PerPage int `json:"per_page"`
+}
+
+type WorkspaceResponse struct {
+ Pagelen int `json:"pagelen"`
+ Page int `json:"page"`
+ Size int `json:"size"`
+ Values []struct {
+ //Type string `json:"type"`
+ //Permission string `json:"permission"`
+ //LastAccessed time.Time `json:"last_accessed"`
+ //AddedOn time.Time `json:"added_on"`
+ Workspace WorkspaceItem `json:"workspace"`
+ } `json:"values"`
+}
+
+type WorkspaceItem struct {
+ //Type string `json:"type"`
+ //Uuid string `json:"uuid"`
+ Slug string `json:"slug"`
+ Name string `json:"name"`
+}
+
+type ReposResponse struct {
+ Pagelen int `json:"pagelen"`
+ Page int `json:"page"`
+ Size int `json:"size"`
+ Values []tasks.BitbucketApiRepo `json:"values"`
+}
+
+const RemoteScopesPerPage int = 100
+const TypeScope string = "scope"
+const TypeGroup string = "group"
+
+// RemoteScopes list all available scope for users
+// @Summary list all available scope for users
+// @Description list all available scope for users
+// @Tags plugins/bitbucket
+// @Accept application/json
+// @Param connectionId path int false "connection ID"
+// @Param groupId query string false "group ID"
+// @Param pageToken query string false "page Token"
+// @Success 200 {object} RemoteScopesOutput
+// @Failure 400 {object} shared.ApiBody "Bad Request"
+// @Failure 500 {object} shared.ApiBody "Internal Error"
+// @Router /plugins/bitbucket/connections/{connectionId}/remote-scopes [GET]
+func RemoteScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
+ connectionId, _ := extractParam(input.Params)
+ if connectionId == 0 {
+ return nil, errors.BadInput.New("invalid connectionId")
+ }
+
+ connection := &models.BitbucketConnection{}
+ err := connectionHelper.First(connection, input.Params)
+ if err != nil {
+ return nil, err
+ }
+
+ groupId, ok := input.Query["groupId"]
+ if !ok || len(groupId) == 0 {
+ groupId = []string{""}
+ }
+
+ pageToken, ok := input.Query["pageToken"]
+ if !ok || len(pageToken) == 0 {
+ pageToken = []string{""}
+ }
+
+ // get gid and pageData
+ gid := groupId[0]
+ pageData, err := GetPageDataFromPageToken(pageToken[0])
+ if err != nil {
+ return nil, errors.BadInput.New("failed to get page token")
+ }
+
+ // create api client
+ apiClient, err := api.NewApiClientFromConnection(context.TODO(), basicRes, connection)
+ if err != nil {
+ return nil, err
+ }
+
+ query, err := GetQueryFromPageData(pageData)
+ if err != nil {
+ return nil, err
+ }
+
+ var res *http.Response
+ outputBody := &RemoteScopesOutput{}
+
+ // list groups part
+ if gid == "" {
+ query.Set("sort", "workspace.slug")
+ query.Set("fields", "values.workspace.slug,values.workspace.name,pagelen,page,size")
+ res, err = apiClient.Get("/user/permissions/workspaces", query, nil)
+ if err != nil {
+ return nil, err
+ }
+
+ resBody := &WorkspaceResponse{}
+ err = api.UnmarshalResponse(res, resBody)
+ if err != nil {
+ return nil, err
+ }
+
+ // append group to output
+ for _, group := range resBody.Values {
+ child := RemoteScopesChild{
+ Type: TypeGroup,
+ Id: group.Workspace.Slug,
+ Name: group.Workspace.Name,
+ // don't need to save group into data
+ Data: nil,
+ }
+ if gid != "" {
+ child.ParentId = &gid
+ }
+
+ outputBody.Children = append(outputBody.Children, child)
+ }
+
+ // check groups count
+ if resBody.Size < pageData.PerPage {
+ pageData = nil
+ }
+ } else {
+ query.Set("sort", "name")
+ query.Set("fields", "values.name,values.full_name,values.language,values.description,values.owner.username,values.created_on,values.updated_on,values.links.clone,values.links.self,pagelen,page,size")
+ // list projects part
+ res, err = apiClient.Get(fmt.Sprintf("/repositories/%s", gid), query, nil)
+ if err != nil {
+ return nil, err
+ }
+
+ resBody := &ReposResponse{}
+ err = api.UnmarshalResponse(res, resBody)
+ if err != nil {
+ return nil, err
+ }
+
+ // append repo to output
+ for _, repo := range resBody.Values {
+ child := RemoteScopesChild{
+ Type: TypeScope,
+ Id: repo.FullName,
+ Name: repo.Name,
+ Data: tasks.ConvertApiRepoToScope(&repo, connection.ID),
+ }
+ child.ParentId = &gid
+ if *child.ParentId == "" {
+ child.ParentId = nil
+ }
+
+ outputBody.Children = append(outputBody.Children, child)
+ }
+
+ // check repo count
+ if resBody.Size < pageData.PerPage {
+ pageData = nil
+ }
+ }
+
+ // get the next page token
+ outputBody.NextPageToken = ""
+ if pageData != nil {
+ pageData.Page += 1
+ outputBody.NextPageToken, err = GetPageTokenFromPageData(pageData)
+ if err != nil {
+ return nil, err
+ }
+ }
+
+ return &plugin.ApiResourceOutput{Body: outputBody, Status: http.StatusOK}, nil
+}
+
+// SearchRemoteScopes use the Search API and only return project
+// @Summary use the Search API and only return project
+// @Description use the Search API and only return project
+// @Tags plugins/bitbucket
+// @Accept application/json
+// @Param connectionId path int false "connection ID"
+// @Param search query string false "search"
+// @Param page query int false "page number"
+// @Param pageSize query int false "page size per page"
+// @Success 200 {object} SearchRemoteScopesOutput
+// @Failure 400 {object} shared.ApiBody "Bad Request"
+// @Failure 500 {object} shared.ApiBody "Internal Error"
+// @Router /plugins/bitbucket/connections/{connectionId}/search-remote-scopes [GET]
+func SearchRemoteScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
+ connectionId, _ := extractParam(input.Params)
+ if connectionId == 0 {
+ return nil, errors.BadInput.New("invalid connectionId")
+ }
+
+ connection := &models.BitbucketConnection{}
+ err := connectionHelper.First(connection, input.Params)
+ if err != nil {
+ return nil, err
+ }
+
+ search, ok := input.Query["search"]
+ if !ok || len(search) == 0 {
+ search = []string{""}
+ }
+ s := search[0]
+
+ p := 1
+ page, ok := input.Query["page"]
+ if ok && len(page) != 0 {
+ p, err = errors.Convert01(strconv.Atoi(page[0]))
+ if err != nil {
+ return nil, errors.BadInput.Wrap(err, fmt.Sprintf("failed to Atoi page:%s", page[0]))
+ }
+ }
+
+ ps := RemoteScopesPerPage
+ pageSize, ok := input.Query["pageSize"]
+ if ok && len(pageSize) != 0 {
+ ps, err = errors.Convert01(strconv.Atoi(pageSize[0]))
+ if err != nil {
+ return nil, errors.BadInput.Wrap(err, fmt.Sprintf("failed to Atoi pageSize:%s", pageSize[0]))
+ }
+ }
+
+ // create api client
+ apiClient, err := api.NewApiClientFromConnection(context.TODO(), basicRes, connection)
+ if err != nil {
+ return nil, err
+ }
+
+ // set query
+ query, err := GetQueryFromPageData(&PageData{p, ps})
+ if err != nil {
+ return nil, err
+ }
+
+ // request search
+ query.Set("sort", "name")
+ query.Set("fields", "values.name,values.full_name,values.language,values.description,values.owner.username,values.created_on,values.updated_on,values.links.clone,values.links.self,pagelen,page,size")
+
+ gid := ``
+ if strings.Contains(s, `/`) {
+ gid = strings.Split(s, `/`)[0]
+ s = strings.Split(s, `/`)[0]
Review Comment:
Bitbucket must search in a workspace(group). So here search text may like `aaa/searchText`
##########
backend/plugins/bitbucket/tasks/pr_extractor.go:
##########
@@ -138,6 +111,7 @@ func ExtractApiPullRequests(taskCtx plugin.SubTaskContext) errors.Error {
}
if rawL.MergeCommit != nil {
bitbucketPr.MergeCommitSha = rawL.MergeCommit.Hash
+ bitbucketPr.MergedAt = rawL.MergeCommit.Date.ToNullableTime()
Review Comment:
fill here instead of in converter.
##########
backend/plugins/bitbucket/tasks/commit_convertor.go:
##########
@@ -38,17 +38,17 @@ var ConvertCommitsMeta = plugin.SubTaskMeta{
}
func ConvertCommits(taskCtx plugin.SubTaskContext) errors.Error {
+ rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_COMMIT_TABLE)
db := taskCtx.GetDal()
- data := taskCtx.GetData().(*BitbucketTaskData)
- repoId := data.Repo.BitbucketId
+ repoId := data.Options.FullName
cursor, err := db.Cursor(
- dal.From("_tool_bitbucket_commits gc"),
Review Comment:
g means `github`, so delete them.
##########
backend/plugins/bitbucket/tasks/pr_comment_extractor.go:
##########
@@ -94,15 +74,14 @@ func ExtractApiPullRequestsComments(taskCtx plugin.SubTaskContext) errors.Error
return nil, err
}
toolprComment.AuthorId = bitbucketUser.AccountId
- toolprComment.AuthorName = bitbucketUser.UserName
+ toolprComment.AuthorName = bitbucketUser.DisplayName
Review Comment:
Other AuthorName is filled by DisplayName, so here changed
##########
backend/plugins/bitbucket/tasks/issue_extractor.go:
##########
@@ -132,8 +109,9 @@ func ExtractApiIssues(taskCtx plugin.SubTaskContext) errors.Error {
results = append(results, relatedUser)
}
if status, ok := issueStatusMap[bitbucketIssue.State]; ok {
- bitbucketIssue.State = status
+ bitbucketIssue.StdState = status
Review Comment:
add a new column to save state after transformation rule
##########
backend/plugins/bitbucket/api/remote.go:
##########
@@ -0,0 +1,371 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package api
+
+import (
+ "context"
+ "encoding/base64"
+ "encoding/json"
+ "fmt"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "github.com/apache/incubator-devlake/plugins/bitbucket/models"
+ "github.com/apache/incubator-devlake/plugins/bitbucket/tasks"
+ "net/http"
+ "net/url"
+ "strconv"
+ "strings"
+)
+
+type RemoteScopesChild struct {
+ Type string `json:"type"`
+ ParentId *string `json:"parentId"`
+ Id string `json:"id"`
+ Name string `json:"name"`
+ Data interface{} `json:"data"`
+}
+
+type RemoteScopesOutput struct {
+ Children []RemoteScopesChild `json:"children"`
+ NextPageToken string `json:"nextPageToken"`
+}
+
+type SearchRemoteScopesOutput struct {
+ Children []RemoteScopesChild `json:"children"`
+ Page int `json:"page"`
+ PageSize int `json:"pageSize"`
+}
+
+type PageData struct {
+ Page int `json:"page"`
+ PerPage int `json:"per_page"`
+}
+
+type WorkspaceResponse struct {
+ Pagelen int `json:"pagelen"`
+ Page int `json:"page"`
+ Size int `json:"size"`
+ Values []struct {
+ //Type string `json:"type"`
+ //Permission string `json:"permission"`
+ //LastAccessed time.Time `json:"last_accessed"`
+ //AddedOn time.Time `json:"added_on"`
+ Workspace WorkspaceItem `json:"workspace"`
+ } `json:"values"`
+}
+
+type WorkspaceItem struct {
+ //Type string `json:"type"`
+ //Uuid string `json:"uuid"`
+ Slug string `json:"slug"`
+ Name string `json:"name"`
+}
+
+type ReposResponse struct {
+ Pagelen int `json:"pagelen"`
+ Page int `json:"page"`
+ Size int `json:"size"`
+ Values []tasks.BitbucketApiRepo `json:"values"`
+}
+
+const RemoteScopesPerPage int = 100
+const TypeScope string = "scope"
+const TypeGroup string = "group"
+
+// RemoteScopes list all available scope for users
+// @Summary list all available scope for users
+// @Description list all available scope for users
+// @Tags plugins/bitbucket
+// @Accept application/json
+// @Param connectionId path int false "connection ID"
+// @Param groupId query string false "group ID"
+// @Param pageToken query string false "page Token"
+// @Success 200 {object} RemoteScopesOutput
+// @Failure 400 {object} shared.ApiBody "Bad Request"
+// @Failure 500 {object} shared.ApiBody "Internal Error"
+// @Router /plugins/bitbucket/connections/{connectionId}/remote-scopes [GET]
+func RemoteScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
+ connectionId, _ := extractParam(input.Params)
+ if connectionId == 0 {
+ return nil, errors.BadInput.New("invalid connectionId")
+ }
+
+ connection := &models.BitbucketConnection{}
+ err := connectionHelper.First(connection, input.Params)
+ if err != nil {
+ return nil, err
+ }
+
+ groupId, ok := input.Query["groupId"]
+ if !ok || len(groupId) == 0 {
+ groupId = []string{""}
+ }
+
+ pageToken, ok := input.Query["pageToken"]
+ if !ok || len(pageToken) == 0 {
+ pageToken = []string{""}
+ }
+
+ // get gid and pageData
+ gid := groupId[0]
+ pageData, err := GetPageDataFromPageToken(pageToken[0])
+ if err != nil {
+ return nil, errors.BadInput.New("failed to get page token")
+ }
+
+ // create api client
+ apiClient, err := api.NewApiClientFromConnection(context.TODO(), basicRes, connection)
+ if err != nil {
+ return nil, err
+ }
+
+ query, err := GetQueryFromPageData(pageData)
+ if err != nil {
+ return nil, err
+ }
+
+ var res *http.Response
+ outputBody := &RemoteScopesOutput{}
+
+ // list groups part
+ if gid == "" {
+ query.Set("sort", "workspace.slug")
+ query.Set("fields", "values.workspace.slug,values.workspace.name,pagelen,page,size")
+ res, err = apiClient.Get("/user/permissions/workspaces", query, nil)
+ if err != nil {
+ return nil, err
+ }
+
+ resBody := &WorkspaceResponse{}
+ err = api.UnmarshalResponse(res, resBody)
+ if err != nil {
+ return nil, err
+ }
+
+ // append group to output
+ for _, group := range resBody.Values {
+ child := RemoteScopesChild{
+ Type: TypeGroup,
+ Id: group.Workspace.Slug,
+ Name: group.Workspace.Name,
+ // don't need to save group into data
+ Data: nil,
+ }
+ if gid != "" {
+ child.ParentId = &gid
+ }
+
+ outputBody.Children = append(outputBody.Children, child)
+ }
+
+ // check groups count
+ if resBody.Size < pageData.PerPage {
+ pageData = nil
+ }
+ } else {
+ query.Set("sort", "name")
+ query.Set("fields", "values.name,values.full_name,values.language,values.description,values.owner.username,values.created_on,values.updated_on,values.links.clone,values.links.self,pagelen,page,size")
+ // list projects part
+ res, err = apiClient.Get(fmt.Sprintf("/repositories/%s", gid), query, nil)
+ if err != nil {
+ return nil, err
+ }
+
+ resBody := &ReposResponse{}
+ err = api.UnmarshalResponse(res, resBody)
+ if err != nil {
+ return nil, err
+ }
+
+ // append repo to output
+ for _, repo := range resBody.Values {
+ child := RemoteScopesChild{
+ Type: TypeScope,
+ Id: repo.FullName,
+ Name: repo.Name,
+ Data: tasks.ConvertApiRepoToScope(&repo, connection.ID),
+ }
+ child.ParentId = &gid
+ if *child.ParentId == "" {
Review Comment:
fixed
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@devlake.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [incubator-devlake] likyh commented on a diff in pull request #4373: feat: add increment & timeFilter for bitbucket
Posted by "likyh (via GitHub)" <gi...@apache.org>.
likyh commented on code in PR #4373:
URL: https://github.com/apache/incubator-devlake/pull/4373#discussion_r1106608577
##########
backend/plugins/bitbucket/impl/impl.go:
##########
@@ -79,6 +90,9 @@ func (p Bitbucket) SubTaskMetas() []plugin.SubTaskMeta {
tasks.CollectApiPrCommitsMeta,
tasks.ExtractApiPrCommitsMeta,
+ tasks.CollectApiCommitsMeta,
+ tasks.ExtractApiCommitsMeta,
Review Comment:
add unused tasks to allow open it on advance mod
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@devlake.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [incubator-devlake] mappjzc commented on a diff in pull request #4373: feat: add increment & timeFilter for bitbucket
Posted by "mappjzc (via GitHub)" <gi...@apache.org>.
mappjzc commented on code in PR #4373:
URL: https://github.com/apache/incubator-devlake/pull/4373#discussion_r1106771410
##########
backend/plugins/bitbucket/api/remote.go:
##########
@@ -0,0 +1,371 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package api
+
+import (
+ "context"
+ "encoding/base64"
+ "encoding/json"
+ "fmt"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "github.com/apache/incubator-devlake/plugins/bitbucket/models"
+ "github.com/apache/incubator-devlake/plugins/bitbucket/tasks"
+ "net/http"
+ "net/url"
+ "strconv"
+ "strings"
+)
+
+type RemoteScopesChild struct {
+ Type string `json:"type"`
+ ParentId *string `json:"parentId"`
+ Id string `json:"id"`
+ Name string `json:"name"`
+ Data interface{} `json:"data"`
+}
+
+type RemoteScopesOutput struct {
+ Children []RemoteScopesChild `json:"children"`
+ NextPageToken string `json:"nextPageToken"`
+}
+
+type SearchRemoteScopesOutput struct {
+ Children []RemoteScopesChild `json:"children"`
+ Page int `json:"page"`
+ PageSize int `json:"pageSize"`
+}
+
+type PageData struct {
+ Page int `json:"page"`
+ PerPage int `json:"per_page"`
+}
+
+type WorkspaceResponse struct {
+ Pagelen int `json:"pagelen"`
+ Page int `json:"page"`
+ Size int `json:"size"`
+ Values []struct {
+ //Type string `json:"type"`
+ //Permission string `json:"permission"`
+ //LastAccessed time.Time `json:"last_accessed"`
+ //AddedOn time.Time `json:"added_on"`
+ Workspace WorkspaceItem `json:"workspace"`
+ } `json:"values"`
+}
+
+type WorkspaceItem struct {
+ //Type string `json:"type"`
+ //Uuid string `json:"uuid"`
+ Slug string `json:"slug"`
+ Name string `json:"name"`
+}
+
+type ReposResponse struct {
+ Pagelen int `json:"pagelen"`
+ Page int `json:"page"`
+ Size int `json:"size"`
+ Values []tasks.BitbucketApiRepo `json:"values"`
+}
+
+const RemoteScopesPerPage int = 100
+const TypeScope string = "scope"
+const TypeGroup string = "group"
+
+// RemoteScopes list all available scope for users
+// @Summary list all available scope for users
+// @Description list all available scope for users
+// @Tags plugins/bitbucket
+// @Accept application/json
+// @Param connectionId path int false "connection ID"
+// @Param groupId query string false "group ID"
+// @Param pageToken query string false "page Token"
+// @Success 200 {object} RemoteScopesOutput
+// @Failure 400 {object} shared.ApiBody "Bad Request"
+// @Failure 500 {object} shared.ApiBody "Internal Error"
+// @Router /plugins/bitbucket/connections/{connectionId}/remote-scopes [GET]
+func RemoteScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
+ connectionId, _ := extractParam(input.Params)
+ if connectionId == 0 {
+ return nil, errors.BadInput.New("invalid connectionId")
+ }
+
+ connection := &models.BitbucketConnection{}
+ err := connectionHelper.First(connection, input.Params)
+ if err != nil {
+ return nil, err
+ }
+
+ groupId, ok := input.Query["groupId"]
+ if !ok || len(groupId) == 0 {
+ groupId = []string{""}
+ }
+
+ pageToken, ok := input.Query["pageToken"]
+ if !ok || len(pageToken) == 0 {
+ pageToken = []string{""}
+ }
+
+ // get gid and pageData
+ gid := groupId[0]
+ pageData, err := GetPageDataFromPageToken(pageToken[0])
+ if err != nil {
+ return nil, errors.BadInput.New("failed to get page token")
+ }
+
+ // create api client
+ apiClient, err := api.NewApiClientFromConnection(context.TODO(), basicRes, connection)
+ if err != nil {
+ return nil, err
+ }
+
+ query, err := GetQueryFromPageData(pageData)
+ if err != nil {
+ return nil, err
+ }
+
+ var res *http.Response
+ outputBody := &RemoteScopesOutput{}
+
+ // list groups part
+ if gid == "" {
+ query.Set("sort", "workspace.slug")
+ query.Set("fields", "values.workspace.slug,values.workspace.name,pagelen,page,size")
+ res, err = apiClient.Get("/user/permissions/workspaces", query, nil)
+ if err != nil {
+ return nil, err
+ }
+
+ resBody := &WorkspaceResponse{}
+ err = api.UnmarshalResponse(res, resBody)
+ if err != nil {
+ return nil, err
+ }
+
+ // append group to output
+ for _, group := range resBody.Values {
+ child := RemoteScopesChild{
+ Type: TypeGroup,
+ Id: group.Workspace.Slug,
+ Name: group.Workspace.Name,
+ // don't need to save group into data
+ Data: nil,
+ }
+ if gid != "" {
Review Comment:
at the first level if,you have checked the gid already.
##########
backend/plugins/bitbucket/api/remote.go:
##########
@@ -0,0 +1,371 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package api
+
+import (
+ "context"
+ "encoding/base64"
+ "encoding/json"
+ "fmt"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "github.com/apache/incubator-devlake/plugins/bitbucket/models"
+ "github.com/apache/incubator-devlake/plugins/bitbucket/tasks"
+ "net/http"
+ "net/url"
+ "strconv"
+ "strings"
+)
+
+type RemoteScopesChild struct {
+ Type string `json:"type"`
+ ParentId *string `json:"parentId"`
+ Id string `json:"id"`
+ Name string `json:"name"`
+ Data interface{} `json:"data"`
+}
+
+type RemoteScopesOutput struct {
+ Children []RemoteScopesChild `json:"children"`
+ NextPageToken string `json:"nextPageToken"`
+}
+
+type SearchRemoteScopesOutput struct {
+ Children []RemoteScopesChild `json:"children"`
+ Page int `json:"page"`
+ PageSize int `json:"pageSize"`
+}
+
+type PageData struct {
+ Page int `json:"page"`
+ PerPage int `json:"per_page"`
+}
+
+type WorkspaceResponse struct {
+ Pagelen int `json:"pagelen"`
+ Page int `json:"page"`
+ Size int `json:"size"`
+ Values []struct {
+ //Type string `json:"type"`
+ //Permission string `json:"permission"`
+ //LastAccessed time.Time `json:"last_accessed"`
+ //AddedOn time.Time `json:"added_on"`
+ Workspace WorkspaceItem `json:"workspace"`
+ } `json:"values"`
+}
+
+type WorkspaceItem struct {
+ //Type string `json:"type"`
+ //Uuid string `json:"uuid"`
+ Slug string `json:"slug"`
+ Name string `json:"name"`
+}
+
+type ReposResponse struct {
+ Pagelen int `json:"pagelen"`
+ Page int `json:"page"`
+ Size int `json:"size"`
+ Values []tasks.BitbucketApiRepo `json:"values"`
+}
+
+const RemoteScopesPerPage int = 100
+const TypeScope string = "scope"
+const TypeGroup string = "group"
+
+// RemoteScopes list all available scope for users
+// @Summary list all available scope for users
+// @Description list all available scope for users
+// @Tags plugins/bitbucket
+// @Accept application/json
+// @Param connectionId path int false "connection ID"
+// @Param groupId query string false "group ID"
+// @Param pageToken query string false "page Token"
+// @Success 200 {object} RemoteScopesOutput
+// @Failure 400 {object} shared.ApiBody "Bad Request"
+// @Failure 500 {object} shared.ApiBody "Internal Error"
+// @Router /plugins/bitbucket/connections/{connectionId}/remote-scopes [GET]
+func RemoteScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
+ connectionId, _ := extractParam(input.Params)
+ if connectionId == 0 {
+ return nil, errors.BadInput.New("invalid connectionId")
+ }
+
+ connection := &models.BitbucketConnection{}
+ err := connectionHelper.First(connection, input.Params)
+ if err != nil {
+ return nil, err
+ }
+
+ groupId, ok := input.Query["groupId"]
+ if !ok || len(groupId) == 0 {
+ groupId = []string{""}
+ }
+
+ pageToken, ok := input.Query["pageToken"]
+ if !ok || len(pageToken) == 0 {
+ pageToken = []string{""}
+ }
+
+ // get gid and pageData
+ gid := groupId[0]
+ pageData, err := GetPageDataFromPageToken(pageToken[0])
+ if err != nil {
+ return nil, errors.BadInput.New("failed to get page token")
+ }
+
+ // create api client
+ apiClient, err := api.NewApiClientFromConnection(context.TODO(), basicRes, connection)
+ if err != nil {
+ return nil, err
+ }
+
+ query, err := GetQueryFromPageData(pageData)
+ if err != nil {
+ return nil, err
+ }
+
+ var res *http.Response
+ outputBody := &RemoteScopesOutput{}
+
+ // list groups part
+ if gid == "" {
+ query.Set("sort", "workspace.slug")
+ query.Set("fields", "values.workspace.slug,values.workspace.name,pagelen,page,size")
+ res, err = apiClient.Get("/user/permissions/workspaces", query, nil)
+ if err != nil {
+ return nil, err
+ }
+
+ resBody := &WorkspaceResponse{}
+ err = api.UnmarshalResponse(res, resBody)
+ if err != nil {
+ return nil, err
+ }
+
+ // append group to output
+ for _, group := range resBody.Values {
+ child := RemoteScopesChild{
+ Type: TypeGroup,
+ Id: group.Workspace.Slug,
+ Name: group.Workspace.Name,
+ // don't need to save group into data
+ Data: nil,
+ }
+ if gid != "" {
+ child.ParentId = &gid
+ }
+
+ outputBody.Children = append(outputBody.Children, child)
+ }
+
+ // check groups count
+ if resBody.Size < pageData.PerPage {
Review Comment:
This if check is same as the else part.They can be taken out together.
##########
backend/plugins/bitbucket/api/remote.go:
##########
@@ -0,0 +1,371 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package api
+
+import (
+ "context"
+ "encoding/base64"
+ "encoding/json"
+ "fmt"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "github.com/apache/incubator-devlake/plugins/bitbucket/models"
+ "github.com/apache/incubator-devlake/plugins/bitbucket/tasks"
+ "net/http"
+ "net/url"
+ "strconv"
+ "strings"
+)
+
+type RemoteScopesChild struct {
+ Type string `json:"type"`
+ ParentId *string `json:"parentId"`
+ Id string `json:"id"`
+ Name string `json:"name"`
+ Data interface{} `json:"data"`
+}
+
+type RemoteScopesOutput struct {
+ Children []RemoteScopesChild `json:"children"`
+ NextPageToken string `json:"nextPageToken"`
+}
+
+type SearchRemoteScopesOutput struct {
+ Children []RemoteScopesChild `json:"children"`
+ Page int `json:"page"`
+ PageSize int `json:"pageSize"`
+}
+
+type PageData struct {
+ Page int `json:"page"`
+ PerPage int `json:"per_page"`
+}
+
+type WorkspaceResponse struct {
+ Pagelen int `json:"pagelen"`
+ Page int `json:"page"`
+ Size int `json:"size"`
+ Values []struct {
+ //Type string `json:"type"`
+ //Permission string `json:"permission"`
+ //LastAccessed time.Time `json:"last_accessed"`
+ //AddedOn time.Time `json:"added_on"`
+ Workspace WorkspaceItem `json:"workspace"`
+ } `json:"values"`
+}
+
+type WorkspaceItem struct {
+ //Type string `json:"type"`
+ //Uuid string `json:"uuid"`
+ Slug string `json:"slug"`
+ Name string `json:"name"`
+}
+
+type ReposResponse struct {
+ Pagelen int `json:"pagelen"`
+ Page int `json:"page"`
+ Size int `json:"size"`
+ Values []tasks.BitbucketApiRepo `json:"values"`
+}
+
+const RemoteScopesPerPage int = 100
+const TypeScope string = "scope"
+const TypeGroup string = "group"
+
+// RemoteScopes list all available scope for users
+// @Summary list all available scope for users
+// @Description list all available scope for users
+// @Tags plugins/bitbucket
+// @Accept application/json
+// @Param connectionId path int false "connection ID"
+// @Param groupId query string false "group ID"
+// @Param pageToken query string false "page Token"
+// @Success 200 {object} RemoteScopesOutput
+// @Failure 400 {object} shared.ApiBody "Bad Request"
+// @Failure 500 {object} shared.ApiBody "Internal Error"
+// @Router /plugins/bitbucket/connections/{connectionId}/remote-scopes [GET]
+func RemoteScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
+ connectionId, _ := extractParam(input.Params)
+ if connectionId == 0 {
+ return nil, errors.BadInput.New("invalid connectionId")
+ }
+
+ connection := &models.BitbucketConnection{}
+ err := connectionHelper.First(connection, input.Params)
+ if err != nil {
+ return nil, err
+ }
+
+ groupId, ok := input.Query["groupId"]
+ if !ok || len(groupId) == 0 {
+ groupId = []string{""}
+ }
+
+ pageToken, ok := input.Query["pageToken"]
+ if !ok || len(pageToken) == 0 {
+ pageToken = []string{""}
+ }
+
+ // get gid and pageData
+ gid := groupId[0]
+ pageData, err := GetPageDataFromPageToken(pageToken[0])
+ if err != nil {
+ return nil, errors.BadInput.New("failed to get page token")
+ }
+
+ // create api client
+ apiClient, err := api.NewApiClientFromConnection(context.TODO(), basicRes, connection)
+ if err != nil {
+ return nil, err
+ }
+
+ query, err := GetQueryFromPageData(pageData)
+ if err != nil {
+ return nil, err
+ }
+
+ var res *http.Response
+ outputBody := &RemoteScopesOutput{}
+
+ // list groups part
+ if gid == "" {
+ query.Set("sort", "workspace.slug")
+ query.Set("fields", "values.workspace.slug,values.workspace.name,pagelen,page,size")
+ res, err = apiClient.Get("/user/permissions/workspaces", query, nil)
+ if err != nil {
+ return nil, err
+ }
+
+ resBody := &WorkspaceResponse{}
+ err = api.UnmarshalResponse(res, resBody)
+ if err != nil {
+ return nil, err
+ }
+
+ // append group to output
+ for _, group := range resBody.Values {
+ child := RemoteScopesChild{
+ Type: TypeGroup,
+ Id: group.Workspace.Slug,
+ Name: group.Workspace.Name,
+ // don't need to save group into data
+ Data: nil,
+ }
+ if gid != "" {
+ child.ParentId = &gid
+ }
+
+ outputBody.Children = append(outputBody.Children, child)
+ }
+
+ // check groups count
+ if resBody.Size < pageData.PerPage {
+ pageData = nil
+ }
+ } else {
+ query.Set("sort", "name")
+ query.Set("fields", "values.name,values.full_name,values.language,values.description,values.owner.username,values.created_on,values.updated_on,values.links.clone,values.links.self,pagelen,page,size")
+ // list projects part
+ res, err = apiClient.Get(fmt.Sprintf("/repositories/%s", gid), query, nil)
+ if err != nil {
+ return nil, err
+ }
+
+ resBody := &ReposResponse{}
+ err = api.UnmarshalResponse(res, resBody)
+ if err != nil {
+ return nil, err
+ }
+
+ // append repo to output
+ for _, repo := range resBody.Values {
+ child := RemoteScopesChild{
+ Type: TypeScope,
+ Id: repo.FullName,
+ Name: repo.Name,
+ Data: tasks.ConvertApiRepoToScope(&repo, connection.ID),
+ }
+ child.ParentId = &gid
+ if *child.ParentId == "" {
+ child.ParentId = nil
+ }
+
+ outputBody.Children = append(outputBody.Children, child)
+ }
+
+ // check repo count
+ if resBody.Size < pageData.PerPage {
+ pageData = nil
+ }
+ }
+
+ // get the next page token
+ outputBody.NextPageToken = ""
+ if pageData != nil {
+ pageData.Page += 1
+ outputBody.NextPageToken, err = GetPageTokenFromPageData(pageData)
+ if err != nil {
+ return nil, err
+ }
+ }
+
+ return &plugin.ApiResourceOutput{Body: outputBody, Status: http.StatusOK}, nil
+}
+
+// SearchRemoteScopes use the Search API and only return project
+// @Summary use the Search API and only return project
+// @Description use the Search API and only return project
+// @Tags plugins/bitbucket
+// @Accept application/json
+// @Param connectionId path int false "connection ID"
+// @Param search query string false "search"
+// @Param page query int false "page number"
+// @Param pageSize query int false "page size per page"
+// @Success 200 {object} SearchRemoteScopesOutput
+// @Failure 400 {object} shared.ApiBody "Bad Request"
+// @Failure 500 {object} shared.ApiBody "Internal Error"
+// @Router /plugins/bitbucket/connections/{connectionId}/search-remote-scopes [GET]
+func SearchRemoteScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
+ connectionId, _ := extractParam(input.Params)
+ if connectionId == 0 {
+ return nil, errors.BadInput.New("invalid connectionId")
+ }
+
+ connection := &models.BitbucketConnection{}
+ err := connectionHelper.First(connection, input.Params)
+ if err != nil {
+ return nil, err
+ }
+
+ search, ok := input.Query["search"]
+ if !ok || len(search) == 0 {
+ search = []string{""}
+ }
+ s := search[0]
+
+ p := 1
+ page, ok := input.Query["page"]
+ if ok && len(page) != 0 {
+ p, err = errors.Convert01(strconv.Atoi(page[0]))
+ if err != nil {
+ return nil, errors.BadInput.Wrap(err, fmt.Sprintf("failed to Atoi page:%s", page[0]))
+ }
+ }
+
+ ps := RemoteScopesPerPage
+ pageSize, ok := input.Query["pageSize"]
+ if ok && len(pageSize) != 0 {
+ ps, err = errors.Convert01(strconv.Atoi(pageSize[0]))
+ if err != nil {
+ return nil, errors.BadInput.Wrap(err, fmt.Sprintf("failed to Atoi pageSize:%s", pageSize[0]))
+ }
+ }
+
+ // create api client
+ apiClient, err := api.NewApiClientFromConnection(context.TODO(), basicRes, connection)
+ if err != nil {
+ return nil, err
+ }
+
+ // set query
+ query, err := GetQueryFromPageData(&PageData{p, ps})
+ if err != nil {
+ return nil, err
+ }
+
+ // request search
+ query.Set("sort", "name")
+ query.Set("fields", "values.name,values.full_name,values.language,values.description,values.owner.username,values.created_on,values.updated_on,values.links.clone,values.links.self,pagelen,page,size")
+
+ gid := ``
+ if strings.Contains(s, `/`) {
+ gid = strings.Split(s, `/`)[0]
+ s = strings.Split(s, `/`)[0]
Review Comment:
s = gid ? and set q = name~gid, and the url of get with gid? It's hard to understand.
##########
backend/plugins/bitbucket/api/remote.go:
##########
@@ -0,0 +1,371 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package api
+
+import (
+ "context"
+ "encoding/base64"
+ "encoding/json"
+ "fmt"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "github.com/apache/incubator-devlake/plugins/bitbucket/models"
+ "github.com/apache/incubator-devlake/plugins/bitbucket/tasks"
+ "net/http"
+ "net/url"
+ "strconv"
+ "strings"
+)
+
+type RemoteScopesChild struct {
+ Type string `json:"type"`
+ ParentId *string `json:"parentId"`
+ Id string `json:"id"`
+ Name string `json:"name"`
+ Data interface{} `json:"data"`
+}
+
+type RemoteScopesOutput struct {
+ Children []RemoteScopesChild `json:"children"`
+ NextPageToken string `json:"nextPageToken"`
+}
+
+type SearchRemoteScopesOutput struct {
+ Children []RemoteScopesChild `json:"children"`
+ Page int `json:"page"`
+ PageSize int `json:"pageSize"`
+}
+
+type PageData struct {
+ Page int `json:"page"`
+ PerPage int `json:"per_page"`
+}
+
+type WorkspaceResponse struct {
+ Pagelen int `json:"pagelen"`
+ Page int `json:"page"`
+ Size int `json:"size"`
+ Values []struct {
+ //Type string `json:"type"`
+ //Permission string `json:"permission"`
+ //LastAccessed time.Time `json:"last_accessed"`
+ //AddedOn time.Time `json:"added_on"`
+ Workspace WorkspaceItem `json:"workspace"`
+ } `json:"values"`
+}
+
+type WorkspaceItem struct {
+ //Type string `json:"type"`
+ //Uuid string `json:"uuid"`
+ Slug string `json:"slug"`
+ Name string `json:"name"`
+}
+
+type ReposResponse struct {
+ Pagelen int `json:"pagelen"`
+ Page int `json:"page"`
+ Size int `json:"size"`
+ Values []tasks.BitbucketApiRepo `json:"values"`
+}
+
+const RemoteScopesPerPage int = 100
+const TypeScope string = "scope"
+const TypeGroup string = "group"
+
+// RemoteScopes list all available scope for users
+// @Summary list all available scope for users
+// @Description list all available scope for users
+// @Tags plugins/bitbucket
+// @Accept application/json
+// @Param connectionId path int false "connection ID"
+// @Param groupId query string false "group ID"
+// @Param pageToken query string false "page Token"
+// @Success 200 {object} RemoteScopesOutput
+// @Failure 400 {object} shared.ApiBody "Bad Request"
+// @Failure 500 {object} shared.ApiBody "Internal Error"
+// @Router /plugins/bitbucket/connections/{connectionId}/remote-scopes [GET]
+func RemoteScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
+ connectionId, _ := extractParam(input.Params)
+ if connectionId == 0 {
+ return nil, errors.BadInput.New("invalid connectionId")
+ }
+
+ connection := &models.BitbucketConnection{}
+ err := connectionHelper.First(connection, input.Params)
+ if err != nil {
+ return nil, err
+ }
+
+ groupId, ok := input.Query["groupId"]
+ if !ok || len(groupId) == 0 {
+ groupId = []string{""}
+ }
+
+ pageToken, ok := input.Query["pageToken"]
+ if !ok || len(pageToken) == 0 {
+ pageToken = []string{""}
+ }
+
+ // get gid and pageData
+ gid := groupId[0]
+ pageData, err := GetPageDataFromPageToken(pageToken[0])
+ if err != nil {
+ return nil, errors.BadInput.New("failed to get page token")
+ }
+
+ // create api client
+ apiClient, err := api.NewApiClientFromConnection(context.TODO(), basicRes, connection)
+ if err != nil {
+ return nil, err
+ }
+
+ query, err := GetQueryFromPageData(pageData)
+ if err != nil {
+ return nil, err
+ }
+
+ var res *http.Response
+ outputBody := &RemoteScopesOutput{}
+
+ // list groups part
+ if gid == "" {
+ query.Set("sort", "workspace.slug")
+ query.Set("fields", "values.workspace.slug,values.workspace.name,pagelen,page,size")
+ res, err = apiClient.Get("/user/permissions/workspaces", query, nil)
+ if err != nil {
+ return nil, err
+ }
+
+ resBody := &WorkspaceResponse{}
+ err = api.UnmarshalResponse(res, resBody)
+ if err != nil {
+ return nil, err
+ }
+
+ // append group to output
+ for _, group := range resBody.Values {
+ child := RemoteScopesChild{
+ Type: TypeGroup,
+ Id: group.Workspace.Slug,
+ Name: group.Workspace.Name,
+ // don't need to save group into data
+ Data: nil,
+ }
+ if gid != "" {
+ child.ParentId = &gid
+ }
+
+ outputBody.Children = append(outputBody.Children, child)
+ }
+
+ // check groups count
+ if resBody.Size < pageData.PerPage {
+ pageData = nil
+ }
+ } else {
+ query.Set("sort", "name")
+ query.Set("fields", "values.name,values.full_name,values.language,values.description,values.owner.username,values.created_on,values.updated_on,values.links.clone,values.links.self,pagelen,page,size")
+ // list projects part
+ res, err = apiClient.Get(fmt.Sprintf("/repositories/%s", gid), query, nil)
+ if err != nil {
+ return nil, err
+ }
+
+ resBody := &ReposResponse{}
+ err = api.UnmarshalResponse(res, resBody)
+ if err != nil {
+ return nil, err
+ }
+
+ // append repo to output
+ for _, repo := range resBody.Values {
+ child := RemoteScopesChild{
+ Type: TypeScope,
+ Id: repo.FullName,
+ Name: repo.Name,
+ Data: tasks.ConvertApiRepoToScope(&repo, connection.ID),
+ }
+ child.ParentId = &gid
+ if *child.ParentId == "" {
Review Comment:
At the first if else,you have checked the gid already. So the same as child.ParentId.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@devlake.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [incubator-devlake] likyh commented on a diff in pull request #4373: feat: add increment & timeFilter for bitbucket
Posted by "likyh (via GitHub)" <gi...@apache.org>.
likyh commented on code in PR #4373:
URL: https://github.com/apache/incubator-devlake/pull/4373#discussion_r1107076814
##########
backend/plugins/bitbucket/api/blueprint_v200.go:
##########
@@ -0,0 +1,198 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package api
+
+import (
+ "fmt"
+ "github.com/apache/incubator-devlake/core/dal"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/models/domainlayer"
+ "github.com/apache/incubator-devlake/core/models/domainlayer/code"
+ "github.com/apache/incubator-devlake/core/models/domainlayer/devops"
+ "github.com/apache/incubator-devlake/core/models/domainlayer/didgen"
+ "github.com/apache/incubator-devlake/core/models/domainlayer/ticket"
+ "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/core/utils"
+ helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "github.com/apache/incubator-devlake/plugins/bitbucket/models"
+ "github.com/apache/incubator-devlake/plugins/bitbucket/tasks"
+ "github.com/go-playground/validator/v10"
+ "net/url"
+ "strings"
+ "time"
+)
+
+func MakeDataSourcePipelinePlanV200(subtaskMetas []plugin.SubTaskMeta, connectionId uint64, bpScopes []*plugin.BlueprintScopeV200, syncPolicy *plugin.BlueprintSyncPolicy) (plugin.PipelinePlan, []plugin.Scope, errors.Error) {
+ connectionHelper := helper.NewConnectionHelper(basicRes, validator.New())
+ // get the connection info for url
+ connection := &models.BitbucketConnection{}
+ err := connectionHelper.FirstById(connection, connectionId)
+ if err != nil {
+ return nil, nil, err
+ }
+
+ plan := make(plugin.PipelinePlan, len(bpScopes))
+ plan, err = makeDataSourcePipelinePlanV200(subtaskMetas, plan, bpScopes, connection, syncPolicy)
+ if err != nil {
+ return nil, nil, err
+ }
+ scopes, err := makeScopesV200(bpScopes, connection)
+ if err != nil {
+ return nil, nil, err
+ }
+
+ return plan, scopes, nil
+}
+
+func makeDataSourcePipelinePlanV200(
+ subtaskMetas []plugin.SubTaskMeta,
+ plan plugin.PipelinePlan,
+ bpScopes []*plugin.BlueprintScopeV200,
+ connection *models.BitbucketConnection,
+ syncPolicy *plugin.BlueprintSyncPolicy,
+) (plugin.PipelinePlan, errors.Error) {
+ var err errors.Error
+ for i, bpScope := range bpScopes {
+ stage := plan[i]
+ if stage == nil {
+ stage = plugin.PipelineStage{}
+ }
+ repo := &models.BitbucketRepo{}
+ // get repo from db
+ err = basicRes.GetDal().First(repo, dal.Where(`connection_id = ? AND bitbucket_id = ?`, connection.ID, bpScope.Id))
+ if err != nil {
+ return nil, errors.Default.Wrap(err, fmt.Sprintf("fail to find repo %s", bpScope.Id))
+ }
+ transformationRule := &models.BitbucketTransformationRule{}
+ // get transformation rules from db
+ db := basicRes.GetDal()
+ err = db.First(transformationRule, dal.Where(`id = ?`, repo.TransformationRuleId))
+ if err != nil && !db.IsErrorNotFound(err) {
+ return nil, err
+ }
+ // refdiff
+ if transformationRule != nil && transformationRule.Refdiff != nil {
+ // add a new task to next stage
+ j := i + 1
+ if j == len(plan) {
+ plan = append(plan, nil)
+ }
+ refdiffOp := transformationRule.Refdiff
+ refdiffOp["repoId"] = didgen.NewDomainIdGenerator(&models.BitbucketRepo{}).Generate(connection.ID, repo.BitbucketId)
+ plan[j] = plugin.PipelineStage{
+ {
+ Plugin: "refdiff",
+ Options: refdiffOp,
+ },
+ }
+ transformationRule.Refdiff = nil
+ }
+
+ // construct task options for bitbucket
+ op := &tasks.BitbucketOptions{
+ ConnectionId: repo.ConnectionId,
+ FullName: repo.BitbucketId,
+ }
+ if syncPolicy.CreatedDateAfter != nil {
+ op.CreatedDateAfter = syncPolicy.CreatedDateAfter.Format(time.RFC3339)
+ }
+ options, err := tasks.EncodeTaskOptions(op)
+ if err != nil {
+ return nil, err
+ }
+
+ subtasks, err := helper.MakePipelinePlanSubtasks(subtaskMetas, bpScope.Entities)
+ if err != nil {
+ return nil, err
+ }
+ stage = append(stage, &plugin.PipelineTask{
+ Plugin: "bitbucket",
+ Subtasks: subtasks,
+ Options: options,
+ })
+ if err != nil {
+ return nil, err
+ }
+
+ // add gitex stage
+ if utils.StringsContains(bpScope.Entities, plugin.DOMAIN_TYPE_CODE) {
+ cloneUrl, err := errors.Convert01(url.Parse(repo.CloneUrl))
+ if err != nil {
+ return nil, err
+ }
+ token := strings.Split(connection.Password, ",")[0]
Review Comment:
fixed
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@devlake.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [incubator-devlake] likyh commented on a diff in pull request #4373: feat: add increment & timeFilter for bitbucket
Posted by "likyh (via GitHub)" <gi...@apache.org>.
likyh commented on code in PR #4373:
URL: https://github.com/apache/incubator-devlake/pull/4373#discussion_r1106606836
##########
backend/plugins/bitbucket/api/remote.go:
##########
@@ -0,0 +1,368 @@
+/*
Review Comment:
this file imitates from https://github.com/apache/incubator-devlake/blob/3285a5aa58d8e678cd4ac1fedd06138d28fc23cc/backend/plugins/gitlab/api/remote.go
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@devlake.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [incubator-devlake] warren830 merged pull request #4373: feat: add increment & timeFilter for bitbucket
Posted by "warren830 (via GitHub)" <gi...@apache.org>.
warren830 merged PR #4373:
URL: https://github.com/apache/incubator-devlake/pull/4373
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@devlake.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [incubator-devlake] likyh commented on a diff in pull request #4373: feat: add increment & timeFilter for bitbucket
Posted by "likyh (via GitHub)" <gi...@apache.org>.
likyh commented on code in PR #4373:
URL: https://github.com/apache/incubator-devlake/pull/4373#discussion_r1106608781
##########
backend/plugins/bitbucket/impl/impl.go:
##########
@@ -166,3 +222,54 @@ func (p Bitbucket) Close(taskCtx plugin.TaskContext) errors.Error {
data.ApiClient.Release()
return nil
}
+
+func EnrichOptions(taskCtx plugin.TaskContext,
+ op *tasks.BitbucketOptions,
+ apiClient *helper.ApiClient) errors.Error {
+ var repo models.BitbucketRepo
+ // validate the op and set name=owner/repo if this is from advanced mode or bpV100
+ err := tasks.ValidateTaskOptions(op)
+ if err != nil {
+ return err
+ }
+ logger := taskCtx.GetLogger()
+ // for advanced mode or others which we only have name, for bp v200, we have githubId
+ err = taskCtx.GetDal().First(&repo, dal.Where(
+ "connection_id = ? AND bitbucket_id = ?",
+ op.ConnectionId, op.FullName))
+ if err == nil {
+ if op.TransformationRuleId == 0 {
+ op.TransformationRuleId = repo.TransformationRuleId
+ }
+ } else {
+ if taskCtx.GetDal().IsErrorNotFound(err) && op.FullName != "" {
+ var repo *tasks.BitbucketApiRepo
Review Comment:
request repo from bitbucket api
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@devlake.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [incubator-devlake] likyh commented on a diff in pull request #4373: feat: add increment & timeFilter for bitbucket
Posted by "likyh (via GitHub)" <gi...@apache.org>.
likyh commented on code in PR #4373:
URL: https://github.com/apache/incubator-devlake/pull/4373#discussion_r1106608040
##########
backend/plugins/bitbucket/api/scope.go:
##########
@@ -0,0 +1,213 @@
+/*
Review Comment:
this file imitates from https://github.com/apache/incubator-devlake/blob/main/backend/plugins/github/api/scope.go
##########
backend/plugins/bitbucket/api/transformation_rule.go:
##########
@@ -0,0 +1,132 @@
+/*
Review Comment:
this file imitates from https://github.com/apache/incubator-devlake/blob/main/backend/plugins/github/api/transformation_rule.go
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@devlake.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [incubator-devlake] likyh commented on a diff in pull request #4373: feat: add increment & timeFilter for bitbucket
Posted by "likyh (via GitHub)" <gi...@apache.org>.
likyh commented on code in PR #4373:
URL: https://github.com/apache/incubator-devlake/pull/4373#discussion_r1106606947
##########
backend/plugins/bitbucket/api/blueprint_v200.go:
##########
@@ -0,0 +1,198 @@
+/*
Review Comment:
this file imitates from https://github.com/apache/incubator-devlake/blob/main/backend/plugins/github/api/blueprint_v200.go
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@devlake.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org