You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by zh...@apache.org on 2022/10/11 04:45:22 UTC
[incubator-devlake] tag v0.14.1 created (now 214ad97e)
This is an automated email from the ASF dual-hosted git repository.
zhangliang2022 pushed a change to tag v0.14.1
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
at 214ad97e (commit)
This tag includes the following new commits:
new 214ad97e fix: bitbucket generated wrong clone url (#3315)
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[incubator-devlake] 01/01: fix: bitbucket generated wrong clone url (#3315)
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhangliang2022 pushed a commit to tag v0.14.1
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit 214ad97e4ab084ab46ef1248c65b21274dbe7e7a
Author: mindlesscloud <li...@merico.dev>
AuthorDate: Sun Oct 9 20:32:02 2022 +0800
fix: bitbucket generated wrong clone url (#3315)
* fix: bitbucket generated wrong clone url
* fix: e2e test
* refactor: unit test instead of e2e
---
plugins/bitbucket/api/blueprint.go | 111 +++++++++++++--------
plugins/bitbucket/api/blueprint_test.go | 57 +++++++++++
.../e2e/snapshot_tables/_tool_bitbucket_repos.csv | 2 +-
plugins/bitbucket/e2e/snapshot_tables/boards.csv | 2 +-
plugins/bitbucket/e2e/snapshot_tables/repos.csv | 2 +-
plugins/bitbucket/tasks/api_common.go | 4 +-
plugins/bitbucket/tasks/repo_extractor.go | 2 +-
7 files changed, 131 insertions(+), 49 deletions(-)
diff --git a/plugins/bitbucket/api/blueprint.go b/plugins/bitbucket/api/blueprint.go
index c0f8ebe5..825670e6 100644
--- a/plugins/bitbucket/api/blueprint.go
+++ b/plugins/bitbucket/api/blueprint.go
@@ -21,13 +21,14 @@ import (
"context"
"encoding/json"
"fmt"
- "github.com/apache/incubator-devlake/errors"
"io"
"net/http"
"net/url"
+ "path"
"strings"
"time"
+ "github.com/apache/incubator-devlake/errors"
"github.com/apache/incubator-devlake/models/domainlayer/didgen"
"github.com/apache/incubator-devlake/plugins/bitbucket/models"
"github.com/apache/incubator-devlake/plugins/bitbucket/tasks"
@@ -36,7 +37,66 @@ import (
"github.com/apache/incubator-devlake/utils"
)
+type repoGetter func(connectionId uint64, owner, repo string) (string, string, errors.Error)
+
func MakePipelinePlan(subtaskMetas []core.SubTaskMeta, connectionId uint64, scope []*core.BlueprintScopeV100) (core.PipelinePlan, errors.Error) {
+ return makePipelinePlan(subtaskMetas, connectionId, getBitbucketApiRepo, scope)
+}
+func getBitbucketApiRepo(connectionId uint64, owner, repo string) (string, string, errors.Error) {
+ // here is the tricky part, we have to obtain the repo id beforehand
+ connection := new(models.BitbucketConnection)
+ err := connectionHelper.FirstById(connection, connectionId)
+ if err != nil {
+ return "", "", err
+ }
+ tokens := strings.Split(connection.GetEncodedToken(), ",")
+ if len(tokens) == 0 {
+ return "", "", errors.Default.New("no token")
+ }
+ token := tokens[0]
+ apiClient, err := helper.NewApiClient(
+ context.TODO(),
+ connection.Endpoint,
+ map[string]string{
+ "Authorization": fmt.Sprintf("Basic %s", token),
+ },
+ 10*time.Second,
+ connection.Proxy,
+ basicRes,
+ )
+ if err != nil {
+ return "", "", err
+ }
+
+ res, err := apiClient.Get(path.Join("repositories", owner, repo), nil, nil)
+ if err != nil {
+ return "", "", err
+ }
+ defer res.Body.Close()
+ if res.StatusCode != http.StatusOK {
+ return "", "", errors.Default.New(fmt.Sprintf(
+ "unexpected status code when requesting repo detail %d %s",
+ res.StatusCode, res.Request.URL.String(),
+ ))
+ }
+ body, err := errors.Convert01(io.ReadAll(res.Body))
+ if err != nil {
+ return "", "", err
+ }
+ apiRepo := new(tasks.BitbucketApiRepo)
+ err = errors.Convert(json.Unmarshal(body, apiRepo))
+ if err != nil {
+ return "", "", err
+ }
+ for _, u := range apiRepo.Links.Clone {
+ if u.Name == "https" {
+ return u.Href, connection.Password, nil
+ }
+ }
+ return "", "", errors.Default.New("no clone url")
+}
+
+func makePipelinePlan(subtaskMetas []core.SubTaskMeta, connectionId uint64, getter repoGetter, scope []*core.BlueprintScopeV100) (core.PipelinePlan, errors.Error) {
var err errors.Error
plan := make(core.PipelinePlan, len(scope))
for i, scopeElem := range scope {
@@ -93,58 +153,23 @@ func MakePipelinePlan(subtaskMetas []core.SubTaskMeta, connectionId uint64, scop
})
// collect git data by gitextractor if CODE was requested
if utils.StringsContains(scopeElem.Entities, core.DOMAIN_TYPE_CODE) {
- // here is the tricky part, we have to obtain the repo id beforehand
- connection := new(models.BitbucketConnection)
- err = connectionHelper.FirstById(connection, connectionId)
- if err != nil {
- return nil, err
- }
- token := strings.Split(connection.GetEncodedToken(), ",")[0]
- apiClient, err := helper.NewApiClient(
- context.TODO(),
- connection.Endpoint,
- map[string]string{
- "Authorization": fmt.Sprintf("Basic %s", token),
- },
- 10*time.Second,
- connection.Proxy,
- basicRes,
- )
- if err != nil {
- return nil, err
- }
- res, err := apiClient.Get(fmt.Sprintf("repositories/%s/%s", op.Owner, op.Repo), nil, nil)
- if err != nil {
- return nil, err
+ original, password, err1 := getter(connectionId, op.Owner, op.Repo)
+ if err1 != nil {
+ return nil, err1
}
- defer res.Body.Close()
- if res.StatusCode != http.StatusOK {
- return nil, errors.Default.New(fmt.Sprintf(
- "unexpected status code when requesting repo detail %d %s",
- res.StatusCode, res.Request.URL.String(),
- ))
- }
- body, err := errors.Convert01(io.ReadAll(res.Body))
- if err != nil {
- return nil, err
- }
- apiRepo := new(tasks.BitbucketApiRepo)
- err = errors.Convert(json.Unmarshal(body, apiRepo))
+ cloneUrl, err := errors.Convert01(url.Parse(original))
if err != nil {
return nil, err
}
- cloneUrl, err := errors.Convert01(url.Parse(apiRepo.Links.Clone[0].Href))
- if err != nil {
- return nil, err
- }
- cloneUrl.User = url.UserPassword("git", token)
+ cloneUrl.User = url.UserPassword(op.Owner, password)
stage = append(stage, &core.PipelineTask{
Plugin: "gitextractor",
Options: map[string]interface{}{
"url": cloneUrl.String(),
- "repoId": didgen.NewDomainIdGenerator(&models.BitbucketRepo{}).Generate(connectionId, apiRepo.BitbucketId),
+ "repoId": didgen.NewDomainIdGenerator(&models.BitbucketRepo{}).Generate(connectionId, fmt.Sprintf("%s/%s", op.Owner, op.Repo)),
},
})
+
}
plan[i] = stage
}
diff --git a/plugins/bitbucket/api/blueprint_test.go b/plugins/bitbucket/api/blueprint_test.go
new file mode 100644
index 00000000..14e284a3
--- /dev/null
+++ b/plugins/bitbucket/api/blueprint_test.go
@@ -0,0 +1,57 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package api
+
+import (
+ "github.com/apache/incubator-devlake/mocks"
+ "testing"
+
+ "github.com/apache/incubator-devlake/errors"
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/stretchr/testify/assert"
+)
+
+func TestMakePipelinePlan(t *testing.T) {
+ var mockGetter repoGetter
+ mockGetter = func(connectionId uint64, owner, repo string) (string, string, errors.Error) {
+ return "https://thenicetgp@bitbucket.org/thenicetgp/lake.git", "secret", nil
+ }
+ scope := &core.BlueprintScopeV100{
+ Entities: []string{core.DOMAIN_TYPE_CODE, core.DOMAIN_TYPE_TICKET, core.DOMAIN_TYPE_CODE_REVIEW, core.DOMAIN_TYPE_CROSS},
+ Options: []byte(`{
+ "owner": "thenicetgp",
+ "repo": "lake"
+ }`),
+ Transformation: nil,
+ }
+ mockMeta := mocks.NewPluginMeta(t)
+ mockMeta.On("RootPkgPath").Return("github.com/apache/incubator-devlake/plugins/bitbucket")
+ err := core.RegisterPlugin("bitbucket", mockMeta)
+ assert.Nil(t, err)
+ plan, err := makePipelinePlan(nil, 1, mockGetter, []*core.BlueprintScopeV100{scope})
+ assert.Nil(t, err)
+ for _, stage := range plan {
+ for _, task := range stage {
+ if task.Plugin == "gitextractor" {
+ assert.Equal(t, task.Options["url"], "https://thenicetgp:secret@bitbucket.org/thenicetgp/lake.git")
+ return
+ }
+ }
+ }
+ t.Fatal("no gitextractor plugin")
+}
diff --git a/plugins/bitbucket/e2e/snapshot_tables/_tool_bitbucket_repos.csv b/plugins/bitbucket/e2e/snapshot_tables/_tool_bitbucket_repos.csv
index a3883fbb..0a13fe85 100644
--- a/plugins/bitbucket/e2e/snapshot_tables/_tool_bitbucket_repos.csv
+++ b/plugins/bitbucket/e2e/snapshot_tables/_tool_bitbucket_repos.csv
@@ -1,2 +1,2 @@
connection_id,bitbucket_id,name,html_url,description,owner_id,language,created_date,updated_date,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
-1,repositories/panjf2000/ants,panjf2000/ants,https://bitbucket.org/panjf2000/ants,,,,2022-06-17T03:27:18.865+00:00,2022-08-12T15:40:12.409+00:00,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_bitbucket_api_repositories,2,
+1,panjf2000/ants,panjf2000/ants,https://bitbucket.org/panjf2000/ants,,,,2022-06-17T03:27:18.865+00:00,2022-08-12T15:40:12.409+00:00,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_bitbucket_api_repositories,2,
diff --git a/plugins/bitbucket/e2e/snapshot_tables/boards.csv b/plugins/bitbucket/e2e/snapshot_tables/boards.csv
index d4cbb227..99f1824d 100644
--- a/plugins/bitbucket/e2e/snapshot_tables/boards.csv
+++ b/plugins/bitbucket/e2e/snapshot_tables/boards.csv
@@ -1,2 +1,2 @@
id,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark,name,description,url,created_date
-bitbucket:BitbucketRepo:1:repositories/panjf2000/ants,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_bitbucket_api_repositories,2,,panjf2000/ants,,https://bitbucket.org/panjf2000/ants/issues,2022-06-17T03:27:18.865+00:00
+bitbucket:BitbucketRepo:1:panjf2000/ants,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_bitbucket_api_repositories,2,,panjf2000/ants,,https://bitbucket.org/panjf2000/ants/issues,2022-06-17T03:27:18.865+00:00
diff --git a/plugins/bitbucket/e2e/snapshot_tables/repos.csv b/plugins/bitbucket/e2e/snapshot_tables/repos.csv
index 32c0ecc0..646a4390 100644
--- a/plugins/bitbucket/e2e/snapshot_tables/repos.csv
+++ b/plugins/bitbucket/e2e/snapshot_tables/repos.csv
@@ -1,2 +1,2 @@
id,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark,name,url,description,owner_id,language,forked_from,deleted
-bitbucket:BitbucketRepo:1:repositories/panjf2000/ants,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_bitbucket_api_repositories,2,,panjf2000/ants,https://bitbucket.org/panjf2000/ants,,,,,0
+bitbucket:BitbucketRepo:1:panjf2000/ants,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_bitbucket_api_repositories,2,,panjf2000/ants,https://bitbucket.org/panjf2000/ants,,,,,0
diff --git a/plugins/bitbucket/tasks/api_common.go b/plugins/bitbucket/tasks/api_common.go
index 000089da..b931a8a4 100644
--- a/plugins/bitbucket/tasks/api_common.go
+++ b/plugins/bitbucket/tasks/api_common.go
@@ -113,7 +113,7 @@ func GetPullRequestsIterator(taskCtx core.SubTaskContext) (*helper.DalCursorIter
dal.From("_tool_bitbucket_pull_requests bpr"),
dal.Where(
`bpr.repo_id = ? and bpr.connection_id = ?`,
- "repositories/"+data.Options.Owner+"/"+data.Options.Repo, data.Options.ConnectionId,
+ fmt.Sprintf("%s/%s", data.Options.Owner, data.Options.Repo), data.Options.ConnectionId,
),
}
// construct the input iterator
@@ -133,7 +133,7 @@ func GetIssuesIterator(taskCtx core.SubTaskContext) (*helper.DalCursorIterator,
dal.From("_tool_bitbucket_issues bpr"),
dal.Where(
`bpr.repo_id = ? and bpr.connection_id = ?`,
- "repositories/"+data.Options.Owner+"/"+data.Options.Repo, data.Options.ConnectionId,
+ fmt.Sprintf("%s/%s", data.Options.Owner, data.Options.Repo), data.Options.ConnectionId,
),
}
// construct the input iterator
diff --git a/plugins/bitbucket/tasks/repo_extractor.go b/plugins/bitbucket/tasks/repo_extractor.go
index d79f5c36..c39e19c6 100644
--- a/plugins/bitbucket/tasks/repo_extractor.go
+++ b/plugins/bitbucket/tasks/repo_extractor.go
@@ -97,7 +97,7 @@ func ExtractApiRepositories(taskCtx core.SubTaskContext) errors.Error {
results := make([]interface{}, 0, 1)
bitbucketRepository := &models.BitbucketRepo{
ConnectionId: data.Options.ConnectionId,
- BitbucketId: "repositories/" + body.FullName,
+ BitbucketId: body.FullName,
Name: body.FullName,
HTMLUrl: body.Links.Html.Href,
Description: body.Description,