You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by wa...@apache.org on 2023/01/06 05:54:35 UTC

[incubator-devlake] branch fix-bitbucket updated: feat: add bitbucket pullrequest_commits collector&extractor&convertor (#3337)

This is an automated email from the ASF dual-hosted git repository.

warren pushed a commit to branch fix-bitbucket
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git


The following commit(s) were added to refs/heads/fix-bitbucket by this push:
     new 4384091df feat: add bitbucket pullrequest_commits collector&extractor&convertor (#3337)
4384091df is described below

commit 4384091df63a7cb6e6914e5ca350fa672b242502
Author: tsoc <47...@users.noreply.github.com>
AuthorDate: Fri Jan 6 13:54:30 2023 +0800

    feat: add bitbucket pullrequest_commits collector&extractor&convertor (#3337)
    
    * feat: add bitbucket pullrequest_commits collector&extractor&convertor
    
    * fix: bitbucket commit response do not have committer
    
    Co-authored-by: Klesh Wong <zh...@merico.dev>
---
 plugins/bitbucket/impl/impl.go                     |  15 +-
 ...egister.go => 20221008_add_prcommits_tables.go} |  27 ++--
 .../{register.go => archived/pr_commit.go}         |  21 +--
 .../bitbucket/models/migrationscripts/register.go  |   1 +
 .../{migrationscripts/register.go => pr_commit.go} |  21 +--
 plugins/bitbucket/tasks/pr_commit_collector.go     |  69 +++++++++
 plugins/bitbucket/tasks/pr_commit_convertor.go     |  86 +++++++++++
 plugins/bitbucket/tasks/pr_commit_extractor.go     | 159 +++++++++++++++++++++
 8 files changed, 369 insertions(+), 30 deletions(-)

diff --git a/plugins/bitbucket/impl/impl.go b/plugins/bitbucket/impl/impl.go
index 54da3c035..95c41518d 100644
--- a/plugins/bitbucket/impl/impl.go
+++ b/plugins/bitbucket/impl/impl.go
@@ -69,22 +69,33 @@ func (plugin Bitbucket) SubTaskMetas() []core.SubTaskMeta {
 	return []core.SubTaskMeta{
 		tasks.CollectApiRepoMeta,
 		tasks.ExtractApiRepoMeta,
+
 		tasks.CollectApiPullRequestsMeta,
 		tasks.ExtractApiPullRequestsMeta,
-		tasks.CollectApiIssuesMeta,
-		tasks.ExtractApiIssuesMeta,
+
 		tasks.CollectApiPrCommentsMeta,
 		tasks.ExtractApiPrCommentsMeta,
+
+		tasks.CollectApiPrCommitsMeta,
+		tasks.ExtractApiPrCommitsMeta,
+
+		tasks.CollectApiIssuesMeta,
+		tasks.ExtractApiIssuesMeta,
+
 		tasks.CollectApiIssueCommentsMeta,
 		tasks.ExtractApiIssueCommentsMeta,
+
 		tasks.CollectApiPipelinesMeta,
 		tasks.ExtractApiPipelinesMeta,
+
 		tasks.CollectApiDeploymentsMeta,
 		tasks.ExtractApiDeploymentsMeta,
+
 		tasks.ConvertRepoMeta,
 		tasks.ConvertAccountsMeta,
 		tasks.ConvertPullRequestsMeta,
 		tasks.ConvertPrCommentsMeta,
+		tasks.ConvertPrCommitsMeta,
 		tasks.ConvertIssuesMeta,
 		tasks.ConvertIssueCommentsMeta,
 		tasks.ConvertPipelineMeta,
diff --git a/plugins/bitbucket/models/migrationscripts/register.go b/plugins/bitbucket/models/migrationscripts/20221008_add_prcommits_tables.go
similarity index 57%
copy from plugins/bitbucket/models/migrationscripts/register.go
copy to plugins/bitbucket/models/migrationscripts/20221008_add_prcommits_tables.go
index d6d7fc8eb..d3a0f0888 100644
--- a/plugins/bitbucket/models/migrationscripts/register.go
+++ b/plugins/bitbucket/models/migrationscripts/20221008_add_prcommits_tables.go
@@ -18,15 +18,26 @@ limitations under the License.
 package migrationscripts
 
 import (
-	"github.com/apache/incubator-devlake/plugins/core"
+	"context"
+	"github.com/apache/incubator-devlake/errors"
+	"github.com/apache/incubator-devlake/plugins/bitbucket/models/migrationscripts/archived"
+	"gorm.io/gorm"
 )
 
-// All return all the migration scripts
-func All() []core.MigrationScript {
-	return []core.MigrationScript{
-		new(addInitTables20220803),
-		new(addPipeline20220914),
-		new(addDeployment20221013),
-		new(addRepoIdAndCommitShaField20221014),
+type addPrCommits20221008 struct{}
+
+func (*addPrCommits20221008) Up(ctx context.Context, db *gorm.DB) errors.Error {
+	err := db.Migrator().AutoMigrate(&archived.BitbucketPrCommit{})
+	if err != nil {
+		return errors.Convert(err)
 	}
+	return nil
+}
+
+func (*addPrCommits20221008) Version() uint64 {
+	return 20221008182354
+}
+
+func (*addPrCommits20221008) Name() string {
+	return "bitbucket add _tool_bitbucket_pull_requests_commits table"
 }
diff --git a/plugins/bitbucket/models/migrationscripts/register.go b/plugins/bitbucket/models/migrationscripts/archived/pr_commit.go
similarity index 66%
copy from plugins/bitbucket/models/migrationscripts/register.go
copy to plugins/bitbucket/models/migrationscripts/archived/pr_commit.go
index d6d7fc8eb..0721a9606 100644
--- a/plugins/bitbucket/models/migrationscripts/register.go
+++ b/plugins/bitbucket/models/migrationscripts/archived/pr_commit.go
@@ -15,18 +15,19 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package migrationscripts
+package archived
 
 import (
-	"github.com/apache/incubator-devlake/plugins/core"
+	"github.com/apache/incubator-devlake/models/migrationscripts/archived"
 )
 
-// All return all the migration scripts
-func All() []core.MigrationScript {
-	return []core.MigrationScript{
-		new(addInitTables20220803),
-		new(addPipeline20220914),
-		new(addDeployment20221013),
-		new(addRepoIdAndCommitShaField20221014),
-	}
+type BitbucketPrCommit struct {
+	ConnectionId  uint64 `gorm:"primaryKey"`
+	CommitSha     string `gorm:"primaryKey;type:varchar(40)"`
+	PullRequestId int    `gorm:"primaryKey;autoIncrement:false"`
+	archived.NoPKModel
+}
+
+func (BitbucketPrCommit) TableName() string {
+	return "_tool_bitbucket_pull_request_commits"
 }
diff --git a/plugins/bitbucket/models/migrationscripts/register.go b/plugins/bitbucket/models/migrationscripts/register.go
index d6d7fc8eb..d5f5374b1 100644
--- a/plugins/bitbucket/models/migrationscripts/register.go
+++ b/plugins/bitbucket/models/migrationscripts/register.go
@@ -26,6 +26,7 @@ func All() []core.MigrationScript {
 	return []core.MigrationScript{
 		new(addInitTables20220803),
 		new(addPipeline20220914),
+		new(addPrCommits20221008),
 		new(addDeployment20221013),
 		new(addRepoIdAndCommitShaField20221014),
 	}
diff --git a/plugins/bitbucket/models/migrationscripts/register.go b/plugins/bitbucket/models/pr_commit.go
similarity index 67%
copy from plugins/bitbucket/models/migrationscripts/register.go
copy to plugins/bitbucket/models/pr_commit.go
index d6d7fc8eb..262093242 100644
--- a/plugins/bitbucket/models/migrationscripts/register.go
+++ b/plugins/bitbucket/models/pr_commit.go
@@ -15,18 +15,19 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package migrationscripts
+package models
 
 import (
-	"github.com/apache/incubator-devlake/plugins/core"
+	"github.com/apache/incubator-devlake/models/common"
 )
 
-// All return all the migration scripts
-func All() []core.MigrationScript {
-	return []core.MigrationScript{
-		new(addInitTables20220803),
-		new(addPipeline20220914),
-		new(addDeployment20221013),
-		new(addRepoIdAndCommitShaField20221014),
-	}
+type BitbucketPrCommit struct {
+	ConnectionId  uint64 `gorm:"primaryKey"`
+	CommitSha     string `gorm:"primaryKey;type:varchar(40)"`
+	PullRequestId int    `gorm:"primaryKey;autoIncrement:false"`
+	common.NoPKModel
+}
+
+func (BitbucketPrCommit) TableName() string {
+	return "_tool_bitbucket_pull_request_commits"
 }
diff --git a/plugins/bitbucket/tasks/pr_commit_collector.go b/plugins/bitbucket/tasks/pr_commit_collector.go
new file mode 100644
index 000000000..f3464c8e3
--- /dev/null
+++ b/plugins/bitbucket/tasks/pr_commit_collector.go
@@ -0,0 +1,69 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+	"fmt"
+	"net/url"
+
+	"github.com/apache/incubator-devlake/errors"
+	"github.com/apache/incubator-devlake/plugins/core"
+	"github.com/apache/incubator-devlake/plugins/helper"
+)
+
+const RAW_PULL_REQUEST_COMMITS_TABLE = "bitbucket_api_pull_request_commits"
+
+var CollectApiPrCommitsMeta = core.SubTaskMeta{
+	Name:             "collectApiPullRequestCommits",
+	EntryPoint:       CollectApiPullRequestCommits,
+	EnabledByDefault: true,
+	Description:      "Collect PullRequestCommits data from Bitbucket api",
+	DomainTypes:      []string{core.DOMAIN_TYPE_CODE_REVIEW},
+}
+
+func CollectApiPullRequestCommits(taskCtx core.SubTaskContext) errors.Error {
+	rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_PULL_REQUEST_COMMITS_TABLE)
+
+	iterator, err := GetPullRequestsIterator(taskCtx)
+	if err != nil {
+		return err
+	}
+	defer iterator.Close()
+
+	collector, err := helper.NewApiCollector(helper.ApiCollectorArgs{
+		RawDataSubTaskArgs: *rawDataSubTaskArgs,
+		ApiClient:          data.ApiClient,
+		PageSize:           100,
+		Incremental:        false,
+		Input:              iterator,
+		UrlTemplate:        "repositories/{{ .Params.Owner }}/{{ .Params.Repo }}/pullrequests/{{ .Input.BitbucketId }}/commits",
+		Query: func(reqData *helper.RequestData) (url.Values, errors.Error) {
+			query := url.Values{}
+			query.Set("state", "all")
+			query.Set("pagelen", fmt.Sprintf("%v", reqData.Pager.Size))
+
+			return query, nil
+		},
+		ResponseParser: GetRawMessageFromResponse,
+	})
+
+	if err != nil {
+		return err
+	}
+	return collector.Execute()
+}
diff --git a/plugins/bitbucket/tasks/pr_commit_convertor.go b/plugins/bitbucket/tasks/pr_commit_convertor.go
new file mode 100644
index 000000000..5e8f58951
--- /dev/null
+++ b/plugins/bitbucket/tasks/pr_commit_convertor.go
@@ -0,0 +1,86 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+	"reflect"
+
+	"github.com/apache/incubator-devlake/errors"
+	"github.com/apache/incubator-devlake/models/domainlayer/code"
+	"github.com/apache/incubator-devlake/models/domainlayer/didgen"
+	bitbucketModels "github.com/apache/incubator-devlake/plugins/bitbucket/models"
+	"github.com/apache/incubator-devlake/plugins/core"
+	"github.com/apache/incubator-devlake/plugins/core/dal"
+	"github.com/apache/incubator-devlake/plugins/helper"
+)
+
+var ConvertPrCommitsMeta = core.SubTaskMeta{
+	Name:             "convertPullRequestCommits",
+	EntryPoint:       ConvertPullRequestCommits,
+	EnabledByDefault: true,
+	Description:      "Convert tool layer table bitbucket_pull_request_commits into  domain layer table pull_request_commits",
+	DomainTypes:      []string{core.DOMAIN_TYPE_CODE_REVIEW},
+}
+
+func ConvertPullRequestCommits(taskCtx core.SubTaskContext) (err errors.Error) {
+	db := taskCtx.GetDal()
+	data := taskCtx.GetData().(*BitbucketTaskData)
+	repoId := data.Repo.BitbucketId
+
+	pullIdGen := didgen.NewDomainIdGenerator(&bitbucketModels.BitbucketPullRequest{})
+
+	cursor, err := db.Cursor(
+		dal.From(&bitbucketModels.BitbucketPrCommit{}),
+		dal.Join(`left join _tool_bitbucket_pull_requests on _tool_bitbucket_pull_requests.bitbucket_id = _tool_bitbucket_pull_request_commits.pull_request_id`),
+		dal.Where("_tool_bitbucket_pull_requests.repo_id = ? and _tool_bitbucket_pull_requests.connection_id = ?", repoId, data.Options.ConnectionId),
+		dal.Orderby("pull_request_id ASC"),
+	)
+	if err != nil {
+		return err
+	}
+	defer cursor.Close()
+
+	converter, err := helper.NewDataConverter(helper.DataConverterArgs{
+		InputRowType: reflect.TypeOf(bitbucketModels.BitbucketPrCommit{}),
+		Input:        cursor,
+		RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
+			Ctx: taskCtx,
+			Params: BitbucketApiParams{
+				ConnectionId: data.Options.ConnectionId,
+				Owner:        data.Options.Owner,
+				Repo:         data.Options.Repo,
+			},
+			Table: RAW_PULL_REQUEST_COMMITS_TABLE,
+		},
+		Convert: func(inputRow interface{}) ([]interface{}, errors.Error) {
+			bitbucketPullRequestCommit := inputRow.(*bitbucketModels.BitbucketPrCommit)
+			domainPrCommit := &code.PullRequestCommit{
+				CommitSha:     bitbucketPullRequestCommit.CommitSha,
+				PullRequestId: pullIdGen.Generate(data.Options.ConnectionId, bitbucketPullRequestCommit.PullRequestId),
+			}
+			return []interface{}{
+				domainPrCommit,
+			}, nil
+		},
+	})
+	if err != nil {
+		return err
+	}
+
+	return converter.Execute()
+}
diff --git a/plugins/bitbucket/tasks/pr_commit_extractor.go b/plugins/bitbucket/tasks/pr_commit_extractor.go
new file mode 100644
index 000000000..38552d996
--- /dev/null
+++ b/plugins/bitbucket/tasks/pr_commit_extractor.go
@@ -0,0 +1,159 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+	"encoding/json"
+	"strings"
+	"time"
+
+	"github.com/apache/incubator-devlake/errors"
+	"github.com/apache/incubator-devlake/plugins/bitbucket/models"
+	"github.com/apache/incubator-devlake/plugins/core"
+	"github.com/apache/incubator-devlake/plugins/helper"
+)
+
+var ExtractApiPrCommitsMeta = core.SubTaskMeta{
+	Name:             "extractApiPullRequestCommits",
+	EntryPoint:       ExtractApiPullRequestCommits,
+	EnabledByDefault: true,
+	Description:      "Extract raw PullRequestCommits data into tool layer table bitbucket_commits",
+	DomainTypes:      []string{core.DOMAIN_TYPE_CODE_REVIEW},
+}
+
+type ApiPrCommitsResponse struct {
+	Type   string    `json:"type"`
+	Hash   string    `json:"hash"`
+	Date   time.Time `json:"date"`
+	Author struct {
+		Type string                   `json:"type"`
+		Raw  string                   `json:"raw"`
+		User BitbucketAccountResponse `json:"user"`
+	} `json:"author"`
+	Message string `json:"message"`
+	Summary struct {
+		Type   string `json:"type"`
+		Raw    string `json:"raw"`
+		Markup string `json:"markup"`
+		HTML   string `json:"html"`
+	} `json:"summary"`
+	Links struct {
+		Self struct {
+			Href string `json:"href"`
+		} `json:"self"`
+		HTML struct {
+			Href string `json:"href"`
+		} `json:"html"`
+	} `json:"links"`
+	Parents []struct {
+		Type  string `json:"type"`
+		Hash  string `json:"hash"`
+		Links struct {
+			Self struct {
+				Href string `json:"href"`
+			} `json:"self"`
+			HTML struct {
+				Href string `json:"href"`
+			} `json:"html"`
+		} `json:"links"`
+	} `json:"parents"`
+	Repository BitbucketApiRepo `json:"repository"`
+}
+
+func ExtractApiPullRequestCommits(taskCtx core.SubTaskContext) errors.Error {
+	data := taskCtx.GetData().(*BitbucketTaskData)
+	repoId := data.Repo.BitbucketId
+	extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{
+		RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
+			Ctx: taskCtx,
+			/*
+				This struct will be JSONEncoded and stored into database along with raw data itself, to identity minimal
+				set of data to be process, for example, we process JiraIssues by Board
+			*/
+			Params: BitbucketApiParams{
+				ConnectionId: data.Options.ConnectionId,
+				Owner:        data.Options.Owner,
+				Repo:         data.Options.Repo,
+			},
+			/*
+				Table store raw data
+			*/
+			Table: RAW_PULL_REQUEST_COMMITS_TABLE,
+		},
+		Extract: func(row *helper.RawData) ([]interface{}, errors.Error) {
+			apiPullRequestCommit := &ApiPrCommitsResponse{}
+			if strings.HasPrefix(string(row.Data), "Not Found") {
+				return nil, nil
+			}
+			err := errors.Convert(json.Unmarshal(row.Data, apiPullRequestCommit))
+			if err != nil {
+				return nil, err
+			}
+			pull := &BitbucketInput{}
+			err = errors.Convert(json.Unmarshal(row.Input, pull))
+			if err != nil {
+				return nil, err
+			}
+			// need to extract 2 kinds of entities here
+			results := make([]interface{}, 0, 3)
+			bitbucketRepoCommit := &models.BitbucketRepoCommit{
+				ConnectionId: data.Options.ConnectionId,
+				RepoId:       repoId,
+				CommitSha:    apiPullRequestCommit.Hash,
+			}
+			results = append(results, bitbucketRepoCommit)
+
+			bitbucketCommit, err := convertPullRequestCommit(apiPullRequestCommit, data.Options.ConnectionId)
+			if err != nil {
+				return nil, err
+			}
+			results = append(results, bitbucketCommit)
+
+			bitbucketPullRequestCommit := &models.BitbucketPrCommit{
+				ConnectionId:  data.Options.ConnectionId,
+				CommitSha:     apiPullRequestCommit.Hash,
+				PullRequestId: pull.BitbucketId,
+			}
+			if err != nil {
+				return nil, err
+			}
+			results = append(results, bitbucketPullRequestCommit)
+			return results, nil
+		},
+	})
+
+	if err != nil {
+		return err
+	}
+
+	return extractor.Execute()
+}
+
+func convertPullRequestCommit(prCommit *ApiPrCommitsResponse, connId uint64) (*models.BitbucketCommit, errors.Error) {
+	bitbucketCommit := &models.BitbucketCommit{
+		Sha:           prCommit.Hash,
+		Message:       prCommit.Message,
+		AuthorId:      prCommit.Author.User.AccountId,
+		AuthorName:    prCommit.Author.User.UserName,
+		AuthorEmail:   prCommit.Author.Raw,
+		AuthoredDate:  prCommit.Date,
+		CommittedDate: prCommit.Date,
+		Url:           prCommit.Links.Self.Href,
+	}
+	return bitbucketCommit, nil
+}