You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by kl...@apache.org on 2022/07/21 07:16:50 UTC
[incubator-devlake] branch main updated: fix: running gitextractor as a standalone process (#2557)
This is an automated email from the ASF dual-hosted git repository.
klesh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new b6c9ff1e fix: running gitextractor as a standalone process (#2557)
b6c9ff1e is described below
commit b6c9ff1e0fc9a2a39e949cde11dffefc20321809
Author: mindlesscloud <li...@merico.dev>
AuthorDate: Thu Jul 21 15:16:46 2022 +0800
fix: running gitextractor as a standalone process (#2557)
---
plugins/gitextractor/gitextractor.go | 6 +++---
plugins/gitextractor/main.go | 6 +++---
plugins/gitextractor/parser/repo.go | 27 ++++++++++++------------
plugins/gitextractor/parser/repo_creator.go | 17 +++++++--------
plugins/gitextractor/tasks/git_repo_collector.go | 4 ++--
5 files changed, 29 insertions(+), 31 deletions(-)
diff --git a/plugins/gitextractor/gitextractor.go b/plugins/gitextractor/gitextractor.go
index e0a6566c..146e869d 100644
--- a/plugins/gitextractor/gitextractor.go
+++ b/plugins/gitextractor/gitextractor.go
@@ -57,7 +57,7 @@ func (plugin GitExtractor) PrepareTaskData(taskCtx core.TaskContext, options map
return nil, err
}
storage := store.NewDatabase(taskCtx, op.Url)
- repo, err := newGitRepo(taskCtx, storage, op)
+ repo, err := newGitRepo(taskCtx.GetLogger(), storage, op)
if err != nil {
return nil, err
}
@@ -77,10 +77,10 @@ func (plugin GitExtractor) RootPkgPath() string {
return "github.com/apache/incubator-devlake/plugins/gitextractor"
}
-func newGitRepo(ctx core.TaskContext, storage models.Store, op tasks.GitExtractorOptions) (*parser.GitRepo, error) {
+func newGitRepo(logger core.Logger, storage models.Store, op tasks.GitExtractorOptions) (*parser.GitRepo, error) {
var err error
var repo *parser.GitRepo
- p := parser.NewGitRepoCreator(storage, ctx)
+ p := parser.NewGitRepoCreator(storage, logger)
if strings.HasPrefix(op.Url, "http") {
repo, err = p.CloneOverHTTP(op.RepoId, op.Url, op.User, op.Password, op.Proxy)
} else if url := strings.TrimPrefix(op.Url, "ssh://"); strings.HasPrefix(url, "git@") {
diff --git a/plugins/gitextractor/main.go b/plugins/gitextractor/main.go
index df435f23..330b004b 100644
--- a/plugins/gitextractor/main.go
+++ b/plugins/gitextractor/main.go
@@ -20,12 +20,12 @@ package main
import (
"context"
"flag"
- "github.com/apache/incubator-devlake/logger"
- "github.com/apache/incubator-devlake/plugins/gitextractor/tasks"
"github.com/apache/incubator-devlake/config"
+ "github.com/apache/incubator-devlake/logger"
"github.com/apache/incubator-devlake/plugins/gitextractor/models"
"github.com/apache/incubator-devlake/plugins/gitextractor/store"
+ "github.com/apache/incubator-devlake/plugins/gitextractor/tasks"
"github.com/apache/incubator-devlake/plugins/helper"
"gorm.io/driver/mysql"
"gorm.io/gorm"
@@ -74,7 +74,7 @@ func main() {
"git extractor",
nil,
)
- repo, err := newGitRepo(subTaskCtx.TaskContext(), storage, tasks.GitExtractorOptions{
+ repo, err := newGitRepo(log, storage, tasks.GitExtractorOptions{
RepoId: *id,
Url: *url,
User: *user,
diff --git a/plugins/gitextractor/parser/repo.go b/plugins/gitextractor/parser/repo.go
index 44368cbb..4d812e8f 100644
--- a/plugins/gitextractor/parser/repo.go
+++ b/plugins/gitextractor/parser/repo.go
@@ -29,7 +29,6 @@ import (
type GitRepo struct {
store models.Store
- ctx context.Context
logger core.Logger
id string
repo *git.Repository
@@ -66,7 +65,7 @@ func (r *GitRepo) CountTags() (int, error) {
return len(tags), nil
}
-func (r *GitRepo) CountBranches() (int, error) {
+func (r *GitRepo) CountBranches(ctx context.Context) (int, error) {
var branchIter *git.BranchIterator
branchIter, err := r.repo.NewBranchIterator(git.BranchAll)
if err != nil {
@@ -75,8 +74,8 @@ func (r *GitRepo) CountBranches() (int, error) {
count := 0
err = branchIter.ForEach(func(branch *git.Branch, branchType git.BranchType) error {
select {
- case <-r.ctx.Done():
- return r.ctx.Err()
+ case <-ctx.Done():
+ return ctx.Err()
default:
}
if branch.IsBranch() || branch.IsRemote() {
@@ -87,7 +86,7 @@ func (r *GitRepo) CountBranches() (int, error) {
return count, err
}
-func (r *GitRepo) CountCommits() (int, error) {
+func (r *GitRepo) CountCommits(ctx context.Context) (int, error) {
odb, err := r.repo.Odb()
if err != nil {
return 0, err
@@ -95,8 +94,8 @@ func (r *GitRepo) CountCommits() (int, error) {
count := 0
err = odb.ForEach(func(id *git.Oid) error {
select {
- case <-r.ctx.Done():
- return r.ctx.Err()
+ case <-ctx.Done():
+ return ctx.Err()
default:
}
commit, _ := r.repo.LookupCommit(id)
@@ -111,8 +110,8 @@ func (r *GitRepo) CountCommits() (int, error) {
func (r *GitRepo) CollectTags(subtaskCtx core.SubTaskContext) error {
return r.repo.Tags.Foreach(func(name string, id *git.Oid) error {
select {
- case <-r.ctx.Done():
- return r.ctx.Err()
+ case <-subtaskCtx.GetContext().Done():
+ return subtaskCtx.GetContext().Err()
default:
}
var err1 error
@@ -124,7 +123,7 @@ func (r *GitRepo) CollectTags(subtaskCtx core.SubTaskContext) error {
} else {
tagCommit = id.String()
}
- r.logger.Info("tagCommit", tagCommit)
+ r.logger.Info("tagCommit:%s", tagCommit)
if tagCommit != "" {
ref := &code.Ref{
DomainEntity: domainlayer.DomainEntity{Id: fmt.Sprintf("%s:%s", r.id, name)},
@@ -151,8 +150,8 @@ func (r *GitRepo) CollectBranches(subtaskCtx core.SubTaskContext) error {
}
return repoInter.ForEach(func(branch *git.Branch, branchType git.BranchType) error {
select {
- case <-r.ctx.Done():
- return r.ctx.Err()
+ case <-subtaskCtx.GetContext().Done():
+ return subtaskCtx.GetContext().Err()
default:
}
if branch.IsBranch() || branch.IsRemote() {
@@ -194,8 +193,8 @@ func (r *GitRepo) CollectCommits(subtaskCtx core.SubTaskContext) error {
}
return odb.ForEach(func(id *git.Oid) error {
select {
- case <-r.ctx.Done():
- return r.ctx.Err()
+ case <-subtaskCtx.GetContext().Done():
+ return subtaskCtx.GetContext().Err()
default:
}
commit, _ := r.repo.LookupCommit(id)
diff --git a/plugins/gitextractor/parser/repo_creator.go b/plugins/gitextractor/parser/repo_creator.go
index d05f3914..62ab06de 100644
--- a/plugins/gitextractor/parser/repo_creator.go
+++ b/plugins/gitextractor/parser/repo_creator.go
@@ -30,14 +30,14 @@ const (
)
type GitRepoCreator struct {
- store models.Store
- taskCtx core.TaskContext
+ store models.Store
+ logger core.Logger
}
-func NewGitRepoCreator(store models.Store, taskCtx core.TaskContext) *GitRepoCreator {
+func NewGitRepoCreator(store models.Store, logger core.Logger) *GitRepoCreator {
return &GitRepoCreator{
- store: store,
- taskCtx: taskCtx,
+ store: store,
+ logger: logger,
}
}
@@ -49,12 +49,11 @@ func (l *GitRepoCreator) LocalRepo(repoPath, repoId string) (*GitRepo, error) {
return l.newGitRepo(repoId, repo), nil
}
-func (l *GitRepoCreator) newGitRepo(id string, repo *git.Repository) *GitRepo {
+func (l *GitRepoCreator) newGitRepo(repoId string, repo *git.Repository) *GitRepo {
return &GitRepo{
store: l.store,
- ctx: l.taskCtx.GetContext(),
- logger: l.taskCtx.GetLogger(),
- id: id,
+ logger: l.logger,
+ id: repoId,
repo: repo,
}
}
diff --git a/plugins/gitextractor/tasks/git_repo_collector.go b/plugins/gitextractor/tasks/git_repo_collector.go
index 3432ed20..12816cba 100644
--- a/plugins/gitextractor/tasks/git_repo_collector.go
+++ b/plugins/gitextractor/tasks/git_repo_collector.go
@@ -54,7 +54,7 @@ func (o GitExtractorOptions) Valid() error {
func CollectGitCommits(subTaskCtx core.SubTaskContext) error {
repo := getGitRepo(subTaskCtx)
- if count, err := repo.CountCommits(); err != nil {
+ if count, err := repo.CountCommits(subTaskCtx.GetContext()); err != nil {
subTaskCtx.GetLogger().Error("unable to get commit count: %v", err)
subTaskCtx.SetProgress(0, -1)
} else {
@@ -65,7 +65,7 @@ func CollectGitCommits(subTaskCtx core.SubTaskContext) error {
func CollectGitBranches(subTaskCtx core.SubTaskContext) error {
repo := getGitRepo(subTaskCtx)
- if count, err := repo.CountBranches(); err != nil {
+ if count, err := repo.CountBranches(subTaskCtx.GetContext()); err != nil {
subTaskCtx.GetLogger().Error("unable to get branch count: %v", err)
subTaskCtx.SetProgress(0, -1)
} else {