You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by kl...@apache.org on 2024/04/22 06:30:15 UTC
(incubator-devlake) branch kw-7343-azdo-git-clone-failed updated (4282ea09d -> e0dd911d5)
This is an automated email from the ASF dual-hosted git repository.
klesh pushed a change to branch kw-7343-azdo-git-clone-failed
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
discard 4282ea09d fix: git clone failing on azuredevops
new e0dd911d5 fix: git clone failing on azuredevops
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (4282ea09d)
\
N -- N -- N refs/heads/kw-7343-azdo-git-clone-failed (e0dd911d5)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
(incubator-devlake) 01/01: fix: git clone failing on azuredevops
Posted by kl...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
klesh pushed a commit to branch kw-7343-azdo-git-clone-failed
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit e0dd911d567f75b3ee79f8a86b0686c9f7c0ace7
Author: Klesh Wong <zh...@merico.dev>
AuthorDate: Mon Apr 22 14:28:00 2024 +0800
fix: git clone failing on azuredevops
---
.../plugins/azuredevops_go/api/blueprint_v200.go | 9 +++--
.../plugins/gitextractor/parser/clone_gitcli.go | 43 +++++++++++++---------
backend/plugins/gitextractor/parser/taskdata.go | 3 +-
.../python/plugins/azuredevops/azuredevops/main.py | 2 +-
.../python/pydevlake/pydevlake/pipeline_tasks.py | 5 ++-
5 files changed, 36 insertions(+), 26 deletions(-)
diff --git a/backend/plugins/azuredevops_go/api/blueprint_v200.go b/backend/plugins/azuredevops_go/api/blueprint_v200.go
index a9d178e47..f1df9be8c 100644
--- a/backend/plugins/azuredevops_go/api/blueprint_v200.go
+++ b/backend/plugins/azuredevops_go/api/blueprint_v200.go
@@ -136,10 +136,11 @@ func makePipelinePlanV200(
stage = append(stage, &coreModels.PipelineTask{
Plugin: "gitextractor",
Options: map[string]interface{}{
- "url": cloneUrl.String(),
- "name": azuredevopsRepo.Name,
- "repoId": didgen.NewDomainIdGenerator(&models.AzuredevopsRepo{}).Generate(connection.ID, azuredevopsRepo.Id),
- "proxy": connection.Proxy,
+ "url": cloneUrl.String(),
+ "name": azuredevopsRepo.Name,
+ "repoId": didgen.NewDomainIdGenerator(&models.AzuredevopsRepo{}).Generate(connection.ID, azuredevopsRepo.Id),
+ "proxy": connection.Proxy,
+ "noShallowClone": true,
},
})
}
diff --git a/backend/plugins/gitextractor/parser/clone_gitcli.go b/backend/plugins/gitextractor/parser/clone_gitcli.go
index 500a30c97..ba8ed1192 100644
--- a/backend/plugins/gitextractor/parser/clone_gitcli.go
+++ b/backend/plugins/gitextractor/parser/clone_gitcli.go
@@ -50,38 +50,45 @@ func NewGitcliCloner(basicRes context.BasicRes) *GitcliCloner {
func (g *GitcliCloner) CloneRepo(ctx plugin.SubTaskContext, localDir string) errors.Error {
taskData := ctx.GetData().(*GitExtractorTaskData)
- // load state
- stateManager, err := api.NewCollectorStateManager(
- ctx,
- ctx.TaskContext().SyncPolicy(),
- "gitextractor",
- fmt.Sprintf(
- `{"RepoId: "%s","SkipCommitStat": %v, "SkipCommitFiles": %v}`,
- taskData.Options.RepoId,
- *taskData.Options.SkipCommitStat,
- *taskData.Options.SkipCommitFiles,
- ),
- )
- if err != nil {
- return err
+ var since *time.Time
+ if !taskData.Options.NoShallowClone {
+ // load state
+ stateManager, err := api.NewCollectorStateManager(
+ ctx,
+ ctx.TaskContext().SyncPolicy(),
+ "gitextractor",
+ fmt.Sprintf(
+ `{"RepoId: "%s","SkipCommitStat": %v, "SkipCommitFiles": %v}`,
+ taskData.Options.RepoId,
+ *taskData.Options.SkipCommitStat,
+ *taskData.Options.SkipCommitFiles,
+ ),
+ )
+ if err != nil {
+ return err
+ }
+ g.stateManager = stateManager
+ since = stateManager.GetSince()
}
- g.stateManager = stateManager
- cmd, err := g.buildCloneCommand(ctx, localDir, g.stateManager.GetSince())
+ cmd, err := g.buildCloneCommand(ctx, localDir, since)
if err != nil {
return err
}
err = g.execCloneCommand(cmd)
if err != nil {
// it is likely that nothing to collect on incrmental mode
- if errors.Is(err, ErrShallowInfoProcessing) && stateManager.IsIncremental() {
+ if errors.Is(err, ErrShallowInfoProcessing) && (g.stateManager == nil || g.stateManager.IsIncremental()) {
return ErrNoDataOnIncrementalMode
}
return err
}
// save state
- return g.stateManager.Close()
+ if !taskData.Options.NoShallowClone {
+ return g.stateManager.Close()
+ }
+ return nil
}
func (g *GitcliCloner) buildCloneCommand(ctx plugin.SubTaskContext, localDir string, since *time.Time) (*exec.Cmd, errors.Error) {
diff --git a/backend/plugins/gitextractor/parser/taskdata.go b/backend/plugins/gitextractor/parser/taskdata.go
index b0a44c660..a2d894d02 100644
--- a/backend/plugins/gitextractor/parser/taskdata.go
+++ b/backend/plugins/gitextractor/parser/taskdata.go
@@ -25,7 +25,7 @@ type GitExtractorTaskData struct {
Options *GitExtractorOptions
ParsedURL *url.URL
GitRepo RepoCollector
- SkipAllSubtasks bool // skip all tasks without error if true
+ SkipAllSubtasks bool // siliently skip all tasks without raising error
}
type GitExtractorOptions struct {
@@ -40,4 +40,5 @@ type GitExtractorOptions struct {
UseGoGit *bool `json:"useGoGit" mapstructure:"useGoGit"`
SkipCommitStat *bool `json:"skipCommitStat" mapstructure:"skipCommitStat" comment:"skip all commit stat including added/deleted lines and commit files as well"`
SkipCommitFiles *bool `json:"skipCommitFiles" mapstructure:"skipCommitFiles"`
+ NoShallowClone bool `json:"noShallowClone" mapstructure:"noShallowClone"`
}
diff --git a/backend/python/plugins/azuredevops/azuredevops/main.py b/backend/python/plugins/azuredevops/azuredevops/main.py
index 2560ab439..e66422522 100644
--- a/backend/python/plugins/azuredevops/azuredevops/main.py
+++ b/backend/python/plugins/azuredevops/azuredevops/main.py
@@ -135,7 +135,7 @@ class AzureDevOpsPlugin(Plugin):
if DomainType.CODE in scope_config.domain_types and not scope.is_external():
url = urlparse(scope.remote_url)
url = url._replace(netloc=f'{url.username}:{connection.token.get_secret_value()}@{url.hostname}')
- yield gitextractor(url.geturl(), scope.name, scope.domain_id(), connection.proxy)
+ yield gitextractor(url.geturl(), scope.name, scope.domain_id(), connection.proxy, True)
def extra_stages(self, scope_config_pairs: list[tuple[GitRepository, GitRepositoryConfig]], _):
for scope, config in scope_config_pairs:
diff --git a/backend/python/pydevlake/pydevlake/pipeline_tasks.py b/backend/python/pydevlake/pydevlake/pipeline_tasks.py
index b62aaeff6..751d0069f 100644
--- a/backend/python/pydevlake/pydevlake/pipeline_tasks.py
+++ b/backend/python/pydevlake/pydevlake/pipeline_tasks.py
@@ -21,7 +21,7 @@ from pydantic import BaseModel
from pydevlake.message import PipelineTask
-def gitextractor(url: str, repo_name: str, scope_id: str, proxy: Optional[str]):
+def gitextractor(url: str, repo_name: str, scope_id: str, proxy: Optional[str], no_shallow_clone: Optional[bool] = False):
return PipelineTask(
plugin="gitextractor",
options={
@@ -29,7 +29,8 @@ def gitextractor(url: str, repo_name: str, scope_id: str, proxy: Optional[str]):
"name": repo_name,
"fullName": repo_name,
"repoId": scope_id,
- "proxy": proxy
+ "proxy": proxy,
+ "noShallowClone": no_shallow_clone,
},
)