You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by kl...@apache.org on 2024/08/22 11:42:03 UTC
(incubator-devlake) branch main updated: feat: double clone against AzDo to reduce libgit2 memory footprint (#7941)
This is an automated email from the ASF dual-hosted git repository.
klesh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 93904521b feat: double clone against AzDo to reduce libgit2 memory footprint (#7941)
93904521b is described below
commit 93904521bd3951158e8aea0774f16b1f975162dd
Author: Klesh Wong <zh...@merico.dev>
AuthorDate: Thu Aug 22 19:41:59 2024 +0800
feat: double clone against AzDo to reduce libgit2 memory footprint (#7941)
---
backend/plugins/gitextractor/gitextractor.go | 2 +
backend/plugins/gitextractor/parser/clone.go | 4 +-
.../plugins/gitextractor/parser/clone_gitcli.go | 360 ++++++++++++---------
backend/plugins/gitextractor/tasks/repo_cloner.go | 8 +-
4 files changed, 222 insertions(+), 152 deletions(-)
diff --git a/backend/plugins/gitextractor/gitextractor.go b/backend/plugins/gitextractor/gitextractor.go
index 0f9db3026..4a6e2f3f2 100644
--- a/backend/plugins/gitextractor/gitextractor.go
+++ b/backend/plugins/gitextractor/gitextractor.go
@@ -38,6 +38,7 @@ func main() {
useGoGit := cmd.Flags().BoolP("useGoGit", "g", false, "use go-git instead of libgit2")
skipCommitStat := cmd.Flags().BoolP("skipCommitStat", "S", false, "")
skipCommitFiles := cmd.Flags().BoolP("skipCommitFiles", "F", true, "")
+ noShallowClone := cmd.Flags().BoolP("noShallowClone", "A", false, "")
timeAfter := cmd.Flags().StringP("timeAfter", "a", "", "collect data that are created after specified time, ie 2006-01-02T15:04:05Z")
_ = cmd.MarkFlagRequired("url")
_ = cmd.MarkFlagRequired("repoId")
@@ -54,6 +55,7 @@ func main() {
"useGoGit": *useGoGit,
"skipCommitStat": skipCommitStat,
"skipCommitFiles": skipCommitFiles,
+ "noShallowClone": noShallowClone,
}, *timeAfter)
}
runner.RunCmd(cmd)
diff --git a/backend/plugins/gitextractor/parser/clone.go b/backend/plugins/gitextractor/parser/clone.go
index 831a639f1..fcd176faa 100644
--- a/backend/plugins/gitextractor/parser/clone.go
+++ b/backend/plugins/gitextractor/parser/clone.go
@@ -19,9 +19,9 @@ package parser
import (
"github.com/apache/incubator-devlake/core/errors"
- "github.com/apache/incubator-devlake/core/plugin"
)
type RepoCloner interface {
- CloneRepo(ctx plugin.SubTaskContext, localDir string) errors.Error
+ CloneRepo() errors.Error
+ CloseRepo() errors.Error
}
diff --git a/backend/plugins/gitextractor/parser/clone_gitcli.go b/backend/plugins/gitextractor/parser/clone_gitcli.go
index 29d821ac4..6dbf5481a 100644
--- a/backend/plugins/gitextractor/parser/clone_gitcli.go
+++ b/backend/plugins/gitextractor/parser/clone_gitcli.go
@@ -26,34 +26,130 @@ import (
"strings"
"time"
- "github.com/apache/incubator-devlake/core/context"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/log"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ giturls "github.com/chainguard-dev/git-urls"
)
var _ RepoCloner = (*GitcliCloner)(nil)
var ErrNoData = errors.NotModified.New("No data to be collected")
+// CloneRepoConfig is the configuration for the CloneRepo method
+// the subtask should run in Full Sync mode whenever the configuration is changed
+type CloneRepoConfig struct {
+ UseGoGit *bool
+ SkipCommitStat *bool
+ SkipCommitFiles *bool
+ NoShallowClone bool
+}
+
type GitcliCloner struct {
+ ctx plugin.SubTaskContext
+ taskData *GitExtractorTaskData
logger log.Logger
stateManager *api.SubtaskStateManager
+ since *time.Time
+ remoteUrl string
+ localDir string
+ success bool
+ syncEnvs []string
+ syncArgs []string
}
-func NewGitcliCloner(basicRes context.BasicRes) *GitcliCloner {
- return &GitcliCloner{
- logger: basicRes.GetLogger().Nested("gitcli"),
+func NewGitcliCloner(ctx plugin.SubTaskContext, localDir string) (*GitcliCloner, errors.Error) {
+ taskData := ctx.GetData().(*GitExtractorTaskData)
+ stateManager := errors.Must1(api.NewSubtaskStateManager(&api.SubtaskCommonArgs{
+ SubTaskContext: ctx,
+ Params: taskData.Options.GitExtractorApiParams,
+ SubtaskConfig: CloneRepoConfig{
+ UseGoGit: taskData.Options.UseGoGit,
+ SkipCommitStat: taskData.Options.SkipCommitStat,
+ SkipCommitFiles: taskData.Options.SkipCommitFiles,
+ NoShallowClone: taskData.Options.NoShallowClone,
+ },
+ }))
+
+ cloner := &GitcliCloner{
+ ctx: ctx,
+ taskData: taskData,
+ logger: ctx.GetLogger().Nested("gitcli"),
+ stateManager: stateManager,
+ since: stateManager.GetSince(),
+ remoteUrl: taskData.Options.Url,
+ localDir: localDir,
+ success: false,
}
+ return cloner, cloner.prepareSync()
}
-// CloneRepoConfig is the configuration for the CloneRepo method
-// the subtask should run in Full Sync mode whenever the configuration is changed
-type CloneRepoConfig struct {
- UseGoGit *bool
- SkipCommitStat *bool
- SkipCommitFiles *bool
- NoShallowClone bool
+func (g *GitcliCloner) prepareSync() errors.Error {
+ taskData := g.taskData
+ if *taskData.Options.SkipCommitStat {
+ g.syncArgs = append(g.syncArgs, "--filter=blob:none")
+ }
+ remoteUrl, e := giturls.Parse(g.remoteUrl)
+ if e != nil {
+ return errors.Convert(e)
+ }
+ // support proxy
+ if remoteUrl.Scheme == "http" || remoteUrl.Scheme == "https" {
+ if taskData.Options.Proxy != "" {
+ g.syncEnvs = append(g.syncEnvs, fmt.Sprintf("HTTPS_PROXY=%s", taskData.Options.Proxy))
+ }
+ if remoteUrl.Scheme == "https" && g.ctx.GetConfigReader().GetBool("IN_SECURE_SKIP_VERIFY") {
+ g.syncEnvs = append(g.syncEnvs, "GIT_SSL_NO_VERIFY=true")
+ }
+ } else if remoteUrl.Scheme == "ssh" {
+ var sshCmdArgs []string
+ if taskData.Options.Proxy != "" {
+ parsedProxyURL, e := url.Parse(taskData.Options.Proxy)
+ if e != nil {
+ return errors.BadInput.Wrap(e, "failed to parse the proxy URL")
+ }
+ proxyCommand := "corkscrew"
+ sshCmdArgs = append(sshCmdArgs, "-o", fmt.Sprintf(`ProxyCommand="%s %s %s %%h %%p"`, proxyCommand, parsedProxyURL.Hostname(), parsedProxyURL.Port()))
+ }
+ // support private key
+ if taskData.Options.PrivateKey != "" {
+ pkFile, err := os.CreateTemp("", "gitext-pk")
+ if err != nil {
+ g.logger.Error(err, "create temp private key file error")
+ return errors.Default.New("failed to handle the private key")
+ }
+ if _, e := pkFile.WriteString(taskData.Options.PrivateKey + "\n"); e != nil {
+ g.logger.Error(err, "write private key file error")
+ return errors.Default.New("failed to write the private key")
+ }
+ pkFile.Close()
+ if e := os.Chmod(pkFile.Name(), 0600); e != nil {
+ g.logger.Error(err, "chmod private key file error")
+ return errors.Default.New("failed to modify the private key")
+ }
+
+ if taskData.Options.Passphrase != "" {
+ pp := exec.CommandContext(
+ g.ctx.GetContext(),
+ "ssh-keygen", "-p",
+ "-P", taskData.Options.Passphrase,
+ "-N", "",
+ "-f", pkFile.Name(),
+ )
+ if ppout, pperr := pp.CombinedOutput(); pperr != nil {
+ g.logger.Error(pperr, "change private key passphrase error")
+ g.logger.Info(string(ppout))
+ return errors.Default.New("failed to decrypt the private key")
+ }
+ }
+ defer os.Remove(pkFile.Name())
+ sshCmdArgs = append(sshCmdArgs, fmt.Sprintf("-i %s -o StrictHostKeyChecking=no", pkFile.Name()))
+ }
+ if len(sshCmdArgs) > 0 {
+ g.syncEnvs = append(g.syncEnvs, fmt.Sprintf("GIT_SSH_COMMAND=ssh %s", strings.Join(sshCmdArgs, " ")))
+ }
+ }
+ return nil
}
func (g *GitcliCloner) IsIncremental() bool {
@@ -67,163 +163,131 @@ func (g *GitcliCloner) IsIncremental() bool {
return false
}
-func (g *GitcliCloner) CloneRepo(ctx plugin.SubTaskContext, localDir string) errors.Error {
- taskData := ctx.GetData().(*GitExtractorTaskData)
- var since *time.Time
- if !taskData.Options.NoShallowClone {
- stateManager, err := api.NewSubtaskStateManager(&api.SubtaskCommonArgs{
- SubTaskContext: ctx,
- Params: taskData.Options.GitExtractorApiParams,
- SubtaskConfig: CloneRepoConfig{
- UseGoGit: taskData.Options.UseGoGit,
- SkipCommitStat: taskData.Options.SkipCommitStat,
- SkipCommitFiles: taskData.Options.SkipCommitFiles,
- NoShallowClone: taskData.Options.NoShallowClone,
- },
- })
- if err != nil {
+func (g *GitcliCloner) CloneRepo() errors.Error {
+ if g.since == nil {
+ // full sync
+ if err := g.fullClone(); err != nil {
return err
}
- g.stateManager = stateManager
- since = stateManager.GetSince()
-
- }
-
- err := g.execGitCloneCommand(ctx, localDir, since)
- if err != nil {
- return err
- }
- // deepen the commits by 1 more step to avoid https://github.com/apache/incubator-devlake/issues/7426
- if since != nil {
- // fixes error described on https://stackoverflow.com/questions/63878612/git-fatal-error-in-object-unshallow-sha-1
- // It might be caused by the commit which being deepen has multiple parent(e.g. a merge commit), not sure.
- if err := g.execGitCommandIn(ctx, localDir, "repack", "-d"); err != nil {
- return errors.Default.Wrap(err, "failed to repack the repo")
+ } else {
+ if g.taskData.Options.NoShallowClone {
+ // data source does not support shallow clone
+ // 1. perform a full clone to accommodate
+ // 2. perform a local shallow clone to reduce the libgit2 memory usage
+ if err := g.doubleClone(); err != nil {
+ return err
+ }
+ } else {
+ // data source support shallow clone
+ if err := g.shallowClone(); err != nil {
+ return err
+ }
}
- // deepen would fail on a EMPTY repo, ignore the error
- if err := g.execGitCommandIn(ctx, localDir, "fetch", "--deepen=1"); err != nil {
- g.logger.Error(err, "failed to deepen the cloned repo")
+ if err := g.deepen(); err != nil {
+ return err
}
+ g.success = true
}
+ return nil
+}
- // save state
- if g.stateManager != nil {
+func (g *GitcliCloner) CloseRepo() errors.Error {
+ if g.success {
+ g.logger.Info("save state")
return g.stateManager.Close()
}
return nil
}
-func (g *GitcliCloner) execGitCloneCommand(ctx plugin.SubTaskContext, localDir string, since *time.Time) errors.Error {
- taskData := ctx.GetData().(*GitExtractorTaskData)
- var args []string
- if *taskData.Options.SkipCommitStat {
- args = append(args, "--filter=blob:none")
+func (g *GitcliCloner) fullClone() errors.Error {
+ return g.gitClone(g.remoteUrl, g.localDir, "--bare")
+}
+
+func (g *GitcliCloner) deepen() errors.Error {
+ // deepen the commits by 1 more step to avoid https://github.com/apache/incubator-devlake/issues/7426
+ // fixes error described on https://stackoverflow.com/questions/63878612/git-fatal-error-in-object-unshallow-sha-1
+ // It might be caused by the commit which being deepen has multiple parent(e.g. a merge commit), not sure.
+ if err := g.gitCmd("repack", "-d"); err != nil {
+ return errors.Default.Wrap(err, "failed to repack the repo")
}
- if since != nil {
- // to fetch newly added commits from ALL branches, we need to the following guide:
- // https://stackoverflow.com/questions/23708231/git-shallow-clone-clone-depth-misses-remote-branches
+ // deepen would fail on a EMPTY repo, ignore the error
+ if err := g.gitFetch("--deepen=1"); err != nil {
+ g.logger.Error(err, "failed to deepen the cloned repo")
+ }
+ return nil
+}
- // 1. clone the repo with depth 1
- cloneArgs := append([]string{"clone", taskData.Options.Url, localDir, "--depth=1", "--bare"}, args...)
- if err := g.execGitCommand(ctx, cloneArgs...); err != nil {
- return err
- }
- // 2. configure to fetch all branches from the remote server, so we can collect new commits from them
- gitConfig, err := os.OpenFile(path.Join(localDir, "config"), os.O_APPEND|os.O_WRONLY, 0644)
- if err != nil {
- return errors.Default.Wrap(err, "failed to open git config file")
- }
- _, err = gitConfig.WriteString("\tfetch = +refs/heads/*:refs/remotes/origin/*\n")
- if err != nil {
- return errors.Default.Wrap(err, "failed to write to git config file")
- }
- // 3. fetch all branches with depth=1 so the next step would collect fewer commits
- // (I don't know why, but it reduced total number of commits from 18k to 7k on https://gitlab.com/gitlab-org/gitlab-foss.git with the same parameters)
- fetchBranchesArgs := append([]string{"fetch", "--depth=1", "origin"}, args...)
- if err := g.execGitCommandIn(ctx, localDir, fetchBranchesArgs...); err != nil {
- return errors.Default.Wrap(err, "failed to fetch all branches from the remote server")
- }
- // 4. fetch all new commits from all branches since the given time
- args = append([]string{"fetch", fmt.Sprintf("--shallow-since=%s", since.Format(time.RFC3339))}, args...)
- if err := g.execGitCommandIn(ctx, localDir, args...); err != nil {
- g.logger.Warn(err, "shallow fetch failed")
- }
- return nil
- } else {
- args = append([]string{"clone", taskData.Options.Url, localDir, "--bare"}, args...)
- return g.execGitCommand(ctx, args...)
+func (g *GitcliCloner) shallowClone() errors.Error {
+ // to fetch newly added commits from ALL branches, we need to the following guide:
+ // https://stackoverflow.com/questions/23708231/git-shallow-clone-clone-depth-misses-remote-branches
+ // 1. clone the repo with depth 1
+ if err := g.gitClone(g.remoteUrl, g.localDir, "--depth=1", "--bare"); err != nil {
+ return err
+ }
+ // 2. configure to fetch all branches from the remote server, so we can collect new commits from them
+ gitConfig, err := os.OpenFile(path.Join(g.localDir, "config"), os.O_APPEND|os.O_WRONLY, 0644)
+ if err != nil {
+ return errors.Default.Wrap(err, "failed to open git config file")
+ }
+ _, err = gitConfig.WriteString("\tfetch = +refs/heads/*:refs/remotes/origin/*\n")
+ if err != nil {
+ return errors.Default.Wrap(err, "failed to write to git config file")
+ }
+ g.logger.Debug("updated git config to fetch all remote branches")
+ // 3. fetch all branches with depth=1 so the next step would collect fewer commits
+ // (I don't know why, but it reduced total number of commits from 18k to 7k on https://gitlab.com/gitlab-org/gitlab-foss.git with the same parameters)
+ if err := g.gitFetch("--depth=1", "origin"); err != nil {
+ return errors.Default.Wrap(err, "failed to fetch all branches from the remote server")
+ }
+ // 4. fetch all new commits from all branches since the given time
+ if err := g.gitFetch(fmt.Sprintf("--shallow-since=%s", g.since.Format(time.RFC3339))); err != nil {
+ g.logger.Warn(err, "shallow fetch failed")
+ }
+ return nil
+}
+
+func (g *GitcliCloner) doubleClone() errors.Error {
+ intermediaryDir, e := os.MkdirTemp("", "gitextint")
+ if e != nil {
+ return errors.Convert(e)
+ }
+ // step 1: full clone into a intermediary dir
+ backup := g.localDir
+ g.localDir = intermediaryDir
+ if err := g.fullClone(); err != nil {
+ return err
+ }
+ g.localDir = backup
+ // step 2: perform shallow clone aginst the intermediary dir
+ backup = g.remoteUrl
+ g.remoteUrl = fmt.Sprintf("file://%s", intermediaryDir) // the file:// prefix is required for shallow clone to work
+ if err := g.shallowClone(); err != nil {
+ return err
}
+ g.remoteUrl = backup
+ return nil
}
-func (g *GitcliCloner) execGitCommand(ctx plugin.SubTaskContext, args ...string) errors.Error {
- return g.execGitCommandIn(ctx, "", args...)
+func (g *GitcliCloner) gitClone(args ...string) errors.Error {
+ args = append(args, g.syncArgs...)
+ return g.git(g.syncEnvs, "", "clone", args...)
}
-func (g *GitcliCloner) execGitCommandIn(ctx plugin.SubTaskContext, workingDir string, args ...string) errors.Error {
- taskData := ctx.GetData().(*GitExtractorTaskData)
- env := []string{}
- if args[0] == "clone" || args[0] == "fetch" {
- // support proxy
- if taskData.ParsedURL.Scheme == "http" || taskData.ParsedURL.Scheme == "https" {
- if taskData.Options.Proxy != "" {
- env = append(env, fmt.Sprintf("HTTPS_PROXY=%s", taskData.Options.Proxy))
- }
- if taskData.ParsedURL.Scheme == "https" && ctx.GetConfigReader().GetBool("IN_SECURE_SKIP_VERIFY") {
- args = append([]string{"-c http.sslVerify=false"}, args...)
- }
- } else if taskData.ParsedURL.Scheme == "ssh" {
- var sshCmdArgs []string
- if taskData.Options.Proxy != "" {
- parsedProxyURL, e := url.Parse(taskData.Options.Proxy)
- if e != nil {
- return errors.BadInput.Wrap(e, "failed to parse the proxy URL")
- }
- proxyCommand := "corkscrew"
- sshCmdArgs = append(sshCmdArgs, "-o", fmt.Sprintf(`ProxyCommand="%s %s %s %%h %%p"`, proxyCommand, parsedProxyURL.Hostname(), parsedProxyURL.Port()))
- }
- // support private key
- if taskData.Options.PrivateKey != "" {
- pkFile, err := os.CreateTemp("", "gitext-pk")
- if err != nil {
- g.logger.Error(err, "create temp private key file error")
- return errors.Default.New("failed to handle the private key")
- }
- if _, e := pkFile.WriteString(taskData.Options.PrivateKey + "\n"); e != nil {
- g.logger.Error(err, "write private key file error")
- return errors.Default.New("failed to write the private key")
- }
- pkFile.Close()
- if e := os.Chmod(pkFile.Name(), 0600); e != nil {
- g.logger.Error(err, "chmod private key file error")
- return errors.Default.New("failed to modify the private key")
- }
+func (g *GitcliCloner) gitFetch(args ...string) errors.Error {
+ args = append(args, g.syncArgs...)
+ return g.git(g.syncEnvs, g.localDir, "fetch", args...)
+}
- if taskData.Options.Passphrase != "" {
- pp := exec.CommandContext(
- ctx.GetContext(),
- "ssh-keygen", "-p",
- "-P", taskData.Options.Passphrase,
- "-N", "",
- "-f", pkFile.Name(),
- )
- if ppout, pperr := pp.CombinedOutput(); pperr != nil {
- g.logger.Error(pperr, "change private key passphrase error")
- g.logger.Info(string(ppout))
- return errors.Default.New("failed to decrypt the private key")
- }
- }
- defer os.Remove(pkFile.Name())
- sshCmdArgs = append(sshCmdArgs, fmt.Sprintf("-i %s -o StrictHostKeyChecking=no", pkFile.Name()))
- }
- if len(sshCmdArgs) > 0 {
- env = append(env, fmt.Sprintf("GIT_SSH_COMMAND=ssh %s", strings.Join(sshCmdArgs, " ")))
- }
- }
- }
- g.logger.Debug("git %v", args)
- cmd := exec.CommandContext(ctx.GetContext(), "git", args...)
+func (g *GitcliCloner) gitCmd(gitcmd string, args ...string) errors.Error {
+ return g.git(nil, g.localDir, gitcmd, args...)
+}
+
+func (g *GitcliCloner) git(env []string, dir string, gitcmd string, args ...string) errors.Error {
+ g.logger.Debug("git %s %v", gitcmd, args)
+ args = append([]string{gitcmd}, args...)
+ cmd := exec.CommandContext(g.ctx.GetContext(), "git", args...)
cmd.Env = env
- cmd.Dir = workingDir
+ cmd.Dir = dir
return g.execCommand(cmd)
}
diff --git a/backend/plugins/gitextractor/tasks/repo_cloner.go b/backend/plugins/gitextractor/tasks/repo_cloner.go
index 45eff7bba..c72841105 100644
--- a/backend/plugins/gitextractor/tasks/repo_cloner.go
+++ b/backend/plugins/gitextractor/tasks/repo_cloner.go
@@ -53,8 +53,11 @@ func CloneGitRepo(subTaskCtx plugin.SubTaskContext) errors.Error {
}
// clone repo
- repoCloner := parser.NewGitcliCloner(subTaskCtx)
- err = repoCloner.CloneRepo(subTaskCtx, localDir)
+ repoCloner, err := parser.NewGitcliCloner(subTaskCtx, localDir)
+ if err != nil {
+ return err
+ }
+ err = repoCloner.CloneRepo()
if err != nil {
if errors.Is(err, parser.ErrNoData) {
taskData.SkipAllSubtasks = true
@@ -79,6 +82,7 @@ func CloneGitRepo(subTaskCtx plugin.SubTaskContext) errors.Error {
// inject clean up callback to remove the cloned dir
cleanup := func() {
_ = os.RemoveAll(localDir)
+ _ = repoCloner.CloseRepo()
}
if e := repoCollector.SetCleanUp(cleanup); e != nil {
return errors.Convert(e)