You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by kl...@apache.org on 2024/07/12 03:49:40 UTC
(incubator-devlake) 01/01: fix: new commits in other branches are not collected
This is an automated email from the ASF dual-hosted git repository.
klesh pushed a commit to branch bugfix-commits-in-nondefaultbranches-are-missing
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit dbb2bdde42a5a2907b05b531d698670eb76e4445
Author: Klesh Wong <zh...@merico.dev>
AuthorDate: Fri Jul 12 11:49:17 2024 +0800
fix: new commits in other branches are not collected
---
.../plugins/gitextractor/parser/clone_gitcli.go | 179 ++++++++++-----------
1 file changed, 85 insertions(+), 94 deletions(-)
diff --git a/backend/plugins/gitextractor/parser/clone_gitcli.go b/backend/plugins/gitextractor/parser/clone_gitcli.go
index 89a004239..4acf4b0d9 100644
--- a/backend/plugins/gitextractor/parser/clone_gitcli.go
+++ b/backend/plugins/gitextractor/parser/clone_gitcli.go
@@ -18,11 +18,11 @@ limitations under the License.
package parser
import (
- "bufio"
"fmt"
"net/url"
"os"
"os/exec"
+ "path"
"strings"
"time"
@@ -115,12 +115,32 @@ func (g *GitcliCloner) CloneRepo(ctx plugin.SubTaskContext, localDir string) err
func (g *GitcliCloner) execGitCloneCommand(ctx plugin.SubTaskContext, localDir string, since *time.Time) errors.Error {
taskData := ctx.GetData().(*GitExtractorTaskData)
- args := []string{"clone", taskData.Options.Url, localDir, "--bare", "--progress"}
+ var args []string
if since != nil {
- args = append(args, fmt.Sprintf("--shallow-since=%s", since.Format(time.RFC3339)))
+ // to fetch newly added commits from ALL branches, we need to the following guide:
+ // https://stackoverflow.com/questions/23708231/git-shallow-clone-clone-depth-misses-remote-branches
+
+ // 1. clone the repo with depth 1
+ if err := g.execGitCommand(ctx, "clone", taskData.Options.Url, localDir, "--depth=1", "--bare"); err != nil {
+ return err
+ }
+ // 2. set remote for all branches
+ // if err := g.execGitCommandIn(ctx, localDir, "remote", "set-branches", "origin", "'*'"); err != nil {
+ // return err
+ // } // someshow it fails siliently on my local machine, don't know why
+ gitConfig, err := os.OpenFile(path.Join(localDir, "config"), os.O_APPEND|os.O_WRONLY, 0644)
+ if err != nil {
+ return errors.Default.Wrap(err, "failed to open git config file")
+ }
+ _, err = gitConfig.WriteString("\tfetch = +refs/heads/*:refs/remotes/origin/*\n")
+ if err != nil {
+ return errors.Default.Wrap(err, "failed to write to git config file")
+ }
+ // 3. fetch all new commits from all branches since the given time
+ args = []string{"fetch", "--progress", fmt.Sprintf("--shallow-since=%s", since.Format(time.RFC3339))}
+ } else {
+ args = []string{"clone", "--progress", "--bare"}
}
- // support time after and diff sync
- // support skipping blobs collection
if *taskData.Options.SkipCommitStat {
args = append(args, "--filter=blob:none")
}
@@ -128,115 +148,86 @@ func (g *GitcliCloner) execGitCloneCommand(ctx plugin.SubTaskContext, localDir s
}
func (g *GitcliCloner) execGitCommand(ctx plugin.SubTaskContext, args ...string) errors.Error {
+ return g.execGitCommandIn(ctx, "", args...)
+}
+
+func (g *GitcliCloner) execGitCommandIn(ctx plugin.SubTaskContext, workingDir string, args ...string) errors.Error {
taskData := ctx.GetData().(*GitExtractorTaskData)
env := []string{}
- // support proxy
- if taskData.ParsedURL.Scheme == "http" || taskData.ParsedURL.Scheme == "https" {
- if taskData.Options.Proxy != "" {
- env = append(env, fmt.Sprintf("HTTPS_PROXY=%s", taskData.Options.Proxy))
- }
- if taskData.ParsedURL.Scheme == "https" && ctx.GetConfigReader().GetBool("IN_SECURE_SKIP_VERIFY") {
- args = append(args, "-c http.sslVerify=false")
- }
- } else if taskData.ParsedURL.Scheme == "ssh" {
- var sshCmdArgs []string
- if taskData.Options.Proxy != "" {
- parsedProxyURL, e := url.Parse(taskData.Options.Proxy)
- if e != nil {
- return errors.BadInput.Wrap(e, "failed to parse the proxy URL")
- }
- proxyCommand := "corkscrew"
- sshCmdArgs = append(sshCmdArgs, "-o", fmt.Sprintf(`ProxyCommand="%s %s %s %%h %%p"`, proxyCommand, parsedProxyURL.Hostname(), parsedProxyURL.Port()))
- }
- // support private key
- if taskData.Options.PrivateKey != "" {
- pkFile, err := os.CreateTemp("", "gitext-pk")
- if err != nil {
- g.logger.Error(err, "create temp private key file error")
- return errors.Default.New("failed to handle the private key")
+ if args[0] == "clone" || args[0] == "fetch" {
+ // support proxy
+ if taskData.ParsedURL.Scheme == "http" || taskData.ParsedURL.Scheme == "https" {
+ if taskData.Options.Proxy != "" {
+ env = append(env, fmt.Sprintf("HTTPS_PROXY=%s", taskData.Options.Proxy))
}
- if _, e := pkFile.WriteString(taskData.Options.PrivateKey + "\n"); e != nil {
- g.logger.Error(err, "write private key file error")
- return errors.Default.New("failed to write the private key")
+ if taskData.ParsedURL.Scheme == "https" && ctx.GetConfigReader().GetBool("IN_SECURE_SKIP_VERIFY") {
+ args = append(args, "-c http.sslVerify=false")
}
- pkFile.Close()
- if e := os.Chmod(pkFile.Name(), 0600); e != nil {
- g.logger.Error(err, "chmod private key file error")
- return errors.Default.New("failed to modify the private key")
+ } else if taskData.ParsedURL.Scheme == "ssh" {
+ var sshCmdArgs []string
+ if taskData.Options.Proxy != "" {
+ parsedProxyURL, e := url.Parse(taskData.Options.Proxy)
+ if e != nil {
+ return errors.BadInput.Wrap(e, "failed to parse the proxy URL")
+ }
+ proxyCommand := "corkscrew"
+ sshCmdArgs = append(sshCmdArgs, "-o", fmt.Sprintf(`ProxyCommand="%s %s %s %%h %%p"`, proxyCommand, parsedProxyURL.Hostname(), parsedProxyURL.Port()))
}
+ // support private key
+ if taskData.Options.PrivateKey != "" {
+ pkFile, err := os.CreateTemp("", "gitext-pk")
+ if err != nil {
+ g.logger.Error(err, "create temp private key file error")
+ return errors.Default.New("failed to handle the private key")
+ }
+ if _, e := pkFile.WriteString(taskData.Options.PrivateKey + "\n"); e != nil {
+ g.logger.Error(err, "write private key file error")
+ return errors.Default.New("failed to write the private key")
+ }
+ pkFile.Close()
+ if e := os.Chmod(pkFile.Name(), 0600); e != nil {
+ g.logger.Error(err, "chmod private key file error")
+ return errors.Default.New("failed to modify the private key")
+ }
- if taskData.Options.Passphrase != "" {
- pp := exec.CommandContext(
- ctx.GetContext(),
- "ssh-keygen", "-p",
- "-P", taskData.Options.Passphrase,
- "-N", "",
- "-f", pkFile.Name(),
- )
- if ppout, pperr := pp.CombinedOutput(); pperr != nil {
- g.logger.Error(pperr, "change private key passphrase error")
- g.logger.Info(string(ppout))
- return errors.Default.New("failed to decrypt the private key")
+ if taskData.Options.Passphrase != "" {
+ pp := exec.CommandContext(
+ ctx.GetContext(),
+ "ssh-keygen", "-p",
+ "-P", taskData.Options.Passphrase,
+ "-N", "",
+ "-f", pkFile.Name(),
+ )
+ if ppout, pperr := pp.CombinedOutput(); pperr != nil {
+ g.logger.Error(pperr, "change private key passphrase error")
+ g.logger.Info(string(ppout))
+ return errors.Default.New("failed to decrypt the private key")
+ }
}
+ defer os.Remove(pkFile.Name())
+ sshCmdArgs = append(sshCmdArgs, fmt.Sprintf("-i %s -o StrictHostKeyChecking=no", pkFile.Name()))
+ }
+ if len(sshCmdArgs) > 0 {
+ env = append(env, fmt.Sprintf("GIT_SSH_COMMAND=ssh %s", strings.Join(sshCmdArgs, " ")))
}
- defer os.Remove(pkFile.Name())
- sshCmdArgs = append(sshCmdArgs, fmt.Sprintf("-i %s -o StrictHostKeyChecking=no", pkFile.Name()))
- }
- if len(sshCmdArgs) > 0 {
- env = append(env, fmt.Sprintf("GIT_SSH_COMMAND=ssh %s", strings.Join(sshCmdArgs, " ")))
}
}
g.logger.Debug("git %v", args)
cmd := exec.CommandContext(ctx.GetContext(), "git", args...)
cmd.Env = env
+ cmd.Dir = workingDir
return g.execCommand(cmd)
}
func (g *GitcliCloner) execCommand(cmd *exec.Cmd) errors.Error {
- stdout, err := cmd.StdoutPipe()
- if err != nil {
- g.logger.Error(err, "stdout pipe error")
- return errors.Default.New("stdout pipe error")
- }
- stderr, err := cmd.StderrPipe()
- if err != nil {
- g.logger.Error(err, "stderr pipe error")
- return errors.Default.New("stderr pipe error")
- }
- combinedOutput := new(strings.Builder)
- stdoutScanner := bufio.NewScanner(stdout)
- stdoutScanner.Split(bufio.ScanLines)
- stderrScanner := bufio.NewScanner(stderr)
- stderrScanner.Split(bufio.ScanLines)
- done := make(chan bool)
- go func() {
- for stdoutScanner.Scan() {
- // TODO: extract progress?
- combinedOutput.WriteString(fmt.Sprintf("stdout: %s\n", stdoutScanner.Text()))
- }
- done <- true
- }()
- go func() {
- // TODO: extract progress?
- for stderrScanner.Scan() {
- combinedOutput.WriteString(fmt.Sprintf("stderr: %s\n", stderrScanner.Text()))
- }
- done <- true
- }()
- if e := cmd.Start(); e != nil {
- g.logger.Error(e, "failed to start\n%s", combinedOutput.String())
- return errors.Default.New("failed to start")
- }
- <-done
- <-done
- err = cmd.Wait()
+ output, err := cmd.CombinedOutput()
if err != nil {
- g.logger.Error(err, "git exited with error\n%s", combinedOutput.String())
- if strings.Contains(combinedOutput.String(), "stderr: fatal: error processing shallow info: 4") ||
- strings.Contains(combinedOutput.String(), "stderr: fatal: the remote end hung up unexpectedly") {
+ outputString := string(output)
+ if strings.Contains(outputString, "fatal: error processing shallow info: 4") ||
+ strings.Contains(outputString, "fatal: the remote end hung up unexpectedly") {
return ErrNoData
}
- return errors.Default.New("git exit error")
+ return errors.Default.New(fmt.Sprintf("git cmd %v in %s failed: %s", cmd.Args, cmd.Dir, outputString))
}
return nil
}