You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by gi...@apache.org on 2024/06/25 06:12:33 UTC
(incubator-devlake) 01/01: fix(gitextractor): update database store, set it incremental by default (#7660)
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a commit to branch release-v1.0-auto-cherry-pick-7660
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit c64d34d0670e4970ef0e0517af02e1683cc3e2b5
Author: Lynwee <li...@merico.dev>
AuthorDate: Tue Jun 25 14:12:15 2024 +0800
fix(gitextractor): update database store, set it incremental by default (#7660)
* fix(gitextractor): update database store, set it incremental by default
* fix(gitextractor): add `SetIncrementalMode` to store
* style(gitextractor): fix ci errors
---
backend/plugins/gitextractor/models/interface.go | 1 +
backend/plugins/gitextractor/parser/clone_gitcli.go | 12 ++++++++++++
backend/plugins/gitextractor/parser/taskdata.go | 2 +-
backend/plugins/gitextractor/store/csv.go | 3 +++
backend/plugins/gitextractor/store/database.go | 5 +++++
backend/plugins/gitextractor/tasks/repo_cloner.go | 4 +++-
6 files changed, 25 insertions(+), 2 deletions(-)
diff --git a/backend/plugins/gitextractor/models/interface.go b/backend/plugins/gitextractor/models/interface.go
index cd176f3b5..0a228de83 100644
--- a/backend/plugins/gitextractor/models/interface.go
+++ b/backend/plugins/gitextractor/models/interface.go
@@ -23,6 +23,7 @@ import (
)
type Store interface {
+ SetIncrementalMode(bool)
RepoCommits(repoCommit *code.RepoCommit) errors.Error
Commits(commit *code.Commit) errors.Error
Refs(ref *code.Ref) errors.Error
diff --git a/backend/plugins/gitextractor/parser/clone_gitcli.go b/backend/plugins/gitextractor/parser/clone_gitcli.go
index 69883bf17..9f2ca7ad5 100644
--- a/backend/plugins/gitextractor/parser/clone_gitcli.go
+++ b/backend/plugins/gitextractor/parser/clone_gitcli.go
@@ -56,6 +56,17 @@ type CloneRepoConfig struct {
NoShallowClone bool
}
+func (g *GitcliCloner) IsIncremental() bool {
+ if g != nil && g.stateManager != nil {
+ if g.stateManager.GetSince() != nil {
+ return true
+ }
+ return g.stateManager.IsIncremental()
+ }
+
+ return false
+}
+
func (g *GitcliCloner) CloneRepo(ctx plugin.SubTaskContext, localDir string) errors.Error {
taskData := ctx.GetData().(*GitExtractorTaskData)
var since *time.Time
@@ -75,6 +86,7 @@ func (g *GitcliCloner) CloneRepo(ctx plugin.SubTaskContext, localDir string) err
}
g.stateManager = stateManager
since = stateManager.GetSince()
+
}
cmd, err := g.buildCloneCommand(ctx, localDir, since)
diff --git a/backend/plugins/gitextractor/parser/taskdata.go b/backend/plugins/gitextractor/parser/taskdata.go
index 4b3ab0172..1b22c9885 100644
--- a/backend/plugins/gitextractor/parser/taskdata.go
+++ b/backend/plugins/gitextractor/parser/taskdata.go
@@ -25,7 +25,7 @@ type GitExtractorTaskData struct {
Options *GitExtractorOptions
ParsedURL *url.URL
GitRepo RepoCollector
- SkipAllSubtasks bool // siliently skip all tasks without raising error
+ SkipAllSubtasks bool // silently skip all tasks without raising errors
}
type GitExtractorApiParams struct {
diff --git a/backend/plugins/gitextractor/store/csv.go b/backend/plugins/gitextractor/store/csv.go
index 8701501c5..982485aea 100644
--- a/backend/plugins/gitextractor/store/csv.go
+++ b/backend/plugins/gitextractor/store/csv.go
@@ -132,6 +132,9 @@ func NewCsvStore(dir string) (*CsvStore, errors.Error) {
return s, nil
}
+func (c *CsvStore) SetIncrementalMode(incrementalMode bool) {
+}
+
func (c *CsvStore) RepoCommits(repoCommit *code.RepoCommit) errors.Error {
return c.repoCommitWriter.Write(repoCommit)
}
diff --git a/backend/plugins/gitextractor/store/database.go b/backend/plugins/gitextractor/store/database.go
index 5e03359f8..c795179ec 100644
--- a/backend/plugins/gitextractor/store/database.go
+++ b/backend/plugins/gitextractor/store/database.go
@@ -38,6 +38,7 @@ type Database struct {
}
func NewDatabase(basicRes context.BasicRes, repoId string) *Database {
+
database := &Database{
table: "gitextractor",
params: repoId,
@@ -56,6 +57,10 @@ func (d *Database) updateRawDataFields(rawData *common.RawDataOrigin) {
rawData.RawDataParams = d.params
}
+func (d *Database) SetIncrementalMode(incrementalMode bool) {
+ d.driver.SetIncrementalMode(incrementalMode)
+}
+
func (d *Database) RepoCommits(repoCommit *code.RepoCommit) errors.Error {
batch, err := d.driver.ForType(reflect.TypeOf(repoCommit))
if err != nil {
diff --git a/backend/plugins/gitextractor/tasks/repo_cloner.go b/backend/plugins/gitextractor/tasks/repo_cloner.go
index a00b447b3..45eff7bba 100644
--- a/backend/plugins/gitextractor/tasks/repo_cloner.go
+++ b/backend/plugins/gitextractor/tasks/repo_cloner.go
@@ -62,7 +62,9 @@ func CloneGitRepo(subTaskCtx plugin.SubTaskContext) errors.Error {
}
return err
}
-
+ if repoCloner.IsIncremental() {
+ storage.SetIncrementalMode(repoCloner.IsIncremental())
+ }
// We have done comparison experiments for git2go and go-git, and the results show that git2go has better performance.
var repoCollector parser.RepoCollector
if *taskData.Options.UseGoGit {