You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by gi...@apache.org on 2024/06/25 06:12:32 UTC
(incubator-devlake) branch release-v1.0-auto-cherry-pick-7660 created (now c64d34d06)
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to branch release-v1.0-auto-cherry-pick-7660
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
at c64d34d06 fix(gitextractor): update database store, set it incremental by default (#7660)
This branch includes the following new commits:
new c64d34d06 fix(gitextractor): update database store, set it incremental by default (#7660)
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(incubator-devlake) 01/01: fix(gitextractor): update database store, set it incremental by default (#7660)
Posted by gi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a commit to branch release-v1.0-auto-cherry-pick-7660
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit c64d34d0670e4970ef0e0517af02e1683cc3e2b5
Author: Lynwee <li...@merico.dev>
AuthorDate: Tue Jun 25 14:12:15 2024 +0800
fix(gitextractor): update database store, set it incremental by default (#7660)
* fix(gitextractor): update database store, set it incremental by default
* fix(gitextractor): add `SetIncrementalMode` to store
* style(gitextractor): fix ci errors
---
backend/plugins/gitextractor/models/interface.go | 1 +
backend/plugins/gitextractor/parser/clone_gitcli.go | 12 ++++++++++++
backend/plugins/gitextractor/parser/taskdata.go | 2 +-
backend/plugins/gitextractor/store/csv.go | 3 +++
backend/plugins/gitextractor/store/database.go | 5 +++++
backend/plugins/gitextractor/tasks/repo_cloner.go | 4 +++-
6 files changed, 25 insertions(+), 2 deletions(-)
diff --git a/backend/plugins/gitextractor/models/interface.go b/backend/plugins/gitextractor/models/interface.go
index cd176f3b5..0a228de83 100644
--- a/backend/plugins/gitextractor/models/interface.go
+++ b/backend/plugins/gitextractor/models/interface.go
@@ -23,6 +23,7 @@ import (
)
type Store interface {
+ SetIncrementalMode(bool)
RepoCommits(repoCommit *code.RepoCommit) errors.Error
Commits(commit *code.Commit) errors.Error
Refs(ref *code.Ref) errors.Error
diff --git a/backend/plugins/gitextractor/parser/clone_gitcli.go b/backend/plugins/gitextractor/parser/clone_gitcli.go
index 69883bf17..9f2ca7ad5 100644
--- a/backend/plugins/gitextractor/parser/clone_gitcli.go
+++ b/backend/plugins/gitextractor/parser/clone_gitcli.go
@@ -56,6 +56,17 @@ type CloneRepoConfig struct {
NoShallowClone bool
}
+func (g *GitcliCloner) IsIncremental() bool {
+ if g != nil && g.stateManager != nil {
+ if g.stateManager.GetSince() != nil {
+ return true
+ }
+ return g.stateManager.IsIncremental()
+ }
+
+ return false
+}
+
func (g *GitcliCloner) CloneRepo(ctx plugin.SubTaskContext, localDir string) errors.Error {
taskData := ctx.GetData().(*GitExtractorTaskData)
var since *time.Time
@@ -75,6 +86,7 @@ func (g *GitcliCloner) CloneRepo(ctx plugin.SubTaskContext, localDir string) err
}
g.stateManager = stateManager
since = stateManager.GetSince()
+
}
cmd, err := g.buildCloneCommand(ctx, localDir, since)
diff --git a/backend/plugins/gitextractor/parser/taskdata.go b/backend/plugins/gitextractor/parser/taskdata.go
index 4b3ab0172..1b22c9885 100644
--- a/backend/plugins/gitextractor/parser/taskdata.go
+++ b/backend/plugins/gitextractor/parser/taskdata.go
@@ -25,7 +25,7 @@ type GitExtractorTaskData struct {
Options *GitExtractorOptions
ParsedURL *url.URL
GitRepo RepoCollector
- SkipAllSubtasks bool // siliently skip all tasks without raising error
+ SkipAllSubtasks bool // silently skip all tasks without raising errors
}
type GitExtractorApiParams struct {
diff --git a/backend/plugins/gitextractor/store/csv.go b/backend/plugins/gitextractor/store/csv.go
index 8701501c5..982485aea 100644
--- a/backend/plugins/gitextractor/store/csv.go
+++ b/backend/plugins/gitextractor/store/csv.go
@@ -132,6 +132,9 @@ func NewCsvStore(dir string) (*CsvStore, errors.Error) {
return s, nil
}
+func (c *CsvStore) SetIncrementalMode(incrementalMode bool) {
+}
+
func (c *CsvStore) RepoCommits(repoCommit *code.RepoCommit) errors.Error {
return c.repoCommitWriter.Write(repoCommit)
}
diff --git a/backend/plugins/gitextractor/store/database.go b/backend/plugins/gitextractor/store/database.go
index 5e03359f8..c795179ec 100644
--- a/backend/plugins/gitextractor/store/database.go
+++ b/backend/plugins/gitextractor/store/database.go
@@ -38,6 +38,7 @@ type Database struct {
}
func NewDatabase(basicRes context.BasicRes, repoId string) *Database {
+
database := &Database{
table: "gitextractor",
params: repoId,
@@ -56,6 +57,10 @@ func (d *Database) updateRawDataFields(rawData *common.RawDataOrigin) {
rawData.RawDataParams = d.params
}
+func (d *Database) SetIncrementalMode(incrementalMode bool) {
+ d.driver.SetIncrementalMode(incrementalMode)
+}
+
func (d *Database) RepoCommits(repoCommit *code.RepoCommit) errors.Error {
batch, err := d.driver.ForType(reflect.TypeOf(repoCommit))
if err != nil {
diff --git a/backend/plugins/gitextractor/tasks/repo_cloner.go b/backend/plugins/gitextractor/tasks/repo_cloner.go
index a00b447b3..45eff7bba 100644
--- a/backend/plugins/gitextractor/tasks/repo_cloner.go
+++ b/backend/plugins/gitextractor/tasks/repo_cloner.go
@@ -62,7 +62,9 @@ func CloneGitRepo(subTaskCtx plugin.SubTaskContext) errors.Error {
}
return err
}
-
+ if repoCloner.IsIncremental() {
+ storage.SetIncrementalMode(repoCloner.IsIncremental())
+ }
// We have done comparison experiments for git2go and go-git, and the results show that git2go has better performance.
var repoCollector parser.RepoCollector
if *taskData.Options.UseGoGit {