You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by ab...@apache.org on 2024/04/02 02:30:06 UTC
(incubator-devlake) branch main updated: feat: gitextractor skips calculating commit stat by default to speed up pipeline (#7268)
This is an automated email from the ASF dual-hosted git repository.
abeizn pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 193850b55 feat: gitextractor skips calculating commit stat by default to speed up pipeline (#7268)
193850b55 is described below
commit 193850b55c0e143848cfeda728a47caa650970ca
Author: Klesh Wong <zh...@merico.dev>
AuthorDate: Tue Apr 2 10:30:02 2024 +0800
feat: gitextractor skips calculating commit stat by default to speed up pipeline (#7268)
* fix: add missing RESUME_PIPELINES setting to env.example
* feat(gitextractor): skip commit stat to speed up pipeline
* refactor: rename key to cfgKey for readibility
* fix: SKIP_COMMIT_FILES and SKIP_COMMIT_STAT shoud default to true
* fix: delete unused TEMPORAL_ variables
---
backend/plugins/gitextractor/gitextractor.go | 60 ++++++++++++++
backend/plugins/gitextractor/impl/impl.go | 27 +++++-
backend/plugins/gitextractor/main.go | 95 ----------------------
backend/plugins/gitextractor/parser/repo_gogit.go | 35 ++++----
.../plugins/gitextractor/parser/repo_libgit2.go | 28 +++----
backend/plugins/gitextractor/parser/taskdata.go | 57 +++++++++++++
backend/plugins/gitextractor/tasks/clone.go | 8 +-
.../gitextractor/tasks/git_repo_collector.go | 35 +-------
env.example | 8 +-
9 files changed, 182 insertions(+), 171 deletions(-)
diff --git a/backend/plugins/gitextractor/gitextractor.go b/backend/plugins/gitextractor/gitextractor.go
new file mode 100644
index 000000000..5032381aa
--- /dev/null
+++ b/backend/plugins/gitextractor/gitextractor.go
@@ -0,0 +1,60 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main // must be main for plugin entry point
+
+import (
+ "github.com/apache/incubator-devlake/core/runner"
+ "github.com/apache/incubator-devlake/plugins/gitextractor/impl"
+ "github.com/spf13/cobra"
+)
+
+var PluginEntry impl.GitExtractor //nolint
+
+// standalone mode for debugging
+func main() {
+ cmd := &cobra.Command{Use: "gitextractor"}
+ url := cmd.Flags().StringP("url", "l", "", "repo url")
+ repoId := cmd.Flags().StringP("repoId", "i", "", "domain layer repo id")
+ user := cmd.Flags().StringP("user", "u", "", "username")
+ password := cmd.Flags().StringP("password", "p", "", "password")
+ // pk := cmd.Flags().StringP("privateKey", "k", "", "private key file")
+ // pkPass := cmd.Flags().StringP("privateKeyPassPhrase", "P", "", "passphrase for private key")
+ proxy := cmd.Flags().StringP("proxy", "x", "", "proxy")
+ useGoGit := cmd.Flags().BoolP("useGoGit", "g", false, "use go-git instead of libgit2")
+ skipCommitStat := cmd.Flags().BoolP("skipCommitStat", "S", true, "")
+ skipCommitFiles := cmd.Flags().BoolP("skipCommitFiles", "F", true, "")
+ timeAfter := cmd.Flags().StringP("timeAfter", "a", "", "collect data that are created after specified time, ie 2006-01-02T15:04:05Z")
+ _ = cmd.MarkFlagRequired("url")
+ _ = cmd.MarkFlagRequired("repoId")
+
+ cmd.Run = func(c *cobra.Command, args []string) {
+ runner.DirectRun(c, args, PluginEntry, map[string]interface{}{
+ "url": *url,
+ "repoId": *repoId,
+ "user": *user,
+ "password": *password,
+ // "privateKey": *
+ // "passphrase"
+ "proxy": *proxy,
+ "useGoGit": *useGoGit,
+ "skipCommitStat": skipCommitStat,
+ "skipCommitFiles": skipCommitFiles,
+ }, *timeAfter)
+ }
+ runner.RunCmd(cmd)
+}
diff --git a/backend/plugins/gitextractor/impl/impl.go b/backend/plugins/gitextractor/impl/impl.go
index d8d5413d4..27cb43e38 100644
--- a/backend/plugins/gitextractor/impl/impl.go
+++ b/backend/plugins/gitextractor/impl/impl.go
@@ -22,6 +22,7 @@ import (
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "github.com/apache/incubator-devlake/plugins/gitextractor/parser"
"github.com/apache/incubator-devlake/plugins/gitextractor/tasks"
)
@@ -58,21 +59,41 @@ func (p GitExtractor) SubTaskMetas() []plugin.SubTaskMeta {
// PrepareTaskData based on task context and user input options, return data that shared among all subtasks
func (p GitExtractor) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]interface{}) (interface{}, errors.Error) {
- var op tasks.GitExtractorOptions
+ log := taskCtx.GetLogger().Nested("gitextractor.PrepareTaskData")
+ var op parser.GitExtractorOptions
if err := helper.Decode(options, &op, nil); err != nil {
return nil, err
}
if err := op.Valid(); err != nil {
return nil, err
}
- taskData := &tasks.GitExtractorTaskData{
+
+ // commit stat, especially commit files(part of stat) are expensive to collect, so we skip them by default
+ cfg := taskCtx.GetConfigReader()
+ loadBool := func(optValue **bool, cfgKey string, defValue bool) {
+ // if user specified the option, use it
+ if *optValue != nil {
+ return
+ }
+ // or fallback to .env configuration
+ if cfg.IsSet(cfgKey) {
+ defValue = cfg.GetBool(cfgKey)
+ }
+ *optValue = &defValue
+ }
+ loadBool(&op.SkipCommitStat, "SKIP_COMMIT_STAT", true)
+ loadBool(&op.SkipCommitFiles, "SKIP_COMMIT_FILES", true)
+ log.Info("SkipCommitStat: %v", *op.SkipCommitStat)
+ log.Info("SkipCommitFiles: %v", *op.SkipCommitFiles)
+
+ taskData := &parser.GitExtractorTaskData{
Options: &op,
}
return taskData, nil
}
func (p GitExtractor) Close(taskCtx plugin.TaskContext) errors.Error {
- if taskData, ok := taskCtx.GetData().(*tasks.GitExtractorTaskData); ok {
+ if taskData, ok := taskCtx.GetData().(*parser.GitExtractorTaskData); ok {
if taskData.GitRepo != nil {
if err := taskData.GitRepo.Close(taskCtx.GetContext()); err != nil {
return errors.Convert(err)
diff --git a/backend/plugins/gitextractor/main.go b/backend/plugins/gitextractor/main.go
deleted file mode 100644
index f9cc30490..000000000
--- a/backend/plugins/gitextractor/main.go
+++ /dev/null
@@ -1,95 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package main
-
-import (
- "context"
- "flag"
- "github.com/apache/incubator-devlake/core/config"
- "github.com/apache/incubator-devlake/core/runner"
- contextimpl "github.com/apache/incubator-devlake/impls/context"
- "github.com/apache/incubator-devlake/impls/dalgorm"
- "github.com/apache/incubator-devlake/impls/logruslog"
- "github.com/apache/incubator-devlake/plugins/gitextractor/impl"
- "github.com/apache/incubator-devlake/plugins/gitextractor/models"
- "github.com/apache/incubator-devlake/plugins/gitextractor/store"
- "github.com/apache/incubator-devlake/plugins/gitextractor/tasks"
-)
-
-// PluginEntry is a variable exported for Framework to search and load
-var PluginEntry impl.GitExtractor //nolint
-
-func main() {
- url := flag.String("url", "", "-url")
- proxy := flag.String("proxy", "", "-proxy")
- id := flag.String("id", "", "-id")
- user := flag.String("user", "", "-user")
- password := flag.String("password", "", "-password")
- output := flag.String("output", "", "-output")
- dbUrl := flag.String("db", "", "-db")
- flag.Parse()
- cfg := config.GetConfig()
- logger := logruslog.Global.Nested("git extractor")
- var storage models.Store
- var err error
- if *url == "" {
- panic("url is missing")
- }
- if *id == "" {
- panic("id is missing")
- }
- db, err := runner.NewGormDb(cfg, logger)
- if err != nil {
- panic(err)
- }
- basicRes := contextimpl.NewDefaultBasicRes(cfg, logger, dalgorm.NewDalgorm(db))
- if *output != "" {
- storage, err = store.NewCsvStore(*output)
- if err != nil {
- panic(err)
- }
- } else if *dbUrl != "" {
- cfg.Set("DB_URL", *dbUrl)
- }
- // If we didn't specify output or dburl, we will use db by default
- if storage == nil {
- storage = store.NewDatabase(basicRes, *id)
- }
- defer storage.Close()
- ctx := context.Background()
- subTaskCtx := contextimpl.NewStandaloneSubTaskContext(
- ctx,
- basicRes,
- "git extractor",
- nil,
- )
- repo, err := tasks.NewGitRepo(subTaskCtx, logger, storage, &tasks.GitExtractorOptions{
- RepoId: *id,
- Url: *url,
- User: *user,
- Password: *password,
- Proxy: *proxy,
- })
- if err != nil {
- panic(err)
- }
- defer repo.Close(ctx)
- if err = repo.CollectAll(subTaskCtx); err != nil {
- panic(err)
- }
-}
diff --git a/backend/plugins/gitextractor/parser/repo_gogit.go b/backend/plugins/gitextractor/parser/repo_gogit.go
index 82be9b365..b12c87c43 100644
--- a/backend/plugins/gitextractor/parser/repo_gogit.go
+++ b/backend/plugins/gitextractor/parser/repo_gogit.go
@@ -22,6 +22,8 @@ import (
"crypto/sha256"
"encoding/hex"
"fmt"
+ "regexp"
+
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/log"
@@ -33,7 +35,6 @@ import (
"github.com/go-git/go-git/v5/plumbing"
"github.com/go-git/go-git/v5/plumbing/object"
"github.com/go-git/go-git/v5/plumbing/storer"
- "regexp"
)
type GoGitRepo struct {
@@ -268,13 +269,13 @@ func (r *GoGitRepo) getComponentMap(subtaskCtx plugin.SubTaskContext) (map[strin
// CollectCommits Collect data from each commit, we can also get the diff line
func (r *GoGitRepo) CollectCommits(subtaskCtx plugin.SubTaskContext) (err error) {
+ taskOpts := subtaskCtx.GetData().(*GitExtractorTaskData).Options
// check it first
componentMap, err := r.getComponentMap(subtaskCtx)
if err != nil {
return err
}
- skipCommitFiles := subtaskCtx.GetConfigReader().GetBool(SkipCommitFiles)
repo := r.repo
store := r.store
@@ -306,19 +307,21 @@ func (r *GoGitRepo) CollectCommits(subtaskCtx plugin.SubTaskContext) (err error)
return err
}
- stats, err := commit.StatsContext(subtaskCtx.GetContext())
- if err != nil {
- return err
- } else {
- for _, stat := range stats {
- codeCommit.Additions += stat.Addition
- // In some repos, deletion may be zero, which is different from git log --stat.
- // It seems go-git doesn't get the correct changes.
- // I have run object.DiffTreeWithOptions manually with different diff algorithms,
- // but get the same result with StatsContext.
- // I cannot reproduce it with another repo.
- // A similar issue: https://github.com/go-git/go-git/issues/367
- codeCommit.Deletions += stat.Deletion
+ if !*taskOpts.SkipCommitStat {
+ stats, err := commit.StatsContext(subtaskCtx.GetContext())
+ if err != nil {
+ return err
+ } else {
+ for _, stat := range stats {
+ codeCommit.Additions += stat.Addition
+ // In some repos, deletion may be zero, which is different from git log --stat.
+ // It seems go-git doesn't get the correct changes.
+ // I have run object.DiffTreeWithOptions manually with different diff algorithms,
+ // but get the same result with StatsContext.
+ // I cannot reproduce it with another repo.
+ // A similar issue: https://github.com/go-git/go-git/issues/367
+ codeCommit.Deletions += stat.Deletion
+ }
}
}
@@ -335,7 +338,7 @@ func (r *GoGitRepo) CollectCommits(subtaskCtx plugin.SubTaskContext) (err error)
if err != nil {
return err
}
- if !skipCommitFiles {
+ if !*taskOpts.SkipCommitFiles {
if err := r.storeDiffCommitFilesComparedToParent(subtaskCtx, componentMap, commit); err != nil {
return err
}
diff --git a/backend/plugins/gitextractor/parser/repo_libgit2.go b/backend/plugins/gitextractor/parser/repo_libgit2.go
index c05c2bfcc..2bf2d6037 100644
--- a/backend/plugins/gitextractor/parser/repo_libgit2.go
+++ b/backend/plugins/gitextractor/parser/repo_libgit2.go
@@ -26,7 +26,6 @@ import (
"sort"
"strconv"
- "github.com/apache/incubator-devlake/core/config"
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/log"
@@ -38,8 +37,6 @@ import (
git "github.com/libgit2/git2go/v33"
)
-const SkipCommitFiles = "SKIP_COMMIT_FILES"
-
var TypeNotMatchError = "the requested type does not match the type in the ODB"
type GitRepo struct {
@@ -231,6 +228,7 @@ func (r *GitRepo) CollectBranches(subtaskCtx plugin.SubTaskContext) error {
// CollectCommits Collect data from each commit, we can also get the diff line
func (r *GitRepo) CollectCommits(subtaskCtx plugin.SubTaskContext) error {
+ taskOpts := subtaskCtx.GetData().(*GitExtractorTaskData).Options
opts, err := getDiffOpts()
if err != nil {
return err
@@ -290,13 +288,16 @@ func (r *GitRepo) CollectCommits(subtaskCtx plugin.SubTaskContext) error {
if commit.ParentCount() > 0 {
parent = commit.Parent(0)
}
- var stats *git.DiffStats
- if stats, err = r.getDiffComparedToParent(c.Sha, commit, parent, opts, componentMap); err != nil {
- return err
+
+ if !*taskOpts.SkipCommitStat {
+ var stats *git.DiffStats
+ if stats, err = r.getDiffComparedToParent(taskOpts, c.Sha, commit, parent, opts, componentMap); err != nil {
+ return err
+ }
+ r.logger.Debug("state: %#+v\n", stats.Deletions())
+ c.Additions += stats.Insertions()
+ c.Deletions += stats.Deletions()
}
- r.logger.Debug("state: %#+v\n", stats.Deletions())
- c.Additions += stats.Insertions()
- c.Deletions += stats.Deletions()
err = r.store.Commits(c)
if err != nil {
@@ -331,7 +332,7 @@ func (r *GitRepo) storeParentCommits(commitSha string, commit *git.Commit) error
return r.store.CommitParents(commitParents)
}
-func (r *GitRepo) getDiffComparedToParent(commitSha string, commit *git.Commit, parent *git.Commit, opts *git.DiffOptions, componentMap map[string]*regexp.Regexp) (*git.DiffStats, errors.Error) {
+func (r *GitRepo) getDiffComparedToParent(taskOpts *GitExtractorOptions, commitSha string, commit *git.Commit, parent *git.Commit, opts *git.DiffOptions, componentMap map[string]*regexp.Regexp) (*git.DiffStats, errors.Error) {
var err error
var parentTree, tree *git.Tree
if parent != nil {
@@ -349,12 +350,7 @@ func (r *GitRepo) getDiffComparedToParent(commitSha string, commit *git.Commit,
if err != nil {
return nil, errors.Convert(err)
}
- cfg := config.GetConfig()
- skipCommitFiles := true
- if cfg.IsSet(SkipCommitFiles) {
- skipCommitFiles = cfg.GetBool(SkipCommitFiles)
- }
- if !skipCommitFiles {
+ if !*taskOpts.SkipCommitFiles {
err = r.storeCommitFilesFromDiff(commitSha, diff, componentMap)
if err != nil {
return nil, errors.Convert(err)
diff --git a/backend/plugins/gitextractor/parser/taskdata.go b/backend/plugins/gitextractor/parser/taskdata.go
new file mode 100644
index 000000000..fcbdf20af
--- /dev/null
+++ b/backend/plugins/gitextractor/parser/taskdata.go
@@ -0,0 +1,57 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package parser
+
+import (
+ "strings"
+
+ "github.com/apache/incubator-devlake/core/errors"
+)
+
+type GitExtractorTaskData struct {
+ Options *GitExtractorOptions
+ GitRepo RepoCollector
+}
+
+type GitExtractorOptions struct {
+ RepoId string `json:"repoId" mapstructure:"repoId"`
+ Name string `json:"name" mapstructure:"name"`
+ Url string `json:"url" mapstructure:"url"`
+ User string `json:"user" mapstructure:"user"`
+ Password string `json:"password" mapstructure:"password"`
+ PrivateKey string `json:"privateKey" mapstructure:"privateKey"`
+ Passphrase string `json:"passphrase" mapstructure:"passphrase"`
+ Proxy string `json:"proxy" mapstructure:"proxy"`
+ UseGoGit bool `json:"useGoGit" mapstructure:"useGoGit"`
+ SkipCommitStat *bool `json:"skipCommitStat" mapstructure:"skipCommitStat" comment:"skip all commit stat including added/deleted lines and commit files as well"`
+ SkipCommitFiles *bool `json:"skipCommitFiles" mapstructure:"skipCommitFiles"`
+}
+
+func (o GitExtractorOptions) Valid() errors.Error {
+ if o.RepoId == "" {
+ return errors.BadInput.New("empty repoId")
+ }
+ if o.Url == "" {
+ return errors.BadInput.New("empty url")
+ }
+ url := strings.TrimPrefix(o.Url, "ssh://")
+ if !(strings.HasPrefix(o.Url, "http") || strings.HasPrefix(url, "git@") || strings.HasPrefix(o.Url, "/")) {
+ return errors.BadInput.New("wrong url")
+ }
+ return nil
+}
diff --git a/backend/plugins/gitextractor/tasks/clone.go b/backend/plugins/gitextractor/tasks/clone.go
index 599d4adb6..14feadbbe 100644
--- a/backend/plugins/gitextractor/tasks/clone.go
+++ b/backend/plugins/gitextractor/tasks/clone.go
@@ -41,7 +41,7 @@ var CloneGitRepoMeta = plugin.SubTaskMeta{
ForceRunOnResume: true,
}
-func useGoGit(subTaskCtx plugin.SubTaskContext, taskData *GitExtractorTaskData) bool {
+func useGoGit(subTaskCtx plugin.SubTaskContext, taskData *parser.GitExtractorTaskData) bool {
if subTaskCtx.GetConfigReader().GetBool(useGoGitInGitExtractor) {
return true
}
@@ -52,7 +52,7 @@ func useGoGit(subTaskCtx plugin.SubTaskContext, taskData *GitExtractorTaskData)
}
func CloneGitRepo(subTaskCtx plugin.SubTaskContext) errors.Error {
- taskData, ok := subTaskCtx.GetData().(*GitExtractorTaskData)
+ taskData, ok := subTaskCtx.GetData().(*parser.GitExtractorTaskData)
if !ok {
panic("git repo reference not found on context")
}
@@ -77,7 +77,7 @@ func CloneGitRepo(subTaskCtx plugin.SubTaskContext) errors.Error {
}
// NewGitRepo create and return a new parser git repo
-func NewGitRepo(ctx plugin.SubTaskContext, logger log.Logger, storage models.Store, op *GitExtractorOptions) (parser.RepoCollector, errors.Error) {
+func NewGitRepo(ctx plugin.SubTaskContext, logger log.Logger, storage models.Store, op *parser.GitExtractorOptions) (parser.RepoCollector, errors.Error) {
var err errors.Error
var repo parser.RepoCollector
p := parser.NewGitRepoCreator(storage, logger)
@@ -94,7 +94,7 @@ func NewGitRepo(ctx plugin.SubTaskContext, logger log.Logger, storage models.Sto
}
// NewGoGitRepo create and return a new parser git repo with go-git
-func NewGoGitRepo(ctx plugin.SubTaskContext, logger log.Logger, storage models.Store, op *GitExtractorOptions) (parser.RepoCollector, errors.Error) {
+func NewGoGitRepo(ctx plugin.SubTaskContext, logger log.Logger, storage models.Store, op *parser.GitExtractorOptions) (parser.RepoCollector, errors.Error) {
var err errors.Error
var repo parser.RepoCollector
p := parser.NewGitRepoCreator(storage, logger)
diff --git a/backend/plugins/gitextractor/tasks/git_repo_collector.go b/backend/plugins/gitextractor/tasks/git_repo_collector.go
index 00abf28cf..c8a287b36 100644
--- a/backend/plugins/gitextractor/tasks/git_repo_collector.go
+++ b/backend/plugins/gitextractor/tasks/git_repo_collector.go
@@ -18,44 +18,11 @@ limitations under the License.
package tasks
import (
- "strings"
-
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/plugins/gitextractor/parser"
)
-type GitExtractorTaskData struct {
- Options *GitExtractorOptions
- GitRepo parser.RepoCollector
-}
-
-type GitExtractorOptions struct {
- RepoId string `json:"repoId"`
- Name string `json:"name"`
- Url string `json:"url"`
- User string `json:"user"`
- Password string `json:"password"`
- PrivateKey string `json:"privateKey"`
- Passphrase string `json:"passphrase"`
- Proxy string `json:"proxy"`
- UseGoGit bool `json:"use_go_git"`
-}
-
-func (o GitExtractorOptions) Valid() errors.Error {
- if o.RepoId == "" {
- return errors.BadInput.New("empty repoId")
- }
- if o.Url == "" {
- return errors.BadInput.New("empty url")
- }
- url := strings.TrimPrefix(o.Url, "ssh://")
- if !(strings.HasPrefix(o.Url, "http") || strings.HasPrefix(url, "git@") || strings.HasPrefix(o.Url, "/")) {
- return errors.BadInput.New("wrong url")
- }
- return nil
-}
-
func CollectGitCommits(subTaskCtx plugin.SubTaskContext) errors.Error {
repo := getGitRepo(subTaskCtx)
if count, err := repo.CountCommits(subTaskCtx.GetContext()); err != nil {
@@ -105,7 +72,7 @@ func CollectGitDiffLines(subTaskCtx plugin.SubTaskContext) errors.Error {
}
func getGitRepo(subTaskCtx plugin.SubTaskContext) parser.RepoCollector {
- taskData, ok := subTaskCtx.GetData().(*GitExtractorTaskData)
+ taskData, ok := subTaskCtx.GetData().(*parser.GitExtractorTaskData)
if !ok {
panic("git repo reference not found on context")
}
diff --git a/env.example b/env.example
index aab1ff22a..a934fbdea 100755
--- a/env.example
+++ b/env.example
@@ -39,9 +39,8 @@ API_TIMEOUT=120s
API_RETRY=3
API_REQUESTS_PER_HOUR=10000
PIPELINE_MAX_PARALLEL=1
-#TEMPORAL_URL=temporal:7233
-TEMPORAL_URL=
-TEMPORAL_TASK_QUEUE=
+# resume undone pipelines on start
+RESUME_PIPELINES=true
# Debug Info Warn Error
LOGGING_LEVEL=
LOGGING_DIR=./logs
@@ -67,6 +66,9 @@ IN_SECURE_SKIP_VERIFY=
# In plugin gitextractor, use go-git to collector repo's data
##########################
USE_GO_GIT_IN_GIT_EXTRACTOR=false
+# NOTE that COMMIT_FILES is part of the COMMIT_STAT
+SKIP_COMMIT_STAT=true
+SKIP_COMMIT_FILES=true
# Set if response error when requesting /connections/{connection_id}/test should be wrapped or not
##########################