You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by ab...@apache.org on 2022/07/22 09:40:05 UTC
[incubator-devlake] branch main updated (21b029f1 -> ff7ccd1f)
This is an automated email from the ASF dual-hosted git repository.
abeizn pushed a change to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
from 21b029f1 `hotfix` config-ui: blueprints service pack 1.0 (QA Bugfixes) (#2534)
new ca2404c3 fix: change migrationscript
new 128279f1 fix: add commitfileComponent table
new 5cedd9e6 fix: update schema
new 172b2183 fix: update migrationscript
new 5ccbc1e3 fix: change table name filecomponent->component; commitfilecomponent->filecomponent
new d9b99a69 fix: change table desgn in components commit_file_components commit_files
new af864443 fix: update db schema
new a7b45969 fix: change migrationscript
new 0cd50247 fix: change component model to a new file and rename some file
new 71466d34 fix: update migrationscript version
new 0a530044 fix: drop commitfile table before automigrate table
new 6f47a524 fix: update csv.go format
new ff7ccd1f fix: update migrationscript drop arichived.commitfile table before
The 13 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
models/domainlayer/code/commit.go | 17 ++++-
.../blueprint.go => domainlayer/code/component.go} | 20 ++----
.../20220722_commitfile_component.go | 82 ++++++++++++++++++++++
models/migrationscripts/register.go | 1 +
plugins/gitextractor/models/interface.go | 1 +
plugins/gitextractor/parser/repo.go | 44 ++++++++++--
plugins/gitextractor/store/csv.go | 4 ++
plugins/gitextractor/store/database.go | 8 +++
8 files changed, 157 insertions(+), 20 deletions(-)
copy models/{migrationscripts/archived/blueprint.go => domainlayer/code/component.go} (76%)
create mode 100644 models/migrationscripts/20220722_commitfile_component.go
[incubator-devlake] 10/13: fix: update migrationscript version
Posted by ab...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
abeizn pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit 71466d34a25ce3185b1846591fd50cb9e6d97b93
Author: xgdyp <zx...@163.com>
AuthorDate: Fri Jul 22 16:25:01 2022 +0800
fix: update migrationscript version
---
...220721_commitfile_component.go => 20220722_commitfile_component.go} | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/models/migrationscripts/20220721_commitfile_component.go b/models/migrationscripts/20220722_commitfile_component.go
similarity index 98%
rename from models/migrationscripts/20220721_commitfile_component.go
rename to models/migrationscripts/20220722_commitfile_component.go
index dc3ff8e6..e4a9c17e 100644
--- a/models/migrationscripts/20220721_commitfile_component.go
+++ b/models/migrationscripts/20220722_commitfile_component.go
@@ -69,9 +69,10 @@ func (*commitfileComponent) Up(ctx context.Context, db *gorm.DB) error {
}
func (*commitfileComponent) Version() uint64 {
- return 202207211505
+ return 20220722162305
}
func (*commitfileComponent) Name() string {
+
return "add commit_file_components components table,update commit_files table"
}
[incubator-devlake] 13/13: fix: update migrationscript drop arichived.commitfile table before
Posted by ab...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
abeizn pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit ff7ccd1f60b3ee494463702da800b3f0f60138ea
Author: xgdyp <zx...@163.com>
AuthorDate: Fri Jul 22 17:02:05 2022 +0800
fix: update migrationscript drop arichived.commitfile table before
---
models/migrationscripts/20220722_commitfile_component.go | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/models/migrationscripts/20220722_commitfile_component.go b/models/migrationscripts/20220722_commitfile_component.go
index 88a346a8..f4bf01c9 100644
--- a/models/migrationscripts/20220722_commitfile_component.go
+++ b/models/migrationscripts/20220722_commitfile_component.go
@@ -21,6 +21,7 @@ import (
"context"
"github.com/apache/incubator-devlake/models/common"
"github.com/apache/incubator-devlake/models/domainlayer"
+ "github.com/apache/incubator-devlake/models/migrationscripts/archived"
"gorm.io/gorm"
)
@@ -59,7 +60,7 @@ func (CommitFileComponent) TableName() string {
type commitfileComponent struct{}
func (*commitfileComponent) Up(ctx context.Context, db *gorm.DB) error {
- err := db.Migrator().DropTable(&CommitFile{})
+ err := db.Migrator().DropTable(&archived.CommitFile{})
if err != nil {
return err
}
@@ -72,7 +73,7 @@ func (*commitfileComponent) Up(ctx context.Context, db *gorm.DB) error {
}
func (*commitfileComponent) Version() uint64 {
- return 20220722162305
+ return 20220722165805
}
func (*commitfileComponent) Name() string {
[incubator-devlake] 06/13: fix: change table desgn in components commit_file_components commit_files
Posted by ab...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
abeizn pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit d9b99a69b221c5204e1681cc1e6071fa5fa71e19
Author: xgdyp <zx...@163.com>
AuthorDate: Fri Jul 15 17:22:50 2022 +0800
fix: change table desgn in components commit_file_components commit_files
---
models/domainlayer/code/commit.go | 21 ++++++-------
models/migrationscripts/updateSchemas20220711.go | 39 +++++++++---------------
plugins/gitextractor/models/interface.go | 2 +-
plugins/gitextractor/parser/repo.go | 25 ++++++++-------
plugins/gitextractor/store/csv.go | 2 +-
plugins/gitextractor/store/database.go | 2 +-
6 files changed, 39 insertions(+), 52 deletions(-)
diff --git a/models/domainlayer/code/commit.go b/models/domainlayer/code/commit.go
index 77065b1e..1fc76bc0 100644
--- a/models/domainlayer/code/commit.go
+++ b/models/domainlayer/code/commit.go
@@ -46,11 +46,11 @@ func (Commit) TableName() string {
type CommitFile struct {
common.NoPKModel
- CommitFileID string `gorm:"primaryKey;type:varchar(255)"`
- CommitSha string `gorm:"type:varchar(40)"`
- FilePath string `gorm:"type:varchar(255)"`
- Additions int
- Deletions int
+ ID string `gorm:"primaryKey;type:varchar(255)"`
+ CommitSha string `gorm:"type:varchar(40)"`
+ FilePath string `gorm:"type:varchar(255)"`
+ Additions int
+ Deletions int
}
func (CommitFile) TableName() string {
@@ -59,21 +59,20 @@ func (CommitFile) TableName() string {
type Component struct {
RepoId string `gorm:"primaryKey;type:varchar(255)"`
- Component string `gorm:"primaryKey;type:varchar(255)"`
+ Name string `gorm:"type:varchar(255)"`
PathRegex string `gorm:"type:varchar(255)"`
}
func (Component) TableName() string {
- return "component"
+ return "components"
}
-type FileComponent struct {
+type CommitFileComponent struct {
common.NoPKModel
CommitFileID string `gorm:"primaryKey;type:varchar(255)"`
- RepoId string `gorm:"primaryKey;type:varchar(255)"`
Component string `gorm:"type:varchar(255)"`
}
-func (FileComponent) TableName() string {
- return "file_component"
+func (CommitFileComponent) TableName() string {
+ return "commit_file_components"
}
diff --git a/models/migrationscripts/updateSchemas20220711.go b/models/migrationscripts/updateSchemas20220711.go
index 4aa8c64b..15ad50f7 100644
--- a/models/migrationscripts/updateSchemas20220711.go
+++ b/models/migrationscripts/updateSchemas20220711.go
@@ -23,55 +23,44 @@ import (
"gorm.io/gorm"
)
-//type CodeComponent20220711 struct {
-// ComponentId string `gorm:"primaryKey;type:varchar(255)"`
-// PathRegex string `gorm:"type:varchar(255)"`
-//}
-//
-//func (CodeComponent20220711) TableName() string {
-// return "code_component_20220711"
-//}
-
type Component struct {
- RepoId string `gorm:"primaryKey;type:varchar(255)"`
- Component string `gorm:"primaryKey;type:varchar(255)"`
+ RepoId string `gorm:"type:varchar(255)"`
+ Name string `gorm:"primaryKey;type:varchar(255)"`
PathRegex string `gorm:"type:varchar(255)"`
}
func (Component) TableName() string {
- return "component"
+ return "components"
}
type CommitFile struct {
common.NoPKModel
- CommitFileID string `gorm:"primaryKey;type:varchar(255)"`
- CommitSha string `gorm:"type:varchar(40)"`
- FilePath string `gorm:"type:varchar(255)"`
- Additions int
- Deletions int
- Component string `gorm:"type:varchar(255)"`
+ ID string `gorm:"primaryKey;type:varchar(255)"`
+ CommitSha string `gorm:"type:varchar(40)"`
+ FilePath string `gorm:"type:varchar(255)"`
+ Additions int
+ Deletions int
}
func (CommitFile) TableName() string {
return "commit_files"
}
-type FileComponent struct {
+type CommitFileComponent struct {
common.NoPKModel
CommitFileID string `gorm:"primaryKey;type:varchar(255)"`
- RepoId string `gorm:"primaryKey;type:varchar(255)"`
Component string `gorm:"type:varchar(255)"`
}
-func (FileComponent) TableName() string {
- return "file_component"
+func (CommitFileComponent) TableName() string {
+ return "commit_file_components"
}
type updateSchemas20220711 struct{}
func (*updateSchemas20220711) Up(ctx context.Context, db *gorm.DB) error {
- err := db.Migrator().AutoMigrate(Component{}, CommitFile{}, FileComponent{})
+ err := db.Migrator().AutoMigrate(Component{}, CommitFile{}, CommitFileComponent{})
if err != nil {
return err
}
@@ -80,9 +69,9 @@ func (*updateSchemas20220711) Up(ctx context.Context, db *gorm.DB) error {
}
func (*updateSchemas20220711) Version() uint64 {
- return 202207151420
+ return 202207151644
}
func (*updateSchemas20220711) Name() string {
- return "file_component table"
+ return "add commit_file_components components table,update commit_files table"
}
diff --git a/plugins/gitextractor/models/interface.go b/plugins/gitextractor/models/interface.go
index a5ae3a86..b3e5ca6c 100644
--- a/plugins/gitextractor/models/interface.go
+++ b/plugins/gitextractor/models/interface.go
@@ -27,6 +27,6 @@ type Store interface {
Refs(ref *code.Ref) error
CommitFiles(file *code.CommitFile) error
CommitParents(pp []*code.CommitParent) error
- FileComponent(component *code.FileComponent) error
+ FileComponent(component *code.CommitFileComponent) error
Close() error
}
diff --git a/plugins/gitextractor/parser/repo.go b/plugins/gitextractor/parser/repo.go
index a796ca23..8807ded0 100644
--- a/plugins/gitextractor/parser/repo.go
+++ b/plugins/gitextractor/parser/repo.go
@@ -198,7 +198,7 @@ func (r *GitRepo) CollectCommits(subtaskCtx core.SubTaskContext) error {
}
componentMap := make(map[string]*regexp.Regexp)
for _, component := range components {
- componentMap[component.Component] = regexp.MustCompile(component.PathRegex)
+ componentMap[component.Name] = regexp.MustCompile(component.PathRegex)
}
odb, err := r.repo.Odb()
if err != nil {
@@ -312,7 +312,7 @@ func (r *GitRepo) getDiffComparedToParent(commitSha string, commit *git.Commit,
func (r *GitRepo) storeCommitFilesFromDiff(commitSha string, diff *git.Diff, componentMap map[string]*regexp.Regexp) error {
var commitFile *code.CommitFile
- var commitfileComponent *code.FileComponent
+ var commitFileComponent *code.CommitFileComponent
var err error
err = diff.ForEach(func(file git.DiffDelta, progress float64) (
git.DiffForEachHunkCallback, error) {
@@ -327,20 +327,19 @@ func (r *GitRepo) storeCommitFilesFromDiff(commitSha string, diff *git.Diff, com
commitFile = new(code.CommitFile)
commitFile.CommitSha = commitSha
commitFile.FilePath = file.NewFile.Path
- commitFile.CommitFileID = commitSha + ":" + file.NewFile.Path
- commitfileComponent = new(code.FileComponent)
+ commitFile.ID = commitSha + ":" + file.NewFile.Path
+ commitFileComponent = new(code.CommitFileComponent)
for component, reg := range componentMap {
if reg.MatchString(commitFile.FilePath) {
- commitfileComponent.Component = component
+ commitFileComponent.Component = component
break
}
}
- commitfileComponent.RepoId = r.id
- commitfileComponent.CommitFileID = commitSha + ":" + file.NewFile.Path
- //commitfileComponent.FilePath = file.NewFile.Path
- //commitfileComponent.CommitSha = commitSha
- if commitfileComponent.Component == "" {
- commitfileComponent.Component = "Default"
+ commitFileComponent.CommitFileID = commitSha + ":" + file.NewFile.Path
+ //commitFileComponent.FilePath = file.NewFile.Path
+ //commitFileComponent.CommitSha = commitSha
+ if commitFileComponent.Component == "" {
+ commitFileComponent.Component = "Default"
}
return func(hunk git.DiffHunk) (git.DiffForEachLineCallback, error) {
return func(line git.DiffLine) error {
@@ -354,8 +353,8 @@ func (r *GitRepo) storeCommitFilesFromDiff(commitSha string, diff *git.Diff, com
}, nil
}, nil
}, git.DiffDetailLines)
- if commitfileComponent != nil {
- err = r.store.FileComponent(commitfileComponent)
+ if commitFileComponent != nil {
+ err = r.store.FileComponent(commitFileComponent)
if err != nil {
r.logger.Error("FileComponent error:", err)
}
diff --git a/plugins/gitextractor/store/csv.go b/plugins/gitextractor/store/csv.go
index bdf4205c..fa2b8845 100644
--- a/plugins/gitextractor/store/csv.go
+++ b/plugins/gitextractor/store/csv.go
@@ -132,7 +132,7 @@ func (c *CsvStore) Refs(ref *code.Ref) error {
func (c *CsvStore) CommitFiles(file *code.CommitFile) error {
return c.commitFileWriter.Write(file)
}
-func (c *CsvStore) FileComponent(component *code.FileComponent) error {
+func (c *CsvStore) FileComponent(component *code.CommitFileComponent) error {
return c.commitFileWriter.Write(component)
}
diff --git a/plugins/gitextractor/store/database.go b/plugins/gitextractor/store/database.go
index 564433d5..8bca2147 100644
--- a/plugins/gitextractor/store/database.go
+++ b/plugins/gitextractor/store/database.go
@@ -91,7 +91,7 @@ func (d *Database) CommitFiles(file *code.CommitFile) error {
return batch.Add(file)
}
-func (d *Database) FileComponent(commitfile *code.FileComponent) error {
+func (d *Database) FileComponent(commitfile *code.CommitFileComponent) error {
batch, err := d.driver.ForType(reflect.TypeOf(commitfile))
if err != nil {
return err
[incubator-devlake] 08/13: fix: change migrationscript
Posted by ab...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
abeizn pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit a7b45969458bdb643cb712d7c4e195a0f4165fa5
Author: xgdyp <zx...@163.com>
AuthorDate: Wed Jul 20 12:20:07 2022 +0800
fix: change migrationscript
---
models/domainlayer/code/commit.go | 12 ++++++------
models/migrationscripts/commitfile_component.go | 18 +++++++++---------
models/migrationscripts/register.go | 3 +--
plugins/gitextractor/models/interface.go | 2 +-
plugins/gitextractor/parser/repo.go | 14 +++++++-------
plugins/gitextractor/store/csv.go | 4 ++--
plugins/gitextractor/store/database.go | 6 +++---
7 files changed, 29 insertions(+), 30 deletions(-)
diff --git a/models/domainlayer/code/commit.go b/models/domainlayer/code/commit.go
index 1fc76bc0..fd5707a1 100644
--- a/models/domainlayer/code/commit.go
+++ b/models/domainlayer/code/commit.go
@@ -18,6 +18,7 @@ limitations under the License.
package code
import (
+ "github.com/apache/incubator-devlake/models/domainlayer"
"time"
"github.com/apache/incubator-devlake/models/common"
@@ -45,8 +46,7 @@ func (Commit) TableName() string {
}
type CommitFile struct {
- common.NoPKModel
- ID string `gorm:"primaryKey;type:varchar(255)"`
+ domainlayer.DomainEntity
CommitSha string `gorm:"type:varchar(40)"`
FilePath string `gorm:"type:varchar(255)"`
Additions int
@@ -58,8 +58,8 @@ func (CommitFile) TableName() string {
}
type Component struct {
- RepoId string `gorm:"primaryKey;type:varchar(255)"`
- Name string `gorm:"type:varchar(255)"`
+ RepoId string `gorm:"type:varchar(255)"`
+ Name string `gorm:"primaryKey;type:varchar(255)"`
PathRegex string `gorm:"type:varchar(255)"`
}
@@ -69,8 +69,8 @@ func (Component) TableName() string {
type CommitFileComponent struct {
common.NoPKModel
- CommitFileID string `gorm:"primaryKey;type:varchar(255)"`
- Component string `gorm:"type:varchar(255)"`
+ CommitFileId string `gorm:"primaryKey;type:varchar(255)"`
+ ComponentName string `gorm:"type:varchar(255)"`
}
func (CommitFileComponent) TableName() string {
diff --git a/models/migrationscripts/commitfile_component.go b/models/migrationscripts/commitfile_component.go
index 15ad50f7..9f1ee69c 100644
--- a/models/migrationscripts/commitfile_component.go
+++ b/models/migrationscripts/commitfile_component.go
@@ -20,6 +20,7 @@ package migrationscripts
import (
"context"
"github.com/apache/incubator-devlake/models/common"
+ "github.com/apache/incubator-devlake/models/domainlayer"
"gorm.io/gorm"
)
@@ -34,8 +35,7 @@ func (Component) TableName() string {
}
type CommitFile struct {
- common.NoPKModel
- ID string `gorm:"primaryKey;type:varchar(255)"`
+ domainlayer.DomainEntity
CommitSha string `gorm:"type:varchar(40)"`
FilePath string `gorm:"type:varchar(255)"`
Additions int
@@ -48,17 +48,17 @@ func (CommitFile) TableName() string {
type CommitFileComponent struct {
common.NoPKModel
- CommitFileID string `gorm:"primaryKey;type:varchar(255)"`
- Component string `gorm:"type:varchar(255)"`
+ CommitFileId string `gorm:"primaryKey;type:varchar(255)"`
+ ComponentName string `gorm:"type:varchar(255)"`
}
func (CommitFileComponent) TableName() string {
return "commit_file_components"
}
-type updateSchemas20220711 struct{}
+type commitfileComponent struct{}
-func (*updateSchemas20220711) Up(ctx context.Context, db *gorm.DB) error {
+func (*commitfileComponent) Up(ctx context.Context, db *gorm.DB) error {
err := db.Migrator().AutoMigrate(Component{}, CommitFile{}, CommitFileComponent{})
if err != nil {
@@ -68,10 +68,10 @@ func (*updateSchemas20220711) Up(ctx context.Context, db *gorm.DB) error {
}
-func (*updateSchemas20220711) Version() uint64 {
- return 202207151644
+func (*commitfileComponent) Version() uint64 {
+ return 202207201009
}
-func (*updateSchemas20220711) Name() string {
+func (*commitfileComponent) Name() string {
return "add commit_file_components components table,update commit_files table"
}
diff --git a/models/migrationscripts/register.go b/models/migrationscripts/register.go
index 2f5351b3..2142ec9a 100644
--- a/models/migrationscripts/register.go
+++ b/models/migrationscripts/register.go
@@ -22,7 +22,6 @@ import "github.com/apache/incubator-devlake/migration"
// All return all the migration scripts of framework
func All() []migration.Script {
return []migration.Script{
-
new(addFrameTables),
new(renameStepToStage),
new(addSubtasksField),
@@ -30,6 +29,6 @@ func All() []migration.Script {
new(renameTasksToPlan),
new(addDomainTables),
new(addTypeField),
- new(updateSchemas20220711),
+ new(commitfileComponent),
}
}
diff --git a/plugins/gitextractor/models/interface.go b/plugins/gitextractor/models/interface.go
index b3e5ca6c..22b5d93e 100644
--- a/plugins/gitextractor/models/interface.go
+++ b/plugins/gitextractor/models/interface.go
@@ -27,6 +27,6 @@ type Store interface {
Refs(ref *code.Ref) error
CommitFiles(file *code.CommitFile) error
CommitParents(pp []*code.CommitParent) error
- FileComponent(component *code.CommitFileComponent) error
+ CommitFileComponents(commitFileComponent *code.CommitFileComponent) error
Close() error
}
diff --git a/plugins/gitextractor/parser/repo.go b/plugins/gitextractor/parser/repo.go
index 8807ded0..cf1d5655 100644
--- a/plugins/gitextractor/parser/repo.go
+++ b/plugins/gitextractor/parser/repo.go
@@ -327,19 +327,19 @@ func (r *GitRepo) storeCommitFilesFromDiff(commitSha string, diff *git.Diff, com
commitFile = new(code.CommitFile)
commitFile.CommitSha = commitSha
commitFile.FilePath = file.NewFile.Path
- commitFile.ID = commitSha + ":" + file.NewFile.Path
+ commitFile.Id = commitSha + ":" + file.NewFile.Path
commitFileComponent = new(code.CommitFileComponent)
for component, reg := range componentMap {
if reg.MatchString(commitFile.FilePath) {
- commitFileComponent.Component = component
+ commitFileComponent.ComponentName = component
break
}
}
- commitFileComponent.CommitFileID = commitSha + ":" + file.NewFile.Path
+ commitFileComponent.CommitFileId = commitSha + ":" + file.NewFile.Path
//commitFileComponent.FilePath = file.NewFile.Path
//commitFileComponent.CommitSha = commitSha
- if commitFileComponent.Component == "" {
- commitFileComponent.Component = "Default"
+ if commitFileComponent.ComponentName == "" {
+ commitFileComponent.ComponentName = "Default"
}
return func(hunk git.DiffHunk) (git.DiffForEachLineCallback, error) {
return func(line git.DiffLine) error {
@@ -354,9 +354,9 @@ func (r *GitRepo) storeCommitFilesFromDiff(commitSha string, diff *git.Diff, com
}, nil
}, git.DiffDetailLines)
if commitFileComponent != nil {
- err = r.store.FileComponent(commitFileComponent)
+ err = r.store.CommitFileComponents(commitFileComponent)
if err != nil {
- r.logger.Error("FileComponent error:", err)
+ r.logger.Error("CommitFileComponents error:", err)
}
}
if commitFile != nil {
diff --git a/plugins/gitextractor/store/csv.go b/plugins/gitextractor/store/csv.go
index fa2b8845..52820c7b 100644
--- a/plugins/gitextractor/store/csv.go
+++ b/plugins/gitextractor/store/csv.go
@@ -132,8 +132,8 @@ func (c *CsvStore) Refs(ref *code.Ref) error {
func (c *CsvStore) CommitFiles(file *code.CommitFile) error {
return c.commitFileWriter.Write(file)
}
-func (c *CsvStore) FileComponent(component *code.CommitFileComponent) error {
- return c.commitFileWriter.Write(component)
+func (c *CsvStore) CommitFileComponents(commitFileComponent *code.CommitFileComponent) error {
+ return c.commitFileWriter.Write(commitFileComponent)
}
func (c *CsvStore) CommitParents(pp []*code.CommitParent) error {
diff --git a/plugins/gitextractor/store/database.go b/plugins/gitextractor/store/database.go
index 8bca2147..9c85aff1 100644
--- a/plugins/gitextractor/store/database.go
+++ b/plugins/gitextractor/store/database.go
@@ -91,12 +91,12 @@ func (d *Database) CommitFiles(file *code.CommitFile) error {
return batch.Add(file)
}
-func (d *Database) FileComponent(commitfile *code.CommitFileComponent) error {
- batch, err := d.driver.ForType(reflect.TypeOf(commitfile))
+func (d *Database) CommitFileComponents(commitFileComponent *code.CommitFileComponent) error {
+ batch, err := d.driver.ForType(reflect.TypeOf(commitFileComponent))
if err != nil {
return err
}
- return batch.Add(commitfile)
+ return batch.Add(commitFileComponent)
}
func (d *Database) CommitParents(pp []*code.CommitParent) error {
[incubator-devlake] 01/13: fix: change migrationscript
Posted by ab...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
abeizn pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit ca2404c31ad4d92e95ff9882223bc6a654dd2aa5
Author: xgdyp <zx...@163.com>
AuthorDate: Mon Jul 11 17:00:28 2022 +0800
fix: change migrationscript
---
models/domainlayer/code/commit.go | 11 ++++
models/migrationscripts/register.go | 2 +
.../updateSchemas20220711.go} | 59 ++++++++++++++--------
plugins/gitextractor/parser/repo.go | 31 ++++++++++--
4 files changed, 79 insertions(+), 24 deletions(-)
diff --git a/models/domainlayer/code/commit.go b/models/domainlayer/code/commit.go
index 551e88aa..6d1ead84 100644
--- a/models/domainlayer/code/commit.go
+++ b/models/domainlayer/code/commit.go
@@ -50,8 +50,19 @@ type CommitFile struct {
FilePath string `gorm:"primaryKey;type:varchar(255)"`
Additions int
Deletions int
+ Component string `gorm:"type:varchar(255)"`
}
func (CommitFile) TableName() string {
return "commit_files"
}
+
+type FileComponent struct {
+ RepoId string `gorm:"primaryKey;type:varchar(255)"`
+ Component string `gorm:"primaryKey;type:varchar(255)"`
+ PathRegex string `gorm:"type:varchar(255)"`
+}
+
+func (FileComponent) TableName() string {
+ return "file_component"
+}
diff --git a/models/migrationscripts/register.go b/models/migrationscripts/register.go
index 5294da8f..2f5351b3 100644
--- a/models/migrationscripts/register.go
+++ b/models/migrationscripts/register.go
@@ -22,6 +22,7 @@ import "github.com/apache/incubator-devlake/migration"
// All return all the migration scripts of framework
func All() []migration.Script {
return []migration.Script{
+
new(addFrameTables),
new(renameStepToStage),
new(addSubtasksField),
@@ -29,5 +30,6 @@ func All() []migration.Script {
new(renameTasksToPlan),
new(addDomainTables),
new(addTypeField),
+ new(updateSchemas20220711),
}
}
diff --git a/models/domainlayer/code/commit.go b/models/migrationscripts/updateSchemas20220711.go
similarity index 52%
copy from models/domainlayer/code/commit.go
copy to models/migrationscripts/updateSchemas20220711.go
index 551e88aa..0268f5d0 100644
--- a/models/domainlayer/code/commit.go
+++ b/models/migrationscripts/updateSchemas20220711.go
@@ -15,33 +15,31 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
-package code
+package migrationscripts
import (
- "time"
-
+ "context"
"github.com/apache/incubator-devlake/models/common"
+ "gorm.io/gorm"
)
-type Commit struct {
- common.NoPKModel
- Sha string `json:"sha" gorm:"primaryKey;type:varchar(40);comment:commit hash"`
- Additions int `json:"additions" gorm:"comment:Added lines of code"`
- Deletions int `json:"deletions" gorm:"comment:Deleted lines of code"`
- DevEq int `json:"deveq" gorm:"comment:Merico developer equivalent from analysis engine"`
- Message string
- AuthorName string `gorm:"type:varchar(255)"`
- AuthorEmail string `gorm:"type:varchar(255)"`
- AuthoredDate time.Time
- AuthorId string `gorm:"type:varchar(255)"`
- CommitterName string `gorm:"type:varchar(255)"`
- CommitterEmail string `gorm:"type:varchar(255)"`
- CommittedDate time.Time
- CommitterId string `gorm:"index;type:varchar(255)"`
+//type CodeComponent20220711 struct {
+// ComponentId string `gorm:"primaryKey;type:varchar(255)"`
+// PathRegex string `gorm:"type:varchar(255)"`
+//}
+//
+//func (CodeComponent20220711) TableName() string {
+// return "code_component_20220711"
+//}
+
+type FileComponent struct {
+ RepoId string `gorm:"primaryKey;type:varchar(255)"`
+ Component string `gorm:"primaryKey;type:varchar(255)"`
+ PathRegex string `gorm:"type:varchar(255)"`
}
-func (Commit) TableName() string {
- return "commits"
+func (FileComponent) TableName() string {
+ return "file_component"
}
type CommitFile struct {
@@ -50,8 +48,29 @@ type CommitFile struct {
FilePath string `gorm:"primaryKey;type:varchar(255)"`
Additions int
Deletions int
+ Component string `gorm:"type:varchar(255)"`
}
func (CommitFile) TableName() string {
return "commit_files"
}
+
+type updateSchemas20220711 struct{}
+
+func (*updateSchemas20220711) Up(ctx context.Context, db *gorm.DB) error {
+
+ err := db.Migrator().AutoMigrate(FileComponent{}, CommitFile{})
+ if err != nil {
+ return err
+ }
+ return nil
+
+}
+
+func (*updateSchemas20220711) Version() uint64 {
+ return 20220711122512
+}
+
+func (*updateSchemas20220711) Name() string {
+ return "file_component table"
+}
diff --git a/plugins/gitextractor/parser/repo.go b/plugins/gitextractor/parser/repo.go
index 4d812e8f..84afbf3a 100644
--- a/plugins/gitextractor/parser/repo.go
+++ b/plugins/gitextractor/parser/repo.go
@@ -23,8 +23,10 @@ import (
"github.com/apache/incubator-devlake/models/domainlayer"
"github.com/apache/incubator-devlake/models/domainlayer/code"
"github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/core/dal"
"github.com/apache/incubator-devlake/plugins/gitextractor/models"
git "github.com/libgit2/git2go/v33"
+ "regexp"
)
type GitRepo struct {
@@ -184,13 +186,25 @@ func (r *GitRepo) CollectBranches(subtaskCtx core.SubTaskContext) error {
func (r *GitRepo) CollectCommits(subtaskCtx core.SubTaskContext) error {
opts, err := getDiffOpts()
+
if err != nil {
return err
}
+ db := subtaskCtx.GetDal()
+ components := make([]code.FileComponent, 0)
+ err = db.All(&components, dal.From(components), dal.Where("repo_id= ?", r.id))
+ if err != nil {
+ return err
+ }
+ componentMap := make(map[string]*regexp.Regexp)
+ for _, component := range components {
+ componentMap[component.Component] = regexp.MustCompile(component.PathRegex)
+ }
odb, err := r.repo.Odb()
if err != nil {
return err
}
+
return odb.ForEach(func(id *git.Oid) error {
select {
case <-subtaskCtx.GetContext().Done():
@@ -228,7 +242,7 @@ func (r *GitRepo) CollectCommits(subtaskCtx core.SubTaskContext) error {
parent := commit.Parent(0)
if parent != nil {
var stats *git.DiffStats
- if stats, err = r.getDiffComparedToParent(c.Sha, commit, parent, opts); err != nil {
+ if stats, err = r.getDiffComparedToParent(c.Sha, commit, parent, opts, componentMap); err != nil {
return err
}
c.Additions += stats.Insertions()
@@ -268,7 +282,7 @@ func (r *GitRepo) storeParentCommits(commitSha string, commit *git.Commit) error
return r.store.CommitParents(commitParents)
}
-func (r *GitRepo) getDiffComparedToParent(commitSha string, commit *git.Commit, parent *git.Commit, opts *git.DiffOptions) (*git.DiffStats, error) {
+func (r *GitRepo) getDiffComparedToParent(commitSha string, commit *git.Commit, parent *git.Commit, opts *git.DiffOptions, componentMap map[string]*regexp.Regexp) (*git.DiffStats, error) {
var err error
var parentTree, tree *git.Tree
parentTree, err = parent.Tree()
@@ -284,7 +298,7 @@ func (r *GitRepo) getDiffComparedToParent(commitSha string, commit *git.Commit,
if err != nil {
return nil, err
}
- err = r.storeCommitFilesFromDiff(commitSha, diff)
+ err = r.storeCommitFilesFromDiff(commitSha, diff, componentMap)
if err != nil {
return nil, err
}
@@ -296,7 +310,7 @@ func (r *GitRepo) getDiffComparedToParent(commitSha string, commit *git.Commit,
return stats, nil
}
-func (r *GitRepo) storeCommitFilesFromDiff(commitSha string, diff *git.Diff) error {
+func (r *GitRepo) storeCommitFilesFromDiff(commitSha string, diff *git.Diff, componentMap map[string]*regexp.Regexp) error {
var commitFile *code.CommitFile
var err error
err = diff.ForEach(func(file git.DiffDelta, progress float64) (
@@ -311,6 +325,15 @@ func (r *GitRepo) storeCommitFilesFromDiff(commitSha string, diff *git.Diff) err
commitFile = new(code.CommitFile)
commitFile.CommitSha = commitSha
commitFile.FilePath = file.NewFile.Path
+ for component, reg := range componentMap {
+ if reg.MatchString(commitFile.FilePath) {
+ commitFile.Component = component
+ break
+ }
+ }
+ if commitFile.Component == "" {
+ commitFile.Component = "Default"
+ }
return func(hunk git.DiffHunk) (git.DiffForEachLineCallback, error) {
return func(line git.DiffLine) error {
if line.Origin == git.DiffLineAddition {
[incubator-devlake] 05/13: fix: change table name filecomponent->component; commitfilecomponent->filecomponent
Posted by ab...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
abeizn pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit 5ccbc1e3826b6f159b64326182e3104c2abfdc58
Author: xgdyp <zx...@163.com>
AuthorDate: Fri Jul 15 14:24:30 2022 +0800
fix: change table name filecomponent->component; commitfilecomponent->filecomponent
---
models/domainlayer/code/commit.go | 12 ++++++------
models/migrationscripts/updateSchemas20220711.go | 16 ++++++++--------
plugins/gitextractor/models/interface.go | 2 +-
plugins/gitextractor/parser/repo.go | 10 +++++-----
plugins/gitextractor/store/csv.go | 2 +-
plugins/gitextractor/store/database.go | 2 +-
6 files changed, 22 insertions(+), 22 deletions(-)
diff --git a/models/domainlayer/code/commit.go b/models/domainlayer/code/commit.go
index b48ef489..77065b1e 100644
--- a/models/domainlayer/code/commit.go
+++ b/models/domainlayer/code/commit.go
@@ -57,23 +57,23 @@ func (CommitFile) TableName() string {
return "commit_files"
}
-type FileComponent struct {
+type Component struct {
RepoId string `gorm:"primaryKey;type:varchar(255)"`
Component string `gorm:"primaryKey;type:varchar(255)"`
PathRegex string `gorm:"type:varchar(255)"`
}
-func (FileComponent) TableName() string {
- return "file_component"
+func (Component) TableName() string {
+ return "component"
}
-type CommitfileComponent struct {
+type FileComponent struct {
common.NoPKModel
CommitFileID string `gorm:"primaryKey;type:varchar(255)"`
RepoId string `gorm:"primaryKey;type:varchar(255)"`
Component string `gorm:"type:varchar(255)"`
}
-func (CommitfileComponent) TableName() string {
- return "commitfile_component"
+func (FileComponent) TableName() string {
+ return "file_component"
}
diff --git a/models/migrationscripts/updateSchemas20220711.go b/models/migrationscripts/updateSchemas20220711.go
index ffbbe3d5..4aa8c64b 100644
--- a/models/migrationscripts/updateSchemas20220711.go
+++ b/models/migrationscripts/updateSchemas20220711.go
@@ -32,14 +32,14 @@ import (
// return "code_component_20220711"
//}
-type FileComponent struct {
+type Component struct {
RepoId string `gorm:"primaryKey;type:varchar(255)"`
Component string `gorm:"primaryKey;type:varchar(255)"`
PathRegex string `gorm:"type:varchar(255)"`
}
-func (FileComponent) TableName() string {
- return "file_component"
+func (Component) TableName() string {
+ return "component"
}
type CommitFile struct {
@@ -56,22 +56,22 @@ func (CommitFile) TableName() string {
return "commit_files"
}
-type CommitfileComponent struct {
+type FileComponent struct {
common.NoPKModel
CommitFileID string `gorm:"primaryKey;type:varchar(255)"`
RepoId string `gorm:"primaryKey;type:varchar(255)"`
Component string `gorm:"type:varchar(255)"`
}
-func (CommitfileComponent) TableName() string {
- return "commitfile_component"
+func (FileComponent) TableName() string {
+ return "file_component"
}
type updateSchemas20220711 struct{}
func (*updateSchemas20220711) Up(ctx context.Context, db *gorm.DB) error {
- err := db.Migrator().AutoMigrate(FileComponent{}, CommitFile{}, CommitfileComponent{})
+ err := db.Migrator().AutoMigrate(Component{}, CommitFile{}, FileComponent{})
if err != nil {
return err
}
@@ -80,7 +80,7 @@ func (*updateSchemas20220711) Up(ctx context.Context, db *gorm.DB) error {
}
func (*updateSchemas20220711) Version() uint64 {
- return 202207151400
+ return 202207151420
}
func (*updateSchemas20220711) Name() string {
diff --git a/plugins/gitextractor/models/interface.go b/plugins/gitextractor/models/interface.go
index 86d73bab..a5ae3a86 100644
--- a/plugins/gitextractor/models/interface.go
+++ b/plugins/gitextractor/models/interface.go
@@ -27,6 +27,6 @@ type Store interface {
Refs(ref *code.Ref) error
CommitFiles(file *code.CommitFile) error
CommitParents(pp []*code.CommitParent) error
- CommitfileComponent(component *code.CommitfileComponent) error
+ FileComponent(component *code.FileComponent) error
Close() error
}
diff --git a/plugins/gitextractor/parser/repo.go b/plugins/gitextractor/parser/repo.go
index 8656a5e1..a796ca23 100644
--- a/plugins/gitextractor/parser/repo.go
+++ b/plugins/gitextractor/parser/repo.go
@@ -191,7 +191,7 @@ func (r *GitRepo) CollectCommits(subtaskCtx core.SubTaskContext) error {
return err
}
db := subtaskCtx.GetDal()
- components := make([]code.FileComponent, 0)
+ components := make([]code.Component, 0)
err = db.All(&components, dal.From(components), dal.Where("repo_id= ?", r.id))
if err != nil {
return err
@@ -312,7 +312,7 @@ func (r *GitRepo) getDiffComparedToParent(commitSha string, commit *git.Commit,
func (r *GitRepo) storeCommitFilesFromDiff(commitSha string, diff *git.Diff, componentMap map[string]*regexp.Regexp) error {
var commitFile *code.CommitFile
- var commitfileComponent *code.CommitfileComponent
+ var commitfileComponent *code.FileComponent
var err error
err = diff.ForEach(func(file git.DiffDelta, progress float64) (
git.DiffForEachHunkCallback, error) {
@@ -328,7 +328,7 @@ func (r *GitRepo) storeCommitFilesFromDiff(commitSha string, diff *git.Diff, com
commitFile.CommitSha = commitSha
commitFile.FilePath = file.NewFile.Path
commitFile.CommitFileID = commitSha + ":" + file.NewFile.Path
- commitfileComponent = new(code.CommitfileComponent)
+ commitfileComponent = new(code.FileComponent)
for component, reg := range componentMap {
if reg.MatchString(commitFile.FilePath) {
commitfileComponent.Component = component
@@ -355,9 +355,9 @@ func (r *GitRepo) storeCommitFilesFromDiff(commitSha string, diff *git.Diff, com
}, nil
}, git.DiffDetailLines)
if commitfileComponent != nil {
- err = r.store.CommitfileComponent(commitfileComponent)
+ err = r.store.FileComponent(commitfileComponent)
if err != nil {
- r.logger.Error("CommitfileComponent error:", err)
+ r.logger.Error("FileComponent error:", err)
}
}
if commitFile != nil {
diff --git a/plugins/gitextractor/store/csv.go b/plugins/gitextractor/store/csv.go
index 8a389116..bdf4205c 100644
--- a/plugins/gitextractor/store/csv.go
+++ b/plugins/gitextractor/store/csv.go
@@ -132,7 +132,7 @@ func (c *CsvStore) Refs(ref *code.Ref) error {
func (c *CsvStore) CommitFiles(file *code.CommitFile) error {
return c.commitFileWriter.Write(file)
}
-func (c *CsvStore) CommitfileComponent(component *code.CommitfileComponent) error {
+func (c *CsvStore) FileComponent(component *code.FileComponent) error {
return c.commitFileWriter.Write(component)
}
diff --git a/plugins/gitextractor/store/database.go b/plugins/gitextractor/store/database.go
index 2821b0f5..564433d5 100644
--- a/plugins/gitextractor/store/database.go
+++ b/plugins/gitextractor/store/database.go
@@ -91,7 +91,7 @@ func (d *Database) CommitFiles(file *code.CommitFile) error {
return batch.Add(file)
}
-func (d *Database) CommitfileComponent(commitfile *code.CommitfileComponent) error {
+func (d *Database) FileComponent(commitfile *code.FileComponent) error {
batch, err := d.driver.ForType(reflect.TypeOf(commitfile))
if err != nil {
return err
[incubator-devlake] 02/13: fix: add commitfileComponent table
Posted by ab...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
abeizn pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit 128279f13c02b60aeafce5b92158cf2f77286ea8
Author: xgdyp <zx...@163.com>
AuthorDate: Fri Jul 15 09:25:29 2022 +0800
fix: add commitfileComponent table
---
models/domainlayer/code/commit.go | 12 ++++++++++++
models/migrationscripts/updateSchemas20220711.go | 16 ++++++++++++++--
plugins/gitextractor/models/interface.go | 1 +
plugins/gitextractor/parser/repo.go | 18 +++++++++++++++---
plugins/gitextractor/store/csv.go | 3 +++
plugins/gitextractor/store/database.go | 8 ++++++++
6 files changed, 53 insertions(+), 5 deletions(-)
diff --git a/models/domainlayer/code/commit.go b/models/domainlayer/code/commit.go
index 6d1ead84..1d777624 100644
--- a/models/domainlayer/code/commit.go
+++ b/models/domainlayer/code/commit.go
@@ -66,3 +66,15 @@ type FileComponent struct {
func (FileComponent) TableName() string {
return "file_component"
}
+
+type CommitfileComponent struct {
+ common.NoPKModel
+ RepoId string `gorm:"primaryKey;type:varchar(255)"`
+ Component string `gorm:"type:varchar(255)"`
+ CommitSha string `gorm:"primaryKey;type:varchar(40)"`
+ FilePath string `gorm:"primaryKey;type:varchar(255)"`
+}
+
+func (CommitfileComponent) TableName() string {
+ return "commitfile_component"
+}
diff --git a/models/migrationscripts/updateSchemas20220711.go b/models/migrationscripts/updateSchemas20220711.go
index 0268f5d0..7df8df0e 100644
--- a/models/migrationscripts/updateSchemas20220711.go
+++ b/models/migrationscripts/updateSchemas20220711.go
@@ -55,11 +55,23 @@ func (CommitFile) TableName() string {
return "commit_files"
}
+type CommitfileComponent struct {
+ common.NoPKModel
+ RepoId string `gorm:"primaryKey;type:varchar(255)"`
+ Component string `gorm:"type:varchar(255)"`
+ CommitSha string `gorm:"primaryKey;type:varchar(40)"`
+ FilePath string `gorm:"primaryKey;type:varchar(255)"`
+}
+
+func (CommitfileComponent) TableName() string {
+ return "commitfile_component"
+}
+
type updateSchemas20220711 struct{}
func (*updateSchemas20220711) Up(ctx context.Context, db *gorm.DB) error {
- err := db.Migrator().AutoMigrate(FileComponent{}, CommitFile{})
+ err := db.Migrator().AutoMigrate(FileComponent{}, CommitFile{}, CommitfileComponent{})
if err != nil {
return err
}
@@ -68,7 +80,7 @@ func (*updateSchemas20220711) Up(ctx context.Context, db *gorm.DB) error {
}
func (*updateSchemas20220711) Version() uint64 {
- return 20220711122512
+ return 20220711122544
}
func (*updateSchemas20220711) Name() string {
diff --git a/plugins/gitextractor/models/interface.go b/plugins/gitextractor/models/interface.go
index 4ed54d7e..86d73bab 100644
--- a/plugins/gitextractor/models/interface.go
+++ b/plugins/gitextractor/models/interface.go
@@ -27,5 +27,6 @@ type Store interface {
Refs(ref *code.Ref) error
CommitFiles(file *code.CommitFile) error
CommitParents(pp []*code.CommitParent) error
+ CommitfileComponent(component *code.CommitfileComponent) error
Close() error
}
diff --git a/plugins/gitextractor/parser/repo.go b/plugins/gitextractor/parser/repo.go
index 84afbf3a..663bbfc5 100644
--- a/plugins/gitextractor/parser/repo.go
+++ b/plugins/gitextractor/parser/repo.go
@@ -312,6 +312,7 @@ func (r *GitRepo) getDiffComparedToParent(commitSha string, commit *git.Commit,
func (r *GitRepo) storeCommitFilesFromDiff(commitSha string, diff *git.Diff, componentMap map[string]*regexp.Regexp) error {
var commitFile *code.CommitFile
+ var commitfileComponent *code.CommitfileComponent
var err error
err = diff.ForEach(func(file git.DiffDelta, progress float64) (
git.DiffForEachHunkCallback, error) {
@@ -322,17 +323,22 @@ func (r *GitRepo) storeCommitFilesFromDiff(commitSha string, diff *git.Diff, com
return nil, err
}
}
+
commitFile = new(code.CommitFile)
commitFile.CommitSha = commitSha
commitFile.FilePath = file.NewFile.Path
+ commitfileComponent = new(code.CommitfileComponent)
for component, reg := range componentMap {
if reg.MatchString(commitFile.FilePath) {
- commitFile.Component = component
+ commitfileComponent.Component = component
break
}
}
- if commitFile.Component == "" {
- commitFile.Component = "Default"
+ commitfileComponent.RepoId = r.id
+ commitfileComponent.FilePath = file.NewFile.Path
+ commitfileComponent.CommitSha = commitSha
+ if commitfileComponent.Component == "" {
+ commitfileComponent.Component = "Default"
}
return func(hunk git.DiffHunk) (git.DiffForEachLineCallback, error) {
return func(line git.DiffLine) error {
@@ -346,6 +352,12 @@ func (r *GitRepo) storeCommitFilesFromDiff(commitSha string, diff *git.Diff, com
}, nil
}, nil
}, git.DiffDetailLines)
+ if commitfileComponent != nil {
+ err = r.store.CommitfileComponent(commitfileComponent)
+ if err != nil {
+ r.logger.Error("CommitfileComponent error:", err)
+ }
+ }
if commitFile != nil {
err = r.store.CommitFiles(commitFile)
if err != nil {
diff --git a/plugins/gitextractor/store/csv.go b/plugins/gitextractor/store/csv.go
index 2eb8edea..8a389116 100644
--- a/plugins/gitextractor/store/csv.go
+++ b/plugins/gitextractor/store/csv.go
@@ -132,6 +132,9 @@ func (c *CsvStore) Refs(ref *code.Ref) error {
func (c *CsvStore) CommitFiles(file *code.CommitFile) error {
return c.commitFileWriter.Write(file)
}
+func (c *CsvStore) CommitfileComponent(component *code.CommitfileComponent) error {
+ return c.commitFileWriter.Write(component)
+}
func (c *CsvStore) CommitParents(pp []*code.CommitParent) error {
var err error
diff --git a/plugins/gitextractor/store/database.go b/plugins/gitextractor/store/database.go
index becb5264..2821b0f5 100644
--- a/plugins/gitextractor/store/database.go
+++ b/plugins/gitextractor/store/database.go
@@ -91,6 +91,14 @@ func (d *Database) CommitFiles(file *code.CommitFile) error {
return batch.Add(file)
}
+func (d *Database) CommitfileComponent(commitfile *code.CommitfileComponent) error {
+ batch, err := d.driver.ForType(reflect.TypeOf(commitfile))
+ if err != nil {
+ return err
+ }
+ return batch.Add(commitfile)
+}
+
func (d *Database) CommitParents(pp []*code.CommitParent) error {
if len(pp) == 0 {
return nil
[incubator-devlake] 12/13: fix: update csv.go format
Posted by ab...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
abeizn pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit 6f47a5240dd99b2bbae3b604372768e25fa4753e
Author: xgdyp <zx...@163.com>
AuthorDate: Fri Jul 22 16:52:16 2022 +0800
fix: update csv.go format
---
plugins/gitextractor/store/csv.go | 1 +
1 file changed, 1 insertion(+)
diff --git a/plugins/gitextractor/store/csv.go b/plugins/gitextractor/store/csv.go
index 52820c7b..df46ee4e 100644
--- a/plugins/gitextractor/store/csv.go
+++ b/plugins/gitextractor/store/csv.go
@@ -132,6 +132,7 @@ func (c *CsvStore) Refs(ref *code.Ref) error {
func (c *CsvStore) CommitFiles(file *code.CommitFile) error {
return c.commitFileWriter.Write(file)
}
+
func (c *CsvStore) CommitFileComponents(commitFileComponent *code.CommitFileComponent) error {
return c.commitFileWriter.Write(commitFileComponent)
}
[incubator-devlake] 03/13: fix: update schema
Posted by ab...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
abeizn pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit 5cedd9e6fc847c0647af6c31831e1bf411758077
Author: xgdyp <zx...@163.com>
AuthorDate: Fri Jul 15 13:41:49 2022 +0800
fix: update schema
---
models/domainlayer/code/commit.go | 1 -
models/migrationscripts/updateSchemas20220711.go | 11 ++++++-----
2 files changed, 6 insertions(+), 6 deletions(-)
diff --git a/models/domainlayer/code/commit.go b/models/domainlayer/code/commit.go
index 1d777624..9b20cd3b 100644
--- a/models/domainlayer/code/commit.go
+++ b/models/domainlayer/code/commit.go
@@ -50,7 +50,6 @@ type CommitFile struct {
FilePath string `gorm:"primaryKey;type:varchar(255)"`
Additions int
Deletions int
- Component string `gorm:"type:varchar(255)"`
}
func (CommitFile) TableName() string {
diff --git a/models/migrationscripts/updateSchemas20220711.go b/models/migrationscripts/updateSchemas20220711.go
index 7df8df0e..f2c3f499 100644
--- a/models/migrationscripts/updateSchemas20220711.go
+++ b/models/migrationscripts/updateSchemas20220711.go
@@ -44,11 +44,12 @@ func (FileComponent) TableName() string {
type CommitFile struct {
common.NoPKModel
- CommitSha string `gorm:"primaryKey;type:varchar(40)"`
- FilePath string `gorm:"primaryKey;type:varchar(255)"`
- Additions int
- Deletions int
- Component string `gorm:"type:varchar(255)"`
+ CommitFileID string `gorm:"primaryKey;"`
+ CommitSha string `gorm:"primaryKey;type:varchar(40)"`
+ FilePath string `gorm:"primaryKey;type:varchar(255)"`
+ Additions int
+ Deletions int
+ Component string `gorm:"type:varchar(255)"`
}
func (CommitFile) TableName() string {
[incubator-devlake] 09/13: fix: change component model to a new file and rename some file
Posted by ab...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
abeizn pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit 0cd5024706849dfc5eaf489761550bd0242b8762
Author: xgdyp <zx...@163.com>
AuthorDate: Thu Jul 21 15:09:50 2022 +0800
fix: change component model to a new file and rename some file
---
models/domainlayer/code/commit.go | 10 --------
models/domainlayer/code/component.go | 28 ++++++++++++++++++++++
...mponent.go => 20220721_commitfile_component.go} | 2 +-
3 files changed, 29 insertions(+), 11 deletions(-)
diff --git a/models/domainlayer/code/commit.go b/models/domainlayer/code/commit.go
index fd5707a1..73145539 100644
--- a/models/domainlayer/code/commit.go
+++ b/models/domainlayer/code/commit.go
@@ -57,16 +57,6 @@ func (CommitFile) TableName() string {
return "commit_files"
}
-type Component struct {
- RepoId string `gorm:"type:varchar(255)"`
- Name string `gorm:"primaryKey;type:varchar(255)"`
- PathRegex string `gorm:"type:varchar(255)"`
-}
-
-func (Component) TableName() string {
- return "components"
-}
-
type CommitFileComponent struct {
common.NoPKModel
CommitFileId string `gorm:"primaryKey;type:varchar(255)"`
diff --git a/models/domainlayer/code/component.go b/models/domainlayer/code/component.go
new file mode 100644
index 00000000..515f422a
--- /dev/null
+++ b/models/domainlayer/code/component.go
@@ -0,0 +1,28 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package code
+
+type Component struct {
+ RepoId string `gorm:"type:varchar(255)"`
+ Name string `gorm:"primaryKey;type:varchar(255)"`
+ PathRegex string `gorm:"type:varchar(255)"`
+}
+
+func (Component) TableName() string {
+ return "components"
+}
diff --git a/models/migrationscripts/commitfile_component.go b/models/migrationscripts/20220721_commitfile_component.go
similarity index 98%
rename from models/migrationscripts/commitfile_component.go
rename to models/migrationscripts/20220721_commitfile_component.go
index 9f1ee69c..dc3ff8e6 100644
--- a/models/migrationscripts/commitfile_component.go
+++ b/models/migrationscripts/20220721_commitfile_component.go
@@ -69,7 +69,7 @@ func (*commitfileComponent) Up(ctx context.Context, db *gorm.DB) error {
}
func (*commitfileComponent) Version() uint64 {
- return 202207201009
+ return 202207211505
}
func (*commitfileComponent) Name() string {
[incubator-devlake] 07/13: fix: update db schema
Posted by ab...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
abeizn pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit af864443f3619d30e4c2b5ac5f7bd1eecbeb0471
Author: xgdyp <zx...@163.com>
AuthorDate: Wed Jul 20 12:19:21 2022 +0800
fix: update db schema
---
.../{updateSchemas20220711.go => commitfile_component.go} | 0
1 file changed, 0 insertions(+), 0 deletions(-)
diff --git a/models/migrationscripts/updateSchemas20220711.go b/models/migrationscripts/commitfile_component.go
similarity index 100%
rename from models/migrationscripts/updateSchemas20220711.go
rename to models/migrationscripts/commitfile_component.go
[incubator-devlake] 04/13: fix: update migrationscript
Posted by ab...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
abeizn pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit 172b2183a202eb4ef253a469f2191f40d07e1289
Author: xgdyp <zx...@163.com>
AuthorDate: Tue Jul 19 16:46:54 2022 +0800
fix: update migrationscript
---
models/domainlayer/code/commit.go | 16 ++++++++--------
models/migrationscripts/updateSchemas20220711.go | 15 +++++++--------
plugins/gitextractor/parser/repo.go | 6 ++++--
3 files changed, 19 insertions(+), 18 deletions(-)
diff --git a/models/domainlayer/code/commit.go b/models/domainlayer/code/commit.go
index 9b20cd3b..b48ef489 100644
--- a/models/domainlayer/code/commit.go
+++ b/models/domainlayer/code/commit.go
@@ -46,10 +46,11 @@ func (Commit) TableName() string {
type CommitFile struct {
common.NoPKModel
- CommitSha string `gorm:"primaryKey;type:varchar(40)"`
- FilePath string `gorm:"primaryKey;type:varchar(255)"`
- Additions int
- Deletions int
+ CommitFileID string `gorm:"primaryKey;type:varchar(255)"`
+ CommitSha string `gorm:"type:varchar(40)"`
+ FilePath string `gorm:"type:varchar(255)"`
+ Additions int
+ Deletions int
}
func (CommitFile) TableName() string {
@@ -68,10 +69,9 @@ func (FileComponent) TableName() string {
type CommitfileComponent struct {
common.NoPKModel
- RepoId string `gorm:"primaryKey;type:varchar(255)"`
- Component string `gorm:"type:varchar(255)"`
- CommitSha string `gorm:"primaryKey;type:varchar(40)"`
- FilePath string `gorm:"primaryKey;type:varchar(255)"`
+ CommitFileID string `gorm:"primaryKey;type:varchar(255)"`
+ RepoId string `gorm:"primaryKey;type:varchar(255)"`
+ Component string `gorm:"type:varchar(255)"`
}
func (CommitfileComponent) TableName() string {
diff --git a/models/migrationscripts/updateSchemas20220711.go b/models/migrationscripts/updateSchemas20220711.go
index f2c3f499..ffbbe3d5 100644
--- a/models/migrationscripts/updateSchemas20220711.go
+++ b/models/migrationscripts/updateSchemas20220711.go
@@ -44,9 +44,9 @@ func (FileComponent) TableName() string {
type CommitFile struct {
common.NoPKModel
- CommitFileID string `gorm:"primaryKey;"`
- CommitSha string `gorm:"primaryKey;type:varchar(40)"`
- FilePath string `gorm:"primaryKey;type:varchar(255)"`
+ CommitFileID string `gorm:"primaryKey;type:varchar(255)"`
+ CommitSha string `gorm:"type:varchar(40)"`
+ FilePath string `gorm:"type:varchar(255)"`
Additions int
Deletions int
Component string `gorm:"type:varchar(255)"`
@@ -58,10 +58,9 @@ func (CommitFile) TableName() string {
type CommitfileComponent struct {
common.NoPKModel
- RepoId string `gorm:"primaryKey;type:varchar(255)"`
- Component string `gorm:"type:varchar(255)"`
- CommitSha string `gorm:"primaryKey;type:varchar(40)"`
- FilePath string `gorm:"primaryKey;type:varchar(255)"`
+ CommitFileID string `gorm:"primaryKey;type:varchar(255)"`
+ RepoId string `gorm:"primaryKey;type:varchar(255)"`
+ Component string `gorm:"type:varchar(255)"`
}
func (CommitfileComponent) TableName() string {
@@ -81,7 +80,7 @@ func (*updateSchemas20220711) Up(ctx context.Context, db *gorm.DB) error {
}
func (*updateSchemas20220711) Version() uint64 {
- return 20220711122544
+ return 202207151400
}
func (*updateSchemas20220711) Name() string {
diff --git a/plugins/gitextractor/parser/repo.go b/plugins/gitextractor/parser/repo.go
index 663bbfc5..8656a5e1 100644
--- a/plugins/gitextractor/parser/repo.go
+++ b/plugins/gitextractor/parser/repo.go
@@ -327,6 +327,7 @@ func (r *GitRepo) storeCommitFilesFromDiff(commitSha string, diff *git.Diff, com
commitFile = new(code.CommitFile)
commitFile.CommitSha = commitSha
commitFile.FilePath = file.NewFile.Path
+ commitFile.CommitFileID = commitSha + ":" + file.NewFile.Path
commitfileComponent = new(code.CommitfileComponent)
for component, reg := range componentMap {
if reg.MatchString(commitFile.FilePath) {
@@ -335,8 +336,9 @@ func (r *GitRepo) storeCommitFilesFromDiff(commitSha string, diff *git.Diff, com
}
}
commitfileComponent.RepoId = r.id
- commitfileComponent.FilePath = file.NewFile.Path
- commitfileComponent.CommitSha = commitSha
+ commitfileComponent.CommitFileID = commitSha + ":" + file.NewFile.Path
+ //commitfileComponent.FilePath = file.NewFile.Path
+ //commitfileComponent.CommitSha = commitSha
if commitfileComponent.Component == "" {
commitfileComponent.Component = "Default"
}
[incubator-devlake] 11/13: fix: drop commitfile table before automigrate table
Posted by ab...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
abeizn pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit 0a530044dc8def0b864c6ff930459b42f7a04706
Author: xgdyp <zx...@163.com>
AuthorDate: Fri Jul 22 16:47:29 2022 +0800
fix: drop commitfile table before automigrate table
---
models/migrationscripts/20220722_commitfile_component.go | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
diff --git a/models/migrationscripts/20220722_commitfile_component.go b/models/migrationscripts/20220722_commitfile_component.go
index e4a9c17e..88a346a8 100644
--- a/models/migrationscripts/20220722_commitfile_component.go
+++ b/models/migrationscripts/20220722_commitfile_component.go
@@ -59,8 +59,11 @@ func (CommitFileComponent) TableName() string {
type commitfileComponent struct{}
func (*commitfileComponent) Up(ctx context.Context, db *gorm.DB) error {
-
- err := db.Migrator().AutoMigrate(Component{}, CommitFile{}, CommitFileComponent{})
+ err := db.Migrator().DropTable(&CommitFile{})
+ if err != nil {
+ return err
+ }
+ err = db.Migrator().AutoMigrate(Component{}, CommitFile{}, CommitFileComponent{})
if err != nil {
return err
}