You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by kl...@apache.org on 2024/11/05 06:13:44 UTC
(incubator-devlake) branch release-v1.0 updated: feat: gitlab extractors and convertors support incremental mode (#7997)
This is an automated email from the ASF dual-hosted git repository.
klesh pushed a commit to branch release-v1.0
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/release-v1.0 by this push:
new f9dfac1a5 feat: gitlab extractors and convertors support incremental mode (#7997)
f9dfac1a5 is described below
commit f9dfac1a5ff6833c0cab24a56c2d7c0133ab5423
Author: Klesh Wong <zh...@merico.dev>
AuthorDate: Fri Sep 27 09:00:15 2024 +0800
feat: gitlab extractors and convertors support incremental mode (#7997)
* feat: gitlab mr_extractor support incremental sync
* feat: incr mode transformation support for deployment/issue/job/mr and others
* feat: removed mr_enricher and all gitlab transformers support incr-mode
* fix: some gitlab subtasks are missing connection in left join clause
* fix: test cases failed due to gitlab transformers support incr-mode
* fix: linting
* fix: do not collect all accounts from jihulab.com
* fix: typo
* fix: gitlab mr comments won't be converted to domain layer till next run
* feat: improve gitlab MR comments/commits collection performance
* fix: gitlab issues/mrs child records are not deleted
* docs: update stateful extractor doc
* fix: gitlab mr detail test
* refactor: unify stateful extractor and convertor helper
* docs: update stateful extractor/convertor doc
* refactor: remove useless type hint
* fix: shoud not be deleting records not extracted by current extractor
* fix: jira issue extractor should not deleting sprint_issue
* refactor: remove commit related subtasks
* fix: remove commit conversion e2e test
* feat(gitlab): update ExtractApiChildPipelines (#8016)
* fix: retransform should run in fullsync mode
* fix: gitlab issue assignees are not being converted
---------
Co-authored-by: Lynwee <15...@qq.com>
---
.../pluginhelper/api/api_extractor_stateful.go | 74 +++++++--
backend/helpers/pluginhelper/api/api_rawdata.go | 2 +-
.../pluginhelper/api/data_convertor_stateful.go | 79 +++++++++-
.../helpers/pluginhelper/api/enrich_with_regex.go | 61 +++++++-
.../pluginhelper/api/subtask_state_manager.go | 2 +-
backend/impls/dalgorm/encdec_serializer.go | 5 +
backend/plugins/gitlab/e2e/mr_commits_test.go | 33 -----
backend/plugins/gitlab/e2e/mr_detail_test.go | 2 +
backend/plugins/gitlab/e2e/mr_enrich_test.go | 79 ----------
..._tool_gitlab_merge_requests_for_enrich_test.csv | 25 ----
...ol_gitlab_merge_requests_for_mr_commit_test.csv | 12 +-
...ool_gitlab_merge_requests_for_mr_notes_test.csv | 34 ++---
.../plugins/gitlab/e2e/snapshot_tables/commits.csv | 65 --------
.../gitlab/e2e/snapshot_tables/repo_commits.csv | 38 -----
.../20240904_remove_mr_review_fields.go | 40 +++++
backend/plugins/gitlab/models/mr.go | 4 +-
backend/plugins/gitlab/tasks/account_collector.go | 4 +-
.../gitlab/tasks/child_pipeline_extractor.go | 29 +---
backend/plugins/gitlab/tasks/commit_collector.go | 80 ----------
backend/plugins/gitlab/tasks/commit_convertor.go | 108 --------------
backend/plugins/gitlab/tasks/commit_extractor.go | 116 ---------------
.../plugins/gitlab/tasks/deployment_convertor.go | 40 +++--
.../plugins/gitlab/tasks/deployment_extractor.go | 16 +-
.../gitlab/tasks/issue_assignee_convertor.go | 49 +++---
backend/plugins/gitlab/tasks/issue_convertor.go | 55 ++++---
backend/plugins/gitlab/tasks/issue_extractor.go | 44 ++++--
.../plugins/gitlab/tasks/issue_label_convertor.go | 53 +++----
backend/plugins/gitlab/tasks/job_convertor.go | 51 +++----
backend/plugins/gitlab/tasks/job_extractor.go | 18 +--
.../plugins/gitlab/tasks/mr_assignee_convertor.go | 52 +++----
.../plugins/gitlab/tasks/mr_commit_convertor.go | 64 ++++----
.../plugins/gitlab/tasks/mr_commit_extractor.go | 56 +++++--
backend/plugins/gitlab/tasks/mr_convertor.go | 57 ++++---
.../plugins/gitlab/tasks/mr_detail_extractor.go | 25 ++--
backend/plugins/gitlab/tasks/mr_enricher.go | 165 ---------------------
backend/plugins/gitlab/tasks/mr_enricher_test.go | 112 --------------
backend/plugins/gitlab/tasks/mr_extractor.go | 78 +++++++---
backend/plugins/gitlab/tasks/mr_label_convertor.go | 53 +++----
...r_comment_convertor.go => mr_note_convertor.go} | 55 +++----
backend/plugins/gitlab/tasks/mr_note_extractor.go | 18 +--
.../plugins/gitlab/tasks/mr_reviewer_convertor.go | 53 +++----
.../gitlab/tasks/pipeline_commit_convertor.go | 45 +++---
.../gitlab/tasks/pipeline_detail_convertor.go | 45 +++---
.../gitlab/tasks/pipeline_detail_extractor.go | 18 +--
backend/plugins/gitlab/tasks/pipeline_extractor.go | 17 +--
backend/plugins/gitlab/tasks/shared.go | 24 ++-
.../plugins/gitlab/tasks/trigger_job_extractor.go | 17 +--
backend/plugins/jira/tasks/epic_extractor.go | 10 +-
.../jira/tasks/issue_changelog_convertor.go | 2 +-
.../jira/tasks/issue_changelog_extractor.go | 9 +-
backend/plugins/jira/tasks/issue_convertor.go | 13 +-
backend/plugins/jira/tasks/issue_extractor.go | 35 +++--
backend/server/api/blueprints/blueprints.go | 1 +
.../src/routes/blueprint/detail/status-panel.tsx | 2 +-
54 files changed, 843 insertions(+), 1401 deletions(-)
diff --git a/backend/helpers/pluginhelper/api/api_extractor_stateful.go b/backend/helpers/pluginhelper/api/api_extractor_stateful.go
index 66ad5281a..2c994afdb 100644
--- a/backend/helpers/pluginhelper/api/api_extractor_stateful.go
+++ b/backend/helpers/pluginhelper/api/api_extractor_stateful.go
@@ -18,6 +18,7 @@ limitations under the License.
package api
import (
+ "encoding/json"
"reflect"
"github.com/apache/incubator-devlake/core/dal"
@@ -27,30 +28,72 @@ import (
)
// StatefulApiExtractorArgs is a struct that contains the arguments for a stateful api extractor
-type StatefulApiExtractorArgs struct {
+type StatefulApiExtractorArgs[InputType any] struct {
*SubtaskCommonArgs
- Extract func(row *RawData) ([]any, errors.Error)
+ BeforeExtract func(issue *InputType, stateManager *SubtaskStateManager) errors.Error
+ Extract func(body *InputType, row *RawData) ([]any, errors.Error)
}
-type StatefulApiExtractor struct {
- *StatefulApiExtractorArgs
+// StatefulApiExtractor is a struct that manages the stateful API extraction process.
+// It facilitates extracting data from a single _raw_data table and saving it into multiple Tool Layer tables.
+// By default, the extractor operates in Incremental Mode, processing only new records added to the raw table since the previous run.
+// This approach reduces the amount of data to process, significantly decreasing the execution time.
+// The extractor automatically detects if the configuration has changed since the last run. If a change is detected,
+// it will automatically switch to Full-Sync mode.
+//
+// Example:
+//
+// extractor, err := api.NewStatefulApiExtractor(&api.StatefulApiExtractorArgs[apiv2models.Issue]{
+// SubtaskCommonArgs: &api.SubtaskCommonArgs{
+// SubTaskContext: subtaskCtx,
+// Table: RAW_ISSUE_TABLE,
+// Params: JiraApiParams{
+// ConnectionId: data.Options.ConnectionId,
+// BoardId: data.Options.BoardId,
+// },
+// SubtaskConfig: config, // The helper stores this configuration in the state and compares it with the previous one
+// // to determine the operating mode (Incremental/FullSync).
+// // Ensure that the configuration is serializable and contains only public fields.
+// // It is also recommended that the configuration includes only the necessary fields used by the extractor.
+// ..},
+// BeforeExtract: func(body *IssuesResponse, stateManager *api.SubtaskStateManager) errors.Error {
+// if stateManager.IsIncremental() {
+// // It is important to delete all existing child-records under DiffSync Mode
+// err := db.Delete(
+// &models.JiraIssueLabel{},
+// dal.Where("connection_id = ? AND issue_id = ?", data.Options.ConnectionId, body.Id),
+// )
+// }
+// return nil
+// },
+// Extract: func(apiIssue *apiv2models.Issue, row *api.RawData) ([]interface{}, errors.Error) {
+// },
+// })
+//
+// if err != nil {
+// return err
+// }
+//
+// return extractor.Execute()
+type StatefulApiExtractor[InputType any] struct {
+ *StatefulApiExtractorArgs[InputType]
*SubtaskStateManager
}
// NewStatefulApiExtractor creates a new StatefulApiExtractor
-func NewStatefulApiExtractor(args *StatefulApiExtractorArgs) (*StatefulApiExtractor, errors.Error) {
+func NewStatefulApiExtractor[InputType any](args *StatefulApiExtractorArgs[InputType]) (*StatefulApiExtractor[InputType], errors.Error) {
stateManager, err := NewSubtaskStateManager(args.SubtaskCommonArgs)
if err != nil {
return nil, err
}
- return &StatefulApiExtractor{
+ return &StatefulApiExtractor[InputType]{
StatefulApiExtractorArgs: args,
SubtaskStateManager: stateManager,
}, nil
}
// Execute sub-task
-func (extractor *StatefulApiExtractor) Execute() errors.Error {
+func (extractor *StatefulApiExtractor[InputType]) Execute() errors.Error {
// load data from database
db := extractor.GetDal()
logger := extractor.GetLogger()
@@ -103,7 +146,20 @@ func (extractor *StatefulApiExtractor) Execute() errors.Error {
return errors.Default.Wrap(err, "error fetching row")
}
- results, err := extractor.Extract(row)
+ body := new(InputType)
+ err = errors.Convert(json.Unmarshal(row.Data, body))
+ if err != nil {
+ return err
+ }
+
+ if extractor.BeforeExtract != nil {
+ err = extractor.BeforeExtract(body, extractor.SubtaskStateManager)
+ if err != nil {
+ return err
+ }
+ }
+
+ results, err := extractor.Extract(body, row)
if err != nil {
return errors.Default.Wrap(err, "error calling plugin Extract implementation")
}
@@ -137,4 +193,4 @@ func (extractor *StatefulApiExtractor) Execute() errors.Error {
return extractor.SubtaskStateManager.Close()
}
-var _ plugin.SubTask = (*StatefulApiExtractor)(nil)
+var _ plugin.SubTask = (*StatefulApiExtractor[any])(nil)
diff --git a/backend/helpers/pluginhelper/api/api_rawdata.go b/backend/helpers/pluginhelper/api/api_rawdata.go
index ef16dde6d..5e9bb7835 100644
--- a/backend/helpers/pluginhelper/api/api_rawdata.go
+++ b/backend/helpers/pluginhelper/api/api_rawdata.go
@@ -34,7 +34,7 @@ type RawData struct {
Data []byte
Url string
Input json.RawMessage `gorm:"type:json"`
- CreatedAt time.Time
+ CreatedAt time.Time `gorm:"index"`
}
type TaskOptions interface {
diff --git a/backend/helpers/pluginhelper/api/data_convertor_stateful.go b/backend/helpers/pluginhelper/api/data_convertor_stateful.go
index ddb710a2f..1c5292dc2 100644
--- a/backend/helpers/pluginhelper/api/data_convertor_stateful.go
+++ b/backend/helpers/pluginhelper/api/data_convertor_stateful.go
@@ -27,18 +27,82 @@ import (
type StatefulDataConverterArgs[InputType any] struct {
*SubtaskCommonArgs
- Input func(*SubtaskStateManager) (dal.Rows, errors.Error)
- Convert func(row *InputType) ([]any, errors.Error)
- BatchSize int
+ Input func(*SubtaskStateManager) (dal.Rows, errors.Error)
+ BeforeConvert func(issue *InputType, stateManager *SubtaskStateManager) errors.Error
+ Convert func(row *InputType) ([]any, errors.Error)
+ BatchSize int
}
+// StatefulDataConverter is a struct that manages the stateful data conversion process.
+// It facilitates converting data from a database cursor and saving it into arbitrary tables.
+// The converter determines the operating mode (Incremental/FullSync) based on the stored state and configuration.
+// It then calls the provided `Input` function to obtain the `dal.Rows` (the database cursor) and processes each
+// record individually through the `Convert` function, saving the results to the database.
+//
+// For Incremental mode to work properly, it is crucial to check `stateManager.IsIncremental()` and utilize
+// `stateManager.GetSince()` to build your query in the `Input` function, ensuring that only the necessary
+// records are fetched.
+//
+// The converter automatically detects if the configuration has changed since the last run. If a change is detected,
+// it will automatically switch to Full-Sync mode.
+//
+// Example:
+//
+// converter, err := api.NewStatefulDataConverter(&api.StatefulDataConverterArgs[models.JiraIssue]{
+// SubtaskCommonArgs: &api.SubtaskCommonArgs{
+// SubTaskContext: subtaskCtx,
+// Table: RAW_ISSUE_TABLE,
+// Params: JiraApiParams{
+// ConnectionId: data.Options.ConnectionId,
+// BoardId: data.Options.BoardId,
+// },
+// SubtaskConfig: mappings,
+// },
+// Input: func(stateManager *api.SubtaskStateManager) (dal.Rows, errors.Error) {
+// clauses := []dal.Clause{
+// dal.Select("_tool_jira_issues.*"),
+// dal.From("_tool_jira_issues"),
+// dal.Join(`left join _tool_jira_board_issues
+// on _tool_jira_board_issues.issue_id = _tool_jira_issues.issue_id
+// and _tool_jira_board_issues.connection_id = _tool_jira_issues.connection_id`),
+// dal.Where(
+// "_tool_jira_board_issues.connection_id = ? AND _tool_jira_board_issues.board_id = ?",
+// data.Options.ConnectionId,
+// data.Options.BoardId,
+// ),
+// }
+// if stateManager.IsIncremental() { // IMPORTANT: to filter records for Incremental Mode
+// since := stateManager.GetSince()
+// if since != nil {
+// clauses = append(clauses, dal.Where("_tool_jira_issues.updated_at >= ? ", since))
+// }
+// }
+// return db.Cursor(clauses...)
+// },
+// BeforeConvert: func(jiraIssue *models.GitlabMergeRequest, stateManager *api.SubtaskStateManager) errors.Error {
+// // It is important to delete all existing child-records under DiffSync Mode
+// issueId := issueIdGen.Generate(data.Options.ConnectionId, jiraIssue.IssueId)
+// if err := db.Delete(&ticket.IssueAssignee{}, dal.Where("issue_id = ?", issueId)); err != nil {
+// return err
+// }
+// ...
+// return nil
+// },
+// Convert: func(jiraIssue *models.JiraIssue) ([]interface{}, errors.Error) {
+// },
+// })
+
+// if err != nil {
+// return err
+// }
+
+// return converter.Execute()
type StatefulDataConverter[InputType any] struct {
*StatefulDataConverterArgs[InputType]
*SubtaskStateManager
}
func NewStatefulDataConverter[
- OptType any,
InputType any,
](
args *StatefulDataConverterArgs[InputType],
@@ -91,6 +155,13 @@ func (converter *StatefulDataConverter[InputType]) Execute() errors.Error {
return errors.Default.Wrap(err, "error fetching rows")
}
+ if converter.BeforeConvert != nil {
+ err = converter.BeforeConvert(inputRow, converter.SubtaskStateManager)
+ if err != nil {
+ return err
+ }
+ }
+
results, err := converter.Convert(inputRow)
if err != nil {
return errors.Default.Wrap(err, "error calling Converter plugin implementation")
diff --git a/backend/helpers/pluginhelper/api/enrich_with_regex.go b/backend/helpers/pluginhelper/api/enrich_with_regex.go
index 5e7bd2a3a..b31fce78b 100644
--- a/backend/helpers/pluginhelper/api/enrich_with_regex.go
+++ b/backend/helpers/pluginhelper/api/enrich_with_regex.go
@@ -28,12 +28,13 @@ import (
// TODO: remove Enricher from naming since it is more like a util function
type RegexEnricher struct {
// This field will store compiled regular expression for every pattern
- regexpMap map[string]*regexp.Regexp
+ regexpMap map[string]*regexp.Regexp
+ regexMapList map[string][]*regexp.Regexp
}
// NewRegexEnricher initialize a regexEnricher
func NewRegexEnricher() *RegexEnricher {
- return &RegexEnricher{regexpMap: make(map[string]*regexp.Regexp)}
+ return &RegexEnricher{regexpMap: make(map[string]*regexp.Regexp), regexMapList: make(map[string][]*regexp.Regexp)}
}
// AddRegexp will add compiled regular expression for pattern to regexpMap
@@ -105,3 +106,59 @@ func (r *RegexEnricher) ReturnNameIfOmittedOrMatched(name string, targets ...str
}
return r.ReturnNameIfMatched(name, targets...)
}
+
+func (r *RegexEnricher) PlainMap() map[string]string {
+ m := make(map[string]string)
+ for k, v := range r.regexpMap {
+ m[k] = v.String()
+ }
+ return m
+}
+
+// TryAdd a named regexp if given pattern is not empty
+func (r *RegexEnricher) TryAddList(name string, patterns ...string) errors.Error {
+ if _, ok := r.regexMapList[name]; ok {
+ return errors.Default.New(fmt.Sprintf("Regex pattern with name: %s already exists", name))
+ }
+ var regexList []*regexp.Regexp
+ for _, pattern := range patterns {
+ if pattern == "" {
+ continue
+ }
+ regex, err := errors.Convert01(regexp.Compile(pattern))
+ if err != nil {
+ return errors.BadInput.Wrap(err, fmt.Sprintf("Fail to compile pattern for regex pattern: %s", pattern))
+ }
+ regexList = append(regexList, regex)
+ }
+
+ // Only save non-empty regexList
+ if len(regexList) > 0 {
+ r.regexMapList[name] = regexList
+ }
+ return nil
+}
+
+// ReturnNameIfMatched will return name if any of the targets matches the regex associated with the given name
+func (r *RegexEnricher) ReturnNameIfMatchedList(name string, targets ...string) string {
+ if regexList, ok := r.regexMapList[name]; !ok {
+ return ""
+ } else {
+ for _, regex := range regexList {
+ for _, target := range targets {
+ if regex.MatchString(target) {
+ return name
+ }
+ }
+ }
+ return "" // If any regex fails to match, return ""
+ }
+}
+
+// ReturnNameIfOmittedOrMatched returns the given name if regex of the given name is omitted or fallback to ReturnNameIfMatched
+func (r *RegexEnricher) ReturnNameIfOmittedOrMatchedList(name string, targets ...string) string {
+ if _, ok := r.regexMapList[name]; !ok {
+ return name
+ }
+ return r.ReturnNameIfMatched(name, targets...)
+}
diff --git a/backend/helpers/pluginhelper/api/subtask_state_manager.go b/backend/helpers/pluginhelper/api/subtask_state_manager.go
index 407a291bf..7dd7d2370 100644
--- a/backend/helpers/pluginhelper/api/subtask_state_manager.go
+++ b/backend/helpers/pluginhelper/api/subtask_state_manager.go
@@ -34,7 +34,7 @@ type SubtaskCommonArgs struct {
plugin.SubTaskContext
Table string // raw table name
Params any // for filtering rows belonging to the scope (jira board, github repo) of the subtask
- SubtaskConfig any // for determining whether the subtask should run in incremental or full sync mode
+ SubtaskConfig any // for determining whether the subtask should run in Incremental or Full-Sync mode by comparing with the previous config to see if it changed
BatchSize int // batch size for saving data
}
diff --git a/backend/impls/dalgorm/encdec_serializer.go b/backend/impls/dalgorm/encdec_serializer.go
index bb6589195..fd7a9d880 100644
--- a/backend/impls/dalgorm/encdec_serializer.go
+++ b/backend/impls/dalgorm/encdec_serializer.go
@@ -91,6 +91,11 @@ func (es *EncDecSerializer) Value(ctx context.Context, field *schema.Field, dst
}
target = string(b)
}
+ if field.GORMDataType == "string" {
+ println("field.GORMDataType == string", field.Size)
+ gormTag, ok := field.Tag.Lookup("gorm")
+ println(ok, gormTag)
+ }
return plugin.Encrypt(es.encryptionSecret, target)
}
diff --git a/backend/plugins/gitlab/e2e/mr_commits_test.go b/backend/plugins/gitlab/e2e/mr_commits_test.go
index d5ef9d2a4..2f85cc775 100644
--- a/backend/plugins/gitlab/e2e/mr_commits_test.go
+++ b/backend/plugins/gitlab/e2e/mr_commits_test.go
@@ -133,37 +133,4 @@ func TestGitlabMrCommitDataFlow(t *testing.T) {
CSVRelPath: "./snapshot_tables/pull_request_commits.csv",
IgnoreTypes: []interface{}{common.Model{}},
})
-
- // verify conversion
- dataflowTester.FlushTabler(&code.Commit{})
- dataflowTester.FlushTabler(&code.RepoCommit{})
- dataflowTester.Subtask(tasks.ConvertCommitsMeta, taskData)
- dataflowTester.VerifyTable(
- code.Commit{},
- "./snapshot_tables/commits.csv",
- e2ehelper.ColumnWithRawData(
- "sha",
- "additions",
- "deletions",
- "dev_eq",
- "message",
- "author_name",
- "author_email",
- "authored_date",
- "author_id",
- "committer_name",
- "committer_email",
- "committed_date",
- "committer_id",
- ),
- )
-
- dataflowTester.VerifyTable(
- code.RepoCommit{},
- "./snapshot_tables/repo_commits.csv",
- e2ehelper.ColumnWithRawData(
- "repo_id",
- "commit_sha",
- ),
- )
}
diff --git a/backend/plugins/gitlab/e2e/mr_detail_test.go b/backend/plugins/gitlab/e2e/mr_detail_test.go
index c92356676..02d1e5501 100644
--- a/backend/plugins/gitlab/e2e/mr_detail_test.go
+++ b/backend/plugins/gitlab/e2e/mr_detail_test.go
@@ -41,6 +41,8 @@ func TestGitlabMrDetailDataFlow(t *testing.T) {
},
}
// import raw data table
+ dataflowTester.FlushTabler(&code.PullRequestAssignee{})
+ dataflowTester.FlushTabler(&code.PullRequestReviewer{})
dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_gitlab_api_merge_requests.csv",
"_raw_gitlab_api_merge_request_details")
diff --git a/backend/plugins/gitlab/e2e/mr_enrich_test.go b/backend/plugins/gitlab/e2e/mr_enrich_test.go
deleted file mode 100644
index fd18003cc..000000000
--- a/backend/plugins/gitlab/e2e/mr_enrich_test.go
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package e2e
-
-import (
- "testing"
-
- "github.com/apache/incubator-devlake/helpers/e2ehelper"
- "github.com/apache/incubator-devlake/plugins/gitlab/impl"
- "github.com/apache/incubator-devlake/plugins/gitlab/models"
- "github.com/apache/incubator-devlake/plugins/gitlab/tasks"
-)
-
-func TestGitlabMrEnrichDataFlow(t *testing.T) {
-
- var gitlab impl.Gitlab
- dataflowTester := e2ehelper.NewDataFlowTester(t, "gitlab", gitlab)
-
- taskData := &tasks.GitlabTaskData{
- Options: &tasks.GitlabOptions{
- ConnectionId: 1,
- ProjectId: 12345678,
- ScopeConfig: new(models.GitlabScopeConfig),
- },
- }
-
- // import data table
- dataflowTester.ImportCsvIntoTabler("./snapshot_tables/_tool_gitlab_merge_requests.csv", &models.GitlabMergeRequest{})
- dataflowTester.ImportCsvIntoTabler("./snapshot_tables/_tool_gitlab_mr_commits.csv", &models.GitlabMrCommit{})
- dataflowTester.ImportCsvIntoTabler("./snapshot_tables/_tool_gitlab_mr_notes.csv", &models.GitlabMrNote{})
-
- // verify extraction
- dataflowTester.Subtask(tasks.EnrichMergeRequestsMeta, taskData)
- dataflowTester.VerifyTable(
- models.GitlabMergeRequest{},
- "./snapshot_tables/_tool_gitlab_merge_requests_for_enrich_test.csv",
- e2ehelper.ColumnWithRawData(
- "connection_id",
- "gitlab_id",
- "iid",
- "project_id",
- "source_project_id",
- "target_project_id",
- "state",
- "title",
- "web_url",
- "user_notes_count",
- "work_in_progress",
- "source_branch",
- "target_branch",
- "merge_commit_sha",
- "merged_at",
- "gitlab_created_at",
- "closed_at",
- "merged_by_username",
- "description",
- "author_username",
- "author_user_id",
- "component",
- "first_comment_time",
- "review_rounds",
- ),
- )
-}
diff --git a/backend/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_merge_requests_for_enrich_test.csv b/backend/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_merge_requests_for_enrich_test.csv
deleted file mode 100644
index 56804c730..000000000
--- a/backend/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_merge_requests_for_enrich_test.csv
+++ /dev/null
@@ -1,25 +0,0 @@
-connection_id,gitlab_id,iid,project_id,source_project_id,target_project_id,state,title,web_url,user_notes_count,work_in_progress,source_branch,target_branch,merge_commit_sha,merged_at,gitlab_created_at,closed_at,merged_by_username,description,author_username,author_user_id,component,first_comment_time,review_rounds,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
-1,32348491,1,12345678,12345678,12345678,merged,"Resolve ""Add documentation to snowflake spend package""",https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/1,1,0,1-add-documentation-to-snowflake-spend-package,master,da1d6dea48f5972ffc683da6cff30934e7d6c52c,2019-06-28T14:32:06.192+00:00,2019-06-28T05:21:43.743+00:00,,tayloramurphy,Closes #1,emilie,2295562,,,2,,,0,
-1,35064956,3,12345678,13835497,12345678,merged,Update README to include steps to resolve a potential dbt-utils conflict,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/3,3,0,5-update-readme-to-include-steps-to-resolve-a-potential-dbt-utils-conflict,master,d678bea9d47b42eb13512d1c9d6a592d80b432d4,2019-08-26T14:15:27.922+00:00,2019-08-15T19:34:32.706+00:00,,emilie,Closes #5,martinguindon,3871284,,,1,,,0,
-1,35841926,4,12345678,12345678,12345678,merged,"Resolve ""Config is not generic enough""",https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/4,0,0,4-config-is-not-generic-enough,master,e95b5db25e15a38e21d11cb45cc21bf17d5c407c,2019-08-26T15:37:50.105+00:00,2019-08-26T15:32:49.557+00:00,,emilie,Closes #4,emilie,2295562,,,1,,,0,
-1,53445063,5,12345678,15706315,12345678,merged,Issue 3 Base model,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/5,20,0,issue_3,master,f2ee4cf121a328ce39723506dc18e4661941971a,2020-03-25T18:36:45.801+00:00,2020-03-24T12:46:15.891+00:00,,tayloramurphy,,nehiljain,783199,,,1,,,0,
-1,53627854,6,12345678,15706063,12345678,merged,Update schema.yml typo in docs,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/6,0,0,patch-1,master,12dcc23a45adce0b12f8687438ec3a28274c7c30,2020-03-25T19:04:19.844+00:00,2020-03-25T19:02:16.747+00:00,,tayloramurphy,,nehiljain,783199,,,1,,,0,
-1,55146687,8,12345678,12345678,12345678,merged,"Resolve ""Document release process""",https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/8,3,0,6-document-release-process,master,7c8245a3a5eda7f502737940aaf7944d99c58f2e,2020-04-08T20:52:11.150+00:00,2020-04-08T20:07:10.223+00:00,,emilie,Closes #6,m_walker,5212782,,,1,,,0,
-1,55146787,9,12345678,15706315,12345678,opened,Issue 3: Transformed model for query performance,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/9,14,0,issue_3,master,,,2020-04-08T20:09:08.130+00:00,,,,nehiljain,783199,,,1,,,0,
-1,58311001,10,12345678,12345678,12345678,merged,Update version in readme,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/10,0,0,emilie-master-patch-23079,master,66c0f1de49a0c876b8f93e8e0dce3327e766f59d,2020-05-11T17:09:20.603+00:00,2020-05-11T17:09:12.265+00:00,,emilie,,emilie,2295562,,,1,,,0,
-1,62519057,11,12345678,19569570,12345678,opened,Clustering metering models,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/11,0,0,clustering-metering,master,,,2020-06-24T12:34:04.792+00:00,,,,jainnehil,842680,,,1,,,0,
-1,65505080,12,12345678,12345678,12345678,merged,"Resolve ""Upgrade package for dbt 0.17""",https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/12,0,0,11-upgrade-package-for-dbt-0-17,master,9bfc136eb90802c2ce59956c34dde01bb3de0d50,2020-07-24T21:13:35.321+00:00,2020-07-24T17:47:08.238+00:00,,tayloramurphy,"Closes #11
-
-* Upgrades to 0.17.0 format
-* Formatting changes to be in line with GitLab SQL Style Guide",tayloramurphy,1942272,,,1,,,0,
-1,68978485,13,12345678,15706315,12345678,closed,Include more snowflake qrt columns,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/13,0,0,include_more_snowflake_qrt_columns,master,,,2020-08-27T20:17:01.825+00:00,2020-08-27T20:20:08.150+00:00,,,aianus,2478227,,,1,,,0,
-1,89243644,14,12345678,24539973,12345678,merged,Update README.md to use the newest version as an example,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/14,0,0,ThomasLaPiana-master-patch-93997,master,21840a7eadb58babe8aeae2960da851a3ed00ddc,2021-02-19T20:13:05.969+00:00,2021-02-19T20:12:14.302+00:00,,tayloramurphy,Update README.md to use the newest version as an example. The old version doesn't work with the current version of dbt,ThomasLaPiana,2061802,,,1,,,0,
-1,110817220,16,12345678,28584714,12345678,merged,Update packages.yml to point to dbt-labs instead of fishtown,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/16,5,0,GJMcClintock-master-patch-24867,master,6f45b467c478df1c67d19cf6d4cbb8e05a710662,2021-08-12T06:12:54.329+00:00,2021-08-03T15:02:54.955+00:00,,vedprakash2021,With the company name change the old repo is deprecated.,GJMcClintock,9439881,,,1,,,0,
-1,111383524,17,12345678,0,12345678,closed,The package name changed -> https://hub.getdbt.com/dbt-labs/dbt_utils/latest/,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/17,1,0,swiffer-master-patch-77533,master,,,2021-08-07T06:50:25.458+00:00,2021-08-07T06:51:14.933+00:00,,,swiffer,156402,,,1,,,0,
-1,114994501,18,12345678,29298577,12345678,opened,Add support for Snowpipe usage monitoring,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/18,0,0,master,master,,,2021-09-01T21:15:30.334+00:00,,,Add models and docs for Snowflake pipes (Snowpipe) usage monitoring based on the views in Snowflake Usage schema,gary-beautypie,9635687,,,1,,,0,
-1,135775405,19,12345678,32935405,12345678,opened,Updates for dbt 1.0,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/19,1,0,updates_for_dbt_1.0,master,,,2022-01-18T19:59:30.723+00:00,,,"This MR sets up the repo for dbt 1.0
-A few configs were renamed.
-
-Could a new release be made for dbt 1.0?",johnj4,10663622,,,1,,,0,
-1,145012495,20,12345678,34491818,12345678,closed,Draft: Update dbt_project.yml,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/20,0,1,PedramNavid-master-patch-20645,master,,,2022-03-15T03:07:06.077+00:00,2022-03-15T03:07:22.665+00:00,,,PedramNavid,9722492,,,1,,,0,
-1,158698019,21,12345678,12345678,12345678,opened,Draft: Corrections for dbt 1,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/21,0,1,updates_for_dbt_1_1,master,,,2022-06-03T09:24:53.707+00:00,,,Closes https://gitlab.com/gitlab-data/analytics/-/issues/12941,paul_armstrong,5618371,,,1,,,0,
diff --git a/backend/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_merge_requests_for_mr_commit_test.csv b/backend/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_merge_requests_for_mr_commit_test.csv
index 2ba28fadb..dc468a58b 100644
--- a/backend/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_merge_requests_for_mr_commit_test.csv
+++ b/backend/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_merge_requests_for_mr_commit_test.csv
@@ -1,5 +1,5 @@
-connection_id,gitlab_id,iid,project_id,source_project_id,target_project_id,state,title,web_url,user_notes_count,work_in_progress,source_branch,target_branch,merge_commit_sha,merged_at,gitlab_created_at,closed_at,merged_by_username,description,author_username,author_user_id,component,first_comment_time,review_rounds,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
-1,32348491,1,12345678,12345678,12345678,merged,"Resolve ""Add documentation to snowflake spend package""",https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/1,1,0,1-add-documentation-to-snowflake-spend-package,master,da1d6dea48f5972ffc683da6cff30934e7d6c52c,2019-06-28T14:32:06.192+00:00,2019-06-28T05:21:43.743+00:00,,tayloramurphy,Closes #1,emilie,2295562,,,0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,1,
+connection_id,gitlab_id,iid,project_id,source_project_id,target_project_id,state,title,web_url,user_notes_count,work_in_progress,source_branch,target_branch,merge_commit_sha,merged_at,gitlab_created_at,closed_at,merged_by_username,description,author_username,author_user_id,component,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
+1,32348491,1,12345678,12345678,12345678,merged,"Resolve ""Add documentation to snowflake spend package""",https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/1,1,0,1-add-documentation-to-snowflake-spend-package,master,da1d6dea48f5972ffc683da6cff30934e7d6c52c,2019-06-28T14:32:06.192+00:00,2019-06-28T05:21:43.743+00:00,,tayloramurphy,Closes #1,emilie,2295562,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,1,
1,34300715,2,12345678,12345678,12345678,merged,"Resolve ""Handle multiple prices and date ranges""",https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/2,9,0,2-handle-multiple-prices-and-date-ranges,master,2fe82576fdbd0a43a2d5e02c17c6dc6ce3e188ee,2019-08-04T22:24:41.219+00:00,2019-08-01T21:36:45.020+00:00,,emilie,"Closes #2
* Adds a model to amortize the snowflake_contract_rates.csv properly
@@ -8,7 +8,7 @@ connection_id,gitlab_id,iid,project_id,source_project_id,target_project_id,state
Testing by running dbt seed on https://gitlab.com/gitlab-data/analytics/merge_requests/1351/diffs and pinning dbt deps to this branch `2-handle...`.
-All models ran fine and the data looks great in my scratch schema!",tayloramurphy,1942272,,,0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,2,
-1,35064956,3,12345678,13835497,12345678,merged,Update README to include steps to resolve a potential dbt-utils conflict,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/3,3,0,5-update-readme-to-include-steps-to-resolve-a-potential-dbt-utils-conflict,master,d678bea9d47b42eb13512d1c9d6a592d80b432d4,2019-08-26T14:15:27.922+00:00,2019-08-15T19:34:32.706+00:00,,emilie,Closes #5,martinguindon,3871284,,,0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,3,
-1,35841926,4,12345678,12345678,12345678,merged,"Resolve ""Config is not generic enough""",https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/4,0,0,4-config-is-not-generic-enough,master,e95b5db25e15a38e21d11cb45cc21bf17d5c407c,2019-08-26T15:37:50.105+00:00,2019-08-26T15:32:49.557+00:00,,emilie,Closes #4,emilie,2295562,,,0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,4,
-1,53445063,5,12345678,15706315,12345678,merged,Issue 3 Base model,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/5,20,0,issue_3,master,f2ee4cf121a328ce39723506dc18e4661941971a,2020-03-25T18:36:45.801+00:00,2020-03-24T12:46:15.891+00:00,,tayloramurphy,,nehiljain,783199,,,0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,5,
+All models ran fine and the data looks great in my scratch schema!",tayloramurphy,1942272,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,2,
+1,35064956,3,12345678,13835497,12345678,merged,Update README to include steps to resolve a potential dbt-utils conflict,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/3,3,0,5-update-readme-to-include-steps-to-resolve-a-potential-dbt-utils-conflict,master,d678bea9d47b42eb13512d1c9d6a592d80b432d4,2019-08-26T14:15:27.922+00:00,2019-08-15T19:34:32.706+00:00,,emilie,Closes #5,martinguindon,3871284,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,3,
+1,35841926,4,12345678,12345678,12345678,merged,"Resolve ""Config is not generic enough""",https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/4,0,0,4-config-is-not-generic-enough,master,e95b5db25e15a38e21d11cb45cc21bf17d5c407c,2019-08-26T15:37:50.105+00:00,2019-08-26T15:32:49.557+00:00,,emilie,Closes #4,emilie,2295562,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,4,
+1,53445063,5,12345678,15706315,12345678,merged,Issue 3 Base model,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/5,20,0,issue_3,master,f2ee4cf121a328ce39723506dc18e4661941971a,2020-03-25T18:36:45.801+00:00,2020-03-24T12:46:15.891+00:00,,tayloramurphy,,nehiljain,783199,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,5,
diff --git a/backend/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_merge_requests_for_mr_notes_test.csv b/backend/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_merge_requests_for_mr_notes_test.csv
index fe38d4100..05d7a0c7d 100644
--- a/backend/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_merge_requests_for_mr_notes_test.csv
+++ b/backend/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_merge_requests_for_mr_notes_test.csv
@@ -1,6 +1,6 @@
-connection_id,gitlab_id,iid,project_id,source_project_id,target_project_id,state,title,web_url,user_notes_count,work_in_progress,source_branch,target_branch,merge_commit_sha,merged_at,gitlab_created_at,closed_at,merged_by_username,description,author_username,author_user_id,component,first_comment_time,review_rounds,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
-1,15869219,37,12345678,12345678,12345678,opened,Draft: Corrections for dbt 1,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/21,0,1,updates_for_dbt_1_1,master,,,2022-06-03T09:24:53.707+00:00,,,Closes https://gitlab.com/gitlab-data/analytics/-/issues/12941,paul_armstrong,5618371,,,0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,21,
-1,32348491,1,12345678,12345678,12345678,merged,"Resolve ""Add documentation to snowflake spend package""",https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/1,1,0,1-add-documentation-to-snowflake-spend-package,master,da1d6dea48f5972ffc683da6cff30934e7d6c52c,2019-06-28T14:32:06.192+00:00,2019-06-28T05:21:43.743+00:00,,tayloramurphy,Closes #1,emilie,2295562,,,0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,1,
+connection_id,gitlab_id,iid,project_id,source_project_id,target_project_id,state,title,web_url,user_notes_count,work_in_progress,source_branch,target_branch,merge_commit_sha,merged_at,gitlab_created_at,closed_at,merged_by_username,description,author_username,author_user_id,component,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
+1,15869219,37,12345678,12345678,12345678,opened,Draft: Corrections for dbt 1,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/21,0,1,updates_for_dbt_1_1,master,,,2022-06-03T09:24:53.707+00:00,,,Closes https://gitlab.com/gitlab-data/analytics/-/issues/12941,paul_armstrong,5618371,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,21,
+1,32348491,1,12345678,12345678,12345678,merged,"Resolve ""Add documentation to snowflake spend package""",https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/1,1,0,1-add-documentation-to-snowflake-spend-package,master,da1d6dea48f5972ffc683da6cff30934e7d6c52c,2019-06-28T14:32:06.192+00:00,2019-06-28T05:21:43.743+00:00,,tayloramurphy,Closes #1,emilie,2295562,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,1,
1,34300715,2,12345678,12345678,12345678,merged,"Resolve ""Handle multiple prices and date ranges""",https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/2,9,0,2-handle-multiple-prices-and-date-ranges,master,2fe82576fdbd0a43a2d5e02c17c6dc6ce3e188ee,2019-08-04T22:24:41.219+00:00,2019-08-01T21:36:45.020+00:00,,emilie,"Closes #2
* Adds a model to amortize the snowflake_contract_rates.csv properly
@@ -9,23 +9,23 @@ connection_id,gitlab_id,iid,project_id,source_project_id,target_project_id,state
Testing by running dbt seed on https://gitlab.com/gitlab-data/analytics/merge_requests/1351/diffs and pinning dbt deps to this branch `2-handle...`.
-All models ran fine and the data looks great in my scratch schema!",tayloramurphy,1942272,,,0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,2,
-1,35064956,3,12345678,13835497,12345678,merged,Update README to include steps to resolve a potential dbt-utils conflict,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/3,3,0,5-update-readme-to-include-steps-to-resolve-a-potential-dbt-utils-conflict,master,d678bea9d47b42eb13512d1c9d6a592d80b432d4,2019-08-26T14:15:27.922+00:00,2019-08-15T19:34:32.706+00:00,,emilie,Closes #5,martinguindon,3871284,,,0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,3,
-1,35841926,4,12345678,12345678,12345678,merged,"Resolve ""Config is not generic enough""",https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/4,0,0,4-config-is-not-generic-enough,master,e95b5db25e15a38e21d11cb45cc21bf17d5c407c,2019-08-26T15:37:50.105+00:00,2019-08-26T15:32:49.557+00:00,,emilie,Closes #4,emilie,2295562,,,0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,4,
-1,53445063,5,12345678,15706315,12345678,merged,Issue 3 Base model,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/5,20,0,issue_3,master,f2ee4cf121a328ce39723506dc18e4661941971a,2020-03-25T18:36:45.801+00:00,2020-03-24T12:46:15.891+00:00,,tayloramurphy,,nehiljain,783199,,,0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,5,
-1,53627854,6,12345678,15706063,12345678,merged,Update schema.yml typo in docs,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/6,0,0,patch-1,master,12dcc23a45adce0b12f8687438ec3a28274c7c30,2020-03-25T19:04:19.844+00:00,2020-03-25T19:02:16.747+00:00,,tayloramurphy,,nehiljain,783199,,,0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,6,
-1,53640548,7,12345678,12345678,12345678,merged,"Resolve ""Add sources and cleanup structure""",https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/7,5,0,7-add-sources-and-cleanup-structure,master,1161de80448bc66072c3ea39600d9940dde87a5b,2020-04-07T12:51:50.096+00:00,2020-03-25T21:58:40.647+00:00,,emilie,Closes #7,tayloramurphy,1942272,,,0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,7,
-1,55146687,8,12345678,12345678,12345678,merged,"Resolve ""Document release process""",https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/8,3,0,6-document-release-process,master,7c8245a3a5eda7f502737940aaf7944d99c58f2e,2020-04-08T20:52:11.150+00:00,2020-04-08T20:07:10.223+00:00,,emilie,Closes #6,m_walker,5212782,,,0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,8,
-1,55146787,9,12345678,15706315,12345678,opened,Issue 3: Transformed model for query performance,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/9,14,0,issue_3,master,,,2020-04-08T20:09:08.130+00:00,,,,nehiljain,783199,,,0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,9,
-1,58311001,10,12345678,12345678,12345678,merged,Update version in readme,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/10,0,0,emilie-master-patch-23079,master,66c0f1de49a0c876b8f93e8e0dce3327e766f59d,2020-05-11T17:09:20.603+00:00,2020-05-11T17:09:12.265+00:00,,emilie,,emilie,2295562,,,0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,10,
-1,62519057,11,12345678,19569570,12345678,opened,Clustering metering models,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/11,0,0,clustering-metering,master,,,2020-06-24T12:34:04.792+00:00,,,,jainnehil,842680,,,0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,11,
+All models ran fine and the data looks great in my scratch schema!",tayloramurphy,1942272,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,2,
+1,35064956,3,12345678,13835497,12345678,merged,Update README to include steps to resolve a potential dbt-utils conflict,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/3,3,0,5-update-readme-to-include-steps-to-resolve-a-potential-dbt-utils-conflict,master,d678bea9d47b42eb13512d1c9d6a592d80b432d4,2019-08-26T14:15:27.922+00:00,2019-08-15T19:34:32.706+00:00,,emilie,Closes #5,martinguindon,3871284,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,3,
+1,35841926,4,12345678,12345678,12345678,merged,"Resolve ""Config is not generic enough""",https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/4,0,0,4-config-is-not-generic-enough,master,e95b5db25e15a38e21d11cb45cc21bf17d5c407c,2019-08-26T15:37:50.105+00:00,2019-08-26T15:32:49.557+00:00,,emilie,Closes #4,emilie,2295562,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,4,
+1,53445063,5,12345678,15706315,12345678,merged,Issue 3 Base model,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/5,20,0,issue_3,master,f2ee4cf121a328ce39723506dc18e4661941971a,2020-03-25T18:36:45.801+00:00,2020-03-24T12:46:15.891+00:00,,tayloramurphy,,nehiljain,783199,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,5,
+1,53627854,6,12345678,15706063,12345678,merged,Update schema.yml typo in docs,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/6,0,0,patch-1,master,12dcc23a45adce0b12f8687438ec3a28274c7c30,2020-03-25T19:04:19.844+00:00,2020-03-25T19:02:16.747+00:00,,tayloramurphy,,nehiljain,783199,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,6,
+1,53640548,7,12345678,12345678,12345678,merged,"Resolve ""Add sources and cleanup structure""",https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/7,5,0,7-add-sources-and-cleanup-structure,master,1161de80448bc66072c3ea39600d9940dde87a5b,2020-04-07T12:51:50.096+00:00,2020-03-25T21:58:40.647+00:00,,emilie,Closes #7,tayloramurphy,1942272,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,7,
+1,55146687,8,12345678,12345678,12345678,merged,"Resolve ""Document release process""",https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/8,3,0,6-document-release-process,master,7c8245a3a5eda7f502737940aaf7944d99c58f2e,2020-04-08T20:52:11.150+00:00,2020-04-08T20:07:10.223+00:00,,emilie,Closes #6,m_walker,5212782,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,8,
+1,55146787,9,12345678,15706315,12345678,opened,Issue 3: Transformed model for query performance,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/9,14,0,issue_3,master,,,2020-04-08T20:09:08.130+00:00,,,,nehiljain,783199,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,9,
+1,58311001,10,12345678,12345678,12345678,merged,Update version in readme,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/10,0,0,emilie-master-patch-23079,master,66c0f1de49a0c876b8f93e8e0dce3327e766f59d,2020-05-11T17:09:20.603+00:00,2020-05-11T17:09:12.265+00:00,,emilie,,emilie,2295562,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,10,
+1,62519057,11,12345678,19569570,12345678,opened,Clustering metering models,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/11,0,0,clustering-metering,master,,,2020-06-24T12:34:04.792+00:00,,,,jainnehil,842680,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,11,
1,65505080,12,12345678,12345678,12345678,merged,"Resolve ""Upgrade package for dbt 0.17""",https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/12,0,0,11-upgrade-package-for-dbt-0-17,master,9bfc136eb90802c2ce59956c34dde01bb3de0d50,2020-07-24T21:13:35.321+00:00,2020-07-24T17:47:08.238+00:00,,tayloramurphy,"Closes #11
* Upgrades to 0.17.0 format
-* Formatting changes to be in line with GitLab SQL Style Guide",tayloramurphy,1942272,,,0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,12,
+* Formatting changes to be in line with GitLab SQL Style Guide",tayloramurphy,1942272,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,12,
1,135772105,30,12345678,32935405,12345678,opened,Updates for dbt 1.0,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/19,1,0,updates_for_dbt_1.0,master,,,2022-01-18T19:59:30.723+00:00,,,"This MR sets up the repo for dbt 1.0
A few configs were renamed.
-Could a new release be made for dbt 1.0?",johnj4,10663622,,,0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,19,
-1,145032495,46,12345678,34491818,12345678,closed,Draft: Update dbt_project.yml,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/20,0,1,PedramNavid-master-patch-20645,master,,,2022-03-15T03:07:06.077+00:00,2022-03-15T03:07:22.665+00:00,,,PedramNavid,9722492,,,0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,20,
-1,1149942101,29,12345678,29298577,12345678,opened,Add support for Snowpipe usage monitoring,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/18,0,0,master,master,,,2021-09-01T21:15:30.334+00:00,,,Add models and docs for Snowflake pipes (Snowpipe) usage monitoring based on the views in Snowflake Usage schema,gary-beautypie,9635687,,,0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,18,
+Could a new release be made for dbt 1.0?",johnj4,10663622,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,19,
+1,145032495,46,12345678,34491818,12345678,closed,Draft: Update dbt_project.yml,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/20,0,1,PedramNavid-master-patch-20645,master,,,2022-03-15T03:07:06.077+00:00,2022-03-15T03:07:22.665+00:00,,,PedramNavid,9722492,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,20,
+1,1149942101,29,12345678,29298577,12345678,opened,Add support for Snowpipe usage monitoring,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/18,0,0,master,master,,,2021-09-01T21:15:30.334+00:00,,,Add models and docs for Snowflake pipes (Snowpipe) usage monitoring based on the views in Snowflake Usage schema,gary-beautypie,9635687,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,18,
diff --git a/backend/plugins/gitlab/e2e/snapshot_tables/commits.csv b/backend/plugins/gitlab/e2e/snapshot_tables/commits.csv
deleted file mode 100644
index ea3ba115e..000000000
--- a/backend/plugins/gitlab/e2e/snapshot_tables/commits.csv
+++ /dev/null
@@ -1,65 +0,0 @@
-sha,additions,deletions,dev_eq,message,author_name,author_email,authored_date,author_id,committer_name,committer_email,committed_date,committer_id,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
-05976c113b02fb71d5c43682bd2f9d96a3454efa,0,0,0,"tab refcatoring
-",Nehil Jain,nehil@snaptravel.com,2020-03-24T18:14:03.000+00:00,nehil@snaptravel.com,Nehil Jain,nehil@snaptravel.com,2020-03-24T18:14:03.000+00:00,nehil@snaptravel.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,34,
-083864201849f6707a34d24e61c43638696f95a6,0,0,0,"Update README to include steps to resolve a potential dbt-utils conflict
-",Martin Guindon,mguindon@absolunet.com,2019-08-15T19:32:19.000+00:00,mguindon@absolunet.com,Martin Guindon,mguindon@absolunet.com,2019-08-15T19:32:19.000+00:00,mguindon@absolunet.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,6,
-08973065d33c6fd2fb1f592b7a1e0fbe2ee72edd,0,0,0,"add usage day
-",Taylor A. Murphy,tmurphy@gitlab.com,2019-08-02T18:35:08.000+00:00,tmurphy@gitlab.com,Taylor A. Murphy,tmurphy@gitlab.com,2019-08-02T18:35:08.000+00:00,tmurphy@gitlab.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,11,
-0a4574d52b5964c37bfcb2952f6c6a8a9c6d5bcf,0,0,0,"fix name
-",Taylor A. Murphy,tmurphy@gitlab.com,2019-08-02T19:00:50.000+00:00,tmurphy@gitlab.com,Taylor A. Murphy,tmurphy@gitlab.com,2019-08-02T19:00:50.000+00:00,tmurphy@gitlab.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,7,
-0dbb8bfdbbf7373656fa3eb036bdcbb25ae162da,0,0,0,Apply suggestion to models/base/snowflake_query_history.sql,Nehil,nehil@getsnaptravel.com,2020-03-24T22:39:40.000+00:00,nehil@getsnaptravel.com,Nehil,nehil@getsnaptravel.com,2020-03-24T22:39:40.000+00:00,nehil@getsnaptravel.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,26,
-1cba71b355d2bd4e1350602cfb9b98cce5a5cb4d,0,0,0,"update documentation
-",Taylor A. Murphy,tmurphy@gitlab.com,2019-08-02T18:53:44.000+00:00,tmurphy@gitlab.com,Taylor A. Murphy,tmurphy@gitlab.com,2019-08-02T18:53:44.000+00:00,tmurphy@gitlab.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,8,
-1f16955d45cec904c8968bb27689d91a71e9a2f2,0,0,0,"change cte address
-",Taylor A. Murphy,tmurphy@gitlab.com,2019-08-02T17:28:44.000+00:00,tmurphy@gitlab.com,Taylor A. Murphy,tmurphy@gitlab.com,2019-08-02T17:28:44.000+00:00,tmurphy@gitlab.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,12,
-211795deaaaaddb23b797261cdcd148e6083d5b2,0,0,0,Update snowflake_query_history.sql,Nehil,nehil@getsnaptravel.com,2020-03-24T22:44:07.000+00:00,nehil@getsnaptravel.com,Nehil,nehil@getsnaptravel.com,2020-03-24T22:44:07.000+00:00,nehil@getsnaptravel.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,21,
-25f2b34099a3e1867fcf73a54e50f78f3c8d8f75,0,0,0,"add documentation
-",Taylor A. Murphy,tmurphy@gitlab.com,2019-08-02T18:47:28.000+00:00,tmurphy@gitlab.com,Taylor A. Murphy,tmurphy@gitlab.com,2019-08-02T18:47:28.000+00:00,tmurphy@gitlab.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,9,
-382084b42697577d3a6adf71ce73d4b5ddd22977,0,0,0,"add info on dashboard
-",emilie,emilie.burke@gmail.com,2019-06-28T10:54:26.000+00:00,emilie.burke@gmail.com,emilie,emilie.burke@gmail.com,2019-06-28T10:54:26.000+00:00,emilie.burke@gmail.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,2,
-3f04e0a61d0c4d2dd736a6bcaa3a06826269a533,0,0,0,"finish top level readme
-",emilie,emilie.burke@gmail.com,2019-06-28T10:48:49.000+00:00,emilie.burke@gmail.com,emilie,emilie.burke@gmail.com,2019-06-28T10:48:49.000+00:00,emilie.burke@gmail.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,3,
-45bed79fcdc6ba3444245f55660f839db50d79c8,0,0,0,Update .gitignore,Nehil,nehil@getsnaptravel.com,2020-03-24T18:15:29.000+00:00,nehil@getsnaptravel.com,Nehil,nehil@getsnaptravel.com,2020-03-24T18:15:29.000+00:00,nehil@getsnaptravel.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,33,
-48295660f8bd0c0a024849d4407830face57d4bc,0,0,0,"add docs and sources
-",emilie,emilie.burke@gmail.com,2020-04-05T14:39:43.000+00:00,emilie.burke@gmail.com,emilie,emilie.burke@gmail.com,2020-04-05T14:39:43.000+00:00,emilie.burke@gmail.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,37,
-4f0d2f7b1a47d21b01a68e59c12d6f8885ab230e,0,0,0,"mr comments
-",Nehil Jain,nehil@snaptravel.com,2020-03-24T22:40:36.000+00:00,nehil@snaptravel.com,Nehil Jain,nehil@snaptravel.com,2020-03-24T22:40:36.000+00:00,nehil@snaptravel.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,24,
-553f7613b178301da4075202afe91ebda5d97ab5,0,0,0,Update schema.yml typo for docs,Nehil,nehil@getsnaptravel.com,2020-03-25T19:01:57.000+00:00,nehil@getsnaptravel.com,Nehil,nehil@getsnaptravel.com,2020-03-25T19:01:57.000+00:00,nehil@getsnaptravel.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,16,
-5a141ad0c28960b02c40e6c476d8df3cc76f58c4,0,0,0,"schema yml, untested
-",Nehil Jain,2761999+nehiljain@users.noreply.github.com,2020-03-24T21:18:16.000+00:00,2761999+nehiljain@users.noreply.github.com,Nehil Jain,2761999+nehiljain@users.noreply.github.com,2020-03-24T21:18:16.000+00:00,2761999+nehiljain@users.noreply.github.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,31,
-5b6a2be437e0c66ba6276030a61a10583c3f83fa,0,0,0,"merging online changes
-",Nehil Jain,nehil@snaptravel.com,2020-03-24T22:42:54.000+00:00,nehil@snaptravel.com,Nehil Jain,nehil@snaptravel.com,2020-03-24T22:42:54.000+00:00,nehil@snaptravel.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,22,
-5d6c369200c633b06d85c9375f61fef297761ef1,0,0,0,Apply suggestion to models/base/snowflake_query_history.sql,"Taylor A Murphy, PhD",tmurphy@gitlab.com,2020-03-25T13:22:42.000+00:00,tmurphy@gitlab.com,"Taylor A Murphy, PhD",tmurphy@gitlab.com,2020-03-25T13:22:42.000+00:00,tmurphy@gitlab.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,19,
-645a724b558a7fa3885fa233d06f68f833d893fc,0,0,0,Apply suggestion to models/base/snowflake_query_history.sql,Nehil,nehil@getsnaptravel.com,2020-03-24T22:39:36.000+00:00,nehil@getsnaptravel.com,Nehil,nehil@getsnaptravel.com,2020-03-24T22:39:36.000+00:00,nehil@getsnaptravel.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,27,
-64abdcdd9ee9c821f56faf93c5bb0ebc57f575e4,0,0,0,Apply suggestion to models/base/snowflake_query_history.sql,Nehil,nehil@getsnaptravel.com,2020-03-24T22:39:31.000+00:00,nehil@getsnaptravel.com,Nehil,nehil@getsnaptravel.com,2020-03-24T22:39:31.000+00:00,nehil@getsnaptravel.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,28,
-6ebf519f25328e49a004f5a574c8b69ba7420be1,0,0,0,"add contract rate and dbt-utils
-",Taylor A. Murphy,tmurphy@gitlab.com,2019-08-02T17:18:15.000+00:00,tmurphy@gitlab.com,Taylor A. Murphy,tmurphy@gitlab.com,2019-08-02T17:18:15.000+00:00,tmurphy@gitlab.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,15,
-81b12c995b2fc12e20fd05113cf36dbd57b6e7b0,0,0,0,Apply suggestion to models/base/snowflake_query_history.sql,"Taylor A Murphy, PhD",tmurphy@gitlab.com,2020-03-25T13:21:04.000+00:00,tmurphy@gitlab.com,"Taylor A Murphy, PhD",tmurphy@gitlab.com,2020-03-25T13:21:04.000+00:00,tmurphy@gitlab.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,20,
-846deac12a1263e4631ced45abbc5e157b23c1be,0,0,0,"testing
-",Nehil Jain,nehil@snaptravel.com,2020-03-24T21:27:46.000+00:00,nehil@snaptravel.com,Nehil Jain,nehil@snaptravel.com,2020-03-24T21:27:46.000+00:00,nehil@snaptravel.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,30,
-8a40765e605966842b05b5d8cdf2dd6890556d09,0,0,0,"change dbt-util version
-",Taylor A. Murphy,tmurphy@gitlab.com,2019-08-02T17:22:09.000+00:00,tmurphy@gitlab.com,Taylor A. Murphy,tmurphy@gitlab.com,2019-08-02T17:22:09.000+00:00,tmurphy@gitlab.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,14,
-8a96911c5350f7575215f02f0242f7e27ad3d1f4,0,0,0,"add documentation
-",Taylor A. Murphy,tmurphy@gitlab.com,2019-08-02T18:46:42.000+00:00,tmurphy@gitlab.com,Taylor A. Murphy,tmurphy@gitlab.com,2019-08-02T18:46:42.000+00:00,tmurphy@gitlab.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,10,
-91e5666bdfe90e5ada3e65aaa9240b2b0b1a072f,0,0,0,"remove config
-",emilie,emilie.burke@gmail.com,2019-08-26T15:34:33.000+00:00,emilie.burke@gmail.com,emilie,emilie.burke@gmail.com,2019-08-26T15:34:33.000+00:00,emilie.burke@gmail.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,17,
-93d13cf14a470995ec35186292bf165e1152dc2a,0,0,0,"comments on sql
-",Nehil Jain,nehiljain@Hansons-iMac.local,2020-03-24T18:55:11.000+00:00,nehiljain@Hansons-iMac.local,Nehil Jain,nehiljain@Hansons-iMac.local,2020-03-24T18:55:11.000+00:00,nehiljain@Hansons-iMac.local,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,32,
-9a80506e158547d87e7ad2902c0196deff5b5c07,0,0,0,"base mmodel for query performance
-",Nehil Jain,nehil@snaptravel.com,2020-03-24T12:52:28.000+00:00,nehil@snaptravel.com,Nehil Jain,nehil@snaptravel.com,2020-03-24T12:52:28.000+00:00,nehil@snaptravel.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,35,
-9e5ec04b0a8273b73936ea150275919b0602c260,0,0,0,"change names
-",Taylor A. Murphy,tmurphy@gitlab.com,2019-08-02T17:24:35.000+00:00,tmurphy@gitlab.com,Taylor A. Murphy,tmurphy@gitlab.com,2019-08-02T17:24:35.000+00:00,tmurphy@gitlab.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,13,
-9ef8183140bae8c39a440d9c7eb47f3bc7dcab75,0,0,0,Apply suggestion to models/base/snowflake_query_history.sql,"Taylor A Murphy, PhD",tmurphy@gitlab.com,2020-03-25T13:22:47.000+00:00,tmurphy@gitlab.com,"Taylor A Murphy, PhD",tmurphy@gitlab.com,2020-03-25T13:22:47.000+00:00,tmurphy@gitlab.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,18,
-a09f2d4ae88e78d7424fba1ae6d0c036b677271d,0,0,0,"update readme
-",emilie,emilie.burke@gmail.com,2020-04-05T14:43:47.000+00:00,emilie.burke@gmail.com,emilie,emilie.burke@gmail.com,2020-04-05T14:43:47.000+00:00,emilie.burke@gmail.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,36,
-abbe0ab2c7bb1dc2cfaa3ef3062f378fb908ba71,0,0,0,"add first bit
-",emilie,emilie.burke@gmail.com,2019-06-28T05:40:36.000+00:00,emilie.burke@gmail.com,emilie,emilie.burke@gmail.com,2019-06-28T05:40:36.000+00:00,emilie.burke@gmail.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,5,
-ad25fcda15e310a7eb34b3e38ff9b962b54f0e03,0,0,0,"add more info to readme
-",emilie,emilie.burke@gmail.com,2019-06-28T11:03:52.000+00:00,emilie.burke@gmail.com,emilie,emilie.burke@gmail.com,2019-06-28T11:03:52.000+00:00,emilie.burke@gmail.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,1,
-bb6dfec8dcdacb5b6b37dbd8eb1aa6f9e8c06138,0,0,0,"merging online changes
-",Nehil Jain,nehil@snaptravel.com,2020-03-24T22:41:35.000+00:00,nehil@snaptravel.com,Nehil Jain,nehil@snaptravel.com,2020-03-24T22:41:35.000+00:00,nehil@snaptravel.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,23,
-c4d406c077af582837d3611a46eded70151f93fb,0,0,0,"tested schema and the model
-",Nehil Jain,nehil@snaptravel.com,2020-03-24T22:37:23.000+00:00,nehil@snaptravel.com,Nehil Jain,nehil@snaptravel.com,2020-03-24T22:37:23.000+00:00,nehil@snaptravel.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,29,
-e01d4f03811cd0da9949848731236e0aa261cf54,0,0,0,"move analyses
-",emilie,emilie.burke@gmail.com,2019-06-28T10:43:17.000+00:00,emilie.burke@gmail.com,emilie,emilie.burke@gmail.com,2019-06-28T10:43:17.000+00:00,emilie.burke@gmail.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,4,
-e24fecacb1a2c85f904bef57ffd5598ca2df023c,0,0,0,Apply suggestion to models/base/snowflake_query_history.sql,Nehil,nehil@getsnaptravel.com,2020-03-24T22:39:42.000+00:00,nehil@getsnaptravel.com,Nehil,nehil@getsnaptravel.com,2020-03-24T22:39:42.000+00:00,nehil@getsnaptravel.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,25,
diff --git a/backend/plugins/gitlab/e2e/snapshot_tables/repo_commits.csv b/backend/plugins/gitlab/e2e/snapshot_tables/repo_commits.csv
deleted file mode 100644
index 1b3ee220d..000000000
--- a/backend/plugins/gitlab/e2e/snapshot_tables/repo_commits.csv
+++ /dev/null
@@ -1,38 +0,0 @@
-repo_id,commit_sha,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
-gitlab:GitlabProject:1:12345678,05976c113b02fb71d5c43682bd2f9d96a3454efa,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,34,
-gitlab:GitlabProject:1:12345678,083864201849f6707a34d24e61c43638696f95a6,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,6,
-gitlab:GitlabProject:1:12345678,08973065d33c6fd2fb1f592b7a1e0fbe2ee72edd,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,11,
-gitlab:GitlabProject:1:12345678,0a4574d52b5964c37bfcb2952f6c6a8a9c6d5bcf,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,7,
-gitlab:GitlabProject:1:12345678,0dbb8bfdbbf7373656fa3eb036bdcbb25ae162da,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,26,
-gitlab:GitlabProject:1:12345678,1cba71b355d2bd4e1350602cfb9b98cce5a5cb4d,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,8,
-gitlab:GitlabProject:1:12345678,1f16955d45cec904c8968bb27689d91a71e9a2f2,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,12,
-gitlab:GitlabProject:1:12345678,211795deaaaaddb23b797261cdcd148e6083d5b2,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,21,
-gitlab:GitlabProject:1:12345678,25f2b34099a3e1867fcf73a54e50f78f3c8d8f75,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,9,
-gitlab:GitlabProject:1:12345678,382084b42697577d3a6adf71ce73d4b5ddd22977,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,2,
-gitlab:GitlabProject:1:12345678,3f04e0a61d0c4d2dd736a6bcaa3a06826269a533,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,3,
-gitlab:GitlabProject:1:12345678,45bed79fcdc6ba3444245f55660f839db50d79c8,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,33,
-gitlab:GitlabProject:1:12345678,48295660f8bd0c0a024849d4407830face57d4bc,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,37,
-gitlab:GitlabProject:1:12345678,4f0d2f7b1a47d21b01a68e59c12d6f8885ab230e,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,24,
-gitlab:GitlabProject:1:12345678,553f7613b178301da4075202afe91ebda5d97ab5,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,16,
-gitlab:GitlabProject:1:12345678,5a141ad0c28960b02c40e6c476d8df3cc76f58c4,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,31,
-gitlab:GitlabProject:1:12345678,5b6a2be437e0c66ba6276030a61a10583c3f83fa,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,22,
-gitlab:GitlabProject:1:12345678,5d6c369200c633b06d85c9375f61fef297761ef1,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,19,
-gitlab:GitlabProject:1:12345678,645a724b558a7fa3885fa233d06f68f833d893fc,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,27,
-gitlab:GitlabProject:1:12345678,64abdcdd9ee9c821f56faf93c5bb0ebc57f575e4,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,28,
-gitlab:GitlabProject:1:12345678,6ebf519f25328e49a004f5a574c8b69ba7420be1,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,15,
-gitlab:GitlabProject:1:12345678,81b12c995b2fc12e20fd05113cf36dbd57b6e7b0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,20,
-gitlab:GitlabProject:1:12345678,846deac12a1263e4631ced45abbc5e157b23c1be,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,30,
-gitlab:GitlabProject:1:12345678,8a40765e605966842b05b5d8cdf2dd6890556d09,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,14,
-gitlab:GitlabProject:1:12345678,8a96911c5350f7575215f02f0242f7e27ad3d1f4,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,10,
-gitlab:GitlabProject:1:12345678,91e5666bdfe90e5ada3e65aaa9240b2b0b1a072f,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,17,
-gitlab:GitlabProject:1:12345678,93d13cf14a470995ec35186292bf165e1152dc2a,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,32,
-gitlab:GitlabProject:1:12345678,9a80506e158547d87e7ad2902c0196deff5b5c07,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,35,
-gitlab:GitlabProject:1:12345678,9e5ec04b0a8273b73936ea150275919b0602c260,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,13,
-gitlab:GitlabProject:1:12345678,9ef8183140bae8c39a440d9c7eb47f3bc7dcab75,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,18,
-gitlab:GitlabProject:1:12345678,a09f2d4ae88e78d7424fba1ae6d0c036b677271d,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,36,
-gitlab:GitlabProject:1:12345678,abbe0ab2c7bb1dc2cfaa3ef3062f378fb908ba71,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,5,
-gitlab:GitlabProject:1:12345678,ad25fcda15e310a7eb34b3e38ff9b962b54f0e03,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,1,
-gitlab:GitlabProject:1:12345678,bb6dfec8dcdacb5b6b37dbd8eb1aa6f9e8c06138,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,23,
-gitlab:GitlabProject:1:12345678,c4d406c077af582837d3611a46eded70151f93fb,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,29,
-gitlab:GitlabProject:1:12345678,e01d4f03811cd0da9949848731236e0aa261cf54,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,4,
-gitlab:GitlabProject:1:12345678,e24fecacb1a2c85f904bef57ffd5598ca2df023c,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,25,
diff --git a/backend/plugins/gitlab/models/migrationscripts/20240904_remove_mr_review_fields.go b/backend/plugins/gitlab/models/migrationscripts/20240904_remove_mr_review_fields.go
new file mode 100644
index 000000000..2628033f0
--- /dev/null
+++ b/backend/plugins/gitlab/models/migrationscripts/20240904_remove_mr_review_fields.go
@@ -0,0 +1,40 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+ "github.com/apache/incubator-devlake/core/context"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
+)
+
+var _ plugin.MigrationScript = (*removeMrEnricherFields)(nil)
+
+type removeMrEnricherFields struct{}
+
+func (script *removeMrEnricherFields) Up(basicRes context.BasicRes) errors.Error {
+ return basicRes.GetDal().DropColumns("_tool_gitlab_merge_requests", "first_comment_time", "review_rounds")
+}
+
+func (*removeMrEnricherFields) Version() uint64 {
+ return 20240904152414
+}
+
+func (*removeMrEnricherFields) Name() string {
+ return "remove mr review fields in favor of dora plugin"
+}
diff --git a/backend/plugins/gitlab/models/mr.go b/backend/plugins/gitlab/models/mr.go
index 6ac3c9b76..c25df4ba0 100644
--- a/backend/plugins/gitlab/models/mr.go
+++ b/backend/plugins/gitlab/models/mr.go
@@ -50,9 +50,7 @@ type GitlabMergeRequest struct {
Description string
AuthorUsername string `gorm:"type:varchar(255)"`
AuthorUserId int
- Component string `gorm:"type:varchar(255)"`
- FirstCommentTime *time.Time `gorm:"comment:Time when the first comment occurred"`
- ReviewRounds int `gorm:"comment:How many rounds of review this MR went through"`
+ Component string `gorm:"type:varchar(255)"`
common.NoPKModel
}
diff --git a/backend/plugins/gitlab/tasks/account_collector.go b/backend/plugins/gitlab/tasks/account_collector.go
index 52ea1a72d..e08498589 100644
--- a/backend/plugins/gitlab/tasks/account_collector.go
+++ b/backend/plugins/gitlab/tasks/account_collector.go
@@ -43,7 +43,7 @@ var CollectAccountsMeta = plugin.SubTaskMeta{
EnabledByDefault: true,
Description: "collect gitlab users, does not support either timeFilter or diffSync.",
DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS},
- Dependencies: []*plugin.SubTaskMeta{&EnrichMergeRequestsMeta},
+ Dependencies: []*plugin.SubTaskMeta{},
}
func CollectAccounts(taskCtx plugin.SubTaskContext) errors.Error {
@@ -58,7 +58,7 @@ func CollectAccounts(taskCtx plugin.SubTaskContext) errors.Error {
}
// Collect all users if endpoint is private gitlab instance
- if !strings.HasPrefix(data.ApiClient.GetEndpoint(), "https://gitlab.com") {
+ if !strings.HasPrefix(data.ApiClient.GetEndpoint(), "https://gitlab.com") && !strings.HasPrefix(data.ApiClient.GetEndpoint(), "https://jihulab.com") {
urlTemplate = "/users"
}
diff --git a/backend/plugins/gitlab/tasks/child_pipeline_extractor.go b/backend/plugins/gitlab/tasks/child_pipeline_extractor.go
index 4c7d8d875..62c5948fc 100644
--- a/backend/plugins/gitlab/tasks/child_pipeline_extractor.go
+++ b/backend/plugins/gitlab/tasks/child_pipeline_extractor.go
@@ -18,8 +18,6 @@ limitations under the License.
package tasks
import (
- "encoding/json"
-
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models/common"
"github.com/apache/incubator-devlake/core/plugin"
@@ -40,32 +38,19 @@ var ExtractApiChildPipelinesMeta = plugin.SubTaskMeta{
Dependencies: []*plugin.SubTaskMeta{&CollectApiChildPipelinesMeta},
}
-func ExtractApiChildPipelines(taskCtx plugin.SubTaskContext) errors.Error {
- rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_CHILD_PIPELINE_TABLE)
-
- extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{
- RawDataSubTaskArgs: *rawDataSubTaskArgs,
- Extract: func(row *api.RawData) ([]interface{}, errors.Error) {
- gitlabApiChildPipeline := &ApiPipeline{}
- err := errors.Convert(json.Unmarshal(row.Data, gitlabApiChildPipeline))
- if err != nil {
- return nil, err
- }
- pipelineProject := convertApiPipelineToGitlabPipelineProject(gitlabApiChildPipeline, data.Options.ConnectionId, data.Options.ProjectId)
+func ExtractApiChildPipelines(subtaskCtx plugin.SubTaskContext) errors.Error {
+ subtaskCommonArgs, data := CreateSubtaskCommonArgs(subtaskCtx, RAW_CHILD_PIPELINE_TABLE)
+ extractor, err := api.NewStatefulApiExtractor(&api.StatefulApiExtractorArgs[ApiPipeline]{
+ SubtaskCommonArgs: subtaskCommonArgs,
+ Extract: func(gitlabApiPipeline *ApiPipeline, row *api.RawData) ([]interface{}, errors.Error) {
+ pipelineProject := convertApiPipelineToGitlabPipelineProject(gitlabApiPipeline, data.Options.ConnectionId, data.Options.ProjectId)
return []interface{}{pipelineProject}, nil
},
})
-
- if err != nil {
- return err
- }
-
- err = extractor.Execute()
if err != nil {
return err
}
-
- return nil
+ return extractor.Execute()
}
func convertApiPipelineToGitlabPipelineProject(gitlabApiChildPipeline *ApiPipeline, connectionId uint64, projectId int) *models.GitlabPipelineProject {
diff --git a/backend/plugins/gitlab/tasks/commit_collector.go b/backend/plugins/gitlab/tasks/commit_collector.go
deleted file mode 100644
index 4bb2e960f..000000000
--- a/backend/plugins/gitlab/tasks/commit_collector.go
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package tasks
-
-import (
- "github.com/apache/incubator-devlake/core/errors"
- "github.com/apache/incubator-devlake/core/models/common"
- "github.com/apache/incubator-devlake/core/plugin"
- "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
-)
-
-func init() {
- RegisterSubtaskMeta(&CollectApiCommitsMeta)
-}
-
-const RAW_COMMIT_TABLE = "gitlab_api_commit"
-
-var CollectApiCommitsMeta = plugin.SubTaskMeta{
- Name: "Collect Commits",
- EntryPoint: CollectApiCommits,
- EnabledByDefault: false,
- Description: "Collect commit data from gitlab api, does not support either timeFilter or diffSync.",
- DomainTypes: []string{plugin.DOMAIN_TYPE_CODE},
- Dependencies: []*plugin.SubTaskMeta{&ConvertJobMeta},
-}
-
-type GitlabApiCommit struct {
- GitlabId string `json:"id"`
- Title string
- Message string
- ProjectId int
- ShortId string `json:"short_id"`
- AuthorName string `json:"author_name"`
- AuthorEmail string `json:"author_email"`
- AuthoredDate common.Iso8601Time `json:"authored_date"`
- CommitterName string `json:"committer_name"`
- CommitterEmail string `json:"committer_email"`
- CommittedDate common.Iso8601Time `json:"committed_date"`
- WebUrl string `json:"web_url"`
- Stats struct {
- Additions int
- Deletions int
- Total int
- }
-}
-
-func CollectApiCommits(taskCtx plugin.SubTaskContext) errors.Error {
- rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_COMMIT_TABLE)
-
- collector, err := api.NewApiCollector(api.ApiCollectorArgs{
- RawDataSubTaskArgs: *rawDataSubTaskArgs,
- ApiClient: data.ApiClient,
- PageSize: 100,
- Incremental: false,
- UrlTemplate: "projects/{{ .Params.ProjectId }}/repository/commits",
- Query: GetQuery,
- ResponseParser: GetRawMessageFromResponse,
- })
-
- if err != nil {
- return err
- }
-
- return collector.Execute()
-}
diff --git a/backend/plugins/gitlab/tasks/commit_convertor.go b/backend/plugins/gitlab/tasks/commit_convertor.go
deleted file mode 100644
index c0fb77a79..000000000
--- a/backend/plugins/gitlab/tasks/commit_convertor.go
+++ /dev/null
@@ -1,108 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package tasks
-
-import (
- "reflect"
-
- "github.com/apache/incubator-devlake/core/dal"
- "github.com/apache/incubator-devlake/core/errors"
- "github.com/apache/incubator-devlake/core/models/domainlayer/code"
- "github.com/apache/incubator-devlake/core/models/domainlayer/didgen"
- "github.com/apache/incubator-devlake/core/plugin"
- helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
- "github.com/apache/incubator-devlake/plugins/gitlab/models"
-)
-
-func init() {
- RegisterSubtaskMeta(&ConvertCommitsMeta)
-}
-
-var ConvertCommitsMeta = plugin.SubTaskMeta{
- Name: "Convert Commits",
- EntryPoint: ConvertApiCommits,
- EnabledByDefault: false,
- Description: "Update domain layer commit according to GitlabCommit",
- DomainTypes: []string{plugin.DOMAIN_TYPE_CODE},
- Dependencies: []*plugin.SubTaskMeta{&ConvertMrLabelsMeta},
-}
-
-func ConvertApiCommits(taskCtx plugin.SubTaskContext) errors.Error {
- rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_COMMIT_TABLE)
- db := taskCtx.GetDal()
-
- // select all commits belongs to the project
- clauses := []dal.Clause{
- dal.Select("gc.*"),
- dal.From("_tool_gitlab_commits gc"),
- dal.Join(`left join _tool_gitlab_project_commits gpc on (
- gpc.commit_sha = gc.sha
- )`),
- dal.Where("gpc.gitlab_project_id = ? and gpc.connection_id = ? ",
- data.Options.ProjectId, data.Options.ConnectionId),
- }
- cursor, err := db.Cursor(clauses...)
- if err != nil {
- return err
- }
- defer cursor.Close()
-
- // TODO: adopt batch indate operation
- //userDidGen := didgen.NewDomainIdGenerator(&models.GitlabAccount{})
-
- converter, err := helper.NewDataConverter(helper.DataConverterArgs{
- RawDataSubTaskArgs: *rawDataSubTaskArgs,
- InputRowType: reflect.TypeOf(models.GitlabCommit{}),
- Input: cursor,
-
- Convert: func(inputRow interface{}) ([]interface{}, errors.Error) {
- gitlabCommit := inputRow.(*models.GitlabCommit)
-
- // convert commit
- commit := &code.Commit{}
- commit.Sha = gitlabCommit.Sha
- commit.Message = gitlabCommit.Message
- commit.Additions = gitlabCommit.Additions
- commit.Deletions = gitlabCommit.Deletions
- commit.AuthorId = gitlabCommit.AuthorEmail
- commit.AuthorName = gitlabCommit.AuthorName
- commit.AuthorEmail = gitlabCommit.AuthorEmail
- commit.AuthoredDate = gitlabCommit.AuthoredDate
- commit.CommitterName = gitlabCommit.CommitterName
- commit.CommitterEmail = gitlabCommit.CommitterEmail
- commit.CommittedDate = gitlabCommit.CommittedDate
- commit.CommitterId = gitlabCommit.CommitterEmail
-
- // convert repo / commits relationship
- repoCommit := &code.RepoCommit{
- RepoId: didgen.NewDomainIdGenerator(&models.GitlabProject{}).Generate(data.Options.ConnectionId, data.Options.ProjectId),
- CommitSha: gitlabCommit.Sha,
- }
-
- return []interface{}{
- commit,
- repoCommit,
- }, nil
- },
- })
- if err != nil {
- return err
- }
-
- return converter.Execute()
-}
diff --git a/backend/plugins/gitlab/tasks/commit_extractor.go b/backend/plugins/gitlab/tasks/commit_extractor.go
deleted file mode 100644
index 664f8eae3..000000000
--- a/backend/plugins/gitlab/tasks/commit_extractor.go
+++ /dev/null
@@ -1,116 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package tasks
-
-import (
- "encoding/json"
-
- "github.com/apache/incubator-devlake/core/errors"
- "github.com/apache/incubator-devlake/core/plugin"
- "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
- "github.com/apache/incubator-devlake/plugins/gitlab/models"
-)
-
-func init() {
- RegisterSubtaskMeta(&ExtractApiCommitsMeta)
-}
-
-var ExtractApiCommitsMeta = plugin.SubTaskMeta{
- Name: "Extract Commits",
- EntryPoint: ExtractApiCommits,
- EnabledByDefault: false,
- Description: "Extract raw commit data into tool layer table GitlabCommit,GitlabAccount and GitlabProjectCommit",
- DomainTypes: []string{plugin.DOMAIN_TYPE_CODE},
- Dependencies: []*plugin.SubTaskMeta{&CollectApiCommitsMeta},
-}
-
-func ExtractApiCommits(taskCtx plugin.SubTaskContext) errors.Error {
- rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_COMMIT_TABLE)
-
- extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{
- RawDataSubTaskArgs: *rawDataSubTaskArgs,
- Extract: func(row *api.RawData) ([]interface{}, errors.Error) {
- // need to extract 3 kinds of entities here
- results := make([]interface{}, 0, 3)
-
- // create gitlab commit
- gitlabApiCommit := &GitlabApiCommit{}
- err := errors.Convert(json.Unmarshal(row.Data, gitlabApiCommit))
- if err != nil {
- return nil, err
- }
- gitlabCommit, err := ConvertCommit(gitlabApiCommit)
- if err != nil {
- return nil, err
- }
-
- // create project/commits relationship
- gitlabProjectCommit := &models.GitlabProjectCommit{GitlabProjectId: data.Options.ProjectId}
- gitlabProjectCommit.CommitSha = gitlabCommit.Sha
-
- // create gitlab user
- GitlabAccountAuthor := &models.GitlabAccount{}
- GitlabAccountAuthor.Email = gitlabCommit.AuthorEmail
- GitlabAccountAuthor.Name = gitlabCommit.AuthorName
-
- gitlabProjectCommit.ConnectionId = data.Options.ConnectionId
- GitlabAccountAuthor.ConnectionId = data.Options.ConnectionId
- results = append(results, gitlabCommit)
- results = append(results, gitlabProjectCommit)
- results = append(results, GitlabAccountAuthor)
-
- // For Commiter Email is not same as AuthorEmail
- if gitlabCommit.CommitterEmail != GitlabAccountAuthor.Email {
- gitlabAccountCommitter := &models.GitlabAccount{}
- gitlabAccountCommitter.Email = gitlabCommit.CommitterEmail
- gitlabAccountCommitter.Name = gitlabCommit.CommitterName
- gitlabAccountCommitter.ConnectionId = data.Options.ConnectionId
- results = append(results, gitlabAccountCommitter)
- }
-
- return results, nil
- },
- })
-
- if err != nil {
- return err
- }
-
- return extractor.Execute()
-}
-
-// Convert the API response to our DB model instance
-func ConvertCommit(commit *GitlabApiCommit) (*models.GitlabCommit, errors.Error) {
- gitlabCommit := &models.GitlabCommit{
- Sha: commit.GitlabId,
- Title: commit.Title,
- Message: commit.Message,
- ShortId: commit.ShortId,
- AuthorName: commit.AuthorName,
- AuthorEmail: commit.AuthorEmail,
- AuthoredDate: commit.AuthoredDate.ToTime(),
- CommitterName: commit.CommitterName,
- CommitterEmail: commit.CommitterEmail,
- CommittedDate: commit.CommittedDate.ToTime(),
- WebUrl: commit.WebUrl,
- Additions: commit.Stats.Additions,
- Deletions: commit.Stats.Deletions,
- Total: commit.Stats.Total,
- }
- return gitlabCommit, nil
-}
diff --git a/backend/plugins/gitlab/tasks/deployment_convertor.go b/backend/plugins/gitlab/tasks/deployment_convertor.go
index d7ff0d774..b211867b4 100644
--- a/backend/plugins/gitlab/tasks/deployment_convertor.go
+++ b/backend/plugins/gitlab/tasks/deployment_convertor.go
@@ -19,7 +19,6 @@ package tasks
import (
"fmt"
- "reflect"
"time"
"github.com/apache/incubator-devlake/core/dal"
@@ -50,9 +49,9 @@ var ConvertDeploymentMeta = plugin.SubTaskMeta{
// ConvertDeployment should be split into two task theoretically
// But in GitLab, all deployments have commits, so there is no need to change it.
-func ConvertDeployment(taskCtx plugin.SubTaskContext) errors.Error {
- rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_DEPLOYMENT)
- db := taskCtx.GetDal()
+func ConvertDeployment(subtaskCtx plugin.SubTaskContext) errors.Error {
+ subtaskCommonArgs, data := CreateSubtaskCommonArgs(subtaskCtx, RAW_DEPLOYMENT)
+ db := subtaskCtx.GetDal()
repo := &models.GitlabProject{}
err := db.First(repo, dal.Where("gitlab_id = ? and connection_id = ?", data.Options.ProjectId, data.Options.ConnectionId))
@@ -61,25 +60,24 @@ func ConvertDeployment(taskCtx plugin.SubTaskContext) errors.Error {
}
projectIdGen := didgen.NewDomainIdGenerator(&models.GitlabProject{})
-
- cursor, err := db.Cursor(
- dal.From(&models.GitlabDeployment{}),
- dal.Where("connection_id = ? AND gitlab_id = ?", data.Options.ConnectionId, data.Options.ProjectId),
- )
- if err != nil {
- return err
- }
- defer cursor.Close()
-
idGen := didgen.NewDomainIdGenerator(&models.GitlabDeployment{})
- converter, err := api.NewDataConverter(api.DataConverterArgs{
- InputRowType: reflect.TypeOf(models.GitlabDeployment{}),
- Input: cursor,
- RawDataSubTaskArgs: *rawDataSubTaskArgs,
- Convert: func(inputRow interface{}) ([]interface{}, errors.Error) {
- gitlabDeployment := inputRow.(*models.GitlabDeployment)
-
+ converter, err := api.NewStatefulDataConverter(&api.StatefulDataConverterArgs[models.GitlabDeployment]{
+ SubtaskCommonArgs: subtaskCommonArgs,
+ Input: func(stateManager *api.SubtaskStateManager) (dal.Rows, errors.Error) {
+ clauses := []dal.Clause{
+ dal.From(&models.GitlabDeployment{}),
+ dal.Where("connection_id = ? AND gitlab_id = ?", data.Options.ConnectionId, data.Options.ProjectId),
+ }
+ if stateManager.IsIncremental() {
+ since := stateManager.GetSince()
+ if since != nil {
+ clauses = append(clauses, dal.Where("updated_at >= ? ", since))
+ }
+ }
+ return db.Cursor(clauses...)
+ },
+ Convert: func(gitlabDeployment *models.GitlabDeployment) ([]interface{}, errors.Error) {
var duration *float64
if gitlabDeployment.DeployableDuration != nil {
deployableDuration := cast.ToFloat64(*gitlabDeployment.DeployableDuration)
diff --git a/backend/plugins/gitlab/tasks/deployment_extractor.go b/backend/plugins/gitlab/tasks/deployment_extractor.go
index 0b439aceb..258471b01 100644
--- a/backend/plugins/gitlab/tasks/deployment_extractor.go
+++ b/backend/plugins/gitlab/tasks/deployment_extractor.go
@@ -18,7 +18,6 @@ limitations under the License.
package tasks
import (
- "encoding/json"
"time"
"github.com/apache/incubator-devlake/core/errors"
@@ -44,16 +43,11 @@ var ExtractDeploymentMeta = plugin.SubTaskMeta{
}
func ExtractDeployment(taskCtx plugin.SubTaskContext) errors.Error {
- rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_DEPLOYMENT)
-
- extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{
- RawDataSubTaskArgs: *rawDataSubTaskArgs,
- Extract: func(row *api.RawData) ([]interface{}, errors.Error) {
- deploymentResp := &GitlabDeploymentResp{}
- err := errors.Convert(json.Unmarshal(row.Data, deploymentResp))
- if err != nil {
- return nil, err
- }
+ subtaskCommonArgs, data := CreateSubtaskCommonArgs(taskCtx, RAW_DEPLOYMENT)
+
+ extractor, err := api.NewStatefulApiExtractor(&api.StatefulApiExtractorArgs[GitlabDeploymentResp]{
+ SubtaskCommonArgs: subtaskCommonArgs,
+ Extract: func(deploymentResp *GitlabDeploymentResp, row *api.RawData) ([]interface{}, errors.Error) {
gitlabDeployment := deploymentResp.toGitlabDeployment(data.Options.ConnectionId, data.Options.ProjectId)
return []interface{}{
gitlabDeployment,
diff --git a/backend/plugins/gitlab/tasks/issue_assignee_convertor.go b/backend/plugins/gitlab/tasks/issue_assignee_convertor.go
index f4a70065d..ce5ce593f 100644
--- a/backend/plugins/gitlab/tasks/issue_assignee_convertor.go
+++ b/backend/plugins/gitlab/tasks/issue_assignee_convertor.go
@@ -18,14 +18,12 @@ limitations under the License.
package tasks
import (
- "reflect"
-
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models/domainlayer/didgen"
"github.com/apache/incubator-devlake/core/models/domainlayer/ticket"
"github.com/apache/incubator-devlake/core/plugin"
- helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/gitlab/models"
)
@@ -40,35 +38,36 @@ var ConvertIssueAssigneeMeta = plugin.SubTaskMeta{
Description: "Convert tool layer table _tool_gitlab_issue_assignees into domain layer table issue_assignees",
DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET},
DependencyTables: []string{models.GitlabIssueAssignee{}.TableName()},
- Dependencies: []*plugin.SubTaskMeta{&ExtractApiIssuesMeta},
+ Dependencies: []*plugin.SubTaskMeta{&ConvertIssuesMeta},
}
-func ConvertIssueAssignee(taskCtx plugin.SubTaskContext) errors.Error {
- db := taskCtx.GetDal()
- rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_ISSUE_TABLE)
-
- cursor, err := db.Cursor(
- dal.From(&models.GitlabIssueAssignee{}),
- dal.Where("project_id = ? and connection_id=?", data.Options.ProjectId, data.Options.ConnectionId),
- )
- if err != nil {
- return err
- }
- defer cursor.Close()
+func ConvertIssueAssignee(subtaskCtx plugin.SubTaskContext) errors.Error {
+ subtaskCommonArgs, data := CreateSubtaskCommonArgs(subtaskCtx, RAW_ISSUE_TABLE)
+ db := subtaskCtx.GetDal()
issueIdGen := didgen.NewDomainIdGenerator(&models.GitlabIssue{})
accountIdGen := didgen.NewDomainIdGenerator(&models.GitlabAccount{})
- converter, err := helper.NewDataConverter(helper.DataConverterArgs{
- RawDataSubTaskArgs: *rawDataSubTaskArgs,
- InputRowType: reflect.TypeOf(models.GitlabIssueAssignee{}),
- Input: cursor,
- Convert: func(inputRow interface{}) ([]interface{}, errors.Error) {
- input := inputRow.(*models.GitlabIssueAssignee)
+ converter, err := api.NewStatefulDataConverter(&api.StatefulDataConverterArgs[models.GitlabIssueAssignee]{
+ SubtaskCommonArgs: subtaskCommonArgs,
+ Input: func(stateManager *api.SubtaskStateManager) (dal.Rows, errors.Error) {
+ clauses := []dal.Clause{
+ dal.From(&models.GitlabIssueAssignee{}),
+ dal.Where("connection_id = ? AND project_id = ?", data.Options.ConnectionId, data.Options.ProjectId),
+ }
+ if stateManager.IsIncremental() {
+ since := stateManager.GetSince()
+ if since != nil {
+ clauses = append(clauses, dal.Where("updated_at >= ? ", since))
+ }
+ }
+ return db.Cursor(clauses...)
+ },
+ Convert: func(issueAssignee *models.GitlabIssueAssignee) ([]interface{}, errors.Error) {
domainIssueAssignee := &ticket.IssueAssignee{
- IssueId: issueIdGen.Generate(data.Options.ConnectionId, input.GitlabId),
- AssigneeId: accountIdGen.Generate(data.Options.ConnectionId, input.AssigneeId),
- AssigneeName: input.AssigneeName,
+ IssueId: issueIdGen.Generate(data.Options.ConnectionId, issueAssignee.GitlabId),
+ AssigneeId: accountIdGen.Generate(data.Options.ConnectionId, issueAssignee.AssigneeId),
+ AssigneeName: issueAssignee.AssigneeName,
}
return []interface{}{domainIssueAssignee}, nil
},
diff --git a/backend/plugins/gitlab/tasks/issue_convertor.go b/backend/plugins/gitlab/tasks/issue_convertor.go
index 9be3c1d18..63af1c659 100644
--- a/backend/plugins/gitlab/tasks/issue_convertor.go
+++ b/backend/plugins/gitlab/tasks/issue_convertor.go
@@ -18,7 +18,6 @@ limitations under the License.
package tasks
import (
- "reflect"
"strconv"
"strings"
@@ -28,7 +27,7 @@ import (
"github.com/apache/incubator-devlake/core/models/domainlayer/didgen"
"github.com/apache/incubator-devlake/core/models/domainlayer/ticket"
"github.com/apache/incubator-devlake/core/plugin"
- helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/gitlab/models"
)
@@ -45,32 +44,41 @@ var ConvertIssuesMeta = plugin.SubTaskMeta{
Dependencies: []*plugin.SubTaskMeta{&ConvertApiMrCommitsMeta},
}
-func ConvertIssues(taskCtx plugin.SubTaskContext) errors.Error {
- db := taskCtx.GetDal()
- rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_ISSUE_TABLE)
- projectId := data.Options.ProjectId
-
- clauses := []dal.Clause{
- dal.Select("issues.*"),
- dal.From("_tool_gitlab_issues issues"),
- dal.Where("project_id = ? and connection_id = ?", projectId, data.Options.ConnectionId),
- }
- cursor, err := db.Cursor(clauses...)
- if err != nil {
- return err
- }
- defer cursor.Close()
+func ConvertIssues(subtaskCtx plugin.SubTaskContext) errors.Error {
+ subtaskCommonArgs, data := CreateSubtaskCommonArgs(subtaskCtx, RAW_ISSUE_TABLE)
+ db := subtaskCtx.GetDal()
+ projectId := data.Options.ProjectId
issueIdGen := didgen.NewDomainIdGenerator(&models.GitlabIssue{})
accountIdGen := didgen.NewDomainIdGenerator(&models.GitlabAccount{})
boardIdGen := didgen.NewDomainIdGenerator(&models.GitlabProject{})
- converter, err := helper.NewDataConverter(helper.DataConverterArgs{
- RawDataSubTaskArgs: *rawDataSubTaskArgs,
- InputRowType: reflect.TypeOf(models.GitlabIssue{}),
- Input: cursor,
- Convert: func(inputRow interface{}) ([]interface{}, errors.Error) {
- issue := inputRow.(*models.GitlabIssue)
+ converter, err := api.NewStatefulDataConverter(&api.StatefulDataConverterArgs[models.GitlabIssue]{
+ SubtaskCommonArgs: subtaskCommonArgs,
+ Input: func(stateManager *api.SubtaskStateManager) (dal.Rows, errors.Error) {
+ clauses := []dal.Clause{
+ dal.From(&models.GitlabIssue{}),
+ dal.Where("connection_id = ? AND project_id = ?", data.Options.ConnectionId, data.Options.ProjectId),
+ }
+ if stateManager.IsIncremental() {
+ since := stateManager.GetSince()
+ if since != nil {
+ clauses = append(clauses, dal.Where("updated_at >= ? ", since))
+ }
+ }
+ return db.Cursor(clauses...)
+ },
+ BeforeConvert: func(issue *models.GitlabIssue, stateManager *api.SubtaskStateManager) errors.Error {
+ issueId := issueIdGen.Generate(data.Options.ConnectionId, issue.GitlabId)
+ if err := db.Delete(&ticket.IssueLabel{}, dal.Where("issue_id = ?", issueId)); err != nil {
+ return err
+ }
+ if err := db.Delete(&ticket.IssueAssignee{}, dal.Where("issue_id = ?", issueId)); err != nil {
+ return err
+ }
+ return nil
+ },
+ Convert: func(issue *models.GitlabIssue) ([]interface{}, errors.Error) {
domainIssue := &ticket.Issue{
DomainEntity: domainlayer.DomainEntity{Id: issueIdGen.Generate(data.Options.ConnectionId, issue.GitlabId)},
IssueKey: strconv.Itoa(issue.Number),
@@ -107,6 +115,7 @@ func ConvertIssues(taskCtx plugin.SubTaskContext) errors.Error {
BoardId: boardIdGen.Generate(data.Options.ConnectionId, projectId),
IssueId: domainIssue.Id,
}
+
return []interface{}{
domainIssue,
boardIssue,
diff --git a/backend/plugins/gitlab/tasks/issue_extractor.go b/backend/plugins/gitlab/tasks/issue_extractor.go
index 717e0ca98..802d58a60 100644
--- a/backend/plugins/gitlab/tasks/issue_extractor.go
+++ b/backend/plugins/gitlab/tasks/issue_extractor.go
@@ -18,9 +18,9 @@ limitations under the License.
package tasks
import (
- "encoding/json"
"regexp"
+ "github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models/common"
"github.com/apache/incubator-devlake/core/plugin"
@@ -133,8 +133,10 @@ type IssuesResponse struct {
}
}
-func ExtractApiIssues(taskCtx plugin.SubTaskContext) errors.Error {
- rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_ISSUE_TABLE)
+func ExtractApiIssues(subtaskCtx plugin.SubTaskContext) errors.Error {
+ subtaskCommonArgs, data := CreateSubtaskCommonArgs(subtaskCtx, RAW_ISSUE_TABLE)
+
+ db := subtaskCtx.GetDal()
config := data.Options.ScopeConfig
var issueSeverityRegex *regexp.Regexp
var issueComponentRegex *regexp.Regexp
@@ -162,15 +164,33 @@ func ExtractApiIssues(taskCtx plugin.SubTaskContext) errors.Error {
return errors.Default.Wrap(err, "regexp Compile issuePriority failed")
}
}
- extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{
- RawDataSubTaskArgs: *rawDataSubTaskArgs,
- Extract: func(row *api.RawData) ([]interface{}, errors.Error) {
- body := &IssuesResponse{}
- err := errors.Convert(json.Unmarshal(row.Data, body))
- if err != nil {
- return nil, err
+ subtaskCommonArgs.SubtaskConfig = map[string]interface{}{
+ "issueSeverity": issueSeverity,
+ "issueComponent": issueComponent,
+ "issuePriorityRegex": issuePriorityRegex,
+ }
+ extractor, err := api.NewStatefulApiExtractor(&api.StatefulApiExtractorArgs[IssuesResponse]{
+ SubtaskCommonArgs: subtaskCommonArgs,
+ BeforeExtract: func(body *IssuesResponse, stateManager *api.SubtaskStateManager) errors.Error {
+ if stateManager.IsIncremental() {
+ err := db.Delete(
+ &models.GitlabIssueLabel{},
+ dal.Where("connection_id = ? AND issue_id = ?", data.Options.ConnectionId, body.Id),
+ )
+ if err != nil {
+ return err
+ }
+ err = db.Delete(
+ &models.GitlabIssueAssignee{},
+ dal.Where("connection_id = ? AND gitlab_id = ?", data.Options.ConnectionId, body.Id),
+ )
+ if err != nil {
+ return err
+ }
}
-
+ return nil
+ },
+ Extract: func(body *IssuesResponse, row *api.RawData) ([]interface{}, errors.Error) {
if body.ProjectId == 0 {
return nil, nil
}
@@ -182,9 +202,9 @@ func ExtractApiIssues(taskCtx plugin.SubTaskContext) errors.Error {
}
for _, label := range body.Labels {
results = append(results, &models.GitlabIssueLabel{
+ ConnectionId: data.Options.ConnectionId,
IssueId: gitlabIssue.GitlabId,
LabelName: label,
- ConnectionId: data.Options.ConnectionId,
})
if issueSeverityRegex != nil && issueSeverityRegex.MatchString(label) {
gitlabIssue.Severity = label
diff --git a/backend/plugins/gitlab/tasks/issue_label_convertor.go b/backend/plugins/gitlab/tasks/issue_label_convertor.go
index c5e8968dd..fab745dda 100644
--- a/backend/plugins/gitlab/tasks/issue_label_convertor.go
+++ b/backend/plugins/gitlab/tasks/issue_label_convertor.go
@@ -18,14 +18,12 @@ limitations under the License.
package tasks
import (
- "reflect"
-
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models/domainlayer/didgen"
"github.com/apache/incubator-devlake/core/models/domainlayer/ticket"
"github.com/apache/incubator-devlake/core/plugin"
- helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/gitlab/models"
)
@@ -42,35 +40,32 @@ var ConvertIssueLabelsMeta = plugin.SubTaskMeta{
Dependencies: []*plugin.SubTaskMeta{&ConvertIssuesMeta},
}
-func ConvertIssueLabels(taskCtx plugin.SubTaskContext) errors.Error {
- db := taskCtx.GetDal()
- rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_ISSUE_TABLE)
- projectId := data.Options.ProjectId
- clauses := []dal.Clause{
- dal.Select("*"),
- dal.From(&models.GitlabIssueLabel{}),
- dal.Join(`left join _tool_gitlab_issues on
- _tool_gitlab_issues.gitlab_id = _tool_gitlab_issue_labels.issue_id`),
- dal.Where(`_tool_gitlab_issues.project_id = ?
- and _tool_gitlab_issues.connection_id = ?`,
- projectId, data.Options.ConnectionId),
- dal.Orderby("issue_id ASC"),
- }
-
- cursor, err := db.Cursor(clauses...)
- if err != nil {
- return err
- }
- defer cursor.Close()
+func ConvertIssueLabels(subtaskCtx plugin.SubTaskContext) errors.Error {
+ subtaskCommonArgs, data := CreateSubtaskCommonArgs(subtaskCtx, RAW_ISSUE_TABLE)
+ db := subtaskCtx.GetDal()
+ projectId := data.Options.ProjectId
issueIdGen := didgen.NewDomainIdGenerator(&models.GitlabIssue{})
- converter, err := helper.NewDataConverter(helper.DataConverterArgs{
- RawDataSubTaskArgs: *rawDataSubTaskArgs,
- InputRowType: reflect.TypeOf(models.GitlabIssueLabel{}),
- Input: cursor,
- Convert: func(inputRow interface{}) ([]interface{}, errors.Error) {
- issueLabel := inputRow.(*models.GitlabIssueLabel)
+ converter, err := api.NewStatefulDataConverter(&api.StatefulDataConverterArgs[models.GitlabIssueLabel]{
+ SubtaskCommonArgs: subtaskCommonArgs,
+ Input: func(stateManager *api.SubtaskStateManager) (dal.Rows, errors.Error) {
+ clauses := []dal.Clause{
+ dal.Select("l.*"),
+ dal.From("_tool_gitlab_issue_labels l"),
+ dal.Join("LEFT JOIN _tool_gitlab_issues s ON s.gitlab_id = l.issue_id AND l.connection_id = s.connection_id"),
+ dal.Where("s.project_id = ? AND s.connection_id = ?", projectId, data.Options.ConnectionId),
+ // dal.Orderby("s.issue_id ASC"),
+ }
+ if stateManager.IsIncremental() {
+ since := stateManager.GetSince()
+ if since != nil {
+ clauses = append(clauses, dal.Where("l.updated_at >= ? ", since))
+ }
+ }
+ return db.Cursor(clauses...)
+ },
+ Convert: func(issueLabel *models.GitlabIssueLabel) ([]interface{}, errors.Error) {
domainIssueLabel := &ticket.IssueLabel{
IssueId: issueIdGen.Generate(data.Options.ConnectionId, issueLabel.IssueId),
LabelName: issueLabel.LabelName,
diff --git a/backend/plugins/gitlab/tasks/job_convertor.go b/backend/plugins/gitlab/tasks/job_convertor.go
index 58bd8f712..910778447 100644
--- a/backend/plugins/gitlab/tasks/job_convertor.go
+++ b/backend/plugins/gitlab/tasks/job_convertor.go
@@ -18,7 +18,6 @@ limitations under the License.
package tasks
import (
- "reflect"
"time"
"github.com/apache/incubator-devlake/core/dal"
@@ -28,7 +27,7 @@ import (
"github.com/apache/incubator-devlake/core/models/domainlayer/didgen"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
- gitlabModels "github.com/apache/incubator-devlake/plugins/gitlab/models"
+ "github.com/apache/incubator-devlake/plugins/gitlab/models"
)
func init() {
@@ -44,35 +43,31 @@ var ConvertJobMeta = plugin.SubTaskMeta{
Dependencies: []*plugin.SubTaskMeta{&ConvertPipelineCommitMeta},
}
-func ConvertJobs(taskCtx plugin.SubTaskContext) (err errors.Error) {
- db := taskCtx.GetDal()
- data := taskCtx.GetData().(*GitlabTaskData)
+func ConvertJobs(subtaskCtx plugin.SubTaskContext) (err errors.Error) {
+ subtaskCommonArgs, data := CreateSubtaskCommonArgs(subtaskCtx, RAW_JOB_TABLE)
+ db := subtaskCtx.GetDal()
regexEnricher := data.RegexEnricher
+ subtaskCommonArgs.SubtaskConfig = regexEnricher.PlainMap()
- cursor, err := db.Cursor(dal.From(gitlabModels.GitlabJob{}),
- dal.Where("project_id = ? and connection_id = ?", data.Options.ProjectId, data.Options.ConnectionId))
- if err != nil {
- return err
- }
- defer cursor.Close()
-
- jobIdGen := didgen.NewDomainIdGenerator(&gitlabModels.GitlabJob{})
- projectIdGen := didgen.NewDomainIdGenerator(&gitlabModels.GitlabProject{})
- pipelineIdGen := didgen.NewDomainIdGenerator(&gitlabModels.GitlabPipeline{})
- converter, err := api.NewDataConverter(api.DataConverterArgs{
- InputRowType: reflect.TypeOf(gitlabModels.GitlabJob{}),
- Input: cursor,
- RawDataSubTaskArgs: api.RawDataSubTaskArgs{
- Ctx: taskCtx,
- Params: gitlabModels.GitlabApiParams{
- ConnectionId: data.Options.ConnectionId,
- ProjectId: data.Options.ProjectId,
- },
- Table: RAW_JOB_TABLE,
+ jobIdGen := didgen.NewDomainIdGenerator(&models.GitlabJob{})
+ projectIdGen := didgen.NewDomainIdGenerator(&models.GitlabProject{})
+ pipelineIdGen := didgen.NewDomainIdGenerator(&models.GitlabPipeline{})
+ converter, err := api.NewStatefulDataConverter(&api.StatefulDataConverterArgs[models.GitlabJob]{
+ SubtaskCommonArgs: subtaskCommonArgs,
+ Input: func(stateManager *api.SubtaskStateManager) (dal.Rows, errors.Error) {
+ clauses := []dal.Clause{
+ dal.From(models.GitlabJob{}),
+ dal.Where("project_id = ? and connection_id = ?", data.Options.ProjectId, data.Options.ConnectionId),
+ }
+ if stateManager.IsIncremental() {
+ since := stateManager.GetSince()
+ if since != nil {
+ clauses = append(clauses, dal.Where("updated_at >= ? ", since))
+ }
+ }
+ return db.Cursor(clauses...)
},
- Convert: func(inputRow interface{}) ([]interface{}, errors.Error) {
- gitlabJob := inputRow.(*gitlabModels.GitlabJob)
-
+ Convert: func(gitlabJob *models.GitlabJob) ([]interface{}, errors.Error) {
createdAt := time.Now()
if gitlabJob.GitlabCreatedAt != nil {
createdAt = *gitlabJob.GitlabCreatedAt
diff --git a/backend/plugins/gitlab/tasks/job_extractor.go b/backend/plugins/gitlab/tasks/job_extractor.go
index 420fe7a30..15b1343dc 100644
--- a/backend/plugins/gitlab/tasks/job_extractor.go
+++ b/backend/plugins/gitlab/tasks/job_extractor.go
@@ -18,8 +18,6 @@ limitations under the License.
package tasks
import (
- "encoding/json"
-
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models/common"
"github.com/apache/incubator-devlake/core/plugin"
@@ -59,18 +57,12 @@ var ExtractApiJobsMeta = plugin.SubTaskMeta{
Dependencies: []*plugin.SubTaskMeta{&CollectApiJobsMeta},
}
-func ExtractApiJobs(taskCtx plugin.SubTaskContext) errors.Error {
- rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_JOB_TABLE)
-
- extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{
- RawDataSubTaskArgs: *rawDataSubTaskArgs,
- Extract: func(row *api.RawData) ([]interface{}, errors.Error) {
- gitlabApiJob := &ApiJob{}
- err := errors.Convert(json.Unmarshal(row.Data, gitlabApiJob))
- if err != nil {
- return nil, err
- }
+func ExtractApiJobs(subtaskCtx plugin.SubTaskContext) errors.Error {
+ subtaskCommonArgs, data := CreateSubtaskCommonArgs(subtaskCtx, RAW_JOB_TABLE)
+ extractor, err := api.NewStatefulApiExtractor(&api.StatefulApiExtractorArgs[ApiJob]{
+ SubtaskCommonArgs: subtaskCommonArgs,
+ Extract: func(gitlabApiJob *ApiJob, row *api.RawData) ([]interface{}, errors.Error) {
gitlabPipeline, err := convertJob(gitlabApiJob, data.Options.ProjectId)
if err != nil {
return nil, err
diff --git a/backend/plugins/gitlab/tasks/mr_assignee_convertor.go b/backend/plugins/gitlab/tasks/mr_assignee_convertor.go
index d638d1dc0..d22714a74 100644
--- a/backend/plugins/gitlab/tasks/mr_assignee_convertor.go
+++ b/backend/plugins/gitlab/tasks/mr_assignee_convertor.go
@@ -18,14 +18,12 @@ limitations under the License.
package tasks
import (
- "reflect"
-
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models/domainlayer/code"
"github.com/apache/incubator-devlake/core/models/domainlayer/didgen"
"github.com/apache/incubator-devlake/core/plugin"
- helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/gitlab/models"
)
@@ -39,39 +37,33 @@ var ConvertMrAssigneesMeta = plugin.SubTaskMeta{
EnabledByDefault: true,
Description: "Convert tool layer table gitlab_assignees into domain layer table pull_request_assignees",
DomainTypes: []string{plugin.DOMAIN_TYPE_CODE_REVIEW},
- Dependencies: []*plugin.SubTaskMeta{&ExtractApiMergeRequestDetailsMeta},
+ Dependencies: []*plugin.SubTaskMeta{&ConvertApiMergeRequestsMeta},
}
-func ConvertMrAssignees(taskCtx plugin.SubTaskContext) errors.Error {
- db := taskCtx.GetDal()
- rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_MERGE_REQUEST_TABLE)
+func ConvertMrAssignees(subtaskCtx plugin.SubTaskContext) errors.Error {
+ subtaskCommonArgs, data := CreateSubtaskCommonArgs(subtaskCtx, RAW_MERGE_REQUEST_TABLE)
+ db := subtaskCtx.GetDal()
projectId := data.Options.ProjectId
- clauses := []dal.Clause{
- dal.Select("_tool_gitlab_assignees.*"),
- dal.From(&models.GitlabAssignee{}),
- dal.Join(`left join _tool_gitlab_merge_requests mr on
- mr.gitlab_id = _tool_gitlab_assignees.merge_request_id`),
- dal.Where(`mr.project_id = ?
- and mr.connection_id = ?`,
- projectId, data.Options.ConnectionId),
- dal.Orderby("_tool_gitlab_assignees.merge_request_id ASC"),
- }
-
- cursor, err := db.Cursor(clauses...)
- if err != nil {
- return err
- }
- defer cursor.Close()
-
mrIdGen := didgen.NewDomainIdGenerator(&models.GitlabMergeRequest{})
accountIdGen := didgen.NewDomainIdGenerator(&models.GitlabAccount{})
- converter, err := helper.NewDataConverter(helper.DataConverterArgs{
- RawDataSubTaskArgs: *rawDataSubTaskArgs,
- InputRowType: reflect.TypeOf(models.GitlabAssignee{}),
- Input: cursor,
- Convert: func(inputRow interface{}) ([]interface{}, errors.Error) {
- mrAssignee := inputRow.(*models.GitlabAssignee)
+ converter, err := api.NewStatefulDataConverter(&api.StatefulDataConverterArgs[models.GitlabAssignee]{
+ SubtaskCommonArgs: subtaskCommonArgs,
+ Input: func(stateManager *api.SubtaskStateManager) (dal.Rows, errors.Error) {
+ clauses := []dal.Clause{
+ dal.From(&models.GitlabAssignee{}),
+ dal.Where(`project_id = ? and connection_id = ?`, projectId, data.Options.ConnectionId),
+ dal.Orderby("merge_request_id ASC"),
+ }
+ if stateManager.IsIncremental() {
+ since := stateManager.GetSince()
+ if since != nil {
+ clauses = append(clauses, dal.Where("updated_at >= ? ", since))
+ }
+ }
+ return db.Cursor(clauses...)
+ },
+ Convert: func(mrAssignee *models.GitlabAssignee) ([]interface{}, errors.Error) {
domainPrAssigne := &code.PullRequestAssignee{
PullRequestId: mrIdGen.Generate(data.Options.ConnectionId, mrAssignee.MergeRequestId),
AssigneeId: accountIdGen.Generate(data.Options.ConnectionId, mrAssignee.AssigneeId),
diff --git a/backend/plugins/gitlab/tasks/mr_commit_convertor.go b/backend/plugins/gitlab/tasks/mr_commit_convertor.go
index 74ae0d926..6f7ff2a5f 100644
--- a/backend/plugins/gitlab/tasks/mr_commit_convertor.go
+++ b/backend/plugins/gitlab/tasks/mr_commit_convertor.go
@@ -18,14 +18,12 @@ limitations under the License.
package tasks
import (
- "reflect"
-
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models/domainlayer/code"
"github.com/apache/incubator-devlake/core/models/domainlayer/didgen"
"github.com/apache/incubator-devlake/core/plugin"
- helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/gitlab/models"
)
@@ -39,45 +37,39 @@ var ConvertApiMrCommitsMeta = plugin.SubTaskMeta{
EnabledByDefault: true,
Description: "Add domain layer PullRequestCommit according to GitlabMrCommit",
DomainTypes: []string{plugin.DOMAIN_TYPE_CODE_REVIEW},
- Dependencies: []*plugin.SubTaskMeta{&ConvertMrCommentMeta},
+ Dependencies: []*plugin.SubTaskMeta{&ConvertApiMergeRequestsMeta},
}
-func ConvertApiMergeRequestsCommits(taskCtx plugin.SubTaskContext) errors.Error {
- rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_MERGE_REQUEST_COMMITS_TABLE)
- db := taskCtx.GetDal()
-
- clauses := []dal.Clause{
- dal.From(&models.GitlabMrCommit{}),
- dal.Join(`left join _tool_gitlab_merge_requests
- on _tool_gitlab_merge_requests.gitlab_id =
- _tool_gitlab_mr_commits.merge_request_id`),
- dal.Where(`_tool_gitlab_merge_requests.project_id = ?
- and _tool_gitlab_merge_requests.connection_id = ?`,
- data.Options.ProjectId, data.Options.ConnectionId),
- dal.Orderby("merge_request_id ASC"),
- }
+func ConvertApiMergeRequestsCommits(subtaskCtx plugin.SubTaskContext) errors.Error {
+ subtaskCommonArgs, data := CreateSubtaskCommonArgs(subtaskCtx, RAW_MERGE_REQUEST_COMMITS_TABLE)
+ db := subtaskCtx.GetDal()
- cursor, err := db.Cursor(clauses...)
- if err != nil {
- return err
- }
-
- // TODO: adopt batch indate operation
domainIdGenerator := didgen.NewDomainIdGenerator(&models.GitlabMergeRequest{})
- converter, err := helper.NewDataConverter(helper.DataConverterArgs{
- RawDataSubTaskArgs: *rawDataSubTaskArgs,
- InputRowType: reflect.TypeOf(models.GitlabMrCommit{}),
- Input: cursor,
-
- Convert: func(inputRow interface{}) ([]interface{}, errors.Error) {
- GitlabMrCommit := inputRow.(*models.GitlabMrCommit)
+ converter, err := api.NewStatefulDataConverter(&api.StatefulDataConverterArgs[models.GitlabMrCommit]{
+ SubtaskCommonArgs: subtaskCommonArgs,
+ Input: func(stateManager *api.SubtaskStateManager) (dal.Rows, errors.Error) {
+ clauses := []dal.Clause{
+ dal.Select("c.*"),
+ dal.From("_tool_gitlab_mr_commits c"),
+ dal.Join(`LEFT JOIN _tool_gitlab_merge_requests mr ON mr.gitlab_id = c.merge_request_id AND c.connection_id = mr.connection_id`),
+ dal.Where(`mr.project_id = ? and mr.connection_id = ?`, data.Options.ProjectId, data.Options.ConnectionId),
+ }
+ if stateManager.IsIncremental() {
+ since := stateManager.GetSince()
+ if since != nil {
+ clauses = append(clauses, dal.Where("c.updated_at >= ? ", since))
+ }
+ }
+ return db.Cursor(clauses...)
+ },
+ Convert: func(mrCommit *models.GitlabMrCommit) ([]interface{}, errors.Error) {
domainPrcommit := &code.PullRequestCommit{
- CommitSha: GitlabMrCommit.CommitSha,
- PullRequestId: domainIdGenerator.Generate(data.Options.ConnectionId, GitlabMrCommit.MergeRequestId),
- CommitAuthorName: GitlabMrCommit.CommitAuthorName,
- CommitAuthorEmail: GitlabMrCommit.CommitAuthorEmail,
- CommitAuthoredDate: *GitlabMrCommit.CommitAuthoredDate,
+ CommitSha: mrCommit.CommitSha,
+ PullRequestId: domainIdGenerator.Generate(data.Options.ConnectionId, mrCommit.MergeRequestId),
+ CommitAuthorName: mrCommit.CommitAuthorName,
+ CommitAuthorEmail: mrCommit.CommitAuthorEmail,
+ CommitAuthoredDate: *mrCommit.CommitAuthoredDate,
}
return []interface{}{
domainPrcommit,
diff --git a/backend/plugins/gitlab/tasks/mr_commit_extractor.go b/backend/plugins/gitlab/tasks/mr_commit_extractor.go
index 5c73c8665..7a94f9fee 100644
--- a/backend/plugins/gitlab/tasks/mr_commit_extractor.go
+++ b/backend/plugins/gitlab/tasks/mr_commit_extractor.go
@@ -40,18 +40,33 @@ var ExtractApiMrCommitsMeta = plugin.SubTaskMeta{
Dependencies: []*plugin.SubTaskMeta{&CollectApiMrCommitsMeta},
}
-func ExtractApiMergeRequestsCommits(taskCtx plugin.SubTaskContext) errors.Error {
- rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_MERGE_REQUEST_COMMITS_TABLE)
+type GitlabApiCommit struct {
+ GitlabId string `json:"id"`
+ Title string
+ Message string
+ ProjectId int
+ ShortId string `json:"short_id"`
+ AuthorName string `json:"author_name"`
+ AuthorEmail string `json:"author_email"`
+ AuthoredDate common.Iso8601Time `json:"authored_date"`
+ CommitterName string `json:"committer_name"`
+ CommitterEmail string `json:"committer_email"`
+ CommittedDate common.Iso8601Time `json:"committed_date"`
+ WebUrl string `json:"web_url"`
+ Stats struct {
+ Additions int
+ Deletions int
+ Total int
+ }
+}
+
+func ExtractApiMergeRequestsCommits(subtaskCtx plugin.SubTaskContext) errors.Error {
+ subtaskCommonArgs, data := CreateSubtaskCommonArgs(subtaskCtx, RAW_MERGE_REQUEST_COMMITS_TABLE)
- extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{
- RawDataSubTaskArgs: *rawDataSubTaskArgs,
- Extract: func(row *api.RawData) ([]interface{}, errors.Error) {
+ extractor, err := api.NewStatefulApiExtractor(&api.StatefulApiExtractorArgs[GitlabApiCommit]{
+ SubtaskCommonArgs: subtaskCommonArgs,
+ Extract: func(gitlabApiCommit *GitlabApiCommit, row *api.RawData) ([]interface{}, errors.Error) {
// create gitlab commit
- gitlabApiCommit := &GitlabApiCommit{}
- err := errors.Convert(json.Unmarshal(row.Data, gitlabApiCommit))
- if err != nil {
- return nil, err
- }
gitlabCommit, err := ConvertCommit(gitlabApiCommit)
if err != nil {
return nil, err
@@ -91,3 +106,24 @@ func ExtractApiMergeRequestsCommits(taskCtx plugin.SubTaskContext) errors.Error
return extractor.Execute()
}
+
+// Convert the API response to our DB model instance
+func ConvertCommit(commit *GitlabApiCommit) (*models.GitlabCommit, errors.Error) {
+ gitlabCommit := &models.GitlabCommit{
+ Sha: commit.GitlabId,
+ Title: commit.Title,
+ Message: commit.Message,
+ ShortId: commit.ShortId,
+ AuthorName: commit.AuthorName,
+ AuthorEmail: commit.AuthorEmail,
+ AuthoredDate: commit.AuthoredDate.ToTime(),
+ CommitterName: commit.CommitterName,
+ CommitterEmail: commit.CommitterEmail,
+ CommittedDate: commit.CommittedDate.ToTime(),
+ WebUrl: commit.WebUrl,
+ Additions: commit.Stats.Additions,
+ Deletions: commit.Stats.Deletions,
+ Total: commit.Stats.Total,
+ }
+ return gitlabCommit, nil
+}
diff --git a/backend/plugins/gitlab/tasks/mr_convertor.go b/backend/plugins/gitlab/tasks/mr_convertor.go
index 478276c71..f2dd24d74 100644
--- a/backend/plugins/gitlab/tasks/mr_convertor.go
+++ b/backend/plugins/gitlab/tasks/mr_convertor.go
@@ -18,15 +18,13 @@ limitations under the License.
package tasks
import (
- "reflect"
-
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models/domainlayer"
"github.com/apache/incubator-devlake/core/models/domainlayer/code"
"github.com/apache/incubator-devlake/core/models/domainlayer/didgen"
"github.com/apache/incubator-devlake/core/plugin"
- helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/gitlab/models"
)
@@ -43,32 +41,43 @@ var ConvertApiMergeRequestsMeta = plugin.SubTaskMeta{
Dependencies: []*plugin.SubTaskMeta{&ConvertProjectMeta},
}
-func ConvertApiMergeRequests(taskCtx plugin.SubTaskContext) errors.Error {
- rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_MERGE_REQUEST_TABLE)
- db := taskCtx.GetDal()
- clauses := []dal.Clause{
- dal.From(&models.GitlabMergeRequest{}),
- dal.Where("project_id=? and connection_id = ?", data.Options.ProjectId, data.Options.ConnectionId),
- }
-
- cursor, err := db.Cursor(clauses...)
- if err != nil {
- return err
- }
- defer cursor.Close()
+func ConvertApiMergeRequests(subtaskCtx plugin.SubTaskContext) errors.Error {
+ subtaskCommonArgs, data := CreateSubtaskCommonArgs(subtaskCtx, RAW_MERGE_REQUEST_TABLE)
+ db := subtaskCtx.GetDal()
domainMrIdGenerator := didgen.NewDomainIdGenerator(&models.GitlabMergeRequest{})
domainRepoIdGenerator := didgen.NewDomainIdGenerator(&models.GitlabProject{})
domainUserIdGen := didgen.NewDomainIdGenerator(&models.GitlabAccount{})
- converter, err := helper.NewDataConverter(helper.DataConverterArgs{
- RawDataSubTaskArgs: *rawDataSubTaskArgs,
- InputRowType: reflect.TypeOf(models.GitlabMergeRequest{}),
- Input: cursor,
-
- Convert: func(inputRow interface{}) ([]interface{}, errors.Error) {
- gitlabMr := inputRow.(*models.GitlabMergeRequest)
-
+ converter, err := api.NewStatefulDataConverter(&api.StatefulDataConverterArgs[models.GitlabMergeRequest]{
+ SubtaskCommonArgs: subtaskCommonArgs,
+ Input: func(stateManager *api.SubtaskStateManager) (dal.Rows, errors.Error) {
+ clauses := []dal.Clause{
+ dal.From(&models.GitlabMergeRequest{}),
+ dal.Where("project_id=? and connection_id = ?", data.Options.ProjectId, data.Options.ConnectionId),
+ }
+ if stateManager.IsIncremental() {
+ since := stateManager.GetSince()
+ if since != nil {
+ clauses = append(clauses, dal.Where("updated_at >= ? ", since))
+ }
+ }
+ return db.Cursor(clauses...)
+ },
+ BeforeConvert: func(gitlabMr *models.GitlabMergeRequest, stateManager *api.SubtaskStateManager) errors.Error {
+ mrId := domainMrIdGenerator.Generate(data.Options.ConnectionId, gitlabMr.GitlabId)
+ if err := db.Delete(&code.PullRequestAssignee{}, dal.Where("pull_request_id = ?", mrId)); err != nil {
+ return err
+ }
+ if err := db.Delete(&code.PullRequestReviewer{}, dal.Where("pull_request_id = ?", mrId)); err != nil {
+ return err
+ }
+ if err := db.Delete(&code.PullRequestLabel{}, dal.Where("pull_request_id = ?", mrId)); err != nil {
+ return err
+ }
+ return nil
+ },
+ Convert: func(gitlabMr *models.GitlabMergeRequest) ([]interface{}, errors.Error) {
domainPr := &code.PullRequest{
DomainEntity: domainlayer.DomainEntity{
Id: domainMrIdGenerator.Generate(data.Options.ConnectionId, gitlabMr.GitlabId),
diff --git a/backend/plugins/gitlab/tasks/mr_detail_extractor.go b/backend/plugins/gitlab/tasks/mr_detail_extractor.go
index 3dbc72e8a..394575246 100644
--- a/backend/plugins/gitlab/tasks/mr_detail_extractor.go
+++ b/backend/plugins/gitlab/tasks/mr_detail_extractor.go
@@ -18,7 +18,6 @@ limitations under the License.
package tasks
import (
- "encoding/json"
"regexp"
"github.com/apache/incubator-devlake/core/errors"
@@ -37,11 +36,12 @@ var ExtractApiMergeRequestDetailsMeta = plugin.SubTaskMeta{
EnabledByDefault: true,
Description: "Extract raw merge request Details data into tool layer table GitlabMergeRequest and GitlabReviewer",
DomainTypes: []string{plugin.DOMAIN_TYPE_CODE_REVIEW},
- Dependencies: []*plugin.SubTaskMeta{&ExtractApiCommitsMeta},
+ Dependencies: []*plugin.SubTaskMeta{&CollectApiMergeRequestDetailsMeta},
}
-func ExtractApiMergeRequestDetails(taskCtx plugin.SubTaskContext) errors.Error {
- rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_MERGE_REQUEST_DETAIL_TABLE)
+func ExtractApiMergeRequestDetails(subtaskCtx plugin.SubTaskContext) errors.Error {
+ subtaskCommonArgs, data := CreateSubtaskCommonArgs(subtaskCtx, RAW_MERGE_REQUEST_DETAIL_TABLE)
+ db := subtaskCtx.GetDal()
config := data.Options.ScopeConfig
var labelTypeRegex *regexp.Regexp
var labelComponentRegex *regexp.Regexp
@@ -60,15 +60,14 @@ func ExtractApiMergeRequestDetails(taskCtx plugin.SubTaskContext) errors.Error {
return errors.Default.Wrap(err, "regexp Compile prComponent failed")
}
}
- extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{
- RawDataSubTaskArgs: *rawDataSubTaskArgs,
- Extract: func(row *api.RawData) ([]interface{}, errors.Error) {
- mr := &MergeRequestRes{}
- err := errors.Convert(json.Unmarshal(row.Data, mr))
- if err != nil {
- return nil, err
- }
-
+ subtaskCommonArgs.SubtaskConfig = map[string]string{
+ "prType": prType,
+ "prComponent": prComponent,
+ }
+ extractor, err := api.NewStatefulApiExtractor(&api.StatefulApiExtractorArgs[MergeRequestRes]{
+ SubtaskCommonArgs: subtaskCommonArgs,
+ BeforeExtract: beforeExtractMr(db, data),
+ Extract: func(mr *MergeRequestRes, row *api.RawData) ([]interface{}, errors.Error) {
gitlabMergeRequest, err := convertMergeRequest(mr)
if err != nil {
return nil, err
diff --git a/backend/plugins/gitlab/tasks/mr_enricher.go b/backend/plugins/gitlab/tasks/mr_enricher.go
deleted file mode 100644
index 327765208..000000000
--- a/backend/plugins/gitlab/tasks/mr_enricher.go
+++ /dev/null
@@ -1,165 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package tasks
-
-import (
- "reflect"
- "time"
-
- "github.com/apache/incubator-devlake/core/dal"
- "github.com/apache/incubator-devlake/core/errors"
- "github.com/apache/incubator-devlake/core/plugin"
- helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
- "github.com/apache/incubator-devlake/plugins/gitlab/models"
-)
-
-func init() {
- RegisterSubtaskMeta(&EnrichMergeRequestsMeta)
-}
-
-var EnrichMergeRequestsMeta = plugin.SubTaskMeta{
- Name: "Enrich Merge Requests",
- EntryPoint: EnrichMergeRequests,
- EnabledByDefault: true,
- Description: "Enrich merge requests data from GitlabCommit, GitlabMrNote and GitlabMergeRequest",
- DomainTypes: []string{plugin.DOMAIN_TYPE_CODE_REVIEW},
- Dependencies: []*plugin.SubTaskMeta{&ExtractApiJobsMeta},
-}
-
-func EnrichMergeRequests(taskCtx plugin.SubTaskContext) errors.Error {
- rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_MERGE_REQUEST_TABLE)
-
- db := taskCtx.GetDal()
- clauses := []dal.Clause{
- dal.From(&models.GitlabMergeRequest{}),
- dal.Where("project_id=? and connection_id = ?", data.Options.ProjectId, data.Options.ConnectionId),
- }
-
- cursor, err := db.Cursor(clauses...)
- if err != nil {
- return err
- } // get mrs from theDB
- defer cursor.Close()
-
- converter, err := helper.NewDataConverter(helper.DataConverterArgs{
- RawDataSubTaskArgs: *rawDataSubTaskArgs,
- InputRowType: reflect.TypeOf(models.GitlabMergeRequest{}),
- Input: cursor,
-
- Convert: func(inputRow interface{}) ([]interface{}, errors.Error) {
- gitlabMr := inputRow.(*models.GitlabMergeRequest)
- // enrich first_comment_time field
- notes := make([]models.GitlabMrNote, 0)
- // `system` = 0 is needed since we only care about human comments
- noteClauses := []dal.Clause{
- dal.From(&models.GitlabMrNote{}),
- dal.Where("merge_request_id = ? AND is_system = ? AND connection_id = ? ",
- gitlabMr.GitlabId, false, data.Options.ConnectionId),
- dal.Orderby("gitlab_created_at asc"),
- }
- err = db.All(¬es, noteClauses...)
- if err != nil {
- return nil, err
- }
-
- commits := make([]models.GitlabCommit, 0)
- commitClauses := []dal.Clause{
- dal.From(&models.GitlabCommit{}),
- dal.Join(`join _tool_gitlab_mr_commits gmrc
- on gmrc.commit_sha = _tool_gitlab_commits.sha`),
- dal.Where("merge_request_id = ? AND gmrc.connection_id = ?",
- gitlabMr.GitlabId, data.Options.ConnectionId),
- dal.Orderby("authored_date asc"),
- }
- err = db.All(&commits, commitClauses...)
- if err != nil {
- return nil, err
- }
-
- // calculate reviewRounds from commits and notes
- reviewRounds := getReviewRounds(commits, notes)
- gitlabMr.ReviewRounds = reviewRounds
-
- if len(notes) > 0 {
- earliestNote, err := findEarliestNote(notes)
- if err != nil {
- return nil, err
- }
- if earliestNote != nil {
- gitlabMr.FirstCommentTime = &earliestNote.GitlabCreatedAt
- }
- }
- return []interface{}{
- gitlabMr,
- }, nil
- },
- })
- if err != nil {
- return err
- }
-
- return converter.Execute()
-}
-
-func findEarliestNote(notes []models.GitlabMrNote) (*models.GitlabMrNote, errors.Error) {
- var earliestNote *models.GitlabMrNote
- earliestTime := time.Now()
- for i := range notes {
- if !notes[i].Resolvable {
- continue
- }
- noteTime := notes[i].GitlabCreatedAt
- if noteTime.Before(earliestTime) {
- earliestTime = noteTime
- earliestNote = ¬es[i]
- }
- }
- return earliestNote, nil
-}
-
-func getReviewRounds(commits []models.GitlabCommit, notes []models.GitlabMrNote) int {
- i := 0
- j := 0
- reviewRounds := 0
- if len(commits) == 0 && len(notes) == 0 {
- return 1
- }
- // state is used to keep track of previous activity
- // 0: init, 1: commit, 2: comment
- // whenever state is switched to comment, we increment reviewRounds by 1
- state := 0 // 0, 1, 2
- for i < len(commits) && j < len(notes) {
- if commits[i].AuthoredDate.Before(notes[j].GitlabCreatedAt) {
- i++
- state = 1
- } else {
- j++
- if state != 2 {
- reviewRounds++
- }
- state = 2
- }
- }
- // There's another implicit round of review in 2 scenarios
- // One: the last state is commit (state == 1)
- // Two: the last state is comment but there're still commits left
- if state == 1 || i < len(commits) {
- reviewRounds++
- }
- return reviewRounds
-}
diff --git a/backend/plugins/gitlab/tasks/mr_enricher_test.go b/backend/plugins/gitlab/tasks/mr_enricher_test.go
deleted file mode 100644
index f22fd275a..000000000
--- a/backend/plugins/gitlab/tasks/mr_enricher_test.go
+++ /dev/null
@@ -1,112 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package tasks
-
-import (
- "github.com/apache/incubator-devlake/plugins/gitlab/models"
- "github.com/stretchr/testify/assert"
- "testing"
- "time"
-)
-
-func TestFindEarliestNote(t *testing.T) {
- baseTime, err := time.Parse(time.RFC3339, "2022-01-02T15:04:05Z")
- assert.Nil(t, err)
- // Create some sample notes
- note1 := models.GitlabMrNote{Resolvable: true, GitlabCreatedAt: baseTime.Add(-time.Hour)}
- note2 := models.GitlabMrNote{Resolvable: false, GitlabCreatedAt: baseTime.Add(-time.Minute)}
- note3 := models.GitlabMrNote{Resolvable: true, GitlabCreatedAt: baseTime.Add(-time.Second)}
-
- // Call the function with the sample notes
- notes := []models.GitlabMrNote{note1, note2, note3}
- earliestNote, err := findEarliestNote(notes)
-
- // Check that no error was returned
- if err != nil {
- t.Errorf("findEarliestNote returned an error: %v", err)
- }
-
- // Check that the correct note was returned
- if earliestNote == nil {
- t.Errorf("findEarliestNote returned nil, expected a note")
- }
- if !assert.Equal(t, note1, *earliestNote) {
- t.Errorf("findEarliestNote returned the wrong note: got %v, expected %v", earliestNote, ¬e1)
- }
-
- // Modify one of the notes to make it unresolvable
- notes[0].Resolvable = false
-
- // Call the function again with the modified notes
- earliestNote, err = findEarliestNote(notes)
-
- // Check that no error was returned
- if err != nil {
- t.Errorf("findEarliestNote returned an error: %v", err)
- }
-
- // Check that the correct note was returned
- if earliestNote == nil {
- t.Errorf("findEarliestNote returned nil, expected a note")
- }
- if !assert.Equal(t, note3, *earliestNote) {
- t.Errorf("findEarliestNote returned the wrong note: got %v, expected %v", earliestNote, ¬e3)
- }
-}
-
-func TestGetReviewRounds(t *testing.T) {
- baseTime, err := time.Parse(time.RFC3339, "2022-01-02T15:04:05Z")
- assert.Nil(t, err)
- // Test case 1: empty input
- var commits []models.GitlabCommit
- var notes []models.GitlabMrNote
- expected := 1
- if got := getReviewRounds(commits, notes); got != expected {
- t.Errorf("getReviewRounds(%v, %v) = %d, expected %d", commits, notes, got, expected)
- }
-
- // Test case 2: single comment
- commits = []models.GitlabCommit{
- {AuthoredDate: baseTime.Add(-time.Hour * 2)},
- {AuthoredDate: baseTime.Add(-time.Hour)},
- }
- notes = []models.GitlabMrNote{
- {GitlabCreatedAt: baseTime},
- }
- expected = 1
- if got := getReviewRounds(commits, notes); got != expected {
- t.Errorf("getReviewRounds(%v, %v) = %d, expected %d", commits, notes, got, expected)
- }
-
- // Test case 3: multiple comments
- commits = []models.GitlabCommit{
- {AuthoredDate: baseTime.Add(-time.Hour * 15)},
- {AuthoredDate: baseTime.Add(-time.Hour * 9)},
- {AuthoredDate: baseTime.Add(-time.Hour * 3)},
- {AuthoredDate: baseTime},
- }
- notes = []models.GitlabMrNote{
- {GitlabCreatedAt: baseTime.Add(-time.Hour * 14)},
- {GitlabCreatedAt: baseTime.Add(-time.Hour * 7)},
- {GitlabCreatedAt: baseTime.Add(-time.Hour * 2)},
- }
- expected = 4
- if got := getReviewRounds(commits, notes); got != expected {
- t.Errorf("getReviewRounds(%v, %v) = %d, expected %d", commits, notes, got, expected)
- }
-}
diff --git a/backend/plugins/gitlab/tasks/mr_extractor.go b/backend/plugins/gitlab/tasks/mr_extractor.go
index 76253833b..3a8457b0c 100644
--- a/backend/plugins/gitlab/tasks/mr_extractor.go
+++ b/backend/plugins/gitlab/tasks/mr_extractor.go
@@ -18,10 +18,10 @@ limitations under the License.
package tasks
import (
- "encoding/json"
"regexp"
"strings"
+ "github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models/common"
"github.com/apache/incubator-devlake/core/plugin"
@@ -96,8 +96,10 @@ var ExtractApiMergeRequestsMeta = plugin.SubTaskMeta{
Dependencies: []*plugin.SubTaskMeta{&CollectApiMergeRequestsMeta},
}
-func ExtractApiMergeRequests(taskCtx plugin.SubTaskContext) errors.Error {
- rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_MERGE_REQUEST_TABLE)
+func ExtractApiMergeRequests(subtaskCtx plugin.SubTaskContext) errors.Error {
+ subtaskCommonArgs, data := CreateSubtaskCommonArgs(subtaskCtx, RAW_MERGE_REQUEST_TABLE)
+
+ db := subtaskCtx.GetDal()
config := data.Options.ScopeConfig
var labelTypeRegex *regexp.Regexp
var labelComponentRegex *regexp.Regexp
@@ -118,17 +120,15 @@ func ExtractApiMergeRequests(taskCtx plugin.SubTaskContext) errors.Error {
}
}
- extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{
- RawDataSubTaskArgs: *rawDataSubTaskArgs,
- Extract: func(row *api.RawData) ([]interface{}, errors.Error) {
-
- mr := &MergeRequestRes{}
- s := string(row.Data)
- err := errors.Convert(json.Unmarshal(row.Data, mr))
- if err != nil {
- return nil, err
- }
+ subtaskCommonArgs.SubtaskConfig = map[string]any{
+ "prType": prType,
+ "prComponent": prComponent,
+ }
+ extractor, err := api.NewStatefulApiExtractor(&api.StatefulApiExtractorArgs[MergeRequestRes]{
+ SubtaskCommonArgs: subtaskCommonArgs,
+ BeforeExtract: beforeExtractMr(db, data),
+ Extract: func(mr *MergeRequestRes, row *api.RawData) ([]interface{}, errors.Error) {
gitlabMergeRequest, err := convertMergeRequest(mr)
if err != nil {
return nil, err
@@ -136,15 +136,23 @@ func ExtractApiMergeRequests(taskCtx plugin.SubTaskContext) errors.Error {
// if we can not find merged_at and closed_at info in the detail
// we need get detail for gitlab v11
+ s := string(row.Data)
if !strings.Contains(s, "\"merged_at\":") {
if !strings.Contains(s, "\"closed_at\":") {
gitlabMergeRequest.IsDetailRequired = true
}
}
- results := make([]interface{}, 0, len(mr.Reviewers)+1)
+ results := make([]interface{}, 0, len(mr.Reviewers)+len(mr.Labels)+1)
gitlabMergeRequest.ConnectionId = data.Options.ConnectionId
results = append(results, gitlabMergeRequest)
+ err = db.Delete(
+ &models.GitlabMrLabel{},
+ dal.Where("connection_id = ? AND mr_id = ?", data.Options.ConnectionId, gitlabMergeRequest.GitlabId),
+ )
+ if err != nil {
+ return nil, err
+ }
for _, label := range mr.Labels {
results = append(results, &models.GitlabMrLabel{
MrId: gitlabMergeRequest.GitlabId,
@@ -160,6 +168,13 @@ func ExtractApiMergeRequests(taskCtx plugin.SubTaskContext) errors.Error {
gitlabMergeRequest.Component = label
}
}
+ err = db.Delete(
+ &models.GitlabReviewer{},
+ dal.Where("connection_id = ? AND merge_request_id = ?", data.Options.ConnectionId, gitlabMergeRequest.GitlabId),
+ )
+ if err != nil {
+ return nil, err
+ }
for _, reviewer := range mr.Reviewers {
gitlabReviewer := &models.GitlabReviewer{
ConnectionId: data.Options.ConnectionId,
@@ -174,6 +189,13 @@ func ExtractApiMergeRequests(taskCtx plugin.SubTaskContext) errors.Error {
}
results = append(results, gitlabReviewer)
}
+ err = db.Delete(
+ &models.GitlabAssignee{},
+ dal.Where("connection_id = ? AND merge_request_id = ?", data.Options.ConnectionId, gitlabMergeRequest.GitlabId),
+ )
+ if err != nil {
+ return nil, err
+ }
for _, assignee := range mr.Assignees {
gitlabAssignee := &models.GitlabAssignee{
ConnectionId: data.Options.ConnectionId,
@@ -188,17 +210,13 @@ func ExtractApiMergeRequests(taskCtx plugin.SubTaskContext) errors.Error {
}
results = append(results, gitlabAssignee)
}
-
return results, nil
},
})
-
if err != nil {
- return errors.Convert(err)
+ return err
}
-
return extractor.Execute()
-
}
func convertMergeRequest(mr *MergeRequestRes) (*models.GitlabMergeRequest, errors.Error) {
@@ -230,3 +248,25 @@ func convertMergeRequest(mr *MergeRequestRes) (*models.GitlabMergeRequest, error
}
return gitlabMergeRequest, nil
}
+
+func beforeExtractMr(db dal.Dal, data *GitlabTaskData) func(mr *MergeRequestRes, stateManager *api.SubtaskStateManager) errors.Error {
+ return func(mr *MergeRequestRes, stateManager *api.SubtaskStateManager) errors.Error {
+ if stateManager.IsIncremental() {
+ err := db.Delete(
+ &models.GitlabMrLabel{},
+ dal.Where("connection_id = ? AND mr_id = ?", data.Options.ConnectionId, mr.GitlabId),
+ )
+ if err != nil {
+ return err
+ }
+ err = db.Delete(
+ &models.GitlabAssignee{},
+ dal.Where("connection_id = ? AND merge_request_id = ?", data.Options.ConnectionId, mr.GitlabId),
+ )
+ if err != nil {
+ return err
+ }
+ }
+ return nil
+ }
+}
diff --git a/backend/plugins/gitlab/tasks/mr_label_convertor.go b/backend/plugins/gitlab/tasks/mr_label_convertor.go
index 934085e81..b5f29a1a8 100644
--- a/backend/plugins/gitlab/tasks/mr_label_convertor.go
+++ b/backend/plugins/gitlab/tasks/mr_label_convertor.go
@@ -18,14 +18,12 @@ limitations under the License.
package tasks
import (
- "reflect"
-
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models/domainlayer/code"
"github.com/apache/incubator-devlake/core/models/domainlayer/didgen"
"github.com/apache/incubator-devlake/core/plugin"
- helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/gitlab/models"
)
@@ -39,38 +37,33 @@ var ConvertMrLabelsMeta = plugin.SubTaskMeta{
EnabledByDefault: true,
Description: "Convert tool layer table gitlab_mr_labels into domain layer table pull_request_labels",
DomainTypes: []string{plugin.DOMAIN_TYPE_CODE_REVIEW},
- Dependencies: []*plugin.SubTaskMeta{&ConvertIssueLabelsMeta},
+ Dependencies: []*plugin.SubTaskMeta{&ConvertApiMergeRequestsMeta},
}
-func ConvertMrLabels(taskCtx plugin.SubTaskContext) errors.Error {
- db := taskCtx.GetDal()
- rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_MERGE_REQUEST_TABLE)
- projectId := data.Options.ProjectId
- clauses := []dal.Clause{
- dal.Select("*"),
- dal.From(&models.GitlabMrLabel{}),
- dal.Join(`left join _tool_gitlab_merge_requests on
- _tool_gitlab_merge_requests.gitlab_id = _tool_gitlab_mr_labels.mr_id`),
- dal.Where(`_tool_gitlab_merge_requests.project_id = ?
- and _tool_gitlab_merge_requests.connection_id = ?`,
- projectId, data.Options.ConnectionId),
- dal.Orderby("mr_id ASC"),
- }
-
- cursor, err := db.Cursor(clauses...)
- if err != nil {
- return err
- }
- defer cursor.Close()
+func ConvertMrLabels(subtaskCtx plugin.SubTaskContext) errors.Error {
+ subtaskCommonArgs, data := CreateSubtaskCommonArgs(subtaskCtx, RAW_MERGE_REQUEST_TABLE)
+ db := subtaskCtx.GetDal()
mrIdGen := didgen.NewDomainIdGenerator(&models.GitlabMergeRequest{})
- converter, err := helper.NewDataConverter(helper.DataConverterArgs{
- RawDataSubTaskArgs: *rawDataSubTaskArgs,
- InputRowType: reflect.TypeOf(models.GitlabMrLabel{}),
- Input: cursor,
- Convert: func(inputRow interface{}) ([]interface{}, errors.Error) {
- mrLabel := inputRow.(*models.GitlabMrLabel)
+ converter, err := api.NewStatefulDataConverter(&api.StatefulDataConverterArgs[models.GitlabMrLabel]{
+ SubtaskCommonArgs: subtaskCommonArgs,
+ Input: func(stateManager *api.SubtaskStateManager) (dal.Rows, errors.Error) {
+ clauses := []dal.Clause{
+ dal.Select("c.*"),
+ dal.From("_tool_gitlab_mr_labels c"),
+ dal.Join(`LEFT JOIN _tool_gitlab_merge_requests mr ON mr.gitlab_id = c.mr_id AND c.connection_id = mr.connection_id`),
+ dal.Where(`mr.project_id = ? and mr.connection_id = ?`, data.Options.ProjectId, data.Options.ConnectionId),
+ }
+ if stateManager.IsIncremental() {
+ since := stateManager.GetSince()
+ if since != nil {
+ clauses = append(clauses, dal.Where("c.updated_at >= ? ", since))
+ }
+ }
+ return db.Cursor(clauses...)
+ },
+ Convert: func(mrLabel *models.GitlabMrLabel) ([]interface{}, errors.Error) {
domainIssueLabel := &code.PullRequestLabel{
PullRequestId: mrIdGen.Generate(data.Options.ConnectionId, mrLabel.MrId),
LabelName: mrLabel.LabelName,
diff --git a/backend/plugins/gitlab/tasks/mr_comment_convertor.go b/backend/plugins/gitlab/tasks/mr_note_convertor.go
similarity index 70%
rename from backend/plugins/gitlab/tasks/mr_comment_convertor.go
rename to backend/plugins/gitlab/tasks/mr_note_convertor.go
index 3b6d1ccee..a55cc5673 100644
--- a/backend/plugins/gitlab/tasks/mr_comment_convertor.go
+++ b/backend/plugins/gitlab/tasks/mr_note_convertor.go
@@ -18,15 +18,13 @@ limitations under the License.
package tasks
import (
- "reflect"
-
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models/domainlayer"
"github.com/apache/incubator-devlake/core/models/domainlayer/code"
"github.com/apache/incubator-devlake/core/models/domainlayer/didgen"
"github.com/apache/incubator-devlake/core/plugin"
- helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/gitlab/models"
)
@@ -36,44 +34,39 @@ func init() {
var ConvertMrCommentMeta = plugin.SubTaskMeta{
Name: "Convert MR Comments",
- EntryPoint: ConvertMergeRequestComment,
+ EntryPoint: ConvertMergeRequestNote,
EnabledByDefault: true,
Description: "Add domain layer Comment according to GitlabMrComment",
DomainTypes: []string{plugin.DOMAIN_TYPE_CODE_REVIEW},
- Dependencies: []*plugin.SubTaskMeta{&ConvertApiMergeRequestsMeta},
+ Dependencies: []*plugin.SubTaskMeta{&ConvertApiMergeRequestsMeta, &ExtractApiMrNotesMeta},
}
-func ConvertMergeRequestComment(taskCtx plugin.SubTaskContext) errors.Error {
- rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_MERGE_REQUEST_NOTES_TABLE)
- db := taskCtx.GetDal()
- clauses := []dal.Clause{
- dal.From(&models.GitlabMrComment{}),
- dal.Join(`left join _tool_gitlab_merge_requests on
- _tool_gitlab_merge_requests.gitlab_id =
- _tool_gitlab_mr_comments.merge_request_id`),
- dal.Where(`_tool_gitlab_merge_requests.project_id = ?
- and _tool_gitlab_mr_comments.connection_id = ?`,
- data.Options.ProjectId, data.Options.ConnectionId),
- }
-
- cursor, err := db.Cursor(clauses...)
- if err != nil {
- return err
- }
- defer cursor.Close()
+func ConvertMergeRequestNote(subtaskCtx plugin.SubTaskContext) errors.Error {
+ subtaskCommonArgs, data := CreateSubtaskCommonArgs(subtaskCtx, RAW_MERGE_REQUEST_NOTES_TABLE)
+ db := subtaskCtx.GetDal()
domainIdGeneratorComment := didgen.NewDomainIdGenerator(&models.GitlabMrComment{})
prIdGen := didgen.NewDomainIdGenerator(&models.GitlabMergeRequest{})
accountIdGen := didgen.NewDomainIdGenerator(&models.GitlabAccount{})
- converter, err := helper.NewDataConverter(helper.DataConverterArgs{
- RawDataSubTaskArgs: *rawDataSubTaskArgs,
- InputRowType: reflect.TypeOf(models.GitlabMrComment{}),
- Input: cursor,
-
- Convert: func(inputRow interface{}) ([]interface{}, errors.Error) {
- gitlabComments := inputRow.(*models.GitlabMrComment)
-
+ converter, err := api.NewStatefulDataConverter(&api.StatefulDataConverterArgs[models.GitlabMrComment]{
+ SubtaskCommonArgs: subtaskCommonArgs,
+ Input: func(stateManager *api.SubtaskStateManager) (dal.Rows, errors.Error) {
+ clauses := []dal.Clause{
+ dal.Select("c.*"),
+ dal.From("_tool_gitlab_mr_comments c"),
+ dal.Join(`LEFT JOIN _tool_gitlab_merge_requests mr ON mr.gitlab_id = c.merge_request_id AND c.connection_id = mr.connection_id`),
+ dal.Where(`mr.project_id = ? and mr.connection_id = ?`, data.Options.ProjectId, data.Options.ConnectionId),
+ }
+ if stateManager.IsIncremental() {
+ since := stateManager.GetSince()
+ if since != nil {
+ clauses = append(clauses, dal.Where("c.updated_at >= ? ", since))
+ }
+ }
+ return db.Cursor(clauses...)
+ },
+ Convert: func(gitlabComments *models.GitlabMrComment) ([]interface{}, errors.Error) {
domainComment := &code.PullRequestComment{
DomainEntity: domainlayer.DomainEntity{
Id: domainIdGeneratorComment.Generate(data.Options.ConnectionId, gitlabComments.GitlabId),
diff --git a/backend/plugins/gitlab/tasks/mr_note_extractor.go b/backend/plugins/gitlab/tasks/mr_note_extractor.go
index a27d1141c..725a2fe48 100644
--- a/backend/plugins/gitlab/tasks/mr_note_extractor.go
+++ b/backend/plugins/gitlab/tasks/mr_note_extractor.go
@@ -18,8 +18,6 @@ limitations under the License.
package tasks
import (
- "encoding/json"
-
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models/common"
"github.com/apache/incubator-devlake/core/plugin"
@@ -57,18 +55,12 @@ var ExtractApiMrNotesMeta = plugin.SubTaskMeta{
Dependencies: []*plugin.SubTaskMeta{&CollectApiMrNotesMeta},
}
-func ExtractApiMergeRequestsNotes(taskCtx plugin.SubTaskContext) errors.Error {
- rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_MERGE_REQUEST_NOTES_TABLE)
-
- extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{
- RawDataSubTaskArgs: *rawDataSubTaskArgs,
- Extract: func(row *api.RawData) ([]interface{}, errors.Error) {
- mrNote := &MergeRequestNote{}
- err := errors.Convert(json.Unmarshal(row.Data, mrNote))
- if err != nil {
- return nil, err
- }
+func ExtractApiMergeRequestsNotes(subtaskCtx plugin.SubTaskContext) errors.Error {
+ subtaskCommonArgs, data := CreateSubtaskCommonArgs(subtaskCtx, RAW_MERGE_REQUEST_NOTES_TABLE)
+ extractor, err := api.NewStatefulApiExtractor(&api.StatefulApiExtractorArgs[MergeRequestNote]{
+ SubtaskCommonArgs: subtaskCommonArgs,
+ Extract: func(mrNote *MergeRequestNote, row *api.RawData) ([]interface{}, errors.Error) {
toolMrNote, err := convertMergeRequestNote(mrNote)
toolMrNote.ConnectionId = data.Options.ConnectionId
if err != nil {
diff --git a/backend/plugins/gitlab/tasks/mr_reviewer_convertor.go b/backend/plugins/gitlab/tasks/mr_reviewer_convertor.go
index b702de573..f9298b093 100644
--- a/backend/plugins/gitlab/tasks/mr_reviewer_convertor.go
+++ b/backend/plugins/gitlab/tasks/mr_reviewer_convertor.go
@@ -18,14 +18,12 @@ limitations under the License.
package tasks
import (
- "reflect"
-
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models/domainlayer/code"
"github.com/apache/incubator-devlake/core/models/domainlayer/didgen"
"github.com/apache/incubator-devlake/core/plugin"
- helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/gitlab/models"
)
@@ -39,39 +37,34 @@ var ConvertMrReviewersMeta = plugin.SubTaskMeta{
EnabledByDefault: true,
Description: "Convert tool layer table gitlab_reviewers into domain layer table pull_request_reviewers",
DomainTypes: []string{plugin.DOMAIN_TYPE_CODE_REVIEW},
- Dependencies: []*plugin.SubTaskMeta{&ExtractApiMergeRequestDetailsMeta},
+ Dependencies: []*plugin.SubTaskMeta{&ConvertApiMergeRequestsMeta},
}
-func ConvertMrReviewers(taskCtx plugin.SubTaskContext) errors.Error {
- db := taskCtx.GetDal()
- rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_MERGE_REQUEST_TABLE)
- projectId := data.Options.ProjectId
- clauses := []dal.Clause{
- dal.Select("_tool_gitlab_reviewers.*"),
- dal.From(&models.GitlabReviewer{}),
- dal.Join(`left join _tool_gitlab_merge_requests mr on
- mr.gitlab_id = _tool_gitlab_reviewers.merge_request_id`),
- dal.Where(`mr.project_id = ?
- and mr.connection_id = ?`,
- projectId, data.Options.ConnectionId),
- dal.Orderby("_tool_gitlab_reviewers.merge_request_id ASC"),
- }
-
- cursor, err := db.Cursor(clauses...)
- if err != nil {
- return err
- }
- defer cursor.Close()
+func ConvertMrReviewers(subtaskCtx plugin.SubTaskContext) errors.Error {
+ subtaskCommonArgs, data := CreateSubtaskCommonArgs(subtaskCtx, RAW_MERGE_REQUEST_TABLE)
+ db := subtaskCtx.GetDal()
mrIdGen := didgen.NewDomainIdGenerator(&models.GitlabMergeRequest{})
accountIdGen := didgen.NewDomainIdGenerator(&models.GitlabAccount{})
- converter, err := helper.NewDataConverter(helper.DataConverterArgs{
- RawDataSubTaskArgs: *rawDataSubTaskArgs,
- InputRowType: reflect.TypeOf(models.GitlabReviewer{}),
- Input: cursor,
- Convert: func(inputRow interface{}) ([]interface{}, errors.Error) {
- mrReviewer := inputRow.(*models.GitlabReviewer)
+ converter, err := api.NewStatefulDataConverter(&api.StatefulDataConverterArgs[models.GitlabReviewer]{
+ SubtaskCommonArgs: subtaskCommonArgs,
+ Input: func(stateManager *api.SubtaskStateManager) (dal.Rows, errors.Error) {
+ clauses := []dal.Clause{
+ dal.Select("c.*"),
+ dal.From("_tool_gitlab_reviewers c"),
+ dal.Join(`LEFT JOIN _tool_gitlab_merge_requests mr ON mr.gitlab_id = c.merge_request_id AND c.connection_id = mr.connection_id`),
+ dal.Where(`mr.project_id = ? and mr.connection_id = ?`, data.Options.ProjectId, data.Options.ConnectionId),
+ }
+ if stateManager.IsIncremental() {
+ since := stateManager.GetSince()
+ if since != nil {
+ clauses = append(clauses, dal.Where("c.updated_at >= ? ", since))
+ }
+ }
+ return db.Cursor(clauses...)
+ },
+ Convert: func(mrReviewer *models.GitlabReviewer) ([]interface{}, errors.Error) {
domainPrReviewer := &code.PullRequestReviewer{
PullRequestId: mrIdGen.Generate(data.Options.ConnectionId, mrReviewer.MergeRequestId),
ReviewerId: accountIdGen.Generate(data.Options.ConnectionId, mrReviewer.ReviewerId),
diff --git a/backend/plugins/gitlab/tasks/pipeline_commit_convertor.go b/backend/plugins/gitlab/tasks/pipeline_commit_convertor.go
index 4984270d9..2064cc6c9 100644
--- a/backend/plugins/gitlab/tasks/pipeline_commit_convertor.go
+++ b/backend/plugins/gitlab/tasks/pipeline_commit_convertor.go
@@ -18,14 +18,12 @@ limitations under the License.
package tasks
import (
- "reflect"
-
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models/domainlayer/devops"
"github.com/apache/incubator-devlake/core/models/domainlayer/didgen"
"github.com/apache/incubator-devlake/core/plugin"
- helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/gitlab/models"
)
@@ -42,9 +40,9 @@ var ConvertPipelineCommitMeta = plugin.SubTaskMeta{
Dependencies: []*plugin.SubTaskMeta{&ConvertDetailPipelineMeta},
}
-func ConvertPipelineCommits(taskCtx plugin.SubTaskContext) errors.Error {
- db := taskCtx.GetDal()
- data := taskCtx.GetData().(*GitlabTaskData)
+func ConvertPipelineCommits(subtaskCtx plugin.SubTaskContext) errors.Error {
+ subtaskCommonArgs, data := CreateSubtaskCommonArgs(subtaskCtx, RAW_PIPELINE_TABLE)
+ db := subtaskCtx.GetDal()
repo := &models.GitlabProject{}
err := db.First(repo, dal.Where("gitlab_id = ? and connection_id = ?", data.Options.ProjectId, data.Options.ConnectionId))
@@ -52,29 +50,24 @@ func ConvertPipelineCommits(taskCtx plugin.SubTaskContext) errors.Error {
return err
}
- cursor, err := db.Cursor(dal.From(models.GitlabPipelineProject{}),
- dal.Where("project_id = ? and connection_id = ?", data.Options.ProjectId, data.Options.ConnectionId))
- if err != nil {
- return err
- }
- defer cursor.Close()
-
pipelineIdGen := didgen.NewDomainIdGenerator(&models.GitlabPipeline{})
- converter, err := helper.NewDataConverter(helper.DataConverterArgs{
- InputRowType: reflect.TypeOf(models.GitlabPipelineProject{}),
- Input: cursor,
- RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
- Ctx: taskCtx,
- Params: models.GitlabApiParams{
- ConnectionId: data.Options.ConnectionId,
- ProjectId: data.Options.ProjectId,
- },
- Table: RAW_PIPELINE_TABLE,
+ converter, err := api.NewStatefulDataConverter(&api.StatefulDataConverterArgs[models.GitlabPipelineProject]{
+ SubtaskCommonArgs: subtaskCommonArgs,
+ Input: func(stateManager *api.SubtaskStateManager) (dal.Rows, errors.Error) {
+ clauses := []dal.Clause{
+ dal.From(models.GitlabPipelineProject{}),
+ dal.Where("project_id = ? and connection_id = ?", data.Options.ProjectId, data.Options.ConnectionId),
+ }
+ if stateManager.IsIncremental() {
+ since := stateManager.GetSince()
+ if since != nil {
+ clauses = append(clauses, dal.Where("updated_at >= ? ", since))
+ }
+ }
+ return db.Cursor(clauses...)
},
- Convert: func(inputRow interface{}) ([]interface{}, errors.Error) {
- gitlabPipelineCommit := inputRow.(*models.GitlabPipelineProject)
-
+ Convert: func(gitlabPipelineCommit *models.GitlabPipelineProject) ([]interface{}, errors.Error) {
domainPipelineCommit := &devops.CiCDPipelineCommit{
PipelineId: pipelineIdGen.Generate(data.Options.ConnectionId, gitlabPipelineCommit.PipelineId),
CommitSha: gitlabPipelineCommit.Sha,
diff --git a/backend/plugins/gitlab/tasks/pipeline_detail_convertor.go b/backend/plugins/gitlab/tasks/pipeline_detail_convertor.go
index aba33b0e4..e808d7822 100644
--- a/backend/plugins/gitlab/tasks/pipeline_detail_convertor.go
+++ b/backend/plugins/gitlab/tasks/pipeline_detail_convertor.go
@@ -18,17 +18,16 @@ limitations under the License.
package tasks
import (
- "reflect"
"time"
"github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/models/domainlayer"
"github.com/apache/incubator-devlake/core/models/domainlayer/devops"
"github.com/apache/incubator-devlake/core/models/domainlayer/didgen"
"github.com/apache/incubator-devlake/core/plugin"
- helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/gitlab/models"
)
@@ -42,37 +41,33 @@ var ConvertDetailPipelineMeta = plugin.SubTaskMeta{
EnabledByDefault: true,
Description: "Convert tool layer table gitlab_detail_pipeline into domain layer table pipeline",
DomainTypes: []string{plugin.DOMAIN_TYPE_CICD},
- Dependencies: []*plugin.SubTaskMeta{&ConvertCommitsMeta},
+ Dependencies: []*plugin.SubTaskMeta{&ExtractApiPipelineDetailsMeta},
}
-func ConvertDetailPipelines(taskCtx plugin.SubTaskContext) errors.Error {
- db := taskCtx.GetDal()
- data := taskCtx.GetData().(*GitlabTaskData)
+func ConvertDetailPipelines(subtaskCtx plugin.SubTaskContext) errors.Error {
+ subtaskCommonArgs, data := CreateSubtaskCommonArgs(subtaskCtx, RAW_PIPELINE_DETAILS_TABLE)
- cursor, err := db.Cursor(dal.From(models.GitlabPipeline{}),
- dal.Where("project_id = ? and connection_id = ?", data.Options.ProjectId, data.Options.ConnectionId))
- if err != nil {
- return err
- }
- defer cursor.Close()
+ db := subtaskCtx.GetDal()
pipelineIdGen := didgen.NewDomainIdGenerator(&models.GitlabPipeline{})
projectIdGen := didgen.NewDomainIdGenerator(&models.GitlabProject{})
- converter, err := helper.NewDataConverter(helper.DataConverterArgs{
- InputRowType: reflect.TypeOf(models.GitlabPipeline{}),
- Input: cursor,
- RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
- Ctx: taskCtx,
- Params: models.GitlabApiParams{
- ConnectionId: data.Options.ConnectionId,
- ProjectId: data.Options.ProjectId,
- },
- Table: RAW_PIPELINE_DETAILS_TABLE,
+ converter, err := api.NewStatefulDataConverter(&api.StatefulDataConverterArgs[models.GitlabPipeline]{
+ SubtaskCommonArgs: subtaskCommonArgs,
+ Input: func(stateManager *api.SubtaskStateManager) (dal.Rows, errors.Error) {
+ clauses := []dal.Clause{
+ dal.From(models.GitlabPipeline{}),
+ dal.Where("project_id = ? and connection_id = ?", data.Options.ProjectId, data.Options.ConnectionId),
+ }
+ if stateManager.IsIncremental() {
+ since := stateManager.GetSince()
+ if since != nil {
+ clauses = append(clauses, dal.Where("updated_at >= ? ", since))
+ }
+ }
+ return db.Cursor(clauses...)
},
- Convert: func(inputRow interface{}) ([]interface{}, errors.Error) {
- gitlabPipeline := inputRow.(*models.GitlabPipeline)
-
+ Convert: func(gitlabPipeline *models.GitlabPipeline) ([]interface{}, errors.Error) {
createdAt := time.Now()
if gitlabPipeline.GitlabCreatedAt != nil {
createdAt = *gitlabPipeline.GitlabCreatedAt
diff --git a/backend/plugins/gitlab/tasks/pipeline_detail_extractor.go b/backend/plugins/gitlab/tasks/pipeline_detail_extractor.go
index 1b78362a1..3fef05109 100644
--- a/backend/plugins/gitlab/tasks/pipeline_detail_extractor.go
+++ b/backend/plugins/gitlab/tasks/pipeline_detail_extractor.go
@@ -18,8 +18,6 @@ limitations under the License.
package tasks
import (
- "encoding/json"
-
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models/common"
"github.com/apache/incubator-devlake/core/models/domainlayer/devops"
@@ -41,18 +39,12 @@ var ExtractApiPipelineDetailsMeta = plugin.SubTaskMeta{
Dependencies: []*plugin.SubTaskMeta{&CollectApiPipelineDetailsMeta},
}
-func ExtractApiPipelineDetails(taskCtx plugin.SubTaskContext) errors.Error {
- rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_PIPELINE_DETAILS_TABLE)
-
- extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{
- RawDataSubTaskArgs: *rawDataSubTaskArgs,
- Extract: func(row *api.RawData) ([]interface{}, errors.Error) {
- gitlabApiPipeline := &ApiPipeline{}
- err := errors.Convert(json.Unmarshal(row.Data, gitlabApiPipeline))
- if err != nil {
- return nil, err
- }
+func ExtractApiPipelineDetails(subtaskCtx plugin.SubTaskContext) errors.Error {
+ subtaskCommonArgs, data := CreateSubtaskCommonArgs(subtaskCtx, RAW_PIPELINE_DETAILS_TABLE)
+ extractor, err := api.NewStatefulApiExtractor(&api.StatefulApiExtractorArgs[ApiPipeline]{
+ SubtaskCommonArgs: subtaskCommonArgs,
+ Extract: func(gitlabApiPipeline *ApiPipeline, row *api.RawData) ([]interface{}, errors.Error) {
gitlabPipeline := &models.GitlabPipeline{
GitlabId: gitlabApiPipeline.Id,
ProjectId: data.Options.ProjectId,
diff --git a/backend/plugins/gitlab/tasks/pipeline_extractor.go b/backend/plugins/gitlab/tasks/pipeline_extractor.go
index d68035db1..0eb2450a5 100644
--- a/backend/plugins/gitlab/tasks/pipeline_extractor.go
+++ b/backend/plugins/gitlab/tasks/pipeline_extractor.go
@@ -18,8 +18,6 @@ limitations under the License.
package tasks
import (
- "encoding/json"
-
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models/common"
"github.com/apache/incubator-devlake/core/plugin"
@@ -69,17 +67,12 @@ var ExtractApiPipelinesMeta = plugin.SubTaskMeta{
Dependencies: []*plugin.SubTaskMeta{&CollectApiPipelinesMeta},
}
-func ExtractApiPipelines(taskCtx plugin.SubTaskContext) errors.Error {
- rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_PIPELINE_TABLE)
+func ExtractApiPipelines(subtaskCtx plugin.SubTaskContext) errors.Error {
+ subtaskCommonArgs, data := CreateSubtaskCommonArgs(subtaskCtx, RAW_PIPELINE_TABLE)
- extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{
- RawDataSubTaskArgs: *rawDataSubTaskArgs,
- Extract: func(row *api.RawData) ([]interface{}, errors.Error) {
- gitlabApiPipeline := &ApiPipeline{}
- err := errors.Convert(json.Unmarshal(row.Data, gitlabApiPipeline))
- if err != nil {
- return nil, err
- }
+ extractor, err := api.NewStatefulApiExtractor(&api.StatefulApiExtractorArgs[ApiPipeline]{
+ SubtaskCommonArgs: subtaskCommonArgs,
+ Extract: func(gitlabApiPipeline *ApiPipeline, row *api.RawData) ([]interface{}, errors.Error) {
pipelineProject := convertApiPipelineToGitlabPipelineProject(gitlabApiPipeline, data.Options.ConnectionId, data.Options.ProjectId)
return []interface{}{pipelineProject}, nil
},
diff --git a/backend/plugins/gitlab/tasks/shared.go b/backend/plugins/gitlab/tasks/shared.go
index 4051c93b5..59ce8b921 100644
--- a/backend/plugins/gitlab/tasks/shared.go
+++ b/backend/plugins/gitlab/tasks/shared.go
@@ -155,10 +155,10 @@ func GetQuery(reqData *helper.RequestData) (url.Values, errors.Error) {
return query, nil
}
-func CreateRawDataSubTaskArgs(taskCtx plugin.SubTaskContext, Table string) (*helper.RawDataSubTaskArgs, *GitlabTaskData) {
- data := taskCtx.GetData().(*GitlabTaskData)
+func CreateRawDataSubTaskArgs(subtaskCtx plugin.SubTaskContext, Table string) (*helper.RawDataSubTaskArgs, *GitlabTaskData) {
+ data := subtaskCtx.GetData().(*GitlabTaskData)
rawDataSubTaskArgs := &helper.RawDataSubTaskArgs{
- Ctx: taskCtx,
+ Ctx: subtaskCtx,
Params: models.GitlabApiParams{
ProjectId: data.Options.ProjectId,
ConnectionId: data.Options.ConnectionId,
@@ -168,15 +168,29 @@ func CreateRawDataSubTaskArgs(taskCtx plugin.SubTaskContext, Table string) (*hel
return rawDataSubTaskArgs, data
}
+func CreateSubtaskCommonArgs(subtaskCtx plugin.SubTaskContext, table string) (*helper.SubtaskCommonArgs, *GitlabTaskData) {
+ data := subtaskCtx.GetData().(*GitlabTaskData)
+ args := &helper.SubtaskCommonArgs{
+ SubTaskContext: subtaskCtx,
+ Table: table,
+ Params: models.GitlabApiParams{
+ ConnectionId: data.Options.ConnectionId,
+ ProjectId: data.Options.ProjectId,
+ },
+ }
+ return args, data
+}
+
func GetMergeRequestsIterator(taskCtx plugin.SubTaskContext, apiCollector *helper.StatefulApiCollector) (*helper.DalCursorIterator, errors.Error) {
db := taskCtx.GetDal()
data := taskCtx.GetData().(*GitlabTaskData)
clauses := []dal.Clause{
dal.Select("gmr.gitlab_id, gmr.iid"),
dal.From("_tool_gitlab_merge_requests gmr"),
+ // collect only openning merge request's notes and commits to speed up the process
dal.Where(
- `gmr.project_id = ? and gmr.connection_id = ?`,
- data.Options.ProjectId, data.Options.ConnectionId,
+ `gmr.project_id = ? and gmr.connection_id = ? AND state = ?`,
+ data.Options.ProjectId, data.Options.ConnectionId, "opened",
),
}
if apiCollector != nil {
diff --git a/backend/plugins/gitlab/tasks/trigger_job_extractor.go b/backend/plugins/gitlab/tasks/trigger_job_extractor.go
index f515b7b85..8a1e7eaeb 100644
--- a/backend/plugins/gitlab/tasks/trigger_job_extractor.go
+++ b/backend/plugins/gitlab/tasks/trigger_job_extractor.go
@@ -18,7 +18,6 @@ limitations under the License.
package tasks
import (
- "encoding/json"
"time"
"github.com/apache/incubator-devlake/core/errors"
@@ -55,18 +54,12 @@ var ExtractApiTriggerJobsMeta = plugin.SubTaskMeta{
Dependencies: []*plugin.SubTaskMeta{&CollectApiTriggerJobsMeta},
}
-func ExtractApiTriggerJobs(taskCtx plugin.SubTaskContext) errors.Error {
- rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_TRIGGER_JOB_TABLE)
-
- extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{
- RawDataSubTaskArgs: *rawDataSubTaskArgs,
- Extract: func(row *api.RawData) ([]interface{}, errors.Error) {
- gitlabApiTriggerJob := &ApiTriggerJob{}
- err := errors.Convert(json.Unmarshal(row.Data, gitlabApiTriggerJob))
- if err != nil {
- return nil, err
- }
+func ExtractApiTriggerJobs(subtaskCtx plugin.SubTaskContext) errors.Error {
+ subtaskCommonArgs, data := CreateSubtaskCommonArgs(subtaskCtx, RAW_TRIGGER_JOB_TABLE)
+ extractor, err := api.NewStatefulApiExtractor(&api.StatefulApiExtractorArgs[ApiTriggerJob]{
+ SubtaskCommonArgs: subtaskCommonArgs,
+ Extract: func(gitlabApiTriggerJob *ApiTriggerJob, row *api.RawData) ([]interface{}, errors.Error) {
gitlabPipeline, err := convertTriggerJob(gitlabApiTriggerJob, data.Options.ProjectId)
if err != nil {
return nil, err
diff --git a/backend/plugins/jira/tasks/epic_extractor.go b/backend/plugins/jira/tasks/epic_extractor.go
index cf1e7f28b..89f2ef70c 100644
--- a/backend/plugins/jira/tasks/epic_extractor.go
+++ b/backend/plugins/jira/tasks/epic_extractor.go
@@ -18,12 +18,15 @@ limitations under the License.
package tasks
import (
+ "encoding/json"
+
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/log"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/jira/models"
+ "github.com/apache/incubator-devlake/plugins/jira/tasks/apiv2models"
)
var _ plugin.SubTaskEntryPoint = ExtractEpics
@@ -61,7 +64,12 @@ func ExtractEpics(taskCtx plugin.SubTaskContext) errors.Error {
Table: RAW_EPIC_TABLE,
},
Extract: func(row *api.RawData) ([]interface{}, errors.Error) {
- return extractIssues(data, mappings, row, userFieldMap)
+ apiIssue := &apiv2models.Issue{}
+ err = errors.Convert(json.Unmarshal(row.Data, apiIssue))
+ if err != nil {
+ return nil, err
+ }
+ return extractIssues(data, mappings, apiIssue, row, userFieldMap)
},
})
if err != nil {
diff --git a/backend/plugins/jira/tasks/issue_changelog_convertor.go b/backend/plugins/jira/tasks/issue_changelog_convertor.go
index 4dc35e5c9..b27fc9666 100644
--- a/backend/plugins/jira/tasks/issue_changelog_convertor.go
+++ b/backend/plugins/jira/tasks/issue_changelog_convertor.go
@@ -78,7 +78,7 @@ func ConvertIssueChangelogs(subtaskCtx plugin.SubTaskContext) errors.Error {
changelogIdGenerator := didgen.NewDomainIdGenerator(&models.JiraIssueChangelogItems{})
accountIdGen := didgen.NewDomainIdGenerator(&models.JiraAccount{})
- converter, err := api.NewStatefulDataConverter[IssueChangelogItemResult](&api.StatefulDataConverterArgs[IssueChangelogItemResult]{
+ converter, err := api.NewStatefulDataConverter(&api.StatefulDataConverterArgs[IssueChangelogItemResult]{
SubtaskCommonArgs: &api.SubtaskCommonArgs{
SubTaskContext: subtaskCtx,
Table: RAW_ISSUE_TABLE,
diff --git a/backend/plugins/jira/tasks/issue_changelog_extractor.go b/backend/plugins/jira/tasks/issue_changelog_extractor.go
index a49142e3f..b5de33a45 100644
--- a/backend/plugins/jira/tasks/issue_changelog_extractor.go
+++ b/backend/plugins/jira/tasks/issue_changelog_extractor.go
@@ -47,7 +47,7 @@ func ExtractIssueChangelogs(subtaskCtx plugin.SubTaskContext) errors.Error {
if err != nil {
return err
}
- extractor, err := api.NewStatefulApiExtractor(&api.StatefulApiExtractorArgs{
+ extractor, err := api.NewStatefulApiExtractor(&api.StatefulApiExtractorArgs[apiv2models.Changelog]{
SubtaskCommonArgs: &api.SubtaskCommonArgs{
SubTaskContext: subtaskCtx,
Table: RAW_CHANGELOG_TABLE,
@@ -56,18 +56,13 @@ func ExtractIssueChangelogs(subtaskCtx plugin.SubTaskContext) errors.Error {
BoardId: data.Options.BoardId,
},
},
- Extract: func(row *api.RawData) ([]interface{}, errors.Error) {
+ Extract: func(changelog *apiv2models.Changelog, row *api.RawData) ([]interface{}, errors.Error) {
// process input
var input apiv2models.Input
err := errors.Convert(json.Unmarshal(row.Input, &input))
if err != nil {
return nil, err
}
- var changelog apiv2models.Changelog
- err = errors.Convert(json.Unmarshal(row.Data, &changelog))
- if err != nil {
- return nil, err
- }
// prepare output
var result []interface{}
cl, user := changelog.ToToolLayer(connectionId, input.IssueId, &input.UpdateTime)
diff --git a/backend/plugins/jira/tasks/issue_convertor.go b/backend/plugins/jira/tasks/issue_convertor.go
index 1c7305758..fb213a638 100644
--- a/backend/plugins/jira/tasks/issue_convertor.go
+++ b/backend/plugins/jira/tasks/issue_convertor.go
@@ -53,7 +53,7 @@ func ConvertIssues(subtaskCtx plugin.SubTaskContext) errors.Error {
boardIdGen := didgen.NewDomainIdGenerator(&models.JiraBoard{})
boardId := boardIdGen.Generate(data.Options.ConnectionId, data.Options.BoardId)
- converter, err := api.NewStatefulDataConverter[models.JiraIssue](&api.StatefulDataConverterArgs[models.JiraIssue]{
+ converter, err := api.NewStatefulDataConverter(&api.StatefulDataConverterArgs[models.JiraIssue]{
SubtaskCommonArgs: &api.SubtaskCommonArgs{
SubTaskContext: subtaskCtx,
Table: RAW_ISSUE_TABLE,
@@ -84,6 +84,17 @@ func ConvertIssues(subtaskCtx plugin.SubTaskContext) errors.Error {
}
return db.Cursor(clauses...)
},
+ // not needed for now due to jira assignee and label are converted in FullSync(Delete+Insert) manner
+ // BeforeConvert: func(jiraIssue *models.JiraIssue, stateManager *api.SubtaskStateManager) errors.Error {
+ // issueId := issueIdGen.Generate(data.Options.ConnectionId, jiraIssue.IssueId)
+ // if err := db.Delete(&ticket.IssueAssignee{}, dal.Where("issue_id = ?", issueId)); err != nil {
+ // return err
+ // }
+ // if err := db.Delete(&ticket.IssueLabel{}, dal.Where("issue_id = ?", issueId)); err != nil {
+ // return err
+ // }
+ // return nil
+ // },
Convert: func(jiraIssue *models.JiraIssue) ([]interface{}, errors.Error) {
var result []interface{}
issue := &ticket.Issue{
diff --git a/backend/plugins/jira/tasks/issue_extractor.go b/backend/plugins/jira/tasks/issue_extractor.go
index 686f23ff4..11277a0a4 100644
--- a/backend/plugins/jira/tasks/issue_extractor.go
+++ b/backend/plugins/jira/tasks/issue_extractor.go
@@ -18,7 +18,6 @@ limitations under the License.
package tasks
import (
- "encoding/json"
"strconv"
"strings"
"time"
@@ -62,7 +61,7 @@ func ExtractIssues(subtaskCtx plugin.SubTaskContext) errors.Error {
if err != nil {
return err
}
- extractor, err := api.NewStatefulApiExtractor(&api.StatefulApiExtractorArgs{
+ extractor, err := api.NewStatefulApiExtractor(&api.StatefulApiExtractorArgs[apiv2models.Issue]{
SubtaskCommonArgs: &api.SubtaskCommonArgs{
SubTaskContext: subtaskCtx,
Table: RAW_ISSUE_TABLE,
@@ -75,8 +74,27 @@ func ExtractIssues(subtaskCtx plugin.SubTaskContext) errors.Error {
"storyPointField": data.Options.ScopeConfig.StoryPointField,
},
},
- Extract: func(row *api.RawData) ([]interface{}, errors.Error) {
- return extractIssues(data, mappings, row, userFieldMap)
+ BeforeExtract: func(apiIssue *apiv2models.Issue, stateManager *api.SubtaskStateManager) errors.Error {
+ if stateManager.IsIncremental() {
+ err := db.Delete(
+ &models.JiraIssueLabel{},
+ dal.Where("connection_id = ? AND issue_id = ?", data.Options.ConnectionId, apiIssue.ID),
+ )
+ if err != nil {
+ return err
+ }
+ err = db.Delete(
+ &models.JiraIssueRelationship{},
+ dal.Where("connection_id = ? AND issue_id = ?", data.Options.ConnectionId, apiIssue.ID),
+ )
+ if err != nil {
+ return err
+ }
+ }
+ return nil
+ },
+ Extract: func(apiIssue *apiv2models.Issue, row *api.RawData) ([]interface{}, errors.Error) {
+ return extractIssues(data, mappings, apiIssue, row, userFieldMap)
},
})
if err != nil {
@@ -85,13 +103,8 @@ func ExtractIssues(subtaskCtx plugin.SubTaskContext) errors.Error {
return extractor.Execute()
}
-func extractIssues(data *JiraTaskData, mappings *typeMappings, row *api.RawData, userFieldMaps map[string]struct{}) ([]interface{}, errors.Error) {
- var apiIssue apiv2models.Issue
- err := errors.Convert(json.Unmarshal(row.Data, &apiIssue))
- if err != nil {
- return nil, err
- }
- err = apiIssue.SetAllFields(row.Data)
+func extractIssues(data *JiraTaskData, mappings *typeMappings, apiIssue *apiv2models.Issue, row *api.RawData, userFieldMaps map[string]struct{}) ([]interface{}, errors.Error) {
+ err := apiIssue.SetAllFields(row.Data)
if err != nil {
return nil, err
}
diff --git a/backend/server/api/blueprints/blueprints.go b/backend/server/api/blueprints/blueprints.go
index 23383d468..841027424 100644
--- a/backend/server/api/blueprints/blueprints.go
+++ b/backend/server/api/blueprints/blueprints.go
@@ -166,6 +166,7 @@ func Patch(c *gin.Context) {
// @Accept application/json
// @Param blueprintId path string true "blueprintId"
// @Param skipCollectors body models.TriggerSyncPolicy false "json"
+// @Param fullSync body models.TriggerSyncPolicy false "json"
// @Success 200 {object} models.Pipeline
// @Failure 400 {object} shared.ApiBody "Bad Request"
// @Failure 500 {object} shared.ApiBody "Internal Error"
diff --git a/config-ui/src/routes/blueprint/detail/status-panel.tsx b/config-ui/src/routes/blueprint/detail/status-panel.tsx
index 50be81fe3..6e19f006f 100644
--- a/config-ui/src/routes/blueprint/detail/status-panel.tsx
+++ b/config-ui/src/routes/blueprint/detail/status-panel.tsx
@@ -119,7 +119,7 @@ export const StatusPanel = ({ from, blueprint, pipelineId, onRefresh }: Props) =
type="primary"
disabled={!blueprint.enable}
loading={operating}
- onClick={() => handleRun({ skipCollectors: true })}
+ onClick={() => handleRun({ skipCollectors: true, fullSync: true })}
>
Re-transform Data
</Button>