You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by li...@apache.org on 2023/02/28 11:26:12 UTC
[incubator-devlake] branch main updated: fix(gitlab): fix bugs to resolve block (#4542)
This is an automated email from the ASF dual-hosted git repository.
likyh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 1697de10d fix(gitlab): fix bugs to resolve block (#4542)
1697de10d is described below
commit 1697de10d6913db15698725a949cbd85f64b34d8
Author: Warren Chen <yi...@merico.dev>
AuthorDate: Tue Feb 28 19:26:06 2023 +0800
fix(gitlab): fix bugs to resolve block (#4542)
* fix(gitlab): fix bugs to resolve block
* test(gitlab): modify e2e
* fix(gitlab): fix fore review
---
.../helpers/pluginhelper/api/iso8601time_test.go | 83 ++++++++++++++++++++
backend/plugins/gitlab/e2e/mr_test.go | 91 ++++------------------
.../_tool_gitlab_merge_requests.csv | 42 +++++-----
.../e2e/snapshot_tables/_tool_gitlab_mr_labels.csv | 30 +++----
.../e2e/snapshot_tables/pull_request_labels.csv | 30 +++----
.../gitlab/e2e/snapshot_tables/pull_requests.csv | 38 ++++-----
.../20230210_add_is_detail_required.go | 4 +-
backend/plugins/gitlab/models/mr.go | 1 +
backend/plugins/gitlab/tasks/job_collector.go | 84 ++++++++++++++++++--
backend/plugins/gitlab/tasks/mr_extractor.go | 2 +
backend/server/api/api.go | 2 +
11 files changed, 254 insertions(+), 153 deletions(-)
diff --git a/backend/helpers/pluginhelper/api/iso8601time_test.go b/backend/helpers/pluginhelper/api/iso8601time_test.go
index 1ce623e00..bc7f9603e 100644
--- a/backend/helpers/pluginhelper/api/iso8601time_test.go
+++ b/backend/helpers/pluginhelper/api/iso8601time_test.go
@@ -18,8 +18,11 @@ limitations under the License.
package api
import (
+ "database/sql/driver"
"encoding/json"
+ "fmt"
"github.com/apache/incubator-devlake/core/errors"
+ "reflect"
"testing"
"time"
@@ -82,3 +85,83 @@ func TestIso8601Time(t *testing.T) {
assert.Equal(t, expected, record4.Created.UTC())
}
}
+
+func TestIso8601Time_Value(t *testing.T) {
+ zeroTime := time.Time{}
+ testCases := []struct {
+ name string
+ input *Iso8601Time
+ output driver.Value
+ err error
+ }{
+ {
+ name: "Nil value",
+ input: nil,
+ output: nil,
+ err: nil,
+ },
+ {
+ name: "Valid time value",
+ input: &Iso8601Time{
+ time: time.Date(2023, 2, 28, 10, 30, 0, 0, time.UTC),
+ format: time.RFC3339,
+ },
+ output: time.Date(2023, 2, 28, 10, 30, 0, 0, time.UTC),
+ err: nil,
+ },
+ {
+ name: "Zero time value",
+ input: &Iso8601Time{
+ time: zeroTime,
+ format: time.RFC3339,
+ },
+ output: nil,
+ err: nil,
+ },
+ }
+ for _, tc := range testCases {
+ t.Run(tc.name, func(t *testing.T) {
+ output, err := tc.input.Value()
+ if output != tc.output {
+ t.Errorf("Expected output to be %v, but got %v", tc.output, output)
+ }
+ if err != tc.err {
+ t.Errorf("Expected error to be %v, but got %v", tc.err, err)
+ }
+ })
+ }
+}
+
+func TestIso8601Time_Scan(t *testing.T) {
+ testCases := []struct {
+ name string
+ input interface{}
+ output *Iso8601Time
+ err error
+ }{
+ {
+ name: "Valid time value",
+ input: time.Date(2023, 2, 28, 10, 30, 0, 0, time.UTC),
+ output: &Iso8601Time{time: time.Date(2023, 2, 28, 10, 30, 0, 0, time.UTC), format: time.RFC3339},
+ err: nil,
+ },
+ {
+ name: "Invalid input value",
+ input: "invalid",
+ output: &Iso8601Time{},
+ err: fmt.Errorf("can not convert %v to timestamp", "invalid"),
+ },
+ }
+ for _, tc := range testCases {
+ t.Run(tc.name, func(t *testing.T) {
+ var output Iso8601Time
+ err := output.Scan(tc.input)
+ if !reflect.DeepEqual(tc.output, &output) {
+ t.Errorf("Expected output to be %v, but got %v", tc.output, output)
+ }
+ if !reflect.DeepEqual(tc.err, err) {
+ t.Errorf("Expected error to be %v, but got %v", tc.err, err)
+ }
+ })
+ }
+}
diff --git a/backend/plugins/gitlab/e2e/mr_test.go b/backend/plugins/gitlab/e2e/mr_test.go
index 2b112e980..8f43e5b1d 100644
--- a/backend/plugins/gitlab/e2e/mr_test.go
+++ b/backend/plugins/gitlab/e2e/mr_test.go
@@ -18,6 +18,7 @@ limitations under the License.
package e2e
import (
+ "github.com/apache/incubator-devlake/core/models/common"
"github.com/apache/incubator-devlake/core/models/domainlayer/code"
"github.com/apache/incubator-devlake/helpers/e2ehelper"
"github.com/apache/incubator-devlake/plugins/gitlab/impl"
@@ -46,86 +47,28 @@ func TestGitlabMrDataFlow(t *testing.T) {
dataflowTester.FlushTabler(&models.GitlabMergeRequest{})
dataflowTester.FlushTabler(&models.GitlabMrLabel{})
dataflowTester.Subtask(tasks.ExtractApiMergeRequestsMeta, taskData)
- dataflowTester.VerifyTable(
- models.GitlabMergeRequest{},
- "./snapshot_tables/_tool_gitlab_merge_requests.csv",
- e2ehelper.ColumnWithRawData(
- "connection_id",
- "gitlab_id",
- "iid",
- "project_id",
- "source_project_id",
- "target_project_id",
- "state",
- "title",
- "web_url",
- "user_notes_count",
- "work_in_progress",
- "source_branch",
- "target_branch",
- "merge_commit_sha",
- "merged_at",
- "gitlab_created_at",
- "closed_at",
- "merged_by_username",
- "description",
- "author_username",
- "author_user_id",
- "component",
- "first_comment_time",
- "review_rounds",
- ),
- )
- dataflowTester.VerifyTable(
- models.GitlabMrLabel{},
- "./snapshot_tables/_tool_gitlab_mr_labels.csv",
- e2ehelper.ColumnWithRawData(
- "connection_id",
- "mr_id",
- "label_name",
- ),
- )
+ dataflowTester.VerifyTableWithOptions(&models.GitlabMergeRequest{}, e2ehelper.TableOptions{
+ CSVRelPath: "./snapshot_tables/_tool_gitlab_merge_requests.csv",
+ IgnoreTypes: []interface{}{common.NoPKModel{}},
+ })
+ dataflowTester.VerifyTableWithOptions(&models.GitlabMrLabel{}, e2ehelper.TableOptions{
+ CSVRelPath: "./snapshot_tables/_tool_gitlab_mr_labels.csv",
+ IgnoreTypes: []interface{}{common.NoPKModel{}},
+ })
// verify conversion
dataflowTester.FlushTabler(&code.PullRequest{})
dataflowTester.Subtask(tasks.ConvertApiMergeRequestsMeta, taskData)
- dataflowTester.VerifyTable(
- code.PullRequest{},
- "./snapshot_tables/pull_requests.csv",
- e2ehelper.ColumnWithRawData(
- "id",
- "base_repo_id",
- "head_repo_id",
- "status",
- "title",
- "description",
- "url",
- "author_name",
- "author_id",
- "parent_pr_id",
- "pull_request_key",
- "created_date",
- "merged_date",
- "closed_date",
- "type",
- "component",
- "merge_commit_sha",
- "head_ref",
- "base_ref",
- "base_commit_sha",
- "head_commit_sha",
- ),
- )
+ dataflowTester.VerifyTableWithOptions(&code.PullRequest{}, e2ehelper.TableOptions{
+ CSVRelPath: "./snapshot_tables/pull_requests.csv",
+ IgnoreTypes: []interface{}{common.NoPKModel{}},
+ })
// verify conversion
dataflowTester.FlushTabler(&code.PullRequestLabel{})
dataflowTester.Subtask(tasks.ConvertMrLabelsMeta, taskData)
- dataflowTester.VerifyTable(
- code.PullRequestLabel{},
- "./snapshot_tables/pull_request_labels.csv",
- e2ehelper.ColumnWithRawData(
- "pull_request_id",
- "label_name",
- ),
- )
+ dataflowTester.VerifyTableWithOptions(&code.PullRequestLabel{}, e2ehelper.TableOptions{
+ CSVRelPath: "./snapshot_tables/pull_request_labels.csv",
+ IgnoreTypes: []interface{}{common.NoPKModel{}},
+ })
}
diff --git a/backend/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_merge_requests.csv b/backend/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_merge_requests.csv
index cace55593..c218b8c48 100644
--- a/backend/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_merge_requests.csv
+++ b/backend/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_merge_requests.csv
@@ -1,25 +1,25 @@
-connection_id,gitlab_id,iid,project_id,source_project_id,target_project_id,state,title,web_url,user_notes_count,work_in_progress,source_branch,target_branch,merge_commit_sha,merged_at,gitlab_created_at,closed_at,merged_by_username,description,author_username,author_user_id,component,first_comment_time,review_rounds,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
-1,32348491,1,12345678,12345678,12345678,merged,"Resolve ""Add documentation to snowflake spend package""",https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/1,1,0,1-add-documentation-to-snowflake-spend-package,master,da1d6dea48f5972ffc683da6cff30934e7d6c52c,2019-06-28T14:32:06.192+00:00,2019-06-28T05:21:43.743+00:00,,tayloramurphy,Closes #1,emilie,2295562,,,0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,1,
-1,35064956,3,12345678,13835497,12345678,merged,Update README to include steps to resolve a potential dbt-utils conflict,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/3,3,0,5-update-readme-to-include-steps-to-resolve-a-potential-dbt-utils-conflict,master,d678bea9d47b42eb13512d1c9d6a592d80b432d4,2019-08-26T14:15:27.922+00:00,2019-08-15T19:34:32.706+00:00,,emilie,Closes #5,martinguindon,3871284,,,0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,3,
-1,35841926,4,12345678,12345678,12345678,merged,"Resolve ""Config is not generic enough""",https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/4,0,0,4-config-is-not-generic-enough,master,e95b5db25e15a38e21d11cb45cc21bf17d5c407c,2019-08-26T15:37:50.105+00:00,2019-08-26T15:32:49.557+00:00,,emilie,Closes #4,emilie,2295562,,,0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,4,
-1,53445063,5,12345678,15706315,12345678,merged,Issue 3 Base model,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/5,20,0,issue_3,master,f2ee4cf121a328ce39723506dc18e4661941971a,2020-03-25T18:36:45.801+00:00,2020-03-24T12:46:15.891+00:00,,tayloramurphy,,nehiljain,783199,,,0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,5,
-1,53627854,6,12345678,15706063,12345678,merged,Update schema.yml typo in docs,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/6,0,0,patch-1,master,12dcc23a45adce0b12f8687438ec3a28274c7c30,2020-03-25T19:04:19.844+00:00,2020-03-25T19:02:16.747+00:00,,tayloramurphy,,nehiljain,783199,,,0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,6,
-1,55146687,8,12345678,12345678,12345678,merged,"Resolve ""Document release process""",https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/8,3,0,6-document-release-process,master,7c8245a3a5eda7f502737940aaf7944d99c58f2e,2020-04-08T20:52:11.150+00:00,2020-04-08T20:07:10.223+00:00,,emilie,Closes #6,m_walker,5212782,,,0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,8,
-1,55146787,9,12345678,15706315,12345678,opened,Issue 3: Transformed model for query performance,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/9,14,0,issue_3,master,,,2020-04-08T20:09:08.130+00:00,,,,nehiljain,783199,,,0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,9,
-1,58311001,10,12345678,12345678,12345678,merged,Update version in readme,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/10,0,0,emilie-master-patch-23079,master,66c0f1de49a0c876b8f93e8e0dce3327e766f59d,2020-05-11T17:09:20.603+00:00,2020-05-11T17:09:12.265+00:00,,emilie,,emilie,2295562,,,0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,10,
-1,62519057,11,12345678,19569570,12345678,opened,Clustering metering models,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/11,0,0,clustering-metering,master,,,2020-06-24T12:34:04.792+00:00,,,,jainnehil,842680,,,0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,11,
-1,65505080,12,12345678,12345678,12345678,merged,"Resolve ""Upgrade package for dbt 0.17""",https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/12,0,0,11-upgrade-package-for-dbt-0-17,master,9bfc136eb90802c2ce59956c34dde01bb3de0d50,2020-07-24T21:13:35.321+00:00,2020-07-24T17:47:08.238+00:00,,tayloramurphy,"Closes #11
+connection_id,gitlab_id,iid,project_id,source_project_id,target_project_id,state,title,web_url,user_notes_count,work_in_progress,is_detail_required,source_branch,target_branch,merge_commit_sha,merged_at,gitlab_created_at,gitlab_updated_at,closed_at,type,merged_by_username,description,author_username,author_user_id,component,first_comment_time,review_rounds
+1,32348491,1,12345678,12345678,12345678,merged,"Resolve ""Add documentation to snowflake spend package""",https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/1,1,0,0,1-add-documentation-to-snowflake-spend-package,master,da1d6dea48f5972ffc683da6cff30934e7d6c52c,2019-06-28T14:32:06.192+00:00,2019-06-28T05:21:43.743+00:00,2019-06-28T14:32:05.270+00:00,,,tayloramurphy,Closes #1,emilie,2295562,,,0
+1,35064956,3,12345678,13835497,12345678,merged,Update README to include steps to resolve a potential dbt-utils conflict,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/3,3,0,0,5-update-readme-to-include-steps-to-resolve-a-potential-dbt-utils-conflict,master,d678bea9d47b42eb13512d1c9d6a592d80b432d4,2019-08-26T14:15:27.922+00:00,2019-08-15T19:34:32.706+00:00,2019-08-26T14:15:27.845+00:00,,,emilie,Closes #5,martinguindon,3871284,,,0
+1,35841926,4,12345678,12345678,12345678,merged,"Resolve ""Config is not generic enough""",https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/4,0,0,0,4-config-is-not-generic-enough,master,e95b5db25e15a38e21d11cb45cc21bf17d5c407c,2019-08-26T15:37:50.105+00:00,2019-08-26T15:32:49.557+00:00,2019-08-26T15:37:49.575+00:00,,,emilie,Closes #4,emilie,2295562,,,0
+1,53445063,5,12345678,15706315,12345678,merged,Issue 3 Base model,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/5,20,0,0,issue_3,master,f2ee4cf121a328ce39723506dc18e4661941971a,2020-03-25T18:36:45.801+00:00,2020-03-24T12:46:15.891+00:00,2020-03-25T18:36:45.303+00:00,,,tayloramurphy,,nehiljain,783199,,,0
+1,53627854,6,12345678,15706063,12345678,merged,Update schema.yml typo in docs,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/6,0,0,0,patch-1,master,12dcc23a45adce0b12f8687438ec3a28274c7c30,2020-03-25T19:04:19.844+00:00,2020-03-25T19:02:16.747+00:00,2020-03-25T19:04:32.235+00:00,,,tayloramurphy,,nehiljain,783199,,,0
+1,55146687,8,12345678,12345678,12345678,merged,"Resolve ""Document release process""",https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/8,3,0,0,6-document-release-process,master,7c8245a3a5eda7f502737940aaf7944d99c58f2e,2020-04-08T20:52:11.150+00:00,2020-04-08T20:07:10.223+00:00,2020-04-08T20:52:10.945+00:00,,,emilie,Closes #6,m_walker,5212782,,,0
+1,55146787,9,12345678,15706315,12345678,opened,Issue 3: Transformed model for query performance,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/9,14,0,0,issue_3,master,,,2020-04-08T20:09:08.130+00:00,2020-06-29T15:47:04.785+00:00,,,,,nehiljain,783199,,,0
+1,58311001,10,12345678,12345678,12345678,merged,Update version in readme,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/10,0,0,0,emilie-master-patch-23079,master,66c0f1de49a0c876b8f93e8e0dce3327e766f59d,2020-05-11T17:09:20.603+00:00,2020-05-11T17:09:12.265+00:00,2020-05-11T17:09:20.565+00:00,,,emilie,,emilie,2295562,,,0
+1,62519057,11,12345678,19569570,12345678,opened,Clustering metering models,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/11,0,0,0,clustering-metering,master,,,2020-06-24T12:34:04.792+00:00,2020-06-24T12:35:01.115+00:00,,,,,jainnehil,842680,,,0
+1,65505080,12,12345678,12345678,12345678,merged,"Resolve ""Upgrade package for dbt 0.17""",https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/12,0,0,0,11-upgrade-package-for-dbt-0-17,master,9bfc136eb90802c2ce59956c34dde01bb3de0d50,2020-07-24T21:13:35.321+00:00,2020-07-24T17:47:08.238+00:00,2020-07-24T21:13:34.907+00:00,,,tayloramurphy,"Closes #11
* Upgrades to 0.17.0 format
-* Formatting changes to be in line with GitLab SQL Style Guide",tayloramurphy,1942272,,,0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,12,
-1,68978485,13,12345678,15706315,12345678,closed,Include more snowflake qrt columns,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/13,0,0,include_more_snowflake_qrt_columns,master,,,2020-08-27T20:17:01.825+00:00,2020-08-27T20:20:08.150+00:00,,,aianus,2478227,,,0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,13,
-1,89243644,14,12345678,24539973,12345678,merged,Update README.md to use the newest version as an example,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/14,0,0,ThomasLaPiana-master-patch-93997,master,21840a7eadb58babe8aeae2960da851a3ed00ddc,2021-02-19T20:13:05.969+00:00,2021-02-19T20:12:14.302+00:00,,tayloramurphy,Update README.md to use the newest version as an example. The old version doesn't work with the current version of dbt,ThomasLaPiana,2061802,,,0,"{""ConnectionI [...]
-1,110817220,16,12345678,28584714,12345678,merged,Update packages.yml to point to dbt-labs instead of fishtown,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/16,5,0,GJMcClintock-master-patch-24867,master,6f45b467c478df1c67d19cf6d4cbb8e05a710662,2021-08-12T06:12:54.329+00:00,2021-08-03T15:02:54.955+00:00,,vedprakash2021,With the company name change the old repo is deprecated.,GJMcClintock,9439881,,,0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,16,
-1,111383524,17,12345678,0,12345678,closed,The package name changed -> https://hub.getdbt.com/dbt-labs/dbt_utils/latest/,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/17,1,0,swiffer-master-patch-77533,master,,,2021-08-07T06:50:25.458+00:00,2021-08-07T06:51:14.933+00:00,,,swiffer,156402,,,0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,17,
-1,114994501,18,12345678,29298577,12345678,opened,Add support for Snowpipe usage monitoring,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/18,0,0,master,master,,,2021-09-01T21:15:30.334+00:00,,,Add models and docs for Snowflake pipes (Snowpipe) usage monitoring based on the views in Snowflake Usage schema,gary-beautypie,9635687,,,0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,18,
-1,135775405,19,12345678,32935405,12345678,opened,Updates for dbt 1.0,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/19,1,0,updates_for_dbt_1.0,master,,,2022-01-18T19:59:30.723+00:00,,,"This MR sets up the repo for dbt 1.0
+* Formatting changes to be in line with GitLab SQL Style Guide",tayloramurphy,1942272,,,0
+1,68978485,13,12345678,15706315,12345678,closed,Include more snowflake qrt columns,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/13,0,0,0,include_more_snowflake_qrt_columns,master,,,2020-08-27T20:17:01.825+00:00,2020-08-27T20:20:08.131+00:00,2020-08-27T20:20:08.150+00:00,,,,aianus,2478227,,,0
+1,89243644,14,12345678,24539973,12345678,merged,Update README.md to use the newest version as an example,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/14,0,0,0,ThomasLaPiana-master-patch-93997,master,21840a7eadb58babe8aeae2960da851a3ed00ddc,2021-02-19T20:13:05.969+00:00,2021-02-19T20:12:14.302+00:00,2021-02-19T20:13:05.913+00:00,,,tayloramurphy,Update README.md to use the newest version as an example. The old version doesn't work with the current version of dbt,ThomasLa [...]
+1,110817220,16,12345678,28584714,12345678,merged,Update packages.yml to point to dbt-labs instead of fishtown,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/16,5,0,0,GJMcClintock-master-patch-24867,master,6f45b467c478df1c67d19cf6d4cbb8e05a710662,2021-08-12T06:12:54.329+00:00,2021-08-03T15:02:54.955+00:00,2021-10-04T16:23:41.057+00:00,,,vedprakash2021,With the company name change the old repo is deprecated.,GJMcClintock,9439881,,,0
+1,111383524,17,12345678,0,12345678,closed,The package name changed -> https://hub.getdbt.com/dbt-labs/dbt_utils/latest/,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/17,1,0,0,swiffer-master-patch-77533,master,,,2021-08-07T06:50:25.458+00:00,2021-08-07T06:51:14.916+00:00,2021-08-07T06:51:14.933+00:00,,,,swiffer,156402,,,0
+1,114994501,18,12345678,29298577,12345678,opened,Add support for Snowpipe usage monitoring,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/18,0,0,0,master,master,,,2021-09-01T21:15:30.334+00:00,2022-06-30T21:41:49.705+00:00,,,,Add models and docs for Snowflake pipes (Snowpipe) usage monitoring based on the views in Snowflake Usage schema,gary-beautypie,9635687,,,0
+1,135775405,19,12345678,32935405,12345678,opened,Updates for dbt 1.0,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/19,1,0,0,updates_for_dbt_1.0,master,,,2022-01-18T19:59:30.723+00:00,2022-02-10T14:13:47.663+00:00,,,,"This MR sets up the repo for dbt 1.0
A few configs were renamed.
-Could a new release be made for dbt 1.0?",johnj4,10663622,,,0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,19,
-1,145012495,20,12345678,34491818,12345678,closed,Draft: Update dbt_project.yml,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/20,0,1,PedramNavid-master-patch-20645,master,,,2022-03-15T03:07:06.077+00:00,2022-03-15T03:07:22.665+00:00,,,PedramNavid,9722492,,,0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,20,
-1,158698019,21,12345678,12345678,12345678,opened,Draft: Corrections for dbt 1,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/21,0,1,updates_for_dbt_1_1,master,,,2022-06-03T09:24:53.707+00:00,,,Closes https://gitlab.com/gitlab-data/analytics/-/issues/12941,paul_armstrong,5618371,,,0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,21,
+Could a new release be made for dbt 1.0?",johnj4,10663622,,,0
+1,145012495,20,12345678,34491818,12345678,closed,Draft: Update dbt_project.yml,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/20,0,1,0,PedramNavid-master-patch-20645,master,,,2022-03-15T03:07:06.077+00:00,2022-03-15T03:07:22.642+00:00,2022-03-15T03:07:22.665+00:00,,,,PedramNavid,9722492,,,0
+1,158698019,21,12345678,12345678,12345678,opened,Draft: Corrections for dbt 1,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/21,0,1,0,updates_for_dbt_1_1,master,,,2022-06-03T09:24:53.707+00:00,2022-06-03T18:06:46.331+00:00,,,,Closes https://gitlab.com/gitlab-data/analytics/-/issues/12941,paul_armstrong,5618371,,,0
diff --git a/backend/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_mr_labels.csv b/backend/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_mr_labels.csv
index 95a3789a0..b7505fb8f 100644
--- a/backend/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_mr_labels.csv
+++ b/backend/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_mr_labels.csv
@@ -1,15 +1,15 @@
-connection_id,mr_id,label_name,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
-1,35841926,Analytics,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,4,
-1,35841926,Data Team,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,4,
-1,35841926,dbt,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,4,
-1,35841926,Refactor,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,4,
-1,55146687,Analytics,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,8,
-1,55146687,Data Team,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,8,
-1,55146687,dbt,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,8,
-1,55146687,Enhancement,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,8,
-1,65505080,Data Team,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,12,
-1,65505080,dbt,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,12,
-1,65505080,Priority::3-Other,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,12,
-1,65505080,Refactor,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,12,
-1,65505080,To Do,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,12,
-1,65505080,workflow::4 - scheduled,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,12,
+connection_id,mr_id,label_name
+1,35841926,Analytics
+1,35841926,Data Team
+1,35841926,Refactor
+1,35841926,dbt
+1,55146687,Analytics
+1,55146687,Data Team
+1,55146687,Enhancement
+1,55146687,dbt
+1,65505080,Data Team
+1,65505080,Priority::3-Other
+1,65505080,Refactor
+1,65505080,To Do
+1,65505080,dbt
+1,65505080,workflow::4 - scheduled
diff --git a/backend/plugins/gitlab/e2e/snapshot_tables/pull_request_labels.csv b/backend/plugins/gitlab/e2e/snapshot_tables/pull_request_labels.csv
index 0817b55ec..ea9795d61 100644
--- a/backend/plugins/gitlab/e2e/snapshot_tables/pull_request_labels.csv
+++ b/backend/plugins/gitlab/e2e/snapshot_tables/pull_request_labels.csv
@@ -1,15 +1,15 @@
-pull_request_id,label_name,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
-gitlab:GitlabMergeRequest:1:35841926,Analytics,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,4,
-gitlab:GitlabMergeRequest:1:35841926,Data Team,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,4,
-gitlab:GitlabMergeRequest:1:35841926,dbt,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,4,
-gitlab:GitlabMergeRequest:1:35841926,Refactor,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,4,
-gitlab:GitlabMergeRequest:1:55146687,Analytics,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,8,
-gitlab:GitlabMergeRequest:1:55146687,Data Team,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,8,
-gitlab:GitlabMergeRequest:1:55146687,dbt,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,8,
-gitlab:GitlabMergeRequest:1:55146687,Enhancement,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,8,
-gitlab:GitlabMergeRequest:1:65505080,Data Team,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,12,
-gitlab:GitlabMergeRequest:1:65505080,dbt,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,12,
-gitlab:GitlabMergeRequest:1:65505080,Priority::3-Other,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,12,
-gitlab:GitlabMergeRequest:1:65505080,Refactor,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,12,
-gitlab:GitlabMergeRequest:1:65505080,To Do,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,12,
-gitlab:GitlabMergeRequest:1:65505080,workflow::4 - scheduled,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,12,
+pull_request_id,label_name
+gitlab:GitlabMergeRequest:1:35841926,Analytics
+gitlab:GitlabMergeRequest:1:35841926,Data Team
+gitlab:GitlabMergeRequest:1:35841926,Refactor
+gitlab:GitlabMergeRequest:1:35841926,dbt
+gitlab:GitlabMergeRequest:1:55146687,Analytics
+gitlab:GitlabMergeRequest:1:55146687,Data Team
+gitlab:GitlabMergeRequest:1:55146687,Enhancement
+gitlab:GitlabMergeRequest:1:55146687,dbt
+gitlab:GitlabMergeRequest:1:65505080,Data Team
+gitlab:GitlabMergeRequest:1:65505080,Priority::3-Other
+gitlab:GitlabMergeRequest:1:65505080,Refactor
+gitlab:GitlabMergeRequest:1:65505080,To Do
+gitlab:GitlabMergeRequest:1:65505080,dbt
+gitlab:GitlabMergeRequest:1:65505080,workflow::4 - scheduled
diff --git a/backend/plugins/gitlab/e2e/snapshot_tables/pull_requests.csv b/backend/plugins/gitlab/e2e/snapshot_tables/pull_requests.csv
index 444deaf3e..00b8207a7 100644
--- a/backend/plugins/gitlab/e2e/snapshot_tables/pull_requests.csv
+++ b/backend/plugins/gitlab/e2e/snapshot_tables/pull_requests.csv
@@ -1,25 +1,25 @@
-id,base_repo_id,head_repo_id,status,title,description,url,author_name,author_id,parent_pr_id,pull_request_key,created_date,merged_date,closed_date,type,component,merge_commit_sha,head_ref,base_ref,base_commit_sha,head_commit_sha,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
-gitlab:GitlabMergeRequest:1:110817220,gitlab:GitlabProject:1:12345678,gitlab:GitlabProject:1:28584714,merged,Update packages.yml to point to dbt-labs instead of fishtown,With the company name change the old repo is deprecated.,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/16,GJMcClintock,gitlab:GitlabAccount:1:9439881,,16,2021-08-03T15:02:54.955+00:00,2021-08-12T06:12:54.329+00:00,,,,6f45b467c478df1c67d19cf6d4cbb8e05a710662,GJMcClintock-master-patch-24867,master,,,"{""C [...]
-gitlab:GitlabMergeRequest:1:111383524,gitlab:GitlabProject:1:12345678,gitlab:GitlabProject:1:0,closed,The package name changed -> https://hub.getdbt.com/dbt-labs/dbt_utils/latest/,,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/17,swiffer,gitlab:GitlabAccount:1:156402,,17,2021-08-07T06:50:25.458+00:00,,2021-08-07T06:51:14.933+00:00,,,,swiffer-master-patch-77533,master,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,17,
-gitlab:GitlabMergeRequest:1:114994501,gitlab:GitlabProject:1:12345678,gitlab:GitlabProject:1:29298577,opened,Add support for Snowpipe usage monitoring,Add models and docs for Snowflake pipes (Snowpipe) usage monitoring based on the views in Snowflake Usage schema,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/18,gary-beautypie,gitlab:GitlabAccount:1:9635687,,18,2021-09-01T21:15:30.334+00:00,,,,,,master,master,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_ap [...]
+id,base_repo_id,head_repo_id,status,title,description,url,author_name,author_id,parent_pr_id,pull_request_key,created_date,merged_date,closed_date,type,component,merge_commit_sha,head_ref,base_ref,base_commit_sha,head_commit_sha
+gitlab:GitlabMergeRequest:1:110817220,gitlab:GitlabProject:1:12345678,gitlab:GitlabProject:1:28584714,merged,Update packages.yml to point to dbt-labs instead of fishtown,With the company name change the old repo is deprecated.,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/16,GJMcClintock,gitlab:GitlabAccount:1:9439881,,16,2021-08-03T15:02:54.955+00:00,2021-08-12T06:12:54.329+00:00,,,,6f45b467c478df1c67d19cf6d4cbb8e05a710662,GJMcClintock-master-patch-24867,master,,
+gitlab:GitlabMergeRequest:1:111383524,gitlab:GitlabProject:1:12345678,gitlab:GitlabProject:1:0,closed,The package name changed -> https://hub.getdbt.com/dbt-labs/dbt_utils/latest/,,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/17,swiffer,gitlab:GitlabAccount:1:156402,,17,2021-08-07T06:50:25.458+00:00,,2021-08-07T06:51:14.933+00:00,,,,swiffer-master-patch-77533,master,,
+gitlab:GitlabMergeRequest:1:114994501,gitlab:GitlabProject:1:12345678,gitlab:GitlabProject:1:29298577,opened,Add support for Snowpipe usage monitoring,Add models and docs for Snowflake pipes (Snowpipe) usage monitoring based on the views in Snowflake Usage schema,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/18,gary-beautypie,gitlab:GitlabAccount:1:9635687,,18,2021-09-01T21:15:30.334+00:00,,,,,,master,master,,
gitlab:GitlabMergeRequest:1:135775405,gitlab:GitlabProject:1:12345678,gitlab:GitlabProject:1:32935405,opened,Updates for dbt 1.0,"This MR sets up the repo for dbt 1.0
A few configs were renamed.
-Could a new release be made for dbt 1.0?",https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/19,johnj4,gitlab:GitlabAccount:1:10663622,,19,2022-01-18T19:59:30.723+00:00,,,,,,updates_for_dbt_1.0,master,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,19,
-gitlab:GitlabMergeRequest:1:145012495,gitlab:GitlabProject:1:12345678,gitlab:GitlabProject:1:34491818,closed,Draft: Update dbt_project.yml,,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/20,PedramNavid,gitlab:GitlabAccount:1:9722492,,20,2022-03-15T03:07:06.077+00:00,,2022-03-15T03:07:22.665+00:00,,,,PedramNavid-master-patch-20645,master,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,20,
-gitlab:GitlabMergeRequest:1:158698019,gitlab:GitlabProject:1:12345678,gitlab:GitlabProject:1:12345678,opened,Draft: Corrections for dbt 1,Closes https://gitlab.com/gitlab-data/analytics/-/issues/12941,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/21,paul_armstrong,gitlab:GitlabAccount:1:5618371,,21,2022-06-03T09:24:53.707+00:00,,,,,,updates_for_dbt_1_1,master,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,21,
-gitlab:GitlabMergeRequest:1:32348491,gitlab:GitlabProject:1:12345678,gitlab:GitlabProject:1:12345678,merged,"Resolve ""Add documentation to snowflake spend package""",Closes #1,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/1,emilie,gitlab:GitlabAccount:1:2295562,,1,2019-06-28T05:21:43.743+00:00,2019-06-28T14:32:06.192+00:00,,,,da1d6dea48f5972ffc683da6cff30934e7d6c52c,1-add-documentation-to-snowflake-spend-package,master,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_r [...]
-gitlab:GitlabMergeRequest:1:35064956,gitlab:GitlabProject:1:12345678,gitlab:GitlabProject:1:13835497,merged,Update README to include steps to resolve a potential dbt-utils conflict,Closes #5,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/3,martinguindon,gitlab:GitlabAccount:1:3871284,,3,2019-08-15T19:34:32.706+00:00,2019-08-26T14:15:27.922+00:00,,,,d678bea9d47b42eb13512d1c9d6a592d80b432d4,5-update-readme-to-include-steps-to-resolve-a-potential-dbt-utils-conflict,master,, [...]
-gitlab:GitlabMergeRequest:1:35841926,gitlab:GitlabProject:1:12345678,gitlab:GitlabProject:1:12345678,merged,"Resolve ""Config is not generic enough""",Closes #4,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/4,emilie,gitlab:GitlabAccount:1:2295562,,4,2019-08-26T15:32:49.557+00:00,2019-08-26T15:37:50.105+00:00,,,,e95b5db25e15a38e21d11cb45cc21bf17d5c407c,4-config-is-not-generic-enough,master,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,4,
-gitlab:GitlabMergeRequest:1:53445063,gitlab:GitlabProject:1:12345678,gitlab:GitlabProject:1:15706315,merged,Issue 3 Base model,,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/5,nehiljain,gitlab:GitlabAccount:1:783199,,5,2020-03-24T12:46:15.891+00:00,2020-03-25T18:36:45.801+00:00,,,,f2ee4cf121a328ce39723506dc18e4661941971a,issue_3,master,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,5,
-gitlab:GitlabMergeRequest:1:53627854,gitlab:GitlabProject:1:12345678,gitlab:GitlabProject:1:15706063,merged,Update schema.yml typo in docs,,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/6,nehiljain,gitlab:GitlabAccount:1:783199,,6,2020-03-25T19:02:16.747+00:00,2020-03-25T19:04:19.844+00:00,,,,12dcc23a45adce0b12f8687438ec3a28274c7c30,patch-1,master,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,6,
-gitlab:GitlabMergeRequest:1:55146687,gitlab:GitlabProject:1:12345678,gitlab:GitlabProject:1:12345678,merged,"Resolve ""Document release process""",Closes #6,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/8,m_walker,gitlab:GitlabAccount:1:5212782,,8,2020-04-08T20:07:10.223+00:00,2020-04-08T20:52:11.150+00:00,,,,7c8245a3a5eda7f502737940aaf7944d99c58f2e,6-document-release-process,master,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,8,
-gitlab:GitlabMergeRequest:1:55146787,gitlab:GitlabProject:1:12345678,gitlab:GitlabProject:1:15706315,opened,Issue 3: Transformed model for query performance,,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/9,nehiljain,gitlab:GitlabAccount:1:783199,,9,2020-04-08T20:09:08.130+00:00,,,,,,issue_3,master,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,9,
-gitlab:GitlabMergeRequest:1:58311001,gitlab:GitlabProject:1:12345678,gitlab:GitlabProject:1:12345678,merged,Update version in readme,,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/10,emilie,gitlab:GitlabAccount:1:2295562,,10,2020-05-11T17:09:12.265+00:00,2020-05-11T17:09:20.603+00:00,,,,66c0f1de49a0c876b8f93e8e0dce3327e766f59d,emilie-master-patch-23079,master,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,10,
-gitlab:GitlabMergeRequest:1:62519057,gitlab:GitlabProject:1:12345678,gitlab:GitlabProject:1:19569570,opened,Clustering metering models,,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/11,jainnehil,gitlab:GitlabAccount:1:842680,,11,2020-06-24T12:34:04.792+00:00,,,,,,clustering-metering,master,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,11,
+Could a new release be made for dbt 1.0?",https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/19,johnj4,gitlab:GitlabAccount:1:10663622,,19,2022-01-18T19:59:30.723+00:00,,,,,,updates_for_dbt_1.0,master,,
+gitlab:GitlabMergeRequest:1:145012495,gitlab:GitlabProject:1:12345678,gitlab:GitlabProject:1:34491818,closed,Draft: Update dbt_project.yml,,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/20,PedramNavid,gitlab:GitlabAccount:1:9722492,,20,2022-03-15T03:07:06.077+00:00,,2022-03-15T03:07:22.665+00:00,,,,PedramNavid-master-patch-20645,master,,
+gitlab:GitlabMergeRequest:1:158698019,gitlab:GitlabProject:1:12345678,gitlab:GitlabProject:1:12345678,opened,Draft: Corrections for dbt 1,Closes https://gitlab.com/gitlab-data/analytics/-/issues/12941,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/21,paul_armstrong,gitlab:GitlabAccount:1:5618371,,21,2022-06-03T09:24:53.707+00:00,,,,,,updates_for_dbt_1_1,master,,
+gitlab:GitlabMergeRequest:1:32348491,gitlab:GitlabProject:1:12345678,gitlab:GitlabProject:1:12345678,merged,"Resolve ""Add documentation to snowflake spend package""",Closes #1,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/1,emilie,gitlab:GitlabAccount:1:2295562,,1,2019-06-28T05:21:43.743+00:00,2019-06-28T14:32:06.192+00:00,,,,da1d6dea48f5972ffc683da6cff30934e7d6c52c,1-add-documentation-to-snowflake-spend-package,master,,
+gitlab:GitlabMergeRequest:1:35064956,gitlab:GitlabProject:1:12345678,gitlab:GitlabProject:1:13835497,merged,Update README to include steps to resolve a potential dbt-utils conflict,Closes #5,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/3,martinguindon,gitlab:GitlabAccount:1:3871284,,3,2019-08-15T19:34:32.706+00:00,2019-08-26T14:15:27.922+00:00,,,,d678bea9d47b42eb13512d1c9d6a592d80b432d4,5-update-readme-to-include-steps-to-resolve-a-potential-dbt-utils-conflict,master,,
+gitlab:GitlabMergeRequest:1:35841926,gitlab:GitlabProject:1:12345678,gitlab:GitlabProject:1:12345678,merged,"Resolve ""Config is not generic enough""",Closes #4,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/4,emilie,gitlab:GitlabAccount:1:2295562,,4,2019-08-26T15:32:49.557+00:00,2019-08-26T15:37:50.105+00:00,,,,e95b5db25e15a38e21d11cb45cc21bf17d5c407c,4-config-is-not-generic-enough,master,,
+gitlab:GitlabMergeRequest:1:53445063,gitlab:GitlabProject:1:12345678,gitlab:GitlabProject:1:15706315,merged,Issue 3 Base model,,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/5,nehiljain,gitlab:GitlabAccount:1:783199,,5,2020-03-24T12:46:15.891+00:00,2020-03-25T18:36:45.801+00:00,,,,f2ee4cf121a328ce39723506dc18e4661941971a,issue_3,master,,
+gitlab:GitlabMergeRequest:1:53627854,gitlab:GitlabProject:1:12345678,gitlab:GitlabProject:1:15706063,merged,Update schema.yml typo in docs,,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/6,nehiljain,gitlab:GitlabAccount:1:783199,,6,2020-03-25T19:02:16.747+00:00,2020-03-25T19:04:19.844+00:00,,,,12dcc23a45adce0b12f8687438ec3a28274c7c30,patch-1,master,,
+gitlab:GitlabMergeRequest:1:55146687,gitlab:GitlabProject:1:12345678,gitlab:GitlabProject:1:12345678,merged,"Resolve ""Document release process""",Closes #6,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/8,m_walker,gitlab:GitlabAccount:1:5212782,,8,2020-04-08T20:07:10.223+00:00,2020-04-08T20:52:11.150+00:00,,,,7c8245a3a5eda7f502737940aaf7944d99c58f2e,6-document-release-process,master,,
+gitlab:GitlabMergeRequest:1:55146787,gitlab:GitlabProject:1:12345678,gitlab:GitlabProject:1:15706315,opened,Issue 3: Transformed model for query performance,,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/9,nehiljain,gitlab:GitlabAccount:1:783199,,9,2020-04-08T20:09:08.130+00:00,,,,,,issue_3,master,,
+gitlab:GitlabMergeRequest:1:58311001,gitlab:GitlabProject:1:12345678,gitlab:GitlabProject:1:12345678,merged,Update version in readme,,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/10,emilie,gitlab:GitlabAccount:1:2295562,,10,2020-05-11T17:09:12.265+00:00,2020-05-11T17:09:20.603+00:00,,,,66c0f1de49a0c876b8f93e8e0dce3327e766f59d,emilie-master-patch-23079,master,,
+gitlab:GitlabMergeRequest:1:62519057,gitlab:GitlabProject:1:12345678,gitlab:GitlabProject:1:19569570,opened,Clustering metering models,,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/11,jainnehil,gitlab:GitlabAccount:1:842680,,11,2020-06-24T12:34:04.792+00:00,,,,,,clustering-metering,master,,
gitlab:GitlabMergeRequest:1:65505080,gitlab:GitlabProject:1:12345678,gitlab:GitlabProject:1:12345678,merged,"Resolve ""Upgrade package for dbt 0.17""","Closes #11
* Upgrades to 0.17.0 format
-* Formatting changes to be in line with GitLab SQL Style Guide",https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/12,tayloramurphy,gitlab:GitlabAccount:1:1942272,,12,2020-07-24T17:47:08.238+00:00,2020-07-24T21:13:35.321+00:00,,,,9bfc136eb90802c2ce59956c34dde01bb3de0d50,11-upgrade-package-for-dbt-0-17,master,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,12,
-gitlab:GitlabMergeRequest:1:68978485,gitlab:GitlabProject:1:12345678,gitlab:GitlabProject:1:15706315,closed,Include more snowflake qrt columns,,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/13,aianus,gitlab:GitlabAccount:1:2478227,,13,2020-08-27T20:17:01.825+00:00,,2020-08-27T20:20:08.150+00:00,,,,include_more_snowflake_qrt_columns,master,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,13,
-gitlab:GitlabMergeRequest:1:89243644,gitlab:GitlabProject:1:12345678,gitlab:GitlabProject:1:24539973,merged,Update README.md to use the newest version as an example,Update README.md to use the newest version as an example. The old version doesn't work with the current version of dbt,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/14,ThomasLaPiana,gitlab:GitlabAccount:1:2061802,,14,2021-02-19T20:12:14.302+00:00,2021-02-19T20:13:05.969+00:00,,,,21840a7eadb58babe8aeae2960da8 [...]
+* Formatting changes to be in line with GitLab SQL Style Guide",https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/12,tayloramurphy,gitlab:GitlabAccount:1:1942272,,12,2020-07-24T17:47:08.238+00:00,2020-07-24T21:13:35.321+00:00,,,,9bfc136eb90802c2ce59956c34dde01bb3de0d50,11-upgrade-package-for-dbt-0-17,master,,
+gitlab:GitlabMergeRequest:1:68978485,gitlab:GitlabProject:1:12345678,gitlab:GitlabProject:1:15706315,closed,Include more snowflake qrt columns,,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/13,aianus,gitlab:GitlabAccount:1:2478227,,13,2020-08-27T20:17:01.825+00:00,,2020-08-27T20:20:08.150+00:00,,,,include_more_snowflake_qrt_columns,master,,
+gitlab:GitlabMergeRequest:1:89243644,gitlab:GitlabProject:1:12345678,gitlab:GitlabProject:1:24539973,merged,Update README.md to use the newest version as an example,Update README.md to use the newest version as an example. The old version doesn't work with the current version of dbt,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/14,ThomasLaPiana,gitlab:GitlabAccount:1:2061802,,14,2021-02-19T20:12:14.302+00:00,2021-02-19T20:13:05.969+00:00,,,,21840a7eadb58babe8aeae2960da8 [...]
diff --git a/backend/plugins/gitlab/models/migrationscripts/20230210_add_is_detail_required.go b/backend/plugins/gitlab/models/migrationscripts/20230210_add_is_detail_required.go
index 96222836c..8d19ce33f 100644
--- a/backend/plugins/gitlab/models/migrationscripts/20230210_add_is_detail_required.go
+++ b/backend/plugins/gitlab/models/migrationscripts/20230210_add_is_detail_required.go
@@ -21,10 +21,12 @@ import (
"github.com/apache/incubator-devlake/core/context"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/helpers/migrationhelper"
+ "time"
)
type gitlabMergeRequests20230210 struct {
IsDetailRequired bool
+ GitlabUpdatedAt *time.Time
}
func (gitlabMergeRequests20230210) TableName() string {
@@ -50,7 +52,7 @@ func (script *addIsDetailRequired20230210) Up(basicRes context.BasicRes) errors.
}
func (*addIsDetailRequired20230210) Version() uint64 {
- return 20230210161031
+ return 20230228161031
}
func (*addIsDetailRequired20230210) Name() string {
diff --git a/backend/plugins/gitlab/models/mr.go b/backend/plugins/gitlab/models/mr.go
index 05dff8fd4..e56b07bc3 100644
--- a/backend/plugins/gitlab/models/mr.go
+++ b/backend/plugins/gitlab/models/mr.go
@@ -41,6 +41,7 @@ type GitlabMergeRequest struct {
MergeCommitSha string `gorm:"type:varchar(255)"`
MergedAt *time.Time
GitlabCreatedAt time.Time
+ GitlabUpdatedAt *time.Time
ClosedAt *time.Time
Type string `gorm:"type:varchar(255)"`
MergedByUsername string `gorm:"type:varchar(255)"`
diff --git a/backend/plugins/gitlab/tasks/job_collector.go b/backend/plugins/gitlab/tasks/job_collector.go
index 5700b173e..9a7a0bf6c 100644
--- a/backend/plugins/gitlab/tasks/job_collector.go
+++ b/backend/plugins/gitlab/tasks/job_collector.go
@@ -18,13 +18,27 @@ limitations under the License.
package tasks
import (
+ "encoding/json"
+ "github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "github.com/apache/incubator-devlake/plugins/gitlab/models"
+ "io"
+ "net/http"
+ "net/url"
+ "reflect"
+ "strconv"
+ "time"
)
const RAW_JOB_TABLE = "gitlab_api_job"
+type SimpleGitlabApiJob struct {
+ GitlabId int
+ CreatedAt helper.Iso8601Time `json:"created_at"`
+}
+
var CollectApiJobsMeta = plugin.SubTaskMeta{
Name: "collectApiJobs",
EntryPoint: CollectApiJobs,
@@ -35,16 +49,70 @@ var CollectApiJobsMeta = plugin.SubTaskMeta{
func CollectApiJobs(taskCtx plugin.SubTaskContext) errors.Error {
rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_JOB_TABLE)
-
- collector, err := helper.NewApiCollector(helper.ApiCollectorArgs{
+ db := taskCtx.GetDal()
+ collector, err := helper.NewStatefulApiCollectorForFinalizableEntity(helper.FinalizableApiCollectorArgs{
RawDataSubTaskArgs: *rawDataSubTaskArgs,
ApiClient: data.ApiClient,
- PageSize: 100,
- Incremental: false,
- UrlTemplate: "projects/{{ .Params.ProjectId }}/jobs",
- Query: GetQuery,
- ResponseParser: GetRawMessageUpdatedAtAfter(data.TimeAfter),
- AfterResponse: ignoreHTTPStatus403, // ignore 403 for CI/CD disable
+ TimeAfter: data.TimeAfter, // set to nil to disable timeFilter
+ CollectNewRecordsByList: helper.FinalizableApiCollectorListArgs{
+ PageSize: 100,
+ Concurrency: 10,
+ FinalizableApiCollectorCommonArgs: helper.FinalizableApiCollectorCommonArgs{
+ UrlTemplate: "projects/{{ .Params.ProjectId }}/jobs",
+ Query: func(reqData *helper.RequestData, createdAfter *time.Time) (url.Values, errors.Error) {
+ query := url.Values{}
+ query.Set("page", strconv.Itoa(reqData.Pager.Page))
+ query.Set("per_page", strconv.Itoa(reqData.Pager.Size))
+ return query, nil
+ },
+ ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) {
+ var items []json.RawMessage
+ err := helper.UnmarshalResponse(res, &items)
+ if err != nil {
+ return nil, err
+ }
+ return items, nil
+ },
+ AfterResponse: ignoreHTTPStatus403, // ignore 403 for CI/CD disable
+ },
+ GetCreated: func(item json.RawMessage) (time.Time, errors.Error) {
+ pr := &SimpleGitlabApiJob{}
+ err := json.Unmarshal(item, pr)
+ if err != nil {
+ return time.Time{}, errors.BadInput.Wrap(err, "failed to unmarshal gitlab job")
+ }
+ return pr.CreatedAt.ToTime(), nil
+ },
+ },
+ CollectUnfinishedDetails: helper.FinalizableApiCollectorDetailArgs{
+ BuildInputIterator: func() (helper.Iterator, errors.Error) {
+ // select pull id from database
+ cursor, err := db.Cursor(
+ dal.Select("gitlab_id"),
+ dal.From(&models.GitlabJob{}),
+ dal.Where(
+ "project_id = ? AND connection_id = ? AND finished_at is null",
+ data.Options.ProjectId, data.Options.ConnectionId,
+ ),
+ )
+ if err != nil {
+ return nil, err
+ }
+ return helper.NewDalCursorIterator(db, cursor, reflect.TypeOf(SimpleGitlabApiJob{}))
+ },
+ FinalizableApiCollectorCommonArgs: helper.FinalizableApiCollectorCommonArgs{
+ UrlTemplate: "projects/{{ .Params.ProjectId }}/jobs/{{ .Input.GitlabId }}",
+ ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) {
+ body, err := io.ReadAll(res.Body)
+ if err != nil {
+ return nil, errors.Convert(err)
+ }
+ res.Body.Close()
+ return []json.RawMessage{body}, nil
+ },
+ AfterResponse: ignoreHTTPStatus403, // ignore 403 for CI/CD disable
+ },
+ },
})
if err != nil {
diff --git a/backend/plugins/gitlab/tasks/mr_extractor.go b/backend/plugins/gitlab/tasks/mr_extractor.go
index bd557f86e..a7e39c230 100644
--- a/backend/plugins/gitlab/tasks/mr_extractor.go
+++ b/backend/plugins/gitlab/tasks/mr_extractor.go
@@ -43,6 +43,7 @@ type MergeRequestRes struct {
SourceBranch string `json:"source_branch"`
TargetBranch string `json:"target_branch"`
GitlabCreatedAt api.Iso8601Time `json:"created_at"`
+ GitlabUpdatedAt *api.Iso8601Time `json:"updated_at"`
MergedAt *api.Iso8601Time `json:"merged_at"`
ClosedAt *api.Iso8601Time `json:"closed_at"`
MergeCommitSha string `json:"merge_commit_sha"`
@@ -193,6 +194,7 @@ func convertMergeRequest(mr *MergeRequestRes) (*models.GitlabMergeRequest, error
MergeCommitSha: mr.MergeCommitSha,
MergedAt: api.Iso8601TimeToTime(mr.MergedAt),
GitlabCreatedAt: mr.GitlabCreatedAt.ToTime(),
+ GitlabUpdatedAt: api.Iso8601TimeToTime(mr.GitlabUpdatedAt),
ClosedAt: api.Iso8601TimeToTime(mr.ClosedAt),
MergedByUsername: mr.MergedBy.Username,
AuthorUsername: mr.Author.Username,
diff --git a/backend/server/api/api.go b/backend/server/api/api.go
index 97dd76136..821437a7e 100644
--- a/backend/server/api/api.go
+++ b/backend/server/api/api.go
@@ -21,6 +21,7 @@ import (
"fmt"
"net/http"
"strconv"
+ "strings"
"time"
"github.com/apache/incubator-devlake/core/config"
@@ -120,6 +121,7 @@ func CreateApiService() {
func bootstrapRemotePlugins(v *viper.Viper) {
port := v.GetString("PORT")
+ port = strings.TrimLeft(port, ":")
portNum, err := strconv.Atoi(port)
if err != nil {
panic(fmt.Errorf("PORT [%s] must be int: %s", port, err.Error()))