You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by kl...@apache.org on 2024/11/05 06:13:44 UTC

(incubator-devlake) branch release-v1.0 updated: feat: gitlab extractors and convertors support incremental mode (#7997)

This is an automated email from the ASF dual-hosted git repository.

klesh pushed a commit to branch release-v1.0
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git


The following commit(s) were added to refs/heads/release-v1.0 by this push:
     new f9dfac1a5 feat: gitlab extractors and convertors support incremental mode (#7997)
f9dfac1a5 is described below

commit f9dfac1a5ff6833c0cab24a56c2d7c0133ab5423
Author: Klesh Wong <zh...@merico.dev>
AuthorDate: Fri Sep 27 09:00:15 2024 +0800

    feat: gitlab extractors and convertors support incremental mode (#7997)
    
    * feat: gitlab mr_extractor support incremental sync
    
    * feat: incr mode transformation support for deployment/issue/job/mr and others
    
    * feat: removed mr_enricher and all gitlab transformers support incr-mode
    
    * fix: some gitlab subtasks are missing connection in left join clause
    
    * fix: test cases failed due to gitlab transformers support incr-mode
    
    * fix: linting
    
    * fix: do not collect all accounts from jihulab.com
    
    * fix: typo
    
    * fix: gitlab mr comments won't be converted to domain layer till next run
    
    * feat: improve gitlab MR comments/commits collection performance
    
    * fix: gitlab issues/mrs child records are not deleted
    
    * docs: update stateful extractor doc
    
    * fix: gitlab mr detail test
    
    * refactor: unify stateful extractor and convertor helper
    
    * docs: update stateful extractor/convertor doc
    
    * refactor: remove useless type hint
    
    * fix: shoud not be deleting records not extracted by current extractor
    
    * fix: jira issue extractor should not deleting sprint_issue
    
    * refactor: remove commit related subtasks
    
    * fix: remove commit conversion e2e test
    
    * feat(gitlab): update ExtractApiChildPipelines (#8016)
    
    * fix: retransform should run in fullsync mode
    
    * fix: gitlab issue assignees are not being converted
    
    ---------
    
    Co-authored-by: Lynwee <15...@qq.com>
---
 .../pluginhelper/api/api_extractor_stateful.go     |  74 +++++++--
 backend/helpers/pluginhelper/api/api_rawdata.go    |   2 +-
 .../pluginhelper/api/data_convertor_stateful.go    |  79 +++++++++-
 .../helpers/pluginhelper/api/enrich_with_regex.go  |  61 +++++++-
 .../pluginhelper/api/subtask_state_manager.go      |   2 +-
 backend/impls/dalgorm/encdec_serializer.go         |   5 +
 backend/plugins/gitlab/e2e/mr_commits_test.go      |  33 -----
 backend/plugins/gitlab/e2e/mr_detail_test.go       |   2 +
 backend/plugins/gitlab/e2e/mr_enrich_test.go       |  79 ----------
 ..._tool_gitlab_merge_requests_for_enrich_test.csv |  25 ----
 ...ol_gitlab_merge_requests_for_mr_commit_test.csv |  12 +-
 ...ool_gitlab_merge_requests_for_mr_notes_test.csv |  34 ++---
 .../plugins/gitlab/e2e/snapshot_tables/commits.csv |  65 --------
 .../gitlab/e2e/snapshot_tables/repo_commits.csv    |  38 -----
 .../20240904_remove_mr_review_fields.go            |  40 +++++
 backend/plugins/gitlab/models/mr.go                |   4 +-
 backend/plugins/gitlab/tasks/account_collector.go  |   4 +-
 .../gitlab/tasks/child_pipeline_extractor.go       |  29 +---
 backend/plugins/gitlab/tasks/commit_collector.go   |  80 ----------
 backend/plugins/gitlab/tasks/commit_convertor.go   | 108 --------------
 backend/plugins/gitlab/tasks/commit_extractor.go   | 116 ---------------
 .../plugins/gitlab/tasks/deployment_convertor.go   |  40 +++--
 .../plugins/gitlab/tasks/deployment_extractor.go   |  16 +-
 .../gitlab/tasks/issue_assignee_convertor.go       |  49 +++---
 backend/plugins/gitlab/tasks/issue_convertor.go    |  55 ++++---
 backend/plugins/gitlab/tasks/issue_extractor.go    |  44 ++++--
 .../plugins/gitlab/tasks/issue_label_convertor.go  |  53 +++----
 backend/plugins/gitlab/tasks/job_convertor.go      |  51 +++----
 backend/plugins/gitlab/tasks/job_extractor.go      |  18 +--
 .../plugins/gitlab/tasks/mr_assignee_convertor.go  |  52 +++----
 .../plugins/gitlab/tasks/mr_commit_convertor.go    |  64 ++++----
 .../plugins/gitlab/tasks/mr_commit_extractor.go    |  56 +++++--
 backend/plugins/gitlab/tasks/mr_convertor.go       |  57 ++++---
 .../plugins/gitlab/tasks/mr_detail_extractor.go    |  25 ++--
 backend/plugins/gitlab/tasks/mr_enricher.go        | 165 ---------------------
 backend/plugins/gitlab/tasks/mr_enricher_test.go   | 112 --------------
 backend/plugins/gitlab/tasks/mr_extractor.go       |  78 +++++++---
 backend/plugins/gitlab/tasks/mr_label_convertor.go |  53 +++----
 ...r_comment_convertor.go => mr_note_convertor.go} |  55 +++----
 backend/plugins/gitlab/tasks/mr_note_extractor.go  |  18 +--
 .../plugins/gitlab/tasks/mr_reviewer_convertor.go  |  53 +++----
 .../gitlab/tasks/pipeline_commit_convertor.go      |  45 +++---
 .../gitlab/tasks/pipeline_detail_convertor.go      |  45 +++---
 .../gitlab/tasks/pipeline_detail_extractor.go      |  18 +--
 backend/plugins/gitlab/tasks/pipeline_extractor.go |  17 +--
 backend/plugins/gitlab/tasks/shared.go             |  24 ++-
 .../plugins/gitlab/tasks/trigger_job_extractor.go  |  17 +--
 backend/plugins/jira/tasks/epic_extractor.go       |  10 +-
 .../jira/tasks/issue_changelog_convertor.go        |   2 +-
 .../jira/tasks/issue_changelog_extractor.go        |   9 +-
 backend/plugins/jira/tasks/issue_convertor.go      |  13 +-
 backend/plugins/jira/tasks/issue_extractor.go      |  35 +++--
 backend/server/api/blueprints/blueprints.go        |   1 +
 .../src/routes/blueprint/detail/status-panel.tsx   |   2 +-
 54 files changed, 843 insertions(+), 1401 deletions(-)

diff --git a/backend/helpers/pluginhelper/api/api_extractor_stateful.go b/backend/helpers/pluginhelper/api/api_extractor_stateful.go
index 66ad5281a..2c994afdb 100644
--- a/backend/helpers/pluginhelper/api/api_extractor_stateful.go
+++ b/backend/helpers/pluginhelper/api/api_extractor_stateful.go
@@ -18,6 +18,7 @@ limitations under the License.
 package api
 
 import (
+	"encoding/json"
 	"reflect"
 
 	"github.com/apache/incubator-devlake/core/dal"
@@ -27,30 +28,72 @@ import (
 )
 
 // StatefulApiExtractorArgs is a struct that contains the arguments for a stateful api extractor
-type StatefulApiExtractorArgs struct {
+type StatefulApiExtractorArgs[InputType any] struct {
 	*SubtaskCommonArgs
-	Extract func(row *RawData) ([]any, errors.Error)
+	BeforeExtract func(issue *InputType, stateManager *SubtaskStateManager) errors.Error
+	Extract       func(body *InputType, row *RawData) ([]any, errors.Error)
 }
 
-type StatefulApiExtractor struct {
-	*StatefulApiExtractorArgs
+// StatefulApiExtractor is a struct that manages the stateful API extraction process.
+// It facilitates extracting data from a single _raw_data table and saving it into multiple Tool Layer tables.
+// By default, the extractor operates in Incremental Mode, processing only new records added to the raw table since the previous run.
+// This approach reduces the amount of data to process, significantly decreasing the execution time.
+// The extractor automatically detects if the configuration has changed since the last run. If a change is detected,
+// it will automatically switch to Full-Sync mode.
+//
+// Example:
+//
+//	extractor, err := api.NewStatefulApiExtractor(&api.StatefulApiExtractorArgs[apiv2models.Issue]{
+//	  SubtaskCommonArgs: &api.SubtaskCommonArgs{
+//	    SubTaskContext: subtaskCtx,
+//	    Table:          RAW_ISSUE_TABLE,
+//	    Params: JiraApiParams{
+//	      ConnectionId: data.Options.ConnectionId,
+//	      BoardId:      data.Options.BoardId,
+//	    },
+//	    SubtaskConfig: config,  // The helper stores this configuration in the state and compares it with the previous one
+//	                            // to determine the operating mode (Incremental/FullSync).
+//	                            // Ensure that the configuration is serializable and contains only public fields.
+//	                            // It is also recommended that the configuration includes only the necessary fields used by the extractor.
+//	..},
+//	  BeforeExtract: func(body *IssuesResponse, stateManager *api.SubtaskStateManager) errors.Error {
+//	    if stateManager.IsIncremental() {
+//	      // It is important to delete all existing child-records under DiffSync Mode
+//	      err := db.Delete(
+//	        &models.JiraIssueLabel{},
+//	        dal.Where("connection_id = ? AND issue_id = ?", data.Options.ConnectionId, body.Id),
+//	      )
+//	    }
+//	    return nil
+//	  },
+//	  Extract: func(apiIssue *apiv2models.Issue, row *api.RawData) ([]interface{}, errors.Error) {
+//	  },
+//	})
+//
+//	if err != nil {
+//	  return err
+//	}
+//
+// return extractor.Execute()
+type StatefulApiExtractor[InputType any] struct {
+	*StatefulApiExtractorArgs[InputType]
 	*SubtaskStateManager
 }
 
 // NewStatefulApiExtractor creates a new StatefulApiExtractor
-func NewStatefulApiExtractor(args *StatefulApiExtractorArgs) (*StatefulApiExtractor, errors.Error) {
+func NewStatefulApiExtractor[InputType any](args *StatefulApiExtractorArgs[InputType]) (*StatefulApiExtractor[InputType], errors.Error) {
 	stateManager, err := NewSubtaskStateManager(args.SubtaskCommonArgs)
 	if err != nil {
 		return nil, err
 	}
-	return &StatefulApiExtractor{
+	return &StatefulApiExtractor[InputType]{
 		StatefulApiExtractorArgs: args,
 		SubtaskStateManager:      stateManager,
 	}, nil
 }
 
 // Execute sub-task
-func (extractor *StatefulApiExtractor) Execute() errors.Error {
+func (extractor *StatefulApiExtractor[InputType]) Execute() errors.Error {
 	// load data from database
 	db := extractor.GetDal()
 	logger := extractor.GetLogger()
@@ -103,7 +146,20 @@ func (extractor *StatefulApiExtractor) Execute() errors.Error {
 			return errors.Default.Wrap(err, "error fetching row")
 		}
 
-		results, err := extractor.Extract(row)
+		body := new(InputType)
+		err = errors.Convert(json.Unmarshal(row.Data, body))
+		if err != nil {
+			return err
+		}
+
+		if extractor.BeforeExtract != nil {
+			err = extractor.BeforeExtract(body, extractor.SubtaskStateManager)
+			if err != nil {
+				return err
+			}
+		}
+
+		results, err := extractor.Extract(body, row)
 		if err != nil {
 			return errors.Default.Wrap(err, "error calling plugin Extract implementation")
 		}
@@ -137,4 +193,4 @@ func (extractor *StatefulApiExtractor) Execute() errors.Error {
 	return extractor.SubtaskStateManager.Close()
 }
 
-var _ plugin.SubTask = (*StatefulApiExtractor)(nil)
+var _ plugin.SubTask = (*StatefulApiExtractor[any])(nil)
diff --git a/backend/helpers/pluginhelper/api/api_rawdata.go b/backend/helpers/pluginhelper/api/api_rawdata.go
index ef16dde6d..5e9bb7835 100644
--- a/backend/helpers/pluginhelper/api/api_rawdata.go
+++ b/backend/helpers/pluginhelper/api/api_rawdata.go
@@ -34,7 +34,7 @@ type RawData struct {
 	Data      []byte
 	Url       string
 	Input     json.RawMessage `gorm:"type:json"`
-	CreatedAt time.Time
+	CreatedAt time.Time       `gorm:"index"`
 }
 
 type TaskOptions interface {
diff --git a/backend/helpers/pluginhelper/api/data_convertor_stateful.go b/backend/helpers/pluginhelper/api/data_convertor_stateful.go
index ddb710a2f..1c5292dc2 100644
--- a/backend/helpers/pluginhelper/api/data_convertor_stateful.go
+++ b/backend/helpers/pluginhelper/api/data_convertor_stateful.go
@@ -27,18 +27,82 @@ import (
 
 type StatefulDataConverterArgs[InputType any] struct {
 	*SubtaskCommonArgs
-	Input     func(*SubtaskStateManager) (dal.Rows, errors.Error)
-	Convert   func(row *InputType) ([]any, errors.Error)
-	BatchSize int
+	Input         func(*SubtaskStateManager) (dal.Rows, errors.Error)
+	BeforeConvert func(issue *InputType, stateManager *SubtaskStateManager) errors.Error
+	Convert       func(row *InputType) ([]any, errors.Error)
+	BatchSize     int
 }
 
+// StatefulDataConverter is a struct that manages the stateful data conversion process.
+// It facilitates converting data from a database cursor and saving it into arbitrary tables.
+// The converter determines the operating mode (Incremental/FullSync) based on the stored state and configuration.
+// It then calls the provided `Input` function to obtain the `dal.Rows` (the database cursor) and processes each
+// record individually through the `Convert` function, saving the results to the database.
+//
+// For Incremental mode to work properly, it is crucial to check `stateManager.IsIncremental()` and utilize
+// `stateManager.GetSince()` to build your query in the `Input` function, ensuring that only the necessary
+// records are fetched.
+//
+// The converter automatically detects if the configuration has changed since the last run. If a change is detected,
+// it will automatically switch to Full-Sync mode.
+//
+// Example:
+//
+// converter, err := api.NewStatefulDataConverter(&api.StatefulDataConverterArgs[models.JiraIssue]{
+// 	SubtaskCommonArgs: &api.SubtaskCommonArgs{
+// 		SubTaskContext: subtaskCtx,
+// 		Table:          RAW_ISSUE_TABLE,
+// 		Params: JiraApiParams{
+// 			ConnectionId: data.Options.ConnectionId,
+// 			BoardId:      data.Options.BoardId,
+// 		},
+// 		SubtaskConfig: mappings,
+// 	},
+// 	Input: func(stateManager *api.SubtaskStateManager) (dal.Rows, errors.Error) {
+// 		clauses := []dal.Clause{
+// 			dal.Select("_tool_jira_issues.*"),
+// 			dal.From("_tool_jira_issues"),
+// 			dal.Join(`left join _tool_jira_board_issues
+// 				on _tool_jira_board_issues.issue_id = _tool_jira_issues.issue_id
+// 				and _tool_jira_board_issues.connection_id = _tool_jira_issues.connection_id`),
+// 			dal.Where(
+// 				"_tool_jira_board_issues.connection_id = ? AND _tool_jira_board_issues.board_id = ?",
+// 				data.Options.ConnectionId,
+// 				data.Options.BoardId,
+// 			),
+// 		}
+// 		if stateManager.IsIncremental() { // IMPORTANT: to filter records for Incremental Mode
+// 			since := stateManager.GetSince()
+// 			if since != nil {
+// 				clauses = append(clauses, dal.Where("_tool_jira_issues.updated_at >= ? ", since))
+// 			}
+// 		}
+// 		return db.Cursor(clauses...)
+// 	},
+//	BeforeConvert: func(jiraIssue *models.GitlabMergeRequest, stateManager *api.SubtaskStateManager) errors.Error {
+//		// It is important to delete all existing child-records under DiffSync Mode
+//		issueId := issueIdGen.Generate(data.Options.ConnectionId, jiraIssue.IssueId)
+//		if err := db.Delete(&ticket.IssueAssignee{}, dal.Where("issue_id = ?", issueId)); err != nil {
+//			return err
+//		}
+//		...
+//		return nil
+//	},
+// 	Convert: func(jiraIssue *models.JiraIssue) ([]interface{}, errors.Error) {
+// 	},
+// })
+
+// if err != nil {
+// 	return err
+// }
+
+// return converter.Execute()
 type StatefulDataConverter[InputType any] struct {
 	*StatefulDataConverterArgs[InputType]
 	*SubtaskStateManager
 }
 
 func NewStatefulDataConverter[
-	OptType any,
 	InputType any,
 ](
 	args *StatefulDataConverterArgs[InputType],
@@ -91,6 +155,13 @@ func (converter *StatefulDataConverter[InputType]) Execute() errors.Error {
 			return errors.Default.Wrap(err, "error fetching rows")
 		}
 
+		if converter.BeforeConvert != nil {
+			err = converter.BeforeConvert(inputRow, converter.SubtaskStateManager)
+			if err != nil {
+				return err
+			}
+		}
+
 		results, err := converter.Convert(inputRow)
 		if err != nil {
 			return errors.Default.Wrap(err, "error calling Converter plugin implementation")
diff --git a/backend/helpers/pluginhelper/api/enrich_with_regex.go b/backend/helpers/pluginhelper/api/enrich_with_regex.go
index 5e7bd2a3a..b31fce78b 100644
--- a/backend/helpers/pluginhelper/api/enrich_with_regex.go
+++ b/backend/helpers/pluginhelper/api/enrich_with_regex.go
@@ -28,12 +28,13 @@ import (
 // TODO: remove Enricher from naming since it is more like a util function
 type RegexEnricher struct {
 	// This field will store compiled regular expression for every pattern
-	regexpMap map[string]*regexp.Regexp
+	regexpMap    map[string]*regexp.Regexp
+	regexMapList map[string][]*regexp.Regexp
 }
 
 // NewRegexEnricher initialize a regexEnricher
 func NewRegexEnricher() *RegexEnricher {
-	return &RegexEnricher{regexpMap: make(map[string]*regexp.Regexp)}
+	return &RegexEnricher{regexpMap: make(map[string]*regexp.Regexp), regexMapList: make(map[string][]*regexp.Regexp)}
 }
 
 // AddRegexp will add compiled regular expression for pattern to regexpMap
@@ -105,3 +106,59 @@ func (r *RegexEnricher) ReturnNameIfOmittedOrMatched(name string, targets ...str
 	}
 	return r.ReturnNameIfMatched(name, targets...)
 }
+
+func (r *RegexEnricher) PlainMap() map[string]string {
+	m := make(map[string]string)
+	for k, v := range r.regexpMap {
+		m[k] = v.String()
+	}
+	return m
+}
+
+// TryAdd a named regexp if given pattern is not empty
+func (r *RegexEnricher) TryAddList(name string, patterns ...string) errors.Error {
+	if _, ok := r.regexMapList[name]; ok {
+		return errors.Default.New(fmt.Sprintf("Regex pattern with name: %s already exists", name))
+	}
+	var regexList []*regexp.Regexp
+	for _, pattern := range patterns {
+		if pattern == "" {
+			continue
+		}
+		regex, err := errors.Convert01(regexp.Compile(pattern))
+		if err != nil {
+			return errors.BadInput.Wrap(err, fmt.Sprintf("Fail to compile pattern for regex pattern: %s", pattern))
+		}
+		regexList = append(regexList, regex)
+	}
+
+	// Only save non-empty regexList
+	if len(regexList) > 0 {
+		r.regexMapList[name] = regexList
+	}
+	return nil
+}
+
+// ReturnNameIfMatched will return name if any of the targets matches the regex associated with the given name
+func (r *RegexEnricher) ReturnNameIfMatchedList(name string, targets ...string) string {
+	if regexList, ok := r.regexMapList[name]; !ok {
+		return ""
+	} else {
+		for _, regex := range regexList {
+			for _, target := range targets {
+				if regex.MatchString(target) {
+					return name
+				}
+			}
+		}
+		return "" // If any regex fails to match, return ""
+	}
+}
+
+// ReturnNameIfOmittedOrMatched returns the given name if regex of the given name is omitted or fallback to ReturnNameIfMatched
+func (r *RegexEnricher) ReturnNameIfOmittedOrMatchedList(name string, targets ...string) string {
+	if _, ok := r.regexMapList[name]; !ok {
+		return name
+	}
+	return r.ReturnNameIfMatched(name, targets...)
+}
diff --git a/backend/helpers/pluginhelper/api/subtask_state_manager.go b/backend/helpers/pluginhelper/api/subtask_state_manager.go
index 407a291bf..7dd7d2370 100644
--- a/backend/helpers/pluginhelper/api/subtask_state_manager.go
+++ b/backend/helpers/pluginhelper/api/subtask_state_manager.go
@@ -34,7 +34,7 @@ type SubtaskCommonArgs struct {
 	plugin.SubTaskContext
 	Table         string // raw table name
 	Params        any    // for filtering rows belonging to the scope (jira board, github repo) of the subtask
-	SubtaskConfig any    // for determining whether the subtask should run in incremental or full sync mode
+	SubtaskConfig any    // for determining whether the subtask should run in Incremental or Full-Sync mode by comparing with the previous config to see if it changed
 	BatchSize     int    // batch size for saving data
 }
 
diff --git a/backend/impls/dalgorm/encdec_serializer.go b/backend/impls/dalgorm/encdec_serializer.go
index bb6589195..fd7a9d880 100644
--- a/backend/impls/dalgorm/encdec_serializer.go
+++ b/backend/impls/dalgorm/encdec_serializer.go
@@ -91,6 +91,11 @@ func (es *EncDecSerializer) Value(ctx context.Context, field *schema.Field, dst
 		}
 		target = string(b)
 	}
+	if field.GORMDataType == "string" {
+		println("field.GORMDataType == string", field.Size)
+		gormTag, ok := field.Tag.Lookup("gorm")
+		println(ok, gormTag)
+	}
 	return plugin.Encrypt(es.encryptionSecret, target)
 }
 
diff --git a/backend/plugins/gitlab/e2e/mr_commits_test.go b/backend/plugins/gitlab/e2e/mr_commits_test.go
index d5ef9d2a4..2f85cc775 100644
--- a/backend/plugins/gitlab/e2e/mr_commits_test.go
+++ b/backend/plugins/gitlab/e2e/mr_commits_test.go
@@ -133,37 +133,4 @@ func TestGitlabMrCommitDataFlow(t *testing.T) {
 		CSVRelPath:  "./snapshot_tables/pull_request_commits.csv",
 		IgnoreTypes: []interface{}{common.Model{}},
 	})
-
-	// verify conversion
-	dataflowTester.FlushTabler(&code.Commit{})
-	dataflowTester.FlushTabler(&code.RepoCommit{})
-	dataflowTester.Subtask(tasks.ConvertCommitsMeta, taskData)
-	dataflowTester.VerifyTable(
-		code.Commit{},
-		"./snapshot_tables/commits.csv",
-		e2ehelper.ColumnWithRawData(
-			"sha",
-			"additions",
-			"deletions",
-			"dev_eq",
-			"message",
-			"author_name",
-			"author_email",
-			"authored_date",
-			"author_id",
-			"committer_name",
-			"committer_email",
-			"committed_date",
-			"committer_id",
-		),
-	)
-
-	dataflowTester.VerifyTable(
-		code.RepoCommit{},
-		"./snapshot_tables/repo_commits.csv",
-		e2ehelper.ColumnWithRawData(
-			"repo_id",
-			"commit_sha",
-		),
-	)
 }
diff --git a/backend/plugins/gitlab/e2e/mr_detail_test.go b/backend/plugins/gitlab/e2e/mr_detail_test.go
index c92356676..02d1e5501 100644
--- a/backend/plugins/gitlab/e2e/mr_detail_test.go
+++ b/backend/plugins/gitlab/e2e/mr_detail_test.go
@@ -41,6 +41,8 @@ func TestGitlabMrDetailDataFlow(t *testing.T) {
 		},
 	}
 	// import raw data table
+	dataflowTester.FlushTabler(&code.PullRequestAssignee{})
+	dataflowTester.FlushTabler(&code.PullRequestReviewer{})
 	dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_gitlab_api_merge_requests.csv",
 		"_raw_gitlab_api_merge_request_details")
 
diff --git a/backend/plugins/gitlab/e2e/mr_enrich_test.go b/backend/plugins/gitlab/e2e/mr_enrich_test.go
deleted file mode 100644
index fd18003cc..000000000
--- a/backend/plugins/gitlab/e2e/mr_enrich_test.go
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements.  See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License.  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package e2e
-
-import (
-	"testing"
-
-	"github.com/apache/incubator-devlake/helpers/e2ehelper"
-	"github.com/apache/incubator-devlake/plugins/gitlab/impl"
-	"github.com/apache/incubator-devlake/plugins/gitlab/models"
-	"github.com/apache/incubator-devlake/plugins/gitlab/tasks"
-)
-
-func TestGitlabMrEnrichDataFlow(t *testing.T) {
-
-	var gitlab impl.Gitlab
-	dataflowTester := e2ehelper.NewDataFlowTester(t, "gitlab", gitlab)
-
-	taskData := &tasks.GitlabTaskData{
-		Options: &tasks.GitlabOptions{
-			ConnectionId: 1,
-			ProjectId:    12345678,
-			ScopeConfig:  new(models.GitlabScopeConfig),
-		},
-	}
-
-	// import data table
-	dataflowTester.ImportCsvIntoTabler("./snapshot_tables/_tool_gitlab_merge_requests.csv", &models.GitlabMergeRequest{})
-	dataflowTester.ImportCsvIntoTabler("./snapshot_tables/_tool_gitlab_mr_commits.csv", &models.GitlabMrCommit{})
-	dataflowTester.ImportCsvIntoTabler("./snapshot_tables/_tool_gitlab_mr_notes.csv", &models.GitlabMrNote{})
-
-	// verify extraction
-	dataflowTester.Subtask(tasks.EnrichMergeRequestsMeta, taskData)
-	dataflowTester.VerifyTable(
-		models.GitlabMergeRequest{},
-		"./snapshot_tables/_tool_gitlab_merge_requests_for_enrich_test.csv",
-		e2ehelper.ColumnWithRawData(
-			"connection_id",
-			"gitlab_id",
-			"iid",
-			"project_id",
-			"source_project_id",
-			"target_project_id",
-			"state",
-			"title",
-			"web_url",
-			"user_notes_count",
-			"work_in_progress",
-			"source_branch",
-			"target_branch",
-			"merge_commit_sha",
-			"merged_at",
-			"gitlab_created_at",
-			"closed_at",
-			"merged_by_username",
-			"description",
-			"author_username",
-			"author_user_id",
-			"component",
-			"first_comment_time",
-			"review_rounds",
-		),
-	)
-}
diff --git a/backend/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_merge_requests_for_enrich_test.csv b/backend/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_merge_requests_for_enrich_test.csv
deleted file mode 100644
index 56804c730..000000000
--- a/backend/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_merge_requests_for_enrich_test.csv
+++ /dev/null
@@ -1,25 +0,0 @@
-connection_id,gitlab_id,iid,project_id,source_project_id,target_project_id,state,title,web_url,user_notes_count,work_in_progress,source_branch,target_branch,merge_commit_sha,merged_at,gitlab_created_at,closed_at,merged_by_username,description,author_username,author_user_id,component,first_comment_time,review_rounds,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
-1,32348491,1,12345678,12345678,12345678,merged,"Resolve ""Add documentation to snowflake spend package""",https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/1,1,0,1-add-documentation-to-snowflake-spend-package,master,da1d6dea48f5972ffc683da6cff30934e7d6c52c,2019-06-28T14:32:06.192+00:00,2019-06-28T05:21:43.743+00:00,,tayloramurphy,Closes #1,emilie,2295562,,,2,,,0,
-1,35064956,3,12345678,13835497,12345678,merged,Update README to include steps to resolve a potential dbt-utils conflict,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/3,3,0,5-update-readme-to-include-steps-to-resolve-a-potential-dbt-utils-conflict,master,d678bea9d47b42eb13512d1c9d6a592d80b432d4,2019-08-26T14:15:27.922+00:00,2019-08-15T19:34:32.706+00:00,,emilie,Closes #5,martinguindon,3871284,,,1,,,0,
-1,35841926,4,12345678,12345678,12345678,merged,"Resolve ""Config is not generic enough""",https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/4,0,0,4-config-is-not-generic-enough,master,e95b5db25e15a38e21d11cb45cc21bf17d5c407c,2019-08-26T15:37:50.105+00:00,2019-08-26T15:32:49.557+00:00,,emilie,Closes #4,emilie,2295562,,,1,,,0,
-1,53445063,5,12345678,15706315,12345678,merged,Issue 3 Base model,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/5,20,0,issue_3,master,f2ee4cf121a328ce39723506dc18e4661941971a,2020-03-25T18:36:45.801+00:00,2020-03-24T12:46:15.891+00:00,,tayloramurphy,,nehiljain,783199,,,1,,,0,
-1,53627854,6,12345678,15706063,12345678,merged,Update schema.yml typo in docs,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/6,0,0,patch-1,master,12dcc23a45adce0b12f8687438ec3a28274c7c30,2020-03-25T19:04:19.844+00:00,2020-03-25T19:02:16.747+00:00,,tayloramurphy,,nehiljain,783199,,,1,,,0,
-1,55146687,8,12345678,12345678,12345678,merged,"Resolve ""Document release process""",https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/8,3,0,6-document-release-process,master,7c8245a3a5eda7f502737940aaf7944d99c58f2e,2020-04-08T20:52:11.150+00:00,2020-04-08T20:07:10.223+00:00,,emilie,Closes #6,m_walker,5212782,,,1,,,0,
-1,55146787,9,12345678,15706315,12345678,opened,Issue 3: Transformed model for query performance,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/9,14,0,issue_3,master,,,2020-04-08T20:09:08.130+00:00,,,,nehiljain,783199,,,1,,,0,
-1,58311001,10,12345678,12345678,12345678,merged,Update version in readme,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/10,0,0,emilie-master-patch-23079,master,66c0f1de49a0c876b8f93e8e0dce3327e766f59d,2020-05-11T17:09:20.603+00:00,2020-05-11T17:09:12.265+00:00,,emilie,,emilie,2295562,,,1,,,0,
-1,62519057,11,12345678,19569570,12345678,opened,Clustering metering models,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/11,0,0,clustering-metering,master,,,2020-06-24T12:34:04.792+00:00,,,,jainnehil,842680,,,1,,,0,
-1,65505080,12,12345678,12345678,12345678,merged,"Resolve ""Upgrade package for dbt 0.17""",https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/12,0,0,11-upgrade-package-for-dbt-0-17,master,9bfc136eb90802c2ce59956c34dde01bb3de0d50,2020-07-24T21:13:35.321+00:00,2020-07-24T17:47:08.238+00:00,,tayloramurphy,"Closes #11 
-
-* Upgrades to 0.17.0 format
-* Formatting changes to be in line with GitLab SQL Style Guide",tayloramurphy,1942272,,,1,,,0,
-1,68978485,13,12345678,15706315,12345678,closed,Include more snowflake qrt columns,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/13,0,0,include_more_snowflake_qrt_columns,master,,,2020-08-27T20:17:01.825+00:00,2020-08-27T20:20:08.150+00:00,,,aianus,2478227,,,1,,,0,
-1,89243644,14,12345678,24539973,12345678,merged,Update README.md to use the newest version as an example,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/14,0,0,ThomasLaPiana-master-patch-93997,master,21840a7eadb58babe8aeae2960da851a3ed00ddc,2021-02-19T20:13:05.969+00:00,2021-02-19T20:12:14.302+00:00,,tayloramurphy,Update README.md to use the newest version as an example. The old version doesn't work with the current version of dbt,ThomasLaPiana,2061802,,,1,,,0,
-1,110817220,16,12345678,28584714,12345678,merged,Update packages.yml to point to dbt-labs instead of fishtown,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/16,5,0,GJMcClintock-master-patch-24867,master,6f45b467c478df1c67d19cf6d4cbb8e05a710662,2021-08-12T06:12:54.329+00:00,2021-08-03T15:02:54.955+00:00,,vedprakash2021,With the company name change the old repo is deprecated.,GJMcClintock,9439881,,,1,,,0,
-1,111383524,17,12345678,0,12345678,closed,The package name changed -> https://hub.getdbt.com/dbt-labs/dbt_utils/latest/,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/17,1,0,swiffer-master-patch-77533,master,,,2021-08-07T06:50:25.458+00:00,2021-08-07T06:51:14.933+00:00,,,swiffer,156402,,,1,,,0,
-1,114994501,18,12345678,29298577,12345678,opened,Add support for Snowpipe usage monitoring,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/18,0,0,master,master,,,2021-09-01T21:15:30.334+00:00,,,Add models and docs for Snowflake pipes (Snowpipe) usage monitoring based on the views in Snowflake Usage schema,gary-beautypie,9635687,,,1,,,0,
-1,135775405,19,12345678,32935405,12345678,opened,Updates for dbt 1.0,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/19,1,0,updates_for_dbt_1.0,master,,,2022-01-18T19:59:30.723+00:00,,,"This MR sets up the repo for dbt 1.0
-A few configs were renamed.
-
-Could a new release be made for dbt 1.0?",johnj4,10663622,,,1,,,0,
-1,145012495,20,12345678,34491818,12345678,closed,Draft: Update dbt_project.yml,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/20,0,1,PedramNavid-master-patch-20645,master,,,2022-03-15T03:07:06.077+00:00,2022-03-15T03:07:22.665+00:00,,,PedramNavid,9722492,,,1,,,0,
-1,158698019,21,12345678,12345678,12345678,opened,Draft: Corrections for dbt 1,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/21,0,1,updates_for_dbt_1_1,master,,,2022-06-03T09:24:53.707+00:00,,,Closes https://gitlab.com/gitlab-data/analytics/-/issues/12941,paul_armstrong,5618371,,,1,,,0,
diff --git a/backend/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_merge_requests_for_mr_commit_test.csv b/backend/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_merge_requests_for_mr_commit_test.csv
index 2ba28fadb..dc468a58b 100644
--- a/backend/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_merge_requests_for_mr_commit_test.csv
+++ b/backend/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_merge_requests_for_mr_commit_test.csv
@@ -1,5 +1,5 @@
-connection_id,gitlab_id,iid,project_id,source_project_id,target_project_id,state,title,web_url,user_notes_count,work_in_progress,source_branch,target_branch,merge_commit_sha,merged_at,gitlab_created_at,closed_at,merged_by_username,description,author_username,author_user_id,component,first_comment_time,review_rounds,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
-1,32348491,1,12345678,12345678,12345678,merged,"Resolve ""Add documentation to snowflake spend package""",https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/1,1,0,1-add-documentation-to-snowflake-spend-package,master,da1d6dea48f5972ffc683da6cff30934e7d6c52c,2019-06-28T14:32:06.192+00:00,2019-06-28T05:21:43.743+00:00,,tayloramurphy,Closes #1,emilie,2295562,,,0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,1,
+connection_id,gitlab_id,iid,project_id,source_project_id,target_project_id,state,title,web_url,user_notes_count,work_in_progress,source_branch,target_branch,merge_commit_sha,merged_at,gitlab_created_at,closed_at,merged_by_username,description,author_username,author_user_id,component,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
+1,32348491,1,12345678,12345678,12345678,merged,"Resolve ""Add documentation to snowflake spend package""",https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/1,1,0,1-add-documentation-to-snowflake-spend-package,master,da1d6dea48f5972ffc683da6cff30934e7d6c52c,2019-06-28T14:32:06.192+00:00,2019-06-28T05:21:43.743+00:00,,tayloramurphy,Closes #1,emilie,2295562,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,1,
 1,34300715,2,12345678,12345678,12345678,merged,"Resolve ""Handle multiple prices and date ranges""",https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/2,9,0,2-handle-multiple-prices-and-date-ranges,master,2fe82576fdbd0a43a2d5e02c17c6dc6ce3e188ee,2019-08-04T22:24:41.219+00:00,2019-08-01T21:36:45.020+00:00,,emilie,"Closes #2 
 
 * Adds a model to amortize the snowflake_contract_rates.csv properly
@@ -8,7 +8,7 @@ connection_id,gitlab_id,iid,project_id,source_project_id,target_project_id,state
 
 Testing by running dbt seed on https://gitlab.com/gitlab-data/analytics/merge_requests/1351/diffs and pinning dbt deps to this branch `2-handle...`.
 
-All models ran fine and the data looks great in my scratch schema!",tayloramurphy,1942272,,,0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,2,
-1,35064956,3,12345678,13835497,12345678,merged,Update README to include steps to resolve a potential dbt-utils conflict,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/3,3,0,5-update-readme-to-include-steps-to-resolve-a-potential-dbt-utils-conflict,master,d678bea9d47b42eb13512d1c9d6a592d80b432d4,2019-08-26T14:15:27.922+00:00,2019-08-15T19:34:32.706+00:00,,emilie,Closes #5,martinguindon,3871284,,,0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,3,
-1,35841926,4,12345678,12345678,12345678,merged,"Resolve ""Config is not generic enough""",https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/4,0,0,4-config-is-not-generic-enough,master,e95b5db25e15a38e21d11cb45cc21bf17d5c407c,2019-08-26T15:37:50.105+00:00,2019-08-26T15:32:49.557+00:00,,emilie,Closes #4,emilie,2295562,,,0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,4,
-1,53445063,5,12345678,15706315,12345678,merged,Issue 3 Base model,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/5,20,0,issue_3,master,f2ee4cf121a328ce39723506dc18e4661941971a,2020-03-25T18:36:45.801+00:00,2020-03-24T12:46:15.891+00:00,,tayloramurphy,,nehiljain,783199,,,0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,5,
+All models ran fine and the data looks great in my scratch schema!",tayloramurphy,1942272,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,2,
+1,35064956,3,12345678,13835497,12345678,merged,Update README to include steps to resolve a potential dbt-utils conflict,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/3,3,0,5-update-readme-to-include-steps-to-resolve-a-potential-dbt-utils-conflict,master,d678bea9d47b42eb13512d1c9d6a592d80b432d4,2019-08-26T14:15:27.922+00:00,2019-08-15T19:34:32.706+00:00,,emilie,Closes #5,martinguindon,3871284,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,3,
+1,35841926,4,12345678,12345678,12345678,merged,"Resolve ""Config is not generic enough""",https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/4,0,0,4-config-is-not-generic-enough,master,e95b5db25e15a38e21d11cb45cc21bf17d5c407c,2019-08-26T15:37:50.105+00:00,2019-08-26T15:32:49.557+00:00,,emilie,Closes #4,emilie,2295562,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,4,
+1,53445063,5,12345678,15706315,12345678,merged,Issue 3 Base model,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/5,20,0,issue_3,master,f2ee4cf121a328ce39723506dc18e4661941971a,2020-03-25T18:36:45.801+00:00,2020-03-24T12:46:15.891+00:00,,tayloramurphy,,nehiljain,783199,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,5,
diff --git a/backend/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_merge_requests_for_mr_notes_test.csv b/backend/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_merge_requests_for_mr_notes_test.csv
index fe38d4100..05d7a0c7d 100644
--- a/backend/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_merge_requests_for_mr_notes_test.csv
+++ b/backend/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_merge_requests_for_mr_notes_test.csv
@@ -1,6 +1,6 @@
-connection_id,gitlab_id,iid,project_id,source_project_id,target_project_id,state,title,web_url,user_notes_count,work_in_progress,source_branch,target_branch,merge_commit_sha,merged_at,gitlab_created_at,closed_at,merged_by_username,description,author_username,author_user_id,component,first_comment_time,review_rounds,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
-1,15869219,37,12345678,12345678,12345678,opened,Draft: Corrections for dbt 1,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/21,0,1,updates_for_dbt_1_1,master,,,2022-06-03T09:24:53.707+00:00,,,Closes https://gitlab.com/gitlab-data/analytics/-/issues/12941,paul_armstrong,5618371,,,0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,21,
-1,32348491,1,12345678,12345678,12345678,merged,"Resolve ""Add documentation to snowflake spend package""",https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/1,1,0,1-add-documentation-to-snowflake-spend-package,master,da1d6dea48f5972ffc683da6cff30934e7d6c52c,2019-06-28T14:32:06.192+00:00,2019-06-28T05:21:43.743+00:00,,tayloramurphy,Closes #1,emilie,2295562,,,0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,1,
+connection_id,gitlab_id,iid,project_id,source_project_id,target_project_id,state,title,web_url,user_notes_count,work_in_progress,source_branch,target_branch,merge_commit_sha,merged_at,gitlab_created_at,closed_at,merged_by_username,description,author_username,author_user_id,component,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
+1,15869219,37,12345678,12345678,12345678,opened,Draft: Corrections for dbt 1,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/21,0,1,updates_for_dbt_1_1,master,,,2022-06-03T09:24:53.707+00:00,,,Closes https://gitlab.com/gitlab-data/analytics/-/issues/12941,paul_armstrong,5618371,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,21,
+1,32348491,1,12345678,12345678,12345678,merged,"Resolve ""Add documentation to snowflake spend package""",https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/1,1,0,1-add-documentation-to-snowflake-spend-package,master,da1d6dea48f5972ffc683da6cff30934e7d6c52c,2019-06-28T14:32:06.192+00:00,2019-06-28T05:21:43.743+00:00,,tayloramurphy,Closes #1,emilie,2295562,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,1,
 1,34300715,2,12345678,12345678,12345678,merged,"Resolve ""Handle multiple prices and date ranges""",https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/2,9,0,2-handle-multiple-prices-and-date-ranges,master,2fe82576fdbd0a43a2d5e02c17c6dc6ce3e188ee,2019-08-04T22:24:41.219+00:00,2019-08-01T21:36:45.020+00:00,,emilie,"Closes #2 
 
 * Adds a model to amortize the snowflake_contract_rates.csv properly
@@ -9,23 +9,23 @@ connection_id,gitlab_id,iid,project_id,source_project_id,target_project_id,state
 
 Testing by running dbt seed on https://gitlab.com/gitlab-data/analytics/merge_requests/1351/diffs and pinning dbt deps to this branch `2-handle...`.
 
-All models ran fine and the data looks great in my scratch schema!",tayloramurphy,1942272,,,0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,2,
-1,35064956,3,12345678,13835497,12345678,merged,Update README to include steps to resolve a potential dbt-utils conflict,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/3,3,0,5-update-readme-to-include-steps-to-resolve-a-potential-dbt-utils-conflict,master,d678bea9d47b42eb13512d1c9d6a592d80b432d4,2019-08-26T14:15:27.922+00:00,2019-08-15T19:34:32.706+00:00,,emilie,Closes #5,martinguindon,3871284,,,0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,3,
-1,35841926,4,12345678,12345678,12345678,merged,"Resolve ""Config is not generic enough""",https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/4,0,0,4-config-is-not-generic-enough,master,e95b5db25e15a38e21d11cb45cc21bf17d5c407c,2019-08-26T15:37:50.105+00:00,2019-08-26T15:32:49.557+00:00,,emilie,Closes #4,emilie,2295562,,,0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,4,
-1,53445063,5,12345678,15706315,12345678,merged,Issue 3 Base model,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/5,20,0,issue_3,master,f2ee4cf121a328ce39723506dc18e4661941971a,2020-03-25T18:36:45.801+00:00,2020-03-24T12:46:15.891+00:00,,tayloramurphy,,nehiljain,783199,,,0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,5,
-1,53627854,6,12345678,15706063,12345678,merged,Update schema.yml typo in docs,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/6,0,0,patch-1,master,12dcc23a45adce0b12f8687438ec3a28274c7c30,2020-03-25T19:04:19.844+00:00,2020-03-25T19:02:16.747+00:00,,tayloramurphy,,nehiljain,783199,,,0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,6,
-1,53640548,7,12345678,12345678,12345678,merged,"Resolve ""Add sources and cleanup structure""",https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/7,5,0,7-add-sources-and-cleanup-structure,master,1161de80448bc66072c3ea39600d9940dde87a5b,2020-04-07T12:51:50.096+00:00,2020-03-25T21:58:40.647+00:00,,emilie,Closes #7,tayloramurphy,1942272,,,0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,7,
-1,55146687,8,12345678,12345678,12345678,merged,"Resolve ""Document release process""",https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/8,3,0,6-document-release-process,master,7c8245a3a5eda7f502737940aaf7944d99c58f2e,2020-04-08T20:52:11.150+00:00,2020-04-08T20:07:10.223+00:00,,emilie,Closes #6,m_walker,5212782,,,0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,8,
-1,55146787,9,12345678,15706315,12345678,opened,Issue 3: Transformed model for query performance,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/9,14,0,issue_3,master,,,2020-04-08T20:09:08.130+00:00,,,,nehiljain,783199,,,0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,9,
-1,58311001,10,12345678,12345678,12345678,merged,Update version in readme,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/10,0,0,emilie-master-patch-23079,master,66c0f1de49a0c876b8f93e8e0dce3327e766f59d,2020-05-11T17:09:20.603+00:00,2020-05-11T17:09:12.265+00:00,,emilie,,emilie,2295562,,,0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,10,
-1,62519057,11,12345678,19569570,12345678,opened,Clustering metering models,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/11,0,0,clustering-metering,master,,,2020-06-24T12:34:04.792+00:00,,,,jainnehil,842680,,,0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,11,
+All models ran fine and the data looks great in my scratch schema!",tayloramurphy,1942272,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,2,
+1,35064956,3,12345678,13835497,12345678,merged,Update README to include steps to resolve a potential dbt-utils conflict,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/3,3,0,5-update-readme-to-include-steps-to-resolve-a-potential-dbt-utils-conflict,master,d678bea9d47b42eb13512d1c9d6a592d80b432d4,2019-08-26T14:15:27.922+00:00,2019-08-15T19:34:32.706+00:00,,emilie,Closes #5,martinguindon,3871284,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,3,
+1,35841926,4,12345678,12345678,12345678,merged,"Resolve ""Config is not generic enough""",https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/4,0,0,4-config-is-not-generic-enough,master,e95b5db25e15a38e21d11cb45cc21bf17d5c407c,2019-08-26T15:37:50.105+00:00,2019-08-26T15:32:49.557+00:00,,emilie,Closes #4,emilie,2295562,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,4,
+1,53445063,5,12345678,15706315,12345678,merged,Issue 3 Base model,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/5,20,0,issue_3,master,f2ee4cf121a328ce39723506dc18e4661941971a,2020-03-25T18:36:45.801+00:00,2020-03-24T12:46:15.891+00:00,,tayloramurphy,,nehiljain,783199,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,5,
+1,53627854,6,12345678,15706063,12345678,merged,Update schema.yml typo in docs,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/6,0,0,patch-1,master,12dcc23a45adce0b12f8687438ec3a28274c7c30,2020-03-25T19:04:19.844+00:00,2020-03-25T19:02:16.747+00:00,,tayloramurphy,,nehiljain,783199,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,6,
+1,53640548,7,12345678,12345678,12345678,merged,"Resolve ""Add sources and cleanup structure""",https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/7,5,0,7-add-sources-and-cleanup-structure,master,1161de80448bc66072c3ea39600d9940dde87a5b,2020-04-07T12:51:50.096+00:00,2020-03-25T21:58:40.647+00:00,,emilie,Closes #7,tayloramurphy,1942272,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,7,
+1,55146687,8,12345678,12345678,12345678,merged,"Resolve ""Document release process""",https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/8,3,0,6-document-release-process,master,7c8245a3a5eda7f502737940aaf7944d99c58f2e,2020-04-08T20:52:11.150+00:00,2020-04-08T20:07:10.223+00:00,,emilie,Closes #6,m_walker,5212782,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,8,
+1,55146787,9,12345678,15706315,12345678,opened,Issue 3: Transformed model for query performance,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/9,14,0,issue_3,master,,,2020-04-08T20:09:08.130+00:00,,,,nehiljain,783199,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,9,
+1,58311001,10,12345678,12345678,12345678,merged,Update version in readme,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/10,0,0,emilie-master-patch-23079,master,66c0f1de49a0c876b8f93e8e0dce3327e766f59d,2020-05-11T17:09:20.603+00:00,2020-05-11T17:09:12.265+00:00,,emilie,,emilie,2295562,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,10,
+1,62519057,11,12345678,19569570,12345678,opened,Clustering metering models,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/11,0,0,clustering-metering,master,,,2020-06-24T12:34:04.792+00:00,,,,jainnehil,842680,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,11,
 1,65505080,12,12345678,12345678,12345678,merged,"Resolve ""Upgrade package for dbt 0.17""",https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/12,0,0,11-upgrade-package-for-dbt-0-17,master,9bfc136eb90802c2ce59956c34dde01bb3de0d50,2020-07-24T21:13:35.321+00:00,2020-07-24T17:47:08.238+00:00,,tayloramurphy,"Closes #11 
 
 * Upgrades to 0.17.0 format
-* Formatting changes to be in line with GitLab SQL Style Guide",tayloramurphy,1942272,,,0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,12,
+* Formatting changes to be in line with GitLab SQL Style Guide",tayloramurphy,1942272,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,12,
 1,135772105,30,12345678,32935405,12345678,opened,Updates for dbt 1.0,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/19,1,0,updates_for_dbt_1.0,master,,,2022-01-18T19:59:30.723+00:00,,,"This MR sets up the repo for dbt 1.0
 A few configs were renamed.
 
-Could a new release be made for dbt 1.0?",johnj4,10663622,,,0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,19,
-1,145032495,46,12345678,34491818,12345678,closed,Draft: Update dbt_project.yml,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/20,0,1,PedramNavid-master-patch-20645,master,,,2022-03-15T03:07:06.077+00:00,2022-03-15T03:07:22.665+00:00,,,PedramNavid,9722492,,,0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,20,
-1,1149942101,29,12345678,29298577,12345678,opened,Add support for Snowpipe usage monitoring,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/18,0,0,master,master,,,2021-09-01T21:15:30.334+00:00,,,Add models and docs for Snowflake pipes (Snowpipe) usage monitoring based on the views in Snowflake Usage schema,gary-beautypie,9635687,,,0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,18,
+Could a new release be made for dbt 1.0?",johnj4,10663622,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,19,
+1,145032495,46,12345678,34491818,12345678,closed,Draft: Update dbt_project.yml,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/20,0,1,PedramNavid-master-patch-20645,master,,,2022-03-15T03:07:06.077+00:00,2022-03-15T03:07:22.665+00:00,,,PedramNavid,9722492,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,20,
+1,1149942101,29,12345678,29298577,12345678,opened,Add support for Snowpipe usage monitoring,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/18,0,0,master,master,,,2021-09-01T21:15:30.334+00:00,,,Add models and docs for Snowflake pipes (Snowpipe) usage monitoring based on the views in Snowflake Usage schema,gary-beautypie,9635687,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,18,
diff --git a/backend/plugins/gitlab/e2e/snapshot_tables/commits.csv b/backend/plugins/gitlab/e2e/snapshot_tables/commits.csv
deleted file mode 100644
index ea3ba115e..000000000
--- a/backend/plugins/gitlab/e2e/snapshot_tables/commits.csv
+++ /dev/null
@@ -1,65 +0,0 @@
-sha,additions,deletions,dev_eq,message,author_name,author_email,authored_date,author_id,committer_name,committer_email,committed_date,committer_id,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
-05976c113b02fb71d5c43682bd2f9d96a3454efa,0,0,0,"tab refcatoring
-",Nehil Jain,nehil@snaptravel.com,2020-03-24T18:14:03.000+00:00,nehil@snaptravel.com,Nehil Jain,nehil@snaptravel.com,2020-03-24T18:14:03.000+00:00,nehil@snaptravel.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,34,
-083864201849f6707a34d24e61c43638696f95a6,0,0,0,"Update README to include steps to resolve a potential dbt-utils conflict
-",Martin Guindon,mguindon@absolunet.com,2019-08-15T19:32:19.000+00:00,mguindon@absolunet.com,Martin Guindon,mguindon@absolunet.com,2019-08-15T19:32:19.000+00:00,mguindon@absolunet.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,6,
-08973065d33c6fd2fb1f592b7a1e0fbe2ee72edd,0,0,0,"add usage day
-",Taylor A. Murphy,tmurphy@gitlab.com,2019-08-02T18:35:08.000+00:00,tmurphy@gitlab.com,Taylor A. Murphy,tmurphy@gitlab.com,2019-08-02T18:35:08.000+00:00,tmurphy@gitlab.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,11,
-0a4574d52b5964c37bfcb2952f6c6a8a9c6d5bcf,0,0,0,"fix name
-",Taylor A. Murphy,tmurphy@gitlab.com,2019-08-02T19:00:50.000+00:00,tmurphy@gitlab.com,Taylor A. Murphy,tmurphy@gitlab.com,2019-08-02T19:00:50.000+00:00,tmurphy@gitlab.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,7,
-0dbb8bfdbbf7373656fa3eb036bdcbb25ae162da,0,0,0,Apply suggestion to models/base/snowflake_query_history.sql,Nehil,nehil@getsnaptravel.com,2020-03-24T22:39:40.000+00:00,nehil@getsnaptravel.com,Nehil,nehil@getsnaptravel.com,2020-03-24T22:39:40.000+00:00,nehil@getsnaptravel.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,26,
-1cba71b355d2bd4e1350602cfb9b98cce5a5cb4d,0,0,0,"update documentation
-",Taylor A. Murphy,tmurphy@gitlab.com,2019-08-02T18:53:44.000+00:00,tmurphy@gitlab.com,Taylor A. Murphy,tmurphy@gitlab.com,2019-08-02T18:53:44.000+00:00,tmurphy@gitlab.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,8,
-1f16955d45cec904c8968bb27689d91a71e9a2f2,0,0,0,"change cte address
-",Taylor A. Murphy,tmurphy@gitlab.com,2019-08-02T17:28:44.000+00:00,tmurphy@gitlab.com,Taylor A. Murphy,tmurphy@gitlab.com,2019-08-02T17:28:44.000+00:00,tmurphy@gitlab.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,12,
-211795deaaaaddb23b797261cdcd148e6083d5b2,0,0,0,Update snowflake_query_history.sql,Nehil,nehil@getsnaptravel.com,2020-03-24T22:44:07.000+00:00,nehil@getsnaptravel.com,Nehil,nehil@getsnaptravel.com,2020-03-24T22:44:07.000+00:00,nehil@getsnaptravel.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,21,
-25f2b34099a3e1867fcf73a54e50f78f3c8d8f75,0,0,0,"add documentation
-",Taylor A. Murphy,tmurphy@gitlab.com,2019-08-02T18:47:28.000+00:00,tmurphy@gitlab.com,Taylor A. Murphy,tmurphy@gitlab.com,2019-08-02T18:47:28.000+00:00,tmurphy@gitlab.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,9,
-382084b42697577d3a6adf71ce73d4b5ddd22977,0,0,0,"add info on dashboard
-",emilie,emilie.burke@gmail.com,2019-06-28T10:54:26.000+00:00,emilie.burke@gmail.com,emilie,emilie.burke@gmail.com,2019-06-28T10:54:26.000+00:00,emilie.burke@gmail.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,2,
-3f04e0a61d0c4d2dd736a6bcaa3a06826269a533,0,0,0,"finish top level readme
-",emilie,emilie.burke@gmail.com,2019-06-28T10:48:49.000+00:00,emilie.burke@gmail.com,emilie,emilie.burke@gmail.com,2019-06-28T10:48:49.000+00:00,emilie.burke@gmail.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,3,
-45bed79fcdc6ba3444245f55660f839db50d79c8,0,0,0,Update .gitignore,Nehil,nehil@getsnaptravel.com,2020-03-24T18:15:29.000+00:00,nehil@getsnaptravel.com,Nehil,nehil@getsnaptravel.com,2020-03-24T18:15:29.000+00:00,nehil@getsnaptravel.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,33,
-48295660f8bd0c0a024849d4407830face57d4bc,0,0,0,"add docs and sources
-",emilie,emilie.burke@gmail.com,2020-04-05T14:39:43.000+00:00,emilie.burke@gmail.com,emilie,emilie.burke@gmail.com,2020-04-05T14:39:43.000+00:00,emilie.burke@gmail.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,37,
-4f0d2f7b1a47d21b01a68e59c12d6f8885ab230e,0,0,0,"mr comments
-",Nehil Jain,nehil@snaptravel.com,2020-03-24T22:40:36.000+00:00,nehil@snaptravel.com,Nehil Jain,nehil@snaptravel.com,2020-03-24T22:40:36.000+00:00,nehil@snaptravel.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,24,
-553f7613b178301da4075202afe91ebda5d97ab5,0,0,0,Update schema.yml typo for docs,Nehil,nehil@getsnaptravel.com,2020-03-25T19:01:57.000+00:00,nehil@getsnaptravel.com,Nehil,nehil@getsnaptravel.com,2020-03-25T19:01:57.000+00:00,nehil@getsnaptravel.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,16,
-5a141ad0c28960b02c40e6c476d8df3cc76f58c4,0,0,0,"schema yml, untested
-",Nehil Jain,2761999+nehiljain@users.noreply.github.com,2020-03-24T21:18:16.000+00:00,2761999+nehiljain@users.noreply.github.com,Nehil Jain,2761999+nehiljain@users.noreply.github.com,2020-03-24T21:18:16.000+00:00,2761999+nehiljain@users.noreply.github.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,31,
-5b6a2be437e0c66ba6276030a61a10583c3f83fa,0,0,0,"merging online changes
-",Nehil Jain,nehil@snaptravel.com,2020-03-24T22:42:54.000+00:00,nehil@snaptravel.com,Nehil Jain,nehil@snaptravel.com,2020-03-24T22:42:54.000+00:00,nehil@snaptravel.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,22,
-5d6c369200c633b06d85c9375f61fef297761ef1,0,0,0,Apply suggestion to models/base/snowflake_query_history.sql,"Taylor A Murphy, PhD",tmurphy@gitlab.com,2020-03-25T13:22:42.000+00:00,tmurphy@gitlab.com,"Taylor A Murphy, PhD",tmurphy@gitlab.com,2020-03-25T13:22:42.000+00:00,tmurphy@gitlab.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,19,
-645a724b558a7fa3885fa233d06f68f833d893fc,0,0,0,Apply suggestion to models/base/snowflake_query_history.sql,Nehil,nehil@getsnaptravel.com,2020-03-24T22:39:36.000+00:00,nehil@getsnaptravel.com,Nehil,nehil@getsnaptravel.com,2020-03-24T22:39:36.000+00:00,nehil@getsnaptravel.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,27,
-64abdcdd9ee9c821f56faf93c5bb0ebc57f575e4,0,0,0,Apply suggestion to models/base/snowflake_query_history.sql,Nehil,nehil@getsnaptravel.com,2020-03-24T22:39:31.000+00:00,nehil@getsnaptravel.com,Nehil,nehil@getsnaptravel.com,2020-03-24T22:39:31.000+00:00,nehil@getsnaptravel.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,28,
-6ebf519f25328e49a004f5a574c8b69ba7420be1,0,0,0,"add contract rate and dbt-utils
-",Taylor A. Murphy,tmurphy@gitlab.com,2019-08-02T17:18:15.000+00:00,tmurphy@gitlab.com,Taylor A. Murphy,tmurphy@gitlab.com,2019-08-02T17:18:15.000+00:00,tmurphy@gitlab.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,15,
-81b12c995b2fc12e20fd05113cf36dbd57b6e7b0,0,0,0,Apply suggestion to models/base/snowflake_query_history.sql,"Taylor A Murphy, PhD",tmurphy@gitlab.com,2020-03-25T13:21:04.000+00:00,tmurphy@gitlab.com,"Taylor A Murphy, PhD",tmurphy@gitlab.com,2020-03-25T13:21:04.000+00:00,tmurphy@gitlab.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,20,
-846deac12a1263e4631ced45abbc5e157b23c1be,0,0,0,"testing
-",Nehil Jain,nehil@snaptravel.com,2020-03-24T21:27:46.000+00:00,nehil@snaptravel.com,Nehil Jain,nehil@snaptravel.com,2020-03-24T21:27:46.000+00:00,nehil@snaptravel.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,30,
-8a40765e605966842b05b5d8cdf2dd6890556d09,0,0,0,"change dbt-util version
-",Taylor A. Murphy,tmurphy@gitlab.com,2019-08-02T17:22:09.000+00:00,tmurphy@gitlab.com,Taylor A. Murphy,tmurphy@gitlab.com,2019-08-02T17:22:09.000+00:00,tmurphy@gitlab.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,14,
-8a96911c5350f7575215f02f0242f7e27ad3d1f4,0,0,0,"add documentation
-",Taylor A. Murphy,tmurphy@gitlab.com,2019-08-02T18:46:42.000+00:00,tmurphy@gitlab.com,Taylor A. Murphy,tmurphy@gitlab.com,2019-08-02T18:46:42.000+00:00,tmurphy@gitlab.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,10,
-91e5666bdfe90e5ada3e65aaa9240b2b0b1a072f,0,0,0,"remove config
-",emilie,emilie.burke@gmail.com,2019-08-26T15:34:33.000+00:00,emilie.burke@gmail.com,emilie,emilie.burke@gmail.com,2019-08-26T15:34:33.000+00:00,emilie.burke@gmail.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,17,
-93d13cf14a470995ec35186292bf165e1152dc2a,0,0,0,"comments on sql
-",Nehil Jain,nehiljain@Hansons-iMac.local,2020-03-24T18:55:11.000+00:00,nehiljain@Hansons-iMac.local,Nehil Jain,nehiljain@Hansons-iMac.local,2020-03-24T18:55:11.000+00:00,nehiljain@Hansons-iMac.local,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,32,
-9a80506e158547d87e7ad2902c0196deff5b5c07,0,0,0,"base mmodel for query performance
-",Nehil Jain,nehil@snaptravel.com,2020-03-24T12:52:28.000+00:00,nehil@snaptravel.com,Nehil Jain,nehil@snaptravel.com,2020-03-24T12:52:28.000+00:00,nehil@snaptravel.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,35,
-9e5ec04b0a8273b73936ea150275919b0602c260,0,0,0,"change names
-",Taylor A. Murphy,tmurphy@gitlab.com,2019-08-02T17:24:35.000+00:00,tmurphy@gitlab.com,Taylor A. Murphy,tmurphy@gitlab.com,2019-08-02T17:24:35.000+00:00,tmurphy@gitlab.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,13,
-9ef8183140bae8c39a440d9c7eb47f3bc7dcab75,0,0,0,Apply suggestion to models/base/snowflake_query_history.sql,"Taylor A Murphy, PhD",tmurphy@gitlab.com,2020-03-25T13:22:47.000+00:00,tmurphy@gitlab.com,"Taylor A Murphy, PhD",tmurphy@gitlab.com,2020-03-25T13:22:47.000+00:00,tmurphy@gitlab.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,18,
-a09f2d4ae88e78d7424fba1ae6d0c036b677271d,0,0,0,"update readme
-",emilie,emilie.burke@gmail.com,2020-04-05T14:43:47.000+00:00,emilie.burke@gmail.com,emilie,emilie.burke@gmail.com,2020-04-05T14:43:47.000+00:00,emilie.burke@gmail.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,36,
-abbe0ab2c7bb1dc2cfaa3ef3062f378fb908ba71,0,0,0,"add first bit
-",emilie,emilie.burke@gmail.com,2019-06-28T05:40:36.000+00:00,emilie.burke@gmail.com,emilie,emilie.burke@gmail.com,2019-06-28T05:40:36.000+00:00,emilie.burke@gmail.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,5,
-ad25fcda15e310a7eb34b3e38ff9b962b54f0e03,0,0,0,"add more info to readme
-",emilie,emilie.burke@gmail.com,2019-06-28T11:03:52.000+00:00,emilie.burke@gmail.com,emilie,emilie.burke@gmail.com,2019-06-28T11:03:52.000+00:00,emilie.burke@gmail.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,1,
-bb6dfec8dcdacb5b6b37dbd8eb1aa6f9e8c06138,0,0,0,"merging online changes
-",Nehil Jain,nehil@snaptravel.com,2020-03-24T22:41:35.000+00:00,nehil@snaptravel.com,Nehil Jain,nehil@snaptravel.com,2020-03-24T22:41:35.000+00:00,nehil@snaptravel.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,23,
-c4d406c077af582837d3611a46eded70151f93fb,0,0,0,"tested schema and the model
-",Nehil Jain,nehil@snaptravel.com,2020-03-24T22:37:23.000+00:00,nehil@snaptravel.com,Nehil Jain,nehil@snaptravel.com,2020-03-24T22:37:23.000+00:00,nehil@snaptravel.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,29,
-e01d4f03811cd0da9949848731236e0aa261cf54,0,0,0,"move analyses
-",emilie,emilie.burke@gmail.com,2019-06-28T10:43:17.000+00:00,emilie.burke@gmail.com,emilie,emilie.burke@gmail.com,2019-06-28T10:43:17.000+00:00,emilie.burke@gmail.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,4,
-e24fecacb1a2c85f904bef57ffd5598ca2df023c,0,0,0,Apply suggestion to models/base/snowflake_query_history.sql,Nehil,nehil@getsnaptravel.com,2020-03-24T22:39:42.000+00:00,nehil@getsnaptravel.com,Nehil,nehil@getsnaptravel.com,2020-03-24T22:39:42.000+00:00,nehil@getsnaptravel.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,25,
diff --git a/backend/plugins/gitlab/e2e/snapshot_tables/repo_commits.csv b/backend/plugins/gitlab/e2e/snapshot_tables/repo_commits.csv
deleted file mode 100644
index 1b3ee220d..000000000
--- a/backend/plugins/gitlab/e2e/snapshot_tables/repo_commits.csv
+++ /dev/null
@@ -1,38 +0,0 @@
-repo_id,commit_sha,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
-gitlab:GitlabProject:1:12345678,05976c113b02fb71d5c43682bd2f9d96a3454efa,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,34,
-gitlab:GitlabProject:1:12345678,083864201849f6707a34d24e61c43638696f95a6,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,6,
-gitlab:GitlabProject:1:12345678,08973065d33c6fd2fb1f592b7a1e0fbe2ee72edd,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,11,
-gitlab:GitlabProject:1:12345678,0a4574d52b5964c37bfcb2952f6c6a8a9c6d5bcf,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,7,
-gitlab:GitlabProject:1:12345678,0dbb8bfdbbf7373656fa3eb036bdcbb25ae162da,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,26,
-gitlab:GitlabProject:1:12345678,1cba71b355d2bd4e1350602cfb9b98cce5a5cb4d,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,8,
-gitlab:GitlabProject:1:12345678,1f16955d45cec904c8968bb27689d91a71e9a2f2,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,12,
-gitlab:GitlabProject:1:12345678,211795deaaaaddb23b797261cdcd148e6083d5b2,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,21,
-gitlab:GitlabProject:1:12345678,25f2b34099a3e1867fcf73a54e50f78f3c8d8f75,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,9,
-gitlab:GitlabProject:1:12345678,382084b42697577d3a6adf71ce73d4b5ddd22977,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,2,
-gitlab:GitlabProject:1:12345678,3f04e0a61d0c4d2dd736a6bcaa3a06826269a533,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,3,
-gitlab:GitlabProject:1:12345678,45bed79fcdc6ba3444245f55660f839db50d79c8,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,33,
-gitlab:GitlabProject:1:12345678,48295660f8bd0c0a024849d4407830face57d4bc,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,37,
-gitlab:GitlabProject:1:12345678,4f0d2f7b1a47d21b01a68e59c12d6f8885ab230e,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,24,
-gitlab:GitlabProject:1:12345678,553f7613b178301da4075202afe91ebda5d97ab5,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,16,
-gitlab:GitlabProject:1:12345678,5a141ad0c28960b02c40e6c476d8df3cc76f58c4,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,31,
-gitlab:GitlabProject:1:12345678,5b6a2be437e0c66ba6276030a61a10583c3f83fa,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,22,
-gitlab:GitlabProject:1:12345678,5d6c369200c633b06d85c9375f61fef297761ef1,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,19,
-gitlab:GitlabProject:1:12345678,645a724b558a7fa3885fa233d06f68f833d893fc,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,27,
-gitlab:GitlabProject:1:12345678,64abdcdd9ee9c821f56faf93c5bb0ebc57f575e4,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,28,
-gitlab:GitlabProject:1:12345678,6ebf519f25328e49a004f5a574c8b69ba7420be1,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,15,
-gitlab:GitlabProject:1:12345678,81b12c995b2fc12e20fd05113cf36dbd57b6e7b0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,20,
-gitlab:GitlabProject:1:12345678,846deac12a1263e4631ced45abbc5e157b23c1be,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,30,
-gitlab:GitlabProject:1:12345678,8a40765e605966842b05b5d8cdf2dd6890556d09,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,14,
-gitlab:GitlabProject:1:12345678,8a96911c5350f7575215f02f0242f7e27ad3d1f4,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,10,
-gitlab:GitlabProject:1:12345678,91e5666bdfe90e5ada3e65aaa9240b2b0b1a072f,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,17,
-gitlab:GitlabProject:1:12345678,93d13cf14a470995ec35186292bf165e1152dc2a,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,32,
-gitlab:GitlabProject:1:12345678,9a80506e158547d87e7ad2902c0196deff5b5c07,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,35,
-gitlab:GitlabProject:1:12345678,9e5ec04b0a8273b73936ea150275919b0602c260,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,13,
-gitlab:GitlabProject:1:12345678,9ef8183140bae8c39a440d9c7eb47f3bc7dcab75,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,18,
-gitlab:GitlabProject:1:12345678,a09f2d4ae88e78d7424fba1ae6d0c036b677271d,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,36,
-gitlab:GitlabProject:1:12345678,abbe0ab2c7bb1dc2cfaa3ef3062f378fb908ba71,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,5,
-gitlab:GitlabProject:1:12345678,ad25fcda15e310a7eb34b3e38ff9b962b54f0e03,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,1,
-gitlab:GitlabProject:1:12345678,bb6dfec8dcdacb5b6b37dbd8eb1aa6f9e8c06138,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,23,
-gitlab:GitlabProject:1:12345678,c4d406c077af582837d3611a46eded70151f93fb,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,29,
-gitlab:GitlabProject:1:12345678,e01d4f03811cd0da9949848731236e0aa261cf54,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,4,
-gitlab:GitlabProject:1:12345678,e24fecacb1a2c85f904bef57ffd5598ca2df023c,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,25,
diff --git a/backend/plugins/gitlab/models/migrationscripts/20240904_remove_mr_review_fields.go b/backend/plugins/gitlab/models/migrationscripts/20240904_remove_mr_review_fields.go
new file mode 100644
index 000000000..2628033f0
--- /dev/null
+++ b/backend/plugins/gitlab/models/migrationscripts/20240904_remove_mr_review_fields.go
@@ -0,0 +1,40 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+	"github.com/apache/incubator-devlake/core/context"
+	"github.com/apache/incubator-devlake/core/errors"
+	"github.com/apache/incubator-devlake/core/plugin"
+)
+
+var _ plugin.MigrationScript = (*removeMrEnricherFields)(nil)
+
+type removeMrEnricherFields struct{}
+
+func (script *removeMrEnricherFields) Up(basicRes context.BasicRes) errors.Error {
+	return basicRes.GetDal().DropColumns("_tool_gitlab_merge_requests", "first_comment_time", "review_rounds")
+}
+
+func (*removeMrEnricherFields) Version() uint64 {
+	return 20240904152414
+}
+
+func (*removeMrEnricherFields) Name() string {
+	return "remove mr review fields in favor of dora plugin"
+}
diff --git a/backend/plugins/gitlab/models/mr.go b/backend/plugins/gitlab/models/mr.go
index 6ac3c9b76..c25df4ba0 100644
--- a/backend/plugins/gitlab/models/mr.go
+++ b/backend/plugins/gitlab/models/mr.go
@@ -50,9 +50,7 @@ type GitlabMergeRequest struct {
 	Description      string
 	AuthorUsername   string `gorm:"type:varchar(255)"`
 	AuthorUserId     int
-	Component        string     `gorm:"type:varchar(255)"`
-	FirstCommentTime *time.Time `gorm:"comment:Time when the first comment occurred"`
-	ReviewRounds     int        `gorm:"comment:How many rounds of review this MR went through"`
+	Component        string `gorm:"type:varchar(255)"`
 	common.NoPKModel
 }
 
diff --git a/backend/plugins/gitlab/tasks/account_collector.go b/backend/plugins/gitlab/tasks/account_collector.go
index 52ea1a72d..e08498589 100644
--- a/backend/plugins/gitlab/tasks/account_collector.go
+++ b/backend/plugins/gitlab/tasks/account_collector.go
@@ -43,7 +43,7 @@ var CollectAccountsMeta = plugin.SubTaskMeta{
 	EnabledByDefault: true,
 	Description:      "collect gitlab users, does not support either timeFilter or diffSync.",
 	DomainTypes:      []string{plugin.DOMAIN_TYPE_CROSS},
-	Dependencies:     []*plugin.SubTaskMeta{&EnrichMergeRequestsMeta},
+	Dependencies:     []*plugin.SubTaskMeta{},
 }
 
 func CollectAccounts(taskCtx plugin.SubTaskContext) errors.Error {
@@ -58,7 +58,7 @@ func CollectAccounts(taskCtx plugin.SubTaskContext) errors.Error {
 	}
 
 	// Collect all users if endpoint is private gitlab instance
-	if !strings.HasPrefix(data.ApiClient.GetEndpoint(), "https://gitlab.com") {
+	if !strings.HasPrefix(data.ApiClient.GetEndpoint(), "https://gitlab.com") && !strings.HasPrefix(data.ApiClient.GetEndpoint(), "https://jihulab.com") {
 		urlTemplate = "/users"
 	}
 
diff --git a/backend/plugins/gitlab/tasks/child_pipeline_extractor.go b/backend/plugins/gitlab/tasks/child_pipeline_extractor.go
index 4c7d8d875..62c5948fc 100644
--- a/backend/plugins/gitlab/tasks/child_pipeline_extractor.go
+++ b/backend/plugins/gitlab/tasks/child_pipeline_extractor.go
@@ -18,8 +18,6 @@ limitations under the License.
 package tasks
 
 import (
-	"encoding/json"
-
 	"github.com/apache/incubator-devlake/core/errors"
 	"github.com/apache/incubator-devlake/core/models/common"
 	"github.com/apache/incubator-devlake/core/plugin"
@@ -40,32 +38,19 @@ var ExtractApiChildPipelinesMeta = plugin.SubTaskMeta{
 	Dependencies:     []*plugin.SubTaskMeta{&CollectApiChildPipelinesMeta},
 }
 
-func ExtractApiChildPipelines(taskCtx plugin.SubTaskContext) errors.Error {
-	rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_CHILD_PIPELINE_TABLE)
-
-	extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{
-		RawDataSubTaskArgs: *rawDataSubTaskArgs,
-		Extract: func(row *api.RawData) ([]interface{}, errors.Error) {
-			gitlabApiChildPipeline := &ApiPipeline{}
-			err := errors.Convert(json.Unmarshal(row.Data, gitlabApiChildPipeline))
-			if err != nil {
-				return nil, err
-			}
-			pipelineProject := convertApiPipelineToGitlabPipelineProject(gitlabApiChildPipeline, data.Options.ConnectionId, data.Options.ProjectId)
+func ExtractApiChildPipelines(subtaskCtx plugin.SubTaskContext) errors.Error {
+	subtaskCommonArgs, data := CreateSubtaskCommonArgs(subtaskCtx, RAW_CHILD_PIPELINE_TABLE)
+	extractor, err := api.NewStatefulApiExtractor(&api.StatefulApiExtractorArgs[ApiPipeline]{
+		SubtaskCommonArgs: subtaskCommonArgs,
+		Extract: func(gitlabApiPipeline *ApiPipeline, row *api.RawData) ([]interface{}, errors.Error) {
+			pipelineProject := convertApiPipelineToGitlabPipelineProject(gitlabApiPipeline, data.Options.ConnectionId, data.Options.ProjectId)
 			return []interface{}{pipelineProject}, nil
 		},
 	})
-
-	if err != nil {
-		return err
-	}
-
-	err = extractor.Execute()
 	if err != nil {
 		return err
 	}
-
-	return nil
+	return extractor.Execute()
 }
 
 func convertApiPipelineToGitlabPipelineProject(gitlabApiChildPipeline *ApiPipeline, connectionId uint64, projectId int) *models.GitlabPipelineProject {
diff --git a/backend/plugins/gitlab/tasks/commit_collector.go b/backend/plugins/gitlab/tasks/commit_collector.go
deleted file mode 100644
index 4bb2e960f..000000000
--- a/backend/plugins/gitlab/tasks/commit_collector.go
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements.  See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License.  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package tasks
-
-import (
-	"github.com/apache/incubator-devlake/core/errors"
-	"github.com/apache/incubator-devlake/core/models/common"
-	"github.com/apache/incubator-devlake/core/plugin"
-	"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
-)
-
-func init() {
-	RegisterSubtaskMeta(&CollectApiCommitsMeta)
-}
-
-const RAW_COMMIT_TABLE = "gitlab_api_commit"
-
-var CollectApiCommitsMeta = plugin.SubTaskMeta{
-	Name:             "Collect Commits",
-	EntryPoint:       CollectApiCommits,
-	EnabledByDefault: false,
-	Description:      "Collect commit data from gitlab api, does not support either timeFilter or diffSync.",
-	DomainTypes:      []string{plugin.DOMAIN_TYPE_CODE},
-	Dependencies:     []*plugin.SubTaskMeta{&ConvertJobMeta},
-}
-
-type GitlabApiCommit struct {
-	GitlabId       string `json:"id"`
-	Title          string
-	Message        string
-	ProjectId      int
-	ShortId        string             `json:"short_id"`
-	AuthorName     string             `json:"author_name"`
-	AuthorEmail    string             `json:"author_email"`
-	AuthoredDate   common.Iso8601Time `json:"authored_date"`
-	CommitterName  string             `json:"committer_name"`
-	CommitterEmail string             `json:"committer_email"`
-	CommittedDate  common.Iso8601Time `json:"committed_date"`
-	WebUrl         string             `json:"web_url"`
-	Stats          struct {
-		Additions int
-		Deletions int
-		Total     int
-	}
-}
-
-func CollectApiCommits(taskCtx plugin.SubTaskContext) errors.Error {
-	rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_COMMIT_TABLE)
-
-	collector, err := api.NewApiCollector(api.ApiCollectorArgs{
-		RawDataSubTaskArgs: *rawDataSubTaskArgs,
-		ApiClient:          data.ApiClient,
-		PageSize:           100,
-		Incremental:        false,
-		UrlTemplate:        "projects/{{ .Params.ProjectId }}/repository/commits",
-		Query:              GetQuery,
-		ResponseParser:     GetRawMessageFromResponse,
-	})
-
-	if err != nil {
-		return err
-	}
-
-	return collector.Execute()
-}
diff --git a/backend/plugins/gitlab/tasks/commit_convertor.go b/backend/plugins/gitlab/tasks/commit_convertor.go
deleted file mode 100644
index c0fb77a79..000000000
--- a/backend/plugins/gitlab/tasks/commit_convertor.go
+++ /dev/null
@@ -1,108 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements.  See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License.  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package tasks
-
-import (
-	"reflect"
-
-	"github.com/apache/incubator-devlake/core/dal"
-	"github.com/apache/incubator-devlake/core/errors"
-	"github.com/apache/incubator-devlake/core/models/domainlayer/code"
-	"github.com/apache/incubator-devlake/core/models/domainlayer/didgen"
-	"github.com/apache/incubator-devlake/core/plugin"
-	helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
-	"github.com/apache/incubator-devlake/plugins/gitlab/models"
-)
-
-func init() {
-	RegisterSubtaskMeta(&ConvertCommitsMeta)
-}
-
-var ConvertCommitsMeta = plugin.SubTaskMeta{
-	Name:             "Convert Commits",
-	EntryPoint:       ConvertApiCommits,
-	EnabledByDefault: false,
-	Description:      "Update domain layer commit according to GitlabCommit",
-	DomainTypes:      []string{plugin.DOMAIN_TYPE_CODE},
-	Dependencies:     []*plugin.SubTaskMeta{&ConvertMrLabelsMeta},
-}
-
-func ConvertApiCommits(taskCtx plugin.SubTaskContext) errors.Error {
-	rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_COMMIT_TABLE)
-	db := taskCtx.GetDal()
-
-	// select all commits belongs to the project
-	clauses := []dal.Clause{
-		dal.Select("gc.*"),
-		dal.From("_tool_gitlab_commits gc"),
-		dal.Join(`left join _tool_gitlab_project_commits gpc on (
-			gpc.commit_sha = gc.sha
-		)`),
-		dal.Where("gpc.gitlab_project_id = ? and gpc.connection_id = ? ",
-			data.Options.ProjectId, data.Options.ConnectionId),
-	}
-	cursor, err := db.Cursor(clauses...)
-	if err != nil {
-		return err
-	}
-	defer cursor.Close()
-
-	// TODO: adopt batch indate operation
-	//userDidGen := didgen.NewDomainIdGenerator(&models.GitlabAccount{})
-
-	converter, err := helper.NewDataConverter(helper.DataConverterArgs{
-		RawDataSubTaskArgs: *rawDataSubTaskArgs,
-		InputRowType:       reflect.TypeOf(models.GitlabCommit{}),
-		Input:              cursor,
-
-		Convert: func(inputRow interface{}) ([]interface{}, errors.Error) {
-			gitlabCommit := inputRow.(*models.GitlabCommit)
-
-			// convert commit
-			commit := &code.Commit{}
-			commit.Sha = gitlabCommit.Sha
-			commit.Message = gitlabCommit.Message
-			commit.Additions = gitlabCommit.Additions
-			commit.Deletions = gitlabCommit.Deletions
-			commit.AuthorId = gitlabCommit.AuthorEmail
-			commit.AuthorName = gitlabCommit.AuthorName
-			commit.AuthorEmail = gitlabCommit.AuthorEmail
-			commit.AuthoredDate = gitlabCommit.AuthoredDate
-			commit.CommitterName = gitlabCommit.CommitterName
-			commit.CommitterEmail = gitlabCommit.CommitterEmail
-			commit.CommittedDate = gitlabCommit.CommittedDate
-			commit.CommitterId = gitlabCommit.CommitterEmail
-
-			// convert repo / commits relationship
-			repoCommit := &code.RepoCommit{
-				RepoId:    didgen.NewDomainIdGenerator(&models.GitlabProject{}).Generate(data.Options.ConnectionId, data.Options.ProjectId),
-				CommitSha: gitlabCommit.Sha,
-			}
-
-			return []interface{}{
-				commit,
-				repoCommit,
-			}, nil
-		},
-	})
-	if err != nil {
-		return err
-	}
-
-	return converter.Execute()
-}
diff --git a/backend/plugins/gitlab/tasks/commit_extractor.go b/backend/plugins/gitlab/tasks/commit_extractor.go
deleted file mode 100644
index 664f8eae3..000000000
--- a/backend/plugins/gitlab/tasks/commit_extractor.go
+++ /dev/null
@@ -1,116 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements.  See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License.  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package tasks
-
-import (
-	"encoding/json"
-
-	"github.com/apache/incubator-devlake/core/errors"
-	"github.com/apache/incubator-devlake/core/plugin"
-	"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
-	"github.com/apache/incubator-devlake/plugins/gitlab/models"
-)
-
-func init() {
-	RegisterSubtaskMeta(&ExtractApiCommitsMeta)
-}
-
-var ExtractApiCommitsMeta = plugin.SubTaskMeta{
-	Name:             "Extract Commits",
-	EntryPoint:       ExtractApiCommits,
-	EnabledByDefault: false,
-	Description:      "Extract raw commit data into tool layer table GitlabCommit,GitlabAccount and GitlabProjectCommit",
-	DomainTypes:      []string{plugin.DOMAIN_TYPE_CODE},
-	Dependencies:     []*plugin.SubTaskMeta{&CollectApiCommitsMeta},
-}
-
-func ExtractApiCommits(taskCtx plugin.SubTaskContext) errors.Error {
-	rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_COMMIT_TABLE)
-
-	extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{
-		RawDataSubTaskArgs: *rawDataSubTaskArgs,
-		Extract: func(row *api.RawData) ([]interface{}, errors.Error) {
-			// need to extract 3 kinds of entities here
-			results := make([]interface{}, 0, 3)
-
-			// create gitlab commit
-			gitlabApiCommit := &GitlabApiCommit{}
-			err := errors.Convert(json.Unmarshal(row.Data, gitlabApiCommit))
-			if err != nil {
-				return nil, err
-			}
-			gitlabCommit, err := ConvertCommit(gitlabApiCommit)
-			if err != nil {
-				return nil, err
-			}
-
-			// create project/commits relationship
-			gitlabProjectCommit := &models.GitlabProjectCommit{GitlabProjectId: data.Options.ProjectId}
-			gitlabProjectCommit.CommitSha = gitlabCommit.Sha
-
-			// create gitlab user
-			GitlabAccountAuthor := &models.GitlabAccount{}
-			GitlabAccountAuthor.Email = gitlabCommit.AuthorEmail
-			GitlabAccountAuthor.Name = gitlabCommit.AuthorName
-
-			gitlabProjectCommit.ConnectionId = data.Options.ConnectionId
-			GitlabAccountAuthor.ConnectionId = data.Options.ConnectionId
-			results = append(results, gitlabCommit)
-			results = append(results, gitlabProjectCommit)
-			results = append(results, GitlabAccountAuthor)
-
-			// For Commiter Email is not same as AuthorEmail
-			if gitlabCommit.CommitterEmail != GitlabAccountAuthor.Email {
-				gitlabAccountCommitter := &models.GitlabAccount{}
-				gitlabAccountCommitter.Email = gitlabCommit.CommitterEmail
-				gitlabAccountCommitter.Name = gitlabCommit.CommitterName
-				gitlabAccountCommitter.ConnectionId = data.Options.ConnectionId
-				results = append(results, gitlabAccountCommitter)
-			}
-
-			return results, nil
-		},
-	})
-
-	if err != nil {
-		return err
-	}
-
-	return extractor.Execute()
-}
-
-// Convert the API response to our DB model instance
-func ConvertCommit(commit *GitlabApiCommit) (*models.GitlabCommit, errors.Error) {
-	gitlabCommit := &models.GitlabCommit{
-		Sha:            commit.GitlabId,
-		Title:          commit.Title,
-		Message:        commit.Message,
-		ShortId:        commit.ShortId,
-		AuthorName:     commit.AuthorName,
-		AuthorEmail:    commit.AuthorEmail,
-		AuthoredDate:   commit.AuthoredDate.ToTime(),
-		CommitterName:  commit.CommitterName,
-		CommitterEmail: commit.CommitterEmail,
-		CommittedDate:  commit.CommittedDate.ToTime(),
-		WebUrl:         commit.WebUrl,
-		Additions:      commit.Stats.Additions,
-		Deletions:      commit.Stats.Deletions,
-		Total:          commit.Stats.Total,
-	}
-	return gitlabCommit, nil
-}
diff --git a/backend/plugins/gitlab/tasks/deployment_convertor.go b/backend/plugins/gitlab/tasks/deployment_convertor.go
index d7ff0d774..b211867b4 100644
--- a/backend/plugins/gitlab/tasks/deployment_convertor.go
+++ b/backend/plugins/gitlab/tasks/deployment_convertor.go
@@ -19,7 +19,6 @@ package tasks
 
 import (
 	"fmt"
-	"reflect"
 	"time"
 
 	"github.com/apache/incubator-devlake/core/dal"
@@ -50,9 +49,9 @@ var ConvertDeploymentMeta = plugin.SubTaskMeta{
 
 // ConvertDeployment should be split into two task theoretically
 // But in GitLab, all deployments have commits, so there is no need to change it.
-func ConvertDeployment(taskCtx plugin.SubTaskContext) errors.Error {
-	rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_DEPLOYMENT)
-	db := taskCtx.GetDal()
+func ConvertDeployment(subtaskCtx plugin.SubTaskContext) errors.Error {
+	subtaskCommonArgs, data := CreateSubtaskCommonArgs(subtaskCtx, RAW_DEPLOYMENT)
+	db := subtaskCtx.GetDal()
 
 	repo := &models.GitlabProject{}
 	err := db.First(repo, dal.Where("gitlab_id = ? and connection_id = ?", data.Options.ProjectId, data.Options.ConnectionId))
@@ -61,25 +60,24 @@ func ConvertDeployment(taskCtx plugin.SubTaskContext) errors.Error {
 	}
 
 	projectIdGen := didgen.NewDomainIdGenerator(&models.GitlabProject{})
-
-	cursor, err := db.Cursor(
-		dal.From(&models.GitlabDeployment{}),
-		dal.Where("connection_id = ? AND gitlab_id = ?", data.Options.ConnectionId, data.Options.ProjectId),
-	)
-	if err != nil {
-		return err
-	}
-	defer cursor.Close()
-
 	idGen := didgen.NewDomainIdGenerator(&models.GitlabDeployment{})
 
-	converter, err := api.NewDataConverter(api.DataConverterArgs{
-		InputRowType:       reflect.TypeOf(models.GitlabDeployment{}),
-		Input:              cursor,
-		RawDataSubTaskArgs: *rawDataSubTaskArgs,
-		Convert: func(inputRow interface{}) ([]interface{}, errors.Error) {
-			gitlabDeployment := inputRow.(*models.GitlabDeployment)
-
+	converter, err := api.NewStatefulDataConverter(&api.StatefulDataConverterArgs[models.GitlabDeployment]{
+		SubtaskCommonArgs: subtaskCommonArgs,
+		Input: func(stateManager *api.SubtaskStateManager) (dal.Rows, errors.Error) {
+			clauses := []dal.Clause{
+				dal.From(&models.GitlabDeployment{}),
+				dal.Where("connection_id = ? AND gitlab_id = ?", data.Options.ConnectionId, data.Options.ProjectId),
+			}
+			if stateManager.IsIncremental() {
+				since := stateManager.GetSince()
+				if since != nil {
+					clauses = append(clauses, dal.Where("updated_at >= ? ", since))
+				}
+			}
+			return db.Cursor(clauses...)
+		},
+		Convert: func(gitlabDeployment *models.GitlabDeployment) ([]interface{}, errors.Error) {
 			var duration *float64
 			if gitlabDeployment.DeployableDuration != nil {
 				deployableDuration := cast.ToFloat64(*gitlabDeployment.DeployableDuration)
diff --git a/backend/plugins/gitlab/tasks/deployment_extractor.go b/backend/plugins/gitlab/tasks/deployment_extractor.go
index 0b439aceb..258471b01 100644
--- a/backend/plugins/gitlab/tasks/deployment_extractor.go
+++ b/backend/plugins/gitlab/tasks/deployment_extractor.go
@@ -18,7 +18,6 @@ limitations under the License.
 package tasks
 
 import (
-	"encoding/json"
 	"time"
 
 	"github.com/apache/incubator-devlake/core/errors"
@@ -44,16 +43,11 @@ var ExtractDeploymentMeta = plugin.SubTaskMeta{
 }
 
 func ExtractDeployment(taskCtx plugin.SubTaskContext) errors.Error {
-	rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_DEPLOYMENT)
-
-	extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{
-		RawDataSubTaskArgs: *rawDataSubTaskArgs,
-		Extract: func(row *api.RawData) ([]interface{}, errors.Error) {
-			deploymentResp := &GitlabDeploymentResp{}
-			err := errors.Convert(json.Unmarshal(row.Data, deploymentResp))
-			if err != nil {
-				return nil, err
-			}
+	subtaskCommonArgs, data := CreateSubtaskCommonArgs(taskCtx, RAW_DEPLOYMENT)
+
+	extractor, err := api.NewStatefulApiExtractor(&api.StatefulApiExtractorArgs[GitlabDeploymentResp]{
+		SubtaskCommonArgs: subtaskCommonArgs,
+		Extract: func(deploymentResp *GitlabDeploymentResp, row *api.RawData) ([]interface{}, errors.Error) {
 			gitlabDeployment := deploymentResp.toGitlabDeployment(data.Options.ConnectionId, data.Options.ProjectId)
 			return []interface{}{
 				gitlabDeployment,
diff --git a/backend/plugins/gitlab/tasks/issue_assignee_convertor.go b/backend/plugins/gitlab/tasks/issue_assignee_convertor.go
index f4a70065d..ce5ce593f 100644
--- a/backend/plugins/gitlab/tasks/issue_assignee_convertor.go
+++ b/backend/plugins/gitlab/tasks/issue_assignee_convertor.go
@@ -18,14 +18,12 @@ limitations under the License.
 package tasks
 
 import (
-	"reflect"
-
 	"github.com/apache/incubator-devlake/core/dal"
 	"github.com/apache/incubator-devlake/core/errors"
 	"github.com/apache/incubator-devlake/core/models/domainlayer/didgen"
 	"github.com/apache/incubator-devlake/core/models/domainlayer/ticket"
 	"github.com/apache/incubator-devlake/core/plugin"
-	helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+	"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
 	"github.com/apache/incubator-devlake/plugins/gitlab/models"
 )
 
@@ -40,35 +38,36 @@ var ConvertIssueAssigneeMeta = plugin.SubTaskMeta{
 	Description:      "Convert tool layer table _tool_gitlab_issue_assignees into  domain layer table issue_assignees",
 	DomainTypes:      []string{plugin.DOMAIN_TYPE_TICKET},
 	DependencyTables: []string{models.GitlabIssueAssignee{}.TableName()},
-	Dependencies:     []*plugin.SubTaskMeta{&ExtractApiIssuesMeta},
+	Dependencies:     []*plugin.SubTaskMeta{&ConvertIssuesMeta},
 }
 
-func ConvertIssueAssignee(taskCtx plugin.SubTaskContext) errors.Error {
-	db := taskCtx.GetDal()
-	rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_ISSUE_TABLE)
-
-	cursor, err := db.Cursor(
-		dal.From(&models.GitlabIssueAssignee{}),
-		dal.Where("project_id = ? and connection_id=?", data.Options.ProjectId, data.Options.ConnectionId),
-	)
-	if err != nil {
-		return err
-	}
-	defer cursor.Close()
+func ConvertIssueAssignee(subtaskCtx plugin.SubTaskContext) errors.Error {
+	subtaskCommonArgs, data := CreateSubtaskCommonArgs(subtaskCtx, RAW_ISSUE_TABLE)
+	db := subtaskCtx.GetDal()
 
 	issueIdGen := didgen.NewDomainIdGenerator(&models.GitlabIssue{})
 	accountIdGen := didgen.NewDomainIdGenerator(&models.GitlabAccount{})
 
-	converter, err := helper.NewDataConverter(helper.DataConverterArgs{
-		RawDataSubTaskArgs: *rawDataSubTaskArgs,
-		InputRowType:       reflect.TypeOf(models.GitlabIssueAssignee{}),
-		Input:              cursor,
-		Convert: func(inputRow interface{}) ([]interface{}, errors.Error) {
-			input := inputRow.(*models.GitlabIssueAssignee)
+	converter, err := api.NewStatefulDataConverter(&api.StatefulDataConverterArgs[models.GitlabIssueAssignee]{
+		SubtaskCommonArgs: subtaskCommonArgs,
+		Input: func(stateManager *api.SubtaskStateManager) (dal.Rows, errors.Error) {
+			clauses := []dal.Clause{
+				dal.From(&models.GitlabIssueAssignee{}),
+				dal.Where("connection_id = ? AND project_id = ?", data.Options.ConnectionId, data.Options.ProjectId),
+			}
+			if stateManager.IsIncremental() {
+				since := stateManager.GetSince()
+				if since != nil {
+					clauses = append(clauses, dal.Where("updated_at >= ? ", since))
+				}
+			}
+			return db.Cursor(clauses...)
+		},
+		Convert: func(issueAssignee *models.GitlabIssueAssignee) ([]interface{}, errors.Error) {
 			domainIssueAssignee := &ticket.IssueAssignee{
-				IssueId:      issueIdGen.Generate(data.Options.ConnectionId, input.GitlabId),
-				AssigneeId:   accountIdGen.Generate(data.Options.ConnectionId, input.AssigneeId),
-				AssigneeName: input.AssigneeName,
+				IssueId:      issueIdGen.Generate(data.Options.ConnectionId, issueAssignee.GitlabId),
+				AssigneeId:   accountIdGen.Generate(data.Options.ConnectionId, issueAssignee.AssigneeId),
+				AssigneeName: issueAssignee.AssigneeName,
 			}
 			return []interface{}{domainIssueAssignee}, nil
 		},
diff --git a/backend/plugins/gitlab/tasks/issue_convertor.go b/backend/plugins/gitlab/tasks/issue_convertor.go
index 9be3c1d18..63af1c659 100644
--- a/backend/plugins/gitlab/tasks/issue_convertor.go
+++ b/backend/plugins/gitlab/tasks/issue_convertor.go
@@ -18,7 +18,6 @@ limitations under the License.
 package tasks
 
 import (
-	"reflect"
 	"strconv"
 	"strings"
 
@@ -28,7 +27,7 @@ import (
 	"github.com/apache/incubator-devlake/core/models/domainlayer/didgen"
 	"github.com/apache/incubator-devlake/core/models/domainlayer/ticket"
 	"github.com/apache/incubator-devlake/core/plugin"
-	helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+	"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
 	"github.com/apache/incubator-devlake/plugins/gitlab/models"
 )
 
@@ -45,32 +44,41 @@ var ConvertIssuesMeta = plugin.SubTaskMeta{
 	Dependencies:     []*plugin.SubTaskMeta{&ConvertApiMrCommitsMeta},
 }
 
-func ConvertIssues(taskCtx plugin.SubTaskContext) errors.Error {
-	db := taskCtx.GetDal()
-	rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_ISSUE_TABLE)
-	projectId := data.Options.ProjectId
-
-	clauses := []dal.Clause{
-		dal.Select("issues.*"),
-		dal.From("_tool_gitlab_issues issues"),
-		dal.Where("project_id = ? and connection_id = ?", projectId, data.Options.ConnectionId),
-	}
-	cursor, err := db.Cursor(clauses...)
-	if err != nil {
-		return err
-	}
-	defer cursor.Close()
+func ConvertIssues(subtaskCtx plugin.SubTaskContext) errors.Error {
+	subtaskCommonArgs, data := CreateSubtaskCommonArgs(subtaskCtx, RAW_ISSUE_TABLE)
 
+	db := subtaskCtx.GetDal()
+	projectId := data.Options.ProjectId
 	issueIdGen := didgen.NewDomainIdGenerator(&models.GitlabIssue{})
 	accountIdGen := didgen.NewDomainIdGenerator(&models.GitlabAccount{})
 	boardIdGen := didgen.NewDomainIdGenerator(&models.GitlabProject{})
 
-	converter, err := helper.NewDataConverter(helper.DataConverterArgs{
-		RawDataSubTaskArgs: *rawDataSubTaskArgs,
-		InputRowType:       reflect.TypeOf(models.GitlabIssue{}),
-		Input:              cursor,
-		Convert: func(inputRow interface{}) ([]interface{}, errors.Error) {
-			issue := inputRow.(*models.GitlabIssue)
+	converter, err := api.NewStatefulDataConverter(&api.StatefulDataConverterArgs[models.GitlabIssue]{
+		SubtaskCommonArgs: subtaskCommonArgs,
+		Input: func(stateManager *api.SubtaskStateManager) (dal.Rows, errors.Error) {
+			clauses := []dal.Clause{
+				dal.From(&models.GitlabIssue{}),
+				dal.Where("connection_id = ? AND project_id = ?", data.Options.ConnectionId, data.Options.ProjectId),
+			}
+			if stateManager.IsIncremental() {
+				since := stateManager.GetSince()
+				if since != nil {
+					clauses = append(clauses, dal.Where("updated_at >= ? ", since))
+				}
+			}
+			return db.Cursor(clauses...)
+		},
+		BeforeConvert: func(issue *models.GitlabIssue, stateManager *api.SubtaskStateManager) errors.Error {
+			issueId := issueIdGen.Generate(data.Options.ConnectionId, issue.GitlabId)
+			if err := db.Delete(&ticket.IssueLabel{}, dal.Where("issue_id = ?", issueId)); err != nil {
+				return err
+			}
+			if err := db.Delete(&ticket.IssueAssignee{}, dal.Where("issue_id = ?", issueId)); err != nil {
+				return err
+			}
+			return nil
+		},
+		Convert: func(issue *models.GitlabIssue) ([]interface{}, errors.Error) {
 			domainIssue := &ticket.Issue{
 				DomainEntity:            domainlayer.DomainEntity{Id: issueIdGen.Generate(data.Options.ConnectionId, issue.GitlabId)},
 				IssueKey:                strconv.Itoa(issue.Number),
@@ -107,6 +115,7 @@ func ConvertIssues(taskCtx plugin.SubTaskContext) errors.Error {
 				BoardId: boardIdGen.Generate(data.Options.ConnectionId, projectId),
 				IssueId: domainIssue.Id,
 			}
+
 			return []interface{}{
 				domainIssue,
 				boardIssue,
diff --git a/backend/plugins/gitlab/tasks/issue_extractor.go b/backend/plugins/gitlab/tasks/issue_extractor.go
index 717e0ca98..802d58a60 100644
--- a/backend/plugins/gitlab/tasks/issue_extractor.go
+++ b/backend/plugins/gitlab/tasks/issue_extractor.go
@@ -18,9 +18,9 @@ limitations under the License.
 package tasks
 
 import (
-	"encoding/json"
 	"regexp"
 
+	"github.com/apache/incubator-devlake/core/dal"
 	"github.com/apache/incubator-devlake/core/errors"
 	"github.com/apache/incubator-devlake/core/models/common"
 	"github.com/apache/incubator-devlake/core/plugin"
@@ -133,8 +133,10 @@ type IssuesResponse struct {
 	}
 }
 
-func ExtractApiIssues(taskCtx plugin.SubTaskContext) errors.Error {
-	rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_ISSUE_TABLE)
+func ExtractApiIssues(subtaskCtx plugin.SubTaskContext) errors.Error {
+	subtaskCommonArgs, data := CreateSubtaskCommonArgs(subtaskCtx, RAW_ISSUE_TABLE)
+
+	db := subtaskCtx.GetDal()
 	config := data.Options.ScopeConfig
 	var issueSeverityRegex *regexp.Regexp
 	var issueComponentRegex *regexp.Regexp
@@ -162,15 +164,33 @@ func ExtractApiIssues(taskCtx plugin.SubTaskContext) errors.Error {
 			return errors.Default.Wrap(err, "regexp Compile issuePriority failed")
 		}
 	}
-	extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{
-		RawDataSubTaskArgs: *rawDataSubTaskArgs,
-		Extract: func(row *api.RawData) ([]interface{}, errors.Error) {
-			body := &IssuesResponse{}
-			err := errors.Convert(json.Unmarshal(row.Data, body))
-			if err != nil {
-				return nil, err
+	subtaskCommonArgs.SubtaskConfig = map[string]interface{}{
+		"issueSeverity":      issueSeverity,
+		"issueComponent":     issueComponent,
+		"issuePriorityRegex": issuePriorityRegex,
+	}
+	extractor, err := api.NewStatefulApiExtractor(&api.StatefulApiExtractorArgs[IssuesResponse]{
+		SubtaskCommonArgs: subtaskCommonArgs,
+		BeforeExtract: func(body *IssuesResponse, stateManager *api.SubtaskStateManager) errors.Error {
+			if stateManager.IsIncremental() {
+				err := db.Delete(
+					&models.GitlabIssueLabel{},
+					dal.Where("connection_id = ? AND issue_id = ?", data.Options.ConnectionId, body.Id),
+				)
+				if err != nil {
+					return err
+				}
+				err = db.Delete(
+					&models.GitlabIssueAssignee{},
+					dal.Where("connection_id = ? AND gitlab_id = ?", data.Options.ConnectionId, body.Id),
+				)
+				if err != nil {
+					return err
+				}
 			}
-
+			return nil
+		},
+		Extract: func(body *IssuesResponse, row *api.RawData) ([]interface{}, errors.Error) {
 			if body.ProjectId == 0 {
 				return nil, nil
 			}
@@ -182,9 +202,9 @@ func ExtractApiIssues(taskCtx plugin.SubTaskContext) errors.Error {
 			}
 			for _, label := range body.Labels {
 				results = append(results, &models.GitlabIssueLabel{
+					ConnectionId: data.Options.ConnectionId,
 					IssueId:      gitlabIssue.GitlabId,
 					LabelName:    label,
-					ConnectionId: data.Options.ConnectionId,
 				})
 				if issueSeverityRegex != nil && issueSeverityRegex.MatchString(label) {
 					gitlabIssue.Severity = label
diff --git a/backend/plugins/gitlab/tasks/issue_label_convertor.go b/backend/plugins/gitlab/tasks/issue_label_convertor.go
index c5e8968dd..fab745dda 100644
--- a/backend/plugins/gitlab/tasks/issue_label_convertor.go
+++ b/backend/plugins/gitlab/tasks/issue_label_convertor.go
@@ -18,14 +18,12 @@ limitations under the License.
 package tasks
 
 import (
-	"reflect"
-
 	"github.com/apache/incubator-devlake/core/dal"
 	"github.com/apache/incubator-devlake/core/errors"
 	"github.com/apache/incubator-devlake/core/models/domainlayer/didgen"
 	"github.com/apache/incubator-devlake/core/models/domainlayer/ticket"
 	"github.com/apache/incubator-devlake/core/plugin"
-	helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+	"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
 	"github.com/apache/incubator-devlake/plugins/gitlab/models"
 )
 
@@ -42,35 +40,32 @@ var ConvertIssueLabelsMeta = plugin.SubTaskMeta{
 	Dependencies:     []*plugin.SubTaskMeta{&ConvertIssuesMeta},
 }
 
-func ConvertIssueLabels(taskCtx plugin.SubTaskContext) errors.Error {
-	db := taskCtx.GetDal()
-	rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_ISSUE_TABLE)
-	projectId := data.Options.ProjectId
-	clauses := []dal.Clause{
-		dal.Select("*"),
-		dal.From(&models.GitlabIssueLabel{}),
-		dal.Join(`left join _tool_gitlab_issues on
-			_tool_gitlab_issues.gitlab_id = _tool_gitlab_issue_labels.issue_id`),
-		dal.Where(`_tool_gitlab_issues.project_id = ?
-			and _tool_gitlab_issues.connection_id = ?`,
-			projectId, data.Options.ConnectionId),
-		dal.Orderby("issue_id ASC"),
-	}
-
-	cursor, err := db.Cursor(clauses...)
-	if err != nil {
-		return err
-	}
-	defer cursor.Close()
+func ConvertIssueLabels(subtaskCtx plugin.SubTaskContext) errors.Error {
+	subtaskCommonArgs, data := CreateSubtaskCommonArgs(subtaskCtx, RAW_ISSUE_TABLE)
 
+	db := subtaskCtx.GetDal()
+	projectId := data.Options.ProjectId
 	issueIdGen := didgen.NewDomainIdGenerator(&models.GitlabIssue{})
 
-	converter, err := helper.NewDataConverter(helper.DataConverterArgs{
-		RawDataSubTaskArgs: *rawDataSubTaskArgs,
-		InputRowType:       reflect.TypeOf(models.GitlabIssueLabel{}),
-		Input:              cursor,
-		Convert: func(inputRow interface{}) ([]interface{}, errors.Error) {
-			issueLabel := inputRow.(*models.GitlabIssueLabel)
+	converter, err := api.NewStatefulDataConverter(&api.StatefulDataConverterArgs[models.GitlabIssueLabel]{
+		SubtaskCommonArgs: subtaskCommonArgs,
+		Input: func(stateManager *api.SubtaskStateManager) (dal.Rows, errors.Error) {
+			clauses := []dal.Clause{
+				dal.Select("l.*"),
+				dal.From("_tool_gitlab_issue_labels l"),
+				dal.Join("LEFT JOIN _tool_gitlab_issues s ON s.gitlab_id = l.issue_id AND l.connection_id = s.connection_id"),
+				dal.Where("s.project_id = ?  AND s.connection_id = ?", projectId, data.Options.ConnectionId),
+				// dal.Orderby("s.issue_id ASC"),
+			}
+			if stateManager.IsIncremental() {
+				since := stateManager.GetSince()
+				if since != nil {
+					clauses = append(clauses, dal.Where("l.updated_at >= ? ", since))
+				}
+			}
+			return db.Cursor(clauses...)
+		},
+		Convert: func(issueLabel *models.GitlabIssueLabel) ([]interface{}, errors.Error) {
 			domainIssueLabel := &ticket.IssueLabel{
 				IssueId:   issueIdGen.Generate(data.Options.ConnectionId, issueLabel.IssueId),
 				LabelName: issueLabel.LabelName,
diff --git a/backend/plugins/gitlab/tasks/job_convertor.go b/backend/plugins/gitlab/tasks/job_convertor.go
index 58bd8f712..910778447 100644
--- a/backend/plugins/gitlab/tasks/job_convertor.go
+++ b/backend/plugins/gitlab/tasks/job_convertor.go
@@ -18,7 +18,6 @@ limitations under the License.
 package tasks
 
 import (
-	"reflect"
 	"time"
 
 	"github.com/apache/incubator-devlake/core/dal"
@@ -28,7 +27,7 @@ import (
 	"github.com/apache/incubator-devlake/core/models/domainlayer/didgen"
 	"github.com/apache/incubator-devlake/core/plugin"
 	"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
-	gitlabModels "github.com/apache/incubator-devlake/plugins/gitlab/models"
+	"github.com/apache/incubator-devlake/plugins/gitlab/models"
 )
 
 func init() {
@@ -44,35 +43,31 @@ var ConvertJobMeta = plugin.SubTaskMeta{
 	Dependencies:     []*plugin.SubTaskMeta{&ConvertPipelineCommitMeta},
 }
 
-func ConvertJobs(taskCtx plugin.SubTaskContext) (err errors.Error) {
-	db := taskCtx.GetDal()
-	data := taskCtx.GetData().(*GitlabTaskData)
+func ConvertJobs(subtaskCtx plugin.SubTaskContext) (err errors.Error) {
+	subtaskCommonArgs, data := CreateSubtaskCommonArgs(subtaskCtx, RAW_JOB_TABLE)
+	db := subtaskCtx.GetDal()
 	regexEnricher := data.RegexEnricher
+	subtaskCommonArgs.SubtaskConfig = regexEnricher.PlainMap()
 
-	cursor, err := db.Cursor(dal.From(gitlabModels.GitlabJob{}),
-		dal.Where("project_id = ? and connection_id = ?", data.Options.ProjectId, data.Options.ConnectionId))
-	if err != nil {
-		return err
-	}
-	defer cursor.Close()
-
-	jobIdGen := didgen.NewDomainIdGenerator(&gitlabModels.GitlabJob{})
-	projectIdGen := didgen.NewDomainIdGenerator(&gitlabModels.GitlabProject{})
-	pipelineIdGen := didgen.NewDomainIdGenerator(&gitlabModels.GitlabPipeline{})
-	converter, err := api.NewDataConverter(api.DataConverterArgs{
-		InputRowType: reflect.TypeOf(gitlabModels.GitlabJob{}),
-		Input:        cursor,
-		RawDataSubTaskArgs: api.RawDataSubTaskArgs{
-			Ctx: taskCtx,
-			Params: gitlabModels.GitlabApiParams{
-				ConnectionId: data.Options.ConnectionId,
-				ProjectId:    data.Options.ProjectId,
-			},
-			Table: RAW_JOB_TABLE,
+	jobIdGen := didgen.NewDomainIdGenerator(&models.GitlabJob{})
+	projectIdGen := didgen.NewDomainIdGenerator(&models.GitlabProject{})
+	pipelineIdGen := didgen.NewDomainIdGenerator(&models.GitlabPipeline{})
+	converter, err := api.NewStatefulDataConverter(&api.StatefulDataConverterArgs[models.GitlabJob]{
+		SubtaskCommonArgs: subtaskCommonArgs,
+		Input: func(stateManager *api.SubtaskStateManager) (dal.Rows, errors.Error) {
+			clauses := []dal.Clause{
+				dal.From(models.GitlabJob{}),
+				dal.Where("project_id = ? and connection_id = ?", data.Options.ProjectId, data.Options.ConnectionId),
+			}
+			if stateManager.IsIncremental() {
+				since := stateManager.GetSince()
+				if since != nil {
+					clauses = append(clauses, dal.Where("updated_at >= ? ", since))
+				}
+			}
+			return db.Cursor(clauses...)
 		},
-		Convert: func(inputRow interface{}) ([]interface{}, errors.Error) {
-			gitlabJob := inputRow.(*gitlabModels.GitlabJob)
-
+		Convert: func(gitlabJob *models.GitlabJob) ([]interface{}, errors.Error) {
 			createdAt := time.Now()
 			if gitlabJob.GitlabCreatedAt != nil {
 				createdAt = *gitlabJob.GitlabCreatedAt
diff --git a/backend/plugins/gitlab/tasks/job_extractor.go b/backend/plugins/gitlab/tasks/job_extractor.go
index 420fe7a30..15b1343dc 100644
--- a/backend/plugins/gitlab/tasks/job_extractor.go
+++ b/backend/plugins/gitlab/tasks/job_extractor.go
@@ -18,8 +18,6 @@ limitations under the License.
 package tasks
 
 import (
-	"encoding/json"
-
 	"github.com/apache/incubator-devlake/core/errors"
 	"github.com/apache/incubator-devlake/core/models/common"
 	"github.com/apache/incubator-devlake/core/plugin"
@@ -59,18 +57,12 @@ var ExtractApiJobsMeta = plugin.SubTaskMeta{
 	Dependencies:     []*plugin.SubTaskMeta{&CollectApiJobsMeta},
 }
 
-func ExtractApiJobs(taskCtx plugin.SubTaskContext) errors.Error {
-	rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_JOB_TABLE)
-
-	extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{
-		RawDataSubTaskArgs: *rawDataSubTaskArgs,
-		Extract: func(row *api.RawData) ([]interface{}, errors.Error) {
-			gitlabApiJob := &ApiJob{}
-			err := errors.Convert(json.Unmarshal(row.Data, gitlabApiJob))
-			if err != nil {
-				return nil, err
-			}
+func ExtractApiJobs(subtaskCtx plugin.SubTaskContext) errors.Error {
+	subtaskCommonArgs, data := CreateSubtaskCommonArgs(subtaskCtx, RAW_JOB_TABLE)
 
+	extractor, err := api.NewStatefulApiExtractor(&api.StatefulApiExtractorArgs[ApiJob]{
+		SubtaskCommonArgs: subtaskCommonArgs,
+		Extract: func(gitlabApiJob *ApiJob, row *api.RawData) ([]interface{}, errors.Error) {
 			gitlabPipeline, err := convertJob(gitlabApiJob, data.Options.ProjectId)
 			if err != nil {
 				return nil, err
diff --git a/backend/plugins/gitlab/tasks/mr_assignee_convertor.go b/backend/plugins/gitlab/tasks/mr_assignee_convertor.go
index d638d1dc0..d22714a74 100644
--- a/backend/plugins/gitlab/tasks/mr_assignee_convertor.go
+++ b/backend/plugins/gitlab/tasks/mr_assignee_convertor.go
@@ -18,14 +18,12 @@ limitations under the License.
 package tasks
 
 import (
-	"reflect"
-
 	"github.com/apache/incubator-devlake/core/dal"
 	"github.com/apache/incubator-devlake/core/errors"
 	"github.com/apache/incubator-devlake/core/models/domainlayer/code"
 	"github.com/apache/incubator-devlake/core/models/domainlayer/didgen"
 	"github.com/apache/incubator-devlake/core/plugin"
-	helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+	"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
 	"github.com/apache/incubator-devlake/plugins/gitlab/models"
 )
 
@@ -39,39 +37,33 @@ var ConvertMrAssigneesMeta = plugin.SubTaskMeta{
 	EnabledByDefault: true,
 	Description:      "Convert tool layer table gitlab_assignees into  domain layer table pull_request_assignees",
 	DomainTypes:      []string{plugin.DOMAIN_TYPE_CODE_REVIEW},
-	Dependencies:     []*plugin.SubTaskMeta{&ExtractApiMergeRequestDetailsMeta},
+	Dependencies:     []*plugin.SubTaskMeta{&ConvertApiMergeRequestsMeta},
 }
 
-func ConvertMrAssignees(taskCtx plugin.SubTaskContext) errors.Error {
-	db := taskCtx.GetDal()
-	rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_MERGE_REQUEST_TABLE)
+func ConvertMrAssignees(subtaskCtx plugin.SubTaskContext) errors.Error {
+	subtaskCommonArgs, data := CreateSubtaskCommonArgs(subtaskCtx, RAW_MERGE_REQUEST_TABLE)
+	db := subtaskCtx.GetDal()
 	projectId := data.Options.ProjectId
-	clauses := []dal.Clause{
-		dal.Select("_tool_gitlab_assignees.*"),
-		dal.From(&models.GitlabAssignee{}),
-		dal.Join(`left join _tool_gitlab_merge_requests mr on
-			mr.gitlab_id = _tool_gitlab_assignees.merge_request_id`),
-		dal.Where(`mr.project_id = ?
-			and mr.connection_id = ?`,
-			projectId, data.Options.ConnectionId),
-		dal.Orderby("_tool_gitlab_assignees.merge_request_id ASC"),
-	}
-
-	cursor, err := db.Cursor(clauses...)
-	if err != nil {
-		return err
-	}
-	defer cursor.Close()
-
 	mrIdGen := didgen.NewDomainIdGenerator(&models.GitlabMergeRequest{})
 	accountIdGen := didgen.NewDomainIdGenerator(&models.GitlabAccount{})
 
-	converter, err := helper.NewDataConverter(helper.DataConverterArgs{
-		RawDataSubTaskArgs: *rawDataSubTaskArgs,
-		InputRowType:       reflect.TypeOf(models.GitlabAssignee{}),
-		Input:              cursor,
-		Convert: func(inputRow interface{}) ([]interface{}, errors.Error) {
-			mrAssignee := inputRow.(*models.GitlabAssignee)
+	converter, err := api.NewStatefulDataConverter(&api.StatefulDataConverterArgs[models.GitlabAssignee]{
+		SubtaskCommonArgs: subtaskCommonArgs,
+		Input: func(stateManager *api.SubtaskStateManager) (dal.Rows, errors.Error) {
+			clauses := []dal.Clause{
+				dal.From(&models.GitlabAssignee{}),
+				dal.Where(`project_id = ? and connection_id = ?`, projectId, data.Options.ConnectionId),
+				dal.Orderby("merge_request_id ASC"),
+			}
+			if stateManager.IsIncremental() {
+				since := stateManager.GetSince()
+				if since != nil {
+					clauses = append(clauses, dal.Where("updated_at >= ? ", since))
+				}
+			}
+			return db.Cursor(clauses...)
+		},
+		Convert: func(mrAssignee *models.GitlabAssignee) ([]interface{}, errors.Error) {
 			domainPrAssigne := &code.PullRequestAssignee{
 				PullRequestId: mrIdGen.Generate(data.Options.ConnectionId, mrAssignee.MergeRequestId),
 				AssigneeId:    accountIdGen.Generate(data.Options.ConnectionId, mrAssignee.AssigneeId),
diff --git a/backend/plugins/gitlab/tasks/mr_commit_convertor.go b/backend/plugins/gitlab/tasks/mr_commit_convertor.go
index 74ae0d926..6f7ff2a5f 100644
--- a/backend/plugins/gitlab/tasks/mr_commit_convertor.go
+++ b/backend/plugins/gitlab/tasks/mr_commit_convertor.go
@@ -18,14 +18,12 @@ limitations under the License.
 package tasks
 
 import (
-	"reflect"
-
 	"github.com/apache/incubator-devlake/core/dal"
 	"github.com/apache/incubator-devlake/core/errors"
 	"github.com/apache/incubator-devlake/core/models/domainlayer/code"
 	"github.com/apache/incubator-devlake/core/models/domainlayer/didgen"
 	"github.com/apache/incubator-devlake/core/plugin"
-	helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+	"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
 	"github.com/apache/incubator-devlake/plugins/gitlab/models"
 )
 
@@ -39,45 +37,39 @@ var ConvertApiMrCommitsMeta = plugin.SubTaskMeta{
 	EnabledByDefault: true,
 	Description:      "Add domain layer PullRequestCommit according to GitlabMrCommit",
 	DomainTypes:      []string{plugin.DOMAIN_TYPE_CODE_REVIEW},
-	Dependencies:     []*plugin.SubTaskMeta{&ConvertMrCommentMeta},
+	Dependencies:     []*plugin.SubTaskMeta{&ConvertApiMergeRequestsMeta},
 }
 
-func ConvertApiMergeRequestsCommits(taskCtx plugin.SubTaskContext) errors.Error {
-	rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_MERGE_REQUEST_COMMITS_TABLE)
-	db := taskCtx.GetDal()
-
-	clauses := []dal.Clause{
-		dal.From(&models.GitlabMrCommit{}),
-		dal.Join(`left join _tool_gitlab_merge_requests
-			on _tool_gitlab_merge_requests.gitlab_id =
-			_tool_gitlab_mr_commits.merge_request_id`),
-		dal.Where(`_tool_gitlab_merge_requests.project_id = ?
-			and _tool_gitlab_merge_requests.connection_id = ?`,
-			data.Options.ProjectId, data.Options.ConnectionId),
-		dal.Orderby("merge_request_id ASC"),
-	}
+func ConvertApiMergeRequestsCommits(subtaskCtx plugin.SubTaskContext) errors.Error {
+	subtaskCommonArgs, data := CreateSubtaskCommonArgs(subtaskCtx, RAW_MERGE_REQUEST_COMMITS_TABLE)
+	db := subtaskCtx.GetDal()
 
-	cursor, err := db.Cursor(clauses...)
-	if err != nil {
-		return err
-	}
-
-	// TODO: adopt batch indate operation
 	domainIdGenerator := didgen.NewDomainIdGenerator(&models.GitlabMergeRequest{})
 
-	converter, err := helper.NewDataConverter(helper.DataConverterArgs{
-		RawDataSubTaskArgs: *rawDataSubTaskArgs,
-		InputRowType:       reflect.TypeOf(models.GitlabMrCommit{}),
-		Input:              cursor,
-
-		Convert: func(inputRow interface{}) ([]interface{}, errors.Error) {
-			GitlabMrCommit := inputRow.(*models.GitlabMrCommit)
+	converter, err := api.NewStatefulDataConverter(&api.StatefulDataConverterArgs[models.GitlabMrCommit]{
+		SubtaskCommonArgs: subtaskCommonArgs,
+		Input: func(stateManager *api.SubtaskStateManager) (dal.Rows, errors.Error) {
+			clauses := []dal.Clause{
+				dal.Select("c.*"),
+				dal.From("_tool_gitlab_mr_commits c"),
+				dal.Join(`LEFT JOIN _tool_gitlab_merge_requests mr ON mr.gitlab_id = c.merge_request_id AND c.connection_id = mr.connection_id`),
+				dal.Where(`mr.project_id = ?  and mr.connection_id = ?`, data.Options.ProjectId, data.Options.ConnectionId),
+			}
+			if stateManager.IsIncremental() {
+				since := stateManager.GetSince()
+				if since != nil {
+					clauses = append(clauses, dal.Where("c.updated_at >= ? ", since))
+				}
+			}
+			return db.Cursor(clauses...)
+		},
+		Convert: func(mrCommit *models.GitlabMrCommit) ([]interface{}, errors.Error) {
 			domainPrcommit := &code.PullRequestCommit{
-				CommitSha:          GitlabMrCommit.CommitSha,
-				PullRequestId:      domainIdGenerator.Generate(data.Options.ConnectionId, GitlabMrCommit.MergeRequestId),
-				CommitAuthorName:   GitlabMrCommit.CommitAuthorName,
-				CommitAuthorEmail:  GitlabMrCommit.CommitAuthorEmail,
-				CommitAuthoredDate: *GitlabMrCommit.CommitAuthoredDate,
+				CommitSha:          mrCommit.CommitSha,
+				PullRequestId:      domainIdGenerator.Generate(data.Options.ConnectionId, mrCommit.MergeRequestId),
+				CommitAuthorName:   mrCommit.CommitAuthorName,
+				CommitAuthorEmail:  mrCommit.CommitAuthorEmail,
+				CommitAuthoredDate: *mrCommit.CommitAuthoredDate,
 			}
 			return []interface{}{
 				domainPrcommit,
diff --git a/backend/plugins/gitlab/tasks/mr_commit_extractor.go b/backend/plugins/gitlab/tasks/mr_commit_extractor.go
index 5c73c8665..7a94f9fee 100644
--- a/backend/plugins/gitlab/tasks/mr_commit_extractor.go
+++ b/backend/plugins/gitlab/tasks/mr_commit_extractor.go
@@ -40,18 +40,33 @@ var ExtractApiMrCommitsMeta = plugin.SubTaskMeta{
 	Dependencies:     []*plugin.SubTaskMeta{&CollectApiMrCommitsMeta},
 }
 
-func ExtractApiMergeRequestsCommits(taskCtx plugin.SubTaskContext) errors.Error {
-	rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_MERGE_REQUEST_COMMITS_TABLE)
+type GitlabApiCommit struct {
+	GitlabId       string `json:"id"`
+	Title          string
+	Message        string
+	ProjectId      int
+	ShortId        string             `json:"short_id"`
+	AuthorName     string             `json:"author_name"`
+	AuthorEmail    string             `json:"author_email"`
+	AuthoredDate   common.Iso8601Time `json:"authored_date"`
+	CommitterName  string             `json:"committer_name"`
+	CommitterEmail string             `json:"committer_email"`
+	CommittedDate  common.Iso8601Time `json:"committed_date"`
+	WebUrl         string             `json:"web_url"`
+	Stats          struct {
+		Additions int
+		Deletions int
+		Total     int
+	}
+}
+
+func ExtractApiMergeRequestsCommits(subtaskCtx plugin.SubTaskContext) errors.Error {
+	subtaskCommonArgs, data := CreateSubtaskCommonArgs(subtaskCtx, RAW_MERGE_REQUEST_COMMITS_TABLE)
 
-	extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{
-		RawDataSubTaskArgs: *rawDataSubTaskArgs,
-		Extract: func(row *api.RawData) ([]interface{}, errors.Error) {
+	extractor, err := api.NewStatefulApiExtractor(&api.StatefulApiExtractorArgs[GitlabApiCommit]{
+		SubtaskCommonArgs: subtaskCommonArgs,
+		Extract: func(gitlabApiCommit *GitlabApiCommit, row *api.RawData) ([]interface{}, errors.Error) {
 			// create gitlab commit
-			gitlabApiCommit := &GitlabApiCommit{}
-			err := errors.Convert(json.Unmarshal(row.Data, gitlabApiCommit))
-			if err != nil {
-				return nil, err
-			}
 			gitlabCommit, err := ConvertCommit(gitlabApiCommit)
 			if err != nil {
 				return nil, err
@@ -91,3 +106,24 @@ func ExtractApiMergeRequestsCommits(taskCtx plugin.SubTaskContext) errors.Error
 
 	return extractor.Execute()
 }
+
+// Convert the API response to our DB model instance
+func ConvertCommit(commit *GitlabApiCommit) (*models.GitlabCommit, errors.Error) {
+	gitlabCommit := &models.GitlabCommit{
+		Sha:            commit.GitlabId,
+		Title:          commit.Title,
+		Message:        commit.Message,
+		ShortId:        commit.ShortId,
+		AuthorName:     commit.AuthorName,
+		AuthorEmail:    commit.AuthorEmail,
+		AuthoredDate:   commit.AuthoredDate.ToTime(),
+		CommitterName:  commit.CommitterName,
+		CommitterEmail: commit.CommitterEmail,
+		CommittedDate:  commit.CommittedDate.ToTime(),
+		WebUrl:         commit.WebUrl,
+		Additions:      commit.Stats.Additions,
+		Deletions:      commit.Stats.Deletions,
+		Total:          commit.Stats.Total,
+	}
+	return gitlabCommit, nil
+}
diff --git a/backend/plugins/gitlab/tasks/mr_convertor.go b/backend/plugins/gitlab/tasks/mr_convertor.go
index 478276c71..f2dd24d74 100644
--- a/backend/plugins/gitlab/tasks/mr_convertor.go
+++ b/backend/plugins/gitlab/tasks/mr_convertor.go
@@ -18,15 +18,13 @@ limitations under the License.
 package tasks
 
 import (
-	"reflect"
-
 	"github.com/apache/incubator-devlake/core/dal"
 	"github.com/apache/incubator-devlake/core/errors"
 	"github.com/apache/incubator-devlake/core/models/domainlayer"
 	"github.com/apache/incubator-devlake/core/models/domainlayer/code"
 	"github.com/apache/incubator-devlake/core/models/domainlayer/didgen"
 	"github.com/apache/incubator-devlake/core/plugin"
-	helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+	"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
 	"github.com/apache/incubator-devlake/plugins/gitlab/models"
 )
 
@@ -43,32 +41,43 @@ var ConvertApiMergeRequestsMeta = plugin.SubTaskMeta{
 	Dependencies:     []*plugin.SubTaskMeta{&ConvertProjectMeta},
 }
 
-func ConvertApiMergeRequests(taskCtx plugin.SubTaskContext) errors.Error {
-	rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_MERGE_REQUEST_TABLE)
-	db := taskCtx.GetDal()
-	clauses := []dal.Clause{
-		dal.From(&models.GitlabMergeRequest{}),
-		dal.Where("project_id=? and connection_id = ?", data.Options.ProjectId, data.Options.ConnectionId),
-	}
-
-	cursor, err := db.Cursor(clauses...)
-	if err != nil {
-		return err
-	}
-	defer cursor.Close()
+func ConvertApiMergeRequests(subtaskCtx plugin.SubTaskContext) errors.Error {
+	subtaskCommonArgs, data := CreateSubtaskCommonArgs(subtaskCtx, RAW_MERGE_REQUEST_TABLE)
+	db := subtaskCtx.GetDal()
 
 	domainMrIdGenerator := didgen.NewDomainIdGenerator(&models.GitlabMergeRequest{})
 	domainRepoIdGenerator := didgen.NewDomainIdGenerator(&models.GitlabProject{})
 	domainUserIdGen := didgen.NewDomainIdGenerator(&models.GitlabAccount{})
 
-	converter, err := helper.NewDataConverter(helper.DataConverterArgs{
-		RawDataSubTaskArgs: *rawDataSubTaskArgs,
-		InputRowType:       reflect.TypeOf(models.GitlabMergeRequest{}),
-		Input:              cursor,
-
-		Convert: func(inputRow interface{}) ([]interface{}, errors.Error) {
-			gitlabMr := inputRow.(*models.GitlabMergeRequest)
-
+	converter, err := api.NewStatefulDataConverter(&api.StatefulDataConverterArgs[models.GitlabMergeRequest]{
+		SubtaskCommonArgs: subtaskCommonArgs,
+		Input: func(stateManager *api.SubtaskStateManager) (dal.Rows, errors.Error) {
+			clauses := []dal.Clause{
+				dal.From(&models.GitlabMergeRequest{}),
+				dal.Where("project_id=? and connection_id = ?", data.Options.ProjectId, data.Options.ConnectionId),
+			}
+			if stateManager.IsIncremental() {
+				since := stateManager.GetSince()
+				if since != nil {
+					clauses = append(clauses, dal.Where("updated_at >= ? ", since))
+				}
+			}
+			return db.Cursor(clauses...)
+		},
+		BeforeConvert: func(gitlabMr *models.GitlabMergeRequest, stateManager *api.SubtaskStateManager) errors.Error {
+			mrId := domainMrIdGenerator.Generate(data.Options.ConnectionId, gitlabMr.GitlabId)
+			if err := db.Delete(&code.PullRequestAssignee{}, dal.Where("pull_request_id = ?", mrId)); err != nil {
+				return err
+			}
+			if err := db.Delete(&code.PullRequestReviewer{}, dal.Where("pull_request_id = ?", mrId)); err != nil {
+				return err
+			}
+			if err := db.Delete(&code.PullRequestLabel{}, dal.Where("pull_request_id = ?", mrId)); err != nil {
+				return err
+			}
+			return nil
+		},
+		Convert: func(gitlabMr *models.GitlabMergeRequest) ([]interface{}, errors.Error) {
 			domainPr := &code.PullRequest{
 				DomainEntity: domainlayer.DomainEntity{
 					Id: domainMrIdGenerator.Generate(data.Options.ConnectionId, gitlabMr.GitlabId),
diff --git a/backend/plugins/gitlab/tasks/mr_detail_extractor.go b/backend/plugins/gitlab/tasks/mr_detail_extractor.go
index 3dbc72e8a..394575246 100644
--- a/backend/plugins/gitlab/tasks/mr_detail_extractor.go
+++ b/backend/plugins/gitlab/tasks/mr_detail_extractor.go
@@ -18,7 +18,6 @@ limitations under the License.
 package tasks
 
 import (
-	"encoding/json"
 	"regexp"
 
 	"github.com/apache/incubator-devlake/core/errors"
@@ -37,11 +36,12 @@ var ExtractApiMergeRequestDetailsMeta = plugin.SubTaskMeta{
 	EnabledByDefault: true,
 	Description:      "Extract raw merge request Details data into tool layer table GitlabMergeRequest and GitlabReviewer",
 	DomainTypes:      []string{plugin.DOMAIN_TYPE_CODE_REVIEW},
-	Dependencies:     []*plugin.SubTaskMeta{&ExtractApiCommitsMeta},
+	Dependencies:     []*plugin.SubTaskMeta{&CollectApiMergeRequestDetailsMeta},
 }
 
-func ExtractApiMergeRequestDetails(taskCtx plugin.SubTaskContext) errors.Error {
-	rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_MERGE_REQUEST_DETAIL_TABLE)
+func ExtractApiMergeRequestDetails(subtaskCtx plugin.SubTaskContext) errors.Error {
+	subtaskCommonArgs, data := CreateSubtaskCommonArgs(subtaskCtx, RAW_MERGE_REQUEST_DETAIL_TABLE)
+	db := subtaskCtx.GetDal()
 	config := data.Options.ScopeConfig
 	var labelTypeRegex *regexp.Regexp
 	var labelComponentRegex *regexp.Regexp
@@ -60,15 +60,14 @@ func ExtractApiMergeRequestDetails(taskCtx plugin.SubTaskContext) errors.Error {
 			return errors.Default.Wrap(err, "regexp Compile prComponent failed")
 		}
 	}
-	extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{
-		RawDataSubTaskArgs: *rawDataSubTaskArgs,
-		Extract: func(row *api.RawData) ([]interface{}, errors.Error) {
-			mr := &MergeRequestRes{}
-			err := errors.Convert(json.Unmarshal(row.Data, mr))
-			if err != nil {
-				return nil, err
-			}
-
+	subtaskCommonArgs.SubtaskConfig = map[string]string{
+		"prType":      prType,
+		"prComponent": prComponent,
+	}
+	extractor, err := api.NewStatefulApiExtractor(&api.StatefulApiExtractorArgs[MergeRequestRes]{
+		SubtaskCommonArgs: subtaskCommonArgs,
+		BeforeExtract:     beforeExtractMr(db, data),
+		Extract: func(mr *MergeRequestRes, row *api.RawData) ([]interface{}, errors.Error) {
 			gitlabMergeRequest, err := convertMergeRequest(mr)
 			if err != nil {
 				return nil, err
diff --git a/backend/plugins/gitlab/tasks/mr_enricher.go b/backend/plugins/gitlab/tasks/mr_enricher.go
deleted file mode 100644
index 327765208..000000000
--- a/backend/plugins/gitlab/tasks/mr_enricher.go
+++ /dev/null
@@ -1,165 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements.  See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License.  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package tasks
-
-import (
-	"reflect"
-	"time"
-
-	"github.com/apache/incubator-devlake/core/dal"
-	"github.com/apache/incubator-devlake/core/errors"
-	"github.com/apache/incubator-devlake/core/plugin"
-	helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
-	"github.com/apache/incubator-devlake/plugins/gitlab/models"
-)
-
-func init() {
-	RegisterSubtaskMeta(&EnrichMergeRequestsMeta)
-}
-
-var EnrichMergeRequestsMeta = plugin.SubTaskMeta{
-	Name:             "Enrich  Merge Requests",
-	EntryPoint:       EnrichMergeRequests,
-	EnabledByDefault: true,
-	Description:      "Enrich merge requests data from GitlabCommit, GitlabMrNote and GitlabMergeRequest",
-	DomainTypes:      []string{plugin.DOMAIN_TYPE_CODE_REVIEW},
-	Dependencies:     []*plugin.SubTaskMeta{&ExtractApiJobsMeta},
-}
-
-func EnrichMergeRequests(taskCtx plugin.SubTaskContext) errors.Error {
-	rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_MERGE_REQUEST_TABLE)
-
-	db := taskCtx.GetDal()
-	clauses := []dal.Clause{
-		dal.From(&models.GitlabMergeRequest{}),
-		dal.Where("project_id=? and connection_id = ?", data.Options.ProjectId, data.Options.ConnectionId),
-	}
-
-	cursor, err := db.Cursor(clauses...)
-	if err != nil {
-		return err
-	} // get mrs from theDB
-	defer cursor.Close()
-
-	converter, err := helper.NewDataConverter(helper.DataConverterArgs{
-		RawDataSubTaskArgs: *rawDataSubTaskArgs,
-		InputRowType:       reflect.TypeOf(models.GitlabMergeRequest{}),
-		Input:              cursor,
-
-		Convert: func(inputRow interface{}) ([]interface{}, errors.Error) {
-			gitlabMr := inputRow.(*models.GitlabMergeRequest)
-			// enrich first_comment_time field
-			notes := make([]models.GitlabMrNote, 0)
-			// `system` = 0 is needed since we only care about human comments
-			noteClauses := []dal.Clause{
-				dal.From(&models.GitlabMrNote{}),
-				dal.Where("merge_request_id = ? AND is_system = ? AND connection_id = ? ",
-					gitlabMr.GitlabId, false, data.Options.ConnectionId),
-				dal.Orderby("gitlab_created_at asc"),
-			}
-			err = db.All(&notes, noteClauses...)
-			if err != nil {
-				return nil, err
-			}
-
-			commits := make([]models.GitlabCommit, 0)
-			commitClauses := []dal.Clause{
-				dal.From(&models.GitlabCommit{}),
-				dal.Join(`join _tool_gitlab_mr_commits gmrc
-					on gmrc.commit_sha = _tool_gitlab_commits.sha`),
-				dal.Where("merge_request_id = ? AND gmrc.connection_id = ?",
-					gitlabMr.GitlabId, data.Options.ConnectionId),
-				dal.Orderby("authored_date asc"),
-			}
-			err = db.All(&commits, commitClauses...)
-			if err != nil {
-				return nil, err
-			}
-
-			// calculate reviewRounds from commits and notes
-			reviewRounds := getReviewRounds(commits, notes)
-			gitlabMr.ReviewRounds = reviewRounds
-
-			if len(notes) > 0 {
-				earliestNote, err := findEarliestNote(notes)
-				if err != nil {
-					return nil, err
-				}
-				if earliestNote != nil {
-					gitlabMr.FirstCommentTime = &earliestNote.GitlabCreatedAt
-				}
-			}
-			return []interface{}{
-				gitlabMr,
-			}, nil
-		},
-	})
-	if err != nil {
-		return err
-	}
-
-	return converter.Execute()
-}
-
-func findEarliestNote(notes []models.GitlabMrNote) (*models.GitlabMrNote, errors.Error) {
-	var earliestNote *models.GitlabMrNote
-	earliestTime := time.Now()
-	for i := range notes {
-		if !notes[i].Resolvable {
-			continue
-		}
-		noteTime := notes[i].GitlabCreatedAt
-		if noteTime.Before(earliestTime) {
-			earliestTime = noteTime
-			earliestNote = &notes[i]
-		}
-	}
-	return earliestNote, nil
-}
-
-func getReviewRounds(commits []models.GitlabCommit, notes []models.GitlabMrNote) int {
-	i := 0
-	j := 0
-	reviewRounds := 0
-	if len(commits) == 0 && len(notes) == 0 {
-		return 1
-	}
-	// state is used to keep track of previous activity
-	// 0: init, 1: commit, 2: comment
-	// whenever state is switched to comment, we increment reviewRounds by 1
-	state := 0 // 0, 1, 2
-	for i < len(commits) && j < len(notes) {
-		if commits[i].AuthoredDate.Before(notes[j].GitlabCreatedAt) {
-			i++
-			state = 1
-		} else {
-			j++
-			if state != 2 {
-				reviewRounds++
-			}
-			state = 2
-		}
-	}
-	// There's another implicit round of review in 2 scenarios
-	// One: the last state is commit (state == 1)
-	// Two: the last state is comment but there're still commits left
-	if state == 1 || i < len(commits) {
-		reviewRounds++
-	}
-	return reviewRounds
-}
diff --git a/backend/plugins/gitlab/tasks/mr_enricher_test.go b/backend/plugins/gitlab/tasks/mr_enricher_test.go
deleted file mode 100644
index f22fd275a..000000000
--- a/backend/plugins/gitlab/tasks/mr_enricher_test.go
+++ /dev/null
@@ -1,112 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements.  See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License.  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package tasks
-
-import (
-	"github.com/apache/incubator-devlake/plugins/gitlab/models"
-	"github.com/stretchr/testify/assert"
-	"testing"
-	"time"
-)
-
-func TestFindEarliestNote(t *testing.T) {
-	baseTime, err := time.Parse(time.RFC3339, "2022-01-02T15:04:05Z")
-	assert.Nil(t, err)
-	// Create some sample notes
-	note1 := models.GitlabMrNote{Resolvable: true, GitlabCreatedAt: baseTime.Add(-time.Hour)}
-	note2 := models.GitlabMrNote{Resolvable: false, GitlabCreatedAt: baseTime.Add(-time.Minute)}
-	note3 := models.GitlabMrNote{Resolvable: true, GitlabCreatedAt: baseTime.Add(-time.Second)}
-
-	// Call the function with the sample notes
-	notes := []models.GitlabMrNote{note1, note2, note3}
-	earliestNote, err := findEarliestNote(notes)
-
-	// Check that no error was returned
-	if err != nil {
-		t.Errorf("findEarliestNote returned an error: %v", err)
-	}
-
-	// Check that the correct note was returned
-	if earliestNote == nil {
-		t.Errorf("findEarliestNote returned nil, expected a note")
-	}
-	if !assert.Equal(t, note1, *earliestNote) {
-		t.Errorf("findEarliestNote returned the wrong note: got %v, expected %v", earliestNote, &note1)
-	}
-
-	// Modify one of the notes to make it unresolvable
-	notes[0].Resolvable = false
-
-	// Call the function again with the modified notes
-	earliestNote, err = findEarliestNote(notes)
-
-	// Check that no error was returned
-	if err != nil {
-		t.Errorf("findEarliestNote returned an error: %v", err)
-	}
-
-	// Check that the correct note was returned
-	if earliestNote == nil {
-		t.Errorf("findEarliestNote returned nil, expected a note")
-	}
-	if !assert.Equal(t, note3, *earliestNote) {
-		t.Errorf("findEarliestNote returned the wrong note: got %v, expected %v", earliestNote, &note3)
-	}
-}
-
-func TestGetReviewRounds(t *testing.T) {
-	baseTime, err := time.Parse(time.RFC3339, "2022-01-02T15:04:05Z")
-	assert.Nil(t, err)
-	// Test case 1: empty input
-	var commits []models.GitlabCommit
-	var notes []models.GitlabMrNote
-	expected := 1
-	if got := getReviewRounds(commits, notes); got != expected {
-		t.Errorf("getReviewRounds(%v, %v) = %d, expected %d", commits, notes, got, expected)
-	}
-
-	// Test case 2: single comment
-	commits = []models.GitlabCommit{
-		{AuthoredDate: baseTime.Add(-time.Hour * 2)},
-		{AuthoredDate: baseTime.Add(-time.Hour)},
-	}
-	notes = []models.GitlabMrNote{
-		{GitlabCreatedAt: baseTime},
-	}
-	expected = 1
-	if got := getReviewRounds(commits, notes); got != expected {
-		t.Errorf("getReviewRounds(%v, %v) = %d, expected %d", commits, notes, got, expected)
-	}
-
-	// Test case 3: multiple comments
-	commits = []models.GitlabCommit{
-		{AuthoredDate: baseTime.Add(-time.Hour * 15)},
-		{AuthoredDate: baseTime.Add(-time.Hour * 9)},
-		{AuthoredDate: baseTime.Add(-time.Hour * 3)},
-		{AuthoredDate: baseTime},
-	}
-	notes = []models.GitlabMrNote{
-		{GitlabCreatedAt: baseTime.Add(-time.Hour * 14)},
-		{GitlabCreatedAt: baseTime.Add(-time.Hour * 7)},
-		{GitlabCreatedAt: baseTime.Add(-time.Hour * 2)},
-	}
-	expected = 4
-	if got := getReviewRounds(commits, notes); got != expected {
-		t.Errorf("getReviewRounds(%v, %v) = %d, expected %d", commits, notes, got, expected)
-	}
-}
diff --git a/backend/plugins/gitlab/tasks/mr_extractor.go b/backend/plugins/gitlab/tasks/mr_extractor.go
index 76253833b..3a8457b0c 100644
--- a/backend/plugins/gitlab/tasks/mr_extractor.go
+++ b/backend/plugins/gitlab/tasks/mr_extractor.go
@@ -18,10 +18,10 @@ limitations under the License.
 package tasks
 
 import (
-	"encoding/json"
 	"regexp"
 	"strings"
 
+	"github.com/apache/incubator-devlake/core/dal"
 	"github.com/apache/incubator-devlake/core/errors"
 	"github.com/apache/incubator-devlake/core/models/common"
 	"github.com/apache/incubator-devlake/core/plugin"
@@ -96,8 +96,10 @@ var ExtractApiMergeRequestsMeta = plugin.SubTaskMeta{
 	Dependencies:     []*plugin.SubTaskMeta{&CollectApiMergeRequestsMeta},
 }
 
-func ExtractApiMergeRequests(taskCtx plugin.SubTaskContext) errors.Error {
-	rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_MERGE_REQUEST_TABLE)
+func ExtractApiMergeRequests(subtaskCtx plugin.SubTaskContext) errors.Error {
+	subtaskCommonArgs, data := CreateSubtaskCommonArgs(subtaskCtx, RAW_MERGE_REQUEST_TABLE)
+
+	db := subtaskCtx.GetDal()
 	config := data.Options.ScopeConfig
 	var labelTypeRegex *regexp.Regexp
 	var labelComponentRegex *regexp.Regexp
@@ -118,17 +120,15 @@ func ExtractApiMergeRequests(taskCtx plugin.SubTaskContext) errors.Error {
 		}
 	}
 
-	extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{
-		RawDataSubTaskArgs: *rawDataSubTaskArgs,
-		Extract: func(row *api.RawData) ([]interface{}, errors.Error) {
-
-			mr := &MergeRequestRes{}
-			s := string(row.Data)
-			err := errors.Convert(json.Unmarshal(row.Data, mr))
-			if err != nil {
-				return nil, err
-			}
+	subtaskCommonArgs.SubtaskConfig = map[string]any{
+		"prType":      prType,
+		"prComponent": prComponent,
+	}
 
+	extractor, err := api.NewStatefulApiExtractor(&api.StatefulApiExtractorArgs[MergeRequestRes]{
+		SubtaskCommonArgs: subtaskCommonArgs,
+		BeforeExtract:     beforeExtractMr(db, data),
+		Extract: func(mr *MergeRequestRes, row *api.RawData) ([]interface{}, errors.Error) {
 			gitlabMergeRequest, err := convertMergeRequest(mr)
 			if err != nil {
 				return nil, err
@@ -136,15 +136,23 @@ func ExtractApiMergeRequests(taskCtx plugin.SubTaskContext) errors.Error {
 
 			// if we can not find merged_at and closed_at info in the detail
 			// we need get detail for gitlab v11
+			s := string(row.Data)
 			if !strings.Contains(s, "\"merged_at\":") {
 				if !strings.Contains(s, "\"closed_at\":") {
 					gitlabMergeRequest.IsDetailRequired = true
 				}
 			}
 
-			results := make([]interface{}, 0, len(mr.Reviewers)+1)
+			results := make([]interface{}, 0, len(mr.Reviewers)+len(mr.Labels)+1)
 			gitlabMergeRequest.ConnectionId = data.Options.ConnectionId
 			results = append(results, gitlabMergeRequest)
+			err = db.Delete(
+				&models.GitlabMrLabel{},
+				dal.Where("connection_id = ? AND mr_id = ?", data.Options.ConnectionId, gitlabMergeRequest.GitlabId),
+			)
+			if err != nil {
+				return nil, err
+			}
 			for _, label := range mr.Labels {
 				results = append(results, &models.GitlabMrLabel{
 					MrId:         gitlabMergeRequest.GitlabId,
@@ -160,6 +168,13 @@ func ExtractApiMergeRequests(taskCtx plugin.SubTaskContext) errors.Error {
 					gitlabMergeRequest.Component = label
 				}
 			}
+			err = db.Delete(
+				&models.GitlabReviewer{},
+				dal.Where("connection_id = ? AND merge_request_id = ?", data.Options.ConnectionId, gitlabMergeRequest.GitlabId),
+			)
+			if err != nil {
+				return nil, err
+			}
 			for _, reviewer := range mr.Reviewers {
 				gitlabReviewer := &models.GitlabReviewer{
 					ConnectionId:   data.Options.ConnectionId,
@@ -174,6 +189,13 @@ func ExtractApiMergeRequests(taskCtx plugin.SubTaskContext) errors.Error {
 				}
 				results = append(results, gitlabReviewer)
 			}
+			err = db.Delete(
+				&models.GitlabAssignee{},
+				dal.Where("connection_id = ? AND merge_request_id = ?", data.Options.ConnectionId, gitlabMergeRequest.GitlabId),
+			)
+			if err != nil {
+				return nil, err
+			}
 			for _, assignee := range mr.Assignees {
 				gitlabAssignee := &models.GitlabAssignee{
 					ConnectionId:   data.Options.ConnectionId,
@@ -188,17 +210,13 @@ func ExtractApiMergeRequests(taskCtx plugin.SubTaskContext) errors.Error {
 				}
 				results = append(results, gitlabAssignee)
 			}
-
 			return results, nil
 		},
 	})
-
 	if err != nil {
-		return errors.Convert(err)
+		return err
 	}
-
 	return extractor.Execute()
-
 }
 
 func convertMergeRequest(mr *MergeRequestRes) (*models.GitlabMergeRequest, errors.Error) {
@@ -230,3 +248,25 @@ func convertMergeRequest(mr *MergeRequestRes) (*models.GitlabMergeRequest, error
 	}
 	return gitlabMergeRequest, nil
 }
+
+func beforeExtractMr(db dal.Dal, data *GitlabTaskData) func(mr *MergeRequestRes, stateManager *api.SubtaskStateManager) errors.Error {
+	return func(mr *MergeRequestRes, stateManager *api.SubtaskStateManager) errors.Error {
+		if stateManager.IsIncremental() {
+			err := db.Delete(
+				&models.GitlabMrLabel{},
+				dal.Where("connection_id = ? AND mr_id = ?", data.Options.ConnectionId, mr.GitlabId),
+			)
+			if err != nil {
+				return err
+			}
+			err = db.Delete(
+				&models.GitlabAssignee{},
+				dal.Where("connection_id = ? AND merge_request_id = ?", data.Options.ConnectionId, mr.GitlabId),
+			)
+			if err != nil {
+				return err
+			}
+		}
+		return nil
+	}
+}
diff --git a/backend/plugins/gitlab/tasks/mr_label_convertor.go b/backend/plugins/gitlab/tasks/mr_label_convertor.go
index 934085e81..b5f29a1a8 100644
--- a/backend/plugins/gitlab/tasks/mr_label_convertor.go
+++ b/backend/plugins/gitlab/tasks/mr_label_convertor.go
@@ -18,14 +18,12 @@ limitations under the License.
 package tasks
 
 import (
-	"reflect"
-
 	"github.com/apache/incubator-devlake/core/dal"
 	"github.com/apache/incubator-devlake/core/errors"
 	"github.com/apache/incubator-devlake/core/models/domainlayer/code"
 	"github.com/apache/incubator-devlake/core/models/domainlayer/didgen"
 	"github.com/apache/incubator-devlake/core/plugin"
-	helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+	"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
 	"github.com/apache/incubator-devlake/plugins/gitlab/models"
 )
 
@@ -39,38 +37,33 @@ var ConvertMrLabelsMeta = plugin.SubTaskMeta{
 	EnabledByDefault: true,
 	Description:      "Convert tool layer table gitlab_mr_labels into  domain layer table pull_request_labels",
 	DomainTypes:      []string{plugin.DOMAIN_TYPE_CODE_REVIEW},
-	Dependencies:     []*plugin.SubTaskMeta{&ConvertIssueLabelsMeta},
+	Dependencies:     []*plugin.SubTaskMeta{&ConvertApiMergeRequestsMeta},
 }
 
-func ConvertMrLabels(taskCtx plugin.SubTaskContext) errors.Error {
-	db := taskCtx.GetDal()
-	rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_MERGE_REQUEST_TABLE)
-	projectId := data.Options.ProjectId
-	clauses := []dal.Clause{
-		dal.Select("*"),
-		dal.From(&models.GitlabMrLabel{}),
-		dal.Join(`left join _tool_gitlab_merge_requests on
-			_tool_gitlab_merge_requests.gitlab_id = _tool_gitlab_mr_labels.mr_id`),
-		dal.Where(`_tool_gitlab_merge_requests.project_id = ?
-			and _tool_gitlab_merge_requests.connection_id = ?`,
-			projectId, data.Options.ConnectionId),
-		dal.Orderby("mr_id ASC"),
-	}
-
-	cursor, err := db.Cursor(clauses...)
-	if err != nil {
-		return err
-	}
-	defer cursor.Close()
+func ConvertMrLabels(subtaskCtx plugin.SubTaskContext) errors.Error {
+	subtaskCommonArgs, data := CreateSubtaskCommonArgs(subtaskCtx, RAW_MERGE_REQUEST_TABLE)
+	db := subtaskCtx.GetDal()
 
 	mrIdGen := didgen.NewDomainIdGenerator(&models.GitlabMergeRequest{})
 
-	converter, err := helper.NewDataConverter(helper.DataConverterArgs{
-		RawDataSubTaskArgs: *rawDataSubTaskArgs,
-		InputRowType:       reflect.TypeOf(models.GitlabMrLabel{}),
-		Input:              cursor,
-		Convert: func(inputRow interface{}) ([]interface{}, errors.Error) {
-			mrLabel := inputRow.(*models.GitlabMrLabel)
+	converter, err := api.NewStatefulDataConverter(&api.StatefulDataConverterArgs[models.GitlabMrLabel]{
+		SubtaskCommonArgs: subtaskCommonArgs,
+		Input: func(stateManager *api.SubtaskStateManager) (dal.Rows, errors.Error) {
+			clauses := []dal.Clause{
+				dal.Select("c.*"),
+				dal.From("_tool_gitlab_mr_labels c"),
+				dal.Join(`LEFT JOIN _tool_gitlab_merge_requests mr ON mr.gitlab_id = c.mr_id AND c.connection_id = mr.connection_id`),
+				dal.Where(`mr.project_id = ?  and mr.connection_id = ?`, data.Options.ProjectId, data.Options.ConnectionId),
+			}
+			if stateManager.IsIncremental() {
+				since := stateManager.GetSince()
+				if since != nil {
+					clauses = append(clauses, dal.Where("c.updated_at >= ? ", since))
+				}
+			}
+			return db.Cursor(clauses...)
+		},
+		Convert: func(mrLabel *models.GitlabMrLabel) ([]interface{}, errors.Error) {
 			domainIssueLabel := &code.PullRequestLabel{
 				PullRequestId: mrIdGen.Generate(data.Options.ConnectionId, mrLabel.MrId),
 				LabelName:     mrLabel.LabelName,
diff --git a/backend/plugins/gitlab/tasks/mr_comment_convertor.go b/backend/plugins/gitlab/tasks/mr_note_convertor.go
similarity index 70%
rename from backend/plugins/gitlab/tasks/mr_comment_convertor.go
rename to backend/plugins/gitlab/tasks/mr_note_convertor.go
index 3b6d1ccee..a55cc5673 100644
--- a/backend/plugins/gitlab/tasks/mr_comment_convertor.go
+++ b/backend/plugins/gitlab/tasks/mr_note_convertor.go
@@ -18,15 +18,13 @@ limitations under the License.
 package tasks
 
 import (
-	"reflect"
-
 	"github.com/apache/incubator-devlake/core/dal"
 	"github.com/apache/incubator-devlake/core/errors"
 	"github.com/apache/incubator-devlake/core/models/domainlayer"
 	"github.com/apache/incubator-devlake/core/models/domainlayer/code"
 	"github.com/apache/incubator-devlake/core/models/domainlayer/didgen"
 	"github.com/apache/incubator-devlake/core/plugin"
-	helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+	"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
 	"github.com/apache/incubator-devlake/plugins/gitlab/models"
 )
 
@@ -36,44 +34,39 @@ func init() {
 
 var ConvertMrCommentMeta = plugin.SubTaskMeta{
 	Name:             "Convert MR Comments",
-	EntryPoint:       ConvertMergeRequestComment,
+	EntryPoint:       ConvertMergeRequestNote,
 	EnabledByDefault: true,
 	Description:      "Add domain layer Comment according to GitlabMrComment",
 	DomainTypes:      []string{plugin.DOMAIN_TYPE_CODE_REVIEW},
-	Dependencies:     []*plugin.SubTaskMeta{&ConvertApiMergeRequestsMeta},
+	Dependencies:     []*plugin.SubTaskMeta{&ConvertApiMergeRequestsMeta, &ExtractApiMrNotesMeta},
 }
 
-func ConvertMergeRequestComment(taskCtx plugin.SubTaskContext) errors.Error {
-	rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_MERGE_REQUEST_NOTES_TABLE)
-	db := taskCtx.GetDal()
-	clauses := []dal.Clause{
-		dal.From(&models.GitlabMrComment{}),
-		dal.Join(`left join _tool_gitlab_merge_requests on
-			_tool_gitlab_merge_requests.gitlab_id =
-			_tool_gitlab_mr_comments.merge_request_id`),
-		dal.Where(`_tool_gitlab_merge_requests.project_id = ?
-			and _tool_gitlab_mr_comments.connection_id = ?`,
-			data.Options.ProjectId, data.Options.ConnectionId),
-	}
-
-	cursor, err := db.Cursor(clauses...)
-	if err != nil {
-		return err
-	}
-	defer cursor.Close()
+func ConvertMergeRequestNote(subtaskCtx plugin.SubTaskContext) errors.Error {
+	subtaskCommonArgs, data := CreateSubtaskCommonArgs(subtaskCtx, RAW_MERGE_REQUEST_NOTES_TABLE)
+	db := subtaskCtx.GetDal()
 
 	domainIdGeneratorComment := didgen.NewDomainIdGenerator(&models.GitlabMrComment{})
 	prIdGen := didgen.NewDomainIdGenerator(&models.GitlabMergeRequest{})
 	accountIdGen := didgen.NewDomainIdGenerator(&models.GitlabAccount{})
 
-	converter, err := helper.NewDataConverter(helper.DataConverterArgs{
-		RawDataSubTaskArgs: *rawDataSubTaskArgs,
-		InputRowType:       reflect.TypeOf(models.GitlabMrComment{}),
-		Input:              cursor,
-
-		Convert: func(inputRow interface{}) ([]interface{}, errors.Error) {
-			gitlabComments := inputRow.(*models.GitlabMrComment)
-
+	converter, err := api.NewStatefulDataConverter(&api.StatefulDataConverterArgs[models.GitlabMrComment]{
+		SubtaskCommonArgs: subtaskCommonArgs,
+		Input: func(stateManager *api.SubtaskStateManager) (dal.Rows, errors.Error) {
+			clauses := []dal.Clause{
+				dal.Select("c.*"),
+				dal.From("_tool_gitlab_mr_comments c"),
+				dal.Join(`LEFT JOIN _tool_gitlab_merge_requests mr ON mr.gitlab_id = c.merge_request_id AND c.connection_id = mr.connection_id`),
+				dal.Where(`mr.project_id = ?  and mr.connection_id = ?`, data.Options.ProjectId, data.Options.ConnectionId),
+			}
+			if stateManager.IsIncremental() {
+				since := stateManager.GetSince()
+				if since != nil {
+					clauses = append(clauses, dal.Where("c.updated_at >= ? ", since))
+				}
+			}
+			return db.Cursor(clauses...)
+		},
+		Convert: func(gitlabComments *models.GitlabMrComment) ([]interface{}, errors.Error) {
 			domainComment := &code.PullRequestComment{
 				DomainEntity: domainlayer.DomainEntity{
 					Id: domainIdGeneratorComment.Generate(data.Options.ConnectionId, gitlabComments.GitlabId),
diff --git a/backend/plugins/gitlab/tasks/mr_note_extractor.go b/backend/plugins/gitlab/tasks/mr_note_extractor.go
index a27d1141c..725a2fe48 100644
--- a/backend/plugins/gitlab/tasks/mr_note_extractor.go
+++ b/backend/plugins/gitlab/tasks/mr_note_extractor.go
@@ -18,8 +18,6 @@ limitations under the License.
 package tasks
 
 import (
-	"encoding/json"
-
 	"github.com/apache/incubator-devlake/core/errors"
 	"github.com/apache/incubator-devlake/core/models/common"
 	"github.com/apache/incubator-devlake/core/plugin"
@@ -57,18 +55,12 @@ var ExtractApiMrNotesMeta = plugin.SubTaskMeta{
 	Dependencies:     []*plugin.SubTaskMeta{&CollectApiMrNotesMeta},
 }
 
-func ExtractApiMergeRequestsNotes(taskCtx plugin.SubTaskContext) errors.Error {
-	rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_MERGE_REQUEST_NOTES_TABLE)
-
-	extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{
-		RawDataSubTaskArgs: *rawDataSubTaskArgs,
-		Extract: func(row *api.RawData) ([]interface{}, errors.Error) {
-			mrNote := &MergeRequestNote{}
-			err := errors.Convert(json.Unmarshal(row.Data, mrNote))
-			if err != nil {
-				return nil, err
-			}
+func ExtractApiMergeRequestsNotes(subtaskCtx plugin.SubTaskContext) errors.Error {
+	subtaskCommonArgs, data := CreateSubtaskCommonArgs(subtaskCtx, RAW_MERGE_REQUEST_NOTES_TABLE)
 
+	extractor, err := api.NewStatefulApiExtractor(&api.StatefulApiExtractorArgs[MergeRequestNote]{
+		SubtaskCommonArgs: subtaskCommonArgs,
+		Extract: func(mrNote *MergeRequestNote, row *api.RawData) ([]interface{}, errors.Error) {
 			toolMrNote, err := convertMergeRequestNote(mrNote)
 			toolMrNote.ConnectionId = data.Options.ConnectionId
 			if err != nil {
diff --git a/backend/plugins/gitlab/tasks/mr_reviewer_convertor.go b/backend/plugins/gitlab/tasks/mr_reviewer_convertor.go
index b702de573..f9298b093 100644
--- a/backend/plugins/gitlab/tasks/mr_reviewer_convertor.go
+++ b/backend/plugins/gitlab/tasks/mr_reviewer_convertor.go
@@ -18,14 +18,12 @@ limitations under the License.
 package tasks
 
 import (
-	"reflect"
-
 	"github.com/apache/incubator-devlake/core/dal"
 	"github.com/apache/incubator-devlake/core/errors"
 	"github.com/apache/incubator-devlake/core/models/domainlayer/code"
 	"github.com/apache/incubator-devlake/core/models/domainlayer/didgen"
 	"github.com/apache/incubator-devlake/core/plugin"
-	helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+	"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
 	"github.com/apache/incubator-devlake/plugins/gitlab/models"
 )
 
@@ -39,39 +37,34 @@ var ConvertMrReviewersMeta = plugin.SubTaskMeta{
 	EnabledByDefault: true,
 	Description:      "Convert tool layer table gitlab_reviewers into domain layer table pull_request_reviewers",
 	DomainTypes:      []string{plugin.DOMAIN_TYPE_CODE_REVIEW},
-	Dependencies:     []*plugin.SubTaskMeta{&ExtractApiMergeRequestDetailsMeta},
+	Dependencies:     []*plugin.SubTaskMeta{&ConvertApiMergeRequestsMeta},
 }
 
-func ConvertMrReviewers(taskCtx plugin.SubTaskContext) errors.Error {
-	db := taskCtx.GetDal()
-	rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_MERGE_REQUEST_TABLE)
-	projectId := data.Options.ProjectId
-	clauses := []dal.Clause{
-		dal.Select("_tool_gitlab_reviewers.*"),
-		dal.From(&models.GitlabReviewer{}),
-		dal.Join(`left join _tool_gitlab_merge_requests mr on
-			mr.gitlab_id = _tool_gitlab_reviewers.merge_request_id`),
-		dal.Where(`mr.project_id = ?
-			and mr.connection_id = ?`,
-			projectId, data.Options.ConnectionId),
-		dal.Orderby("_tool_gitlab_reviewers.merge_request_id ASC"),
-	}
-
-	cursor, err := db.Cursor(clauses...)
-	if err != nil {
-		return err
-	}
-	defer cursor.Close()
+func ConvertMrReviewers(subtaskCtx plugin.SubTaskContext) errors.Error {
+	subtaskCommonArgs, data := CreateSubtaskCommonArgs(subtaskCtx, RAW_MERGE_REQUEST_TABLE)
+	db := subtaskCtx.GetDal()
 
 	mrIdGen := didgen.NewDomainIdGenerator(&models.GitlabMergeRequest{})
 	accountIdGen := didgen.NewDomainIdGenerator(&models.GitlabAccount{})
 
-	converter, err := helper.NewDataConverter(helper.DataConverterArgs{
-		RawDataSubTaskArgs: *rawDataSubTaskArgs,
-		InputRowType:       reflect.TypeOf(models.GitlabReviewer{}),
-		Input:              cursor,
-		Convert: func(inputRow interface{}) ([]interface{}, errors.Error) {
-			mrReviewer := inputRow.(*models.GitlabReviewer)
+	converter, err := api.NewStatefulDataConverter(&api.StatefulDataConverterArgs[models.GitlabReviewer]{
+		SubtaskCommonArgs: subtaskCommonArgs,
+		Input: func(stateManager *api.SubtaskStateManager) (dal.Rows, errors.Error) {
+			clauses := []dal.Clause{
+				dal.Select("c.*"),
+				dal.From("_tool_gitlab_reviewers c"),
+				dal.Join(`LEFT JOIN _tool_gitlab_merge_requests mr ON mr.gitlab_id = c.merge_request_id AND c.connection_id = mr.connection_id`),
+				dal.Where(`mr.project_id = ?  and mr.connection_id = ?`, data.Options.ProjectId, data.Options.ConnectionId),
+			}
+			if stateManager.IsIncremental() {
+				since := stateManager.GetSince()
+				if since != nil {
+					clauses = append(clauses, dal.Where("c.updated_at >= ? ", since))
+				}
+			}
+			return db.Cursor(clauses...)
+		},
+		Convert: func(mrReviewer *models.GitlabReviewer) ([]interface{}, errors.Error) {
 			domainPrReviewer := &code.PullRequestReviewer{
 				PullRequestId: mrIdGen.Generate(data.Options.ConnectionId, mrReviewer.MergeRequestId),
 				ReviewerId:    accountIdGen.Generate(data.Options.ConnectionId, mrReviewer.ReviewerId),
diff --git a/backend/plugins/gitlab/tasks/pipeline_commit_convertor.go b/backend/plugins/gitlab/tasks/pipeline_commit_convertor.go
index 4984270d9..2064cc6c9 100644
--- a/backend/plugins/gitlab/tasks/pipeline_commit_convertor.go
+++ b/backend/plugins/gitlab/tasks/pipeline_commit_convertor.go
@@ -18,14 +18,12 @@ limitations under the License.
 package tasks
 
 import (
-	"reflect"
-
 	"github.com/apache/incubator-devlake/core/dal"
 	"github.com/apache/incubator-devlake/core/errors"
 	"github.com/apache/incubator-devlake/core/models/domainlayer/devops"
 	"github.com/apache/incubator-devlake/core/models/domainlayer/didgen"
 	"github.com/apache/incubator-devlake/core/plugin"
-	helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+	"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
 	"github.com/apache/incubator-devlake/plugins/gitlab/models"
 )
 
@@ -42,9 +40,9 @@ var ConvertPipelineCommitMeta = plugin.SubTaskMeta{
 	Dependencies:     []*plugin.SubTaskMeta{&ConvertDetailPipelineMeta},
 }
 
-func ConvertPipelineCommits(taskCtx plugin.SubTaskContext) errors.Error {
-	db := taskCtx.GetDal()
-	data := taskCtx.GetData().(*GitlabTaskData)
+func ConvertPipelineCommits(subtaskCtx plugin.SubTaskContext) errors.Error {
+	subtaskCommonArgs, data := CreateSubtaskCommonArgs(subtaskCtx, RAW_PIPELINE_TABLE)
+	db := subtaskCtx.GetDal()
 
 	repo := &models.GitlabProject{}
 	err := db.First(repo, dal.Where("gitlab_id = ? and connection_id = ?", data.Options.ProjectId, data.Options.ConnectionId))
@@ -52,29 +50,24 @@ func ConvertPipelineCommits(taskCtx plugin.SubTaskContext) errors.Error {
 		return err
 	}
 
-	cursor, err := db.Cursor(dal.From(models.GitlabPipelineProject{}),
-		dal.Where("project_id = ? and connection_id = ?", data.Options.ProjectId, data.Options.ConnectionId))
-	if err != nil {
-		return err
-	}
-	defer cursor.Close()
-
 	pipelineIdGen := didgen.NewDomainIdGenerator(&models.GitlabPipeline{})
 
-	converter, err := helper.NewDataConverter(helper.DataConverterArgs{
-		InputRowType: reflect.TypeOf(models.GitlabPipelineProject{}),
-		Input:        cursor,
-		RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
-			Ctx: taskCtx,
-			Params: models.GitlabApiParams{
-				ConnectionId: data.Options.ConnectionId,
-				ProjectId:    data.Options.ProjectId,
-			},
-			Table: RAW_PIPELINE_TABLE,
+	converter, err := api.NewStatefulDataConverter(&api.StatefulDataConverterArgs[models.GitlabPipelineProject]{
+		SubtaskCommonArgs: subtaskCommonArgs,
+		Input: func(stateManager *api.SubtaskStateManager) (dal.Rows, errors.Error) {
+			clauses := []dal.Clause{
+				dal.From(models.GitlabPipelineProject{}),
+				dal.Where("project_id = ? and connection_id = ?", data.Options.ProjectId, data.Options.ConnectionId),
+			}
+			if stateManager.IsIncremental() {
+				since := stateManager.GetSince()
+				if since != nil {
+					clauses = append(clauses, dal.Where("updated_at >= ? ", since))
+				}
+			}
+			return db.Cursor(clauses...)
 		},
-		Convert: func(inputRow interface{}) ([]interface{}, errors.Error) {
-			gitlabPipelineCommit := inputRow.(*models.GitlabPipelineProject)
-
+		Convert: func(gitlabPipelineCommit *models.GitlabPipelineProject) ([]interface{}, errors.Error) {
 			domainPipelineCommit := &devops.CiCDPipelineCommit{
 				PipelineId: pipelineIdGen.Generate(data.Options.ConnectionId, gitlabPipelineCommit.PipelineId),
 				CommitSha:  gitlabPipelineCommit.Sha,
diff --git a/backend/plugins/gitlab/tasks/pipeline_detail_convertor.go b/backend/plugins/gitlab/tasks/pipeline_detail_convertor.go
index aba33b0e4..e808d7822 100644
--- a/backend/plugins/gitlab/tasks/pipeline_detail_convertor.go
+++ b/backend/plugins/gitlab/tasks/pipeline_detail_convertor.go
@@ -18,17 +18,16 @@ limitations under the License.
 package tasks
 
 import (
-	"reflect"
 	"time"
 
 	"github.com/apache/incubator-devlake/core/errors"
+	"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
 
 	"github.com/apache/incubator-devlake/core/dal"
 	"github.com/apache/incubator-devlake/core/models/domainlayer"
 	"github.com/apache/incubator-devlake/core/models/domainlayer/devops"
 	"github.com/apache/incubator-devlake/core/models/domainlayer/didgen"
 	"github.com/apache/incubator-devlake/core/plugin"
-	helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
 	"github.com/apache/incubator-devlake/plugins/gitlab/models"
 )
 
@@ -42,37 +41,33 @@ var ConvertDetailPipelineMeta = plugin.SubTaskMeta{
 	EnabledByDefault: true,
 	Description:      "Convert tool layer table gitlab_detail_pipeline into domain layer table pipeline",
 	DomainTypes:      []string{plugin.DOMAIN_TYPE_CICD},
-	Dependencies:     []*plugin.SubTaskMeta{&ConvertCommitsMeta},
+	Dependencies:     []*plugin.SubTaskMeta{&ExtractApiPipelineDetailsMeta},
 }
 
-func ConvertDetailPipelines(taskCtx plugin.SubTaskContext) errors.Error {
-	db := taskCtx.GetDal()
-	data := taskCtx.GetData().(*GitlabTaskData)
+func ConvertDetailPipelines(subtaskCtx plugin.SubTaskContext) errors.Error {
+	subtaskCommonArgs, data := CreateSubtaskCommonArgs(subtaskCtx, RAW_PIPELINE_DETAILS_TABLE)
 
-	cursor, err := db.Cursor(dal.From(models.GitlabPipeline{}),
-		dal.Where("project_id = ? and connection_id = ?", data.Options.ProjectId, data.Options.ConnectionId))
-	if err != nil {
-		return err
-	}
-	defer cursor.Close()
+	db := subtaskCtx.GetDal()
 
 	pipelineIdGen := didgen.NewDomainIdGenerator(&models.GitlabPipeline{})
 	projectIdGen := didgen.NewDomainIdGenerator(&models.GitlabProject{})
 
-	converter, err := helper.NewDataConverter(helper.DataConverterArgs{
-		InputRowType: reflect.TypeOf(models.GitlabPipeline{}),
-		Input:        cursor,
-		RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
-			Ctx: taskCtx,
-			Params: models.GitlabApiParams{
-				ConnectionId: data.Options.ConnectionId,
-				ProjectId:    data.Options.ProjectId,
-			},
-			Table: RAW_PIPELINE_DETAILS_TABLE,
+	converter, err := api.NewStatefulDataConverter(&api.StatefulDataConverterArgs[models.GitlabPipeline]{
+		SubtaskCommonArgs: subtaskCommonArgs,
+		Input: func(stateManager *api.SubtaskStateManager) (dal.Rows, errors.Error) {
+			clauses := []dal.Clause{
+				dal.From(models.GitlabPipeline{}),
+				dal.Where("project_id = ? and connection_id = ?", data.Options.ProjectId, data.Options.ConnectionId),
+			}
+			if stateManager.IsIncremental() {
+				since := stateManager.GetSince()
+				if since != nil {
+					clauses = append(clauses, dal.Where("updated_at >= ? ", since))
+				}
+			}
+			return db.Cursor(clauses...)
 		},
-		Convert: func(inputRow interface{}) ([]interface{}, errors.Error) {
-			gitlabPipeline := inputRow.(*models.GitlabPipeline)
-
+		Convert: func(gitlabPipeline *models.GitlabPipeline) ([]interface{}, errors.Error) {
 			createdAt := time.Now()
 			if gitlabPipeline.GitlabCreatedAt != nil {
 				createdAt = *gitlabPipeline.GitlabCreatedAt
diff --git a/backend/plugins/gitlab/tasks/pipeline_detail_extractor.go b/backend/plugins/gitlab/tasks/pipeline_detail_extractor.go
index 1b78362a1..3fef05109 100644
--- a/backend/plugins/gitlab/tasks/pipeline_detail_extractor.go
+++ b/backend/plugins/gitlab/tasks/pipeline_detail_extractor.go
@@ -18,8 +18,6 @@ limitations under the License.
 package tasks
 
 import (
-	"encoding/json"
-
 	"github.com/apache/incubator-devlake/core/errors"
 	"github.com/apache/incubator-devlake/core/models/common"
 	"github.com/apache/incubator-devlake/core/models/domainlayer/devops"
@@ -41,18 +39,12 @@ var ExtractApiPipelineDetailsMeta = plugin.SubTaskMeta{
 	Dependencies:     []*plugin.SubTaskMeta{&CollectApiPipelineDetailsMeta},
 }
 
-func ExtractApiPipelineDetails(taskCtx plugin.SubTaskContext) errors.Error {
-	rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_PIPELINE_DETAILS_TABLE)
-
-	extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{
-		RawDataSubTaskArgs: *rawDataSubTaskArgs,
-		Extract: func(row *api.RawData) ([]interface{}, errors.Error) {
-			gitlabApiPipeline := &ApiPipeline{}
-			err := errors.Convert(json.Unmarshal(row.Data, gitlabApiPipeline))
-			if err != nil {
-				return nil, err
-			}
+func ExtractApiPipelineDetails(subtaskCtx plugin.SubTaskContext) errors.Error {
+	subtaskCommonArgs, data := CreateSubtaskCommonArgs(subtaskCtx, RAW_PIPELINE_DETAILS_TABLE)
 
+	extractor, err := api.NewStatefulApiExtractor(&api.StatefulApiExtractorArgs[ApiPipeline]{
+		SubtaskCommonArgs: subtaskCommonArgs,
+		Extract: func(gitlabApiPipeline *ApiPipeline, row *api.RawData) ([]interface{}, errors.Error) {
 			gitlabPipeline := &models.GitlabPipeline{
 				GitlabId:        gitlabApiPipeline.Id,
 				ProjectId:       data.Options.ProjectId,
diff --git a/backend/plugins/gitlab/tasks/pipeline_extractor.go b/backend/plugins/gitlab/tasks/pipeline_extractor.go
index d68035db1..0eb2450a5 100644
--- a/backend/plugins/gitlab/tasks/pipeline_extractor.go
+++ b/backend/plugins/gitlab/tasks/pipeline_extractor.go
@@ -18,8 +18,6 @@ limitations under the License.
 package tasks
 
 import (
-	"encoding/json"
-
 	"github.com/apache/incubator-devlake/core/errors"
 	"github.com/apache/incubator-devlake/core/models/common"
 	"github.com/apache/incubator-devlake/core/plugin"
@@ -69,17 +67,12 @@ var ExtractApiPipelinesMeta = plugin.SubTaskMeta{
 	Dependencies:     []*plugin.SubTaskMeta{&CollectApiPipelinesMeta},
 }
 
-func ExtractApiPipelines(taskCtx plugin.SubTaskContext) errors.Error {
-	rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_PIPELINE_TABLE)
+func ExtractApiPipelines(subtaskCtx plugin.SubTaskContext) errors.Error {
+	subtaskCommonArgs, data := CreateSubtaskCommonArgs(subtaskCtx, RAW_PIPELINE_TABLE)
 
-	extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{
-		RawDataSubTaskArgs: *rawDataSubTaskArgs,
-		Extract: func(row *api.RawData) ([]interface{}, errors.Error) {
-			gitlabApiPipeline := &ApiPipeline{}
-			err := errors.Convert(json.Unmarshal(row.Data, gitlabApiPipeline))
-			if err != nil {
-				return nil, err
-			}
+	extractor, err := api.NewStatefulApiExtractor(&api.StatefulApiExtractorArgs[ApiPipeline]{
+		SubtaskCommonArgs: subtaskCommonArgs,
+		Extract: func(gitlabApiPipeline *ApiPipeline, row *api.RawData) ([]interface{}, errors.Error) {
 			pipelineProject := convertApiPipelineToGitlabPipelineProject(gitlabApiPipeline, data.Options.ConnectionId, data.Options.ProjectId)
 			return []interface{}{pipelineProject}, nil
 		},
diff --git a/backend/plugins/gitlab/tasks/shared.go b/backend/plugins/gitlab/tasks/shared.go
index 4051c93b5..59ce8b921 100644
--- a/backend/plugins/gitlab/tasks/shared.go
+++ b/backend/plugins/gitlab/tasks/shared.go
@@ -155,10 +155,10 @@ func GetQuery(reqData *helper.RequestData) (url.Values, errors.Error) {
 	return query, nil
 }
 
-func CreateRawDataSubTaskArgs(taskCtx plugin.SubTaskContext, Table string) (*helper.RawDataSubTaskArgs, *GitlabTaskData) {
-	data := taskCtx.GetData().(*GitlabTaskData)
+func CreateRawDataSubTaskArgs(subtaskCtx plugin.SubTaskContext, Table string) (*helper.RawDataSubTaskArgs, *GitlabTaskData) {
+	data := subtaskCtx.GetData().(*GitlabTaskData)
 	rawDataSubTaskArgs := &helper.RawDataSubTaskArgs{
-		Ctx: taskCtx,
+		Ctx: subtaskCtx,
 		Params: models.GitlabApiParams{
 			ProjectId:    data.Options.ProjectId,
 			ConnectionId: data.Options.ConnectionId,
@@ -168,15 +168,29 @@ func CreateRawDataSubTaskArgs(taskCtx plugin.SubTaskContext, Table string) (*hel
 	return rawDataSubTaskArgs, data
 }
 
+func CreateSubtaskCommonArgs(subtaskCtx plugin.SubTaskContext, table string) (*helper.SubtaskCommonArgs, *GitlabTaskData) {
+	data := subtaskCtx.GetData().(*GitlabTaskData)
+	args := &helper.SubtaskCommonArgs{
+		SubTaskContext: subtaskCtx,
+		Table:          table,
+		Params: models.GitlabApiParams{
+			ConnectionId: data.Options.ConnectionId,
+			ProjectId:    data.Options.ProjectId,
+		},
+	}
+	return args, data
+}
+
 func GetMergeRequestsIterator(taskCtx plugin.SubTaskContext, apiCollector *helper.StatefulApiCollector) (*helper.DalCursorIterator, errors.Error) {
 	db := taskCtx.GetDal()
 	data := taskCtx.GetData().(*GitlabTaskData)
 	clauses := []dal.Clause{
 		dal.Select("gmr.gitlab_id, gmr.iid"),
 		dal.From("_tool_gitlab_merge_requests gmr"),
+		// collect only openning merge request's notes and commits to speed up the process
 		dal.Where(
-			`gmr.project_id = ? and gmr.connection_id = ?`,
-			data.Options.ProjectId, data.Options.ConnectionId,
+			`gmr.project_id = ? and gmr.connection_id = ? AND state = ?`,
+			data.Options.ProjectId, data.Options.ConnectionId, "opened",
 		),
 	}
 	if apiCollector != nil {
diff --git a/backend/plugins/gitlab/tasks/trigger_job_extractor.go b/backend/plugins/gitlab/tasks/trigger_job_extractor.go
index f515b7b85..8a1e7eaeb 100644
--- a/backend/plugins/gitlab/tasks/trigger_job_extractor.go
+++ b/backend/plugins/gitlab/tasks/trigger_job_extractor.go
@@ -18,7 +18,6 @@ limitations under the License.
 package tasks
 
 import (
-	"encoding/json"
 	"time"
 
 	"github.com/apache/incubator-devlake/core/errors"
@@ -55,18 +54,12 @@ var ExtractApiTriggerJobsMeta = plugin.SubTaskMeta{
 	Dependencies:     []*plugin.SubTaskMeta{&CollectApiTriggerJobsMeta},
 }
 
-func ExtractApiTriggerJobs(taskCtx plugin.SubTaskContext) errors.Error {
-	rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_TRIGGER_JOB_TABLE)
-
-	extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{
-		RawDataSubTaskArgs: *rawDataSubTaskArgs,
-		Extract: func(row *api.RawData) ([]interface{}, errors.Error) {
-			gitlabApiTriggerJob := &ApiTriggerJob{}
-			err := errors.Convert(json.Unmarshal(row.Data, gitlabApiTriggerJob))
-			if err != nil {
-				return nil, err
-			}
+func ExtractApiTriggerJobs(subtaskCtx plugin.SubTaskContext) errors.Error {
+	subtaskCommonArgs, data := CreateSubtaskCommonArgs(subtaskCtx, RAW_TRIGGER_JOB_TABLE)
 
+	extractor, err := api.NewStatefulApiExtractor(&api.StatefulApiExtractorArgs[ApiTriggerJob]{
+		SubtaskCommonArgs: subtaskCommonArgs,
+		Extract: func(gitlabApiTriggerJob *ApiTriggerJob, row *api.RawData) ([]interface{}, errors.Error) {
 			gitlabPipeline, err := convertTriggerJob(gitlabApiTriggerJob, data.Options.ProjectId)
 			if err != nil {
 				return nil, err
diff --git a/backend/plugins/jira/tasks/epic_extractor.go b/backend/plugins/jira/tasks/epic_extractor.go
index cf1e7f28b..89f2ef70c 100644
--- a/backend/plugins/jira/tasks/epic_extractor.go
+++ b/backend/plugins/jira/tasks/epic_extractor.go
@@ -18,12 +18,15 @@ limitations under the License.
 package tasks
 
 import (
+	"encoding/json"
+
 	"github.com/apache/incubator-devlake/core/dal"
 	"github.com/apache/incubator-devlake/core/errors"
 	"github.com/apache/incubator-devlake/core/log"
 	"github.com/apache/incubator-devlake/core/plugin"
 	"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
 	"github.com/apache/incubator-devlake/plugins/jira/models"
+	"github.com/apache/incubator-devlake/plugins/jira/tasks/apiv2models"
 )
 
 var _ plugin.SubTaskEntryPoint = ExtractEpics
@@ -61,7 +64,12 @@ func ExtractEpics(taskCtx plugin.SubTaskContext) errors.Error {
 			Table: RAW_EPIC_TABLE,
 		},
 		Extract: func(row *api.RawData) ([]interface{}, errors.Error) {
-			return extractIssues(data, mappings, row, userFieldMap)
+			apiIssue := &apiv2models.Issue{}
+			err = errors.Convert(json.Unmarshal(row.Data, apiIssue))
+			if err != nil {
+				return nil, err
+			}
+			return extractIssues(data, mappings, apiIssue, row, userFieldMap)
 		},
 	})
 	if err != nil {
diff --git a/backend/plugins/jira/tasks/issue_changelog_convertor.go b/backend/plugins/jira/tasks/issue_changelog_convertor.go
index 4dc35e5c9..b27fc9666 100644
--- a/backend/plugins/jira/tasks/issue_changelog_convertor.go
+++ b/backend/plugins/jira/tasks/issue_changelog_convertor.go
@@ -78,7 +78,7 @@ func ConvertIssueChangelogs(subtaskCtx plugin.SubTaskContext) errors.Error {
 	changelogIdGenerator := didgen.NewDomainIdGenerator(&models.JiraIssueChangelogItems{})
 	accountIdGen := didgen.NewDomainIdGenerator(&models.JiraAccount{})
 
-	converter, err := api.NewStatefulDataConverter[IssueChangelogItemResult](&api.StatefulDataConverterArgs[IssueChangelogItemResult]{
+	converter, err := api.NewStatefulDataConverter(&api.StatefulDataConverterArgs[IssueChangelogItemResult]{
 		SubtaskCommonArgs: &api.SubtaskCommonArgs{
 			SubTaskContext: subtaskCtx,
 			Table:          RAW_ISSUE_TABLE,
diff --git a/backend/plugins/jira/tasks/issue_changelog_extractor.go b/backend/plugins/jira/tasks/issue_changelog_extractor.go
index a49142e3f..b5de33a45 100644
--- a/backend/plugins/jira/tasks/issue_changelog_extractor.go
+++ b/backend/plugins/jira/tasks/issue_changelog_extractor.go
@@ -47,7 +47,7 @@ func ExtractIssueChangelogs(subtaskCtx plugin.SubTaskContext) errors.Error {
 	if err != nil {
 		return err
 	}
-	extractor, err := api.NewStatefulApiExtractor(&api.StatefulApiExtractorArgs{
+	extractor, err := api.NewStatefulApiExtractor(&api.StatefulApiExtractorArgs[apiv2models.Changelog]{
 		SubtaskCommonArgs: &api.SubtaskCommonArgs{
 			SubTaskContext: subtaskCtx,
 			Table:          RAW_CHANGELOG_TABLE,
@@ -56,18 +56,13 @@ func ExtractIssueChangelogs(subtaskCtx plugin.SubTaskContext) errors.Error {
 				BoardId:      data.Options.BoardId,
 			},
 		},
-		Extract: func(row *api.RawData) ([]interface{}, errors.Error) {
+		Extract: func(changelog *apiv2models.Changelog, row *api.RawData) ([]interface{}, errors.Error) {
 			// process input
 			var input apiv2models.Input
 			err := errors.Convert(json.Unmarshal(row.Input, &input))
 			if err != nil {
 				return nil, err
 			}
-			var changelog apiv2models.Changelog
-			err = errors.Convert(json.Unmarshal(row.Data, &changelog))
-			if err != nil {
-				return nil, err
-			}
 			// prepare output
 			var result []interface{}
 			cl, user := changelog.ToToolLayer(connectionId, input.IssueId, &input.UpdateTime)
diff --git a/backend/plugins/jira/tasks/issue_convertor.go b/backend/plugins/jira/tasks/issue_convertor.go
index 1c7305758..fb213a638 100644
--- a/backend/plugins/jira/tasks/issue_convertor.go
+++ b/backend/plugins/jira/tasks/issue_convertor.go
@@ -53,7 +53,7 @@ func ConvertIssues(subtaskCtx plugin.SubTaskContext) errors.Error {
 	boardIdGen := didgen.NewDomainIdGenerator(&models.JiraBoard{})
 	boardId := boardIdGen.Generate(data.Options.ConnectionId, data.Options.BoardId)
 
-	converter, err := api.NewStatefulDataConverter[models.JiraIssue](&api.StatefulDataConverterArgs[models.JiraIssue]{
+	converter, err := api.NewStatefulDataConverter(&api.StatefulDataConverterArgs[models.JiraIssue]{
 		SubtaskCommonArgs: &api.SubtaskCommonArgs{
 			SubTaskContext: subtaskCtx,
 			Table:          RAW_ISSUE_TABLE,
@@ -84,6 +84,17 @@ func ConvertIssues(subtaskCtx plugin.SubTaskContext) errors.Error {
 			}
 			return db.Cursor(clauses...)
 		},
+		// not needed for now due to jira assignee and label are converted in FullSync(Delete+Insert) manner
+		// BeforeConvert: func(jiraIssue *models.JiraIssue, stateManager *api.SubtaskStateManager) errors.Error {
+		// 	issueId := issueIdGen.Generate(data.Options.ConnectionId, jiraIssue.IssueId)
+		// 	if err := db.Delete(&ticket.IssueAssignee{}, dal.Where("issue_id = ?", issueId)); err != nil {
+		// 		return err
+		// 	}
+		// 	if err := db.Delete(&ticket.IssueLabel{}, dal.Where("issue_id = ?", issueId)); err != nil {
+		// 		return err
+		// 	}
+		// 	return nil
+		// },
 		Convert: func(jiraIssue *models.JiraIssue) ([]interface{}, errors.Error) {
 			var result []interface{}
 			issue := &ticket.Issue{
diff --git a/backend/plugins/jira/tasks/issue_extractor.go b/backend/plugins/jira/tasks/issue_extractor.go
index 686f23ff4..11277a0a4 100644
--- a/backend/plugins/jira/tasks/issue_extractor.go
+++ b/backend/plugins/jira/tasks/issue_extractor.go
@@ -18,7 +18,6 @@ limitations under the License.
 package tasks
 
 import (
-	"encoding/json"
 	"strconv"
 	"strings"
 	"time"
@@ -62,7 +61,7 @@ func ExtractIssues(subtaskCtx plugin.SubTaskContext) errors.Error {
 	if err != nil {
 		return err
 	}
-	extractor, err := api.NewStatefulApiExtractor(&api.StatefulApiExtractorArgs{
+	extractor, err := api.NewStatefulApiExtractor(&api.StatefulApiExtractorArgs[apiv2models.Issue]{
 		SubtaskCommonArgs: &api.SubtaskCommonArgs{
 			SubTaskContext: subtaskCtx,
 			Table:          RAW_ISSUE_TABLE,
@@ -75,8 +74,27 @@ func ExtractIssues(subtaskCtx plugin.SubTaskContext) errors.Error {
 				"storyPointField": data.Options.ScopeConfig.StoryPointField,
 			},
 		},
-		Extract: func(row *api.RawData) ([]interface{}, errors.Error) {
-			return extractIssues(data, mappings, row, userFieldMap)
+		BeforeExtract: func(apiIssue *apiv2models.Issue, stateManager *api.SubtaskStateManager) errors.Error {
+			if stateManager.IsIncremental() {
+				err := db.Delete(
+					&models.JiraIssueLabel{},
+					dal.Where("connection_id = ? AND issue_id = ?", data.Options.ConnectionId, apiIssue.ID),
+				)
+				if err != nil {
+					return err
+				}
+				err = db.Delete(
+					&models.JiraIssueRelationship{},
+					dal.Where("connection_id = ? AND issue_id = ?", data.Options.ConnectionId, apiIssue.ID),
+				)
+				if err != nil {
+					return err
+				}
+			}
+			return nil
+		},
+		Extract: func(apiIssue *apiv2models.Issue, row *api.RawData) ([]interface{}, errors.Error) {
+			return extractIssues(data, mappings, apiIssue, row, userFieldMap)
 		},
 	})
 	if err != nil {
@@ -85,13 +103,8 @@ func ExtractIssues(subtaskCtx plugin.SubTaskContext) errors.Error {
 	return extractor.Execute()
 }
 
-func extractIssues(data *JiraTaskData, mappings *typeMappings, row *api.RawData, userFieldMaps map[string]struct{}) ([]interface{}, errors.Error) {
-	var apiIssue apiv2models.Issue
-	err := errors.Convert(json.Unmarshal(row.Data, &apiIssue))
-	if err != nil {
-		return nil, err
-	}
-	err = apiIssue.SetAllFields(row.Data)
+func extractIssues(data *JiraTaskData, mappings *typeMappings, apiIssue *apiv2models.Issue, row *api.RawData, userFieldMaps map[string]struct{}) ([]interface{}, errors.Error) {
+	err := apiIssue.SetAllFields(row.Data)
 	if err != nil {
 		return nil, err
 	}
diff --git a/backend/server/api/blueprints/blueprints.go b/backend/server/api/blueprints/blueprints.go
index 23383d468..841027424 100644
--- a/backend/server/api/blueprints/blueprints.go
+++ b/backend/server/api/blueprints/blueprints.go
@@ -166,6 +166,7 @@ func Patch(c *gin.Context) {
 // @Accept application/json
 // @Param blueprintId path string true "blueprintId"
 // @Param skipCollectors body models.TriggerSyncPolicy false "json"
+// @Param fullSync body models.TriggerSyncPolicy false "json"
 // @Success 200 {object} models.Pipeline
 // @Failure 400 {object} shared.ApiBody "Bad Request"
 // @Failure 500 {object} shared.ApiBody "Internal Error"
diff --git a/config-ui/src/routes/blueprint/detail/status-panel.tsx b/config-ui/src/routes/blueprint/detail/status-panel.tsx
index 50be81fe3..6e19f006f 100644
--- a/config-ui/src/routes/blueprint/detail/status-panel.tsx
+++ b/config-ui/src/routes/blueprint/detail/status-panel.tsx
@@ -119,7 +119,7 @@ export const StatusPanel = ({ from, blueprint, pipelineId, onRefresh }: Props) =
                 type="primary"
                 disabled={!blueprint.enable}
                 loading={operating}
-                onClick={() => handleRun({ skipCollectors: true })}
+                onClick={() => handleRun({ skipCollectors: true, fullSync: true })}
               >
                 Re-transform Data
               </Button>