You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by kl...@apache.org on 2023/12/07 13:03:06 UTC

(incubator-devlake) branch main updated: fix: option time after for plugins (#6585)

This is an automated email from the ASF dual-hosted git repository.

klesh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git


The following commit(s) were added to refs/heads/main by this push:
     new 1f103140d fix: option time after for plugins (#6585)
1f103140d is described below

commit 1f103140def8845861d1f7e3b363f47581e6074d
Author: abeizn <zi...@merico.dev>
AuthorDate: Thu Dec 7 21:03:01 2023 +0800

    fix: option time after for plugins (#6585)
    
    * fix: option time after for plugins
    
    * fix: lint and add omitempty tag
    
    * fix: e2e
    
    * fix: e2e
    
    * fix: zentao e2e
    
    * fix: add mapstructure:squash
---
 .../pluginhelper/api/api_collector_with_state.go   | 34 ++++++++++++++++++++--
 backend/plugins/ae/tasks/task_data.go              |  5 ++--
 backend/plugins/bamboo/models/task.go              | 13 +++++----
 .../plugins/bitbucket/api/blueprint_V200_test.go   |  1 +
 backend/plugins/bitbucket/tasks/task_data.go       |  1 +
 backend/plugins/circleci/tasks/task_data.go        | 11 +++----
 backend/plugins/customize/tasks/task_data.go       |  5 +++-
 backend/plugins/feishu/tasks/task_data.go          |  5 ++--
 backend/plugins/gitee/tasks/task_data.go           |  9 +++---
 backend/plugins/github/tasks/task_data.go          | 15 +++++-----
 backend/plugins/gitlab/tasks/task_data.go          |  9 +++---
 backend/plugins/icla/tasks/task_data.go            |  3 +-
 backend/plugins/jenkins/tasks/task_data.go         | 17 ++++++-----
 backend/plugins/jira/tasks/task_data.go            | 11 +++----
 backend/plugins/opsgenie/tasks/task_data.go        |  9 +++---
 backend/plugins/pagerduty/tasks/task_data.go       |  9 +++---
 backend/plugins/slack/tasks/task_data.go           |  3 +-
 backend/plugins/sonarqube/tasks/task_data.go       |  7 +++--
 backend/plugins/tapd/tasks/task_data.go            | 13 +++++----
 backend/plugins/teambition/tasks/task_data.go      | 11 +++----
 backend/plugins/trello/tasks/task_data.go          |  9 +++---
 backend/plugins/zentao/api/blueprint_V200_test.go  |  1 +
 backend/plugins/zentao/tasks/task_data.go          |  5 ++--
 23 files changed, 131 insertions(+), 75 deletions(-)

diff --git a/backend/helpers/pluginhelper/api/api_collector_with_state.go b/backend/helpers/pluginhelper/api/api_collector_with_state.go
index 7eb54ac14..51c9d7ec3 100644
--- a/backend/helpers/pluginhelper/api/api_collector_with_state.go
+++ b/backend/helpers/pluginhelper/api/api_collector_with_state.go
@@ -21,6 +21,7 @@ import (
 	"encoding/json"
 	"net/http"
 	"net/url"
+	"reflect"
 	"time"
 
 	"github.com/apache/incubator-devlake/core/dal"
@@ -41,15 +42,45 @@ type ApiCollectorStateManager struct {
 	Before        *time.Time
 }
 
+type CollectorOptions struct {
+	TimeAfter string `json:"timeAfter,omitempty"`
+}
+
 // NewStatefulApiCollector create a new ApiCollectorStateManager
 func NewStatefulApiCollector(args RawDataSubTaskArgs) (*ApiCollectorStateManager, errors.Error) {
 	db := args.Ctx.GetDal()
-
+	syncPolicy := args.Ctx.TaskContext().SyncPolicy()
 	rawDataSubTask, err := NewRawDataSubTask(args)
 	if err != nil {
 		return nil, errors.Default.Wrap(err, "Couldn't resolve raw subtask args")
 	}
 
+	// get optionTimeAfter from options
+	data := args.Ctx.GetData()
+	value := reflect.ValueOf(data)
+	if value.Kind() == reflect.Ptr && value.Elem().Kind() == reflect.Struct {
+		options := value.Elem().FieldByName("Options")
+		if options.IsValid() && options.Kind() == reflect.Ptr && options.Elem().Kind() == reflect.Struct {
+			collectorOptions := options.Elem().FieldByName("CollectorOptions")
+			if collectorOptions.IsValid() && collectorOptions.Kind() == reflect.Struct {
+				timeAfter := collectorOptions.FieldByName("TimeAfter")
+				if timeAfter.IsValid() && timeAfter.Kind() == reflect.String && timeAfter.String() != "" {
+					optionTimeAfter, parseErr := time.Parse(time.RFC3339, timeAfter.String())
+					if parseErr != nil {
+						return nil, errors.Default.Wrap(parseErr, "Failed to parse timeAfter!")
+					}
+					if syncPolicy != nil {
+						syncPolicy.TimeAfter = &optionTimeAfter
+					} else {
+						syncPolicy = &models.SyncPolicy{
+							TimeAfter: &optionTimeAfter,
+						}
+					}
+				}
+			}
+		}
+	}
+
 	// CollectorLatestState retrieves the latest collector state from the database
 	oldState := models.CollectorLatestState{}
 	err = db.First(&oldState, dal.Where(`raw_data_table = ? AND raw_data_params = ?`, rawDataSubTask.table, rawDataSubTask.params))
@@ -68,7 +99,6 @@ func NewStatefulApiCollector(args RawDataSubTaskArgs) (*ApiCollectorStateManager
 	oldLatestSuccessStart := oldState.LatestSuccessStart
 
 	// Calculate incremental and since based on syncPolicy and old state
-	syncPolicy := args.Ctx.TaskContext().SyncPolicy()
 	var isIncremental bool
 	var since *time.Time
 
diff --git a/backend/plugins/ae/tasks/task_data.go b/backend/plugins/ae/tasks/task_data.go
index 3a933536c..e8e93d9fc 100644
--- a/backend/plugins/ae/tasks/task_data.go
+++ b/backend/plugins/ae/tasks/task_data.go
@@ -22,8 +22,9 @@ import (
 )
 
 type AeOptions struct {
-	ConnectionId uint64 `json:"connectionId"`
-	ProjectId    int
+	ConnectionId            uint64 `json:"connectionId"`
+	ProjectId               int
+	helper.CollectorOptions `mapstructure:",squash"`
 }
 
 type AeTaskData struct {
diff --git a/backend/plugins/bamboo/models/task.go b/backend/plugins/bamboo/models/task.go
index a3578cf80..ff2baf62f 100644
--- a/backend/plugins/bamboo/models/task.go
+++ b/backend/plugins/bamboo/models/task.go
@@ -17,6 +17,8 @@ limitations under the License.
 
 package models
 
+import "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+
 type BambooApiParams struct {
 	ConnectionId uint64 `json:"connectionId"`
 	PlanKey      string
@@ -27,9 +29,10 @@ type BambooOptions struct {
 	// options means some custom params required by plugin running.
 	// Such As How many rows do your want
 	// You can use it in sub tasks and you need pass it in main.go and pipelines.
-	ConnectionId       uint64   `json:"connectionId"`
-	PlanKey            string   `json:"planKey"`
-	Tasks              []string `json:"tasks,omitempty"`
-	ScopeConfigId      uint64   `mapstructure:"scopeConfigId" json:"scopeConfigId"`
-	*BambooScopeConfig `mapstructure:"scopeConfig" json:"scopeConfig"`
+	ConnectionId         uint64   `json:"connectionId"`
+	PlanKey              string   `json:"planKey"`
+	Tasks                []string `json:"tasks,omitempty"`
+	ScopeConfigId        uint64   `mapstructure:"scopeConfigId" json:"scopeConfigId"`
+	*BambooScopeConfig   `mapstructure:"scopeConfig" json:"scopeConfig"`
+	api.CollectorOptions `mapstructure:",squash"`
 }
diff --git a/backend/plugins/bitbucket/api/blueprint_V200_test.go b/backend/plugins/bitbucket/api/blueprint_V200_test.go
index 93b08bb29..b26ae731b 100644
--- a/backend/plugins/bitbucket/api/blueprint_V200_test.go
+++ b/backend/plugins/bitbucket/api/blueprint_V200_test.go
@@ -83,6 +83,7 @@ func TestMakeDataSourcePipelinePlanV200(t *testing.T) {
 				Options: map[string]interface{}{
 					"fullName":     "likyh/likyhphp",
 					"connectionId": uint64(1),
+					"TimeAfter":    "",
 				},
 			},
 			{
diff --git a/backend/plugins/bitbucket/tasks/task_data.go b/backend/plugins/bitbucket/tasks/task_data.go
index 235b05080..87d8f297c 100644
--- a/backend/plugins/bitbucket/tasks/task_data.go
+++ b/backend/plugins/bitbucket/tasks/task_data.go
@@ -29,6 +29,7 @@ type BitbucketOptions struct {
 	FullName                     string   `json:"fullName" mapstructure:"fullName"`
 	ScopeConfigId                uint64   `json:"scopeConfigId" mapstructure:"scopeConfigId,omitempty"`
 	*models.BitbucketScopeConfig `mapstructure:"scopeConfig,omitempty" json:"scopeConfig"`
+	api.CollectorOptions         `mapstructure:",squash"`
 }
 
 type BitbucketTaskData struct {
diff --git a/backend/plugins/circleci/tasks/task_data.go b/backend/plugins/circleci/tasks/task_data.go
index d9398e017..ddc6335ce 100644
--- a/backend/plugins/circleci/tasks/task_data.go
+++ b/backend/plugins/circleci/tasks/task_data.go
@@ -24,11 +24,12 @@ import (
 )
 
 type CircleciOptions struct {
-	ConnectionId  uint64                      `json:"connectionId"`
-	ProjectSlug   string                      `json:"projectSlug"`
-	PageSize      uint64                      `mapstruct:"pageSize"`
-	ScopeConfigId uint64                      `json:"scopeConfigId" mapstructure:"scopeConfigId,omitempty"`
-	ScopeConfig   *models.CircleciScopeConfig `json:"scopeConfig" mapstructure:"scopeConfig,omitempty"`
+	ConnectionId            uint64                      `json:"connectionId"`
+	ProjectSlug             string                      `json:"projectSlug"`
+	PageSize                uint64                      `mapstruct:"pageSize"`
+	ScopeConfigId           uint64                      `json:"scopeConfigId" mapstructure:"scopeConfigId,omitempty"`
+	ScopeConfig             *models.CircleciScopeConfig `json:"scopeConfig" mapstructure:"scopeConfig,omitempty"`
+	helper.CollectorOptions `mapstructure:",squash"`
 }
 
 type CircleciTaskData struct {
diff --git a/backend/plugins/customize/tasks/task_data.go b/backend/plugins/customize/tasks/task_data.go
index 6850bd725..16c2b5108 100644
--- a/backend/plugins/customize/tasks/task_data.go
+++ b/backend/plugins/customize/tasks/task_data.go
@@ -17,6 +17,8 @@ limitations under the License.
 
 package tasks
 
+import "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+
 type MappingRules struct {
 	Table         string            `json:"table" example:"issues"`
 	RawDataTable  string            `json:"rawDataTable" example:"_raw_jira_api_issues"`
@@ -25,7 +27,8 @@ type MappingRules struct {
 }
 
 type Options struct {
-	TransformationRules []MappingRules `json:"transformationRules"`
+	TransformationRules  []MappingRules `json:"transformationRules"`
+	api.CollectorOptions `mapstructure:",squash"`
 }
 
 type TaskData struct {
diff --git a/backend/plugins/feishu/tasks/task_data.go b/backend/plugins/feishu/tasks/task_data.go
index 428c06739..23b30466f 100644
--- a/backend/plugins/feishu/tasks/task_data.go
+++ b/backend/plugins/feishu/tasks/task_data.go
@@ -26,8 +26,9 @@ type FeishuApiParams struct {
 }
 
 type FeishuOptions struct {
-	ConnectionId       uint64  `json:"connectionId"`
-	NumOfDaysToCollect float64 `json:"numOfDaysToCollect"`
+	ConnectionId            uint64  `json:"connectionId"`
+	NumOfDaysToCollect      float64 `json:"numOfDaysToCollect"`
+	helper.CollectorOptions `mapstructure:",squash"`
 }
 
 type FeishuTaskData struct {
diff --git a/backend/plugins/gitee/tasks/task_data.go b/backend/plugins/gitee/tasks/task_data.go
index aa551fbc3..df1913321 100644
--- a/backend/plugins/gitee/tasks/task_data.go
+++ b/backend/plugins/gitee/tasks/task_data.go
@@ -26,10 +26,11 @@ import (
 )
 
 type GiteeOptions struct {
-	ConnectionId uint64 `json:"connectionId"`
-	Owner        string
-	Repo         string
-	ScopeConfig  models.GiteeScopeConfig `mapstructure:"scopeConfig" json:"scopeConfig"`
+	ConnectionId         uint64 `json:"connectionId"`
+	Owner                string
+	Repo                 string
+	ScopeConfig          models.GiteeScopeConfig `mapstructure:"scopeConfig" json:"scopeConfig"`
+	api.CollectorOptions `mapstructure:",squash"`
 }
 
 type GiteeTaskData struct {
diff --git a/backend/plugins/github/tasks/task_data.go b/backend/plugins/github/tasks/task_data.go
index f87d57694..b20b63669 100644
--- a/backend/plugins/github/tasks/task_data.go
+++ b/backend/plugins/github/tasks/task_data.go
@@ -27,13 +27,14 @@ import (
 )
 
 type GithubOptions struct {
-	ConnectionId  uint64                    `json:"connectionId" mapstructure:"connectionId,omitempty"`
-	ScopeConfigId uint64                    `json:"scopeConfigId" mapstructure:"scopeConfigId,omitempty"`
-	GithubId      int                       `json:"githubId" mapstructure:"githubId,omitempty"`
-	Owner         string                    `json:"owner" mapstructure:"owner,omitempty"`
-	Repo          string                    `json:"repo"  mapstructure:"repo,omitempty"`
-	Name          string                    `json:"name"  mapstructure:"name,omitempty"`
-	ScopeConfig   *models.GithubScopeConfig `mapstructure:"scopeConfig,omitempty" json:"scopeConfig"`
+	ConnectionId            uint64                    `json:"connectionId" mapstructure:"connectionId,omitempty"`
+	ScopeConfigId           uint64                    `json:"scopeConfigId" mapstructure:"scopeConfigId,omitempty"`
+	GithubId                int                       `json:"githubId" mapstructure:"githubId,omitempty"`
+	Owner                   string                    `json:"owner" mapstructure:"owner,omitempty"`
+	Repo                    string                    `json:"repo"  mapstructure:"repo,omitempty"`
+	Name                    string                    `json:"name"  mapstructure:"name,omitempty"`
+	ScopeConfig             *models.GithubScopeConfig `mapstructure:"scopeConfig,omitempty" json:"scopeConfig"`
+	helper.CollectorOptions `mapstructure:",squash"`
 }
 
 type GithubTaskData struct {
diff --git a/backend/plugins/gitlab/tasks/task_data.go b/backend/plugins/gitlab/tasks/task_data.go
index 0eaa36e45..a2a7fead4 100644
--- a/backend/plugins/gitlab/tasks/task_data.go
+++ b/backend/plugins/gitlab/tasks/task_data.go
@@ -24,10 +24,11 @@ import (
 )
 
 type GitlabOptions struct {
-	ConnectionId  uint64                    `mapstructure:"connectionId" json:"connectionId"`
-	ProjectId     int                       `mapstructure:"projectId" json:"projectId"`
-	ScopeConfigId uint64                    `mapstructure:"scopeConfigId" json:"scopeConfigId"`
-	ScopeConfig   *models.GitlabScopeConfig `mapstructure:"scopeConfig" json:"scopeConfig"`
+	ConnectionId            uint64                    `mapstructure:"connectionId" json:"connectionId"`
+	ProjectId               int                       `mapstructure:"projectId" json:"projectId"`
+	ScopeConfigId           uint64                    `mapstructure:"scopeConfigId" json:"scopeConfigId"`
+	ScopeConfig             *models.GitlabScopeConfig `mapstructure:"scopeConfig" json:"scopeConfig"`
+	helper.CollectorOptions `mapstructure:",squash"`
 }
 
 type GitlabTaskData struct {
diff --git a/backend/plugins/icla/tasks/task_data.go b/backend/plugins/icla/tasks/task_data.go
index dfa35a0ea..7573bb780 100644
--- a/backend/plugins/icla/tasks/task_data.go
+++ b/backend/plugins/icla/tasks/task_data.go
@@ -29,7 +29,8 @@ type IclaOptions struct {
 	// options means some custom params required by plugin running.
 	// Such As How many rows do your want
 	// You can use it in subtasks, and you need to pass it to main.go and pipelines.
-	Tasks []string `json:"tasks,omitempty"`
+	Tasks                   []string `json:"tasks,omitempty"`
+	helper.CollectorOptions `mapstructure:",squash"`
 }
 
 type IclaTaskData struct {
diff --git a/backend/plugins/jenkins/tasks/task_data.go b/backend/plugins/jenkins/tasks/task_data.go
index d34b4e6ac..42cf44bb8 100644
--- a/backend/plugins/jenkins/tasks/task_data.go
+++ b/backend/plugins/jenkins/tasks/task_data.go
@@ -27,14 +27,15 @@ import (
 
 type JenkinsApiParams models.JenkinsApiParams
 type JenkinsOptions struct {
-	ConnectionId  uint64 `json:"connectionId"`
-	ScopeId       string
-	ScopeConfigId uint64                     `json:"scopeConfigId"`
-	JobFullName   string                     `json:"jobFullName"` // "path1/path2/job name"
-	JobName       string                     `json:"jobName"`     // "job name"
-	JobPath       string                     `json:"jobPath"`     // "job/path1/job/path2"
-	Tasks         []string                   `json:"tasks,omitempty"`
-	ScopeConfig   *models.JenkinsScopeConfig `mapstructure:"scopeConfig" json:"scopeConfig"`
+	ConnectionId         uint64 `json:"connectionId"`
+	ScopeId              string
+	ScopeConfigId        uint64                     `json:"scopeConfigId"`
+	JobFullName          string                     `json:"jobFullName"` // "path1/path2/job name"
+	JobName              string                     `json:"jobName"`     // "job name"
+	JobPath              string                     `json:"jobPath"`     // "job/path1/job/path2"
+	Tasks                []string                   `json:"tasks,omitempty"`
+	ScopeConfig          *models.JenkinsScopeConfig `mapstructure:"scopeConfig" json:"scopeConfig"`
+	api.CollectorOptions `mapstructure:",squash"`
 }
 
 type JenkinsTaskData struct {
diff --git a/backend/plugins/jira/tasks/task_data.go b/backend/plugins/jira/tasks/task_data.go
index 42433bf9c..185eb4d43 100644
--- a/backend/plugins/jira/tasks/task_data.go
+++ b/backend/plugins/jira/tasks/task_data.go
@@ -26,11 +26,12 @@ import (
 )
 
 type JiraOptions struct {
-	ConnectionId  uint64                  `json:"connectionId"`
-	BoardId       uint64                  `json:"boardId"`
-	ScopeConfig   *models.JiraScopeConfig `json:"scopeConfig"`
-	ScopeConfigId uint64
-	PageSize      int
+	ConnectionId         uint64                  `json:"connectionId"`
+	BoardId              uint64                  `json:"boardId"`
+	ScopeConfig          *models.JiraScopeConfig `json:"scopeConfig"`
+	ScopeConfigId        uint64
+	PageSize             int
+	api.CollectorOptions `mapstructure:",squash"`
 }
 
 type JiraTaskData struct {
diff --git a/backend/plugins/opsgenie/tasks/task_data.go b/backend/plugins/opsgenie/tasks/task_data.go
index 5d20fc820..22800c866 100644
--- a/backend/plugins/opsgenie/tasks/task_data.go
+++ b/backend/plugins/opsgenie/tasks/task_data.go
@@ -24,10 +24,11 @@ import (
 )
 
 type OpsgenieOptions struct {
-	ConnectionId uint64   `json:"connectionId"`
-	ServiceId    string   `json:"service_id,omitempty"`
-	ServiceName  string   `json:"service_name,omitempty"`
-	Tasks        []string `json:"tasks,omitempty"`
+	ConnectionId         uint64   `json:"connectionId"`
+	ServiceId            string   `json:"service_id,omitempty"`
+	ServiceName          string   `json:"service_name,omitempty"`
+	Tasks                []string `json:"tasks,omitempty"`
+	api.CollectorOptions `mapstructure:",squash"`
 	*models.OpsenieScopeConfig
 }
 
diff --git a/backend/plugins/pagerduty/tasks/task_data.go b/backend/plugins/pagerduty/tasks/task_data.go
index 5f456f725..7718f0120 100644
--- a/backend/plugins/pagerduty/tasks/task_data.go
+++ b/backend/plugins/pagerduty/tasks/task_data.go
@@ -24,10 +24,11 @@ import (
 )
 
 type PagerDutyOptions struct {
-	ConnectionId uint64   `json:"connectionId"`
-	ServiceId    string   `json:"service_id,omitempty"`
-	ServiceName  string   `json:"service_name,omitempty"`
-	Tasks        []string `json:"tasks,omitempty"`
+	ConnectionId         uint64   `json:"connectionId"`
+	ServiceId            string   `json:"service_id,omitempty"`
+	ServiceName          string   `json:"service_name,omitempty"`
+	Tasks                []string `json:"tasks,omitempty"`
+	api.CollectorOptions `mapstructure:",squash"`
 	*models.PagerdutyScopeConfig
 }
 
diff --git a/backend/plugins/slack/tasks/task_data.go b/backend/plugins/slack/tasks/task_data.go
index 41c811fb6..a280657b6 100644
--- a/backend/plugins/slack/tasks/task_data.go
+++ b/backend/plugins/slack/tasks/task_data.go
@@ -26,7 +26,8 @@ type SlackApiParams struct {
 }
 
 type SlackOptions struct {
-	ConnectionId uint64 `json:"connectionId"`
+	ConnectionId            uint64 `json:"connectionId"`
+	helper.CollectorOptions `mapstructure:",squash"`
 }
 
 type SlackTaskData struct {
diff --git a/backend/plugins/sonarqube/tasks/task_data.go b/backend/plugins/sonarqube/tasks/task_data.go
index 998054a69..cec529707 100644
--- a/backend/plugins/sonarqube/tasks/task_data.go
+++ b/backend/plugins/sonarqube/tasks/task_data.go
@@ -28,9 +28,10 @@ type SonarqubeOptions struct {
 	// options means some custom params required by plugin running.
 	// Such As How many rows do your want
 	// You can use it in subtasks, and you need to pass it to main.go and pipelines.
-	ConnectionId uint64   `json:"connectionId"`
-	ProjectKey   string   `json:"projectKey"`
-	Tasks        []string `json:"tasks,omitempty"`
+	ConnectionId         uint64   `json:"connectionId"`
+	ProjectKey           string   `json:"projectKey"`
+	Tasks                []string `json:"tasks,omitempty"`
+	api.CollectorOptions `mapstructure:",squash"`
 }
 
 type SonarqubeTaskData struct {
diff --git a/backend/plugins/tapd/tasks/task_data.go b/backend/plugins/tapd/tasks/task_data.go
index 16b077bbb..b796cb5eb 100644
--- a/backend/plugins/tapd/tasks/task_data.go
+++ b/backend/plugins/tapd/tasks/task_data.go
@@ -26,12 +26,13 @@ import (
 )
 
 type TapdOptions struct {
-	ConnectionId  uint64 `mapstruct:"connectionId"`
-	WorkspaceId   uint64 `mapstruct:"workspaceId"`
-	PageSize      uint64 `mapstruct:"pageSize"`
-	CstZone       *time.Location
-	ScopeConfigId uint64
-	ScopeConfig   *models.TapdScopeConfig `json:"scopeConfig"`
+	ConnectionId            uint64 `mapstruct:"connectionId"`
+	WorkspaceId             uint64 `mapstruct:"workspaceId"`
+	PageSize                uint64 `mapstruct:"pageSize"`
+	CstZone                 *time.Location
+	ScopeConfigId           uint64
+	ScopeConfig             *models.TapdScopeConfig `json:"scopeConfig"`
+	helper.CollectorOptions `mapstructure:",squash"`
 }
 
 type TapdTaskData struct {
diff --git a/backend/plugins/teambition/tasks/task_data.go b/backend/plugins/teambition/tasks/task_data.go
index 961f1f33d..7b992e1ac 100644
--- a/backend/plugins/teambition/tasks/task_data.go
+++ b/backend/plugins/teambition/tasks/task_data.go
@@ -25,11 +25,12 @@ import (
 )
 
 type TeambitionOptions struct {
-	ConnectionId        uint64 `json:"connectionId"`
-	ProjectId           string `json:"projectId"`
-	PageSize            uint64 `mapstruct:"pageSize"`
-	CstZone             *time.Location
-	TransformationRules TransformationRules `json:"transformationRules"`
+	ConnectionId            uint64 `json:"connectionId"`
+	ProjectId               string `json:"projectId"`
+	PageSize                uint64 `mapstruct:"pageSize"`
+	CstZone                 *time.Location
+	TransformationRules     TransformationRules `json:"transformationRules"`
+	helper.CollectorOptions `mapstructure:",squash"`
 }
 
 type TeambitionTaskData struct {
diff --git a/backend/plugins/trello/tasks/task_data.go b/backend/plugins/trello/tasks/task_data.go
index 273fc3f71..76f61602e 100644
--- a/backend/plugins/trello/tasks/task_data.go
+++ b/backend/plugins/trello/tasks/task_data.go
@@ -23,10 +23,11 @@ import (
 )
 
 type TrelloOptions struct {
-	ConnectionId  uint64 `json:"connectionId"`
-	BoardId       string `json:"boardId"`
-	ScopeId       string
-	ScopeConfigId uint64
+	ConnectionId         uint64 `json:"connectionId"`
+	BoardId              string `json:"boardId"`
+	ScopeId              string
+	ScopeConfigId        uint64
+	api.CollectorOptions `mapstructure:",squash"`
 }
 
 type TrelloTaskData struct {
diff --git a/backend/plugins/zentao/api/blueprint_V200_test.go b/backend/plugins/zentao/api/blueprint_V200_test.go
index 53a55690e..bb2dd7ca8 100644
--- a/backend/plugins/zentao/api/blueprint_V200_test.go
+++ b/backend/plugins/zentao/api/blueprint_V200_test.go
@@ -84,6 +84,7 @@ func TestMakeDataSourcePipelinePlanV200(t *testing.T) {
 				Options: map[string]interface{}{
 					"ConnectionId": uint64(1),
 					"projectId":    int64(1),
+					"TimeAfter":    "",
 				},
 			},
 		},
diff --git a/backend/plugins/zentao/tasks/task_data.go b/backend/plugins/zentao/tasks/task_data.go
index 26bc9734c..177c8710f 100644
--- a/backend/plugins/zentao/tasks/task_data.go
+++ b/backend/plugins/zentao/tasks/task_data.go
@@ -36,8 +36,9 @@ type ZentaoOptions struct {
 	ConnectionId uint64 `json:"connectionId"`
 	ProjectId    int64  `json:"projectId" mapstructure:"projectId"`
 	// TODO not support now
-	ScopeConfigId uint64                    `json:"scopeConfigId" mapstructure:"scopeConfigId,omitempty"`
-	ScopeConfig   *models.ZentaoScopeConfig `json:"scopeConfig" mapstructure:"scopeConfig,omitempty"`
+	ScopeConfigId           uint64                    `json:"scopeConfigId" mapstructure:"scopeConfigId,omitempty"`
+	ScopeConfig             *models.ZentaoScopeConfig `json:"scopeConfig" mapstructure:"scopeConfig,omitempty"`
+	helper.CollectorOptions `mapstructure:",squash"`
 }
 
 func (o *ZentaoOptions) GetParams() any {