(incubator-devlake) branch dev-9 updated: feat(gitlab): collect child pipelines

[ly...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

lynwee pushed a commit to branch dev-9
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git


The following commit(s) were added to refs/heads/dev-9 by this push:
     new 7773cc9e7 feat(gitlab): collect child pipelines
7773cc9e7 is described below

commit 7773cc9e734de2fb7880db98eecfc04ea24e4423
Author: d4x1 <15...@qq.com>
AuthorDate: Thu Sep 5 12:50:55 2024 +0800

    feat(gitlab): collect child pipelines
---
 .../gitlab/tasks/child_pipeline_collector.go       | 83 ++++++++++++++++++++++
 1 file changed, 83 insertions(+)

diff --git a/backend/plugins/gitlab/tasks/child_pipeline_collector.go b/backend/plugins/gitlab/tasks/child_pipeline_collector.go
new file mode 100644
index 000000000..022f80410
--- /dev/null
+++ b/backend/plugins/gitlab/tasks/child_pipeline_collector.go
@@ -0,0 +1,83 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+	"fmt"
+	"net/url"
+	"time"
+
+	"github.com/apache/incubator-devlake/core/errors"
+	"github.com/apache/incubator-devlake/core/plugin"
+	helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+)
+
+func init() {
+	RegisterSubtaskMeta(&CollectApiChildPipelinesMeta)
+}
+
+const RAW_CHILD_PIPELINE_TABLE = "gitlab_api_child_pipeline"
+
+var CollectApiChildPipelinesMeta = plugin.SubTaskMeta{
+	Name:             "Collect Child Pipelines",
+	EntryPoint:       CollectApiChildPipelines,
+	EnabledByDefault: true,
+	Description:      "Collect child pipeline data from gitlab api, supports both timeFilter and diffSync.",
+	DomainTypes:      []string{plugin.DOMAIN_TYPE_CICD},
+	Dependencies:     []*plugin.SubTaskMeta{&CollectApiPipelinesMeta},
+}
+
+func CollectApiChildPipelines(taskCtx plugin.SubTaskContext) errors.Error {
+	rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_CHILD_PIPELINE_TABLE)
+	apiCollector, err := helper.NewStatefulApiCollector(*rawDataSubTaskArgs)
+	if err != nil {
+		return err
+	}
+
+	tickInterval, err := helper.CalcTickInterval(200, 1*time.Minute)
+	if err != nil {
+		return err
+	}
+
+	err = apiCollector.InitCollector(helper.ApiCollectorArgs{
+		RawDataSubTaskArgs: *rawDataSubTaskArgs,
+		ApiClient:          data.ApiClient,
+		MinTickInterval:    &tickInterval,
+		PageSize:           100,
+		UrlTemplate:        "projects/{{ .Params.ProjectId }}/pipelines",
+		Query: func(reqData *helper.RequestData) (url.Values, errors.Error) {
+			query := url.Values{}
+			if apiCollector.GetSince() != nil {
+				query.Set("updated_after", apiCollector.GetSince().Format(time.RFC3339))
+			}
+			query.Set("with_stats", "true")
+			query.Set("sort", "asc")
+			query.Set("source", "parent_pipeline")
+			query.Set("page", fmt.Sprintf("%v", reqData.Pager.Page))
+			query.Set("per_page", fmt.Sprintf("%v", reqData.Pager.Size))
+			return query, nil
+		},
+		ResponseParser: GetRawMessageFromResponse,
+		AfterResponse:  ignoreHTTPStatus403, // ignore 403 for CI/CD disable
+	})
+	if err != nil {
+		return err
+	}
+
+	return apiCollector.Execute()
+}