You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by zh...@apache.org on 2022/06/08 11:47:26 UTC
[incubator-devlake] branch release-v0.11 updated: fix(gitlab): delete gitlab child pipeline subtask (#2120)
This is an automated email from the ASF dual-hosted git repository.
zhangliang2022 pushed a commit to branch release-v0.11
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/release-v0.11 by this push:
new 0677f875 fix(gitlab): delete gitlab child pipeline subtask (#2120)
0677f875 is described below
commit 0677f875a3e1705d274f97fd4334fe7de3008025
Author: Warren Chen <yi...@merico.dev>
AuthorDate: Wed Jun 8 19:47:23 2022 +0800
fix(gitlab): delete gitlab child pipeline subtask (#2120)
closes #2118
---
plugins/gitlab/impl/impl.go | 2 -
plugins/gitlab/tasks/pipeline_collector.go | 33 -----------------
plugins/gitlab/tasks/pipeline_extractor.go | 59 ------------------------------
3 files changed, 94 deletions(-)
diff --git a/plugins/gitlab/impl/impl.go b/plugins/gitlab/impl/impl.go
index 9a21548a..2c48224b 100644
--- a/plugins/gitlab/impl/impl.go
+++ b/plugins/gitlab/impl/impl.go
@@ -62,8 +62,6 @@ func (plugin Gitlab) SubTaskMetas() []core.SubTaskMeta {
tasks.ExtractApiMergeRequestsCommitsMeta,
tasks.CollectApiPipelinesMeta,
tasks.ExtractApiPipelinesMeta,
- tasks.CollectApiChildrenOnPipelinesMeta,
- tasks.ExtractApiChildrenOnPipelinesMeta,
tasks.EnrichMergeRequestsMeta,
tasks.ConvertProjectMeta,
tasks.ConvertIssuesMeta,
diff --git a/plugins/gitlab/tasks/pipeline_collector.go b/plugins/gitlab/tasks/pipeline_collector.go
index 237c0e6c..3dfcff4d 100644
--- a/plugins/gitlab/tasks/pipeline_collector.go
+++ b/plugins/gitlab/tasks/pipeline_collector.go
@@ -23,7 +23,6 @@ import (
)
const RAW_PIPELINE_TABLE = "gitlab_api_pipeline"
-const RAW_CHILDREN_ON_PIPELINE_TABLE = "gitlab_api_children_on_pipeline"
var CollectApiPipelinesMeta = core.SubTaskMeta{
Name: "collectApiPipelines",
@@ -32,13 +31,6 @@ var CollectApiPipelinesMeta = core.SubTaskMeta{
Description: "Collect pipeline data from gitlab api",
}
-var CollectApiChildrenOnPipelinesMeta = core.SubTaskMeta{
- Name: "collectApiChildrenOnPipelines",
- EntryPoint: CollectApiChildrenOnPipelines,
- EnabledByDefault: true,
- Description: "Collect pipline child data from gitlab api",
-}
-
func CollectApiPipelines(taskCtx core.SubTaskContext) error {
rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_PIPELINE_TABLE)
@@ -58,28 +50,3 @@ func CollectApiPipelines(taskCtx core.SubTaskContext) error {
return collector.Execute()
}
-
-func CollectApiChildrenOnPipelines(taskCtx core.SubTaskContext) error {
- rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_CHILDREN_ON_PIPELINE_TABLE)
-
- iterator, err := GetPipelinesIterator(taskCtx)
- if err != nil {
- return err
- }
-
- collector, err := helper.NewApiCollector(helper.ApiCollectorArgs{
- RawDataSubTaskArgs: *rawDataSubTaskArgs,
- ApiClient: data.ApiClient,
- Incremental: false,
- Input: iterator,
- UrlTemplate: "projects/{{ .Params.ProjectId }}/pipelines/{{ .Input.GitlabId }}",
- Query: GetQuery,
- ResponseParser: helper.GetRawMessageDirectFromResponse,
- })
-
- if err != nil {
- return err
- }
-
- return collector.Execute()
-}
diff --git a/plugins/gitlab/tasks/pipeline_extractor.go b/plugins/gitlab/tasks/pipeline_extractor.go
index 3e0203b6..bed10494 100644
--- a/plugins/gitlab/tasks/pipeline_extractor.go
+++ b/plugins/gitlab/tasks/pipeline_extractor.go
@@ -57,13 +57,6 @@ var ExtractApiPipelinesMeta = core.SubTaskMeta{
Description: "Extract raw pipelines data into tool layer table GitlabPipeline",
}
-var ExtractApiChildrenOnPipelinesMeta = core.SubTaskMeta{
- Name: "extractApiChildrenOnPipelines",
- EntryPoint: ExtractApiChildrenOnPipelines,
- EnabledByDefault: true,
- Description: "Extract raw pipelines data into tool layer table GitlabPipeline",
-}
-
func ExtractApiPipelines(taskCtx core.SubTaskContext) error {
rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_PIPELINE_TABLE)
@@ -100,58 +93,6 @@ func ExtractApiPipelines(taskCtx core.SubTaskContext) error {
return extractor.Execute()
}
-func ExtractApiChildrenOnPipelines(taskCtx core.SubTaskContext) error {
- rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_CHILDREN_ON_PIPELINE_TABLE)
-
- extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{
- RawDataSubTaskArgs: *rawDataSubTaskArgs,
- Extract: func(row *helper.RawData) ([]interface{}, error) {
- pipelineRes := &ApiSinglePipelineResponse{}
- err := json.Unmarshal(row.Data, pipelineRes)
- if err != nil {
- return nil, err
- }
- duration := int(pipelineRes.UpdatedAt.ToTime().Sub(pipelineRes.GitlabCreatedAt.ToTime()).Seconds())
- pipelineRes.Duration = duration
- gitlabPipeline, err := convertSinglePipeline(pipelineRes)
- if err != nil {
- return nil, err
- }
-
- // use data.Options.ProjectId to set the value of ProjectId for it
- gitlabPipeline.ProjectId = data.Options.ProjectId
-
- results := make([]interface{}, 0, 1)
- results = append(results, gitlabPipeline)
-
- return results, nil
- },
- })
-
- if err != nil {
- return err
- }
-
- return extractor.Execute()
-}
-
-func convertSinglePipeline(pipeline *ApiSinglePipelineResponse) (*models.GitlabPipeline, error) {
- gitlabPipeline := &models.GitlabPipeline{
- GitlabId: pipeline.GitlabId,
- ProjectId: pipeline.ProjectId,
- GitlabCreatedAt: pipeline.GitlabCreatedAt.ToTime(),
- Ref: pipeline.Ref,
- Sha: pipeline.Sha,
- WebUrl: pipeline.WebUrl,
- Duration: pipeline.Duration,
- StartedAt: helper.Iso8601TimeToTime(pipeline.GitlabCreatedAt),
- FinishedAt: helper.Iso8601TimeToTime(pipeline.UpdatedAt),
- Coverage: pipeline.Coverage,
- Status: pipeline.Status,
- }
- return gitlabPipeline, nil
-}
-
func convertPipeline(pipeline *ApiPipeline) (*models.GitlabPipeline, error) {
gitlabPipeline := &models.GitlabPipeline{
GitlabId: pipeline.GitlabId,