[incubator-devlake] 04/12: update project collector

[wa...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

warren pushed a commit to branch feat-plugin-zentao
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git

commit ef1c75fb7ec98d0e958ea19a24ca2c90cac49b1b
Author: Yingchu Chen <yi...@merico.dev>
AuthorDate: Sat Sep 10 15:50:00 2022 +0800

    update project collector
---
 plugins/zentao/tasks/api_client.go        |  4 ++++
 plugins/zentao/tasks/project_collector.go | 11 +++-------
 plugins/zentao/tasks/shared.go            | 34 +++++++++++++++++++++++++++++++
 3 files changed, 41 insertions(+), 8 deletions(-)

diff --git a/plugins/zentao/tasks/api_client.go b/plugins/zentao/tasks/api_client.go
index 17d490fb..e5105ded 100644
--- a/plugins/zentao/tasks/api_client.go
+++ b/plugins/zentao/tasks/api_client.go
@@ -89,3 +89,7 @@ func NewZentaoApiClient(taskCtx core.TaskContext, connection *models.ZentaoConne
 	}
 	return asyncApiClient, nil
 }
+
+type ZentaoPagination struct {
+	Page int `json:"page"`
+}
diff --git a/plugins/zentao/tasks/project_collector.go b/plugins/zentao/tasks/project_collector.go
index 7af9dc2c..1aafec69 100644
--- a/plugins/zentao/tasks/project_collector.go
+++ b/plugins/zentao/tasks/project_collector.go
@@ -32,11 +32,6 @@ var _ core.SubTaskEntryPoint = CollectProject
 
 func CollectProject(taskCtx core.SubTaskContext) error {
 	data := taskCtx.GetData().(*ZentaoTaskData)
-	iterator, err := helper.NewDateIterator(365)
-	if err != nil {
-		return err
-	}
-
 	collector, err := helper.NewApiCollector(helper.ApiCollectorArgs{
 		RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
 			Ctx:    taskCtx,
@@ -45,7 +40,6 @@ func CollectProject(taskCtx core.SubTaskContext) error {
 		},
 		ApiClient:   data.ApiClient,
 		Incremental: false,
-		Input:       iterator,
 		PageSize:    100,
 		// TODO write which api would you want request
 		UrlTemplate: "projects",
@@ -55,11 +49,12 @@ func CollectProject(taskCtx core.SubTaskContext) error {
 			query.Set("limit", fmt.Sprintf("%v", reqData.Pager.Size))
 			return query, nil
 		},
+		GetTotalPages: GetTotalPagesFromResponse,
 		ResponseParser: func(res *http.Response) ([]json.RawMessage, error) {
 			var data struct {
-				Projects []json.RawMessage `json:"data"`
+				Projects []json.RawMessage `json:"projects"`
 			}
-			err = helper.UnmarshalResponse(res, &data)
+			err := helper.UnmarshalResponse(res, &data)
 			return data.Projects, err
 		},
 	})
diff --git a/plugins/zentao/tasks/shared.go b/plugins/zentao/tasks/shared.go
new file mode 100644
index 00000000..503e1f54
--- /dev/null
+++ b/plugins/zentao/tasks/shared.go
@@ -0,0 +1,34 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+	"net/http"
+
+	"github.com/apache/incubator-devlake/plugins/helper"
+)
+
+func GetTotalPagesFromResponse(res *http.Response, args *helper.ApiCollectorArgs) (int, error) {
+	body := &ZentaoPagination{}
+	err := helper.UnmarshalResponse(res, body)
+	if err != nil {
+		return 0, err
+	}
+	return body.Page, nil
+
+}