You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by wa...@apache.org on 2022/10/17 06:20:09 UTC
[incubator-devlake] 01/12: feat(zentao): create new plugin
This is an automated email from the ASF dual-hosted git repository.
warren pushed a commit to branch feat-plugin-zentao
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit d0852c8257b5e2ea3b54f7249d91983fd156c15f
Author: Yingchu Chen <yi...@merico.dev>
AuthorDate: Tue Sep 6 18:03:04 2022 +0800
feat(zentao): create new plugin
Relate to #2961
---
plugins/zentao/api/blueprint.go | 69 ++++++++++
plugins/zentao/api/connection.go | 149 +++++++++++++++++++++
plugins/zentao/api/init.go | 39 ++++++
plugins/zentao/impl/impl.go | 124 +++++++++++++++++
plugins/zentao/models/archived/connection.go | 70 ++++++++++
plugins/zentao/models/connection.go | 51 +++++++
.../migrationscripts/20220906_add_init_tables.go | 39 ++++++
plugins/zentao/models/migrationscripts/register.go | 27 ++++
plugins/zentao/tasks/api_client.go | 92 +++++++++++++
plugins/zentao/tasks/project_collector.go | 78 +++++++++++
plugins/zentao/tasks/task_data.go | 61 +++++++++
plugins/zentao/zentao.go | 47 +++++++
12 files changed, 846 insertions(+)
diff --git a/plugins/zentao/api/blueprint.go b/plugins/zentao/api/blueprint.go
new file mode 100644
index 00000000..3fa0e25f
--- /dev/null
+++ b/plugins/zentao/api/blueprint.go
@@ -0,0 +1,69 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package api
+
+import (
+ "encoding/json"
+
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/helper"
+ "github.com/apache/incubator-devlake/plugins/zentao/tasks"
+)
+
+func MakePipelinePlan(subtaskMetas []core.SubTaskMeta, connectionId uint64, scope []*core.BlueprintScopeV100) (core.PipelinePlan, error) {
+ var err error
+ plan := make(core.PipelinePlan, len(scope))
+ for i, scopeElem := range scope {
+ taskOptions := make(map[string]interface{})
+ err = json.Unmarshal(scopeElem.Options, &taskOptions)
+ if err != nil {
+ return nil, err
+ }
+ taskOptions["connectionId"] = connectionId
+
+ //TODO Add transformation rules to task options
+
+ /*
+ var transformationRules tasks.TransformationRules
+ if len(scopeElem.Transformation) > 0 {
+ err = json.Unmarshal(scopeElem.Transformation, &transformationRules)
+ if err != nil {
+ return nil, err
+ }
+ }
+ */
+ //taskOptions["transformationRules"] = transformationRules
+ _, err := tasks.DecodeAndValidateTaskOptions(taskOptions)
+ if err != nil {
+ return nil, err
+ }
+ // subtasks
+ subtasks, err := helper.MakePipelinePlanSubtasks(subtaskMetas, scopeElem.Entities)
+ if err != nil {
+ return nil, err
+ }
+ plan[i] = core.PipelineStage{
+ {
+ Plugin: "zentao",
+ Subtasks: subtasks,
+ Options: taskOptions,
+ },
+ }
+ }
+ return plan, nil
+}
diff --git a/plugins/zentao/api/connection.go b/plugins/zentao/api/connection.go
new file mode 100644
index 00000000..2f8183d5
--- /dev/null
+++ b/plugins/zentao/api/connection.go
@@ -0,0 +1,149 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package api
+
+import (
+ "context"
+ "github.com/apache/incubator-devlake/errors"
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/feishu/apimodels"
+ "github.com/apache/incubator-devlake/plugins/helper"
+ "github.com/apache/incubator-devlake/plugins/zentao/models"
+ "github.com/mitchellh/mapstructure"
+ "net/http"
+)
+
+//TODO Please modify the following code to fit your needs
+func TestConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) {
+ // process input
+ var params models.TestConnectionRequest
+ err := mapstructure.Decode(input.Body, ¶ms)
+ if err != nil {
+ return nil, errors.BadInput.Wrap(err, "could not decode request parameters", errors.AsUserMessage())
+ }
+ err = vld.Struct(params)
+ if err != nil {
+ return nil, errors.BadInput.Wrap(err, "could not validate request parameters", errors.AsUserMessage())
+ }
+
+ authApiClient, err := helper.NewApiClient(context.TODO(), params.Endpoint, nil, 0, params.Proxy, basicRes)
+ if err != nil {
+ return nil, err
+ }
+
+ // request for access token
+ tokenReqBody := &apimodels.ApiAccessTokenRequest{
+ AppId: params.Username,
+ AppSecret: params.Password,
+ }
+ tokenRes, err := authApiClient.Post("/tokens", nil, tokenReqBody, nil)
+ if err != nil {
+ return nil, err
+ }
+ tokenResBody := &apimodels.ApiAccessTokenResponse{}
+ err = helper.UnmarshalResponse(tokenRes, tokenResBody)
+ if err != nil {
+ return nil, err
+ }
+ if tokenResBody.AppAccessToken == "" && tokenResBody.TenantAccessToken == "" {
+ return nil, errors.Default.New("failed to request access token")
+ }
+
+ // output
+ return nil, nil
+}
+
+//TODO Please modify the folowing code to adapt to your plugin
+/*
+POST /plugins/Zentao/connections
+{
+ "name": "Zentao data connection name",
+ "endpoint": "Zentao api endpoint, i.e. https://example.com",
+ "username": "username, usually should be email address",
+ "password": "Zentao api access token"
+}
+*/
+func PostConnections(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) {
+ // update from request and save to database
+ connection := &models.ZentaoConnection{}
+ err := connectionHelper.Create(connection, input)
+ if err != nil {
+ return nil, err
+ }
+ return &core.ApiResourceOutput{Body: connection, Status: http.StatusOK}, nil
+}
+
+//TODO Please modify the folowing code to adapt to your plugin
+/*
+PATCH /plugins/Zentao/connections/:connectionId
+{
+ "name": "Zentao data connection name",
+ "endpoint": "Zentao api endpoint, i.e. https://example.com",
+ "username": "username, usually should be email address",
+ "password": "Zentao api access token"
+}
+*/
+func PatchConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) {
+ connection := &models.ZentaoConnection{}
+ err := connectionHelper.Patch(connection, input)
+ if err != nil {
+ return nil, err
+ }
+ return &core.ApiResourceOutput{Body: connection}, nil
+}
+
+/*
+DELETE /plugins/Zentao/connections/:connectionId
+*/
+func DeleteConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) {
+ connection := &models.ZentaoConnection{}
+ err := connectionHelper.First(connection, input.Params)
+ if err != nil {
+ return nil, err
+ }
+ err = connectionHelper.Delete(connection)
+ return &core.ApiResourceOutput{Body: connection}, err
+}
+
+/*
+GET /plugins/Zentao/connections
+*/
+func ListConnections(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) {
+ var connections []models.ZentaoConnection
+ err := connectionHelper.List(&connections)
+ if err != nil {
+ return nil, err
+ }
+ return &core.ApiResourceOutput{Body: connections, Status: http.StatusOK}, nil
+}
+
+//TODO Please modify the folowing code to adapt to your plugin
+/*
+GET /plugins/Zentao/connections/:connectionId
+{
+ "name": "Zentao data connection name",
+ "endpoint": "Zentao api endpoint, i.e. https://merico.atlassian.net/rest",
+ "username": "username, usually should be email address",
+ "password": "Zentao api access token"
+}
+*/
+func GetConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) {
+ connection := &models.ZentaoConnection{}
+ err := connectionHelper.First(connection, input.Params)
+ return &core.ApiResourceOutput{Body: connection}, err
+}
diff --git a/plugins/zentao/api/init.go b/plugins/zentao/api/init.go
new file mode 100644
index 00000000..6774e148
--- /dev/null
+++ b/plugins/zentao/api/init.go
@@ -0,0 +1,39 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package api
+
+import (
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/helper"
+ "github.com/go-playground/validator/v10"
+ "github.com/spf13/viper"
+ "gorm.io/gorm"
+)
+
+var vld *validator.Validate
+var connectionHelper *helper.ConnectionApiHelper
+var basicRes core.BasicRes
+
+func Init(config *viper.Viper, logger core.Logger, database *gorm.DB) {
+ basicRes = helper.NewDefaultBasicRes(config, logger, database)
+ vld = validator.New()
+ connectionHelper = helper.NewConnectionHelper(
+ basicRes,
+ vld,
+ )
+}
diff --git a/plugins/zentao/impl/impl.go b/plugins/zentao/impl/impl.go
new file mode 100644
index 00000000..cc81c226
--- /dev/null
+++ b/plugins/zentao/impl/impl.go
@@ -0,0 +1,124 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package impl
+
+import (
+ "fmt"
+ "github.com/apache/incubator-devlake/migration"
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/zentao/api"
+ "github.com/apache/incubator-devlake/plugins/zentao/models"
+ "github.com/apache/incubator-devlake/plugins/zentao/models/migrationscripts"
+ "github.com/apache/incubator-devlake/plugins/zentao/tasks"
+ "github.com/apache/incubator-devlake/plugins/helper"
+ "github.com/spf13/viper"
+ "gorm.io/gorm"
+)
+
+// make sure interface is implemented
+var _ core.PluginMeta = (*Zentao)(nil)
+var _ core.PluginInit = (*Zentao)(nil)
+var _ core.PluginTask = (*Zentao)(nil)
+var _ core.PluginApi = (*Zentao)(nil)
+var _ core.PluginBlueprintV100 = (*Zentao)(nil)
+var _ core.CloseablePluginTask = (*Zentao)(nil)
+
+
+
+type Zentao struct{}
+
+func (plugin Zentao) Description() string {
+ return "collect some Zentao data"
+}
+
+func (plugin Zentao) Init(config *viper.Viper, logger core.Logger, db *gorm.DB) error {
+ api.Init(config, logger, db)
+ return nil
+}
+
+func (plugin Zentao) SubTaskMetas() []core.SubTaskMeta {
+ // TODO add your sub task here
+ return []core.SubTaskMeta{
+ tasks.CollectProjectMeta,
+ }
+}
+
+func (plugin Zentao) PrepareTaskData(taskCtx core.TaskContext, options map[string]interface{}) (interface{}, error) {
+ op, err := tasks.DecodeAndValidateTaskOptions(options)
+ if err != nil {
+ return nil, err
+ }
+ connectionHelper := helper.NewConnectionHelper(
+ taskCtx,
+ nil,
+ )
+ connection := &models.ZentaoConnection{}
+ err = connectionHelper.FirstById(connection, op.ConnectionId)
+ if err != nil {
+ return nil, fmt.Errorf("unable to get Zentao connection by the given connection ID: %v", err)
+ }
+
+ apiClient, err := tasks.NewZentaoApiClient(taskCtx, connection)
+ if err != nil {
+ return nil, fmt.Errorf("unable to get Zentao API client instance: %v", err)
+ }
+
+ return &tasks.ZentaoTaskData{
+ Options: op,
+ ApiClient: apiClient,
+ }, nil
+}
+
+// PkgPath information lost when compiled as plugin(.so)
+func (plugin Zentao) RootPkgPath() string {
+ return "github.com/apache/incubator-devlake/plugins/zentao"
+}
+
+func (plugin Zentao) MigrationScripts() []migration.Script {
+ return migrationscripts.All()
+}
+
+func (plugin Zentao) ApiResources() map[string]map[string]core.ApiResourceHandler {
+ return map[string]map[string]core.ApiResourceHandler{
+ "test": {
+ "POST": api.TestConnection,
+ },
+ "connections": {
+ "POST": api.PostConnections,
+ "GET": api.ListConnections,
+ },
+ "connections/:connectionId": {
+ "GET": api.GetConnection,
+ "PATCH": api.PatchConnection,
+ "DELETE": api.DeleteConnection,
+ },
+ }
+}
+
+func (plugin Zentao) MakePipelinePlan(connectionId uint64, scope []*core.BlueprintScopeV100) (core.PipelinePlan, error) {
+ return api.MakePipelinePlan(plugin.SubTaskMetas(), connectionId, scope)
+}
+
+func (plugin Zentao) Close(taskCtx core.TaskContext) error {
+ data, ok := taskCtx.GetData().(*tasks.ZentaoTaskData)
+ if !ok {
+ return fmt.Errorf("GetData failed when try to close %+v", taskCtx)
+ }
+ data.ApiClient.Release()
+ return nil
+}
diff --git a/plugins/zentao/models/archived/connection.go b/plugins/zentao/models/archived/connection.go
new file mode 100644
index 00000000..60d7178c
--- /dev/null
+++ b/plugins/zentao/models/archived/connection.go
@@ -0,0 +1,70 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package models
+
+import (
+ "github.com/apache/incubator-devlake/models/migrationscripts/archived"
+)
+
+//TODO Please modify the following code to fit your needs
+// This object conforms to what the frontend currently sends.
+type ZentaoConnection struct {
+ RestConnection `mapstructure:",squash"`
+ //TODO you may need to use helper.BasicAuth instead of helper.AccessToken
+ BasicAuth `mapstructure:",squash"`
+}
+
+type TestConnectionRequest struct {
+ Endpoint string `json:"endpoint"`
+ Proxy string `json:"proxy"`
+ BasicAuth `mapstructure:",squash"`
+}
+
+// This object conforms to what the frontend currently expects.
+type ZentaoResponse struct {
+ Name string `json:"name"`
+ ID int `json:"id"`
+ ZentaoConnection
+}
+
+// Using User because it requires authentication.
+type ApiUserResponse struct {
+ Id int
+ Name string `json:"name"`
+}
+
+func (ZentaoConnection) TableName() string {
+ return "_tool_zentao_connections"
+}
+
+type BasicAuth struct {
+ Username string `mapstructure:"username" validate:"required" json:"username"`
+ Password string `mapstructure:"password" validate:"required" json:"password"`
+}
+
+type RestConnection struct {
+ BaseConnection `mapstructure:",squash"`
+ Endpoint string `mapstructure:"endpoint" validate:"required" json:"endpoint"`
+ Proxy string `mapstructure:"proxy" json:"proxy"`
+ RateLimitPerHour int `comment:"api request rate limt per hour" json:"rateLimit"`
+}
+
+type BaseConnection struct {
+ Name string `gorm:"type:varchar(100);uniqueIndex" json:"name" validate:"required"`
+ archived.Model
+}
diff --git a/plugins/zentao/models/connection.go b/plugins/zentao/models/connection.go
new file mode 100644
index 00000000..6140d7f4
--- /dev/null
+++ b/plugins/zentao/models/connection.go
@@ -0,0 +1,51 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package models
+
+import "github.com/apache/incubator-devlake/plugins/helper"
+
+//TODO Please modify the following code to fit your needs
+// This object conforms to what the frontend currently sends.
+type ZentaoConnection struct {
+ helper.RestConnection `mapstructure:",squash"`
+ //TODO you may need to use helper.BasicAuth instead of helper.AccessToken
+ helper.BasicAuth `mapstructure:",squash"`
+}
+
+type TestConnectionRequest struct {
+ Endpoint string `json:"endpoint"`
+ Proxy string `json:"proxy"`
+ helper.BasicAuth `mapstructure:",squash"`
+}
+
+// This object conforms to what the frontend currently expects.
+type ZentaoResponse struct {
+ Name string `json:"name"`
+ ID int `json:"id"`
+ ZentaoConnection
+}
+
+// Using User because it requires authentication.
+type ApiUserResponse struct {
+ Id int
+ Name string `json:"name"`
+}
+
+func (ZentaoConnection) TableName() string {
+ return "_tool_zentao_connections"
+}
diff --git a/plugins/zentao/models/migrationscripts/20220906_add_init_tables.go b/plugins/zentao/models/migrationscripts/20220906_add_init_tables.go
new file mode 100644
index 00000000..181bbc77
--- /dev/null
+++ b/plugins/zentao/models/migrationscripts/20220906_add_init_tables.go
@@ -0,0 +1,39 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+ "context"
+ "gorm.io/gorm"
+)
+
+type addInitTables struct {}
+
+func (u *addInitTables) Up(ctx context.Context, db *gorm.DB) error {
+ return db.Migrator().AutoMigrate(
+ // TODO add you models
+ )
+}
+
+func (*addInitTables) Version() uint64 {
+ return 20220906000001
+}
+
+func (*addInitTables) Name() string {
+ return "zentao init schemas"
+}
diff --git a/plugins/zentao/models/migrationscripts/register.go b/plugins/zentao/models/migrationscripts/register.go
new file mode 100644
index 00000000..92e20c01
--- /dev/null
+++ b/plugins/zentao/models/migrationscripts/register.go
@@ -0,0 +1,27 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import "github.com/apache/incubator-devlake/migration"
+
+// All return all the migration scripts
+func All() []migration.Script {
+ return []migration.Script{
+ new(addInitTables),
+ }
+}
diff --git a/plugins/zentao/tasks/api_client.go b/plugins/zentao/tasks/api_client.go
new file mode 100644
index 00000000..62d2d954
--- /dev/null
+++ b/plugins/zentao/tasks/api_client.go
@@ -0,0 +1,92 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "fmt"
+ "github.com/apache/incubator-devlake/errors"
+ "github.com/apache/incubator-devlake/plugins/feishu/apimodels"
+ "net/http"
+ "strconv"
+ "time"
+
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/helper"
+ "github.com/apache/incubator-devlake/plugins/zentao/models"
+)
+
+func NewZentaoApiClient(taskCtx core.TaskContext, connection *models.ZentaoConnection) (*helper.ApiAsyncClient, error) {
+ authApiClient, err := helper.NewApiClient(taskCtx.GetContext(), connection.Endpoint, nil, 0, connection.Proxy, taskCtx)
+ if err != nil {
+ return nil, err
+ }
+
+ // request for access token
+ tokenReqBody := &apimodels.ApiAccessTokenRequest{
+ AppId: connection.Username,
+ AppSecret: connection.Password,
+ }
+ tokenRes, err := authApiClient.Post("/tokens", nil, tokenReqBody, nil)
+ if err != nil {
+ return nil, err
+ }
+ tokenResBody := &apimodels.ApiAccessTokenResponse{}
+ err = helper.UnmarshalResponse(tokenRes, tokenResBody)
+ if err != nil {
+ return nil, err
+ }
+ if tokenResBody.AppAccessToken == "" && tokenResBody.TenantAccessToken == "" {
+ return nil, errors.Default.New("failed to request access token")
+ }
+ // real request apiClient
+ apiClient, err := helper.NewApiClient(taskCtx.GetContext(), connection.Endpoint, nil, 0, connection.Proxy, taskCtx)
+ if err != nil {
+ return nil, err
+ }
+ // set token
+ apiClient.SetHeaders(map[string]string{
+ "Token": fmt.Sprintf("%v", tokenResBody.TenantAccessToken),
+ })
+
+ // create rate limit calculator
+ rateLimiter := &helper.ApiRateLimitCalculator{
+ UserRateLimitPerHour: connection.RateLimitPerHour,
+ DynamicRateLimit: func(res *http.Response) (int, time.Duration, error) {
+ rateLimitHeader := res.Header.Get("RateLimit-Limit")
+ if rateLimitHeader == "" {
+ // use default
+ return 0, 0, nil
+ }
+ rateLimit, err := strconv.Atoi(rateLimitHeader)
+ if err != nil {
+ return 0, 0, fmt.Errorf("failed to parse RateLimit-Limit header: %w", err)
+ }
+ // seems like {{ .plugin-ame }} rate limit is on minute basis
+ return rateLimit, 1 * time.Minute, nil
+ },
+ }
+ asyncApiClient, err := helper.CreateAsyncApiClient(
+ taskCtx,
+ apiClient,
+ rateLimiter,
+ )
+ if err != nil {
+ return nil, err
+ }
+ return asyncApiClient, nil
+}
diff --git a/plugins/zentao/tasks/project_collector.go b/plugins/zentao/tasks/project_collector.go
new file mode 100644
index 00000000..7af9dc2c
--- /dev/null
+++ b/plugins/zentao/tasks/project_collector.go
@@ -0,0 +1,78 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "encoding/json"
+ "fmt"
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/helper"
+ "net/http"
+ "net/url"
+)
+
+const RAW_PROJECT_TABLE = "zentao_project"
+
+var _ core.SubTaskEntryPoint = CollectProject
+
+func CollectProject(taskCtx core.SubTaskContext) error {
+ data := taskCtx.GetData().(*ZentaoTaskData)
+ iterator, err := helper.NewDateIterator(365)
+ if err != nil {
+ return err
+ }
+
+ collector, err := helper.NewApiCollector(helper.ApiCollectorArgs{
+ RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
+ Ctx: taskCtx,
+ Params: ZentaoApiParams{},
+ Table: RAW_PROJECT_TABLE,
+ },
+ ApiClient: data.ApiClient,
+ Incremental: false,
+ Input: iterator,
+ PageSize: 100,
+ // TODO write which api would you want request
+ UrlTemplate: "projects",
+ Query: func(reqData *helper.RequestData) (url.Values, error) {
+ query := url.Values{}
+ query.Set("page", fmt.Sprintf("%v", reqData.Pager.Page))
+ query.Set("limit", fmt.Sprintf("%v", reqData.Pager.Size))
+ return query, nil
+ },
+ ResponseParser: func(res *http.Response) ([]json.RawMessage, error) {
+ var data struct {
+ Projects []json.RawMessage `json:"data"`
+ }
+ err = helper.UnmarshalResponse(res, &data)
+ return data.Projects, err
+ },
+ })
+ if err != nil {
+ return err
+ }
+
+ return collector.Execute()
+}
+
+var CollectProjectMeta = core.SubTaskMeta{
+ Name: "CollectProject",
+ EntryPoint: CollectProject,
+ EnabledByDefault: true,
+ Description: "Collect Project data from Zentao api",
+}
diff --git a/plugins/zentao/tasks/task_data.go b/plugins/zentao/tasks/task_data.go
new file mode 100644
index 00000000..a2d88f22
--- /dev/null
+++ b/plugins/zentao/tasks/task_data.go
@@ -0,0 +1,61 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "fmt"
+ "github.com/apache/incubator-devlake/plugins/helper"
+ "github.com/mitchellh/mapstructure"
+)
+
+type ZentaoApiParams struct {
+ ProductId uint64
+ ExecutionId uint64
+ ProjectId uint64
+}
+
+type ZentaoOptions struct {
+ // TODO add some custom options here if necessary
+ // options means some custom params required by plugin running.
+ // Such As How many rows do your want
+ // You can use it in sub tasks and you need pass it in main.go and pipelines.
+ ConnectionId uint64 `json:"connectionId"`
+ ProductId uint64
+ ExecutionId uint64
+ ProjectId uint64
+ Tasks []string `json:"tasks,omitempty"`
+ Since string
+}
+
+type ZentaoTaskData struct {
+ Options *ZentaoOptions
+ ApiClient *helper.ApiAsyncClient
+}
+
+func DecodeAndValidateTaskOptions(options map[string]interface{}) (*ZentaoOptions, error) {
+ var op ZentaoOptions
+ err := mapstructure.Decode(options, &op)
+ if err != nil {
+ return nil, err
+ }
+
+ if op.ConnectionId == 0 {
+ return nil, fmt.Errorf("connectionId is invalid")
+ }
+ return &op, nil
+}
diff --git a/plugins/zentao/zentao.go b/plugins/zentao/zentao.go
new file mode 100644
index 00000000..32845c90
--- /dev/null
+++ b/plugins/zentao/zentao.go
@@ -0,0 +1,47 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+ "github.com/apache/incubator-devlake/plugins/Zentao/impl"
+ "github.com/apache/incubator-devlake/runner"
+ "github.com/spf13/cobra"
+)
+
+// Export a variable named PluginEntry for Framework to search and load
+var PluginEntry impl.Zentao //nolint
+
+// standalone mode for debugging
+func main() {
+ cmd := &cobra.Command{Use: "zentao"}
+
+ connectionId := cmd.Flags().Uint64P("connectionId", "c", 0, "zentao connection id")
+ executionId := cmd.Flags().IntP("executionId", "e", 8, "execution id")
+ productId := cmd.Flags().IntP("productId", "o", 8, "product id")
+ projectId := cmd.Flags().IntP("projectId", "p", 8, "project id")
+
+ cmd.Run = func(cmd *cobra.Command, args []string) {
+ runner.DirectRun(cmd, args, PluginEntry, map[string]interface{}{
+ "connectionId": *connectionId,
+ "executionId": *executionId,
+ "productId": *productId,
+ "projectId": *projectId,
+ })
+ }
+ runner.RunCmd(cmd)
+}