You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by wa...@apache.org on 2022/09/10 07:03:18 UTC
[incubator-devlake] branch feat-plugin-zentao updated (ef013b89 -> cdf63f3d)
This is an automated email from the ASF dual-hosted git repository.
warren pushed a change to branch feat-plugin-zentao
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
from ef013b89 fix: golangci-lint error (#3032)
new e75bcd36 feat(zentao): create new plugin
new cdf63f3d feat(zentao): create new plugin
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../zentao/api/blueprint.go | 4 +-
.../zentao/api/connection.go | 93 ++++++++++------------
plugins/{ae => zentao}/api/init.go | 0
.../zentao/impl/impl.go | 57 ++++++-------
.../model.go => zentao/models/access_token.go} | 9 ++-
.../models}/archived/connection.go | 60 ++++++++------
.../zentao/models/connection.go | 20 ++---
.../migrationscripts/20220906_add_init_tables.go | 4 +-
.../models/migrationscripts/register.go | 0
.../zentao/tasks/api_client.go | 40 +++++++---
.../tasks/project_collector.go} | 58 +++++++-------
.../zentao/tasks/task_data.go | 28 ++++---
.../zentao/zentao.go | 18 +++--
13 files changed, 212 insertions(+), 179 deletions(-)
copy generator/template/plugin/api/blueprint.go-template => plugins/zentao/api/blueprint.go (95%)
copy generator/template/plugin/api/connection.go-template => plugins/zentao/api/connection.go (61%)
copy plugins/{ae => zentao}/api/init.go (100%)
copy generator/template/plugin/impl/impl_complete_plugin.go-template => plugins/zentao/impl/impl.go (52%)
copy plugins/{customize/models/model.go => zentao/models/access_token.go} (82%)
copy plugins/{azure/models/migrationscripts => zentao/models}/archived/connection.go (71%)
copy generator/template/plugin/models/connection.go-template => plugins/zentao/models/connection.go (73%)
copy generator/template/migrationscripts/add_init_tables.go-template => plugins/zentao/models/migrationscripts/20220906_add_init_tables.go (94%)
copy plugins/{dora => zentao}/models/migrationscripts/register.go (100%)
copy generator/template/plugin/tasks/api_client.go-template => plugins/zentao/tasks/api_client.go (62%)
copy plugins/{ae/tasks/commits_collector.go => zentao/tasks/project_collector.go} (62%)
copy generator/template/plugin/tasks/task_data_complete_plugin.go-template => plugins/zentao/tasks/task_data.go (75%)
copy generator/template/plugin/plugin_main_complete_plugin.go-template => plugins/zentao/zentao.go (67%)
[incubator-devlake] 01/02: feat(zentao): create new plugin
Posted by wa...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
warren pushed a commit to branch feat-plugin-zentao
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit e75bcd36c00a5d3736a13e400860b49065207038
Author: Yingchu Chen <yi...@merico.dev>
AuthorDate: Tue Sep 6 18:03:04 2022 +0800
feat(zentao): create new plugin
Relate to #2961
---
plugins/zentao/api/blueprint.go | 69 ++++++++++
plugins/zentao/api/connection.go | 149 +++++++++++++++++++++
plugins/zentao/api/init.go | 39 ++++++
plugins/zentao/impl/impl.go | 124 +++++++++++++++++
plugins/zentao/models/archived/connection.go | 70 ++++++++++
plugins/zentao/models/connection.go | 51 +++++++
.../migrationscripts/20220906_add_init_tables.go | 39 ++++++
plugins/zentao/models/migrationscripts/register.go | 27 ++++
plugins/zentao/tasks/api_client.go | 92 +++++++++++++
plugins/zentao/tasks/project_collector.go | 78 +++++++++++
plugins/zentao/tasks/task_data.go | 61 +++++++++
plugins/zentao/zentao.go | 47 +++++++
12 files changed, 846 insertions(+)
diff --git a/plugins/zentao/api/blueprint.go b/plugins/zentao/api/blueprint.go
new file mode 100644
index 00000000..3fa0e25f
--- /dev/null
+++ b/plugins/zentao/api/blueprint.go
@@ -0,0 +1,69 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package api
+
+import (
+ "encoding/json"
+
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/helper"
+ "github.com/apache/incubator-devlake/plugins/zentao/tasks"
+)
+
+func MakePipelinePlan(subtaskMetas []core.SubTaskMeta, connectionId uint64, scope []*core.BlueprintScopeV100) (core.PipelinePlan, error) {
+ var err error
+ plan := make(core.PipelinePlan, len(scope))
+ for i, scopeElem := range scope {
+ taskOptions := make(map[string]interface{})
+ err = json.Unmarshal(scopeElem.Options, &taskOptions)
+ if err != nil {
+ return nil, err
+ }
+ taskOptions["connectionId"] = connectionId
+
+ //TODO Add transformation rules to task options
+
+ /*
+ var transformationRules tasks.TransformationRules
+ if len(scopeElem.Transformation) > 0 {
+ err = json.Unmarshal(scopeElem.Transformation, &transformationRules)
+ if err != nil {
+ return nil, err
+ }
+ }
+ */
+ //taskOptions["transformationRules"] = transformationRules
+ _, err := tasks.DecodeAndValidateTaskOptions(taskOptions)
+ if err != nil {
+ return nil, err
+ }
+ // subtasks
+ subtasks, err := helper.MakePipelinePlanSubtasks(subtaskMetas, scopeElem.Entities)
+ if err != nil {
+ return nil, err
+ }
+ plan[i] = core.PipelineStage{
+ {
+ Plugin: "zentao",
+ Subtasks: subtasks,
+ Options: taskOptions,
+ },
+ }
+ }
+ return plan, nil
+}
diff --git a/plugins/zentao/api/connection.go b/plugins/zentao/api/connection.go
new file mode 100644
index 00000000..2f8183d5
--- /dev/null
+++ b/plugins/zentao/api/connection.go
@@ -0,0 +1,149 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package api
+
+import (
+ "context"
+ "github.com/apache/incubator-devlake/errors"
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/feishu/apimodels"
+ "github.com/apache/incubator-devlake/plugins/helper"
+ "github.com/apache/incubator-devlake/plugins/zentao/models"
+ "github.com/mitchellh/mapstructure"
+ "net/http"
+)
+
+//TODO Please modify the following code to fit your needs
+func TestConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) {
+ // process input
+ var params models.TestConnectionRequest
+ err := mapstructure.Decode(input.Body, ¶ms)
+ if err != nil {
+ return nil, errors.BadInput.Wrap(err, "could not decode request parameters", errors.AsUserMessage())
+ }
+ err = vld.Struct(params)
+ if err != nil {
+ return nil, errors.BadInput.Wrap(err, "could not validate request parameters", errors.AsUserMessage())
+ }
+
+ authApiClient, err := helper.NewApiClient(context.TODO(), params.Endpoint, nil, 0, params.Proxy, basicRes)
+ if err != nil {
+ return nil, err
+ }
+
+ // request for access token
+ tokenReqBody := &apimodels.ApiAccessTokenRequest{
+ AppId: params.Username,
+ AppSecret: params.Password,
+ }
+ tokenRes, err := authApiClient.Post("/tokens", nil, tokenReqBody, nil)
+ if err != nil {
+ return nil, err
+ }
+ tokenResBody := &apimodels.ApiAccessTokenResponse{}
+ err = helper.UnmarshalResponse(tokenRes, tokenResBody)
+ if err != nil {
+ return nil, err
+ }
+ if tokenResBody.AppAccessToken == "" && tokenResBody.TenantAccessToken == "" {
+ return nil, errors.Default.New("failed to request access token")
+ }
+
+ // output
+ return nil, nil
+}
+
+//TODO Please modify the folowing code to adapt to your plugin
+/*
+POST /plugins/Zentao/connections
+{
+ "name": "Zentao data connection name",
+ "endpoint": "Zentao api endpoint, i.e. https://example.com",
+ "username": "username, usually should be email address",
+ "password": "Zentao api access token"
+}
+*/
+func PostConnections(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) {
+ // update from request and save to database
+ connection := &models.ZentaoConnection{}
+ err := connectionHelper.Create(connection, input)
+ if err != nil {
+ return nil, err
+ }
+ return &core.ApiResourceOutput{Body: connection, Status: http.StatusOK}, nil
+}
+
+//TODO Please modify the folowing code to adapt to your plugin
+/*
+PATCH /plugins/Zentao/connections/:connectionId
+{
+ "name": "Zentao data connection name",
+ "endpoint": "Zentao api endpoint, i.e. https://example.com",
+ "username": "username, usually should be email address",
+ "password": "Zentao api access token"
+}
+*/
+func PatchConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) {
+ connection := &models.ZentaoConnection{}
+ err := connectionHelper.Patch(connection, input)
+ if err != nil {
+ return nil, err
+ }
+ return &core.ApiResourceOutput{Body: connection}, nil
+}
+
+/*
+DELETE /plugins/Zentao/connections/:connectionId
+*/
+func DeleteConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) {
+ connection := &models.ZentaoConnection{}
+ err := connectionHelper.First(connection, input.Params)
+ if err != nil {
+ return nil, err
+ }
+ err = connectionHelper.Delete(connection)
+ return &core.ApiResourceOutput{Body: connection}, err
+}
+
+/*
+GET /plugins/Zentao/connections
+*/
+func ListConnections(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) {
+ var connections []models.ZentaoConnection
+ err := connectionHelper.List(&connections)
+ if err != nil {
+ return nil, err
+ }
+ return &core.ApiResourceOutput{Body: connections, Status: http.StatusOK}, nil
+}
+
+//TODO Please modify the folowing code to adapt to your plugin
+/*
+GET /plugins/Zentao/connections/:connectionId
+{
+ "name": "Zentao data connection name",
+ "endpoint": "Zentao api endpoint, i.e. https://merico.atlassian.net/rest",
+ "username": "username, usually should be email address",
+ "password": "Zentao api access token"
+}
+*/
+func GetConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) {
+ connection := &models.ZentaoConnection{}
+ err := connectionHelper.First(connection, input.Params)
+ return &core.ApiResourceOutput{Body: connection}, err
+}
diff --git a/plugins/zentao/api/init.go b/plugins/zentao/api/init.go
new file mode 100644
index 00000000..6774e148
--- /dev/null
+++ b/plugins/zentao/api/init.go
@@ -0,0 +1,39 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package api
+
+import (
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/helper"
+ "github.com/go-playground/validator/v10"
+ "github.com/spf13/viper"
+ "gorm.io/gorm"
+)
+
+var vld *validator.Validate
+var connectionHelper *helper.ConnectionApiHelper
+var basicRes core.BasicRes
+
+func Init(config *viper.Viper, logger core.Logger, database *gorm.DB) {
+ basicRes = helper.NewDefaultBasicRes(config, logger, database)
+ vld = validator.New()
+ connectionHelper = helper.NewConnectionHelper(
+ basicRes,
+ vld,
+ )
+}
diff --git a/plugins/zentao/impl/impl.go b/plugins/zentao/impl/impl.go
new file mode 100644
index 00000000..cc81c226
--- /dev/null
+++ b/plugins/zentao/impl/impl.go
@@ -0,0 +1,124 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package impl
+
+import (
+ "fmt"
+ "github.com/apache/incubator-devlake/migration"
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/zentao/api"
+ "github.com/apache/incubator-devlake/plugins/zentao/models"
+ "github.com/apache/incubator-devlake/plugins/zentao/models/migrationscripts"
+ "github.com/apache/incubator-devlake/plugins/zentao/tasks"
+ "github.com/apache/incubator-devlake/plugins/helper"
+ "github.com/spf13/viper"
+ "gorm.io/gorm"
+)
+
+// make sure interface is implemented
+var _ core.PluginMeta = (*Zentao)(nil)
+var _ core.PluginInit = (*Zentao)(nil)
+var _ core.PluginTask = (*Zentao)(nil)
+var _ core.PluginApi = (*Zentao)(nil)
+var _ core.PluginBlueprintV100 = (*Zentao)(nil)
+var _ core.CloseablePluginTask = (*Zentao)(nil)
+
+
+
+type Zentao struct{}
+
+func (plugin Zentao) Description() string {
+ return "collect some Zentao data"
+}
+
+func (plugin Zentao) Init(config *viper.Viper, logger core.Logger, db *gorm.DB) error {
+ api.Init(config, logger, db)
+ return nil
+}
+
+func (plugin Zentao) SubTaskMetas() []core.SubTaskMeta {
+ // TODO add your sub task here
+ return []core.SubTaskMeta{
+ tasks.CollectProjectMeta,
+ }
+}
+
+func (plugin Zentao) PrepareTaskData(taskCtx core.TaskContext, options map[string]interface{}) (interface{}, error) {
+ op, err := tasks.DecodeAndValidateTaskOptions(options)
+ if err != nil {
+ return nil, err
+ }
+ connectionHelper := helper.NewConnectionHelper(
+ taskCtx,
+ nil,
+ )
+ connection := &models.ZentaoConnection{}
+ err = connectionHelper.FirstById(connection, op.ConnectionId)
+ if err != nil {
+ return nil, fmt.Errorf("unable to get Zentao connection by the given connection ID: %v", err)
+ }
+
+ apiClient, err := tasks.NewZentaoApiClient(taskCtx, connection)
+ if err != nil {
+ return nil, fmt.Errorf("unable to get Zentao API client instance: %v", err)
+ }
+
+ return &tasks.ZentaoTaskData{
+ Options: op,
+ ApiClient: apiClient,
+ }, nil
+}
+
+// PkgPath information lost when compiled as plugin(.so)
+func (plugin Zentao) RootPkgPath() string {
+ return "github.com/apache/incubator-devlake/plugins/zentao"
+}
+
+func (plugin Zentao) MigrationScripts() []migration.Script {
+ return migrationscripts.All()
+}
+
+func (plugin Zentao) ApiResources() map[string]map[string]core.ApiResourceHandler {
+ return map[string]map[string]core.ApiResourceHandler{
+ "test": {
+ "POST": api.TestConnection,
+ },
+ "connections": {
+ "POST": api.PostConnections,
+ "GET": api.ListConnections,
+ },
+ "connections/:connectionId": {
+ "GET": api.GetConnection,
+ "PATCH": api.PatchConnection,
+ "DELETE": api.DeleteConnection,
+ },
+ }
+}
+
+func (plugin Zentao) MakePipelinePlan(connectionId uint64, scope []*core.BlueprintScopeV100) (core.PipelinePlan, error) {
+ return api.MakePipelinePlan(plugin.SubTaskMetas(), connectionId, scope)
+}
+
+func (plugin Zentao) Close(taskCtx core.TaskContext) error {
+ data, ok := taskCtx.GetData().(*tasks.ZentaoTaskData)
+ if !ok {
+ return fmt.Errorf("GetData failed when try to close %+v", taskCtx)
+ }
+ data.ApiClient.Release()
+ return nil
+}
diff --git a/plugins/zentao/models/archived/connection.go b/plugins/zentao/models/archived/connection.go
new file mode 100644
index 00000000..60d7178c
--- /dev/null
+++ b/plugins/zentao/models/archived/connection.go
@@ -0,0 +1,70 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package models
+
+import (
+ "github.com/apache/incubator-devlake/models/migrationscripts/archived"
+)
+
+//TODO Please modify the following code to fit your needs
+// This object conforms to what the frontend currently sends.
+type ZentaoConnection struct {
+ RestConnection `mapstructure:",squash"`
+ //TODO you may need to use helper.BasicAuth instead of helper.AccessToken
+ BasicAuth `mapstructure:",squash"`
+}
+
+type TestConnectionRequest struct {
+ Endpoint string `json:"endpoint"`
+ Proxy string `json:"proxy"`
+ BasicAuth `mapstructure:",squash"`
+}
+
+// This object conforms to what the frontend currently expects.
+type ZentaoResponse struct {
+ Name string `json:"name"`
+ ID int `json:"id"`
+ ZentaoConnection
+}
+
+// Using User because it requires authentication.
+type ApiUserResponse struct {
+ Id int
+ Name string `json:"name"`
+}
+
+func (ZentaoConnection) TableName() string {
+ return "_tool_zentao_connections"
+}
+
+type BasicAuth struct {
+ Username string `mapstructure:"username" validate:"required" json:"username"`
+ Password string `mapstructure:"password" validate:"required" json:"password"`
+}
+
+type RestConnection struct {
+ BaseConnection `mapstructure:",squash"`
+ Endpoint string `mapstructure:"endpoint" validate:"required" json:"endpoint"`
+ Proxy string `mapstructure:"proxy" json:"proxy"`
+ RateLimitPerHour int `comment:"api request rate limt per hour" json:"rateLimit"`
+}
+
+type BaseConnection struct {
+ Name string `gorm:"type:varchar(100);uniqueIndex" json:"name" validate:"required"`
+ archived.Model
+}
diff --git a/plugins/zentao/models/connection.go b/plugins/zentao/models/connection.go
new file mode 100644
index 00000000..6140d7f4
--- /dev/null
+++ b/plugins/zentao/models/connection.go
@@ -0,0 +1,51 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package models
+
+import "github.com/apache/incubator-devlake/plugins/helper"
+
+//TODO Please modify the following code to fit your needs
+// This object conforms to what the frontend currently sends.
+type ZentaoConnection struct {
+ helper.RestConnection `mapstructure:",squash"`
+ //TODO you may need to use helper.BasicAuth instead of helper.AccessToken
+ helper.BasicAuth `mapstructure:",squash"`
+}
+
+type TestConnectionRequest struct {
+ Endpoint string `json:"endpoint"`
+ Proxy string `json:"proxy"`
+ helper.BasicAuth `mapstructure:",squash"`
+}
+
+// This object conforms to what the frontend currently expects.
+type ZentaoResponse struct {
+ Name string `json:"name"`
+ ID int `json:"id"`
+ ZentaoConnection
+}
+
+// Using User because it requires authentication.
+type ApiUserResponse struct {
+ Id int
+ Name string `json:"name"`
+}
+
+func (ZentaoConnection) TableName() string {
+ return "_tool_zentao_connections"
+}
diff --git a/plugins/zentao/models/migrationscripts/20220906_add_init_tables.go b/plugins/zentao/models/migrationscripts/20220906_add_init_tables.go
new file mode 100644
index 00000000..181bbc77
--- /dev/null
+++ b/plugins/zentao/models/migrationscripts/20220906_add_init_tables.go
@@ -0,0 +1,39 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+ "context"
+ "gorm.io/gorm"
+)
+
+type addInitTables struct {}
+
+func (u *addInitTables) Up(ctx context.Context, db *gorm.DB) error {
+ return db.Migrator().AutoMigrate(
+ // TODO add you models
+ )
+}
+
+func (*addInitTables) Version() uint64 {
+ return 20220906000001
+}
+
+func (*addInitTables) Name() string {
+ return "zentao init schemas"
+}
diff --git a/plugins/zentao/models/migrationscripts/register.go b/plugins/zentao/models/migrationscripts/register.go
new file mode 100644
index 00000000..92e20c01
--- /dev/null
+++ b/plugins/zentao/models/migrationscripts/register.go
@@ -0,0 +1,27 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import "github.com/apache/incubator-devlake/migration"
+
+// All return all the migration scripts
+func All() []migration.Script {
+ return []migration.Script{
+ new(addInitTables),
+ }
+}
diff --git a/plugins/zentao/tasks/api_client.go b/plugins/zentao/tasks/api_client.go
new file mode 100644
index 00000000..62d2d954
--- /dev/null
+++ b/plugins/zentao/tasks/api_client.go
@@ -0,0 +1,92 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "fmt"
+ "github.com/apache/incubator-devlake/errors"
+ "github.com/apache/incubator-devlake/plugins/feishu/apimodels"
+ "net/http"
+ "strconv"
+ "time"
+
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/helper"
+ "github.com/apache/incubator-devlake/plugins/zentao/models"
+)
+
+func NewZentaoApiClient(taskCtx core.TaskContext, connection *models.ZentaoConnection) (*helper.ApiAsyncClient, error) {
+ authApiClient, err := helper.NewApiClient(taskCtx.GetContext(), connection.Endpoint, nil, 0, connection.Proxy, taskCtx)
+ if err != nil {
+ return nil, err
+ }
+
+ // request for access token
+ tokenReqBody := &apimodels.ApiAccessTokenRequest{
+ AppId: connection.Username,
+ AppSecret: connection.Password,
+ }
+ tokenRes, err := authApiClient.Post("/tokens", nil, tokenReqBody, nil)
+ if err != nil {
+ return nil, err
+ }
+ tokenResBody := &apimodels.ApiAccessTokenResponse{}
+ err = helper.UnmarshalResponse(tokenRes, tokenResBody)
+ if err != nil {
+ return nil, err
+ }
+ if tokenResBody.AppAccessToken == "" && tokenResBody.TenantAccessToken == "" {
+ return nil, errors.Default.New("failed to request access token")
+ }
+ // real request apiClient
+ apiClient, err := helper.NewApiClient(taskCtx.GetContext(), connection.Endpoint, nil, 0, connection.Proxy, taskCtx)
+ if err != nil {
+ return nil, err
+ }
+ // set token
+ apiClient.SetHeaders(map[string]string{
+ "Token": fmt.Sprintf("%v", tokenResBody.TenantAccessToken),
+ })
+
+ // create rate limit calculator
+ rateLimiter := &helper.ApiRateLimitCalculator{
+ UserRateLimitPerHour: connection.RateLimitPerHour,
+ DynamicRateLimit: func(res *http.Response) (int, time.Duration, error) {
+ rateLimitHeader := res.Header.Get("RateLimit-Limit")
+ if rateLimitHeader == "" {
+ // use default
+ return 0, 0, nil
+ }
+ rateLimit, err := strconv.Atoi(rateLimitHeader)
+ if err != nil {
+ return 0, 0, fmt.Errorf("failed to parse RateLimit-Limit header: %w", err)
+ }
+ // seems like {{ .plugin-ame }} rate limit is on minute basis
+ return rateLimit, 1 * time.Minute, nil
+ },
+ }
+ asyncApiClient, err := helper.CreateAsyncApiClient(
+ taskCtx,
+ apiClient,
+ rateLimiter,
+ )
+ if err != nil {
+ return nil, err
+ }
+ return asyncApiClient, nil
+}
diff --git a/plugins/zentao/tasks/project_collector.go b/plugins/zentao/tasks/project_collector.go
new file mode 100644
index 00000000..7af9dc2c
--- /dev/null
+++ b/plugins/zentao/tasks/project_collector.go
@@ -0,0 +1,78 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "encoding/json"
+ "fmt"
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/helper"
+ "net/http"
+ "net/url"
+)
+
+const RAW_PROJECT_TABLE = "zentao_project"
+
+var _ core.SubTaskEntryPoint = CollectProject
+
+func CollectProject(taskCtx core.SubTaskContext) error {
+ data := taskCtx.GetData().(*ZentaoTaskData)
+ iterator, err := helper.NewDateIterator(365)
+ if err != nil {
+ return err
+ }
+
+ collector, err := helper.NewApiCollector(helper.ApiCollectorArgs{
+ RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
+ Ctx: taskCtx,
+ Params: ZentaoApiParams{},
+ Table: RAW_PROJECT_TABLE,
+ },
+ ApiClient: data.ApiClient,
+ Incremental: false,
+ Input: iterator,
+ PageSize: 100,
+ // TODO write which api would you want request
+ UrlTemplate: "projects",
+ Query: func(reqData *helper.RequestData) (url.Values, error) {
+ query := url.Values{}
+ query.Set("page", fmt.Sprintf("%v", reqData.Pager.Page))
+ query.Set("limit", fmt.Sprintf("%v", reqData.Pager.Size))
+ return query, nil
+ },
+ ResponseParser: func(res *http.Response) ([]json.RawMessage, error) {
+ var data struct {
+ Projects []json.RawMessage `json:"data"`
+ }
+ err = helper.UnmarshalResponse(res, &data)
+ return data.Projects, err
+ },
+ })
+ if err != nil {
+ return err
+ }
+
+ return collector.Execute()
+}
+
+var CollectProjectMeta = core.SubTaskMeta{
+ Name: "CollectProject",
+ EntryPoint: CollectProject,
+ EnabledByDefault: true,
+ Description: "Collect Project data from Zentao api",
+}
diff --git a/plugins/zentao/tasks/task_data.go b/plugins/zentao/tasks/task_data.go
new file mode 100644
index 00000000..a2d88f22
--- /dev/null
+++ b/plugins/zentao/tasks/task_data.go
@@ -0,0 +1,61 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "fmt"
+ "github.com/apache/incubator-devlake/plugins/helper"
+ "github.com/mitchellh/mapstructure"
+)
+
+type ZentaoApiParams struct {
+ ProductId uint64
+ ExecutionId uint64
+ ProjectId uint64
+}
+
+type ZentaoOptions struct {
+ // TODO add some custom options here if necessary
+ // options means some custom params required by plugin running.
+ // Such As How many rows do your want
+ // You can use it in sub tasks and you need pass it in main.go and pipelines.
+ ConnectionId uint64 `json:"connectionId"`
+ ProductId uint64
+ ExecutionId uint64
+ ProjectId uint64
+ Tasks []string `json:"tasks,omitempty"`
+ Since string
+}
+
+type ZentaoTaskData struct {
+ Options *ZentaoOptions
+ ApiClient *helper.ApiAsyncClient
+}
+
+func DecodeAndValidateTaskOptions(options map[string]interface{}) (*ZentaoOptions, error) {
+ var op ZentaoOptions
+ err := mapstructure.Decode(options, &op)
+ if err != nil {
+ return nil, err
+ }
+
+ if op.ConnectionId == 0 {
+ return nil, fmt.Errorf("connectionId is invalid")
+ }
+ return &op, nil
+}
diff --git a/plugins/zentao/zentao.go b/plugins/zentao/zentao.go
new file mode 100644
index 00000000..32845c90
--- /dev/null
+++ b/plugins/zentao/zentao.go
@@ -0,0 +1,47 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+ "github.com/apache/incubator-devlake/plugins/Zentao/impl"
+ "github.com/apache/incubator-devlake/runner"
+ "github.com/spf13/cobra"
+)
+
+// Export a variable named PluginEntry for Framework to search and load
+var PluginEntry impl.Zentao //nolint
+
+// standalone mode for debugging
+func main() {
+ cmd := &cobra.Command{Use: "zentao"}
+
+ connectionId := cmd.Flags().Uint64P("connectionId", "c", 0, "zentao connection id")
+ executionId := cmd.Flags().IntP("executionId", "e", 8, "execution id")
+ productId := cmd.Flags().IntP("productId", "o", 8, "product id")
+ projectId := cmd.Flags().IntP("projectId", "p", 8, "project id")
+
+ cmd.Run = func(cmd *cobra.Command, args []string) {
+ runner.DirectRun(cmd, args, PluginEntry, map[string]interface{}{
+ "connectionId": *connectionId,
+ "executionId": *executionId,
+ "productId": *productId,
+ "projectId": *projectId,
+ })
+ }
+ runner.RunCmd(cmd)
+}
[incubator-devlake] 02/02: feat(zentao): create new plugin
Posted by wa...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
warren pushed a commit to branch feat-plugin-zentao
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit cdf63f3db8e6cdcbf2180c3df033e1c85eb5f283
Author: Yingchu Chen <yi...@merico.dev>
AuthorDate: Tue Sep 6 18:03:04 2022 +0800
feat(zentao): create new plugin
Relate to #2961
---
plugins/zentao/api/connection.go | 11 +++++------
plugins/zentao/models/access_token.go | 27 +++++++++++++++++++++++++++
plugins/zentao/tasks/api_client.go | 13 ++++++-------
3 files changed, 38 insertions(+), 13 deletions(-)
diff --git a/plugins/zentao/api/connection.go b/plugins/zentao/api/connection.go
index 2f8183d5..fe702185 100644
--- a/plugins/zentao/api/connection.go
+++ b/plugins/zentao/api/connection.go
@@ -21,7 +21,6 @@ import (
"context"
"github.com/apache/incubator-devlake/errors"
"github.com/apache/incubator-devlake/plugins/core"
- "github.com/apache/incubator-devlake/plugins/feishu/apimodels"
"github.com/apache/incubator-devlake/plugins/helper"
"github.com/apache/incubator-devlake/plugins/zentao/models"
"github.com/mitchellh/mapstructure"
@@ -47,20 +46,20 @@ func TestConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput, erro
}
// request for access token
- tokenReqBody := &apimodels.ApiAccessTokenRequest{
- AppId: params.Username,
- AppSecret: params.Password,
+ tokenReqBody := &models.ApiAccessTokenRequest{
+ Account: params.Username,
+ Password: params.Password,
}
tokenRes, err := authApiClient.Post("/tokens", nil, tokenReqBody, nil)
if err != nil {
return nil, err
}
- tokenResBody := &apimodels.ApiAccessTokenResponse{}
+ tokenResBody := &models.ApiAccessTokenResponse{}
err = helper.UnmarshalResponse(tokenRes, tokenResBody)
if err != nil {
return nil, err
}
- if tokenResBody.AppAccessToken == "" && tokenResBody.TenantAccessToken == "" {
+ if tokenResBody.Token == "" {
return nil, errors.Default.New("failed to request access token")
}
diff --git a/plugins/zentao/models/access_token.go b/plugins/zentao/models/access_token.go
new file mode 100644
index 00000000..60bf2617
--- /dev/null
+++ b/plugins/zentao/models/access_token.go
@@ -0,0 +1,27 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package models
+
+type ApiAccessTokenRequest struct {
+ Account string `json:"account"`
+ Password string `json:"password"`
+}
+
+type ApiAccessTokenResponse struct {
+ Token string `json:"token"`
+}
diff --git a/plugins/zentao/tasks/api_client.go b/plugins/zentao/tasks/api_client.go
index 62d2d954..17d490fb 100644
--- a/plugins/zentao/tasks/api_client.go
+++ b/plugins/zentao/tasks/api_client.go
@@ -20,7 +20,6 @@ package tasks
import (
"fmt"
"github.com/apache/incubator-devlake/errors"
- "github.com/apache/incubator-devlake/plugins/feishu/apimodels"
"net/http"
"strconv"
"time"
@@ -37,20 +36,20 @@ func NewZentaoApiClient(taskCtx core.TaskContext, connection *models.ZentaoConne
}
// request for access token
- tokenReqBody := &apimodels.ApiAccessTokenRequest{
- AppId: connection.Username,
- AppSecret: connection.Password,
+ tokenReqBody := &models.ApiAccessTokenRequest{
+ Account: connection.Username,
+ Password: connection.Password,
}
tokenRes, err := authApiClient.Post("/tokens", nil, tokenReqBody, nil)
if err != nil {
return nil, err
}
- tokenResBody := &apimodels.ApiAccessTokenResponse{}
+ tokenResBody := &models.ApiAccessTokenResponse{}
err = helper.UnmarshalResponse(tokenRes, tokenResBody)
if err != nil {
return nil, err
}
- if tokenResBody.AppAccessToken == "" && tokenResBody.TenantAccessToken == "" {
+ if tokenResBody.Token == "" {
return nil, errors.Default.New("failed to request access token")
}
// real request apiClient
@@ -60,7 +59,7 @@ func NewZentaoApiClient(taskCtx core.TaskContext, connection *models.ZentaoConne
}
// set token
apiClient.SetHeaders(map[string]string{
- "Token": fmt.Sprintf("%v", tokenResBody.TenantAccessToken),
+ "Token": fmt.Sprintf("%v", tokenResBody.Token),
})
// create rate limit calculator