You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by ab...@apache.org on 2022/08/18 08:45:51 UTC
[incubator-devlake] branch main updated: feat(azure): add azure plugin
This is an automated email from the ASF dual-hosted git repository.
abeizn pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 195c1496 feat(azure): add azure plugin
195c1496 is described below
commit 195c149645c3670cb27e2def8d429c0489fa46a5
Author: Yingchu Chen <yi...@merico.dev>
AuthorDate: Thu Aug 18 15:51:27 2022 +0800
feat(azure): add azure plugin
---
plugins/azure/api/blueprint.go | 59 ++++++++
plugins/azure/api/connection.go | 148 +++++++++++++++++++++
plugins/azure/api/init.go | 39 ++++++
plugins/azure/azure.go | 42 ++++++
plugins/azure/impl/impl.go | 113 ++++++++++++++++
plugins/azure/models/build.go | 76 +++++++++++
plugins/azure/models/build_definition.go | 27 ++++
plugins/azure/models/connection.go | 43 ++++++
.../migrationscripts/20220727_add_init_tables.go | 52 ++++++++
.../models/migrationscripts/archived/build.go | 1 +
.../migrationscripts/archived/build_definition.go | 27 ++++
.../models/migrationscripts/archived/connection.go | 62 +++++++++
.../azure/models/migrationscripts/archived/repo.go | 41 ++++++
plugins/azure/models/migrationscripts/register.go | 29 ++++
plugins/azure/models/repo.go | 41 ++++++
plugins/azure/models/response.go | 18 +++
plugins/azure/tasks/api_client.go | 62 +++++++++
plugins/azure/tasks/build_definition_collector.go | 74 +++++++++++
plugins/azure/tasks/build_definition_extractor.go | 138 +++++++++++++++++++
plugins/azure/tasks/repo_collector.go | 79 +++++++++++
plugins/azure/tasks/repo_extractor.go | 115 ++++++++++++++++
plugins/azure/tasks/task_data.go | 57 ++++++++
22 files changed, 1343 insertions(+)
diff --git a/plugins/azure/api/blueprint.go b/plugins/azure/api/blueprint.go
new file mode 100644
index 00000000..93f028ff
--- /dev/null
+++ b/plugins/azure/api/blueprint.go
@@ -0,0 +1,59 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package api
+
+import (
+ "encoding/json"
+
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/helper"
+ "github.com/apache/incubator-devlake/plugins/jenkins/tasks"
+)
+
+func MakePipelinePlan(subtaskMetas []core.SubTaskMeta, connectionId uint64, scope []*core.BlueprintScopeV100) (core.PipelinePlan, error) {
+ var err error
+ plan := make(core.PipelinePlan, len(scope))
+ for i, scopeElem := range scope {
+ // handle taskOptions and transformationRules, by dumping them to taskOptions
+ taskOptions := make(map[string]interface{})
+ err = json.Unmarshal(scopeElem.Options, &taskOptions)
+ if err != nil {
+ return nil, err
+ }
+ taskOptions["connectionId"] = connectionId
+ _, err := tasks.DecodeAndValidateTaskOptions(taskOptions)
+ if err != nil {
+ return nil, err
+ }
+ // subtasks
+ subtasks, err := helper.MakePipelinePlanSubtasks(subtaskMetas, scopeElem.Entities)
+ if err != nil {
+ return nil, err
+ }
+ stage := core.PipelineStage{
+ {
+ Plugin: "azure",
+ Subtasks: subtasks,
+ Options: taskOptions,
+ },
+ }
+
+ plan[i] = stage
+ }
+ return plan, nil
+}
diff --git a/plugins/azure/api/connection.go b/plugins/azure/api/connection.go
new file mode 100644
index 00000000..b54ef456
--- /dev/null
+++ b/plugins/azure/api/connection.go
@@ -0,0 +1,148 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package api
+
+import (
+ "context"
+ "fmt"
+ "net/http"
+ "time"
+
+ "github.com/apache/incubator-devlake/plugins/azure/models"
+
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/helper"
+ "github.com/apache/incubator-devlake/utils"
+ "github.com/mitchellh/mapstructure"
+)
+
+func TestConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) {
+ // decode
+ var err error
+ var connection models.TestConnectionRequest
+ err = mapstructure.Decode(input.Body, &connection)
+ if err != nil {
+ return nil, err
+ }
+ // validate
+ err = vld.Struct(connection)
+ if err != nil {
+ return nil, err
+ }
+ // test connection
+ encodedToken := utils.GetEncodedToken(connection.Username, connection.Password)
+
+ apiClient, err := helper.NewApiClient(
+ context.TODO(),
+ connection.Endpoint,
+ map[string]string{
+ "Authorization": fmt.Sprintf("Basic %v", encodedToken),
+ },
+ 3*time.Second,
+ connection.Proxy,
+ basicRes,
+ )
+ if err != nil {
+ return nil, err
+ }
+ res, err := apiClient.Get("", nil, nil)
+ if err != nil {
+ return nil, err
+ }
+
+ if res.StatusCode != http.StatusOK {
+ return nil, fmt.Errorf("unexpected status code: %d", res.StatusCode)
+ }
+ return nil, nil
+}
+
+/*
+POST /plugins/zaure/connections
+{
+ "name": "zaure data connection name",
+ "endpoint": "zaure api endpoint, i.e. https://ci.zaure.io/",
+ "username": "username, usually should be email address",
+ "password": "zaure api access token"
+}
+*/
+func PostConnections(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) {
+ // create a new connection
+ connection := &models.AzureConnection{}
+
+ // update from request and save to database
+ err := connectionHelper.Create(connection, input)
+ if err != nil {
+ return nil, err
+ }
+ return &core.ApiResourceOutput{Body: connection, Status: http.StatusOK}, nil
+}
+
+/*
+PATCH /plugins/zaure/connections/connectionId
+{
+ "name": "zaure data connection name",
+ "endpoint": "zaure api endpoint, i.e. https://ci.zaure.io/",
+ "username": "username, usually should be email address",
+ "password": "zaure api access token"
+}
+*/
+
+func PatchConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) {
+ connection := &models.AzureConnection{}
+ err := connectionHelper.Patch(connection, input)
+ if err != nil {
+ return nil, err
+ }
+
+ return &core.ApiResourceOutput{Body: connection}, nil
+}
+
+/*
+DELETE /plugins/zaure/connections/connectionId
+*/
+func DeleteConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) {
+ connection := &models.AzureConnection{}
+ err := connectionHelper.First(connection, input.Params)
+ if err != nil {
+ return nil, err
+ }
+ err = connectionHelper.Delete(connection)
+ return &core.ApiResourceOutput{Body: connection}, err
+}
+
+/*
+GET /plugins/zaure/connections
+*/
+func ListConnections(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) {
+ var connections []models.AzureConnection
+ err := connectionHelper.List(&connections)
+ if err != nil {
+ return nil, err
+ }
+
+ return &core.ApiResourceOutput{Body: connections, Status: http.StatusOK}, nil
+}
+
+/*
+GET /plugins/zaure/connections/connectionId
+*/
+func GetConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) {
+ connection := &models.AzureConnection{}
+ err := connectionHelper.First(connection, input.Params)
+ return &core.ApiResourceOutput{Body: connection}, err
+}
diff --git a/plugins/azure/api/init.go b/plugins/azure/api/init.go
new file mode 100644
index 00000000..6774e148
--- /dev/null
+++ b/plugins/azure/api/init.go
@@ -0,0 +1,39 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package api
+
+import (
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/helper"
+ "github.com/go-playground/validator/v10"
+ "github.com/spf13/viper"
+ "gorm.io/gorm"
+)
+
+var vld *validator.Validate
+var connectionHelper *helper.ConnectionApiHelper
+var basicRes core.BasicRes
+
+func Init(config *viper.Viper, logger core.Logger, database *gorm.DB) {
+ basicRes = helper.NewDefaultBasicRes(config, logger, database)
+ vld = validator.New()
+ connectionHelper = helper.NewConnectionHelper(
+ basicRes,
+ vld,
+ )
+}
diff --git a/plugins/azure/azure.go b/plugins/azure/azure.go
new file mode 100644
index 00000000..bbdd47e0
--- /dev/null
+++ b/plugins/azure/azure.go
@@ -0,0 +1,42 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+ "github.com/apache/incubator-devlake/plugins/azure/impl"
+ "github.com/apache/incubator-devlake/runner"
+ "github.com/spf13/cobra"
+)
+
+var PluginEntry impl.Azure
+
+// standalone mode for debugging
+func main() {
+ cmd := &cobra.Command{Use: "azure"}
+
+ connectionId := cmd.Flags().Uint64P("connection", "c", 1, "azure connection id")
+ project := cmd.Flags().StringP("project", "p", "", "azure project name")
+
+ cmd.Run = func(cmd *cobra.Command, args []string) {
+ runner.DirectRun(cmd, args, PluginEntry, map[string]interface{}{
+ "connectionId": *connectionId,
+ "project": *project,
+ })
+ }
+ runner.RunCmd(cmd)
+}
diff --git a/plugins/azure/impl/impl.go b/plugins/azure/impl/impl.go
new file mode 100644
index 00000000..f0d0888c
--- /dev/null
+++ b/plugins/azure/impl/impl.go
@@ -0,0 +1,113 @@
+package impl
+
+import (
+ "fmt"
+ "github.com/apache/incubator-devlake/migration"
+ "github.com/apache/incubator-devlake/plugins/azure/api"
+ "github.com/apache/incubator-devlake/plugins/azure/models"
+ "github.com/apache/incubator-devlake/plugins/azure/models/migrationscripts"
+ "github.com/apache/incubator-devlake/plugins/azure/tasks"
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/helper"
+
+ "github.com/spf13/viper"
+ "gorm.io/gorm"
+)
+
+// make sure interface is implemented
+var _ core.PluginMeta = (*Azure)(nil)
+var _ core.PluginInit = (*Azure)(nil)
+var _ core.PluginTask = (*Azure)(nil)
+var _ core.PluginApi = (*Azure)(nil)
+var _ core.CloseablePluginTask = (*Azure)(nil)
+
+// Export a variable named PluginEntry for Framework to search and load
+var PluginEntry Azure //nolint
+
+type Azure struct{}
+
+func (plugin Azure) Description() string {
+ return "collect some Azure data"
+}
+
+func (plugin Azure) Init(config *viper.Viper, logger core.Logger, db *gorm.DB) error {
+ api.Init(config, logger, db)
+ return nil
+}
+
+func (plugin Azure) SubTaskMetas() []core.SubTaskMeta {
+ return []core.SubTaskMeta{
+ tasks.CollectApiRepoMeta,
+ tasks.ExtractApiRepoMeta,
+ tasks.CollectApiBuildDefinitionMeta,
+ tasks.ExtractApiBuildDefinitionMeta,
+ }
+}
+
+func (plugin Azure) PrepareTaskData(taskCtx core.TaskContext, options map[string]interface{}) (interface{}, error) {
+ op, err := tasks.DecodeAndValidateTaskOptions(options)
+ if err != nil {
+ return nil, err
+ }
+ if op.ConnectionId == 0 {
+ return nil, fmt.Errorf("connectionId is invalid")
+ }
+
+ connection := &models.AzureConnection{}
+ connectionHelper := helper.NewConnectionHelper(
+ taskCtx,
+ nil,
+ )
+ if err != nil {
+ return nil, err
+ }
+ err = connectionHelper.FirstById(connection, op.ConnectionId)
+ if err != nil {
+ return nil, err
+ }
+
+ apiClient, err := tasks.CreateApiClient(taskCtx, connection)
+ if err != nil {
+ return nil, err
+ }
+ return &tasks.AzureTaskData{
+ Options: op,
+ ApiClient: apiClient,
+ Connection: connection,
+ }, nil
+}
+
+// PkgPath information lost when compiled as plugin(.so)
+func (plugin Azure) RootPkgPath() string {
+ return "github.com/apache/incubator-devlake/plugins/azure"
+}
+
+func (plugin Azure) ApiResources() map[string]map[string]core.ApiResourceHandler {
+ return map[string]map[string]core.ApiResourceHandler{
+ "test": {
+ "POST": api.TestConnection,
+ },
+ "connections": {
+ "POST": api.PostConnections,
+ "GET": api.ListConnections,
+ },
+ "connections/:connectionId": {
+ "GET": api.GetConnection,
+ "PATCH": api.PatchConnection,
+ "DELETE": api.DeleteConnection,
+ },
+ }
+}
+
+func (plugin Azure) MigrationScripts() []migration.Script {
+ return migrationscripts.All()
+}
+
+func (plugin Azure) Close(taskCtx core.TaskContext) error {
+ data, ok := taskCtx.GetData().(*tasks.AzureTaskData)
+ if !ok {
+ return fmt.Errorf("GetData failed when try to close %+v", taskCtx)
+ }
+ data.ApiClient.Release()
+ return nil
+}
diff --git a/plugins/azure/models/build.go b/plugins/azure/models/build.go
new file mode 100644
index 00000000..aeeace28
--- /dev/null
+++ b/plugins/azure/models/build.go
@@ -0,0 +1,76 @@
+package models
+
+import (
+ "github.com/apache/incubator-devlake/models/common"
+ "time"
+)
+
+type AzureBuild struct {
+ common.NoPKModel
+ // collected fields
+ ConnectionId uint64 `gorm:"primaryKey"`
+ JobName string `gorm:"primaryKey;type:varchar(255)"`
+ Duration float64 // build time
+ DisplayName string `gorm:"type:varchar(255)"` // "#7"
+ EstimatedDuration float64 // EstimatedDuration
+ Number int64 `gorm:"primaryKey"`
+ Result string // Result
+ Timestamp int64 // start time
+ StartTime time.Time // convered by timestamp
+ CommitSha string `gorm:"type:varchar(255)"`
+}
+
+func (AzureBuild) TableName() string {
+ return "_tool_azure_builds"
+}
+
+type AutoGenerated struct {
+ Quality string `json:"quality"`
+ AuthoredBy struct {
+ DisplayName string `json:"displayName"`
+ URL string `json:"url"`
+ Links struct {
+ Avatar struct {
+ Href string `json:"href"`
+ } `json:"avatar"`
+ } `json:"_links"`
+ ID string `json:"id"`
+ UniqueName string `json:"uniqueName"`
+ ImageURL string `json:"imageUrl"`
+ Descriptor string `json:"descriptor"`
+ } `json:"authoredBy"`
+ Drafts []interface{} `json:"drafts"`
+ Queue struct {
+ Links struct {
+ Self struct {
+ Href string `json:"href"`
+ } `json:"self"`
+ } `json:"_links"`
+ ID int `json:"id"`
+ Name string `json:"name"`
+ URL string `json:"url"`
+ Pool struct {
+ ID int `json:"id"`
+ Name string `json:"name"`
+ IsHosted bool `json:"isHosted"`
+ } `json:"pool"`
+ } `json:"queue"`
+ ID int `json:"id"`
+ Name string `json:"name"`
+ URL string `json:"url"`
+ URI string `json:"uri"`
+ Path string `json:"path"`
+ Type string `json:"type"`
+ QueueStatus string `json:"queueStatus"`
+ Revision int `json:"revision"`
+ CreatedDate time.Time `json:"createdDate"`
+ Project struct {
+ ID string `json:"id"`
+ Name string `json:"name"`
+ URL string `json:"url"`
+ State string `json:"state"`
+ Revision int `json:"revision"`
+ Visibility string `json:"visibility"`
+ LastUpdateTime time.Time `json:"lastUpdateTime"`
+ } `json:"project"`
+}
diff --git a/plugins/azure/models/build_definition.go b/plugins/azure/models/build_definition.go
new file mode 100644
index 00000000..a2af71cd
--- /dev/null
+++ b/plugins/azure/models/build_definition.go
@@ -0,0 +1,27 @@
+package models
+
+import (
+ "github.com/apache/incubator-devlake/models/common"
+ "time"
+)
+
+type AzureBuildDefinition struct {
+ common.NoPKModel
+ // collected fields
+ ConnectionId uint64 `gorm:"primaryKey"`
+ ProjectId string `gorm:"primaryKey;type:varchar(255)"`
+ AzureId int `gorm:"primaryKey"`
+ AuthorId string `gorm:"type:varchar(255)"`
+ QueueId int
+ Url string `gorm:"type:varchar(255)"`
+ Name string `gorm:"type:varchar(255)"`
+ Path string `gorm:"type:varchar(255)"`
+ Type string `gorm:"type:varchar(255)"`
+ QueueStatus string `json:"queueStatus" gorm:"type:varchar(255)"`
+ Revision int `json:"revision"`
+ AzureCreatedDate time.Time `json:"createdDate"`
+}
+
+func (AzureBuildDefinition) TableName() string {
+ return "_tool_azure_build_definitions"
+}
diff --git a/plugins/azure/models/connection.go b/plugins/azure/models/connection.go
new file mode 100644
index 00000000..6a9025c5
--- /dev/null
+++ b/plugins/azure/models/connection.go
@@ -0,0 +1,43 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package models
+
+import "github.com/apache/incubator-devlake/plugins/helper"
+
+// This object conforms to what the frontend currently sends.
+type AzureConnection struct {
+ helper.RestConnection `mapstructure:",squash"`
+ helper.BasicAuth `mapstructure:",squash"`
+}
+
+type AzureResponse struct {
+ ID int `json:"id"`
+ Name string `json:"name"`
+ AzureConnection
+}
+
+type TestConnectionRequest struct {
+ Endpoint string `json:"endpoint" validate:"required"`
+ Username string `json:"username" validate:"required"`
+ Password string `json:"password" validate:"required"`
+ Proxy string `json:"proxy"`
+}
+
+func (AzureConnection) TableName() string {
+ return "_tool_azure_connections"
+}
diff --git a/plugins/azure/models/migrationscripts/20220727_add_init_tables.go b/plugins/azure/models/migrationscripts/20220727_add_init_tables.go
new file mode 100644
index 00000000..84861e00
--- /dev/null
+++ b/plugins/azure/models/migrationscripts/20220727_add_init_tables.go
@@ -0,0 +1,52 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+ "context"
+ "github.com/apache/incubator-devlake/plugins/azure/models/migrationscripts/archived"
+ "gorm.io/gorm"
+)
+
+type addInitTables struct{}
+
+func (*addInitTables) Up(ctx context.Context, db *gorm.DB) error {
+ if !db.Migrator().HasTable(&archived.AzureConnection{}) {
+ err := db.Migrator().AutoMigrate(&archived.AzureConnection{})
+ if err != nil {
+ return err
+ }
+ }
+ err := db.Migrator().AutoMigrate(
+ //&archived.AzureRepo{},
+ &archived.AzureBuildDefinition{},
+ )
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func (*addInitTables) Version() uint64 {
+ return 20220727231237
+}
+
+func (*addInitTables) Name() string {
+ return "Azure init schemas"
+}
diff --git a/plugins/azure/models/migrationscripts/archived/build.go b/plugins/azure/models/migrationscripts/archived/build.go
new file mode 100644
index 00000000..c24c8254
--- /dev/null
+++ b/plugins/azure/models/migrationscripts/archived/build.go
@@ -0,0 +1 @@
+package archived
diff --git a/plugins/azure/models/migrationscripts/archived/build_definition.go b/plugins/azure/models/migrationscripts/archived/build_definition.go
new file mode 100644
index 00000000..cf4c739f
--- /dev/null
+++ b/plugins/azure/models/migrationscripts/archived/build_definition.go
@@ -0,0 +1,27 @@
+package archived
+
+import (
+ "github.com/apache/incubator-devlake/models/migrationscripts/archived"
+ "time"
+)
+
+type AzureBuildDefinition struct {
+ archived.NoPKModel
+ // collected fields
+ ConnectionId uint64 `gorm:"primaryKey"`
+ ProjectId string `gorm:"primaryKey;type:varchar(255)"`
+ AzureId int `gorm:"primaryKey"`
+ AuthorId string `gorm:"type:varchar(255)"`
+ QueueId int
+ Url string `gorm:"type:varchar(255)"`
+ Name string `gorm:"type:varchar(255)"`
+ Path string `gorm:"type:varchar(255)"`
+ Type string `gorm:"type:varchar(255)"`
+ QueueStatus string `json:"queueStatus" gorm:"type:varchar(255)"`
+ Revision int `json:"revision"`
+ AzureCreatedDate time.Time `json:"createdDate"`
+}
+
+func (AzureBuildDefinition) TableName() string {
+ return "_tool_azure_build_definitions"
+}
diff --git a/plugins/azure/models/migrationscripts/archived/connection.go b/plugins/azure/models/migrationscripts/archived/connection.go
new file mode 100644
index 00000000..d289941a
--- /dev/null
+++ b/plugins/azure/models/migrationscripts/archived/connection.go
@@ -0,0 +1,62 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package archived
+
+import (
+ "github.com/apache/incubator-devlake/models/migrationscripts/archived"
+)
+
+type BaseConnection struct {
+ Name string `gorm:"type:varchar(100);uniqueIndex" json:"name" validate:"required"`
+ archived.Model
+}
+
+type BasicAuth struct {
+ Username string `mapstructure:"username" validate:"required" json:"username"`
+ Password string `mapstructure:"password" validate:"required" json:"password" encrypt:"yes"`
+}
+
+type RestConnection struct {
+ BaseConnection `mapstructure:",squash"`
+ Endpoint string `mapstructure:"endpoint" validate:"required" json:"endpoint"`
+ Proxy string `mapstructure:"proxy" json:"proxy"`
+ RateLimitPerHour int `comment:"api request rate limt per hour" json:"rateLimit"`
+}
+
+// This object conforms to what the frontend currently sends.
+type AzureConnection struct {
+ RestConnection `mapstructure:",squash"`
+ BasicAuth `mapstructure:",squash"`
+}
+
+type AzureResponse struct {
+ ID int `json:"id"`
+ Name string `json:"name"`
+ AzureConnection
+}
+
+type TestConnectionRequest struct {
+ Endpoint string `json:"endpoint" validate:"required"`
+ Username string `json:"username" validate:"required"`
+ Password string `json:"password" validate:"required"`
+ Proxy string `json:"proxy"`
+}
+
+func (AzureConnection) TableName() string {
+ return "_tool_azure_connections"
+}
diff --git a/plugins/azure/models/migrationscripts/archived/repo.go b/plugins/azure/models/migrationscripts/archived/repo.go
new file mode 100644
index 00000000..022acd17
--- /dev/null
+++ b/plugins/azure/models/migrationscripts/archived/repo.go
@@ -0,0 +1,41 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package archived
+
+import (
+ "github.com/apache/incubator-devlake/models/migrationscripts/archived"
+)
+
+type AzureRepo struct {
+ ConnectionId uint64 `gorm:"primaryKey"`
+ AzureId string `gorm:"primaryKey;type:varchar(255)" json:"id"`
+ Name string `gorm:"type:varchar(255)" json:"name"`
+ Url string `gorm:"type:varchar(255)" json:"url"`
+ ProjectId string `gorm:"type:varchar(255);index"`
+ DefaultBranch string `json:"defaultBranch"`
+ Size int `json:"size"`
+ RemoteURL string `json:"remoteUrl"`
+ SshUrl string `gorm:"type:varchar(255)" json:"sshUrl"`
+ WebUrl string `gorm:"type:varchar(255)" json:"webUrl"`
+ IsDisabled bool `json:"isDisabled"`
+ archived.NoPKModel
+}
+
+func (AzureRepo) TableName() string {
+ return "_tool_azure_repos"
+}
diff --git a/plugins/azure/models/migrationscripts/register.go b/plugins/azure/models/migrationscripts/register.go
new file mode 100644
index 00000000..c1365f7d
--- /dev/null
+++ b/plugins/azure/models/migrationscripts/register.go
@@ -0,0 +1,29 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+ "github.com/apache/incubator-devlake/migration"
+)
+
+// All return all the migration scripts
+func All() []migration.Script {
+ return []migration.Script{
+ new(addInitTables),
+ }
+}
diff --git a/plugins/azure/models/repo.go b/plugins/azure/models/repo.go
new file mode 100644
index 00000000..b7444945
--- /dev/null
+++ b/plugins/azure/models/repo.go
@@ -0,0 +1,41 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package models
+
+import (
+ "github.com/apache/incubator-devlake/models/common"
+)
+
+type AzureRepo struct {
+ ConnectionId uint64 `gorm:"primaryKey"`
+ AzureId string `gorm:"primaryKey;type:varchar(255)" json:"id"`
+ Name string `gorm:"type:varchar(255)" json:"name"`
+ Url string `gorm:"type:varchar(255)" json:"url"`
+ ProjectId string `gorm:"type:varchar(255);index"`
+ DefaultBranch string `json:"defaultBranch"`
+ Size int `json:"size"`
+ RemoteURL string `json:"remoteUrl"`
+ SshUrl string `gorm:"type:varchar(255)" json:"sshUrl"`
+ WebUrl string `gorm:"type:varchar(255)" json:"webUrl"`
+ IsDisabled bool `json:"isDisabled"`
+ common.NoPKModel
+}
+
+func (AzureRepo) TableName() string {
+ return "_tool_azure_repos"
+}
diff --git a/plugins/azure/models/response.go b/plugins/azure/models/response.go
new file mode 100644
index 00000000..556be189
--- /dev/null
+++ b/plugins/azure/models/response.go
@@ -0,0 +1,18 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package models
diff --git a/plugins/azure/tasks/api_client.go b/plugins/azure/tasks/api_client.go
new file mode 100644
index 00000000..d25693af
--- /dev/null
+++ b/plugins/azure/tasks/api_client.go
@@ -0,0 +1,62 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "fmt"
+ "github.com/apache/incubator-devlake/plugins/azure/models"
+ "net/http"
+
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/helper"
+)
+
+func CreateApiClient(taskCtx core.TaskContext, connection *models.AzureConnection) (*helper.ApiAsyncClient, error) {
+ // create synchronize api client so we can calculate api rate limit dynamically
+ headers := map[string]string{
+ "Authorization": fmt.Sprintf("Basic %v", connection.GetEncodedToken()),
+ }
+
+ apiClient, err := helper.NewApiClient(taskCtx.GetContext(), connection.Endpoint, headers, 0, connection.Proxy, taskCtx)
+ if err != nil {
+ return nil, err
+ }
+
+ apiClient.SetAfterFunction(func(res *http.Response) error {
+ if res.StatusCode == http.StatusUnauthorized {
+ return fmt.Errorf("authentication failed, please check your Username/Password")
+ }
+ return nil
+ })
+
+ // TODO add some check after request if necessary
+ // create rate limit calculator
+ rateLimiter := &helper.ApiRateLimitCalculator{
+ UserRateLimitPerHour: connection.RateLimitPerHour,
+ }
+ asyncApiClient, err := helper.CreateAsyncApiClient(
+ taskCtx,
+ apiClient,
+ rateLimiter,
+ )
+ if err != nil {
+ return nil, err
+ }
+
+ return asyncApiClient, nil
+}
diff --git a/plugins/azure/tasks/build_definition_collector.go b/plugins/azure/tasks/build_definition_collector.go
new file mode 100644
index 00000000..0480058a
--- /dev/null
+++ b/plugins/azure/tasks/build_definition_collector.go
@@ -0,0 +1,74 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "encoding/json"
+ "net/http"
+ "net/url"
+
+ "github.com/apache/incubator-devlake/plugins/helper"
+
+ "github.com/apache/incubator-devlake/plugins/core"
+)
+
+const RAW_BUILD_DEFINITION_TABLE = "azure_api_build_definitions"
+
+var CollectApiBuildDefinitionMeta = core.SubTaskMeta{
+ Name: "collectApiBuild",
+ EntryPoint: CollectApiBuildDefinitions,
+ Required: true,
+ Description: "Collect BuildDefinition data from Azure api",
+ DomainTypes: []string{core.DOMAIN_TYPE_CICD},
+}
+
+func CollectApiBuildDefinitions(taskCtx core.SubTaskContext) error {
+ data := taskCtx.GetData().(*AzureTaskData)
+
+ collector, err := helper.NewApiCollector(helper.ApiCollectorArgs{
+ RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
+ Ctx: taskCtx,
+ Params: AzureApiParams{
+ ConnectionId: data.Options.ConnectionId,
+ Project: data.Options.Project,
+ },
+ Table: RAW_BUILD_DEFINITION_TABLE,
+ },
+ ApiClient: data.ApiClient,
+
+ UrlTemplate: "{{ .Params.Project }}/_apis/build/definitions?api-version=7.1-preview.7",
+ Query: func(reqData *helper.RequestData) (url.Values, error) {
+ query := url.Values{}
+
+ return query, nil
+ },
+ ResponseParser: func(res *http.Response) ([]json.RawMessage, error) {
+ var data struct {
+ Builds []json.RawMessage `json:"value"`
+ }
+ err := helper.UnmarshalResponse(res, &data)
+ return data.Builds, err
+ },
+ })
+
+ if err != nil {
+ return err
+ }
+
+ return collector.Execute()
+}
diff --git a/plugins/azure/tasks/build_definition_extractor.go b/plugins/azure/tasks/build_definition_extractor.go
new file mode 100644
index 00000000..921d7dcd
--- /dev/null
+++ b/plugins/azure/tasks/build_definition_extractor.go
@@ -0,0 +1,138 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "encoding/json"
+ "time"
+
+ "github.com/apache/incubator-devlake/plugins/azure/models"
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/helper"
+)
+
+type AzureApiBuildDefinition struct {
+ Quality string `json:"quality"`
+ AuthoredBy struct {
+ DisplayName string `json:"displayName"`
+ URL string `json:"url"`
+ Links struct {
+ Avatar struct {
+ Href string `json:"href"`
+ } `json:"avatar"`
+ } `json:"_links"`
+ ID string `json:"id"`
+ UniqueName string `json:"uniqueName"`
+ ImageURL string `json:"imageUrl"`
+ Descriptor string `json:"descriptor"`
+ } `json:"authoredBy"`
+ Queue struct {
+ Links struct {
+ Self struct {
+ Href string `json:"href"`
+ } `json:"self"`
+ } `json:"_links"`
+ ID int `json:"id"`
+ Name string `json:"name"`
+ URL string `json:"url"`
+ Pool struct {
+ ID int `json:"id"`
+ Name string `json:"name"`
+ IsHosted bool `json:"isHosted"`
+ } `json:"pool"`
+ } `json:"queue"`
+ ID int `json:"id"`
+ Name string `json:"name"`
+ URL string `json:"url"`
+ URI string `json:"uri"`
+ Path string `json:"path"`
+ Type string `json:"type"`
+ QueueStatus string `json:"queueStatus"`
+ Revision int `json:"revision"`
+ CreatedDate time.Time `json:"createdDate"`
+ Project struct {
+ ID string `json:"id"`
+ Name string `json:"name"`
+ URL string `json:"url"`
+ State string `json:"state"`
+ Revision int `json:"revision"`
+ Visibility string `json:"visibility"`
+ LastUpdateTime time.Time `json:"lastUpdateTime"`
+ } `json:"project"`
+}
+
+var ExtractApiBuildDefinitionMeta = core.SubTaskMeta{
+ Name: "extractApiBuild",
+ EntryPoint: ExtractApiBuildDefinition,
+ Required: true,
+ Description: "Extract raw BuildDefinition data into tool layer table azure_repos",
+ DomainTypes: []string{core.DOMAIN_TYPE_CICD},
+}
+
+func ExtractApiBuildDefinition(taskCtx core.SubTaskContext) error {
+ data := taskCtx.GetData().(*AzureTaskData)
+ extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{
+ RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
+ Ctx: taskCtx,
+ /*
+ This struct will be JSONEncoded and stored into database along with raw data itself, to identity minimal
+ set of data to be process, for example, we process JiraIssues by Board
+ */
+ Params: AzureApiParams{
+ ConnectionId: data.Options.ConnectionId,
+ Project: data.Options.Project,
+ },
+ /*
+ Table store raw data
+ */
+ Table: RAW_BUILD_DEFINITION_TABLE,
+ },
+ Extract: func(row *helper.RawData) ([]interface{}, error) {
+ body := &AzureApiBuildDefinition{}
+ err := json.Unmarshal(row.Data, body)
+ if err != nil {
+ return nil, err
+ }
+
+ results := make([]interface{}, 0, 1)
+ azureBuildDefinition := &models.AzureBuildDefinition{
+ ConnectionId: data.Options.ConnectionId,
+ ProjectId: body.Project.ID,
+ AzureId: body.ID,
+ AuthorId: body.AuthoredBy.ID,
+ QueueId: body.Queue.ID,
+ Url: body.URL,
+ Name: body.Name,
+ Path: body.Path,
+ Type: body.Type,
+ QueueStatus: body.QueueStatus,
+ Revision: body.Revision,
+ AzureCreatedDate: body.CreatedDate,
+ }
+ results = append(results, azureBuildDefinition)
+
+ return results, nil
+ },
+ })
+
+ if err != nil {
+ return err
+ }
+
+ return extractor.Execute()
+}
diff --git a/plugins/azure/tasks/repo_collector.go b/plugins/azure/tasks/repo_collector.go
new file mode 100644
index 00000000..4109f460
--- /dev/null
+++ b/plugins/azure/tasks/repo_collector.go
@@ -0,0 +1,79 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "encoding/json"
+ "fmt"
+ "net/http"
+ "net/url"
+
+ "github.com/apache/incubator-devlake/plugins/helper"
+
+ "github.com/apache/incubator-devlake/plugins/core"
+)
+
+const RAW_REPOSITORIES_TABLE = "azure_api_repositories"
+
+var CollectApiRepoMeta = core.SubTaskMeta{
+ Name: "collectApiRepo",
+ EntryPoint: CollectApiRepositories,
+ Required: true,
+ Description: "Collect repositories data from Azure api",
+ DomainTypes: []string{core.DOMAIN_TYPE_CODE},
+}
+
+func CollectApiRepositories(taskCtx core.SubTaskContext) error {
+ data := taskCtx.GetData().(*AzureTaskData)
+
+ collector, err := helper.NewApiCollector(helper.ApiCollectorArgs{
+ RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
+ Ctx: taskCtx,
+ Params: AzureApiParams{
+ ConnectionId: data.Options.ConnectionId,
+ Project: data.Options.Project,
+ },
+ Table: RAW_REPOSITORIES_TABLE,
+ },
+ ApiClient: data.ApiClient,
+
+ UrlTemplate: "{{ .Params.Project }}/_apis/git/repositories?api-version=7.1-preview.1",
+ Query: func(reqData *helper.RequestData) (url.Values, error) {
+ query := url.Values{}
+ query.Set("state", "all")
+ query.Set("page", fmt.Sprintf("%v", reqData.Pager.Page))
+ query.Set("direction", "asc")
+ query.Set("per_page", fmt.Sprintf("%v", reqData.Pager.Size))
+
+ return query, nil
+ },
+ ResponseParser: func(res *http.Response) ([]json.RawMessage, error) {
+ var data struct {
+ Repos []json.RawMessage `json:"value"`
+ }
+ err := helper.UnmarshalResponse(res, &data)
+ return data.Repos, err
+ },
+ })
+
+ if err != nil {
+ return err
+ }
+
+ return collector.Execute()
+}
diff --git a/plugins/azure/tasks/repo_extractor.go b/plugins/azure/tasks/repo_extractor.go
new file mode 100644
index 00000000..391c2258
--- /dev/null
+++ b/plugins/azure/tasks/repo_extractor.go
@@ -0,0 +1,115 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "encoding/json"
+ "fmt"
+ "time"
+
+ "github.com/apache/incubator-devlake/plugins/azure/models"
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/helper"
+)
+
+type AzureApiRepo struct {
+ ID string `json:"id"`
+ Name string `json:"name"`
+ URL string `json:"url"`
+ Project struct {
+ ID string `json:"id"`
+ Name string `json:"name"`
+ URL string `json:"url"`
+ State string `json:"state"`
+ Revision int `json:"revision"`
+ Visibility string `json:"visibility"`
+ LastUpdateTime time.Time `json:"lastUpdateTime"`
+ } `json:"project"`
+ DefaultBranch string `json:"defaultBranch"`
+ Size int `json:"size"`
+ RemoteURL string `json:"remoteUrl"`
+ SSHURL string `json:"sshUrl"`
+ WebURL string `json:"webUrl"`
+ IsDisabled bool `json:"isDisabled"`
+}
+
+var ExtractApiRepoMeta = core.SubTaskMeta{
+ Name: "extractApiRepo",
+ EntryPoint: ExtractApiRepositories,
+ Required: true,
+ Description: "Extract raw Repositories data into tool layer table azure_repos",
+ DomainTypes: []string{core.DOMAIN_TYPE_CODE},
+}
+
+type ApiRepoResponse AzureApiRepo
+
+func ExtractApiRepositories(taskCtx core.SubTaskContext) error {
+ data := taskCtx.GetData().(*AzureTaskData)
+ extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{
+ RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
+ Ctx: taskCtx,
+ /*
+ This struct will be JSONEncoded and stored into database along with raw data itself, to identity minimal
+ set of data to be process, for example, we process JiraIssues by Board
+ */
+ Params: AzureApiParams{
+ ConnectionId: data.Options.ConnectionId,
+ Project: data.Options.Project,
+ },
+ /*
+ Table store raw data
+ */
+ Table: RAW_REPOSITORIES_TABLE,
+ },
+ Extract: func(row *helper.RawData) ([]interface{}, error) {
+ body := &ApiRepoResponse{}
+ err := json.Unmarshal(row.Data, body)
+ if err != nil {
+ return nil, err
+ }
+ if body.ID == "" {
+ return nil, fmt.Errorf("repo %s not found", data.Options.Project)
+ }
+ results := make([]interface{}, 0, 1)
+ azureRepository := &models.AzureRepo{
+ ConnectionId: data.Options.ConnectionId,
+ AzureId: body.ID,
+ Name: body.Name,
+ Url: body.URL,
+ ProjectId: body.Project.ID,
+ DefaultBranch: body.DefaultBranch,
+ Size: body.Size,
+ RemoteURL: body.RemoteURL,
+ SshUrl: body.SSHURL,
+ WebUrl: body.WebURL,
+ IsDisabled: body.IsDisabled,
+ }
+ data.Repo = azureRepository
+
+ results = append(results, azureRepository)
+
+ return results, nil
+ },
+ })
+
+ if err != nil {
+ return err
+ }
+
+ return extractor.Execute()
+}
diff --git a/plugins/azure/tasks/task_data.go b/plugins/azure/tasks/task_data.go
new file mode 100644
index 00000000..abcd12cf
--- /dev/null
+++ b/plugins/azure/tasks/task_data.go
@@ -0,0 +1,57 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "fmt"
+ "github.com/apache/incubator-devlake/plugins/azure/models"
+ "github.com/apache/incubator-devlake/plugins/helper"
+ "github.com/mitchellh/mapstructure"
+)
+
+type AzureApiParams struct {
+ ConnectionId uint64
+ Project string
+}
+
+type AzureOptions struct {
+ ConnectionId uint64 `json:"connectionId"`
+ Project string
+ Since string
+ Tasks []string `json:"tasks,omitempty"`
+}
+
+type AzureTaskData struct {
+ Options *AzureOptions
+ ApiClient *helper.ApiAsyncClient
+ Connection *models.AzureConnection
+ Repo *models.AzureRepo
+}
+
+func DecodeAndValidateTaskOptions(options map[string]interface{}) (*AzureOptions, error) {
+ var op AzureOptions
+ err := mapstructure.Decode(options, &op)
+ if err != nil {
+ return nil, err
+ }
+ // find the needed Azure now
+ if op.ConnectionId == 0 {
+ return nil, fmt.Errorf("connectionId is invalid")
+ }
+ return &op, nil
+}