You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by wa...@apache.org on 2022/06/16 05:02:36 UTC
[incubator-devlake] 01/05: feat: multi-data connections support for Jenkins
This is an automated email from the ASF dual-hosted git repository.
warren pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit 0e5fede4ef5f5a82a65d266f0f83f1a5eb1db8dd
Author: abeizn <zi...@merico.dev>
AuthorDate: Wed Jun 15 19:38:25 2022 +0800
feat: multi-data connections support for Jenkins
---
models/migrationscripts/register.go | 4 +-
.../migrationscripts/updateSchemas20220612.go | 36 +++++----
plugins/jenkins/api/connection.go | 93 +++++++++++-----------
.../jenkins/{tasks/task_data.go => api/init.go} | 29 +++----
plugins/jenkins/{jenkins.go => impl/impl.go} | 53 +++++++-----
plugins/jenkins/jenkins.go | 83 ++-----------------
plugins/jenkins/models/build.go | 30 +++----
plugins/jenkins/models/connection.go | 12 ++-
plugins/jenkins/models/job.go | 10 ++-
.../models/migrationscripts/archived/build.go | 15 ++--
.../{ => migrationscripts/archived}/connection.go | 33 ++++++--
.../models/migrationscripts/archived/job.go | 10 ++-
.../jenkins/models/migrationscripts/init_schema.go | 55 ++++++++++++-
plugins/jenkins/tasks/build_collector.go | 21 +++--
plugins/jenkins/tasks/build_convertor.go | 22 +++--
plugins/jenkins/tasks/build_extractor.go | 12 ++-
plugins/jenkins/tasks/client.go | 32 ++------
plugins/jenkins/tasks/job_collector.go | 11 ++-
plugins/jenkins/tasks/job_convertor.go | 24 ++++--
plugins/jenkins/tasks/job_extractor.go | 12 ++-
plugins/jenkins/tasks/task_data.go | 20 +++--
21 files changed, 334 insertions(+), 283 deletions(-)
diff --git a/models/migrationscripts/register.go b/models/migrationscripts/register.go
index 78fe8d13..d9adfcbb 100644
--- a/models/migrationscripts/register.go
+++ b/models/migrationscripts/register.go
@@ -26,7 +26,7 @@ func All() []migration.Script {
new(updateSchemas20220505), new(updateSchemas20220507), new(updateSchemas20220510),
new(updateSchemas20220513), new(updateSchemas20220524), new(updateSchemas20220526),
new(updateSchemas20220527), new(updateSchemas20220528), new(updateSchemas20220601),
- new(updateSchemas20220602), new(updateSchemas20220613), new(updateSchemas20220614),
- new(updateSchemas2022061402),
+ new(updateSchemas20220602), new(updateSchemas20220612), new(updateSchemas20220613),
+ new(updateSchemas20220614), new(updateSchemas2022061402),
}
}
diff --git a/plugins/jenkins/models/migrationscripts/archived/job.go b/models/migrationscripts/updateSchemas20220612.go
similarity index 61%
copy from plugins/jenkins/models/migrationscripts/archived/job.go
copy to models/migrationscripts/updateSchemas20220612.go
index 0dda257f..625d1bdb 100644
--- a/plugins/jenkins/models/migrationscripts/archived/job.go
+++ b/models/migrationscripts/updateSchemas20220612.go
@@ -5,9 +5,7 @@ this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
-
http://www.apache.org/licenses/LICENSE-2.0
-
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -15,26 +13,34 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
-package archived
+package migrationscripts
import (
+ "context"
+
"github.com/apache/incubator-devlake/models/migrationscripts/archived"
+ "gorm.io/gorm"
)
-// JenkinsJobProps current used jenkins job props
-type JenkinsJobProps struct {
- Name string `gorm:"primaryKey;type:varchar(255)"`
- Class string `gorm:"type:varchar(255)"`
- Color string `gorm:"type:varchar(255)"`
- Base string `gorm:"type:varchar(255)"`
+type updateSchemas20220612 struct{}
+
+func (*updateSchemas20220612) Up(ctx context.Context, db *gorm.DB) error {
+ err := db.Migrator().DropTable(&archived.Job{}, &archived.Build{})
+ if err != nil {
+ return err
+ }
+ err = db.Migrator().CreateTable(&archived.Job{}, &archived.Build{})
+ if err != nil {
+ return err
+ }
+
+ return nil
}
-// JenkinsJob db entity for jenkins job
-type JenkinsJob struct {
- JenkinsJobProps
- archived.NoPKModel
+func (*updateSchemas20220612) Version() uint64 {
+ return 20220612154333
}
-func (JenkinsJob) TableName() string {
- return "_tool_jenkins_jobs"
+func (*updateSchemas20220612) Name() string {
+ return "Regenerate tables jobs and builds"
}
diff --git a/plugins/jenkins/api/connection.go b/plugins/jenkins/api/connection.go
index 857589fd..bf446e8b 100644
--- a/plugins/jenkins/api/connection.go
+++ b/plugins/jenkins/api/connection.go
@@ -19,20 +19,17 @@ package api
import (
"fmt"
- "github.com/apache/incubator-devlake/config"
- "github.com/apache/incubator-devlake/plugins/jenkins/models"
"net/http"
"time"
+ "github.com/apache/incubator-devlake/plugins/jenkins/models"
+
"github.com/apache/incubator-devlake/plugins/core"
"github.com/apache/incubator-devlake/plugins/helper"
"github.com/apache/incubator-devlake/utils"
- "github.com/go-playground/validator/v10"
"github.com/mitchellh/mapstructure"
)
-var vld = validator.New()
-
func TestConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) {
// decode
var err error
@@ -41,7 +38,7 @@ func TestConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput, erro
if err != nil {
return nil, err
}
- // validate
+ // validateplugins/jenkins/tasks/job_collector.go
err = vld.Struct(connection)
if err != nil {
return nil, err
@@ -72,73 +69,77 @@ func TestConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput, erro
}
/*
-PATCH /plugins/jenkins/connections/:connectionId
+POST /plugins/jenkins/connections
{
- "Endpoint": "your-endpoint",
- "Username": "your-username",
- "Password": "your-password",
+ "name": "jenkins data connection name",
+ "endpoint": "jenkins api endpoint, i.e. https://ci.jenkins.io/",
+ "username": "username, usually should be email address",
+ "password": "jenkins api access token"
}
*/
-func PatchConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) {
- v := config.GetConfig()
+func PostConnections(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) {
+ // create a new connection
connection := &models.JenkinsConnection{}
- err := helper.EncodeStruct(v, connection, "env")
+
+ // update from request and save to database
+ err := connectionHelper.Create(connection, input)
if err != nil {
return nil, err
}
- // update from request and save to .env
- err = helper.DecodeStruct(v, connection, input.Body, "env")
+ return &core.ApiResourceOutput{Body: connection, Status: http.StatusOK}, nil
+}
+
+/*
+PATCH /plugins/jenkins/connections/connectionId
+{
+ "name": "jenkins data connection name",
+ "endpoint": "jenkins api endpoint, i.e. https://ci.jenkins.io/",
+ "username": "username, usually should be email address",
+ "password": "jenkins api access token"
+}
+*/
+
+func PatchConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) {
+ connection := &models.JenkinsConnection{}
+ err := connectionHelper.Patch(connection, input)
if err != nil {
return nil, err
}
- err = config.WriteConfig(v)
+
+ return &core.ApiResourceOutput{Body: connection}, nil
+}
+
+/*
+DELETE /plugins/jenkins/connections/connectionId
+*/
+func DeleteConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) {
+ connection := &models.JenkinsConnection{}
+ err := connectionHelper.First(connection, input.Params)
if err != nil {
return nil, err
}
- response := models.JenkinsResponse{
- JenkinsConnection: *connection,
- Name: "Jenkins",
- ID: 1,
- }
- return &core.ApiResourceOutput{Body: response, Status: http.StatusOK}, nil
+ err = connectionHelper.Delete(connection)
+ return &core.ApiResourceOutput{Body: connection}, err
}
/*
GET /plugins/jenkins/connections
*/
func ListConnections(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) {
- // RETURN ONLY 1 SOURCE (FROM ENV) until multi-connection is developed.
- v := config.GetConfig()
- connection := &models.JenkinsConnection{}
-
- err := helper.EncodeStruct(v, connection, "env")
+ var connections []models.JenkinsConnection
+ err := connectionHelper.List(&connections)
if err != nil {
return nil, err
}
- response := models.JenkinsResponse{
- JenkinsConnection: *connection,
- Name: "Jenkins",
- ID: 1,
- }
- return &core.ApiResourceOutput{Body: []models.JenkinsResponse{response}}, nil
+ return &core.ApiResourceOutput{Body: connections, Status: http.StatusOK}, nil
}
/*
-GET /plugins/jenkins/connections/:connectionId
+GET /plugins/jenkins/connections/connectionId
*/
func GetConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) {
- // RETURN ONLY 1 SOURCE FROM ENV (Ignore ID until multi-connection is developed.)
- v := config.GetConfig()
connection := &models.JenkinsConnection{}
- err := helper.EncodeStruct(v, connection, "env")
- if err != nil {
- return nil, err
- }
- response := &models.JenkinsResponse{
- JenkinsConnection: *connection,
- Name: "Jenkins",
- ID: 1,
- }
- return &core.ApiResourceOutput{Body: response}, nil
+ err := connectionHelper.First(connection, input.Params)
+ return &core.ApiResourceOutput{Body: connection}, err
}
diff --git a/plugins/jenkins/tasks/task_data.go b/plugins/jenkins/api/init.go
similarity index 63%
copy from plugins/jenkins/tasks/task_data.go
copy to plugins/jenkins/api/init.go
index ad76d098..6774e148 100644
--- a/plugins/jenkins/tasks/task_data.go
+++ b/plugins/jenkins/api/init.go
@@ -15,24 +15,25 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
-package tasks
+package api
import (
- "time"
-
+ "github.com/apache/incubator-devlake/plugins/core"
"github.com/apache/incubator-devlake/plugins/helper"
+ "github.com/go-playground/validator/v10"
+ "github.com/spf13/viper"
+ "gorm.io/gorm"
)
-type JenkinsOptions struct {
- Host string
- Username string
- Password string
- Since string
- Tasks []string `json:"tasks,omitempty"`
-}
+var vld *validator.Validate
+var connectionHelper *helper.ConnectionApiHelper
+var basicRes core.BasicRes
-type JenkinsTaskData struct {
- Options *JenkinsOptions
- ApiClient *helper.ApiAsyncClient
- Since *time.Time
+func Init(config *viper.Viper, logger core.Logger, database *gorm.DB) {
+ basicRes = helper.NewDefaultBasicRes(config, logger, database)
+ vld = validator.New()
+ connectionHelper = helper.NewConnectionHelper(
+ basicRes,
+ vld,
+ )
}
diff --git a/plugins/jenkins/jenkins.go b/plugins/jenkins/impl/impl.go
similarity index 75%
copy from plugins/jenkins/jenkins.go
copy to plugins/jenkins/impl/impl.go
index 4edc89df..c7bf3d89 100644
--- a/plugins/jenkins/jenkins.go
+++ b/plugins/jenkins/impl/impl.go
@@ -15,19 +15,19 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
-package main
+package impl
import (
"fmt"
"github.com/apache/incubator-devlake/migration"
"github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/helper"
"github.com/apache/incubator-devlake/plugins/jenkins/api"
+ "github.com/apache/incubator-devlake/plugins/jenkins/models"
"github.com/apache/incubator-devlake/plugins/jenkins/models/migrationscripts"
"github.com/apache/incubator-devlake/plugins/jenkins/tasks"
- "github.com/apache/incubator-devlake/runner"
"github.com/mitchellh/mapstructure"
- "github.com/spf13/cobra"
"github.com/spf13/viper"
"gorm.io/gorm"
)
@@ -41,6 +41,7 @@ var _ core.Migratable = (*Jenkins)(nil)
type Jenkins struct{}
func (plugin Jenkins) Init(config *viper.Viper, logger core.Logger, db *gorm.DB) error {
+ api.Init(config, logger, db)
return nil
}
@@ -64,13 +65,31 @@ func (plugin Jenkins) PrepareTaskData(taskCtx core.TaskContext, options map[stri
if err != nil {
return nil, fmt.Errorf("Failed to decode options: %v", err)
}
- apiClient, err := tasks.CreateApiClient(taskCtx)
+ if op.ConnectionId == 0 {
+ return nil, fmt.Errorf("connectionId is invalid")
+ }
+
+ connection := &models.JenkinsConnection{}
+ connectionHelper := helper.NewConnectionHelper(
+ taskCtx,
+ nil,
+ )
+ if err != nil {
+ return nil, err
+ }
+ err = connectionHelper.FirstById(connection, op.ConnectionId)
+ if err != nil {
+ return nil, err
+ }
+
+ apiClient, err := tasks.CreateApiClient(taskCtx, connection)
if err != nil {
return nil, err
}
return &tasks.JenkinsTaskData{
- Options: &op,
- ApiClient: apiClient,
+ Options: &op,
+ ApiClient: apiClient,
+ Connection: connection,
}, nil
}
@@ -79,7 +98,9 @@ func (plugin Jenkins) RootPkgPath() string {
}
func (plugin Jenkins) MigrationScripts() []migration.Script {
- return []migration.Script{new(migrationscripts.InitSchemas)}
+ return []migration.Script{
+ new(migrationscripts.InitSchemas),
+ }
}
func (plugin Jenkins) ApiResources() map[string]map[string]core.ApiResourceHandler {
@@ -88,21 +109,13 @@ func (plugin Jenkins) ApiResources() map[string]map[string]core.ApiResourceHandl
"POST": api.TestConnection,
},
"connections": {
- "GET": api.ListConnections,
+ "POST": api.PostConnections,
+ "GET": api.ListConnections,
},
"connections/:connectionId": {
- "GET": api.GetConnection,
- "PATCH": api.PatchConnection,
+ "PATCH": api.PatchConnection,
+ "DELETE": api.DeleteConnection,
+ "GET": api.GetConnection,
},
}
}
-
-var PluginEntry Jenkins //nolint
-
-func main() {
- jenkinsCmd := &cobra.Command{Use: "jenkins"}
- jenkinsCmd.Run = func(cmd *cobra.Command, args []string) {
- runner.DirectRun(cmd, args, PluginEntry, map[string]interface{}{})
- }
- runner.RunCmd(jenkinsCmd)
-}
diff --git a/plugins/jenkins/jenkins.go b/plugins/jenkins/jenkins.go
index 4edc89df..fcbd14b2 100644
--- a/plugins/jenkins/jenkins.go
+++ b/plugins/jenkins/jenkins.go
@@ -18,91 +18,20 @@ limitations under the License.
package main
import (
- "fmt"
-
- "github.com/apache/incubator-devlake/migration"
- "github.com/apache/incubator-devlake/plugins/core"
- "github.com/apache/incubator-devlake/plugins/jenkins/api"
- "github.com/apache/incubator-devlake/plugins/jenkins/models/migrationscripts"
- "github.com/apache/incubator-devlake/plugins/jenkins/tasks"
+ "github.com/apache/incubator-devlake/plugins/jenkins/impl"
"github.com/apache/incubator-devlake/runner"
- "github.com/mitchellh/mapstructure"
"github.com/spf13/cobra"
- "github.com/spf13/viper"
- "gorm.io/gorm"
)
-var _ core.PluginMeta = (*Jenkins)(nil)
-var _ core.PluginInit = (*Jenkins)(nil)
-var _ core.PluginTask = (*Jenkins)(nil)
-var _ core.PluginApi = (*Jenkins)(nil)
-var _ core.Migratable = (*Jenkins)(nil)
-
-type Jenkins struct{}
-
-func (plugin Jenkins) Init(config *viper.Viper, logger core.Logger, db *gorm.DB) error {
- return nil
-}
-
-func (plugin Jenkins) Description() string {
- return "To collect and enrich data from Jenkins"
-}
-
-func (plugin Jenkins) SubTaskMetas() []core.SubTaskMeta {
- return []core.SubTaskMeta{
- tasks.CollectApiJobsMeta,
- tasks.ExtractApiJobsMeta,
- tasks.CollectApiBuildsMeta,
- tasks.ExtractApiBuildsMeta,
- tasks.ConvertJobsMeta,
- tasks.ConvertBuildsMeta,
- }
-}
-func (plugin Jenkins) PrepareTaskData(taskCtx core.TaskContext, options map[string]interface{}) (interface{}, error) {
- var op tasks.JenkinsOptions
- err := mapstructure.Decode(options, &op)
- if err != nil {
- return nil, fmt.Errorf("Failed to decode options: %v", err)
- }
- apiClient, err := tasks.CreateApiClient(taskCtx)
- if err != nil {
- return nil, err
- }
- return &tasks.JenkinsTaskData{
- Options: &op,
- ApiClient: apiClient,
- }, nil
-}
-
-func (plugin Jenkins) RootPkgPath() string {
- return "github.com/apache/incubator-devlake/plugins/jenkins"
-}
-
-func (plugin Jenkins) MigrationScripts() []migration.Script {
- return []migration.Script{new(migrationscripts.InitSchemas)}
-}
-
-func (plugin Jenkins) ApiResources() map[string]map[string]core.ApiResourceHandler {
- return map[string]map[string]core.ApiResourceHandler{
- "test": {
- "POST": api.TestConnection,
- },
- "connections": {
- "GET": api.ListConnections,
- },
- "connections/:connectionId": {
- "GET": api.GetConnection,
- "PATCH": api.PatchConnection,
- },
- }
-}
-
-var PluginEntry Jenkins //nolint
+var PluginEntry impl.Jenkins
func main() {
jenkinsCmd := &cobra.Command{Use: "jenkins"}
+ connectionId := jenkinsCmd.Flags().Uint64P("connection", "c", 0, "jenkins connection id")
jenkinsCmd.Run = func(cmd *cobra.Command, args []string) {
- runner.DirectRun(cmd, args, PluginEntry, map[string]interface{}{})
+ runner.DirectRun(cmd, args, PluginEntry, map[string]interface{}{
+ "connectionId": *connectionId,
+ })
}
runner.RunCmd(jenkinsCmd)
}
diff --git a/plugins/jenkins/models/build.go b/plugins/jenkins/models/build.go
index 26c2339f..98d4b974 100644
--- a/plugins/jenkins/models/build.go
+++ b/plugins/jenkins/models/build.go
@@ -18,31 +18,23 @@ limitations under the License.
package models
import (
- "github.com/apache/incubator-devlake/models/common"
"time"
-)
-// JenkinsBuildProps current used jenkins build props
-type JenkinsBuildProps struct {
- Duration float64 // build time
- DisplayName string // "#7"
- EstimatedDuration float64
- Number int64
- Result string
- Timestamp int64 // start time
- StartTime time.Time // convered by timestamp
- CommitSha string
-}
+ "github.com/apache/incubator-devlake/models/common"
+)
// JenkinsBuild db entity for jenkins build
type JenkinsBuild struct {
common.NoPKModel
- JobName string `gorm:"primaryKey;type:varchar(255)"`
- Duration float64 // build time
- DisplayName string `gorm:"type:varchar(255)"` // "#7"
- EstimatedDuration float64
- Number int64 `gorm:"primaryKey"`
- Result string
+
+ // collected fields
+ ConnectionId uint64 `gorm:"primaryKey"`
+ JobName string `gorm:"primaryKey;type:varchar(255)"`
+ Duration float64 // build time
+ DisplayName string `gorm:"type:varchar(255)"` // "#7"
+ EstimatedDuration float64 // EstimatedDuration
+ Number int64 `gorm:"primaryKey"`
+ Result string // Result
Timestamp int64 // start time
StartTime time.Time // convered by timestamp
CommitSha string `gorm:"type:varchar(255)"`
diff --git a/plugins/jenkins/models/connection.go b/plugins/jenkins/models/connection.go
index 2049fece..aa030327 100644
--- a/plugins/jenkins/models/connection.go
+++ b/plugins/jenkins/models/connection.go
@@ -17,12 +17,12 @@ limitations under the License.
package models
+import "github.com/apache/incubator-devlake/plugins/helper"
+
// This object conforms to what the frontend currently sends.
type JenkinsConnection struct {
- Endpoint string `mapstructure:"endpoint" validate:"required" env:"JENKINS_ENDPOINT" json:"endpoint"`
- Username string `mapstructure:"username" validate:"required" env:"JENKINS_USERNAME" json:"username"`
- Password string `mapstructure:"password" validate:"required" env:"JENKINS_PASSWORD" json:"password"`
- Proxy string `mapstructure:"proxy" env:"JENKINS_PROXY" json:"proxy"`
+ helper.RestConnection `mapstructure:",squash"`
+ helper.BasicAuth `mapstructure:",squash"`
}
type JenkinsResponse struct {
@@ -37,3 +37,7 @@ type TestConnectionRequest struct {
Password string `json:"password" validate:"required"`
Proxy string `json:"proxy"`
}
+
+func (JenkinsConnection) TableName() string {
+ return "_tool_jenkins_connections"
+}
diff --git a/plugins/jenkins/models/job.go b/plugins/jenkins/models/job.go
index 961e55fe..dbac6366 100644
--- a/plugins/jenkins/models/job.go
+++ b/plugins/jenkins/models/job.go
@@ -21,10 +21,12 @@ import "github.com/apache/incubator-devlake/models/common"
// JenkinsJobProps current used jenkins job props
type JenkinsJobProps struct {
- Name string `gorm:"primaryKey;type:varchar(255)"`
- Class string `gorm:"type:varchar(255)"`
- Color string `gorm:"type:varchar(255)"`
- Base string `gorm:"type:varchar(255)"`
+ // collected fields
+ ConnectionId uint64 `gorm:"primaryKey"`
+ Name string `gorm:"primaryKey;type:varchar(255)"`
+ Class string `gorm:"type:varchar(255)"`
+ Color string `gorm:"type:varchar(255)"`
+ Base string `gorm:"type:varchar(255)"`
}
// JenkinsJob db entity for jenkins job
diff --git a/plugins/jenkins/models/migrationscripts/archived/build.go b/plugins/jenkins/models/migrationscripts/archived/build.go
index f0a133a3..39dfd31c 100644
--- a/plugins/jenkins/models/migrationscripts/archived/build.go
+++ b/plugins/jenkins/models/migrationscripts/archived/build.go
@@ -26,12 +26,15 @@ import (
// JenkinsBuild db entity for jenkins build
type JenkinsBuild struct {
archived.NoPKModel
- JobName string `gorm:"primaryKey;type:varchar(255)"`
- Duration float64 // build time
- DisplayName string `gorm:"type:varchar(255)"` // "#7"
- EstimatedDuration float64
- Number int64 `gorm:"primaryKey"`
- Result string
+
+ // collected fields
+ ConnectionId uint64 `gorm:"primaryKey"`
+ JobName string `gorm:"primaryKey;type:varchar(255)"`
+ Duration float64 // build time
+ DisplayName string `gorm:"type:varchar(255)"`
+ EstimatedDuration float64 // EstimatedDuration
+ Number int64 `gorm:"primaryKey"`
+ Result string // Result
Timestamp int64 // start time
StartTime time.Time // convered by timestamp
CommitSha string `gorm:"type:varchar(255)"`
diff --git a/plugins/jenkins/models/connection.go b/plugins/jenkins/models/migrationscripts/archived/connection.go
similarity index 57%
copy from plugins/jenkins/models/connection.go
copy to plugins/jenkins/models/migrationscripts/archived/connection.go
index 2049fece..5dc0990c 100644
--- a/plugins/jenkins/models/connection.go
+++ b/plugins/jenkins/models/migrationscripts/archived/connection.go
@@ -15,14 +15,33 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
-package models
+package archived
+
+import (
+ "github.com/apache/incubator-devlake/models/migrationscripts/archived"
+)
+
+type BaseConnection struct {
+ Name string `gorm:"type:varchar(100);uniqueIndex" json:"name" validate:"required"`
+ archived.Model
+}
+
+type BasicAuth struct {
+ Username string `mapstructure:"username" validate:"required" json:"username"`
+ Password string `mapstructure:"password" validate:"required" json:"password" encrypt:"yes"`
+}
+
+type RestConnection struct {
+ BaseConnection `mapstructure:",squash"`
+ Endpoint string `mapstructure:"endpoint" validate:"required" json:"endpoint"`
+ Proxy string `mapstructure:"proxy" json:"proxy"`
+ RateLimit int `comment:"api request rate limt per hour" json:"rateLimit"`
+}
// This object conforms to what the frontend currently sends.
type JenkinsConnection struct {
- Endpoint string `mapstructure:"endpoint" validate:"required" env:"JENKINS_ENDPOINT" json:"endpoint"`
- Username string `mapstructure:"username" validate:"required" env:"JENKINS_USERNAME" json:"username"`
- Password string `mapstructure:"password" validate:"required" env:"JENKINS_PASSWORD" json:"password"`
- Proxy string `mapstructure:"proxy" env:"JENKINS_PROXY" json:"proxy"`
+ RestConnection `mapstructure:",squash"`
+ BasicAuth `mapstructure:",squash"`
}
type JenkinsResponse struct {
@@ -37,3 +56,7 @@ type TestConnectionRequest struct {
Password string `json:"password" validate:"required"`
Proxy string `json:"proxy"`
}
+
+func (JenkinsConnection) TableName() string {
+ return "_tool_jenkins_connections"
+}
diff --git a/plugins/jenkins/models/migrationscripts/archived/job.go b/plugins/jenkins/models/migrationscripts/archived/job.go
index 0dda257f..95310876 100644
--- a/plugins/jenkins/models/migrationscripts/archived/job.go
+++ b/plugins/jenkins/models/migrationscripts/archived/job.go
@@ -23,10 +23,12 @@ import (
// JenkinsJobProps current used jenkins job props
type JenkinsJobProps struct {
- Name string `gorm:"primaryKey;type:varchar(255)"`
- Class string `gorm:"type:varchar(255)"`
- Color string `gorm:"type:varchar(255)"`
- Base string `gorm:"type:varchar(255)"`
+ // collected fields
+ ConnectionId uint64 `gorm:"primaryKey"`
+ Name string `gorm:"primaryKey;type:varchar(255)"`
+ Class string `gorm:"type:varchar(255)"`
+ Color string `gorm:"type:varchar(255)"`
+ Base string `gorm:"type:varchar(255)"`
}
// JenkinsJob db entity for jenkins job
diff --git a/plugins/jenkins/models/migrationscripts/init_schema.go b/plugins/jenkins/models/migrationscripts/init_schema.go
index 9fcdcc06..b9465976 100644
--- a/plugins/jenkins/models/migrationscripts/init_schema.go
+++ b/plugins/jenkins/models/migrationscripts/init_schema.go
@@ -19,22 +19,73 @@ package migrationscripts
import (
"context"
+ "fmt"
+ "github.com/apache/incubator-devlake/config"
+ "github.com/apache/incubator-devlake/plugins/core"
"github.com/apache/incubator-devlake/plugins/jenkins/models/migrationscripts/archived"
"gorm.io/gorm"
+ "gorm.io/gorm/clause"
)
type InitSchemas struct{}
func (*InitSchemas) Up(ctx context.Context, db *gorm.DB) error {
- return db.Migrator().AutoMigrate(
+
+ rawTableList := []string{
+ "_raw_jenkins_api_jobs",
+ "_raw_jenkins_api_builds",
+ }
+ for _, v := range rawTableList {
+ err := db.Exec(fmt.Sprintf("DROP TABLE IF EXISTS %s CASCADE", v)).Error
+ if err != nil {
+ return err
+ }
+ }
+
+ err := db.Migrator().DropTable(
+ &archived.JenkinsJob{},
+ &archived.JenkinsBuild{},
+ )
+ if err != nil {
+ return err
+ }
+
+ err = db.Migrator().AutoMigrate(
&archived.JenkinsJob{},
&archived.JenkinsBuild{},
+ &archived.JenkinsConnection{},
)
+ if err != nil {
+ return err
+ }
+
+ conn := &archived.JenkinsConnection{}
+ v := config.GetConfig()
+
+ conn.Name = "init jenkins connection"
+ conn.ID = 1
+ conn.Endpoint = v.GetString("JENKINS_ENDPOINT")
+ conn.Proxy = v.GetString("JENKINS_PROXY")
+ conn.RateLimit = v.GetInt("JENKINS_API_REQUESTS_PER_HOUR")
+ conn.Username = v.GetString("JENKINS_USERNAME")
+ encKey := v.GetString(core.EncodeKeyEnvStr)
+ conn.Password, err = core.Encrypt(encKey, v.GetString("JENKINS_PASSWORD"))
+ if err != nil {
+ return err
+ }
+ err = db.Clauses(clause.OnConflict{DoNothing: true}).Create(conn).Error
+
+ if err != nil {
+ return err
+ }
+
+ return nil
+
}
func (*InitSchemas) Version() uint64 {
- return 20220407201137
+ return 20220614201236
}
func (*InitSchemas) Name() string {
diff --git a/plugins/jenkins/tasks/build_collector.go b/plugins/jenkins/tasks/build_collector.go
index 9a3339d2..ca711333 100644
--- a/plugins/jenkins/tasks/build_collector.go
+++ b/plugins/jenkins/tasks/build_collector.go
@@ -25,8 +25,8 @@ import (
"reflect"
"github.com/apache/incubator-devlake/plugins/core"
+ . "github.com/apache/incubator-devlake/plugins/core/dal"
"github.com/apache/incubator-devlake/plugins/helper"
- "github.com/apache/incubator-devlake/plugins/jenkins/models"
)
const RAW_BUILD_TABLE = "jenkins_api_builds"
@@ -43,19 +43,30 @@ type SimpleJob struct {
}
func CollectApiBuilds(taskCtx core.SubTaskContext) error {
- db := taskCtx.GetDb()
+ db := taskCtx.GetDal()
data := taskCtx.GetData().(*JenkinsTaskData)
- cursor, err := db.Model(&models.JenkinsJob{}).Select("name").Rows()
+ clauses := []Clause{
+ Select("tjj.name"),
+ From("_tool_jenkins_jobs tjj"),
+ Where(`tjj.connection_id = ?`, data.Options.ConnectionId),
+ }
+
+ cursor, err := db.Cursor(clauses...)
if err != nil {
return err
}
- iterator, err := helper.NewCursorIterator(db, cursor, reflect.TypeOf(SimpleJob{}))
+ defer cursor.Close()
+
+ iterator, err := helper.NewDalCursorIterator(db, cursor, reflect.TypeOf(SimpleJob{}))
if err != nil {
return err
}
collector, err := helper.NewApiCollector(helper.ApiCollectorArgs{
RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
+ Params: JenkinsApiParams{
+ ConnectionId: data.Connection.ID,
+ },
Ctx: taskCtx,
Table: RAW_BUILD_TABLE,
},
@@ -70,7 +81,7 @@ func CollectApiBuilds(taskCtx core.SubTaskContext) error {
query := url.Values{}
treeValue := fmt.Sprintf(
"allBuilds[number,timestamp,duration,estimatedDuration,displayName,result,actions[lastBuiltRevision[SHA1],mercurialRevisionNumber],changeSet[kind,revisions[revision]]]{%d,%d}",
- reqData.Pager.Skip, reqData.Pager.Skip+reqData.Pager.Size)
+ reqData.Pager.Skip+1, reqData.Pager.Skip+reqData.Pager.Size)
query.Set("tree", treeValue)
return query, nil
},
diff --git a/plugins/jenkins/tasks/build_convertor.go b/plugins/jenkins/tasks/build_convertor.go
index be4687ed..5f12ef47 100644
--- a/plugins/jenkins/tasks/build_convertor.go
+++ b/plugins/jenkins/tasks/build_convertor.go
@@ -18,13 +18,15 @@ limitations under the License.
package tasks
import (
+ "reflect"
+
"github.com/apache/incubator-devlake/models/domainlayer"
"github.com/apache/incubator-devlake/models/domainlayer/devops"
"github.com/apache/incubator-devlake/models/domainlayer/didgen"
"github.com/apache/incubator-devlake/plugins/core"
+ . "github.com/apache/incubator-devlake/plugins/core/dal"
"github.com/apache/incubator-devlake/plugins/helper"
models "github.com/apache/incubator-devlake/plugins/jenkins/models"
- "reflect"
)
var ConvertBuildsMeta = core.SubTaskMeta{
@@ -35,11 +37,14 @@ var ConvertBuildsMeta = core.SubTaskMeta{
}
func ConvertBuilds(taskCtx core.SubTaskContext) error {
- db := taskCtx.GetDb()
+ db := taskCtx.GetDal()
+ data := taskCtx.GetData().(*JenkinsTaskData)
- jenkinsBuild := &models.JenkinsBuild{}
-
- cursor, err := db.Model(jenkinsBuild).Rows()
+ clauses := []Clause{
+ Select("*"),
+ From("_tool_jenkins_builds"),
+ }
+ cursor, err := db.Cursor(clauses...)
if err != nil {
return err
}
@@ -52,6 +57,9 @@ func ConvertBuilds(taskCtx core.SubTaskContext) error {
InputRowType: reflect.TypeOf(models.JenkinsBuild{}),
Input: cursor,
RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
+ Params: JenkinsApiParams{
+ ConnectionId: data.Connection.ID,
+ },
Ctx: taskCtx,
Table: RAW_BUILD_TABLE,
},
@@ -59,9 +67,9 @@ func ConvertBuilds(taskCtx core.SubTaskContext) error {
jenkinsBuild := inputRow.(*models.JenkinsBuild)
build := &devops.Build{
DomainEntity: domainlayer.DomainEntity{
- Id: buildIdGen.Generate(jenkinsBuild.JobName, jenkinsBuild.Number),
+ Id: buildIdGen.Generate(jenkinsBuild.ConnectionId, jenkinsBuild.JobName, jenkinsBuild.Number),
},
- JobId: jobIdGen.Generate(jenkinsBuild.JobName),
+ JobId: jobIdGen.Generate(jenkinsBuild.ConnectionId, jenkinsBuild.JobName),
Name: jenkinsBuild.DisplayName,
DurationSec: uint64(jenkinsBuild.Duration / 1000),
Status: jenkinsBuild.Result,
diff --git a/plugins/jenkins/tasks/build_extractor.go b/plugins/jenkins/tasks/build_extractor.go
index 514b5155..ac57728e 100644
--- a/plugins/jenkins/tasks/build_extractor.go
+++ b/plugins/jenkins/tasks/build_extractor.go
@@ -19,11 +19,12 @@ package tasks
import (
"encoding/json"
+ "strconv"
+ "time"
+
"github.com/apache/incubator-devlake/plugins/core"
"github.com/apache/incubator-devlake/plugins/helper"
"github.com/apache/incubator-devlake/plugins/jenkins/models"
- "strconv"
- "time"
)
// this struct should be moved to `gitub_api_common.go`
@@ -36,8 +37,12 @@ var ExtractApiBuildsMeta = core.SubTaskMeta{
}
func ExtractApiBuilds(taskCtx core.SubTaskContext) error {
+ data := taskCtx.GetData().(*JenkinsTaskData)
extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{
RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
+ Params: JenkinsApiParams{
+ ConnectionId: data.Connection.ID,
+ },
Ctx: taskCtx,
/*
This struct will be JSONEncoded and stored into database along with raw data itself, to identity minimal
@@ -54,7 +59,6 @@ func ExtractApiBuilds(taskCtx core.SubTaskContext) error {
if err != nil {
return nil, err
}
-
input := &SimpleJob{}
err = json.Unmarshal(row.Input, input)
if err != nil {
@@ -64,6 +68,7 @@ func ExtractApiBuilds(taskCtx core.SubTaskContext) error {
results := make([]interface{}, 0, 1)
build := &models.JenkinsBuild{
+ ConnectionId: data.Connection.ID,
JobName: input.Name,
Duration: body.Duration,
DisplayName: body.DisplayName,
@@ -73,7 +78,6 @@ func ExtractApiBuilds(taskCtx core.SubTaskContext) error {
Timestamp: body.Timestamp,
StartTime: time.Unix(body.Timestamp/1000, 0),
}
-
vcs := body.ChangeSet.Kind
if vcs == "git" || vcs == "hg" {
for _, a := range body.Actions {
diff --git a/plugins/jenkins/tasks/client.go b/plugins/jenkins/tasks/client.go
index 09a8ceb3..059bfdbd 100644
--- a/plugins/jenkins/tasks/client.go
+++ b/plugins/jenkins/tasks/client.go
@@ -23,47 +23,31 @@ import (
"github.com/apache/incubator-devlake/plugins/core"
"github.com/apache/incubator-devlake/plugins/helper"
- "github.com/apache/incubator-devlake/utils"
+ "github.com/apache/incubator-devlake/plugins/jenkins/models"
)
-func CreateApiClient(taskCtx core.TaskContext) (*helper.ApiAsyncClient, error) {
- // load configuration
- endpoint := taskCtx.GetConfig("JENKINS_ENDPOINT")
- if endpoint == "" {
- return nil, fmt.Errorf("JENKINS_ENDPOINT is required")
- }
- userRateLimit, err := utils.StrToIntOr(taskCtx.GetConfig("JENKINS_API_REQUESTS_PER_HOUR"), 0)
- if err != nil {
- return nil, err
- }
- username := taskCtx.GetConfig("JENKINS_USERNAME")
- if username == "" {
- return nil, fmt.Errorf("JENKINS_USERNAME is required")
- }
- password := taskCtx.GetConfig("JENKINS_PASSWORD")
- if password == "" {
- return nil, fmt.Errorf("JENKINS_PASSWORD is required")
- }
- encodedToken := utils.GetEncodedToken(username, password)
- proxy := taskCtx.GetConfig("JENKINS_PROXY")
+func CreateApiClient(taskCtx core.TaskContext, connection *models.JenkinsConnection) (*helper.ApiAsyncClient, error) {
// create synchronize api client so we can calculate api rate limit dynamically
headers := map[string]string{
- "Authorization": fmt.Sprintf("Basic %v", encodedToken),
+ "Authorization": fmt.Sprintf("Basic %v", connection.GetEncodedToken()),
}
- apiClient, err := helper.NewApiClient(endpoint, headers, 0, proxy, taskCtx.GetContext())
+
+ apiClient, err := helper.NewApiClient(connection.Endpoint, headers, 0, connection.Proxy, taskCtx.GetContext())
if err != nil {
return nil, err
}
+
apiClient.SetAfterFunction(func(res *http.Response) error {
if res.StatusCode == http.StatusUnauthorized {
return fmt.Errorf("authentication failed, please check your Username/Password")
}
return nil
})
+
// create rate limit calculator
rateLimiter := &helper.ApiRateLimitCalculator{
- UserRateLimitPerHour: userRateLimit,
+ UserRateLimitPerHour: connection.RateLimit,
}
asyncApiClient, err := helper.CreateAsyncApiClient(
taskCtx,
diff --git a/plugins/jenkins/tasks/job_collector.go b/plugins/jenkins/tasks/job_collector.go
index bfad4879..a1dcbb36 100644
--- a/plugins/jenkins/tasks/job_collector.go
+++ b/plugins/jenkins/tasks/job_collector.go
@@ -37,15 +37,13 @@ var CollectApiJobsMeta = core.SubTaskMeta{
}
func CollectApiJobs(taskCtx core.SubTaskContext) error {
- //db := taskCtx.GetDb()
data := taskCtx.GetData().(*JenkinsTaskData)
-
- //since := data.Since
incremental := false
- // user didn't specify a time range to sync, try load from database
-
collector, err := helper.NewApiCollector(helper.ApiCollectorArgs{
RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
+ Params: JenkinsApiParams{
+ ConnectionId: data.Connection.ID,
+ },
Ctx: taskCtx,
/*
This struct will be JSONEncoded and stored into database along with raw data itself, to identity minimal
@@ -59,6 +57,7 @@ func CollectApiJobs(taskCtx core.SubTaskContext) error {
ApiClient: data.ApiClient,
PageSize: 100,
Incremental: incremental,
+ // jenkins api is special, 1. If the concurrency is larger than 1, then it will report 500.
Concurrency: 1,
UrlTemplate: "api/json",
@@ -66,7 +65,7 @@ func CollectApiJobs(taskCtx core.SubTaskContext) error {
query := url.Values{}
treeValue := fmt.Sprintf(
"jobs[name,class,color,base]{%d,%d}",
- reqData.Pager.Skip, reqData.Pager.Skip+reqData.Pager.Size)
+ reqData.Pager.Skip+1, reqData.Pager.Skip+reqData.Pager.Size)
query.Set("tree", treeValue)
return query, nil
},
diff --git a/plugins/jenkins/tasks/job_convertor.go b/plugins/jenkins/tasks/job_convertor.go
index ceac1687..25461a2a 100644
--- a/plugins/jenkins/tasks/job_convertor.go
+++ b/plugins/jenkins/tasks/job_convertor.go
@@ -18,13 +18,15 @@ limitations under the License.
package tasks
import (
+ "reflect"
+
"github.com/apache/incubator-devlake/models/domainlayer"
"github.com/apache/incubator-devlake/models/domainlayer/devops"
"github.com/apache/incubator-devlake/models/domainlayer/didgen"
"github.com/apache/incubator-devlake/plugins/core"
+ . "github.com/apache/incubator-devlake/plugins/core/dal"
"github.com/apache/incubator-devlake/plugins/helper"
"github.com/apache/incubator-devlake/plugins/jenkins/models"
- "reflect"
)
var ConvertJobsMeta = core.SubTaskMeta{
@@ -35,22 +37,28 @@ var ConvertJobsMeta = core.SubTaskMeta{
}
func ConvertJobs(taskCtx core.SubTaskContext) error {
- db := taskCtx.GetDb()
-
- jenkinsJob := &models.JenkinsJob{}
+ db := taskCtx.GetDal()
+ data := taskCtx.GetData().(*JenkinsTaskData)
- jobIdGen := didgen.NewDomainIdGenerator(jenkinsJob)
-
- cursor, err := db.Model(jenkinsJob).Rows()
+ clauses := []Clause{
+ Select("*"),
+ From("_tool_jenkins_jobs"),
+ }
+ cursor, err := db.Cursor(clauses...)
if err != nil {
return err
}
defer cursor.Close()
+ jobIdGen := didgen.NewDomainIdGenerator(&models.JenkinsJob{})
+
converter, err := helper.NewDataConverter(helper.DataConverterArgs{
InputRowType: reflect.TypeOf(models.JenkinsJob{}),
Input: cursor,
RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
+ Params: JenkinsApiParams{
+ ConnectionId: data.Connection.ID,
+ },
Ctx: taskCtx,
Table: RAW_JOB_TABLE,
},
@@ -58,7 +66,7 @@ func ConvertJobs(taskCtx core.SubTaskContext) error {
jenkinsJob := inputRow.(*models.JenkinsJob)
job := &devops.Job{
DomainEntity: domainlayer.DomainEntity{
- Id: jobIdGen.Generate(jenkinsJob.Name),
+ Id: jobIdGen.Generate(jenkinsJob.ConnectionId, jenkinsJob.Name),
},
Name: jenkinsJob.Name,
}
diff --git a/plugins/jenkins/tasks/job_extractor.go b/plugins/jenkins/tasks/job_extractor.go
index 56075a8b..762aa37d 100644
--- a/plugins/jenkins/tasks/job_extractor.go
+++ b/plugins/jenkins/tasks/job_extractor.go
@@ -19,6 +19,7 @@ package tasks
import (
"encoding/json"
+
"github.com/apache/incubator-devlake/plugins/core"
"github.com/apache/incubator-devlake/plugins/helper"
"github.com/apache/incubator-devlake/plugins/jenkins/models"
@@ -34,8 +35,12 @@ var ExtractApiJobsMeta = core.SubTaskMeta{
}
func ExtractApiJobs(taskCtx core.SubTaskContext) error {
+ data := taskCtx.GetData().(*JenkinsTaskData)
extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{
RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
+ Params: JenkinsApiParams{
+ ConnectionId: data.Connection.ID,
+ },
Ctx: taskCtx,
/*
This struct will be JSONEncoded and stored into database along with raw data itself, to identity minimal
@@ -56,9 +61,10 @@ func ExtractApiJobs(taskCtx core.SubTaskContext) error {
job := &models.JenkinsJob{
JenkinsJobProps: models.JenkinsJobProps{
- Name: body.Name,
- Class: body.Class,
- Color: body.Color,
+ ConnectionId: data.Connection.ID,
+ Name: body.Name,
+ Class: body.Class,
+ Color: body.Color,
},
}
results = append(results, job)
diff --git a/plugins/jenkins/tasks/task_data.go b/plugins/jenkins/tasks/task_data.go
index ad76d098..1ee92532 100644
--- a/plugins/jenkins/tasks/task_data.go
+++ b/plugins/jenkins/tasks/task_data.go
@@ -21,18 +21,22 @@ import (
"time"
"github.com/apache/incubator-devlake/plugins/helper"
+ "github.com/apache/incubator-devlake/plugins/jenkins/models"
)
+type JenkinsApiParams struct {
+ ConnectionId uint64
+}
+
type JenkinsOptions struct {
- Host string
- Username string
- Password string
- Since string
- Tasks []string `json:"tasks,omitempty"`
+ ConnectionId uint64 `json:"connectionId"`
+ Since string
+ Tasks []string `json:"tasks,omitempty"`
}
type JenkinsTaskData struct {
- Options *JenkinsOptions
- ApiClient *helper.ApiAsyncClient
- Since *time.Time
+ Options *JenkinsOptions
+ ApiClient *helper.ApiAsyncClient
+ Connection *models.JenkinsConnection
+ Since *time.Time
}