You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by zh...@apache.org on 2023/05/31 04:03:16 UTC
[incubator-devlake] branch main updated: refactor: rename transformation rule to scope config - jenkins (#5314)
This is an automated email from the ASF dual-hosted git repository.
zhangliang2022 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 1e3bc0df1 refactor: rename transformation rule to scope config - jenkins (#5314)
1e3bc0df1 is described below
commit 1e3bc0df1fcbd6f7ee52894ab5f1d431385ae1d2
Author: Klesh Wong <zh...@merico.dev>
AuthorDate: Wed May 31 12:03:12 2023 +0800
refactor: rename transformation rule to scope config - jenkins (#5314)
---
backend/plugins/jenkins/api/blueprint_v100.go | 145 ---------------------
backend/plugins/jenkins/api/blueprint_v100_test.go | 126 ------------------
backend/plugins/jenkins/api/connection.go | 43 ------
backend/plugins/jenkins/api/init.go | 10 +-
backend/plugins/jenkins/api/scope.go | 2 +-
backend/plugins/jenkins/api/scope_config.go | 83 ++++++++++++
.../register.go => api/swagger.go} | 27 ++--
backend/plugins/jenkins/api/transformation_rule.go | 83 ------------
backend/plugins/jenkins/e2e/jobs_test.go | 13 +-
backend/plugins/jenkins/impl/impl.go | 70 +++++-----
backend/plugins/jenkins/models/job.go | 2 +-
...22_add_connection_id_to_transformation_rules.go | 23 +++-
.../migrationscripts/20230530_scope_config.go | 56 ++++++++
.../jenkins/models/migrationscripts/register.go | 1 +
.../{transformation_rule.go => scope_config.go} | 16 +--
backend/plugins/jenkins/tasks/task_data.go | 22 ++--
16 files changed, 233 insertions(+), 489 deletions(-)
diff --git a/backend/plugins/jenkins/api/blueprint_v100.go b/backend/plugins/jenkins/api/blueprint_v100.go
deleted file mode 100644
index fa7985ce9..000000000
--- a/backend/plugins/jenkins/api/blueprint_v100.go
+++ /dev/null
@@ -1,145 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package api
-
-import (
- "context"
- "encoding/json"
-
- "github.com/apache/incubator-devlake/plugins/jenkins/models"
-
- "github.com/apache/incubator-devlake/core/errors"
- "github.com/apache/incubator-devlake/core/plugin"
- helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
- aha "github.com/apache/incubator-devlake/helpers/pluginhelper/api/apihelperabstract"
- "github.com/apache/incubator-devlake/plugins/jenkins/tasks"
-)
-
-func MakePipelinePlanV100(subtaskMetas []plugin.SubTaskMeta, connectionId uint64, scope []*plugin.BlueprintScopeV100) (plugin.PipelinePlan, errors.Error) {
- var err errors.Error
- connection := new(models.JenkinsConnection)
- err = connectionHelper.FirstById(connection, connectionId)
- if err != nil {
- return nil, err
- }
-
- apiClient, err := helper.NewApiClientFromConnection(context.TODO(), basicRes, connection)
- if err != nil {
- return nil, err
- }
-
- plan, err := makePipelinePlanV100(subtaskMetas, scope, connection, apiClient)
- if err != nil {
- return nil, err
- }
- return plan, nil
-}
-
-func makePipelinePlanV100(
- subtaskMetas []plugin.SubTaskMeta,
- scope []*plugin.BlueprintScopeV100,
- connection *models.JenkinsConnection,
- apiClient aha.ApiClientAbstract,
-) (plugin.PipelinePlan, errors.Error) {
- var err errors.Error
- plans := make(plugin.PipelinePlan, 0, len(scope))
-
- if err != nil {
- return nil, err
- }
-
- for _, scopeElem := range scope {
- // handle taskOptions and transformationRules, by dumping them to taskOptions
- transformationRules := make(map[string]interface{})
- if len(scopeElem.Transformation) > 0 {
- err = errors.Convert(json.Unmarshal(scopeElem.Transformation, &transformationRules))
- if err != nil {
- return nil, err
- }
- }
-
- // check productionPattern and separate it from transformationRules
- productionPattern, ok := transformationRules["productionPattern"]
- if ok && productionPattern != nil {
- delete(transformationRules, "productionPattern")
- } else {
- productionPattern = nil
- }
-
- err = GetAllJobs(apiClient, "", "", 100, func(job *models.Job, isPath bool) errors.Error {
- // do not mind path
- if isPath {
- return nil
- }
-
- taskOptions := make(map[string]interface{})
- err = errors.Convert(json.Unmarshal(scopeElem.Options, &taskOptions))
- if err != nil {
- return err
- }
- taskOptions["connectionId"] = connection.ID
- taskOptions["transformationRules"] = transformationRules
- taskOptions["jobFullName"] = job.FullName
-
- op, err := tasks.DecodeTaskOptions(taskOptions)
- if err != nil {
- return err
- }
- _, err = tasks.ValidateTaskOptions(op)
- if err != nil {
- return err
- }
-
- // subtasks
- subtasks, err := helper.MakePipelinePlanSubtasks(subtaskMetas, scopeElem.Entities)
- if err != nil {
- return err
- }
- stage := plugin.PipelineStage{
- {
- Plugin: "jenkins",
- Subtasks: subtasks,
- Options: taskOptions,
- },
- }
-
- plans = append(plans, stage)
-
- return nil
- })
- if err != nil {
- return nil, err
- }
- // This is just to add a dora subtask, then we can add another two subtasks at the end of plans
- // The only purpose is to adapt old blueprints
- // DEPRECATED, will be removed in v0.17
- // Dora part
- if productionPattern != nil {
- stageDora := plugin.PipelineStage{
- {
- Plugin: "dora",
- Subtasks: []string{"EnrichTaskEnv"},
- Options: map[string]interface{}{},
- },
- }
-
- plans = append(plans, stageDora)
- }
- }
- return plans, nil
-}
diff --git a/backend/plugins/jenkins/api/blueprint_v100_test.go b/backend/plugins/jenkins/api/blueprint_v100_test.go
deleted file mode 100644
index 5c82444f2..000000000
--- a/backend/plugins/jenkins/api/blueprint_v100_test.go
+++ /dev/null
@@ -1,126 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package api
-
-import (
- "bytes"
- "encoding/json"
- "io"
- "net/http"
- "testing"
-
- mockaha "github.com/apache/incubator-devlake/mocks/helpers/pluginhelper/api/apihelperabstract"
-
- "github.com/apache/incubator-devlake/core/models/common"
- "github.com/apache/incubator-devlake/core/plugin"
- helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
- "github.com/apache/incubator-devlake/plugins/jenkins/models"
- "github.com/stretchr/testify/assert"
- "github.com/stretchr/testify/mock"
-)
-
-func TestProcessScope(t *testing.T) {
- connection := &models.JenkinsConnection{
- BaseConnection: helper.BaseConnection{
- Name: "jenkins-test",
- Model: common.Model{
- ID: 1,
- },
- },
- JenkinsConn: models.JenkinsConn{
- RestConnection: helper.RestConnection{
- Endpoint: "https://api.github.com/",
- Proxy: "",
- RateLimitPerHour: 0,
- },
- BasicAuth: helper.BasicAuth{
- Username: "Username",
- Password: "Password",
- },
- },
- }
-
- bs := &plugin.BlueprintScopeV100{
- Entities: []string{"CICD"},
- Options: json.RawMessage(`{}`),
- Transformation: json.RawMessage(`{
- "productionPattern": "(?i)build-and-deploy",
- "deploymentPattern": "deploy"
- }`),
- }
- scopes := make([]*plugin.BlueprintScopeV100, 0)
- scopes = append(scopes, bs)
-
- mockApiClient := mockaha.NewApiClientAbstract(t)
-
- var remoteData = []*models.Job{
- {
- Name: "devlake",
- Color: "blue",
- Class: "hudson.model.FreeStyleProject",
- Base: "",
- URL: "https://test.nddtf.com/job/devlake/",
- Description: "",
- },
- }
-
- var data struct {
- Jobs []json.RawMessage `json:"jobs"`
- }
-
- // job to apiClient
- js, err1 := json.Marshal(remoteData[0])
- assert.Nil(t, err1)
- data.Jobs = append(data.Jobs, js)
-
- js, err1 = json.Marshal(data)
- assert.Nil(t, err1)
-
- res := &http.Response{}
- res.Body = io.NopCloser(bytes.NewBuffer(js))
- res.StatusCode = http.StatusOK
-
- mockApiClient.On("Get", mock.Anything, mock.Anything, mock.Anything).Return(res, nil).Once()
-
- plan, err := makePipelinePlanV100(nil, scopes, connection, mockApiClient)
- assert.Nil(t, err)
-
- expectPlan := plugin.PipelinePlan{
- plugin.PipelineStage{
- {
- Plugin: "jenkins",
- Subtasks: []string{},
- Options: map[string]interface{}{
- "jobFullName": "devlake",
- "connectionId": uint64(1),
- "transformationRules": map[string]interface{}{
- "deploymentPattern": "deploy",
- },
- },
- },
- },
- plugin.PipelineStage{
- {
- Plugin: "dora",
- Subtasks: []string{"EnrichTaskEnv"},
- Options: map[string]interface{}{},
- },
- },
- }
- assert.Equal(t, expectPlan, plan)
-}
diff --git a/backend/plugins/jenkins/api/connection.go b/backend/plugins/jenkins/api/connection.go
index 0eb22e636..9ade3fa6b 100644
--- a/backend/plugins/jenkins/api/connection.go
+++ b/backend/plugins/jenkins/api/connection.go
@@ -167,46 +167,3 @@ func GetConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, e
}
return &plugin.ApiResourceOutput{Body: connection}, err
}
-
-// @Summary blueprints setting for jenkins
-// @Description blueprint setting for jenkins
-// @Tags plugins/jenkins
-// @Accept application/json
-// @Param blueprint body JenkinsBlueprintSetting true "json"
-// @Router /blueprints/jenkins/blueprint-setting [post]
-func PostJenkinsBluePrint(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
- blueprint := &JenkinsBlueprintSetting{}
- return &plugin.ApiResourceOutput{Body: blueprint, Status: http.StatusOK}, nil
-}
-
-type JenkinsBlueprintSetting []struct {
- Version string `json:"version"`
- Connections []struct {
- Plugin string `json:"plugin"`
- ConnectionID int `json:"connectionId"`
- Scope []struct {
- Options struct {
- } `json:"options"`
- Entities []string `json:"entities"`
- } `json:"scopes"`
- } `json:"connections"`
-}
-
-// @Summary pipelines plan for jenkins
-// @Description pipelines plan for jenkins
-// @Tags plugins/jenkins
-// @Accept application/json
-// @Param blueprint body JenkinsPipelinePlan true "json"
-// @Router /pipelines/jenkins/pipeline-plan [post]
-func PostJenkinsPipeline(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
- blueprint := &JenkinsPipelinePlan{}
- return &plugin.ApiResourceOutput{Body: blueprint, Status: http.StatusOK}, nil
-}
-
-type JenkinsPipelinePlan [][]struct {
- Plugin string `json:"plugin"`
- Subtasks []string `json:"subtasks"`
- Options struct {
- ConnectionID int `json:"connectionId"`
- } `json:"options"`
-}
diff --git a/backend/plugins/jenkins/api/init.go b/backend/plugins/jenkins/api/init.go
index 8fe30ce1e..ae2a1e10d 100644
--- a/backend/plugins/jenkins/api/init.go
+++ b/backend/plugins/jenkins/api/init.go
@@ -26,9 +26,9 @@ import (
var vld *validator.Validate
var connectionHelper *api.ConnectionApiHelper
-var scopeHelper *api.ScopeApiHelper[models.JenkinsConnection, models.JenkinsJob, models.JenkinsTransformationRule]
+var scopeHelper *api.ScopeApiHelper[models.JenkinsConnection, models.JenkinsJob, models.JenkinsScopeConfig]
var basicRes context.BasicRes
-var trHelper *api.TransformationRuleHelper[models.JenkinsTransformationRule]
+var scHelper *api.ScopeConfigHelper[models.JenkinsScopeConfig]
func Init(br context.BasicRes) {
basicRes = br
@@ -41,16 +41,16 @@ func Init(br context.BasicRes) {
ScopeIdFieldName: "FullName",
ScopeIdColumnName: "full_name",
}
- scopeHelper = api.NewScopeHelper[models.JenkinsConnection, models.JenkinsJob, models.JenkinsTransformationRule](
+ scopeHelper = api.NewScopeHelper[models.JenkinsConnection, models.JenkinsJob, models.JenkinsScopeConfig](
basicRes,
vld,
connectionHelper,
- api.NewScopeDatabaseHelperImpl[models.JenkinsConnection, models.JenkinsJob, models.JenkinsTransformationRule](
+ api.NewScopeDatabaseHelperImpl[models.JenkinsConnection, models.JenkinsJob, models.JenkinsScopeConfig](
basicRes, connectionHelper, params),
params,
nil,
)
- trHelper = api.NewTransformationRuleHelper[models.JenkinsTransformationRule](
+ scHelper = api.NewScopeConfigHelper[models.JenkinsScopeConfig](
basicRes,
vld,
)
diff --git a/backend/plugins/jenkins/api/scope.go b/backend/plugins/jenkins/api/scope.go
index 525aa52cd..c36629896 100644
--- a/backend/plugins/jenkins/api/scope.go
+++ b/backend/plugins/jenkins/api/scope.go
@@ -27,7 +27,7 @@ import (
type ScopeRes struct {
models.JenkinsJob
- TransformationRuleName string `json:"transformationRuleName,omitempty"`
+ ScopeConfigName string `json:"scopeConfigName,omitempty"`
}
type ScopeReq api.ScopeReq[models.JenkinsJob]
diff --git a/backend/plugins/jenkins/api/scope_config.go b/backend/plugins/jenkins/api/scope_config.go
new file mode 100644
index 000000000..36369c443
--- /dev/null
+++ b/backend/plugins/jenkins/api/scope_config.go
@@ -0,0 +1,83 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package api
+
+import (
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
+)
+
+// CreateScopeConfig create scope config for Jenkins
+// @Summary create scope config for Jenkins
+// @Description create scope config for Jenkins
+// @Tags plugins/jenkins
+// @Accept application/json
+// @Param connectionId path int true "connectionId"
+// @Param scopeConfig body models.JenkinsScopeConfig true "scope config"
+// @Success 200 {object} models.JenkinsScopeConfig
+// @Failure 400 {object} shared.ApiBody "Bad Request"
+// @Failure 500 {object} shared.ApiBody "Internal Error"
+// @Router /plugins/jenkins/connections/{connectionId}/scope_configs [POST]
+func CreateScopeConfig(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
+ return scHelper.Create(input)
+}
+
+// UpdateScopeConfig update scope config for Jenkins
+// @Summary update scope config for Jenkins
+// @Description update scope config for Jenkins
+// @Tags plugins/jenkins
+// @Accept application/json
+// @Param id path int true "id"
+// @Param scopeConfig body models.JenkinsScopeConfig true "scope config"
+// @Param connectionId path int true "connectionId"
+// @Success 200 {object} models.JenkinsScopeConfig
+// @Failure 400 {object} shared.ApiBody "Bad Request"
+// @Failure 500 {object} shared.ApiBody "Internal Error"
+// @Router /plugins/jenkins/connections/{connectionId}/scope_configs/{id} [PATCH]
+func UpdateScopeConfig(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
+ return scHelper.Update(input)
+}
+
+// GetScopeConfig return one scope config
+// @Summary return one scope config
+// @Description return one scope config
+// @Tags plugins/jenkins
+// @Param id path int true "id"
+// @Param connectionId path int true "connectionId"
+// @Success 200 {object} models.JenkinsScopeConfig
+// @Failure 400 {object} shared.ApiBody "Bad Request"
+// @Failure 500 {object} shared.ApiBody "Internal Error"
+// @Router /plugins/jenkins/connections/{connectionId}/scope_configs/{id} [GET]
+func GetScopeConfig(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
+ return scHelper.Update(input)
+}
+
+// GetScopeConfigList return all scope configs
+// @Summary return all scope configs
+// @Description return all scope configs
+// @Tags plugins/jenkins
+// @Param pageSize query int false "page size, default 50"
+// @Param page query int false "page size, default 1"
+// @Param connectionId path int true "connectionId"
+// @Success 200 {object} []models.JenkinsScopeConfig
+// @Failure 400 {object} shared.ApiBody "Bad Request"
+// @Failure 500 {object} shared.ApiBody "Internal Error"
+// @Router /plugins/jenkins/connections/{connectionId}/scope_configs [GET]
+func GetScopeConfigList(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
+ return scHelper.List(input)
+}
diff --git a/backend/plugins/jenkins/models/migrationscripts/register.go b/backend/plugins/jenkins/api/swagger.go
similarity index 62%
copy from backend/plugins/jenkins/models/migrationscripts/register.go
copy to backend/plugins/jenkins/api/swagger.go
index bc0d54f57..0007b0ae5 100644
--- a/backend/plugins/jenkins/models/migrationscripts/register.go
+++ b/backend/plugins/jenkins/api/swagger.go
@@ -15,23 +15,18 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
-package migrationscripts
+package api
import (
- "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/plugins/jenkins/tasks"
)
-// All return all the migration scripts
-func All() []plugin.MigrationScript {
- return []plugin.MigrationScript{
- new(addInitTables),
- new(modifyAllEntities),
- new(modifyJenkinsBuild),
- new(addJobFields),
- new(addJobPathForBuilds),
- new(changeIndexOfJobPath),
- new(addTransformationRule20221128),
- new(addFullNameForBuilds),
- new(addConnectionIdToTransformationRule),
- }
-}
+type JenkinsTaskOptions tasks.JenkinsOptions
+
+// @Summary jenkins task options for pipelines
+// @Description This is a dummy API to demonstrate the available task options for jenkins pipelines
+// @Tags plugins/jenkins
+// @Accept application/json
+// @Param pipeline body JenkinsTaskOptions true "json"
+// @Router /pipelines/jenkins/pipeline-task [post]
+func _() {}
diff --git a/backend/plugins/jenkins/api/transformation_rule.go b/backend/plugins/jenkins/api/transformation_rule.go
deleted file mode 100644
index 31377baa5..000000000
--- a/backend/plugins/jenkins/api/transformation_rule.go
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package api
-
-import (
- "github.com/apache/incubator-devlake/core/errors"
- "github.com/apache/incubator-devlake/core/plugin"
-)
-
-// CreateTransformationRule create transformation rule for Jenkins
-// @Summary create transformation rule for Jenkins
-// @Description create transformation rule for Jenkins
-// @Tags plugins/jenkins
-// @Accept application/json
-// @Param connectionId path int true "connectionId"
-// @Param transformationRule body models.JenkinsTransformationRule true "transformation rule"
-// @Success 200 {object} models.JenkinsTransformationRule
-// @Failure 400 {object} shared.ApiBody "Bad Request"
-// @Failure 500 {object} shared.ApiBody "Internal Error"
-// @Router /plugins/jenkins/connections/{connectionId}/transformation_rules [POST]
-func CreateTransformationRule(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
- return trHelper.Create(input)
-}
-
-// UpdateTransformationRule update transformation rule for Jenkins
-// @Summary update transformation rule for Jenkins
-// @Description update transformation rule for Jenkins
-// @Tags plugins/jenkins
-// @Accept application/json
-// @Param id path int true "id"
-// @Param transformationRule body models.JenkinsTransformationRule true "transformation rule"
-// @Param connectionId path int true "connectionId"
-// @Success 200 {object} models.JenkinsTransformationRule
-// @Failure 400 {object} shared.ApiBody "Bad Request"
-// @Failure 500 {object} shared.ApiBody "Internal Error"
-// @Router /plugins/jenkins/connections/{connectionId}/transformation_rules/{id} [PATCH]
-func UpdateTransformationRule(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
- return trHelper.Update(input)
-}
-
-// GetTransformationRule return one transformation rule
-// @Summary return one transformation rule
-// @Description return one transformation rule
-// @Tags plugins/jenkins
-// @Param id path int true "id"
-// @Param connectionId path int true "connectionId"
-// @Success 200 {object} models.JenkinsTransformationRule
-// @Failure 400 {object} shared.ApiBody "Bad Request"
-// @Failure 500 {object} shared.ApiBody "Internal Error"
-// @Router /plugins/jenkins/connections/{connectionId}/transformation_rules/{id} [GET]
-func GetTransformationRule(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
- return trHelper.Update(input)
-}
-
-// GetTransformationRuleList return all transformation rules
-// @Summary return all transformation rules
-// @Description return all transformation rules
-// @Tags plugins/jenkins
-// @Param pageSize query int false "page size, default 50"
-// @Param page query int false "page size, default 1"
-// @Param connectionId path int true "connectionId"
-// @Success 200 {object} []models.JenkinsTransformationRule
-// @Failure 400 {object} shared.ApiBody "Bad Request"
-// @Failure 500 {object} shared.ApiBody "Internal Error"
-// @Router /plugins/jenkins/connections/{connectionId}/transformation_rules [GET]
-func GetTransformationRuleList(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
- return trHelper.List(input)
-}
diff --git a/backend/plugins/jenkins/e2e/jobs_test.go b/backend/plugins/jenkins/e2e/jobs_test.go
index d2cb60d6c..816cb377b 100644
--- a/backend/plugins/jenkins/e2e/jobs_test.go
+++ b/backend/plugins/jenkins/e2e/jobs_test.go
@@ -18,9 +18,10 @@ limitations under the License.
package e2e
import (
+ "testing"
+
"github.com/apache/incubator-devlake/core/models/common"
"github.com/apache/incubator-devlake/core/models/domainlayer/devops"
- "testing"
"github.com/apache/incubator-devlake/helpers/e2ehelper"
"github.com/apache/incubator-devlake/plugins/jenkins/impl"
@@ -35,11 +36,11 @@ func TestJenkinsJobsDataFlow(t *testing.T) {
taskData := &tasks.JenkinsTaskData{
Options: &tasks.JenkinsOptions{
- ConnectionId: 1,
- JobName: `devlake`,
- JobFullName: `Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake`,
- JobPath: `job/Test-jenkins-dir/job/test-jenkins-sub-dir/job/test-sub-sub-dir/`,
- JenkinsTransformationRule: new(models.JenkinsTransformationRule),
+ ConnectionId: 1,
+ JobName: `devlake`,
+ JobFullName: `Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake`,
+ JobPath: `job/Test-jenkins-dir/job/test-jenkins-sub-dir/job/test-sub-sub-dir/`,
+ ScopeConfig: new(models.JenkinsScopeConfig),
},
}
diff --git a/backend/plugins/jenkins/impl/impl.go b/backend/plugins/jenkins/impl/impl.go
index f60447451..01fb34425 100644
--- a/backend/plugins/jenkins/impl/impl.go
+++ b/backend/plugins/jenkins/impl/impl.go
@@ -43,6 +43,7 @@ var _ plugin.PluginModel = (*Jenkins)(nil)
var _ plugin.PluginMigration = (*Jenkins)(nil)
var _ plugin.CloseablePluginTask = (*Jenkins)(nil)
var _ plugin.PluginSource = (*Jenkins)(nil)
+var _ plugin.DataSourcePluginBlueprintV200 = (*Jenkins)(nil)
type Jenkins struct{}
@@ -59,8 +60,8 @@ func (p Jenkins) Scope() interface{} {
return &models.JenkinsJob{}
}
-func (p Jenkins) TransformationRule() interface{} {
- return &models.JenkinsTransformationRule{}
+func (p Jenkins) ScopeConfig() interface{} {
+ return &models.JenkinsScopeConfig{}
}
func (p Jenkins) GetTablesInfo() []dal.Tabler {
@@ -133,10 +134,10 @@ func (p Jenkins) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]
}
}
regexEnricher := helper.NewRegexEnricher()
- if err := regexEnricher.TryAdd(devops.DEPLOYMENT, op.DeploymentPattern); err != nil {
+ if err := regexEnricher.TryAdd(devops.DEPLOYMENT, op.ScopeConfig.DeploymentPattern); err != nil {
return nil, errors.BadInput.Wrap(err, "invalid value for `deploymentPattern`")
}
- if err := regexEnricher.TryAdd(devops.PRODUCTION, op.ProductionPattern); err != nil {
+ if err := regexEnricher.TryAdd(devops.PRODUCTION, op.ScopeConfig.ProductionPattern); err != nil {
return nil, errors.BadInput.Wrap(err, "invalid value for `productionPattern`")
}
taskData := &tasks.JenkinsTaskData{
@@ -160,10 +161,6 @@ func (p Jenkins) MigrationScripts() []plugin.MigrationScript {
return migrationscripts.All()
}
-func (p Jenkins) MakePipelinePlan(connectionId uint64, scope []*plugin.BlueprintScopeV100) (plugin.PipelinePlan, errors.Error) {
- return api.MakePipelinePlanV100(p.SubTaskMetas(), connectionId, scope)
-}
-
func (p Jenkins) MakeDataSourcePipelinePlanV200(connectionId uint64, scopes []*plugin.BlueprintScopeV200, syncPolicy plugin.BlueprintSyncPolicy) (pp plugin.PipelinePlan, sc []plugin.Scope, err errors.Error) {
return api.MakeDataSourcePipelinePlanV200(p.SubTaskMetas(), connectionId, scopes, &syncPolicy)
}
@@ -191,13 +188,13 @@ func (p Jenkins) ApiResources() map[string]map[string]plugin.ApiResourceHandler
"GET": api.GetScopeList,
"PUT": api.PutScope,
},
- "connections/:connectionId/transformation_rules": {
- "POST": api.CreateTransformationRule,
- "GET": api.GetTransformationRuleList,
+ "connections/:connectionId/scope_configs": {
+ "POST": api.CreateScopeConfig,
+ "GET": api.GetScopeConfigList,
},
- "connections/:connectionId/transformation_rules/:id": {
- "PATCH": api.UpdateTransformationRule,
- "GET": api.GetTransformationRule,
+ "connections/:connectionId/scope_configs/:id": {
+ "PATCH": api.UpdateScopeConfig,
+ "GET": api.GetScopeConfig,
},
"connections/:connectionId/proxy/rest/*path": {
"GET": api.Proxy,
@@ -229,19 +226,18 @@ func EnrichOptions(taskCtx plugin.TaskContext,
}
log := taskCtx.GetLogger()
- // for advanced mode or others which we only have name, for bp v200, we have TransformationRuleId
+ // for advanced mode or others which we only have name, for bp v200, we have ScopeConfigId
err = taskCtx.GetDal().First(jenkinsJob,
dal.Where(`connection_id = ? and full_name = ?`,
op.ConnectionId, op.JobFullName))
if err == nil {
- if op.TransformationRuleId == 0 {
- op.TransformationRuleId = jenkinsJob.TransformationRuleId
+ if op.ScopeConfigId == 0 {
+ op.ScopeConfigId = jenkinsJob.ScopeConfigId
}
}
err = api.GetJob(apiClient, op.JobPath, op.JobName, op.JobFullName, 100, func(job *models.Job, isPath bool) errors.Error {
log.Debug(fmt.Sprintf("Current job: %s", job.FullName))
- op.Name = job.Name
op.JobPath = job.Path
jenkinsJob := ConvertJobToJenkinsJob(job, op)
err = taskCtx.GetDal().CreateIfNotExist(jenkinsJob)
@@ -254,18 +250,18 @@ func EnrichOptions(taskCtx plugin.TaskContext,
if !strings.HasSuffix(op.JobPath, "/") {
op.JobPath = fmt.Sprintf("%s/", op.JobPath)
}
- // We only set op.JenkinsTransformationRule when it's nil and we have op.TransformationRuleId != 0
- if op.JenkinsTransformationRule.DeploymentPattern == "" && op.JenkinsTransformationRule.ProductionPattern == "" && op.TransformationRuleId != 0 {
- var transformationRule models.JenkinsTransformationRule
- err = taskCtx.GetDal().First(&transformationRule, dal.Where("id = ?", op.TransformationRuleId))
+ // We only set op.JenkinsScopeConfig when it's nil and we have op.ScopeConfigId != 0
+ if op.ScopeConfig.DeploymentPattern == "" && op.ScopeConfig.ProductionPattern == "" && op.ScopeConfigId != 0 {
+ var scopeConfig models.JenkinsScopeConfig
+ err = taskCtx.GetDal().First(&scopeConfig, dal.Where("id = ?", op.ScopeConfigId))
if err != nil {
- return errors.BadInput.Wrap(err, "fail to get transformationRule")
+ return errors.BadInput.Wrap(err, "fail to get scopeConfig")
}
- op.JenkinsTransformationRule = &transformationRule
+ op.ScopeConfig = &scopeConfig
}
- if op.JenkinsTransformationRule.DeploymentPattern == "" && op.JenkinsTransformationRule.ProductionPattern == "" && op.TransformationRuleId == 0 {
- op.JenkinsTransformationRule = new(models.JenkinsTransformationRule)
+ if op.ScopeConfig.DeploymentPattern == "" && op.ScopeConfig.ProductionPattern == "" && op.ScopeConfigId == 0 {
+ op.ScopeConfig = new(models.JenkinsScopeConfig)
}
return nil
@@ -273,16 +269,16 @@ func EnrichOptions(taskCtx plugin.TaskContext,
func ConvertJobToJenkinsJob(job *models.Job, op *tasks.JenkinsOptions) *models.JenkinsJob {
return &models.JenkinsJob{
- ConnectionId: op.ConnectionId,
- FullName: job.FullName,
- TransformationRuleId: op.TransformationRuleId,
- Name: job.Name,
- Path: job.Path,
- Class: job.Class,
- Color: job.Color,
- Base: job.Base,
- Url: job.URL,
- Description: job.Description,
- PrimaryView: job.URL + job.Path + job.Class,
+ ConnectionId: op.ConnectionId,
+ FullName: job.FullName,
+ ScopeConfigId: op.ScopeConfigId,
+ Name: job.Name,
+ Path: job.Path,
+ Class: job.Class,
+ Color: job.Color,
+ Base: job.Base,
+ Url: job.URL,
+ Description: job.Description,
+ PrimaryView: job.URL + job.Path + job.Class,
}
}
diff --git a/backend/plugins/jenkins/models/job.go b/backend/plugins/jenkins/models/job.go
index 3f328cab2..b9c854e2a 100644
--- a/backend/plugins/jenkins/models/job.go
+++ b/backend/plugins/jenkins/models/job.go
@@ -25,7 +25,7 @@ import (
type JenkinsJob struct {
ConnectionId uint64 `gorm:"primaryKey" mapstructure:"connectionId,omitempty" validate:"required" json:"connectionId"`
FullName string `gorm:"primaryKey;type:varchar(255)" mapstructure:"jobFullName" validate:"required" json:"jobFullName"` // "path1/path2/job name"
- TransformationRuleId uint64 `mapstructure:"transformationRuleId,omitempty" json:"transformationRuleId,omitempty"`
+ ScopeConfigId uint64 `mapstructure:"scopeConfigId,omitempty" json:"scopeConfigId,omitempty"`
Name string `gorm:"index;type:varchar(255)" mapstructure:"name" json:"name"` // scope name now is same to `jobFullName`
Path string `gorm:"index;type:varchar(511)" mapstructure:"-,omitempty" json:"-"` // "job/path1/job/path2"
Class string `gorm:"type:varchar(255)" mapstructure:"class,omitempty" json:"class"`
diff --git a/backend/plugins/jenkins/models/migrationscripts/20230322_add_connection_id_to_transformation_rules.go b/backend/plugins/jenkins/models/migrationscripts/20230322_add_connection_id_to_transformation_rules.go
index 38236b7f4..34c38fbd3 100644
--- a/backend/plugins/jenkins/models/migrationscripts/20230322_add_connection_id_to_transformation_rules.go
+++ b/backend/plugins/jenkins/models/migrationscripts/20230322_add_connection_id_to_transformation_rules.go
@@ -22,25 +22,34 @@ import (
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/helpers/migrationhelper"
- "github.com/apache/incubator-devlake/plugins/jenkins/models"
)
type addConnectionIdToTransformationRule struct{}
-type transformationRule20220322 struct {
+type job20230322 struct {
+ ConnectionId uint64 `gorm:"primaryKey" mapstructure:"connectionId,omitempty" validate:"required" json:"connectionId"`
+ FullName string `gorm:"primaryKey;type:varchar(255)" mapstructure:"jobFullName" validate:"required" json:"jobFullName"` // "path1/path2/job name"
+ TransformationRuleId uint64 `mapstructure:"transformationRuleId,omitempty" json:"transformationRuleId,omitempty"`
+}
+
+func (job20230322) TableName() string {
+ return "_tool_jenkins_jobs"
+}
+
+type transformationRule20230322 struct {
ConnectionId uint64
}
-func (transformationRule20220322) TableName() string {
+func (transformationRule20230322) TableName() string {
return "_tool_jenkins_transformation_rules"
}
func (u *addConnectionIdToTransformationRule) Up(baseRes context.BasicRes) errors.Error {
- err := migrationhelper.AutoMigrateTables(baseRes, &transformationRule20220322{})
+ err := migrationhelper.AutoMigrateTables(baseRes, &transformationRule20230322{})
if err != nil {
return err
}
- var scopes []models.JenkinsJob
+ var scopes []job20230322
err = baseRes.GetDal().All(&scopes)
if err != nil {
return err
@@ -55,14 +64,14 @@ func (u *addConnectionIdToTransformationRule) Up(baseRes context.BasicRes) error
// set connection_id for rules
for trId, cId := range idMap {
err = baseRes.GetDal().UpdateColumn(
- &models.JenkinsTransformationRule{}, "connection_id", cId,
+ &job20230322{}, "connection_id", cId,
dal.Where("id = ?", trId))
if err != nil {
return err
}
}
// delete all rules that are not referenced.
- return baseRes.GetDal().Delete(&models.JenkinsTransformationRule{}, dal.Where("connection_id IS NULL OR connection_id = 0"))
+ return baseRes.GetDal().Delete(&job20230322{}, dal.Where("connection_id IS NULL OR connection_id = 0"))
}
func (*addConnectionIdToTransformationRule) Version() uint64 {
diff --git a/backend/plugins/jenkins/models/migrationscripts/20230530_scope_config.go b/backend/plugins/jenkins/models/migrationscripts/20230530_scope_config.go
new file mode 100644
index 000000000..1cdbe9cb8
--- /dev/null
+++ b/backend/plugins/jenkins/models/migrationscripts/20230530_scope_config.go
@@ -0,0 +1,56 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+ "github.com/apache/incubator-devlake/core/context"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/helpers/migrationhelper"
+)
+
+type renameTr2ScopeConfig struct {
+}
+
+type scopeConfig20230530 struct {
+ Entities []string `gorm:"type:json" json:"entities"`
+}
+
+func (scopeConfig20230530) TableName() string {
+ return "_tool_jenkins_scope_configs"
+}
+
+func (u *renameTr2ScopeConfig) Up(baseRes context.BasicRes) errors.Error {
+ db := baseRes.GetDal()
+ err := db.RenameColumn("_tool_jenkins_jobs", "transformation_rule_id", "scope_config_id")
+ if err != nil {
+ return err
+ }
+ err = db.RenameTable("_tool_jenkins_transformation_rules", "_tool_jenkins_scope_configs")
+ if err != nil {
+ return err
+ }
+ return migrationhelper.AutoMigrateTables(baseRes, &scopeConfig20230530{})
+}
+
+func (*renameTr2ScopeConfig) Version() uint64 {
+ return 20230530134905
+}
+
+func (*renameTr2ScopeConfig) Name() string {
+ return "rename transformation rule to scope config for jenkins"
+}
diff --git a/backend/plugins/jenkins/models/migrationscripts/register.go b/backend/plugins/jenkins/models/migrationscripts/register.go
index bc0d54f57..b75403117 100644
--- a/backend/plugins/jenkins/models/migrationscripts/register.go
+++ b/backend/plugins/jenkins/models/migrationscripts/register.go
@@ -33,5 +33,6 @@ func All() []plugin.MigrationScript {
new(addTransformationRule20221128),
new(addFullNameForBuilds),
new(addConnectionIdToTransformationRule),
+ new(renameTr2ScopeConfig),
}
}
diff --git a/backend/plugins/jenkins/models/transformation_rule.go b/backend/plugins/jenkins/models/scope_config.go
similarity index 56%
rename from backend/plugins/jenkins/models/transformation_rule.go
rename to backend/plugins/jenkins/models/scope_config.go
index 2845f0a04..35dbf3705 100644
--- a/backend/plugins/jenkins/models/transformation_rule.go
+++ b/backend/plugins/jenkins/models/scope_config.go
@@ -21,14 +21,14 @@ import (
"github.com/apache/incubator-devlake/core/models/common"
)
-type JenkinsTransformationRule struct {
- common.Model `mapstructure:"-"`
- ConnectionId uint64 `mapstructure:"connectionId" json:"connectionId"`
- Name string `gorm:"type:varchar(255);index:idx_name_jenkins,unique" validate:"required" mapstructure:"name" json:"name"`
- DeploymentPattern string `gorm:"type:varchar(255)" mapstructure:"deploymentPattern,omitempty" json:"deploymentPattern"`
- ProductionPattern string `gorm:"type:varchar(255)" mapstructure:"productionPattern,omitempty" json:"productionPattern"`
+type JenkinsScopeConfig struct {
+ common.ScopeConfig `mapstructure:",squash" json:",inline" gorm:"embedded"`
+ ConnectionId uint64 `mapstructure:"connectionId" json:"connectionId"`
+ Name string `gorm:"type:varchar(255);index:idx_name_jenkins,unique" validate:"required" mapstructure:"name" json:"name"`
+ DeploymentPattern string `gorm:"type:varchar(255)" mapstructure:"deploymentPattern,omitempty" json:"deploymentPattern"`
+ ProductionPattern string `gorm:"type:varchar(255)" mapstructure:"productionPattern,omitempty" json:"productionPattern"`
}
-func (t JenkinsTransformationRule) TableName() string {
- return "_tool_jenkins_transformation_rules"
+func (t JenkinsScopeConfig) TableName() string {
+ return "_tool_jenkins_scope_configs"
}
diff --git a/backend/plugins/jenkins/tasks/task_data.go b/backend/plugins/jenkins/tasks/task_data.go
index 1635fb504..43e562b7c 100644
--- a/backend/plugins/jenkins/tasks/task_data.go
+++ b/backend/plugins/jenkins/tasks/task_data.go
@@ -32,15 +32,15 @@ type JenkinsApiParams struct {
}
type JenkinsOptions struct {
- ConnectionId uint64 `json:"connectionId"`
- ScopeId string
- TransformationRuleId uint64 `json:"transformationRuleId"`
- JobFullName string `json:"jobFullName"` // "path1/path2/job name"
- JobName string `json:"jobName"` // "job name"
- JobPath string `json:"jobPath"` // "job/path1/job/path2"
- TimeAfter string
- Tasks []string `json:"tasks,omitempty"`
- *models.JenkinsTransformationRule `mapstructure:"transformationRules" json:"transformationRules"`
+ ConnectionId uint64 `json:"connectionId"`
+ ScopeId string
+ ScopeConfigId uint64 `json:"scopeConfigId"`
+ JobFullName string `json:"jobFullName"` // "path1/path2/job name"
+ JobName string `json:"jobName"` // "job name"
+ JobPath string `json:"jobPath"` // "job/path1/job/path2"
+ TimeAfter string
+ Tasks []string `json:"tasks,omitempty"`
+ ScopeConfig *models.JenkinsScopeConfig `mapstructure:"scopeConfig" json:"scopeConfig"`
}
type JenkinsTaskData struct {
@@ -74,8 +74,8 @@ func ValidateTaskOptions(op *JenkinsOptions) (*JenkinsOptions, errors.Error) {
op.JobName = op.JobFullName
op.JobPath = `view/all`
}
- if op.JenkinsTransformationRule == nil && op.TransformationRuleId == 0 {
- op.JenkinsTransformationRule = new(models.JenkinsTransformationRule)
+ if op.ScopeConfig == nil && op.ScopeConfigId == 0 {
+ op.ScopeConfig = new(models.JenkinsScopeConfig)
}
return op, nil
}