You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by wa...@apache.org on 2022/08/03 01:29:03 UTC
[incubator-devlake] branch main updated: fix: patch /blueprints/:id wouldn't handle byte[]
This is an automated email from the ASF dual-hosted git repository.
warren pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 4f05cdca fix: patch /blueprints/:id wouldn't handle byte[]
4f05cdca is described below
commit 4f05cdca151d5cd716cb90ad809ced5d3f7e2711
Author: Klesh Wong <zh...@merico.dev>
AuthorDate: Tue Aug 2 22:13:13 2022 +0800
fix: patch /blueprints/:id wouldn't handle byte[]
---
models/blueprint.go | 33 +++++++-------
plugins/helper/iso8601time.go | 46 -------------------
plugins/helper/mapstructure.go | 76 ++++++++++++++++++++++++++++++++
plugins/helper/mapstructure_test.go | 58 ++++++++++++++++++++++++
scripts/pm/framework/blueprint-create.sh | 65 +++++++++++++++++++++++++++
scripts/pm/framework/blueprint-update.sh | 68 ++++++++++++++++++++++++++++
services/blueprint.go | 14 +++---
7 files changed, 292 insertions(+), 68 deletions(-)
diff --git a/models/blueprint.go b/models/blueprint.go
index adced10e..413bb9aa 100644
--- a/models/blueprint.go
+++ b/models/blueprint.go
@@ -22,20 +22,21 @@ import (
"github.com/apache/incubator-devlake/models/common"
"github.com/apache/incubator-devlake/plugins/core"
- "gorm.io/datatypes"
)
-const BLUEPRINT_MODE_NORMAL = "NORMAL"
-const BLUEPRINT_MODE_ADVANCED = "ADVANCED"
+const (
+ BLUEPRINT_MODE_NORMAL = "NORMAL"
+ BLUEPRINT_MODE_ADVANCED = "ADVANCED"
+)
type Blueprint struct {
- Name string `json:"name" validate:"required"`
- Mode string `json:"mode" gorm:"varchar(20)" validate:"required,oneof=NORMAL ADVANCED"`
- Plan datatypes.JSON `json:"plan"`
- Enable bool `json:"enable"`
- CronConfig string `json:"cronConfig"`
- IsManual bool `json:"isManual"`
- Settings datatypes.JSON `json:"settings"`
+ Name string `json:"name" validate:"required"`
+ Mode string `json:"mode" gorm:"varchar(20)" validate:"required,oneof=NORMAL ADVANCED"`
+ Plan json.RawMessage `json:"plan"`
+ Enable bool `json:"enable"`
+ CronConfig string `json:"cronConfig"`
+ IsManual bool `json:"isManual"`
+ Settings json.RawMessage `json:"settings"`
common.Model
}
@@ -50,10 +51,10 @@ type BlueprintSettings struct {
// UnmarshalPlan unmarshals Plan in JSON to strong-typed core.PipelinePlan
func (bp *Blueprint) UnmarshalPlan() (core.PipelinePlan, error) {
- var plan core.PipelinePlan
- err := json.Unmarshal(bp.Plan, &plan)
- if err != nil {
- return nil, err
- }
- return plan, nil
+ var plan core.PipelinePlan
+ err := json.Unmarshal(bp.Plan, &plan)
+ if err != nil {
+ return nil, err
+ }
+ return plan, nil
}
diff --git a/plugins/helper/iso8601time.go b/plugins/helper/iso8601time.go
index 6232eb83..5b02e9c9 100644
--- a/plugins/helper/iso8601time.go
+++ b/plugins/helper/iso8601time.go
@@ -19,12 +19,9 @@ package helper
import (
"fmt"
- "reflect"
"regexp"
"strings"
"time"
-
- "github.com/mitchellh/mapstructure"
)
/*
@@ -134,46 +131,3 @@ func Iso8601TimeToTime(iso8601Time *Iso8601Time) *time.Time {
t := iso8601Time.ToTime()
return &t
}
-
-// DecodeMapStruct with time.Time and Iso8601Time support
-func DecodeMapStruct(input map[string]interface{}, result interface{}) error {
- decoder, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{
- Metadata: nil,
- DecodeHook: mapstructure.ComposeDecodeHookFunc(
- func(f reflect.Type, t reflect.Type, data interface{}) (interface{}, error) {
- if t != reflect.TypeOf(Iso8601Time{}) && t != reflect.TypeOf(time.Time{}) {
- return data, nil
- }
-
- var tt time.Time
- var err error
-
- switch f.Kind() {
- case reflect.String:
- tt, err = ConvertStringToTime(data.(string))
- case reflect.Float64:
- tt = time.Unix(0, int64(data.(float64))*int64(time.Millisecond))
- case reflect.Int64:
- tt = time.Unix(0, data.(int64)*int64(time.Millisecond))
- }
- if err != nil {
- return data, nil
- }
-
- if t == reflect.TypeOf(Iso8601Time{}) {
- return Iso8601Time{time: tt}, nil
- }
- return tt, nil
- },
- ),
- Result: result,
- })
- if err != nil {
- return err
- }
-
- if err := decoder.Decode(input); err != nil {
- return err
- }
- return err
-}
diff --git a/plugins/helper/mapstructure.go b/plugins/helper/mapstructure.go
new file mode 100644
index 00000000..17e74df4
--- /dev/null
+++ b/plugins/helper/mapstructure.go
@@ -0,0 +1,76 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package helper
+
+import (
+ "encoding/json"
+ "reflect"
+ "time"
+
+ "github.com/mitchellh/mapstructure"
+)
+
+// DecodeMapStruct with time.Time and Iso8601Time support
+func DecodeMapStruct(input map[string]interface{}, result interface{}) error {
+ decoder, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{
+ ZeroFields: true,
+ DecodeHook: mapstructure.ComposeDecodeHookFunc(
+ func(f reflect.Type, t reflect.Type, data interface{}) (interface{}, error) {
+ if data == nil {
+ return nil, nil
+ }
+ if t == reflect.TypeOf(json.RawMessage{}) {
+ return json.Marshal(data)
+ }
+
+ if t != reflect.TypeOf(Iso8601Time{}) && t != reflect.TypeOf(time.Time{}) {
+ return data, nil
+ }
+
+ var tt time.Time
+ var err error
+
+ switch f.Kind() {
+ case reflect.String:
+ tt, err = ConvertStringToTime(data.(string))
+ case reflect.Float64:
+ tt = time.Unix(0, int64(data.(float64))*int64(time.Millisecond))
+ case reflect.Int64:
+ tt = time.Unix(0, data.(int64)*int64(time.Millisecond))
+ }
+ if err != nil {
+ return data, nil
+ }
+
+ if t == reflect.TypeOf(Iso8601Time{}) {
+ return Iso8601Time{time: tt}, nil
+ }
+ return tt, nil
+ },
+ ),
+ Result: result,
+ })
+ if err != nil {
+ return err
+ }
+
+ if err := decoder.Decode(input); err != nil {
+ return err
+ }
+ return err
+}
diff --git a/plugins/helper/mapstructure_test.go b/plugins/helper/mapstructure_test.go
new file mode 100644
index 00000000..34ab85e3
--- /dev/null
+++ b/plugins/helper/mapstructure_test.go
@@ -0,0 +1,58 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package helper
+
+import (
+ "encoding/json"
+ "fmt"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+)
+
+type DecodeMapStructJson struct {
+ Id int
+ Settings json.RawMessage
+ Plan json.RawMessage
+ Existing json.RawMessage
+}
+
+func TestDecodeMapStructJsonRawMessage(t *testing.T) {
+ input := map[string]interface{}{
+ "id": 100,
+ "settings": map[string]interface{}{
+ "version": "1.0.0",
+ },
+ }
+
+ decoded := &DecodeMapStructJson{
+ Settings: json.RawMessage(`{"version": "1.0.101"}`),
+ Existing: json.RawMessage(`{"hello", "world"}`),
+ }
+ err := DecodeMapStruct(input, decoded)
+ fmt.Println(string(decoded.Settings))
+ assert.Nil(t, err)
+ assert.Equal(t, decoded.Id, 100)
+ assert.Nil(t, decoded.Plan)
+ assert.NotNil(t, decoded.Settings)
+ settings := make(map[string]string)
+ err = json.Unmarshal(decoded.Settings, &settings)
+ assert.Nil(t, err)
+ assert.Equal(t, settings["version"], "1.0.0")
+ assert.Equal(t, decoded.Existing, json.RawMessage(`{"hello", "world"}`))
+}
diff --git a/scripts/pm/framework/blueprint-create.sh b/scripts/pm/framework/blueprint-create.sh
new file mode 100755
index 00000000..be6cabcd
--- /dev/null
+++ b/scripts/pm/framework/blueprint-create.sh
@@ -0,0 +1,65 @@
+#!/bin/sh
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+. "$(dirname $0)/../vars/active-vars.sh"
+
+curl -sv $LAKE_ENDPOINT/blueprints -H "Content-Type: application/json" --data @- <<JSON | jq
+{
+ "name": "MY BLUEPRINT2",
+ "cronConfig": "0 0 * * *",
+ "settings": {
+ "version": "1.0.0",
+ "connections": [
+ {
+ "plugin": "jira",
+ "connectionId": 1,
+ "scope": [
+ {
+ "transformation": {
+ "epicKeyField": "customfield_10014",
+ "typeMappings": {
+ "缺陷": {
+ "standardType": "Bug"
+ },
+ "线上事故": {
+ "standardType": "Incident"
+ },
+ "故事": {
+ "standardType": "Requirement"
+ }
+ },
+ "storyPointField": "customfield_10024",
+ "remotelinkCommitShaPattern": "/commit/([0-9a-f]{40})$"
+ },
+ "options": {
+ "boardId": 70
+ },
+ "entities": [
+ "TICKET",
+ "CROSS"
+ ]
+ }
+ ]
+ }
+ ]
+ },
+ "enable": true,
+ "mode": "NORMAL",
+ "isManual": false
+}
+JSON
diff --git a/scripts/pm/framework/blueprint-update.sh b/scripts/pm/framework/blueprint-update.sh
new file mode 100755
index 00000000..13d8a71c
--- /dev/null
+++ b/scripts/pm/framework/blueprint-update.sh
@@ -0,0 +1,68 @@
+#!/bin/sh
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+. "$(dirname $0)/../vars/active-vars.sh"
+
+pipeline_id=${1-8}
+
+curl -sv -XPATCH $LAKE_ENDPOINT/blueprints/$pipeline_id \
+ -H "Content-Type: application/json" --data @- <<JSON | jq
+{
+ "name": "MY BLUEPRINT2",
+ "cronConfig": "0 0 * * *",
+ "settings": {
+ "version": "1.0.0",
+ "connections": [
+ {
+ "plugin": "jira",
+ "connectionId": 1,
+ "scope": [
+ {
+ "transformation": {
+ "epicKeyField": "customfield_10014",
+ "typeMappings": {
+ "缺陷": {
+ "standardType": "Bug"
+ },
+ "线上事故": {
+ "standardType": "Incident"
+ },
+ "故事": {
+ "standardType": "Requirement"
+ }
+ },
+ "storyPointField": "customfield_10024",
+ "remotelinkCommitShaPattern": "/commit/([0-9a-f]{40})$"
+ },
+ "options": {
+ "boardId": 70
+ },
+ "entities": [
+ "TICKET",
+ "CROSS"
+ ]
+ }
+ ]
+ }
+ ]
+ },
+ "enable": true,
+ "mode": "NORMAL",
+ "isManual": false
+}
+JSON
diff --git a/services/blueprint.go b/services/blueprint.go
index c49599b9..4709a664 100644
--- a/services/blueprint.go
+++ b/services/blueprint.go
@@ -26,10 +26,9 @@ import (
"github.com/apache/incubator-devlake/logger"
"github.com/apache/incubator-devlake/models"
"github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/helper"
"github.com/go-playground/validator/v10"
- "github.com/mitchellh/mapstructure"
"github.com/robfig/cron/v3"
- "gorm.io/datatypes"
"gorm.io/gorm"
)
@@ -40,8 +39,10 @@ type BlueprintQuery struct {
PageSize int `form:"pageSize"`
}
-var blueprintLog = logger.Global.Nested("blueprint")
-var vld = validator.New()
+var (
+ blueprintLog = logger.Global.Nested("blueprint")
+ vld = validator.New()
+)
// CreateBlueprint accepts a Blueprint instance and insert it to database
func CreateBlueprint(blueprint *models.Blueprint) error {
@@ -139,7 +140,7 @@ func PatchBlueprint(id uint64, body map[string]interface{}) (*models.Blueprint,
return nil, err
}
originMode := blueprint.Mode
- err = mapstructure.Decode(body, blueprint)
+ err = helper.DecodeMapStruct(body, blueprint)
if err != nil {
return nil, err
}
@@ -234,10 +235,11 @@ func createPipelineByBlueprint(blueprintId uint64, name string, plan core.Pipeli
}
// GeneratePlanJson generates pipeline plan by version
-func GeneratePlanJson(settings datatypes.JSON) (datatypes.JSON, error) {
+func GeneratePlanJson(settings json.RawMessage) (json.RawMessage, error) {
bpSettings := new(models.BlueprintSettings)
err := json.Unmarshal(settings, bpSettings)
if err != nil {
+ fmt.Println(string(settings))
return nil, err
}
var plan interface{}