[trafficcontrol] branch master updated: Changed ruleset from json to yaml - Human friendly - Added a way to add parameters as well although doesn't check with existing profile - Doesn't handle JSON input anymore. Next step although everybody should use yaml for this

[fr...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

friede pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/trafficcontrol.git


The following commit(s) were added to refs/heads/master by this push:
     new e27cab4  Changed ruleset from json to yaml - Human friendly - Added a way to add parameters as well although doesn't check with existing profile - Doesn't handle JSON input anymore. Next step although everybody should use yaml for this
e27cab4 is described below

commit e27cab46e19299608a8ae500ee5d73960f44db2c
Author: Steve Malenfant <st...@cox.com>
AuthorDate: Tue Mar 19 14:46:39 2019 -0400

    Changed ruleset from json to yaml
    - Human friendly
    - Added a way to add parameters as well although doesn't check with existing profile
    - Doesn't handle JSON input anymore. Next step although everybody should use yaml for this
---
 .../bin/convert_profile/convert622to713.yaml       | 140 +++++++++++++++++++++
 .../install/bin/convert_profile/convert_profile.go |  64 ++++++----
 2 files changed, 180 insertions(+), 24 deletions(-)

diff --git a/traffic_ops/install/bin/convert_profile/convert622to713.yaml b/traffic_ops/install/bin/convert_profile/convert622to713.yaml
new file mode 100644
index 0000000..a7a5523
--- /dev/null
+++ b/traffic_ops/install/bin/convert_profile/convert622to713.yaml
@@ -0,0 +1,140 @@
+---
+description: Upgrade profile from ATS6 to ATS7 based on https://cwiki.apache.org/confluence/display/TS/Upgrading+to+v7.0
+replace_name:
+  old: '621'
+  new: '713'
+replace_description:
+  old: v6.2.1
+  new: v7.1.3
+validate_parameters:
+- name: trafficserver
+  config_file: package
+  value: 6.2.1-62.3f63b3a.el7.centos.x86_64
+conversion_actions:
+- match_parameter:
+    name: trafficserver
+    config_file: package
+    value: 6.2.1-62.3f63b3a.el7.centos.x86_64
+  new_value: "<PLACE TRAFFICSERVER7 VERSION STRING HERE>"
+- match_parameter:
+    name: trafficserver-debuginfo
+    config_file: package
+    value: 6.2.1-62.3f63b3a.el7.centos.x86_64
+  new_value: "<PLACE TRAFFICSERVER7 VERSION STRING HERE>"
+- match_parameter:
+    name: location
+    config_file: logs_xml\.config
+    value: ".*"
+  new_config_file: logging.config
+- match_parameter:
+    name: LogFormat\.*
+    config_file: logs_xml\.config
+    value: ".*"
+  new_config_file: logging.config
+- match_parameter:
+    name: CONFIG proxy\.config\.log\.xml_config_file
+    config_file: records\.config
+    value: STRING logs_xml\.config
+  new_name: CONFIG proxy.config.log.config.filename
+  new_value: STRING logging.confxsig
+- match_parameter:
+    name: CONFIG proxy\.config\.hostdb\.timeout
+    config_file: records\.config
+    value: INT 1440
+  new_value: INT 86400
+- match_parameter:
+    name: CONFIG proxy\.config\.http\.keep_alive_no_activity_timeout_in
+    config_file: records\.config
+    value: INT 115
+  new_value: INT 120
+- match_parameter:
+    name: CONFIG proxy\.config\.http\.cache\.fuzz\.time
+    config_file: records\.config
+    value: INT 240
+  new_value: INT 0
+- match_parameter:
+    name: CONFIG proxy\.config\.http\.cache\.fuzz\.probability
+    config_file: records\.config
+    value: FLOAT 0.005
+  new_value: INT 0
+- match_parameter:
+    name: CONFIG proxy\.config\.http\.anonymize_insert_client_ip
+    config_file: records\.config
+    value: ".*"
+  new_name: CONFIG proxy.config.http.insert_client_ip
+- match_parameter:
+    name: CONFIG proxy\.node\.num_processes
+    config_file: records\.config
+    value: ".*"
+  action: delete
+- match_parameter:
+    name: CONFIG proxy\.config\.ssl\.compression
+    config_file: records\.config
+    value: ".*"
+  action: delete
+- match_parameter:
+    name: CONFIG proxy\.config\.ssl\.number\.threads
+    config_file: records\.config
+    value: ".*"
+  action: delete
+- match_parameter:
+    name: CONFIG proxy\.config\.system\.mmap_max
+    config_file: records\.config
+    value: ".*"
+  action: delete
+- match_parameter:
+    name: CONFIG proxy\.config\.stats\.enable_lua
+    config_file: records\.config
+    value: ".*"
+  action: delete
+- match_parameter:
+    name: CONFIG proxy\.config\.hostdb\.size
+    config_file: records\.config
+    value: ".*"
+  action: delete
+- match_parameter:
+    name: CONFIG proxy\.config\.hostdb\.storage_size
+    config_file: records\.config
+    value: ".*"
+  action: delete
+- match_parameter:
+    name: CONFIG proxy\.config\.ping\.npack_to_trans
+    config_file: records\.config
+    value: ".*"
+  action: delete
+- match_parameter:
+    name: CONFIG proxy\.config\.ping\.timeout_sec
+    config_file: records\.config
+    value: ".*"
+  action: delete
+- match_parameter:
+    name: CONFIG proxy\.config\.http\.enable_url_expandomatic
+    config_file: records\.config
+    value: ".*"
+  action: delete
+- match_parameter:
+    name: CONFIG proxy\.config\.dns\.url_expansions
+    config_file: records\.config
+    value: ".*"
+  action: delete
+- match_parameter:
+    name: CONFIG proxy\.config\.ssl\.SSLv2
+    config_file: records\.config
+    value: ".*"
+  action: delete
+- match_parameter:
+    name: CONFIG proxy\.config\.icp\..*
+    config_file: records\.config
+    value: ".*"
+  action: delete
+- match_parameter:
+    name: CONFIG proxy\.config\.spdy\..*
+    config_file: records\.config
+    value: ".*"
+  action: delete
+- match_parameter:
+    name: CONFIG proxy\.cluster\..*
+    config_file: records\.config
+    value: ".*"
+  action: delete
+
diff --git a/traffic_ops/install/bin/convert_profile/convert_profile.go b/traffic_ops/install/bin/convert_profile/convert_profile.go
index 143677e..448a0cd 100644
--- a/traffic_ops/install/bin/convert_profile/convert_profile.go
+++ b/traffic_ops/install/bin/convert_profile/convert_profile.go
@@ -22,6 +22,7 @@ package main
 import (
 	"bytes"
 	"encoding/json"
+	"gopkg.in/yaml.v2"
 	"flag"
 	"fmt"
 	"io/ioutil"
@@ -39,41 +40,43 @@ type InputConfigParams struct {
 
 // TrafficOps Profile Parsing
 type Profile struct {
-	Parameters  []Parameter `json:"parameters"`
-	Description ProfileDesc `json:"profile"`
+	Parameters  []Parameter `json:"parameters" yaml:"parameters"`
+	Description ProfileDesc `json:"profile" yaml:"profile"`
 }
 
 type Parameter struct {
-	Name       string `json:"name"`
-	ConfigFile string `json:"config_file"`
-	Value      string `json:"value"`
+	Name       string `json:"name" yaml:"name"`
+	ConfigFile string `json:"config_file" yaml:"config_file"`
+	Value      string `json:"value" yaml:"value"`
 }
 
 type ProfileDesc struct {
-	Description string `json:"description"`
-	Name        string `json:"name"`
-	Type        string `json:"type"`
+	Description string `json:"description" yaml:"description"`
+	Name        string `json:"name" yaml:"name"`
+	Type        string `json:"type" yaml:"type"`
+	Cdn         string `json:"cdn" yaml:"cdn"`
 }
 
 // ConversionPolicy Parsing
 type ConversionPolicy struct {
-	ValidateParameters []Parameter      `json:"validate_parameters"`
-	ReplaceName        ReplaceRule      `json:"replace_name"`
-	ReplaceDescription ReplaceRule      `json:"replace_description"`
-	ConversionRules    []ConversionRule `json:"conversion_actions"`
+	ValidateParameters []Parameter      `json:"validate_parameters" yaml:"validate_parameters"`
+	ReplaceName        ReplaceRule      `json:"replace_name" yaml:"replace_name"`
+	ReplaceDescription ReplaceRule      `json:"replace_description" yaml:"replace_description"`
+	ConversionRules    []ConversionRule `json:"conversion_actions" yaml:"conversion_actions"`
+	AddParameters      []Parameter      `json:"add_parameters" yaml:"add_parameters"`
 }
 
 type ReplaceRule struct {
-	Old string `json:"old"`
-	New string `json:"new"`
+	Old string `json:"old" yaml:"old"`
+	New string `json:"new" yaml:"new"`
 }
 
 type ConversionRule struct {
-	MatchParameter Parameter `json:"match_parameter"`
-	NewName        string    `json:"new_name"`
-	NewConfigFile  string    `json:"new_config_file"`
-	NewValue       string    `json:"new_value"`
-	Action         string    `json:"action"`
+	MatchParameter Parameter `json:"match_parameter" yaml:"match_parameter"`
+	NewName        string    `json:"new_name" yaml:"new_name"`
+	NewConfigFile  string    `json:"new_config_file" yaml:"new_config_file"`
+	NewValue       string    `json:"new_value" yaml:"new_value"`
+	Action         string    `json:"action" yaml:"action"`
 }
 
 func formatParam(p Parameter) string {
@@ -87,9 +90,8 @@ func (cr ConversionRule) Apply(param Parameter) (Parameter, bool) {
 	inParam := formatParam(param)
 
 	if cr.Action == "delete" {
-		fmt.Fprintf(os.Stderr, "Deleting parameter %s\n", inParam)
+		fmt.Fprintf(os.Stdout, "Deleting parameter %s\n", inParam)
 		return param, false
-
 	} else if cr.Action != "" {
 		fmt.Fprintf(os.Stderr, "[WARNING] Unknown action %s, skipping action\n", cr.Action)
 	}
@@ -105,7 +107,7 @@ func (cr ConversionRule) Apply(param Parameter) (Parameter, bool) {
 	if cr.NewValue != "" {
 		param.Value = cr.NewValue
 	}
-	fmt.Fprintf(os.Stderr, "Updating parameter %s to %s\n", inParam, formatParam(param))
+	fmt.Fprintf(os.Stdout, "Updating parameter %s to %s\n", inParam, formatParam(param))
 
 	return param, true
 }
@@ -143,7 +145,7 @@ func readFile(inFile string) []byte {
 
 func parseInputProfile(inFile string) *Profile {
 	var pt Profile
-	err := json.Unmarshal(readFile(inFile), &pt)
+	err := yaml.Unmarshal(readFile(inFile), &pt)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "[ERROR] Cannot parse input profile\n")
 		panic(err)
@@ -154,7 +156,7 @@ func parseInputProfile(inFile string) *Profile {
 
 func parseInputRules(inFile string) *ConversionPolicy {
 	var cp ConversionPolicy
-	err := json.Unmarshal(readFile(inFile), &cp)
+	err := yaml.Unmarshal(readFile(inFile), &cp)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "[ERROR] Cannot parse conversion rules\n")
 		panic(err)
@@ -191,6 +193,17 @@ func ValidateParameters(profile *Profile,
 	return true
 }
 
+// There is currently no check if the name, config_file or value is already present
+func AddParameters(profile *Profile, params []Parameter) {
+	filteredParams := profile.Parameters
+	for _, param := range params {
+		// fmt.Fprintf(os.Stdout, "Adding parameter %s\n", add.Name)
+		fmt.Fprintf(os.Stdout, "Adding parameter %s\n", formatParam(param))
+		filteredParams = append(filteredParams, param)
+	}
+	profile.Parameters = filteredParams
+}
+
 // ConvertProfile will modify paramaters as described by matching entries in conversionActions
 // If ignoreValue is set to true, the Value field in matcher will be ignored, effectively matching
 // all values
@@ -202,6 +215,7 @@ func ConvertProfile(profile *Profile,
 	for _, param := range profile.Parameters {
 
 		matched := false
+		//fmt.Fprintf(os.Stdout, "ConvertProfile: %s\n", param.Name)
 		for _, rule := range rules {
 			if paramsMatch(rule.MatchParameter, param, ignoreValue) {
 				matched = true
@@ -230,6 +244,7 @@ func paramsMatch(matcher Parameter, param Parameter, ignoreValue bool) bool {
 	cfgRe := regexp.MustCompile(matcher.ConfigFile)
 	valueRe := regexp.MustCompile(matcher.Value)
 
+
 	if nil != nameRe.FindStringIndex(param.Name) &&
 		nil != cfgRe.FindStringIndex(param.ConfigFile) {
 
@@ -267,6 +282,7 @@ func main() {
 	}
 	ConvertProfile(inProfile, rules.ConversionRules, cfgParam.Force)
 	UpdateDetails(inProfile, rules)
+	AddParameters(inProfile, rules.AddParameters)
 
 	// Can't use the standard JSON Marshaller because it forces HTML escape
 	buf := new(bytes.Buffer)