You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@trafficcontrol.apache.org by oc...@apache.org on 2020/07/17 01:34:50 UTC
[trafficcontrol] branch master updated: Add ORT inferring location
params (#4861)
This is an automated email from the ASF dual-hosted git repository.
ocket8888 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/trafficcontrol.git
The following commit(s) were added to refs/heads/master by this push:
new cdf4679 Add ORT inferring location params (#4861)
cdf4679 is described below
commit cdf46794759b663fc50284e9ceea5d94f4628b57
Author: Robert O Butts <ro...@users.noreply.github.com>
AuthorDate: Thu Jul 16 19:34:35 2020 -0600
Add ORT inferring location params (#4861)
Adds ORT location param inference. If location Parameters do not exist
the files are created and added anyway with the directory determined
from the local ATS install, for:
- cache.config
- hosting.config
- ip_allow.config
- parent.config
- plugin.config
- records.config
- remap.config
- storage.config
- volume.config
- Delivery Services with
- Edge Header Rewrite
- Mid Header Rewrite,
- Cache URL
- URL Sig
- URI Signing
Note this is not an exhaustive of required files, and many files
must be dynamic and cannot be inferred.
But this does remove the manual configuration for this list.
More files may be added in the future.
---
lib/go-atscfg/meta.go | 197 +++++++++++++++++++---
lib/go-atscfg/meta_test.go | 88 ++++++----
lib/go-tc/ats.go | 4 +-
traffic_ops/testing/api/v1/atsconfig_meta_test.go | 8 -
traffic_ops_ort/atstccfg/atstccfg.go | 2 +-
traffic_ops_ort/atstccfg/cfgfile/all.go | 20 ++-
traffic_ops_ort/atstccfg/cfgfile/cfgfile_test.go | 9 +-
traffic_ops_ort/atstccfg/cfgfile/meta.go | 15 +-
traffic_ops_ort/atstccfg/cfgfile/routing.go | 24 +--
traffic_ops_ort/atstccfg/config/config.go | 4 +
traffic_ops_ort/traffic_ops_ort.pl | 24 ++-
11 files changed, 299 insertions(+), 96 deletions(-)
diff --git a/lib/go-atscfg/meta.go b/lib/go-atscfg/meta.go
index d2583ce..fd80993 100644
--- a/lib/go-atscfg/meta.go
+++ b/lib/go-atscfg/meta.go
@@ -21,6 +21,8 @@ package atscfg
import (
"encoding/json"
+ "errors"
+ "path/filepath"
"strings"
"github.com/apache/trafficcontrol/lib/go-log"
@@ -31,7 +33,6 @@ type ConfigProfileParams struct {
FileNameOnDisk string
Location string
URL string
- APIURI string
}
// APIVersion is the Traffic Ops API version for config fiels.
@@ -40,6 +41,23 @@ type ConfigProfileParams struct {
// TODO change the config system to not use old API paths, and remove this.
const APIVersion = "2.0"
+// requiredFiles is a constant (because Go doesn't allow const slices).
+// Note these are not exhaustive. This is only used to error if these are missing.
+// The presence of these is no guarantee the location Parameters are complete and correct.
+func requiredFiles() []string {
+ return []string{
+ "cache.config",
+ "hosting.config",
+ "ip_allow.config",
+ "parent.config",
+ "plugin.config",
+ "records.config",
+ "remap.config",
+ "storage.config",
+ "volume.config",
+ }
+}
+
func MakeMetaConfig(
serverHostName tc.CacheName,
server *ServerInfo,
@@ -50,10 +68,16 @@ func MakeMetaConfig(
scopeParams map[string]string, // map[configFileName]scopeParam
dsNames map[tc.DeliveryServiceName]struct{},
) string {
- return MetaObjToMetaConfig(MakeMetaObj(serverHostName, server, tmURL, tmReverseProxyURL, locationParams, uriSignedDSes, scopeParams, dsNames))
-}
-
-func MetaObjToMetaConfig(atsData tc.ATSConfigMetaData) string {
+ // dses are only used when configDir is not empty
+ dses := map[tc.DeliveryServiceName]tc.DeliveryServiceNullable{}
+ for dsName, _ := range dsNames {
+ dses[dsName] = tc.DeliveryServiceNullable{}
+ }
+ configDir := ""
+ atsData, err := MakeMetaObj(serverHostName, server, tmURL, tmReverseProxyURL, locationParams, uriSignedDSes, scopeParams, dses, configDir)
+ if err != nil {
+ return "error creating meta config: " + err.Error()
+ }
bts, err := json.Marshal(atsData)
if err != nil {
// should never happen
@@ -63,6 +87,146 @@ func MetaObjToMetaConfig(atsData tc.ATSConfigMetaData) string {
return string(bts)
}
+// AddMetaObjConfigDir takes the Meta Object generated from TO data, and the ATS config directory
+// and prepends the config directory to all relative paths,
+// and creates MetaObj entries for all required config files which have no location parameter.
+// If configDir is empty and any location Parameters have relative paths, returns an error.
+func AddMetaObjConfigDir(
+ metaObj tc.ATSConfigMetaData,
+ configDir string,
+ serverHostName tc.CacheName,
+ server *ServerInfo,
+ tmURL string, // global tm.url Parameter
+ tmReverseProxyURL string, // global tm.rev_proxy.url Parameter
+ locationParams map[string]ConfigProfileParams, // map[configFile]params; 'location' and 'URL' Parameters on serverHostName's Profile
+ uriSignedDSes []tc.DeliveryServiceName,
+ scopeParams map[string]string, // map[configFileName]scopeParam
+ dses map[tc.DeliveryServiceName]tc.DeliveryServiceNullable,
+) (tc.ATSConfigMetaData, error) {
+
+ // Note there may be multiple files with the same name in different directories.
+ configFilesM := map[string][]tc.ATSConfigMetaDataConfigFile{} // map[fileShortName]tc.ATSConfigMetaDataConfigFile
+ for _, fi := range metaObj.ConfigFiles {
+ configFilesM[fi.FileNameOnDisk] = append(configFilesM[fi.FileNameOnDisk], fi)
+ }
+
+ // add all strictly required files, all of which should be in the base config directory.
+ // If they don't exist, create them.
+ // If they exist with a relative path, prepend configDir.
+ // If any exist with a relative path, or don't exist, and configDir is empty, return an error.
+ for _, fileName := range requiredFiles() {
+ if _, ok := configFilesM[fileName]; ok {
+ continue
+ }
+ if configDir == "" {
+ return metaObj, errors.New("required file '" + fileName + "' has no location Parameter, and ATS config directory not found.")
+ }
+ configFilesM[fileName] = []tc.ATSConfigMetaDataConfigFile{{
+ FileNameOnDisk: fileName,
+ Location: configDir,
+ Scope: string(getServerScope(fileName, server.Type, nil)),
+ }}
+ }
+
+ for fileName, fis := range configFilesM {
+ newFis := []tc.ATSConfigMetaDataConfigFile{}
+ for _, fi := range fis {
+ if !filepath.IsAbs(fi.Location) {
+ if configDir == "" {
+ return metaObj, errors.New("file '" + fileName + "' has location Parameter with relative path '" + fi.Location + "', but ATS config directory was not found.")
+ }
+ absPath := filepath.Join(configDir, fi.Location)
+ fi.Location = absPath
+ }
+ newFis = append(newFis, fi)
+ }
+ configFilesM[fileName] = newFis
+ }
+
+ for _, ds := range dses {
+ if ds.XMLID == nil {
+ log.Errorln("meta config generation got Delivery Service with nil XMLID - not considering!")
+ continue
+ }
+ err := error(nil)
+ // Note we log errors, but don't return them.
+ // If an individual DS has an error, we don't want to break the rest of the CDN.
+ if (ds.EdgeHeaderRewrite != nil || ds.MaxOriginConnections != nil) &&
+ strings.HasPrefix(server.Type, tc.EdgeTypePrefix) {
+ fileName := "hdr_rw_" + *ds.XMLID + ".config"
+ scope := tc.ATSConfigMetaDataConfigFileScopeCDNs
+ if configFilesM, err = ensureConfigFile(configFilesM, fileName, configDir, scope); err != nil {
+ log.Errorln("meta config generation: " + err.Error())
+ }
+ }
+ if (ds.MidHeaderRewrite != nil || ds.MaxOriginConnections != nil) &&
+ ds.Type != nil && ds.Type.UsesMidCache() &&
+ strings.HasPrefix(server.Type, tc.MidTypePrefix) {
+ fileName := "hdr_rw_mid_" + *ds.XMLID + ".config"
+ scope := tc.ATSConfigMetaDataConfigFileScopeCDNs
+ if configFilesM, err = ensureConfigFile(configFilesM, fileName, configDir, scope); err != nil {
+ log.Errorln("meta config generation: " + err.Error())
+ }
+ }
+ if ds.RegexRemap != nil {
+ configFile := "regex_remap_" + *ds.XMLID + ".config"
+ scope := tc.ATSConfigMetaDataConfigFileScopeCDNs
+ if configFilesM, err = ensureConfigFile(configFilesM, configFile, configDir, scope); err != nil {
+ log.Errorln("meta config generation: " + err.Error())
+ }
+ }
+ if ds.CacheURL != nil {
+ configFile := "cacheurl_" + *ds.XMLID + ".config"
+ scope := tc.ATSConfigMetaDataConfigFileScopeCDNs
+ if configFilesM, err = ensureConfigFile(configFilesM, configFile, configDir, scope); err != nil {
+ log.Errorln("meta config generation: " + err.Error())
+ }
+ }
+ if ds.SigningAlgorithm != nil && *ds.SigningAlgorithm == tc.SigningAlgorithmURLSig {
+ configFile := "url_sig_" + *ds.XMLID + ".config"
+ scope := tc.ATSConfigMetaDataConfigFileScopeProfiles
+ if configFilesM, err = ensureConfigFile(configFilesM, configFile, configDir, scope); err != nil {
+ log.Errorln("meta config generation: " + err.Error())
+ }
+ }
+ if ds.SigningAlgorithm != nil && *ds.SigningAlgorithm == tc.SigningAlgorithmURISigning {
+ configFile := "uri_signing_" + *ds.XMLID + ".config"
+ scope := tc.ATSConfigMetaDataConfigFileScopeProfiles
+ if configFilesM, err = ensureConfigFile(configFilesM, configFile, configDir, scope); err != nil {
+ log.Errorln("meta config generation: " + err.Error())
+ }
+ }
+ }
+ // TODO add location params for ds ensure garbage here
+
+ newFiles := []tc.ATSConfigMetaDataConfigFile{}
+ for _, fis := range configFilesM {
+ for _, fi := range fis {
+ newFiles = append(newFiles, fi)
+ }
+ }
+ metaObj.ConfigFiles = newFiles
+ return metaObj, nil
+}
+
+// ensureConfigFile ensures files contains the given fileName. If so, returns files unmodified.
+// If not, if configDir is empty, returns an error.
+// If not, and configDir is nonempty, creates the given file, configDir location, and scope, and returns files.
+func ensureConfigFile(files map[string][]tc.ATSConfigMetaDataConfigFile, fileName string, configDir string, scope tc.ATSConfigMetaDataConfigFileScope) (map[string][]tc.ATSConfigMetaDataConfigFile, error) {
+ if _, ok := files[fileName]; ok {
+ return files, nil
+ }
+ if configDir == "" {
+ return files, errors.New("required file '" + fileName + "' has no location Parameter, and ATS config directory not found.")
+ }
+ files[fileName] = []tc.ATSConfigMetaDataConfigFile{{
+ FileNameOnDisk: fileName,
+ Location: configDir,
+ Scope: string(scope),
+ }}
+ return files, nil
+}
+
func MakeMetaObj(
serverHostName tc.CacheName,
server *ServerInfo,
@@ -71,8 +235,9 @@ func MakeMetaObj(
locationParams map[string]ConfigProfileParams, // map[configFile]params; 'location' and 'URL' Parameters on serverHostName's Profile
uriSignedDSes []tc.DeliveryServiceName,
scopeParams map[string]string, // map[configFileName]scopeParam
- dsNames map[tc.DeliveryServiceName]struct{},
-) tc.ATSConfigMetaData {
+ dses map[tc.DeliveryServiceName]tc.DeliveryServiceNullable,
+ configDir string,
+) (tc.ATSConfigMetaData, error) {
if tmURL == "" {
log.Errorln("ats.GetConfigMetadata: global tm.url parameter missing or empty! Setting empty in meta config!")
}
@@ -121,7 +286,7 @@ locationParamsFor:
for _, prefix := range dsConfigFilePrefixes {
if strings.HasPrefix(cfgFile, prefix) {
dsName := strings.TrimSuffix(strings.TrimPrefix(cfgFile, prefix), ".config")
- if _, ok := dsNames[tc.DeliveryServiceName(dsName)]; !ok {
+ if _, ok := dses[tc.DeliveryServiceName(dsName)]; !ok {
log.Warnln("Server Profile had 'location' Parameter '" + cfgFile + "', but delivery Service '" + dsName + "' is not assigned to this Server! Not including in meta config!")
continue locationParamsFor
}
@@ -138,25 +303,17 @@ locationParamsFor:
scope := getServerScope(cfgFile, server.Type, scopeParams)
if cfgParams.URL != "" {
+ // TODO this is legacy, from when a custom URL could be set via Parameters.
+ // verify nobody is relying on it in a production system, and remove.
scope = tc.ATSConfigMetaDataConfigFileScopeCDNs
- atsCfg.URL = cfgParams.URL
- } else {
- scopeID := ""
- if scope == tc.ATSConfigMetaDataConfigFileScopeCDNs {
- scopeID = string(server.CDN)
- } else if scope == tc.ATSConfigMetaDataConfigFileScopeProfiles {
- scopeID = server.ProfileName
- } else { // ATSConfigMetaDataConfigFileScopeServers
- scopeID = server.HostName
- }
- atsCfg.APIURI = "/api/" + APIVersion + "/" + string(scope) + "/" + scopeID + "/configfiles/ats/" + cfgFile
}
atsCfg.Scope = string(scope)
atsData.ConfigFiles = append(atsData.ConfigFiles, atsCfg)
}
- return atsData
+
+ return AddMetaObjConfigDir(atsData, configDir, serverHostName, server, tmURL, tmReverseProxyURL, locationParams, uriSignedDSes, scopeParams, dses)
}
func getServerScope(cfgFile string, serverType string, scopeParams map[string]string) tc.ATSConfigMetaDataConfigFileScope {
diff --git a/lib/go-atscfg/meta_test.go b/lib/go-atscfg/meta_test.go
index dbd7b52..e735c75 100644
--- a/lib/go-atscfg/meta_test.go
+++ b/lib/go-atscfg/meta_test.go
@@ -20,7 +20,6 @@ package atscfg
*/
import (
- "encoding/json"
"strings"
"testing"
@@ -51,15 +50,10 @@ func TestMakeMetaConfig(t *testing.T) {
tmURL := "https://myto.invalid"
tmReverseProxyURL := "https://myrp.myto.invalid"
locationParams := map[string]ConfigProfileParams{
- "remap.config": ConfigProfileParams{
- FileNameOnDisk: "remap.config",
- Location: "/my/location/",
- },
"regex_revalidate.config": ConfigProfileParams{
FileNameOnDisk: "regex_revalidate.config",
Location: "/my/location/",
URL: "http://myurl/remap.config", // cdn-scoped endpoint
- APIURI: "http://myapi/remap.config",
},
"cache.config": ConfigProfileParams{
FileNameOnDisk: "cache.config", // cache.config on mids is server-scoped
@@ -116,15 +110,14 @@ func TestMakeMetaConfig(t *testing.T) {
},
}
uriSignedDSes := []tc.DeliveryServiceName{"mydsname"}
- dses := map[tc.DeliveryServiceName]struct{}{"mydsname": {}}
+ dses := map[tc.DeliveryServiceName]tc.DeliveryServiceNullable{"mydsname": {}}
scopeParams := map[string]string{"custom.config": string(tc.ATSConfigMetaDataConfigFileScopeProfiles)}
- txt := MakeMetaConfig(serverHostName, server, tmURL, tmReverseProxyURL, locationParams, uriSignedDSes, scopeParams, dses)
-
- cfg := tc.ATSConfigMetaData{}
- if err := json.Unmarshal([]byte(txt), &cfg); err != nil {
- t.Fatalf("MakeMetaConfig returned invalid JSON: " + err.Error())
+ cfgPath := "/etc/foo/trafficserver"
+ cfg, err := MakeMetaObj(serverHostName, server, tmURL, tmReverseProxyURL, locationParams, uriSignedDSes, scopeParams, dses, cfgPath)
+ if err != nil {
+ t.Fatalf("MakeMetaObj: " + err.Error())
}
if cfg.Info.ProfileID != int(server.ProfileID) {
@@ -223,7 +216,7 @@ func TestMakeMetaConfig(t *testing.T) {
}
},
"remap.config": func(cf tc.ATSConfigMetaDataConfigFile) {
- if expected := "/my/location/"; cf.Location != expected {
+ if expected := cfgPath; cf.Location != expected {
t.Errorf("expected location '%v', actual '%v'", expected, cf.Location)
}
if expected := string(tc.ATSConfigMetaDataConfigFileScopeServers); cf.Scope != expected {
@@ -246,6 +239,46 @@ func TestMakeMetaConfig(t *testing.T) {
t.Errorf("expected scope '%v', actual '%v'", expected, cf.Scope)
}
},
+ "hosting.config": func(cf tc.ATSConfigMetaDataConfigFile) {
+ if expected := cfgPath; cf.Location != expected {
+ t.Errorf("expected location '%v', actual '%v'", expected, cf.Location)
+ }
+ if expected := string(tc.ATSConfigMetaDataConfigFileScopeServers); cf.Scope != expected {
+ t.Errorf("expected scope for %v is '%v', actual '%v'", cf.FileNameOnDisk, expected, cf.Scope)
+ }
+ },
+ "parent.config": func(cf tc.ATSConfigMetaDataConfigFile) {
+ if expected := cfgPath; cf.Location != expected {
+ t.Errorf("expected location '%v', actual '%v'", expected, cf.Location)
+ }
+ if expected := string(tc.ATSConfigMetaDataConfigFileScopeServers); cf.Scope != expected {
+ t.Errorf("expected scope for %v is '%v', actual '%v'", cf.FileNameOnDisk, expected, cf.Scope)
+ }
+ },
+ "plugin.config": func(cf tc.ATSConfigMetaDataConfigFile) {
+ if expected := cfgPath; cf.Location != expected {
+ t.Errorf("expected location '%v', actual '%v'", expected, cf.Location)
+ }
+ if expected := string(tc.ATSConfigMetaDataConfigFileScopeProfiles); cf.Scope != expected {
+ t.Errorf("expected scope for %v is '%v', actual '%v'", cf.FileNameOnDisk, expected, cf.Scope)
+ }
+ },
+ "records.config": func(cf tc.ATSConfigMetaDataConfigFile) {
+ if expected := cfgPath; cf.Location != expected {
+ t.Errorf("expected location '%v', actual '%v'", expected, cf.Location)
+ }
+ if expected := string(tc.ATSConfigMetaDataConfigFileScopeProfiles); cf.Scope != expected {
+ t.Errorf("expected scope for %v is '%v', actual '%v'", cf.FileNameOnDisk, expected, cf.Scope)
+ }
+ },
+ "storage.config": func(cf tc.ATSConfigMetaDataConfigFile) {
+ if expected := cfgPath; cf.Location != expected {
+ t.Errorf("expected location '%v', actual '%v'", expected, cf.Location)
+ }
+ if expected := string(tc.ATSConfigMetaDataConfigFileScopeProfiles); cf.Scope != expected {
+ t.Errorf("expected scope for %v is '%v', actual '%v'", cf.FileNameOnDisk, expected, cf.Scope)
+ }
+ },
}
for _, cfgFile := range cfg.ConfigFiles {
@@ -258,10 +291,9 @@ func TestMakeMetaConfig(t *testing.T) {
}
server.Type = "MID"
- txt = MakeMetaConfig(serverHostName, server, tmURL, tmReverseProxyURL, locationParams, uriSignedDSes, scopeParams, dses)
- cfg = tc.ATSConfigMetaData{}
- if err := json.Unmarshal([]byte(txt), &cfg); err != nil {
- t.Fatalf("MakeMetaConfig returned invalid JSON: " + err.Error())
+ cfg, err = MakeMetaObj(serverHostName, server, tmURL, tmReverseProxyURL, locationParams, uriSignedDSes, scopeParams, dses, cfgPath)
+ if err != nil {
+ t.Fatalf("MakeMetaObj: " + err.Error())
}
for _, cfgFile := range cfg.ConfigFiles {
if cfgFile.FileNameOnDisk != "cache.config" {
@@ -272,23 +304,15 @@ func TestMakeMetaConfig(t *testing.T) {
}
break
}
- if strings.Contains(txt, "nonexistentds") {
- t.Errorf("expected location parameters for nonexistent delivery services to not be added to config, actual '%v'", txt)
- }
-
- // check for expected apiUri vs url keys (if values are empty strings, they should be omitted from the json)
- m := map[string]interface{}{}
- if err := json.Unmarshal([]byte(txt), &m); err != nil {
- t.Fatalf("MakeMetaConfig returned invalid JSON: " + err.Error())
- }
- cfl := m["configFiles"].([]interface{})
- for _, cf := range cfl {
- c := cf.(map[string]interface{})
- if c["fnameOnDisk"] == "external.config" {
- if _, exists := c["apiUri"]; exists {
+ for _, fi := range cfg.ConfigFiles {
+ if strings.Contains(fi.FileNameOnDisk, "nonexistentds") {
+ t.Errorf("expected location parameters for nonexistent delivery services to not be added to config, actual '%v'", fi.FileNameOnDisk)
+ }
+ if fi.FileNameOnDisk == "external.config" {
+ if fi.APIURI != "" {
t.Errorf("expected: apiUri field to be omitted for external.config, actual: present")
}
- if _, exists := c["url"]; !exists {
+ if fi.URL != "" {
t.Errorf("expected: url field to be present for external.config, actual: omitted")
}
}
diff --git a/lib/go-tc/ats.go b/lib/go-tc/ats.go
index 4990c63..359d2a8 100644
--- a/lib/go-tc/ats.go
+++ b/lib/go-tc/ats.go
@@ -46,8 +46,8 @@ type ATSConfigMetaDataInfo struct {
type ATSConfigMetaDataConfigFile struct {
FileNameOnDisk string `json:"fnameOnDisk"`
Location string `json:"location"`
- APIURI string `json:"apiUri,omitempty"`
- URL string `json:"url,omitempty"`
+ APIURI string `json:"apiUri,omitempty"` // APIURI is deprecated, do not use.
+ URL string `json:"url,omitempty"` // URL is deprecated, do not use.
Scope string `json:"scope"`
}
diff --git a/traffic_ops/testing/api/v1/atsconfig_meta_test.go b/traffic_ops/testing/api/v1/atsconfig_meta_test.go
index c267b9e..0372173 100644
--- a/traffic_ops/testing/api/v1/atsconfig_meta_test.go
+++ b/traffic_ops/testing/api/v1/atsconfig_meta_test.go
@@ -54,7 +54,6 @@ func GetTestATSConfigMeta(t *testing.T) {
expected := tc.ATSConfigMetaDataConfigFile{
FileNameOnDisk: "hdr_rw_ds1.config",
Location: "/remap/config/location/parameter",
- APIURI: "cdns/cdn1/configfiles/ats/hdr_rw_ds1.config", // expected suffix; config gen doesn't care about API version
URL: "",
Scope: "cdns",
}
@@ -76,9 +75,6 @@ func GetTestATSConfigMeta(t *testing.T) {
if expected.Location != actual.Location {
t.Errorf("Getting server '"+server.HostName+"' config list: expected: %+v actual: %+v\n", expected, *actual)
}
- if !strings.HasSuffix(actual.APIURI, expected.APIURI) {
- t.Errorf("Getting server '"+server.HostName+"' config list: expected: %+v actual: %+v\n", expected, *actual)
- }
if actual.Scope != expected.Scope {
t.Errorf("Getting server '"+server.HostName+"' config list: expected: %+v actual: %+v\n", expected, *actual)
}
@@ -114,7 +110,6 @@ func GetTestATSConfigMetaMidHdrRw(t *testing.T) {
expected := tc.ATSConfigMetaDataConfigFile{
FileNameOnDisk: "hdr_rw_mid_ds1nat.config",
Location: "/remap/config/location/parameter",
- APIURI: "cdns/cdn1/configfiles/ats/hdr_rw_mid_ds1nat.config", // expected suffix; config gen doesn't care about API version
URL: "",
Scope: "cdns",
}
@@ -136,9 +131,6 @@ func GetTestATSConfigMetaMidHdrRw(t *testing.T) {
if expected.Location != actual.Location {
t.Errorf("Getting server '"+server.HostName+"' config list: expected: %+v actual: %+v\n", expected, *actual)
}
- if !strings.HasSuffix(actual.APIURI, expected.APIURI) {
- t.Errorf("Getting server '"+server.HostName+"' config list: expected: %+v actual: %+v\n", expected, *actual)
- }
if actual.Scope != expected.Scope {
t.Errorf("Getting server '"+server.HostName+"' config list: expected: %+v actual: %+v\n", expected, *actual)
}
diff --git a/traffic_ops_ort/atstccfg/atstccfg.go b/traffic_ops_ort/atstccfg/atstccfg.go
index cb222cf..ed7409b 100644
--- a/traffic_ops_ort/atstccfg/atstccfg.go
+++ b/traffic_ops_ort/atstccfg/atstccfg.go
@@ -109,7 +109,7 @@ func main() {
os.Exit(config.ExitCodeErrGeneric)
}
- configs, err := cfgfile.GetAllConfigs(toData, tccfg.RevalOnly)
+ configs, err := cfgfile.GetAllConfigs(toData, tccfg.RevalOnly, tccfg.Dir)
if err != nil {
log.Errorln("Getting config for'" + cfg.CacheHostName + "': " + err.Error())
os.Exit(config.ExitCodeErrGeneric)
diff --git a/traffic_ops_ort/atstccfg/cfgfile/all.go b/traffic_ops_ort/atstccfg/cfgfile/all.go
index 9072edc..d855200 100644
--- a/traffic_ops_ort/atstccfg/cfgfile/all.go
+++ b/traffic_ops_ort/atstccfg/cfgfile/all.go
@@ -26,6 +26,7 @@ import (
"mime/multipart"
"path/filepath"
"regexp"
+ "sort"
"strconv"
"strings"
@@ -36,8 +37,8 @@ import (
)
// GetAllConfigs gets all config files for cfg.CacheHostName.
-func GetAllConfigs(toData *config.TOData, revalOnly bool) ([]config.ATSConfigFile, error) {
- meta, err := GetMeta(toData)
+func GetAllConfigs(toData *config.TOData, revalOnly bool, dir string) ([]config.ATSConfigFile, error) {
+ meta, err := GetMeta(toData, dir)
if err != nil {
return nil, errors.New("creating meta: " + err.Error())
}
@@ -50,7 +51,7 @@ func GetAllConfigs(toData *config.TOData, revalOnly bool) ([]config.ATSConfigFil
}
txt, contentType, lineComment, err := GetConfigFile(toData, fi)
if err != nil {
- return nil, errors.New("getting config file '" + fi.APIURI + "': " + err.Error())
+ return nil, errors.New("getting config file '" + fi.FileNameOnDisk + "': " + err.Error())
}
if fi.FileNameOnDisk == atscfg.SSLMultiCertConfigFileName {
hasSSLMultiCertConfig = true
@@ -75,6 +76,7 @@ const HdrLineComment = "Line-Comment"
// WriteConfigs writes the given configs as a RFC2046ยง5.1 MIME multipart/mixed message.
func WriteConfigs(configs []config.ATSConfigFile, output io.Writer) error {
+ sort.Sort(ATSConfigFiles(configs))
w := multipart.NewWriter(output)
// Create a unique boundary. Because we're using a text encoding, we need to make sure the boundary text doesn't occur in any body.
@@ -113,6 +115,18 @@ func WriteConfigs(configs []config.ATSConfigFile, output io.Writer) error {
return nil
}
+// ATSConfigFiles implements sort.Interface to sort by path.
+type ATSConfigFiles []config.ATSConfigFile
+
+func (p ATSConfigFiles) Len() int { return len(p) }
+func (p ATSConfigFiles) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
+func (p ATSConfigFiles) Less(i, j int) bool {
+ if p[i].Location != p[j].Location {
+ return p[i].Location < p[j].Location
+ }
+ return p[i].FileNameOnDisk < p[j].FileNameOnDisk
+}
+
var returnRegex = regexp.MustCompile(`\s*__RETURN__\s*`)
// PreprocessConfigFile does global preprocessing on the given config file cfgFile.
diff --git a/traffic_ops_ort/atstccfg/cfgfile/cfgfile_test.go b/traffic_ops_ort/atstccfg/cfgfile/cfgfile_test.go
index a2e88bf..c05bc40 100644
--- a/traffic_ops_ort/atstccfg/cfgfile/cfgfile_test.go
+++ b/traffic_ops_ort/atstccfg/cfgfile/cfgfile_test.go
@@ -21,7 +21,6 @@ package cfgfile
import (
"bytes"
-
"math/rand"
"strings"
"testing"
@@ -29,6 +28,7 @@ import (
"github.com/apache/trafficcontrol/lib/go-atscfg"
"github.com/apache/trafficcontrol/lib/go-tc"
+ "github.com/apache/trafficcontrol/lib/go-util"
"github.com/apache/trafficcontrol/traffic_ops_ort/atstccfg/config"
)
@@ -125,7 +125,8 @@ func TestGetAllConfigsWriteConfigsDeterministic(t *testing.T) {
// TODO expand fake data. Currently, it's only making a remap.config.
toData := MakeFakeTOData()
revalOnly := false
- configs, err := GetAllConfigs(toData, revalOnly)
+ cfgPath := "/etc/trafficserver/"
+ configs, err := GetAllConfigs(toData, revalOnly, cfgPath)
if err != nil {
t.Fatalf("error getting configs: " + err.Error())
}
@@ -138,7 +139,7 @@ func TestGetAllConfigsWriteConfigsDeterministic(t *testing.T) {
configStr = removeComments(configStr)
for i := 0; i < 10; i++ {
- configs2, err := GetAllConfigs(toData, revalOnly)
+ configs2, err := GetAllConfigs(toData, revalOnly, cfgPath)
if err != nil {
t.Fatalf("error getting configs2: " + err.Error())
}
@@ -268,7 +269,7 @@ func randDS() *tc.DeliveryServiceNullable {
ds.MissLong = randFloat64()
ds.MultiSiteOrigin = randBool()
ds.OriginShield = randStr()
- ds.OrgServerFQDN = randStr()
+ ds.OrgServerFQDN = util.StrPtr("http://" + *(randStr()))
ds.ProfileDesc = randStr()
ds.ProfileID = randInt()
ds.ProfileName = randStr()
diff --git a/traffic_ops_ort/atstccfg/cfgfile/meta.go b/traffic_ops_ort/atstccfg/cfgfile/meta.go
index a9851d2..b4b4490 100644
--- a/traffic_ops_ort/atstccfg/cfgfile/meta.go
+++ b/traffic_ops_ort/atstccfg/cfgfile/meta.go
@@ -27,7 +27,7 @@ import (
"github.com/apache/trafficcontrol/traffic_ops_ort/atstccfg/config"
)
-func GetMeta(toData *config.TOData) (*tc.ATSConfigMetaData, error) {
+func GetMeta(toData *config.TOData, dir string) (*tc.ATSConfigMetaData, error) {
cgMap := map[string]tc.CacheGroupNullable{}
for _, cg := range toData.CacheGroups {
if cg.Name == nil {
@@ -125,7 +125,7 @@ func GetMeta(toData *config.TOData) (*tc.ATSConfigMetaData, error) {
}
}
- dsNames := map[tc.DeliveryServiceName]struct{}{}
+ dses := map[tc.DeliveryServiceName]tc.DeliveryServiceNullable{}
if tc.CacheTypeFromString(toData.Server.Type) != tc.CacheTypeMid {
dsIDs := map[int]struct{}{}
for _, ds := range toData.DeliveryServices {
@@ -163,7 +163,7 @@ func GetMeta(toData *config.TOData) (*tc.ATSConfigMetaData, error) {
if _, ok := dssMap[*ds.ID]; !ok {
continue
}
- dsNames[tc.DeliveryServiceName(*ds.XMLID)] = struct{}{}
+ dses[tc.DeliveryServiceName(*ds.XMLID)] = ds
}
} else {
for _, ds := range toData.DeliveryServices {
@@ -176,7 +176,7 @@ func GetMeta(toData *config.TOData) (*tc.ATSConfigMetaData, error) {
if ds.CDNID == nil || *ds.CDNID != toData.Server.CDNID {
continue
}
- dsNames[tc.DeliveryServiceName(*ds.XMLID)] = struct{}{}
+ dses[tc.DeliveryServiceName(*ds.XMLID)] = ds
}
}
@@ -188,7 +188,7 @@ func GetMeta(toData *config.TOData) (*tc.ATSConfigMetaData, error) {
if ds.XMLID == nil {
continue // TODO log?
}
- if _, ok := dsNames[tc.DeliveryServiceName(*ds.XMLID)]; !ok {
+ if _, ok := dses[tc.DeliveryServiceName(*ds.XMLID)]; !ok {
continue
}
if ds.SigningAlgorithm == nil || *ds.SigningAlgorithm != tc.SigningAlgorithmURISigning {
@@ -197,6 +197,9 @@ func GetMeta(toData *config.TOData) (*tc.ATSConfigMetaData, error) {
uriSignedDSes = append(uriSignedDSes, tc.DeliveryServiceName(*ds.XMLID))
}
- metaObj := atscfg.MakeMetaObj(tc.CacheName(toData.Server.HostName), &serverInfo, toURL, toReverseProxyURL, locationParams, uriSignedDSes, scopeParams, dsNames)
+ metaObj, err := atscfg.MakeMetaObj(tc.CacheName(toData.Server.HostName), &serverInfo, toURL, toReverseProxyURL, locationParams, uriSignedDSes, scopeParams, dses, dir)
+ if err != nil {
+ return nil, errors.New("generating: " + err.Error())
+ }
return &metaObj, nil
}
diff --git a/traffic_ops_ort/atstccfg/cfgfile/routing.go b/traffic_ops_ort/atstccfg/cfgfile/routing.go
index 62be2d8..01fd4e9 100644
--- a/traffic_ops_ort/atstccfg/cfgfile/routing.go
+++ b/traffic_ops_ort/atstccfg/cfgfile/routing.go
@@ -37,29 +37,15 @@ var scopeConfigFileFuncs = map[string]func(toData *config.TOData, fileName strin
// GetConfigFile returns the text of the generated config file, the MIME Content Type of the config file, and any error.
func GetConfigFile(toData *config.TOData, fileInfo tc.ATSConfigMetaDataConfigFile) (string, string, string, error) {
- path := fileInfo.APIURI
- // TODO remove the URL path parsing. It's a legacy from when config files were endpoints in the meta config.
- // We should replace it with actually calling the right file and name directly.
start := time.Now()
defer func() {
- log.Infof("GetConfigFile %v took %v\n", path, time.Since(start).Round(time.Millisecond))
+ log.Infof("GetConfigFile %v took %v\n", fileInfo.FileNameOnDisk, time.Since(start).Round(time.Millisecond))
}()
-
- pathParts := strings.Split(path, "/")
- if len(pathParts) < 8 {
- return "", "", "", errors.New("unknown config file '" + path + "'")
+ log.Infoln("GetConfigFile scope '" + fileInfo.Scope + "' fileName '" + fileInfo.FileNameOnDisk + "'")
+ if scopeConfigFileFunc, ok := scopeConfigFileFuncs[fileInfo.Scope]; ok {
+ return scopeConfigFileFunc(toData, fileInfo.FileNameOnDisk)
}
- scope := pathParts[3]
- resource := pathParts[4]
- fileName := pathParts[7]
-
- log.Infoln("GetConfigFile scope '" + scope + "' resource '" + resource + "' fileName '" + fileName + "'")
-
- if scopeConfigFileFunc, ok := scopeConfigFileFuncs[scope]; ok {
- return scopeConfigFileFunc(toData, fileName)
- }
-
- return "", "", "", errors.New("unknown config file '" + fileInfo.APIURI + "'")
+ return "", "", "", errors.New("unknown config file '" + fileInfo.FileNameOnDisk + "'")
}
type ConfigFilePrefixSuffixFunc struct {
diff --git a/traffic_ops_ort/atstccfg/config/config.go b/traffic_ops_ort/atstccfg/config/config.go
index a725095..5197dee 100644
--- a/traffic_ops_ort/atstccfg/config/config.go
+++ b/traffic_ops_ort/atstccfg/config/config.go
@@ -65,6 +65,7 @@ type Cfg struct {
TOTimeout time.Duration
TOURL *url.URL
TOUser string
+ Dir string
}
type TCCfg struct {
@@ -99,6 +100,7 @@ func GetCfg() (Cfg, error) {
setRevalStatusPtr := flag.StringP("set-reval-status", "a", "", "POSTs to Traffic Ops setting the revalidate status of the server. Must be 'true' or 'false'. Requires --set-queue-status also be set")
revalOnlyPtr := flag.BoolP("revalidate-only", "y", false, "Whether to exclude files not named 'regex_revalidate.config'")
disableProxyPtr := flag.BoolP("traffic-ops-disable-proxy", "p", false, "Whether to not use the Traffic Ops proxy specified in the GLOBAL Parameter tm.rev_proxy.url")
+ dirPtr := flag.StringP("dir", "D", "", "ATS config directory, used for config files without location parameters or with relative paths. May be blank. If blank and any required config file location parameter is missing or relative, will error.")
flag.Parse()
@@ -128,6 +130,7 @@ func GetCfg() (Cfg, error) {
setRevalStatus := *setRevalStatusPtr
revalOnly := *revalOnlyPtr
disableProxy := *disableProxyPtr
+ dir := *dirPtr
urlSourceStr := "argument" // for error messages
if toURL == "" {
@@ -186,6 +189,7 @@ func GetCfg() (Cfg, error) {
SetQueueStatus: setQueueStatus,
RevalOnly: revalOnly,
DisableProxy: disableProxy,
+ Dir: dir,
}
if err := log.InitCfg(cfg); err != nil {
return Cfg{}, errors.New("Initializing loggers: " + err.Error() + "\n")
diff --git a/traffic_ops_ort/traffic_ops_ort.pl b/traffic_ops_ort/traffic_ops_ort.pl
index 8a353bb..7f3177b 100755
--- a/traffic_ops_ort/traffic_ops_ort.pl
+++ b/traffic_ops_ort/traffic_ops_ort.pl
@@ -189,6 +189,8 @@ my %install_tracker;
my $cfg_file_tracker = undef;
+my $ats_config_dir = get_ats_config_dir();
+
####-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-####
#### Start main flow
####-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-####
@@ -270,6 +272,26 @@ if ( $script_mode != $REPORT ) {
#### Subroutines
####-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-####
+# Returns the ATS config directory, if it can find it.
+# Tries rpm (yum) first, then falls back to find.
+# If it fails to find it, logs an error and returns the empty string.
+sub get_ats_config_dir {
+ my $dir = `rpm -ql trafficserver | grep -E 'etc/trafficserver\$' | tail -1`;
+ $dir =~ s/^\s+|\s+$//g; # trim leading and trailing whitespace
+ if ( $dir eq "" ) {
+ $dir = `find / -type d -path '*/etc/trafficserver' | tail -1`;
+ $dir =~ s/^\s+|\s+$//g; # trim leading and trailing whitespace
+ }
+ if ( ! length $dir ) {
+ # if it became undefined somehow, make sure we're returning ""
+ $dir = "";
+ }
+ if ( $dir eq "" ) {
+ ( $log_level >> $ERROR ) && print "ERROR Failed to find config directory, using empty string!\n";
+ }
+ return $dir;
+}
+
sub revalidate_while_sleeping {
$syncds_update = &check_revalidate_state(1);
if ( $syncds_update > 0 ) {
@@ -1583,7 +1605,7 @@ sub get_cfg_file_list {
$atstccfg_reval_arg = '--revalidate-only';
}
- my $result = `$atstccfg_cmd $atstccfg_timeout_arg $atstccfg_arg_disable_proxy --traffic-ops-user='$TO_USER' --traffic-ops-password='$TO_PASS' --traffic-ops-url='$TO_URL' --cache-host-name='$host_name' $atstccfg_reval_arg --log-location-error=stderr --log-location-warning=stderr --log-location-info=null 2>$atstccfg_log_path`;
+ my $result = `$atstccfg_cmd --dir='$ats_config_dir' $atstccfg_timeout_arg $atstccfg_arg_disable_proxy --traffic-ops-user='$TO_USER' --traffic-ops-password='$TO_PASS' --traffic-ops-url='$TO_URL' --cache-host-name='$host_name' $atstccfg_reval_arg --log-location-error=stderr --log-location-warning=stderr --log-location-info=null 2>$atstccfg_log_path`;
my $atstccfg_exit_code = $?;
if ($atstccfg_exit_code != 0) {
( $log_level >> $ERROR ) && printf("ERROR getting config files from atstccfg via Traffic Ops. See $atstccfg_log_path for details\n");