You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@trafficcontrol.apache.org by oc...@apache.org on 2022/02/07 17:29:45 UTC
[trafficcontrol] branch master updated: Add DS parameter LastRawRemap(Pre|Post) to allow raw text in remap.config (#6529)
This is an automated email from the ASF dual-hosted git repository.
ocket8888 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/trafficcontrol.git
The following commit(s) were added to refs/heads/master by this push:
new f487c44 Add DS parameter LastRawRemap(Pre|Post) to allow raw text in remap.config (#6529)
f487c44 is described below
commit f487c44e418b7c5240dc47f6ab8f023980967136
Author: Brian Olsen <br...@comcast.com>
AuthorDate: Mon Feb 7 10:28:03 2022 -0700
Add DS parameter LastRawRemap(Pre|Post) to allow raw text in remap.config (#6529)
* Add DS parameter LastRawRemap(Pre/Post) to allow raw text in remap.config
* refactor to account for last in topology
---
CHANGELOG.md | 1 +
docs/source/overview/profiles_and_parameters.rst | 5 +
lib/go-atscfg/remapdotconfig.go | 112 ++++++++++--
lib/go-atscfg/remapdotconfig_test.go | 208 +++++++++++++++++++++++
4 files changed, 310 insertions(+), 16 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 50c3353..ca7ec1c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -35,6 +35,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
- [#6405](https://github.com/apache/trafficcontrol/issues/6405) Added cache config version to all t3c apps and config file headers
- Traffic Vault: Added additional flag to TV Riak (Deprecated) Util
- Added Traffic Vault Postgres columns, a Traffic Ops API endpoint, and Traffic Portal page to show SSL certificate expiration information.
+- Added support for a DS profile parameter 'LastRawRemapPre' and 'LastRawRemapPost' which allows raw text lines to be pre or post pended to remap.config.
### Fixed
- [#6411](https://github.com/apache/trafficcontrol/pull/6411) Removes invalid 'ALL cdn' options from TP
diff --git a/docs/source/overview/profiles_and_parameters.rst b/docs/source/overview/profiles_and_parameters.rst
index c2c8305..713b529 100644
--- a/docs/source/overview/profiles_and_parameters.rst
+++ b/docs/source/overview/profiles_and_parameters.rst
@@ -621,6 +621,11 @@ The following plugins have support for adding args with following parameter Conf
| cachekey.pparam | cachekey.pparam | ``-o`` | ``@pparam=-o`` |
+------------------------+---------------------+------------------------------+--------------------------------------+
+In order to support difficult configurations at MID/LAST, a
+:term:`Delivery Service` profile parameter is available with parameters
+``LastRawRemapPre`` and ``LastRawRemapPost``, config file ``remap.config``
+and Value the raw remap lines. The Value in this parameter will be pre
+or post pended to the end of ``remap.config``.
.. seealso:: For an explanation of the syntax of this configuration file, refer to `the Apache Traffic Server remap.config documentation <https://docs.trafficserver.apache.org/en/7.1.x/admin-guide/files/remap.config.en.html>`_.
diff --git a/lib/go-atscfg/remapdotconfig.go b/lib/go-atscfg/remapdotconfig.go
index 7803dcf..40d0c75 100644
--- a/lib/go-atscfg/remapdotconfig.go
+++ b/lib/go-atscfg/remapdotconfig.go
@@ -120,7 +120,7 @@ func MakeRemapDotConfig(
// remap.config parameters use "<plugin>.pparam" key
// cachekey.config parameters retain the 'cachekey.config' key
func classifyConfigParams(configParams []tc.Parameter) map[string][]tc.Parameter {
- var configParamMap = map[string][]tc.Parameter{}
+ configParamMap := map[string][]tc.Parameter{}
for _, param := range configParams {
key := param.ConfigFile
if "remap.config" == key {
@@ -140,7 +140,7 @@ func paramsStringFor(parameters []tc.Parameter, warnings *[]string) (paramsStrin
// Try to extract argument
index := strings.IndexAny(param.Value, "= ")
- var arg string
+ arg := ""
if 0 < index {
arg = param.Value[:index]
} else {
@@ -202,6 +202,28 @@ func cachekeyArgsFor(configParamsMap map[string][]tc.Parameter, warnings *[]stri
return
}
+// lastPrePostRemapLinesFor Returns any pre or post raw remap lines.
+func lastPrePostRemapLinesFor(dsConfigParamsMap map[string][]tc.Parameter, dsid string) ([]string, []string) {
+ preRemapLines := []string{}
+ postRemapLines := []string{}
+
+ // Any raw pre pend
+ if params, ok := dsConfigParamsMap["LastRawRemapPre"]; ok {
+ for _, param := range params {
+ preRemapLines = append(preRemapLines, param.Value+" # Raw: "+dsid+"\n")
+ }
+ }
+
+ // Any raw post pend
+ if params, ok := dsConfigParamsMap["LastRawRemapPost"]; ok {
+ for _, param := range params {
+ postRemapLines = append(postRemapLines, param.Value+" # Raw: "+dsid+"\n")
+ }
+ }
+
+ return preRemapLines, postRemapLines
+}
+
// getServerConfigRemapDotConfigForMid returns the remap lines, any warnings, and any error.
func getServerConfigRemapDotConfigForMid(
atsMajorVersion int,
@@ -217,6 +239,8 @@ func getServerConfigRemapDotConfigForMid(
) (string, []string, error) {
warnings := []string{}
midRemaps := map[string]string{}
+ preRemapLines := []string{}
+ postRemapLines := []string{}
for _, ds := range dses {
if !hasRequiredCapabilities(serverCapabilities[*server.ID], dsRequiredCapabilities[*ds.ID]) {
continue
@@ -261,19 +285,19 @@ func getServerConfigRemapDotConfigForMid(
}
// Logic for handling cachekey params
- var cachekeyArgs string
+ cachekeyArgs := ""
// qstring ignore
if ds.QStringIgnore != nil && *ds.QStringIgnore == tc.QueryStringIgnoreIgnoreInCacheKeyAndPassUp {
cachekeyArgs = getQStringIgnoreRemap(atsMajorVersion)
}
- var dsConfigParamsMap map[string][]tc.Parameter
+ dsConfigParamsMap := map[string][]tc.Parameter{}
if nil != ds.ProfileID {
dsConfigParamsMap = classifyConfigParams(profilesConfigParams[*ds.ProfileID])
}
- if 0 < len(dsConfigParamsMap) {
+ if len(dsConfigParamsMap) > 0 {
cachekeyArgs += cachekeyArgsFor(dsConfigParamsMap, &warnings)
}
@@ -289,16 +313,41 @@ func getServerConfigRemapDotConfigForMid(
if midRemap != "" {
midRemaps[remapFrom] = *ds.OrgServerFQDN + midRemap
}
+
+ // Any raw pre or post pend
+ dsPreRemaps, dsPostRemaps := lastPrePostRemapLinesFor(dsConfigParamsMap, *ds.XMLID)
+
+ // Add to pre/post remap lines if this is last tier
+ if len(dsPreRemaps) > 0 || len(dsPostRemaps) > 0 {
+ isLastCache, err := serverIsLastCacheForDS(server, &ds, nameTopologies, cacheGroups)
+ if err != nil {
+ return "", warnings, errors.New("determining if cache is the last tier for ds '" + *ds.XMLID + "': " + err.Error())
+ }
+
+ if isLastCache {
+ preRemapLines = append(preRemapLines, dsPreRemaps...)
+ postRemapLines = append(postRemapLines, dsPostRemaps...)
+ }
+ }
}
textLines := []string{}
+
for originFQDN, midRemap := range midRemaps {
textLines = append(textLines, "map "+originFQDN+" "+midRemap+"\n")
}
+
+ sort.Strings(preRemapLines)
sort.Strings(textLines)
+ sort.Strings(postRemapLines)
- text := header
- text += strings.Join(textLines, "")
+ // Prepend any pre remap lines
+ remapLinesAll := append(preRemapLines, textLines...)
+
+ // Append on any post raw remap lines
+ remapLinesAll = append(remapLinesAll, postRemapLines...)
+
+ text := header + strings.Join(remapLinesAll, "")
return text, warnings, nil
}
@@ -319,6 +368,8 @@ func getServerConfigRemapDotConfigForEdge(
) (string, []string, error) {
warnings := []string{}
textLines := []string{}
+ preRemapLines := []string{}
+ postRemapLines := []string{}
for _, ds := range dses {
if !hasRequiredCapabilities(serverCapabilities[*server.ID], dsRequiredCapabilities[*ds.ID]) {
@@ -365,8 +416,16 @@ func getServerConfigRemapDotConfigForEdge(
profileremapConfigParams = profilesRemapConfigParams[*ds.ProfileID]
}
remapWarns := []string{}
- remapText, remapWarns, err = buildEdgeRemapLine(atsMajorVersion, server, serverPackageParamData, remapText, ds, line.From, line.To, profileremapConfigParams, cacheGroups, nameTopologies)
+ dsLines := RemapLines{}
+ dsLines, remapWarns, err = buildEdgeRemapLine(atsMajorVersion, server, serverPackageParamData, remapText, ds, line.From, line.To, profileremapConfigParams, cacheGroups, nameTopologies)
warnings = append(warnings, remapWarns...)
+ remapText = dsLines.Text
+
+ // Add to pre/post remap lines if this is last tier
+ if len(dsLines.Pre) > 0 || len(dsLines.Post) > 0 {
+ preRemapLines = append(preRemapLines, dsLines.Pre...)
+ postRemapLines = append(postRemapLines, dsLines.Post...)
+ }
if err != nil {
return "", warnings, err
@@ -381,12 +440,24 @@ func getServerConfigRemapDotConfigForEdge(
textLines = append(textLines, remapText)
}
- text := header
+ sort.Strings(preRemapLines)
sort.Strings(textLines)
- text += strings.Join(textLines, "")
+ sort.Strings(postRemapLines)
+
+ remapLinesAll := append(preRemapLines, textLines...)
+ remapLinesAll = append(remapLinesAll, postRemapLines...)
+
+ text := header
+ text += strings.Join(remapLinesAll, "")
return text, warnings, nil
}
+type RemapLines struct {
+ Pre []string
+ Text string
+ Post []string
+}
+
// buildEdgeRemapLine builds the remap line for the given server and delivery service.
// The cacheKeyConfigParams map may be nil, if this ds profile had no cache key config params.
// Returns the remap line, any warnings, and any error.
@@ -401,14 +472,16 @@ func buildEdgeRemapLine(
remapConfigParams []tc.Parameter,
cacheGroups map[tc.CacheGroupName]tc.CacheGroupNullable,
nameTopologies map[TopologyName]tc.Topology,
-) (string, []string, error) {
+) (RemapLines, []string, error) {
warnings := []string{}
+ remapLines := RemapLines{}
+
// ds = 'remap' in perl
mapFrom = strings.Replace(mapFrom, `__http__`, *server.HostName, -1)
isLastCache, err := serverIsLastCacheForDS(server, &ds, nameTopologies, cacheGroups)
if err != nil {
- return "", warnings, errors.New("determining if cache is the last tier: " + err.Error())
+ return remapLines, warnings, errors.New("determining if cache is the last tier: " + err.Error())
}
// if this remap is going to a parent, use http not https.
@@ -426,7 +499,7 @@ func buildEdgeRemapLine(
if *ds.Topology != "" {
topoTxt, err := makeDSTopologyHeaderRewriteTxt(ds, tc.CacheGroupName(*server.Cachegroup), nameTopologies[TopologyName(*ds.Topology)], cacheGroups)
if err != nil {
- return "", warnings, err
+ return remapLines, warnings, err
}
text += topoTxt
} else if (ds.EdgeHeaderRewrite != nil && *ds.EdgeHeaderRewrite != "") || (ds.ServiceCategory != nil && *ds.ServiceCategory != "") || (ds.MaxOriginConnections != nil && *ds.MaxOriginConnections != 0) {
@@ -447,7 +520,7 @@ func buildEdgeRemapLine(
// Form the cachekey args string, qstring ignore, then
// remap.config then cachekey.config
- var cachekeyArgs string
+ cachekeyArgs := ""
if ds.QStringIgnore != nil {
if *ds.QStringIgnore == tc.QueryStringIgnoreDropAtEdge {
@@ -458,7 +531,7 @@ func buildEdgeRemapLine(
}
}
- if 0 < len(dsConfigParamsMap) {
+ if len(dsConfigParamsMap) > 0 {
cachekeyArgs += cachekeyArgsFor(dsConfigParamsMap, &warnings)
}
@@ -513,7 +586,14 @@ func buildEdgeRemapLine(
text += ` @plugin=fq_pacing.so @pparam=--rate=` + strconv.Itoa(*ds.FQPacingRate)
}
- return text, warnings, nil
+ remapLines.Text = text
+
+ // Any raw pre or post pend lines?
+ if isLastCache {
+ remapLines.Pre, remapLines.Post = lastPrePostRemapLinesFor(dsConfigParamsMap, *ds.XMLID)
+ }
+
+ return remapLines, warnings, nil
}
// makeDSTopologyHeaderRewriteTxt returns the appropriate header rewrite remap line text for the given DS on the given server, and any error.
diff --git a/lib/go-atscfg/remapdotconfig_test.go b/lib/go-atscfg/remapdotconfig_test.go
index e9b6466..7ba240c 100644
--- a/lib/go-atscfg/remapdotconfig_test.go
+++ b/lib/go-atscfg/remapdotconfig_test.go
@@ -7734,3 +7734,211 @@ func TestMakeRemapDotConfigMidHTTPSOriginHTTPRemapTopology(t *testing.T) {
t.Errorf("expected topology mid https origin to create remap from http to https (edge->mid communication always uses http, but the origin needs to still be https), actual: ''%v'''", txt)
}
}
+
+func TestMakeRemapDotConfigMidLastRawRemap(t *testing.T) {
+ hdr := "myHeaderComment"
+
+ server := makeTestRemapServer()
+ server.Type = "MID"
+ server.Cachegroup = util.StrPtr("midCG")
+
+ ds := DeliveryService{}
+ ds.ID = util.IntPtr(48)
+ dsType := tc.DSType("DNS")
+ ds.Type = &dsType
+ ds.OrgServerFQDN = util.StrPtr("http://origin.example.test")
+ /*
+ ds.RangeRequestHandling = util.IntPtr(tc.RangeRequestHandlingCacheRangeRequest)
+ ds.RemapText = util.StrPtr("@plugin=tslua.so @pparam=my-range-manipulator.lua")
+ */
+ ds.SigningAlgorithm = util.StrPtr("foo")
+ ds.XMLID = util.StrPtr("mydsname")
+ ds.QStringIgnore = util.IntPtr(int(tc.QueryStringIgnoreIgnoreInCacheKeyAndPassUp))
+ ds.RegexRemap = util.StrPtr("")
+ ds.FQPacingRate = util.IntPtr(314159)
+ ds.DSCP = util.IntPtr(0)
+ ds.RoutingName = util.StrPtr("myroutingname")
+ ds.MultiSiteOrigin = util.BoolPtr(false)
+ ds.OriginShield = util.StrPtr("myoriginshield")
+ ds.ProfileID = util.IntPtr(49)
+ ds.ProfileName = util.StrPtr("dsprofile")
+ ds.Protocol = util.IntPtr(int(tc.DSProtocolHTTP))
+ ds.AnonymousBlockingEnabled = util.BoolPtr(false)
+ ds.Active = util.BoolPtr(true)
+ ds.Topology = util.StrPtr("t0")
+
+ // non-nil default values should not trigger header rewrite plugin directive
+ ds.EdgeHeaderRewrite = util.StrPtr("")
+ ds.MidHeaderRewrite = util.StrPtr("")
+ ds.ServiceCategory = util.StrPtr("")
+ ds.MaxOriginConnections = util.IntPtr(0)
+
+ dses := []DeliveryService{ds}
+
+ dss := []DeliveryServiceServer{
+ DeliveryServiceServer{
+ Server: *server.ID,
+ DeliveryService: *ds.ID,
+ },
+ }
+
+ dsRegexes := []tc.DeliveryServiceRegexes{
+ tc.DeliveryServiceRegexes{
+ DSName: *ds.XMLID,
+ Regexes: []tc.DeliveryServiceRegex{
+ tc.DeliveryServiceRegex{
+ Type: string(tc.DSMatchTypeHostRegex),
+ SetNumber: 0,
+ Pattern: `.*\.mypattern\..*`,
+ },
+ },
+ },
+ }
+
+ serverParams := []tc.Parameter{
+ tc.Parameter{
+ Name: "trafficserver",
+ ConfigFile: "package",
+ Value: "7",
+ Profiles: []byte(`["global"]`),
+ },
+ tc.Parameter{
+ Name: "serverpkgval",
+ ConfigFile: "package",
+ Value: "serverpkgval __HOSTNAME__ foo",
+ Profiles: []byte(*server.Profile),
+ },
+ tc.Parameter{
+ Name: "dscp_remap_no",
+ ConfigFile: "package",
+ Value: "notused",
+ Profiles: []byte(*server.Profile),
+ },
+ }
+
+ remapConfigParams := []tc.Parameter{
+ tc.Parameter{
+ Name: "not_location",
+ ConfigFile: "cachekey.config",
+ Value: "notinconfig",
+ Profiles: []byte(`["global"]`),
+ },
+ tc.Parameter{
+ Name: "LastRawRemapPost",
+ ConfigFile: "remap.config",
+ Value: "remap http://penraw/ http://penraw0/",
+ Profiles: []byte(`["dsprofile"]`),
+ },
+ tc.Parameter{
+ Name: "LastRawRemapPost",
+ ConfigFile: "remap.config",
+ Value: "remap http://lastraw/ http://lastraw0/",
+ Profiles: []byte(`["dsprofile"]`),
+ },
+ tc.Parameter{
+ Name: "LastRawRemapPre",
+ ConfigFile: "remap.config",
+ Value: "map_with_recp_port http://firstraw:8000/ http://firstraw0/",
+ Profiles: []byte(`["dsprofile"]`),
+ },
+ }
+
+ cdn := &tc.CDN{
+ DomainName: "cdndomain.example",
+ Name: "my-cdn-name",
+ }
+
+ topologies := []tc.Topology{
+ {
+ Name: "t0",
+ Nodes: []tc.TopologyNode{
+ {
+ Cachegroup: "edgeCG",
+ Parents: []int{1, 2},
+ },
+ {
+ Cachegroup: "midCG",
+ },
+ {
+ Cachegroup: "midCG2",
+ },
+ },
+ },
+ }
+
+ mid0 := makeTestParentServer()
+ mid0.Cachegroup = util.StrPtr("midCG")
+ mid0.HostName = util.StrPtr("mymid0")
+ mid0.ID = util.IntPtr(45)
+ setIP(mid0, "192.168.2.2")
+
+ mid1 := makeTestParentServer()
+ mid1.Cachegroup = util.StrPtr("midCG")
+ mid1.HostName = util.StrPtr("mymid1")
+ mid1.ID = util.IntPtr(46)
+ setIP(mid1, "192.168.2.3")
+
+ eCG := &tc.CacheGroupNullable{}
+ eCG.Name = server.Cachegroup
+ eCG.ID = server.CachegroupID
+ eCG.ParentName = mid0.Cachegroup
+ eCG.ParentCachegroupID = mid0.CachegroupID
+ eCG.SecondaryParentName = mid1.Cachegroup
+ eCG.SecondaryParentCachegroupID = mid1.CachegroupID
+ eCGType := tc.CacheGroupEdgeTypeName
+ eCG.Type = &eCGType
+
+ mCG := &tc.CacheGroupNullable{}
+ mCG.Name = mid0.Cachegroup
+ mCG.ID = mid0.CachegroupID
+ mCGType := tc.CacheGroupMidTypeName
+ mCG.Type = &mCGType
+
+ mCG2 := &tc.CacheGroupNullable{}
+ mCG2.Name = mid1.Cachegroup
+ mCG2.ID = mid1.CachegroupID
+ mCGType2 := tc.CacheGroupMidTypeName
+ mCG2.Type = &mCGType2
+
+ cgs := []tc.CacheGroupNullable{*eCG, *mCG, *mCG2}
+ serverCapabilities := map[int]map[ServerCapability]struct{}{}
+ dsRequiredCapabilities := map[int]map[ServerCapability]struct{}{}
+
+ cfg, err := MakeRemapDotConfig(server, dses, dss, dsRegexes, serverParams, cdn, remapConfigParams, topologies, cgs, serverCapabilities, dsRequiredCapabilities, &RemapDotConfigOpts{HdrComment: hdr})
+ if err != nil {
+ t.Fatal(err)
+ }
+ txt := cfg.Text
+
+ txt = strings.TrimSpace(txt)
+
+ testComment(t, txt, hdr)
+
+ txtLines := strings.Split(txt, "\n")
+
+ linesexp := 5
+
+ if len(txtLines) != linesexp {
+ t.Fatalf("expected %d lines in comment, actual: '%v' count %v", linesexp, txt, len(txtLines))
+ } else {
+ var commentstr string
+
+ commentstr = txtLines[0]
+ if len(commentstr) == 0 || '#' != commentstr[0] {
+ t.Fatalf("expected [1] as comment, actual: \n'%v' got %v", txt, commentstr)
+ }
+
+ firststr := txtLines[1]
+ if !strings.Contains(firststr, "firstraw") {
+ t.Fatalf("expected [1] with 'firstraw', actual: '%v' got %v", txt, firststr)
+ }
+ laststr := txtLines[len(txtLines)-2]
+ if !strings.Contains(laststr, "last") {
+ t.Fatalf("expected [-2] last with 'lastraw', actual: '%v' got %v", txt, laststr)
+ }
+ penstr := txtLines[len(txtLines)-1]
+ if !strings.Contains(penstr, "penraw") {
+ t.Fatalf("expected [-1] with 'penraw', actual: '%v' got %v", txt, penstr)
+ }
+ }
+}