[GitHub] rawlinp commented on a change in pull request #3075: Traffic Ops Golang parent.config

[GitBox <gi...@apache.org>]

rawlinp commented on a change in pull request #3075: Traffic Ops Golang parent.config
URL: https://github.com/apache/trafficcontrol/pull/3075#discussion_r261437762
 
 

 ##########
 File path: traffic_ops/traffic_ops_golang/ats/parentdotconfig.go
 ##########
 @@ -0,0 +1,1393 @@
+package ats
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import (
+	"database/sql"
+	"errors"
+	"net/http"
+	"net/url"
+	"regexp"
+	"sort"
+	"strconv"
+	"strings"
+
+	"github.com/apache/trafficcontrol/lib/go-log"
+	"github.com/apache/trafficcontrol/lib/go-tc"
+	"github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/api"
+
+	"github.com/lib/pq"
+)
+
+const TypeCacheGroupOrigin = "ORG_LOC"
+const DefaultATSVersion = "5" // emulates Perl
+
+func GetParentDotConfig(w http.ResponseWriter, r *http.Request) {
+	inf, userErr, sysErr, errCode := api.NewInfo(r, []string{"id-or-host"}, nil)
+	if userErr != nil || sysErr != nil {
+		api.HandleErr(w, r, inf.Tx.Tx, errCode, userErr, sysErr)
+		return
+	}
+	defer inf.Close()
+
+	idOrHost := strings.TrimSuffix(inf.Params["id-or-host"], ".json")
+	hostName := ""
+	isHost := false
+	id, err := strconv.Atoi(idOrHost)
+	if err != nil {
+		isHost = true
+		hostName = idOrHost
+	}
+
+	serverInfo, ok, err := &ServerInfo{}, false, error(nil)
+	if isHost {
+		serverInfo, ok, err = getServerInfoByHost(inf.Tx.Tx, hostName)
+	} else {
+		serverInfo, ok, err = getServerInfoByID(inf.Tx.Tx, id)
+	}
+	if err != nil {
+		api.HandleErr(w, r, inf.Tx.Tx, http.StatusInternalServerError, nil, errors.New("Getting server info: "+err.Error()))
+		return
+	}
+	if !ok {
+		api.HandleErr(w, r, inf.Tx.Tx, http.StatusNotFound, errors.New("server not found"), nil)
+		return
+	}
+
+	atsMajorVer, err := GetATSMajorVersion(inf.Tx.Tx, serverInfo.ProfileID)
+	if err != nil {
+		api.HandleErr(w, r, inf.Tx.Tx, http.StatusInternalServerError, nil, errors.New("Getting ATS major version: "+err.Error()))
+		return
+	}
+
+	hdr, err := headerComment(inf.Tx.Tx, serverInfo.HostName)
+	if err != nil {
+		api.HandleErr(w, r, inf.Tx.Tx, http.StatusInternalServerError, nil, errors.New("Getting header comment: "+err.Error()))
+		return
+	}
+
+	textArr := []string{}
+	text := ""
+	// TODO put these in separate functions. No if-statement should be this long.
+	if serverInfo.IsTopLevelCache() {
+		log.Errorf("DEBUG PCGen isTopLevel\n")
+		uniqueOrigins := map[string]struct{}{}
+
+		data, err := getParentConfigDSTopLevel(inf.Tx.Tx, serverInfo.CDN)
+		if err != nil {
+			api.HandleErr(w, r, inf.Tx.Tx, http.StatusInternalServerError, nil, errors.New("Getting parent config DS data: "+err.Error()))
+			return
+		}
+
+		parentInfos := map[string][]ParentInfo{} // TODO better names (this was transliterated from Perl)
+
+		log.Errorf("DEBUG PCGTL len(data) %+v\n", len(data))
+
+		for _, ds := range data {
+			parentQStr := "ignore"
+			if ds.QStringHandling == "" && ds.MSOAlgorithm == AlgorithmConsistentHash && ds.QStringIgnore == 0 {
+				parentQStr = "consider"
+			}
+
+			orgURIStr := ds.OriginFQDN
+			orgURI, err := url.Parse(orgURIStr) // TODO verify origin is always a host:port
+			if err != nil {
+				log.Errorln("Malformed ds '" + string(ds.Name) + "' origin  URI: '" + orgURIStr + "', skipping! : " + err.Error())
+				continue
+			}
+			// TODO put in function, to remove duplication
+			if orgURI.Port() == "" {
+				if orgURI.Scheme == "http" {
+					orgURI.Host += ":80"
+				} else if orgURI.Scheme == "https" {
+					orgURI.Host += ":443"
+				} else {
+					log.Errorln("parent.config generation: fds '" + string(ds.Name) + "' origin  URI: '" + orgURIStr + "' is unknown scheme '" + orgURI.Scheme + "', but has no port! Using as-is! ")
+				}
+			}
+
+			if _, ok := uniqueOrigins[ds.OriginFQDN]; ok {
+				// log.Errorf("DEBUG PCGT uniqueorigins skipping %+v\n", ds.OriginFQDN)
+				continue // TODO warn?
+			}
+			uniqueOrigins[ds.OriginFQDN] = struct{}{}
+
+			log.Errorf("DEBUG PCGTQ origin %+v\n", ds.OriginFQDN)
+
+			// log.Errorf("DEBUG PCGT uniqueorigins adding ds '%+v' ds.OriginfQDN '%+v' orgURI.Hostname '%+v'\n", ds.Name, ds.OriginFQDN, orgURI.Hostname())
+
+			textLine := ""
+
+			if ds.OriginShield != "" {
+				log.Errorf("DEBUG PCGT ds '%+v' is origin shield\n", ds.Name)
+				// TODO fix to only call once
+				serverParams, err := getParentConfigServerProfileParams(inf.Tx.Tx, serverInfo.ID)
+				if err != nil {
+					api.HandleErr(w, r, inf.Tx.Tx, http.StatusInternalServerError, nil, errors.New("Getting server params: "+err.Error()))
+					return
+				}
+				// log.Errorf("DEBUG PCGT getParentConfigServerProfileParams %+v len %+v\n", serverInfo.ID, len(serverParams))
+
+				algorithm := ""
+				if parentSelectAlg, hasParentSelectAlg := serverParams[ParentConfigParamAlgorithm]; hasParentSelectAlg {
+					algorithm = "round_robin=" + parentSelectAlg
+				}
+				textLine += "dest_domain=" + orgURI.Hostname() + " port=" + orgURI.Port() + " parent=" + ds.OriginShield + " " + algorithm + " go_direct=true\n"
+			} else if ds.MultiSiteOrigin {
+				log.Errorf("DEBUG PCGT ds '%+v' is multisite\n", ds.Name)
+				textLine += "dest_domain=" + orgURI.Hostname() + " port=" + orgURI.Port() + " "
+				if len(parentInfos) == 0 {
+					// If we have multi-site origin, get parent_data once
+					parentInfos, err = getParentInfo(inf.Tx.Tx, serverInfo)
+					// log.Errorf("DEBUG PCGT getParentInfo len %+v\n", len(parentInfos))
+					if err != nil {
+						api.HandleErr(w, r, inf.Tx.Tx, http.StatusInternalServerError, nil, errors.New("Getting server parent info: "+err.Error()))
+						return
+					}
+				}
+
+				if len(parentInfos[orgURI.Hostname()]) == 0 {
+					// TODO error? emulates Perl
+					log.Warnln("ParentInfo: delivery service " + ds.Name + " has no parent servers")
+				}
+
+				// log.Errorf("DEBUG PCGT parentInfos len %+v\n", len(parentInfos))
+				// log.Errorf("DEBUG PCGT parentInfos origin '"+orgURI.Hostname()+"' len %+v\n", len(parentInfos[orgURI.Hostname()]))
+				// log.Errorf("DEBUG PCGT parentInfos %++v\n\n", parentInfos)
+
+				rankedParents := ParentInfoSortByRank(parentInfos[orgURI.Hostname()])
+				sort.Sort(rankedParents)
+
+				parentInfo := []string{}
+				secondaryParentInfo := []string{}
+				nullParentInfo := []string{}
+				for _, parent := range ([]ParentInfo)(rankedParents) {
+					if parent.PrimaryParent {
+						parentInfo = append(parentInfo, parent.Format())
+					} else if parent.SecondaryParent {
+						secondaryParentInfo = append(secondaryParentInfo, parent.Format())
+					} else {
+						nullParentInfo = append(nullParentInfo, parent.Format())
+					}
+				}
+
+				if len(parentInfo) == 0 {
+					// If no parents are found in the secondary parent either, then set the null parent list (parents in neither secondary or primary)
+					// as the secondary parent list and clear the null parent list.
+					if len(secondaryParentInfo) == 0 {
+						secondaryParentInfo = nullParentInfo
+						nullParentInfo = []string{}
+					}
+					parentInfo = secondaryParentInfo
+					secondaryParentInfo = []string{} // TODO should thi be '= secondary'? Currently emulates Perl
+				}
+
+				// TODO benchmark, verify this isn't slow. if it is, it could easily be made faster
+				seen := map[string]struct{}{} // TODO change to host+port? host isn't unique
+				parentInfo, seen = removeStrDuplicates(parentInfo, seen)
+				secondaryParentInfo, seen = removeStrDuplicates(secondaryParentInfo, seen)
+				nullParentInfo, seen = removeStrDuplicates(nullParentInfo, seen)
+
+				// If the ats version supports it and the algorithm is consistent hash, put secondary and non-primary parents into secondary parent group.
+				// This will ensure that secondary and tertiary parents will be unused unless all hosts in the primary group are unavailable.
+
+				parents := ""
+
+				if atsMajorVer >= 6 && ds.MSOAlgorithm == "consistent_hash" && (len(secondaryParentInfo) > 0 || len(nullParentInfo) > 0) {
+					parents = `parent="` + strings.Join(parentInfo, "") + `" secondary_parent="` + strings.Join(secondaryParentInfo, "") + strings.Join(nullParentInfo, "") + `"`
+				} else {
+					parents = `parent="` + strings.Join(parentInfo, "") + strings.Join(secondaryParentInfo, "") + strings.Join(nullParentInfo, "") + `"`
+				}
+				textLine += parents + ` round_robin=` + ds.MSOAlgorithm + ` qstring=` + parentQStr + ` go_direct=false parent_is_proxy=false`
+
+				parentRetry := ds.MSOParentRetry
+				if atsMajorVer >= 6 && parentRetry != "" {
+					if unavailableServerRetryResponsesValid(ds.MSOUnavailableServerRetryResponses) {
+						log.Errorf("DEBUG unavailableServerRetryResponsesValid '%+v'\n", ds.MSOUnavailableServerRetryResponses)
+						textLine += ` parent_retry=` + parentRetry + ` unavailable_server_retry_responses=` + ds.MSOUnavailableServerRetryResponses
+					} else {
+						log.Errorf("DEBUG unavailableServerRetryResponsesValid '%+v' NOT\n", ds.MSOUnavailableServerRetryResponses)
+						if ds.MSOUnavailableServerRetryResponses != "" {
+							log.Errorln("Malformed unavailable_server_retry_responses parameter '" + ds.MSOUnavailableServerRetryResponses + "', not using!")
+						}
+						textLine += ` parent_retry=` + parentRetry
+					}
+					textLine += ` max_simple_retries=` + ds.MSOMaxSimpleRetries + ` max_unavailable_server_retries=` + ds.MSOMaxUnavailableServerRetries
+				}
+				textLine += "\n"
+				textArr = append(textArr, textLine)
+			} else {
+				// log.Errorf("DEBUG PCGT ds '%+v' neither origin shield nor multisite\n", ds.Name)
+			}
+		}
+		sort.Sort(sort.StringSlice(textArr))
+		text = hdr + strings.Join(textArr, "")
+	} else {
+		log.Errorf("DEBUG PCGen not top level\n")
+		// not a top level cache
+		data, err := getParentConfigDS(inf.Tx.Tx, serverInfo.ID) // TODO rename
+		if err != nil {
+			api.HandleErr(w, r, inf.Tx.Tx, http.StatusInternalServerError, nil, errors.New("Getting parent config DS data (non-top-level): "+err.Error()))
+			return
+		}
+
+		log.Errorf("DEBUG PCGen got data len %+v\n", len(data))
+
+		parentInfos, err := getParentInfo(inf.Tx.Tx, serverInfo)
+		if err != nil {
+			api.HandleErr(w, r, inf.Tx.Tx, http.StatusInternalServerError, nil, errors.New("Getting server parent info (non-top-level: "+err.Error()))
+			return
+		}
+
+		log.Errorf("DEBUG PCGen got parent info len %+v\n", len(parentInfos))
+
+		done := map[string]tc.DeliveryServiceName{}                                       // map[originHost]ds
+		serverParams, err := getParentConfigServerProfileParams(inf.Tx.Tx, serverInfo.ID) // (map[string]string, error) {
+		if err != nil {
+			api.HandleErr(w, r, inf.Tx.Tx, http.StatusInternalServerError, nil, errors.New("Getting parent config server profile params: "+err.Error()))
+			return
+		}
+
+		log.Errorf("DEBUG PCGen got server params len %+v\n", len(serverParams))
+
+		qsh := serverParams[ParentConfigParamQStringHandling] // TODO rename
+		parentInfo := []string{}
+		secondaryParentInfo := []string{}
+
+		log.Errorf("DEBUG PCGen ranging over len(parentInfos[DeliveryServicesAllParentsKey]) %+v\n", len(parentInfos[DeliveryServicesAllParentsKey]))
+
+		parentInfosAllParents := parentInfos[DeliveryServicesAllParentsKey]
+		sort.Sort(ParentInfoSortByRank(parentInfosAllParents))
+
+		for _, parent := range parentInfosAllParents { // TODO fix magic key
+			pTxt := parent.Format()
+			if parent.PrimaryParent {
+				parentInfo = append(parentInfo, pTxt)
+			} else if parent.SecondaryParent {
+				secondaryParentInfo = append(secondaryParentInfo, pTxt)
+			}
+		}
+
+		if len(parentInfo) == 0 {
+			parentInfo = secondaryParentInfo
+			secondaryParentInfo = []string{}
+		}
+
+		// TODO remove duplicate code with top level if block
+		seen := map[string]struct{}{} // TODO change to host+port? host isn't unique
+		parentInfo, seen = removeStrDuplicates(parentInfo, seen)
+		secondaryParentInfo, seen = removeStrDuplicates(secondaryParentInfo, seen)
+
+		parents := ""
+		secParents := "" // TODO rename
+		sort.Sort(sort.StringSlice(parentInfo))
+		sort.Sort(sort.StringSlice(secondaryParentInfo))
+		if atsMajorVer >= 6 && len(secondaryParentInfo) > 0 {
+			parents = `parent="` + strings.Join(parentInfo, "") + `"`
+			secParents = ` secondary_parent="` + strings.Join(secondaryParentInfo, "") + `"`
+		} else {
+			parents = `parent="` + strings.Join(parentInfo, "") + strings.Join(secondaryParentInfo, "") + `"`
+		}
+
+		roundRobin := `round_robin=consistent_hash`
+		goDirect := `go_direct=false` // TODO double-check with Perl, that this isn't ever different
+
+		sort.Sort(ParentConfigDSSortByName(data))
+		for _, ds := range data {
+			text := ""
+			originFQDN := ds.OriginFQDN
+			if originFQDN == "" {
+				continue // TODO warn? (Perl doesn't)
+			}
+
+			orgURI, err := url.Parse(originFQDN) // TODO verify
+			if err != nil {
+				log.Errorln("Malformed ds '" + string(ds.Name) + "' origin  URI: '" + originFQDN + "': skipping!" + err.Error())
+				continue
+			}
+
+			if existingDS, ok := done[originFQDN]; ok {
+				log.Errorln("parent.config generation: duplicate origin! services '" + string(ds.Name) + "' and '" + string(existingDS) + "' share origin '" + orgURI.Host + "': skipping '" + string(ds.Name) + "'!")
+				continue
+			}
+
+			// TODO put in function, to remove duplication
+			if orgURI.Port() == "" {
+				if orgURI.Scheme == "http" {
+					orgURI.Host += ":80"
+				} else if orgURI.Scheme == "https" {
+					orgURI.Host += ":443"
+				} else {
+					log.Errorln("parent.config generation non-top-level: ds '" + string(ds.Name) + "' origin  URI: '" + originFQDN + "' is unknown scheme '" + orgURI.Scheme + "', but has no port! Using as-is! ")
+				}
+			}
+
+			// debug
+			if orgURI.Hostname() == "odol-cim-linear-hds.cmc.co.ndcwest.comcast.net" {
+				log.Errorf("DEBUG ds '%+v' orgURI '%+v' is %++v\n", ds.Name, orgURI.Hostname(), ds)
+			}
+			// TODO encode this in a DSType func, IsGoDirect() ?
+			if dsType := tc.DSType(ds.Type); dsType == tc.DSTypeHTTPNoCache || dsType == tc.DSTypeHTTPLive || dsType == tc.DSTypeDNSLive {
+				text += `dest_domain=` + orgURI.Hostname() + ` port=` + orgURI.Port() + ` go_direct=true` + "\n"
+			} else {
+
+				// check for profile psel.qstring_handling.  If this parameter is assigned to the server profile,
+				// then edges will use the qstring handling value specified in the parameter for all profiles.
+
+				// If there is no defined parameter in the profile, then check the delivery service profile.
+				// If psel.qstring_handling exists in the DS profile, then we use that value for the specified DS only.
+				// This is used only if not overridden by a server profile qstring handling parameter.
+
+				// TODO refactor this logic, hard to understand (transliterated from Perl)
+				dsQSH := qsh
+				if dsQSH == "" {
+					dsQSH = ds.QStringHandling
+				}
+				parentQStr := dsQSH
+				if parentQStr == "" {
+					parentQStr = "ignore"
+				}
+				if ds.QStringIgnore == 0 && dsQSH == "" {
+					parentQStr = "consider"
+				}
+
+				text += `dest_domain=` + orgURI.Hostname() + ` port=` + orgURI.Port() + ` ` + parents + ` ` + secParents + ` ` + roundRobin + ` ` + goDirect + ` qstring=` + parentQStr + "\n"
+			}
+			textArr = append(textArr, text)
+			done[originFQDN] = ds.Name
+		}
+
+		defaultDestText := `dest_domain=. ` + parents
+		if serverParams[ParentConfigParamAlgorithm] == AlgorithmConsistentHash {
+			defaultDestText += secParents
+		}
+		defaultDestText += ` round_robin=consistent_hash go_direct=false`
+
+		if qStr := serverParams[ParentConfigParamQString]; qStr != "" {
+			defaultDestText += ` qstring=` + qStr
+		}
+		defaultDestText += "\n"
+
+		sort.Sort(sort.StringSlice(textArr))
+		text = hdr + strings.Join(textArr, "") + defaultDestText
+	}
+	w.Header().Set("Content-Type", "text/plain")
+	w.Write([]byte(text))
+}
+
+// unavailableServerRetryResponsesValid returns whether a unavailable_server_retry_responses parameter is valid for an ATS parent rule.
+func unavailableServerRetryResponsesValid(s string) bool {
+	// optimization if param is empty
+	if s == "" {
+		return false
+	}
+	re := regexp.MustCompile(`^"(:?\d{3},)+\d{3}"\s*$`) // TODO benchmark, cache if performance matters
+	return re.MatchString(s)
+}
+func removeStrDuplicates(pi []string, seen map[string]struct{}) ([]string, map[string]struct{}) {
+	npi := []string{}
+	for _, p := range pi {
+		if _, ok := seen[p]; !ok {
+			npi = append(npi, p)
+			seen[p] = struct{}{}
+		}
+	}
+	return npi, seen
+}
+
+type OriginHost string
+
+type ParentInfos map[OriginHost]ParentInfo
+
+func (p ParentInfo) Format() string {
+	host := ""
+	if p.UseIP {
+		host = p.IP
+	} else {
+		host = p.Host + "." + p.Domain
+	}
+	return host + ":" + strconv.Itoa(p.Port) + "|" + p.Weight + ";"
+}
+
+type ParentInfoSortByRank []ParentInfo
+
+func (s ParentInfoSortByRank) Len() int           { return len(([]ParentInfo)(s)) }
+func (s ParentInfoSortByRank) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }
+func (s ParentInfoSortByRank) Less(i, j int) bool { return s[i].Rank < s[j].Rank }
+
+type ParentConfigDSSortByName []ParentConfigDS
+
+func (s ParentConfigDSSortByName) Len() int      { return len(([]ParentConfigDS)(s)) }
+func (s ParentConfigDSSortByName) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
+func (s ParentConfigDSSortByName) Less(i, j int) bool {
+	// TODO make this match the Perl sort "foreach my $ds ( sort @{ $data->{dslist} } )" ?
+	return strings.Compare(string(s[i].Name), string(s[j].Name)) < 0
+}
+
+//ParentConfigDS
+
+const AlgorithmConsistentHash = "consistent_hash"
+
+type ServerInfo struct {
+	CacheGroupID                  int
+	CDN                           tc.CDNName
+	CDNID                         int
+	DomainName                    string
+	HostName                      string
+	ID                            int
+	IP                            string
+	ParentCacheGroupID            int
+	ParentCacheGroupType          string
+	ProfileID                     ProfileID
+	ProfileName                   string
+	Port                          int
+	SecondaryParentCacheGroupID   int
+	SecondaryParentCacheGroupType string
+	Type                          string
+}
+
+func (s *ServerInfo) IsTopLevelCache() bool {
+	return (s.ParentCacheGroupType == TypeCacheGroupOrigin || s.ParentCacheGroupID == 0) &&
+		(s.ParentCacheGroupType == TypeCacheGroupOrigin || s.SecondaryParentCacheGroupID == 0)
 
 Review comment:
   Did you mean for this to be `s.SecondaryParentCacheGroupType == ...`? Also, the query below coalesces the IDs into -1; is that what it should be checking for here?

----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
users@infra.apache.org


With regards,
Apache Git Services