You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@trafficcontrol.apache.org by oc...@apache.org on 2023/08/14 17:42:38 UTC
[trafficcontrol] branch master updated: Add parents to varnish cache (#7669)
This is an automated email from the ASF dual-hosted git repository.
ocket8888 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/trafficcontrol.git
The following commit(s) were added to refs/heads/master by this push:
new fe12a38608 Add parents to varnish cache (#7669)
fe12a38608 is described below
commit fe12a386082d13d78015b0bbd0558b9414c1b304
Author: AbdelrahmanElawady <60...@users.noreply.github.com>
AuthorDate: Mon Aug 14 20:42:32 2023 +0300
Add parents to varnish cache (#7669)
* Add varnishcfg package and parent configuration
* Add Varnish Dockerfile to be used in CIAB
* Add license to Varnsih Dockerfile
* Add systemctl.sh to handle Varnish service and integrate Varnish with t3c-apply
* Move host changes to BE fetch, change varnish dir and make test more readable
* Remove Varnish package release and arch, add GoDoc and move licenses
* Move license text
---
cache-config/t3c-apply/config/config.go | 6 +
cache-config/t3c-apply/t3c-apply.go | 8 +-
cache-config/t3c-apply/torequest/cmd.go | 1 +
cache-config/t3c-apply/torequest/torequest.go | 30 +-
cache-config/t3c-generate/cfgfile/varnish.go | 46 +++
cache-config/t3c-generate/config/config.go | 3 +
cache-config/t3c-generate/t3c-generate.go | 14 +
infrastructure/cdn-in-a-box/enroller/Dockerfile | 3 +
infrastructure/cdn-in-a-box/varnish/Dockerfile | 63 ++++
infrastructure/cdn-in-a-box/varnish/run.sh | 93 ++++++
infrastructure/cdn-in-a-box/varnish/systemctl.sh | 94 ++++++
.../cdn-in-a-box/varnish/traffic_ops_ort.crontab | 18 ++
lib/go-atscfg/parentabstraction.go | 3 +
lib/go-atscfg/parentdotconfig.go | 5 +-
lib/go-atscfg/remapdotconfig.go | 14 +-
lib/go-atscfg/remapdotconfig_test.go | 2 +-
lib/go-atscfg/sslservernamedotyaml.go | 6 +-
lib/go-atscfg/strategiesdotconfig.go | 4 +-
lib/varnishcfg/backends.go | 183 +++++++++++
lib/varnishcfg/backends_test.go | 354 +++++++++++++++++++++
lib/varnishcfg/vcl.go | 101 ++++++
lib/varnishcfg/vclbuilder.go | 75 +++++
22 files changed, 1097 insertions(+), 29 deletions(-)
diff --git a/cache-config/t3c-apply/config/config.go b/cache-config/t3c-apply/config/config.go
index e7956e1eba..3797c80061 100644
--- a/cache-config/t3c-apply/config/config.go
+++ b/cache-config/t3c-apply/config/config.go
@@ -123,6 +123,7 @@ type Cfg struct {
Version string
GitRevision string
LocalATSVersion string
+ CacheType string
}
func (cfg Cfg) AppVersion() string { return t3cutil.VersionStr(AppName, cfg.Version, cfg.GitRevision) }
@@ -277,6 +278,7 @@ func GetCfg(appVersion string, gitRevision string) (Cfg, error) {
defaultClientTLSVersions := getopt.StringLong("default-client-tls-versions", 'V', "", "Comma-delimited list of default TLS versions for Delivery Services with no Parameter, e.g. --default-tls-versions='1.1,1.2,1.3'. If omitted, all versions are enabled.")
maxmindLocationPtr := getopt.StringLong("maxmind-location", 'M', "", "URL of a maxmind gzipped database file, to be installed into the trafficserver etc directory.")
verbosePtr := getopt.CounterLong("verbose", 'v', `Log verbosity. Logging is output to stderr. By default, errors are logged. To log warnings, pass '-v'. To log info, pass '-vv'. To omit error logging, see '-s'`)
+ cache := getopt.StringLong("cache", 'T', "ats", "Cache server type. Generate configuration files for specific cache server type, e.g. 'ats', 'varnish'.")
const silentFlagName = "silent"
silentPtr := getopt.BoolLong(silentFlagName, 's', `Silent. Errors are not logged, and the 'verbose' flag is ignored. If a fatal error occurs, the return code will be non-zero but no text will be output to stderr`)
@@ -533,6 +535,9 @@ If any of the related flags are also set, they override the mode's default behav
if tsHome != "" {
TSHome = tsHome
tsConfigDir = tsHome + "/etc/trafficserver"
+ if cache != nil && *cache == "varnish" {
+ tsConfigDir = tsHome + "/etc/varnish"
+ }
toInfoLog = append(toInfoLog, fmt.Sprintf("TSHome: %s, TSConfigDir: %s\n", TSHome, tsConfigDir))
}
@@ -612,6 +617,7 @@ If any of the related flags are also set, they override the mode's default behav
Version: appVersion,
GitRevision: gitRevision,
LocalATSVersion: atsVersionStr,
+ CacheType: *cache,
}
if err = log.InitCfg(cfg); err != nil {
diff --git a/cache-config/t3c-apply/t3c-apply.go b/cache-config/t3c-apply/t3c-apply.go
index 32692822b6..75876f129a 100644
--- a/cache-config/t3c-apply/t3c-apply.go
+++ b/cache-config/t3c-apply/t3c-apply.go
@@ -220,7 +220,7 @@ func Main() int {
}
} else {
- syncdsUpdate, err = trops.CheckSyncDSState(metaData)
+ syncdsUpdate, err = trops.CheckSyncDSState(metaData, cfg)
if err != nil {
log.Errorln("Checking syncds state: " + err.Error())
return GitCommitAndExit(ExitCodeSyncDSError, FailureExitMsg, cfg, metaData, oldMetaData)
@@ -241,7 +241,7 @@ func Main() int {
} else if rc == 0 {
log.Infoln("updated the remap.config for reloading.")
}
- if err := trops.StartServices(&syncdsUpdate, metaData); err != nil {
+ if err := trops.StartServices(&syncdsUpdate, metaData, cfg); err != nil {
log.Errorln("failed to start services: " + err.Error())
metaData.PartialSuccess = true
return GitCommitAndExit(ExitCodeServicesError, PostConfigFailureExitMsg, cfg, metaData, oldMetaData)
@@ -311,7 +311,7 @@ func Main() int {
}
}
- if err := trops.StartServices(&syncdsUpdate, metaData); err != nil {
+ if err := trops.StartServices(&syncdsUpdate, metaData, cfg); err != nil {
log.Errorln("failed to start services: " + err.Error())
metaData.PartialSuccess = true
return GitCommitAndExit(ExitCodeServicesError, PostConfigFailureExitMsg, cfg, metaData, oldMetaData)
@@ -377,7 +377,7 @@ func GitCommitAndExit(exitCode int, exitMsg string, cfg config.Cfg, metaData *t3
// so add the old files to the new metadata.
// This is especially important for reval runs, which don't add all files.
metaData.OwnedFilePaths = t3cutil.CombineOwnedFilePaths(metaData, oldMetaData)
- if len(metaData.InstalledPackages) == 0 {
+ if len(metaData.InstalledPackages) == 0 && oldMetaData != nil {
metaData.InstalledPackages = oldMetaData.InstalledPackages
}
WriteMetaData(cfg, metaData)
diff --git a/cache-config/t3c-apply/torequest/cmd.go b/cache-config/t3c-apply/torequest/cmd.go
index efc52a327c..597d19f358 100644
--- a/cache-config/t3c-apply/torequest/cmd.go
+++ b/cache-config/t3c-apply/torequest/cmd.go
@@ -72,6 +72,7 @@ func generate(cfg config.Cfg) ([]t3cutil.ATSConfigFile, error) {
args := []string{
`generate`,
"--dir=" + cfg.TsConfigDir,
+ "--cache=" + cfg.CacheType,
}
if cfg.LogLocationErr == log.LogLocationNull {
diff --git a/cache-config/t3c-apply/torequest/torequest.go b/cache-config/t3c-apply/torequest/torequest.go
index 3d6d7e34c9..990b71c852 100644
--- a/cache-config/t3c-apply/torequest/torequest.go
+++ b/cache-config/t3c-apply/torequest/torequest.go
@@ -748,7 +748,7 @@ func (r *TrafficOpsReq) CheckRevalidateState(sleepOverride bool) (UpdateStatus,
// CheckSyncDSState retrieves and returns the DS Update status from Traffic Ops.
// The metaData is this run's metadata. It must not be nil, and this function may add to it.
-func (r *TrafficOpsReq) CheckSyncDSState(metaData *t3cutil.ApplyMetaData) (UpdateStatus, error) {
+func (r *TrafficOpsReq) CheckSyncDSState(metaData *t3cutil.ApplyMetaData, cfg config.Cfg) (UpdateStatus, error) {
updateStatus := UpdateTropsNotNeeded
randDispSec := time.Duration(0)
log.Debugln("Checking syncds state.")
@@ -785,7 +785,7 @@ func (r *TrafficOpsReq) CheckSyncDSState(metaData *t3cutil.ApplyMetaData) (Updat
}
} else if !r.Cfg.IgnoreUpdateFlag {
log.Errorln("no queued update needs to be applied. Running revalidation before exiting.")
- r.RevalidateWhileSleeping(metaData)
+ r.RevalidateWhileSleeping(metaData, cfg)
return UpdateTropsNotNeeded, nil
} else {
log.Errorln("Traffic Ops is signaling that no update is waiting to be applied.")
@@ -1091,7 +1091,7 @@ func (r *TrafficOpsReq) ProcessPackagesWithMetaData(packageMetaData []string) er
return nil
}
-func (r *TrafficOpsReq) RevalidateWhileSleeping(metaData *t3cutil.ApplyMetaData) (UpdateStatus, error) {
+func (r *TrafficOpsReq) RevalidateWhileSleeping(metaData *t3cutil.ApplyMetaData, cfg config.Cfg) (UpdateStatus, error) {
updateStatus, err := r.CheckRevalidateState(true)
if err != nil {
return updateStatus, err
@@ -1115,7 +1115,7 @@ func (r *TrafficOpsReq) RevalidateWhileSleeping(metaData *t3cutil.ApplyMetaData)
t3cutil.WriteActionLog(t3cutil.ActionLogActionUpdateFilesReval, t3cutil.ActionLogStatusSuccess, metaData)
}
- if err := r.StartServices(&updateStatus, metaData); err != nil {
+ if err := r.StartServices(&updateStatus, metaData, cfg); err != nil {
return updateStatus, errors.New("failed to start services: " + err.Error())
}
@@ -1132,7 +1132,7 @@ func (r *TrafficOpsReq) RevalidateWhileSleeping(metaData *t3cutil.ApplyMetaData)
// StartServices reloads, restarts, or starts ATS as necessary,
// according to the changed config files and run mode.
// Returns nil on success or any error.
-func (r *TrafficOpsReq) StartServices(syncdsUpdate *UpdateStatus, metaData *t3cutil.ApplyMetaData) error {
+func (r *TrafficOpsReq) StartServices(syncdsUpdate *UpdateStatus, metaData *t3cutil.ApplyMetaData, cfg config.Cfg) error {
serviceNeeds := t3cutil.ServiceNeedsNothing
if r.Cfg.ServiceAction == t3cutil.ApplyServiceActionFlagRestart {
serviceNeeds = t3cutil.ServiceNeedsRestart
@@ -1154,13 +1154,17 @@ func (r *TrafficOpsReq) StartServices(syncdsUpdate *UpdateStatus, metaData *t3cu
serviceNeeds = t3cutil.ServiceNeedsReload
}
}
+ packageName := "trafficserver"
+ if cfg.CacheType == "varnish" {
+ packageName = "varnish"
+ }
- if (serviceNeeds == t3cutil.ServiceNeedsRestart || serviceNeeds == t3cutil.ServiceNeedsReload) && !r.IsPackageInstalled("trafficserver") {
+ if (serviceNeeds == t3cutil.ServiceNeedsRestart || serviceNeeds == t3cutil.ServiceNeedsReload) && !r.IsPackageInstalled(packageName) {
// TODO try to reload/restart anyway? To allow non-RPM installs?
- return errors.New("trafficserver needs " + serviceNeeds.String() + " but is not installed.")
+ return errors.New(packageName + " needs " + serviceNeeds.String() + " but is not installed.")
}
- svcStatus, _, err := util.GetServiceStatus("trafficserver")
+ svcStatus, _, err := util.GetServiceStatus(packageName)
if err != nil {
return errors.New("getting trafficserver service status: " + err.Error())
}
@@ -1177,7 +1181,7 @@ func (r *TrafficOpsReq) StartServices(syncdsUpdate *UpdateStatus, metaData *t3cu
if svcStatus != util.SvcRunning {
startStr = "start"
}
- if _, err := util.ServiceStart("trafficserver", startStr); err != nil {
+ if _, err := util.ServiceStart(packageName, startStr); err != nil {
t3cutil.WriteActionLog(t3cutil.ActionLogActionATSRestart, t3cutil.ActionLogStatusFailure, metaData)
return errors.New("failed to restart trafficserver")
}
@@ -1204,7 +1208,13 @@ func (r *TrafficOpsReq) StartServices(syncdsUpdate *UpdateStatus, metaData *t3cu
log.Errorln("ATS configuration has changed. The new config will be picked up the next time ATS is started.")
} else if serviceNeeds == t3cutil.ServiceNeedsReload {
log.Infoln("ATS configuration has changed, Running 'traffic_ctl config reload' now.")
- if _, _, err := util.ExecCommand(config.TSHome+config.TrafficCtl, "config", "reload"); err != nil {
+ reloadCommand := config.TSHome + config.TrafficCtl
+ reloadArgs := []string{"config", "reload"}
+ if cfg.CacheType == "varnish" {
+ reloadCommand = "varnishreload"
+ reloadArgs = []string{}
+ }
+ if _, _, err := util.ExecCommand(reloadCommand, reloadArgs...); err != nil {
t3cutil.WriteActionLog(t3cutil.ActionLogActionATSReload, t3cutil.ActionLogStatusFailure, metaData)
if *syncdsUpdate == UpdateTropsNeeded {
diff --git a/cache-config/t3c-generate/cfgfile/varnish.go b/cache-config/t3c-generate/cfgfile/varnish.go
new file mode 100644
index 0000000000..060445a75a
--- /dev/null
+++ b/cache-config/t3c-generate/cfgfile/varnish.go
@@ -0,0 +1,46 @@
+package cfgfile
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import (
+ "github.com/apache/trafficcontrol/cache-config/t3c-generate/config"
+ "github.com/apache/trafficcontrol/cache-config/t3cutil"
+ "github.com/apache/trafficcontrol/lib/varnishcfg"
+)
+
+// GetVarnishConfigs returns varnish configuration files
+// TODO: add varnishncsa and hitch configs
+func GetVarnishConfigs(toData *t3cutil.ConfigData, cfg config.Cfg) ([]t3cutil.ATSConfigFile, error) {
+ vclBuilder := varnishcfg.NewVCLBuilder(toData)
+ vcl, warnings, err := vclBuilder.BuildVCLFile()
+ logWarnings("Generating varnish configuration files: ", warnings)
+
+ configs := make([]t3cutil.ATSConfigFile, 0)
+ // TODO: should be parameterized and generated from varnishcfg
+ configs = append(configs, t3cutil.ATSConfigFile{
+ Name: "default.vcl",
+ Text: vcl,
+ Path: cfg.Dir,
+ ContentType: "text/plain; charset=us-ascii",
+ LineComment: "//",
+ Secure: false,
+ })
+ return configs, err
+}
diff --git a/cache-config/t3c-generate/config/config.go b/cache-config/t3c-generate/config/config.go
index 77a01df771..47814756eb 100644
--- a/cache-config/t3c-generate/config/config.go
+++ b/cache-config/t3c-generate/config/config.go
@@ -62,6 +62,7 @@ type Cfg struct {
DefaultTLSVersions []atscfg.TLSVersion
Version string
GitRevision string
+ Cache string
}
func (cfg Cfg) ErrorLog() log.LogLocation { return log.LogLocation(cfg.LogLocationErr) }
@@ -88,6 +89,7 @@ func GetCfg(appVersion string, gitRevision string) (Cfg, error) {
atsVersion := getopt.StringLong("ats-version", 'a', "", "The ATS version, e.g. 9.1.2-42.abc123.el7.x86_64. If omitted, generation will attempt to get the ATS version from the Server Parameters, and fall back to lib/go-atscfg.DefaultATSVersion")
verbosePtr := getopt.CounterLong("verbose", 'v', `Log verbosity. Logging is output to stderr. By default, errors are logged. To log warnings, pass '-v'. To log info, pass '-vv'. To omit error logging, see '-s'`)
silentPtr := getopt.BoolLong("silent", 's', `Silent. Errors are not logged, and the 'verbose' flag is ignored. If a fatal error occurs, the return code will be non-zero but no text will be output to stderr`)
+ cache := getopt.StringLong("cache", 'C', "ats", "Cache server type. Generate configuration files for specific cache server type, e.g. 'ats', 'varnish'.")
const useStrategiesFlagName = "use-strategies"
const defaultUseStrategies = t3cutil.UseStrategiesFlagFalse
@@ -185,6 +187,7 @@ func GetCfg(appVersion string, gitRevision string) (Cfg, error) {
GitRevision: gitRevision,
UseStrategies: t3cutil.UseStrategiesFlag(*useStrategiesPtr),
GoDirect: *goDirectPtr,
+ Cache: *cache,
}
if err := log.InitCfg(cfg); err != nil {
return Cfg{}, errors.New("Initializing loggers: " + err.Error() + "\n")
diff --git a/cache-config/t3c-generate/t3c-generate.go b/cache-config/t3c-generate/t3c-generate.go
index 9211c8ade4..f73646df6c 100644
--- a/cache-config/t3c-generate/t3c-generate.go
+++ b/cache-config/t3c-generate/t3c-generate.go
@@ -85,6 +85,20 @@ func main() {
os.Exit(config.ExitCodeErrGeneric)
}
+ if cfg.Cache == "varnish" {
+ configs, err := cfgfile.GetVarnishConfigs(toData, cfg)
+ if err != nil {
+ log.Errorln("Generating varnish config for'" + *toData.Server.HostName + "': " + err.Error())
+ os.Exit(config.ExitCodeErrGeneric)
+ }
+ err = cfgfile.WriteConfigs(configs, os.Stdout)
+ if err != nil {
+ log.Errorln("Writing configs for '" + *toData.Server.HostName + "': " + err.Error())
+ os.Exit(config.ExitCodeErrGeneric)
+ }
+ os.Exit(config.ExitCodeSuccess)
+ }
+
configs, err := cfgfile.GetAllConfigs(toData, cfg)
if err != nil {
log.Errorln("Getting config for'" + *toData.Server.HostName + "': " + err.Error())
diff --git a/infrastructure/cdn-in-a-box/enroller/Dockerfile b/infrastructure/cdn-in-a-box/enroller/Dockerfile
index 909a1fb6b7..9c431ae837 100644
--- a/infrastructure/cdn-in-a-box/enroller/Dockerfile
+++ b/infrastructure/cdn-in-a-box/enroller/Dockerfile
@@ -42,6 +42,9 @@ COPY ./traffic_ops/toclientlib/ /go/src/github.com/apache/trafficcontrol/traffic
COPY ./traffic_ops/v4-client/ /go/src/github.com/apache/trafficcontrol/traffic_ops/v4-client/
COPY ./infrastructure/cdn-in-a-box/ /go/src/github.com/apache/trafficcontrol/infrastructure/cdn-in-a-box/
+# varnishcfg requires t3c for ToData struct and not needed for enroller
+RUN rm -rf /go/src/github.com/apache/trafficcontrol/lib/varnishcfg
+
WORKDIR /go/src/github.com/apache/trafficcontrol/infrastructure/cdn-in-a-box/enroller
RUN set -o errexit -o nounset; \
go clean; \
diff --git a/infrastructure/cdn-in-a-box/varnish/Dockerfile b/infrastructure/cdn-in-a-box/varnish/Dockerfile
new file mode 100644
index 0000000000..c31e54c47b
--- /dev/null
+++ b/infrastructure/cdn-in-a-box/varnish/Dockerfile
@@ -0,0 +1,63 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+ARG BASE_IMAGE=rockylinux \
+ RHEL_VERSION=8
+FROM ${BASE_IMAGE}:${RHEL_VERSION} AS common-varnish-cache-config-layers
+ARG RHEL_VERSION=8
+# Makes RHEL_VERSION available at runtime
+ENV RHEL_VERSION="$RHEL_VERSION"
+
+RUN dnf module disable varnish -y && yum install -y epel-release
+
+RUN curl -s https://packagecloud.io/install/repositories/varnishcache/varnish73/script.rpm.sh | bash
+
+RUN yum install varnish-7.3.0 -y
+
+RUN dnf install -y bind-utils kyotocabinet-libs initscripts iproute net-tools nmap-ncat gettext autoconf automake libtool gcc-c++ cronie glibc-devel openssl-devel git perl && \
+ dnf install -y jq logrotate findutils && \
+ dnf clean all
+
+
+COPY infrastructure/cdn-in-a-box/varnish/run.sh infrastructure/cdn-in-a-box/traffic_ops/to-access.sh infrastructure/cdn-in-a-box/enroller/server_template.json /
+
+COPY infrastructure/cdn-in-a-box/dns/set-dns.sh \
+ infrastructure/cdn-in-a-box/dns/insert-self-into-dns.sh \
+ /usr/local/sbin/
+
+
+COPY infrastructure/cdn-in-a-box/varnish/systemctl.sh /usr/bin/systemctl
+
+ARG ORT_RPM=infrastructure/cdn-in-a-box/cache/trafficcontrol-cache-config.rpm
+COPY $ORT_RPM /
+RUN rpm -Uvh /$(basename $ORT_RPM) &&\
+ rm /$(basename $ORT_RPM)
+
+COPY infrastructure/cdn-in-a-box/varnish/traffic_ops_ort.crontab /etc/cron.d/traffic_ops_ort-cron-template
+
+
+CMD /run.sh
+
+FROM common-varnish-cache-config-layers AS mid
+ENV CACHE_TYPE=mid
+COPY infrastructure/cdn-in-a-box/mid/init.d/ /opt/init.d/
+
+FROM common-varnish-cache-config-layers AS edge
+ENV CACHE_TYPE=edge
+COPY infrastructure/cdn-in-a-box/edge/init.d/ /opt/init.d/
+
+
diff --git a/infrastructure/cdn-in-a-box/varnish/run.sh b/infrastructure/cdn-in-a-box/varnish/run.sh
new file mode 100755
index 0000000000..eb9ccd4f62
--- /dev/null
+++ b/infrastructure/cdn-in-a-box/varnish/run.sh
@@ -0,0 +1,93 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+trap 'echo "Error on line ${LINENO} of ${0}"; exit 1' ERR
+set -o errexit -o nounset -o pipefail -o xtrace -o monitor
+env > /ciab.env
+
+mkdir /tmp/ort
+
+set-dns.sh
+insert-self-into-dns.sh
+
+source /to-access.sh
+
+# Wait on SSL certificate generation
+until [[ -f "$X509_CA_ENV_FILE" ]]
+do
+ echo "Waiting on Shared SSL certificate generation"
+ sleep 3
+done
+
+# Source the CIAB-CA shared SSL environment
+until [[ -v X509_GENERATION_COMPLETE && -n "$X509_GENERATION_COMPLETE" ]]
+do
+ echo "Waiting on X509 vars to be defined"
+ sleep 1
+ source "$X509_CA_ENV_FILE"
+done
+
+# Trust the CIAB-CA at the System level
+cp "$X509_CA_CERT_FULL_CHAIN_FILE" /etc/pki/ca-trust/source/anchors
+update-ca-trust extract
+
+while ! to-ping 2>/dev/null; do
+ echo "waiting for Traffic Ops"
+ sleep 5
+done
+
+export TO_USER=$TO_ADMIN_USER
+export TO_PASSWORD=$TO_ADMIN_PASSWORD
+
+# wait until the CDN has been registered
+found=
+while [[ -z $found ]]; do
+ echo 'waiting for enroller setup'
+ sleep 3
+ found=$(to-get api/4.0/cdns?name="$CDN_NAME" | jq -r '.response[].name')
+done
+
+for f in /opt/init.d/*; do
+ echo "$f"
+ source "$f"
+done
+
+# Wait for SSL keys to exist
+until [[ $(to-get "api/4.0/cdns/name/$CDN_NAME/sslkeys" | jq '.response | length') -ge 2 ]]; do
+ echo 'waiting for SSL keys to exist'
+ sleep 3
+done
+mkdir -p /tmp/trafficcontrol-cache-config
+mkdir -p /opt/cache/etc/varnish
+
+# hostname is already defined in /etc/init.d/99-run.sh
+hostname="${hostname//-/_}" # replace - with _
+hostname="${hostname^^}" # uppercase
+debug_variable_name="T3C_DEBUG_COMPONENT_${hostname}"
+debug_binary="${!debug_variable_name}"
+if ! type -p "$debug_binary"; then
+ t3c apply --cache=varnish --trafficserver-home=/opt/cache --run-mode=badass --traffic-ops-url="$TO_URL" --traffic-ops-user="$TO_USER" --traffic-ops-password="$TO_PASSWORD" --git=yes -vv || { echo "Failed"; }
+fi
+
+envsubst < "/etc/cron.d/traffic_ops_ort-cron-template" > "/etc/cron.d/traffic_ops_ort-cron" && rm -f "/etc/cron.d/traffic_ops_ort-cron-template"
+chmod "0644" "/etc/cron.d/traffic_ops_ort-cron" && crontab "/etc/cron.d/traffic_ops_ort-cron"
+
+crond -im off
+
+varnishlog
diff --git a/infrastructure/cdn-in-a-box/varnish/systemctl.sh b/infrastructure/cdn-in-a-box/varnish/systemctl.sh
new file mode 100755
index 0000000000..cfe2a55c24
--- /dev/null
+++ b/infrastructure/cdn-in-a-box/varnish/systemctl.sh
@@ -0,0 +1,94 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+VARNISHD_EXECUTABLE="/usr/sbin/varnishd"
+
+is_varnishd_running() {
+ pgrep -x "$(basename "$VARNISHD_EXECUTABLE")" >/dev/null
+}
+
+start_varnishd() {
+ if is_varnishd_running; then
+ echo "varnishd is already running."
+ else
+ echo "Starting varnishd..."
+ "$VARNISHD_EXECUTABLE" -f /opt/cache/etc/varnish/default.vcl
+ echo "varnishd is now running."
+ fi
+}
+
+stop_varnishd() {
+ if is_varnishd_running; then
+ echo "Stopping varnishd..."
+
+ # Send SIGTERM signal to varnishd to terminate gracefully
+ pkill -x "$(basename "$VARNISHD_EXECUTABLE")"
+
+ # Wait for varnishd to stop, giving it a timeout of 10 seconds
+ timeout=10
+ while is_varnishd_running; do
+ if ((timeout-- == 0)); then
+ echo "Timed out waiting for varnishd to stop. Sending SIGKILL..."
+ pkill -9 -x "$(basename "$VARNISHD_EXECUTABLE")"
+ break
+ fi
+ sleep 1
+ done
+
+ if is_varnishd_running; then
+ echo "Failed to stop varnishd."
+ else
+ echo "varnishd is stopped."
+ fi
+ else
+ echo "varnishd is not running."
+ fi
+}
+
+restart_varnishd() {
+ echo "Restarting varnishd..."
+ stop_varnishd
+ start_varnishd
+}
+
+case "$1" in
+ enable)
+ exit 0
+ ;;
+ start)
+ start_varnishd
+ ;;
+ stop)
+ stop_varnishd
+ ;;
+ restart)
+ restart_varnishd
+ ;;
+ status)
+ if is_varnishd_running; then
+ exit 0
+ fi
+ exit 3
+ ;;
+ *)
+ echo "Usage: $0 {start|stop|restart|enable|status}"
+ exit 1
+esac
+
+exit 0
diff --git a/infrastructure/cdn-in-a-box/varnish/traffic_ops_ort.crontab b/infrastructure/cdn-in-a-box/varnish/traffic_ops_ort.crontab
new file mode 100644
index 0000000000..d830ed0062
--- /dev/null
+++ b/infrastructure/cdn-in-a-box/varnish/traffic_ops_ort.crontab
@@ -0,0 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+*/1 * * * * root t3c apply --cache=varnish --run-mode=syncds --traffic-ops-url=$TO_URL --traffic-ops-user=$TO_USER --traffic-ops-password=$TO_PASSWORD --git=yes -vv --cache-host-name=$(hostname -s) >> /var/log/ort.log 2>&1
diff --git a/lib/go-atscfg/parentabstraction.go b/lib/go-atscfg/parentabstraction.go
index b827c7ab6b..4c604e58a3 100644
--- a/lib/go-atscfg/parentabstraction.go
+++ b/lib/go-atscfg/parentabstraction.go
@@ -130,6 +130,9 @@ type ParentAbstractionService struct {
// Becomes parent.config weight directive
// Becomes strategies.yaml TODO
Weight float64
+
+ // DS is the delivery service associated with the service
+ DS DeliveryService
}
// ParentAbstractionServices implements sort.Interface
diff --git a/lib/go-atscfg/parentdotconfig.go b/lib/go-atscfg/parentdotconfig.go
index 282d9861d7..1ff5aebcc6 100644
--- a/lib/go-atscfg/parentdotconfig.go
+++ b/lib/go-atscfg/parentdotconfig.go
@@ -141,7 +141,7 @@ func MakeParentDotConfig(
warnings := []string{}
atsMajorVersion := getATSMajorVersion(opt.ATSMajorVersion, tcServerParams, &warnings)
- parentAbstraction, dataWarns, err := makeParentDotConfigData(
+ parentAbstraction, dataWarns, err := MakeParentDotConfigData(
dses,
server,
servers,
@@ -292,7 +292,7 @@ func createTopology(server *Server, ds DeliveryService, nameTopologies map[Topol
return topoName, topo, warns
}
-func makeParentDotConfigData(
+func MakeParentDotConfigData(
dses []DeliveryService,
server *Server,
servers []Server,
@@ -1057,6 +1057,7 @@ func getTopologyParentConfigLine(
}
pasvc := &ParentAbstractionService{}
+ pasvc.DS = *ds
pasvc.Name = *ds.XMLID
pasvc.Comment = makeParentComment(addComments, *ds.XMLID, *ds.Topology)
pasvc.DestDomain = orgURI.Hostname()
diff --git a/lib/go-atscfg/remapdotconfig.go b/lib/go-atscfg/remapdotconfig.go
index db610bfcd9..2e059fe38e 100644
--- a/lib/go-atscfg/remapdotconfig.go
+++ b/lib/go-atscfg/remapdotconfig.go
@@ -153,7 +153,7 @@ func MakeRemapDotConfig(
}
cdnDomain := cdn.DomainName
- dsRegexes := makeDSRegexMap(dsRegexArr)
+ dsRegexes := MakeDSRegexMap(dsRegexArr)
// Returned DSes are guaranteed to have a non-nil XMLID, Type, DSCP, ID, and Active.
dses, dsWarns := remapFilterDSes(server, dss, unfilteredDSes)
warnings = append(warnings, dsWarns...)
@@ -174,7 +174,7 @@ func MakeRemapDotConfig(
}
nameTopologies := makeTopologyNameMap(topologies)
- anyCastPartners := getAnyCastPartners(server, servers)
+ anyCastPartners := GetAnyCastPartners(server, servers)
hdr := makeHdrComment(opt.HdrComment)
txt := ""
@@ -543,7 +543,7 @@ func getServerConfigRemapDotConfigForEdge(
continue
}
- requestFQDNs, err := getDSRequestFQDNs(&ds, dsRegexes[tc.DeliveryServiceName(*ds.XMLID)], server, anyCastPartners, cdnDomain)
+ requestFQDNs, err := GetDSRequestFQDNs(&ds, dsRegexes[tc.DeliveryServiceName(*ds.XMLID)], server, anyCastPartners, cdnDomain)
if err != nil {
warnings = append(warnings, "error getting ds '"+*ds.XMLID+"' request fqdns, skipping! Error: "+err.Error())
continue
@@ -1053,7 +1053,7 @@ func (r deliveryServiceRegexesSortByTypeThenSetNum) Less(i, j int) bool {
}
func (r deliveryServiceRegexesSortByTypeThenSetNum) Swap(i, j int) { r[i], r[j] = r[j], r[i] }
-func makeDSRegexMap(regexes []tc.DeliveryServiceRegexes) map[tc.DeliveryServiceName][]tc.DeliveryServiceRegex {
+func MakeDSRegexMap(regexes []tc.DeliveryServiceRegexes) map[tc.DeliveryServiceName][]tc.DeliveryServiceRegex {
dsRegexMap := map[tc.DeliveryServiceName][]tc.DeliveryServiceRegex{}
for _, dsRegex := range regexes {
sort.Sort(deliveryServiceRegexesSortByTypeThenSetNum(dsRegex.Regexes))
@@ -1062,7 +1062,7 @@ func makeDSRegexMap(regexes []tc.DeliveryServiceRegexes) map[tc.DeliveryServiceN
return dsRegexMap
}
-func getAnyCastPartners(server *Server, servers []Server) map[string][]string {
+func GetAnyCastPartners(server *Server, servers []Server) map[string][]string {
anyCastIPs := make(map[string][]string)
for _, int := range server.Interfaces {
if int.Name == "lo" {
@@ -1104,8 +1104,8 @@ func (ks keyVals) Less(i, j int) bool {
return ks[i].Val < ks[j].Val
}
-// getDSRequestFQDNs returns the FQDNs that clients will request from the edge.
-func getDSRequestFQDNs(ds *DeliveryService, regexes []tc.DeliveryServiceRegex, server *Server, anyCastPartners map[string][]string, cdnDomain string) ([]string, error) {
+// GetDSRequestFQDNs returns the FQDNs that clients will request from the edge.
+func GetDSRequestFQDNs(ds *DeliveryService, regexes []tc.DeliveryServiceRegex, server *Server, anyCastPartners map[string][]string, cdnDomain string) ([]string, error) {
if server.HostName == nil {
return nil, errors.New("server missing hostname")
}
diff --git a/lib/go-atscfg/remapdotconfig_test.go b/lib/go-atscfg/remapdotconfig_test.go
index a4ee79e70c..22ed2eede8 100644
--- a/lib/go-atscfg/remapdotconfig_test.go
+++ b/lib/go-atscfg/remapdotconfig_test.go
@@ -110,7 +110,7 @@ func TestAnyCastRemapDotConfig(t *testing.T) {
server.Interfaces = []tc.ServerInterfaceInfoV40{}
setIPInfo(server, "lo", "192.168.2.6", "fdf8:f53b:82e4::53")
servers := makeTestAnyCastServers()
- for _, anyCsstServer := range getAnyCastPartners(server, servers) {
+ for _, anyCsstServer := range GetAnyCastPartners(server, servers) {
if len(anyCsstServer) != 2 {
t.Errorf("expected 2 edges in anycast group, actual '%v'", len(anyCsstServer))
}
diff --git a/lib/go-atscfg/sslservernamedotyaml.go b/lib/go-atscfg/sslservernamedotyaml.go
index 3d5e0aed05..5bc87aa03e 100644
--- a/lib/go-atscfg/sslservernamedotyaml.go
+++ b/lib/go-atscfg/sslservernamedotyaml.go
@@ -217,7 +217,7 @@ func GetServerSSLData(
return nil, warnings, errors.New("this server missing Profiles")
}
- dsRegexes := makeDSRegexMap(dsRegexArr)
+ dsRegexes := MakeDSRegexMap(dsRegexArr)
parentConfigParamsWithProfiles, err := tcParamsToParamsWithProfiles(tcParentConfigParams)
if err != nil {
@@ -241,7 +241,7 @@ func GetServerSSLData(
}
nameTopologies := makeTopologyNameMap(topologies)
- anyCastPartners := getAnyCastPartners(server, servers)
+ anyCastPartners := GetAnyCastPartners(server, servers)
sort.Sort(dsesSortByName(dses))
@@ -262,7 +262,7 @@ func GetServerSSLData(
dsParentConfigParams = profileParentConfigParams[*ds.ProfileName]
}
- requestFQDNs, err := getDSRequestFQDNs(&ds, dsRegexes[tc.DeliveryServiceName(*ds.XMLID)], server, anyCastPartners, cdn.DomainName)
+ requestFQDNs, err := GetDSRequestFQDNs(&ds, dsRegexes[tc.DeliveryServiceName(*ds.XMLID)], server, anyCastPartners, cdn.DomainName)
if err != nil {
warnings = append(warnings, "error getting ds '"+*ds.XMLID+"' request fqdns, skipping! Error: "+err.Error())
continue
diff --git a/lib/go-atscfg/strategiesdotconfig.go b/lib/go-atscfg/strategiesdotconfig.go
index df2c88cb54..ee51d4d32c 100644
--- a/lib/go-atscfg/strategiesdotconfig.go
+++ b/lib/go-atscfg/strategiesdotconfig.go
@@ -81,7 +81,7 @@ func MakeStrategiesDotYAML(
atsMajorVersion := getATSMajorVersion(opt.ATSMajorVersion, tcServerParams, &warnings)
- parentAbstraction, dataWarns, err := makeParentDotConfigData(
+ parentAbstraction, dataWarns, err := MakeParentDotConfigData(
dses,
server,
servers,
@@ -99,7 +99,7 @@ func MakeStrategiesDotYAML(
ATSMajorVersion: opt.ATSMajorVersion,
GoDirect: opt.GoDirect,
ParentIsProxy: opt.ParentIsProxy,
- }, // TODO change makeParentDotConfigData to its own opt?
+ }, // TODO change MakeParentDotConfigData to its own opt?
atsMajorVersion,
)
warnings = append(warnings, dataWarns...)
diff --git a/lib/varnishcfg/backends.go b/lib/varnishcfg/backends.go
new file mode 100644
index 0000000000..7ff67796eb
--- /dev/null
+++ b/lib/varnishcfg/backends.go
@@ -0,0 +1,183 @@
+package varnishcfg
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import (
+ "fmt"
+ "strings"
+
+ "github.com/apache/trafficcontrol/lib/go-atscfg"
+ "github.com/apache/trafficcontrol/lib/go-tc"
+)
+
+func (v *VCLBuilder) configureDirectors(vclFile *vclFile, parents *atscfg.ParentAbstraction) ([]string, error) {
+ warnings := []string{}
+
+ vclFile.imports = append(vclFile.imports, "directors")
+ var err error
+ requestFQDNs := make([]string, 0)
+
+ for _, svc := range parents.Services {
+ addBackends(vclFile.backends, append(svc.Parents, svc.SecondaryParents...), svc.DestDomain, svc.Port)
+ addDirectors(vclFile.subroutines, svc)
+
+ requestFQDNs = []string{svc.DestDomain}
+
+ if v.toData.Server.Type == tc.CacheTypeEdge.String() {
+ dsRegexes := atscfg.MakeDSRegexMap(v.toData.DeliveryServiceRegexes)
+ anyCastPartners := atscfg.GetAnyCastPartners(v.toData.Server, v.toData.Servers)
+ requestFQDNs, err = atscfg.GetDSRequestFQDNs(
+ &svc.DS,
+ dsRegexes[tc.DeliveryServiceName(*svc.DS.XMLID)],
+ v.toData.Server,
+ anyCastPartners,
+ v.toData.CDN.DomainName,
+ )
+ if err != nil {
+ warnings = append(warnings, "error getting ds '"+*svc.DS.XMLID+"' request fqdns, skipping! Error: "+err.Error())
+ continue
+ }
+ }
+
+ assignBackends(vclFile.subroutines, svc, requestFQDNs)
+ }
+
+ return warnings, nil
+}
+
+func assignBackends(subroutines map[string][]string, svc *atscfg.ParentAbstractionService, requestFQDNs []string) {
+ lines := make([]string, 0)
+ hostHeaderLines := make([]string, 0)
+
+ conditions := make([]string, 0)
+ backendConditions := make([]string, 0)
+ for _, fqdn := range requestFQDNs {
+ conditions = append(conditions, fmt.Sprintf(`req.http.host == "%s"`, fqdn))
+ backendConditions = append(backendConditions, fmt.Sprintf(`bereq.http.host == "%s"`, fqdn))
+ }
+
+ lines = append(lines, fmt.Sprintf("if (%s) {", strings.Join(conditions, " || ")))
+ lines = append(lines, fmt.Sprintf("\tset req.backend_hint = %s.backend();", svc.Name))
+
+ // only change request host from edge servers which typically has multiple request FQDNs or
+ // one request FQDN that is not the origin.
+ if len(requestFQDNs) > 1 || (len(requestFQDNs) == 1 && requestFQDNs[0] != svc.DestDomain) {
+ hostHeaderLines = append(hostHeaderLines, fmt.Sprintf("if (%s) {", strings.Join(backendConditions, " || ")))
+ hostHeaderLines = append(hostHeaderLines, fmt.Sprintf("\tset bereq.http.host = \"%s\";", svc.DestDomain))
+ hostHeaderLines = append(hostHeaderLines, "}")
+ }
+
+ lines = append(lines, "}")
+
+ if _, ok := subroutines["vcl_recv"]; !ok {
+ subroutines["vcl_recv"] = make([]string, 0)
+ }
+ subroutines["vcl_recv"] = append(subroutines["vcl_recv"], lines...)
+ if len(hostHeaderLines) == 0 {
+ return
+ }
+
+ if _, ok := subroutines["vcl_backend_fetch"]; !ok {
+ subroutines["vcl_backend_fetch"] = make([]string, 0)
+ }
+ subroutines["vcl_backend_fetch"] = append(subroutines["vcl_backend_fetch"], hostHeaderLines...)
+}
+
+func addBackends(backends map[string]backend, parents []*atscfg.ParentAbstractionServiceParent, originDomain string, originPort int) {
+ for _, parent := range parents {
+ backendName := fmt.Sprintf("%s", getBackendName(parent.FQDN, parent.Port))
+ if _, ok := backends[backendName]; ok {
+ continue
+ }
+ backends[backendName] = backend{
+ host: parent.FQDN,
+ port: parent.Port,
+ }
+ }
+ backendName := getBackendName(originDomain, originPort)
+ if _, ok := backends[backendName]; ok {
+ return
+ }
+ backends[backendName] = backend{
+ host: originDomain,
+ port: originPort,
+ }
+}
+
+func addDirectors(subroutines map[string][]string, svc *atscfg.ParentAbstractionService) {
+ lines := make([]string, 0)
+ fallbackDirectorLines := make([]string, 0)
+ fallbackDirectorLines = append(fallbackDirectorLines, fmt.Sprintf("new %s = directors.fallback();", svc.Name))
+
+ if len(svc.Parents) != 0 {
+ lines = append(lines, addBackendsToDirector(svc.Name+"_primary", svc.RetryPolicy, svc.Parents)...)
+ fallbackDirectorLines = append(fallbackDirectorLines, fmt.Sprintf("%s.add_backend(%s_primary.backend());", svc.Name, svc.Name))
+ }
+ if len(svc.SecondaryParents) != 0 {
+ lines = append(lines, addBackendsToDirector(svc.Name+"_secondary", svc.RetryPolicy, svc.SecondaryParents)...)
+ fallbackDirectorLines = append(fallbackDirectorLines, fmt.Sprintf("%s.add_backend(%s_secondary.backend());", svc.Name, svc.Name))
+ }
+ fallbackDirectorLines = append(fallbackDirectorLines, fmt.Sprintf("%s.add_backend(%s);", svc.Name, getBackendName(svc.DestDomain, svc.Port)))
+
+ lines = append(lines, fallbackDirectorLines...)
+
+ if _, ok := subroutines["vcl_init"]; !ok {
+ subroutines["vcl_init"] = make([]string, 0)
+ }
+ subroutines["vcl_init"] = append(subroutines["vcl_init"], lines...)
+}
+
+func addBackendsToDirector(name string, retryPolicy atscfg.ParentAbstractionServiceRetryPolicy, parents []*atscfg.ParentAbstractionServiceParent) []string {
+ lines := make([]string, 0)
+ directorType, sticky := getDirectorType(retryPolicy)
+ lines = append(lines, fmt.Sprintf("new %s = directors.%s(%s);", name, directorType, sticky))
+ for _, parent := range parents {
+ lines = append(lines, fmt.Sprintf("%s.add_backend(%s);", name, getBackendName(parent.FQDN, parent.Port)))
+ }
+ return lines
+}
+
+func getDirectorType(retryPolicy atscfg.ParentAbstractionServiceRetryPolicy) (director string, sticky string) {
+ switch retryPolicy {
+ case atscfg.ParentAbstractionServiceRetryPolicyRoundRobinIP:
+ fallthrough
+ case atscfg.ParentAbstractionServiceRetryPolicyRoundRobinStrict:
+ director = "round_robin"
+ case atscfg.ParentAbstractionServiceRetryPolicyFirst:
+ director = "fallback"
+ case atscfg.ParentAbstractionServiceRetryPolicyLatched:
+ director = "fallback"
+ sticky = "1"
+ case atscfg.ParentAbstractionServiceRetryPolicyConsistentHash:
+ director = "shard"
+ default:
+ director = "shard"
+ }
+ return
+}
+
+func getBackendName(host string, port int) string {
+ // maybe a better way to ensure backend names are unique?
+
+ if port <= 0 {
+ return strings.ReplaceAll(host, ".", "_")
+ }
+ return fmt.Sprintf("%s_%d", strings.ReplaceAll(host, ".", "_"), port)
+}
diff --git a/lib/varnishcfg/backends_test.go b/lib/varnishcfg/backends_test.go
new file mode 100644
index 0000000000..276ef10f40
--- /dev/null
+++ b/lib/varnishcfg/backends_test.go
@@ -0,0 +1,354 @@
+package varnishcfg
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import (
+ "reflect"
+ "testing"
+
+ "github.com/apache/trafficcontrol/lib/go-atscfg"
+)
+
+func TestAddBackends(t *testing.T) {
+ testCases := []struct {
+ name string
+ backends map[string]backend
+ parents []*atscfg.ParentAbstractionServiceParent
+ originDomain string
+ originPort int
+ expectedBackends map[string]backend
+ }{
+ {
+ name: "no parents",
+ backends: make(map[string]backend),
+ parents: []*atscfg.ParentAbstractionServiceParent{},
+ originDomain: "origin.example.com",
+ originPort: 80,
+ expectedBackends: map[string]backend{
+ "origin_example_com_80": {
+ host: "origin.example.com",
+ port: 80,
+ },
+ },
+ },
+ {
+ name: "single parent",
+ backends: make(map[string]backend),
+ parents: []*atscfg.ParentAbstractionServiceParent{
+ {FQDN: "parent.example.com", Port: 444},
+ },
+ originDomain: "origin.example.com",
+ originPort: 80,
+ expectedBackends: map[string]backend{
+ "parent_example_com_444": {
+ host: "parent.example.com",
+ port: 444,
+ },
+ "origin_example_com_80": {
+ host: "origin.example.com",
+ port: 80,
+ },
+ },
+ },
+ {
+ name: "multiple parent",
+ backends: make(map[string]backend),
+ parents: []*atscfg.ParentAbstractionServiceParent{
+ {FQDN: "parent.example.com", Port: 444},
+ {FQDN: "parent2.example.com", Port: 80},
+ },
+ originDomain: "origin.example.com",
+ originPort: 80,
+ expectedBackends: map[string]backend{
+ "parent_example_com_444": {
+ host: "parent.example.com",
+ port: 444,
+ },
+ "parent2_example_com_80": {
+ host: "parent2.example.com",
+ port: 80,
+ },
+ "origin_example_com_80": {
+ host: "origin.example.com",
+ port: 80,
+ },
+ },
+ },
+ {
+ name: "already added parents",
+ backends: map[string]backend{
+ "parent_example_com_444": {
+ host: "parent.example.com",
+ port: 444,
+ },
+ "origin_example_com_80": {
+ host: "origin.example.com",
+ port: 80,
+ },
+ },
+ parents: []*atscfg.ParentAbstractionServiceParent{
+ {FQDN: "parent.example.com", Port: 444},
+ },
+ originDomain: "origin.example.com",
+ originPort: 80,
+ expectedBackends: map[string]backend{
+ "parent_example_com_444": {
+ host: "parent.example.com",
+ port: 444,
+ },
+ "origin_example_com_80": {
+ host: "origin.example.com",
+ port: 80,
+ },
+ },
+ },
+ }
+ for _, tC := range testCases {
+ t.Run(tC.name, func(t *testing.T) {
+ addBackends(tC.backends, tC.parents, tC.originDomain, tC.originPort)
+ if !reflect.DeepEqual(tC.expectedBackends, tC.backends) {
+ t.Errorf("expected %v got %v", tC.expectedBackends, tC.backends)
+ }
+ })
+ }
+}
+
+func TestAddBackendsToDirector(t *testing.T) {
+ testCases := []struct {
+ name string
+ directorName string
+ retryPolicy atscfg.ParentAbstractionServiceRetryPolicy
+ parents []*atscfg.ParentAbstractionServiceParent
+ expectedLines []string
+ }{
+ {
+ name: "round robin",
+ directorName: "dir",
+ retryPolicy: atscfg.ParentAbstractionServiceRetryPolicyRoundRobinStrict,
+ parents: []*atscfg.ParentAbstractionServiceParent{
+ {FQDN: "parent.example.com", Port: 80},
+ {FQDN: "parent2.example.com", Port: 80},
+ },
+ expectedLines: []string{
+ `new dir = directors.round_robin();`,
+ `dir.add_backend(parent_example_com_80);`,
+ `dir.add_backend(parent2_example_com_80);`,
+ },
+ },
+ {
+ name: "fallback",
+ directorName: "dir",
+ retryPolicy: atscfg.ParentAbstractionServiceRetryPolicyFirst,
+ parents: []*atscfg.ParentAbstractionServiceParent{
+ {FQDN: "parent.example.com", Port: 80},
+ {FQDN: "parent2.example.com", Port: 80},
+ },
+ expectedLines: []string{
+ `new dir = directors.fallback();`,
+ `dir.add_backend(parent_example_com_80);`,
+ `dir.add_backend(parent2_example_com_80);`,
+ },
+ },
+ {
+ name: "fallback sticky",
+ directorName: "dir",
+ retryPolicy: atscfg.ParentAbstractionServiceRetryPolicyLatched,
+ parents: []*atscfg.ParentAbstractionServiceParent{
+ {FQDN: "parent.example.com", Port: 80},
+ {FQDN: "parent2.example.com", Port: 80},
+ },
+ expectedLines: []string{
+ `new dir = directors.fallback(1);`,
+ `dir.add_backend(parent_example_com_80);`,
+ `dir.add_backend(parent2_example_com_80);`,
+ },
+ },
+ }
+ for _, tC := range testCases {
+ t.Run(tC.name, func(t *testing.T) {
+ lines := addBackendsToDirector(tC.directorName, tC.retryPolicy, tC.parents)
+ if !reflect.DeepEqual(tC.expectedLines, lines) {
+ t.Errorf("expected %v got %v", tC.expectedLines, lines)
+ }
+ })
+ }
+}
+
+func TestAddDirectors(t *testing.T) {
+ testCases := []struct {
+ name string
+ subroutines map[string][]string
+ svc *atscfg.ParentAbstractionService
+ expectedSubroutines map[string][]string
+ }{
+ {
+ name: "no parents",
+ subroutines: make(map[string][]string),
+ svc: &atscfg.ParentAbstractionService{
+ Name: "demo",
+ RetryPolicy: atscfg.ParentAbstractionServiceRetryPolicyConsistentHash,
+ Parents: []*atscfg.ParentAbstractionServiceParent{},
+ DestDomain: "origin.example.com",
+ Port: 80,
+ },
+ expectedSubroutines: map[string][]string{
+ "vcl_init": {
+ `new demo = directors.fallback();`,
+ `demo.add_backend(origin_example_com_80);`,
+ },
+ },
+ },
+ {
+ name: "primary parents",
+ subroutines: make(map[string][]string),
+ svc: &atscfg.ParentAbstractionService{
+ Name: "demo",
+ RetryPolicy: atscfg.ParentAbstractionServiceRetryPolicyConsistentHash,
+ Parents: []*atscfg.ParentAbstractionServiceParent{
+ {FQDN: "parent.example.com", Port: 80},
+ },
+ DestDomain: "origin.example.com",
+ Port: 80,
+ },
+ expectedSubroutines: map[string][]string{
+ "vcl_init": {
+ `new demo_primary = directors.shard();`,
+ `demo_primary.add_backend(parent_example_com_80);`,
+ `new demo = directors.fallback();`,
+ `demo.add_backend(demo_primary.backend());`,
+ `demo.add_backend(origin_example_com_80);`,
+ },
+ },
+ },
+ {
+ name: "primary and secondary parents",
+ subroutines: make(map[string][]string),
+ svc: &atscfg.ParentAbstractionService{
+ Name: "demo",
+ RetryPolicy: atscfg.ParentAbstractionServiceRetryPolicyLatched,
+ Parents: []*atscfg.ParentAbstractionServiceParent{
+ {FQDN: "parent.example.com", Port: 80},
+ },
+ SecondaryParents: []*atscfg.ParentAbstractionServiceParent{
+ {FQDN: "parent2.example.com", Port: 80},
+ },
+ DestDomain: "origin.example.com",
+ Port: 80,
+ },
+ expectedSubroutines: map[string][]string{
+ "vcl_init": {
+ `new demo_primary = directors.fallback(1);`,
+ `demo_primary.add_backend(parent_example_com_80);`,
+ `new demo_secondary = directors.fallback(1);`,
+ `demo_secondary.add_backend(parent2_example_com_80);`,
+ `new demo = directors.fallback();`,
+ `demo.add_backend(demo_primary.backend());`,
+ `demo.add_backend(demo_secondary.backend());`,
+ `demo.add_backend(origin_example_com_80);`,
+ },
+ },
+ },
+ }
+ for _, tC := range testCases {
+ t.Run(tC.name, func(t *testing.T) {
+ addDirectors(tC.subroutines, tC.svc)
+ if !reflect.DeepEqual(tC.expectedSubroutines, tC.subroutines) {
+ t.Errorf("expected %v got %v", tC.expectedSubroutines, tC.subroutines)
+ }
+ })
+ }
+}
+
+func TestAssignBackends(t *testing.T) {
+ testCases := []struct {
+ name string
+ subroutines map[string][]string
+ svc *atscfg.ParentAbstractionService
+ requestFQDNs []string
+ expectedSubroutines map[string][]string
+ }{
+ {
+ name: "edge with one request FQDN",
+ subroutines: make(map[string][]string),
+ svc: &atscfg.ParentAbstractionService{
+ Name: "demo",
+ DestDomain: "origin.example.com",
+ },
+ requestFQDNs: []string{"example.com"},
+ expectedSubroutines: map[string][]string{
+ "vcl_recv": {
+ `if (req.http.host == "example.com") {`,
+ ` set req.backend_hint = demo.backend();`,
+ `}`,
+ },
+ "vcl_backend_fetch": {
+ `if (bereq.http.host == "example.com") {`,
+ ` set bereq.http.host = "origin.example.com";`,
+ `}`,
+ },
+ },
+ },
+ {
+ name: "edge with multiple request FQDNs",
+ subroutines: make(map[string][]string),
+ svc: &atscfg.ParentAbstractionService{
+ Name: "demo",
+ DestDomain: "origin.example.com",
+ },
+ requestFQDNs: []string{"example.com", "another.example.com"},
+ expectedSubroutines: map[string][]string{
+ "vcl_recv": {
+ `if (req.http.host == "example.com" || req.http.host == "another.example.com") {`,
+ ` set req.backend_hint = demo.backend();`,
+ `}`,
+ },
+ "vcl_backend_fetch": {
+ `if (bereq.http.host == "example.com" || bereq.http.host == "another.example.com") {`,
+ ` set bereq.http.host = "origin.example.com";`,
+ `}`,
+ },
+ },
+ },
+ {
+ name: "mid",
+ subroutines: make(map[string][]string),
+ svc: &atscfg.ParentAbstractionService{
+ Name: "demo",
+ DestDomain: "origin.example.com",
+ },
+ requestFQDNs: []string{"origin.example.com"},
+ expectedSubroutines: map[string][]string{
+ "vcl_recv": {
+ `if (req.http.host == "origin.example.com") {`,
+ ` set req.backend_hint = demo.backend();`,
+ `}`,
+ },
+ },
+ },
+ }
+ for _, tC := range testCases {
+ t.Run(tC.name, func(t *testing.T) {
+ assignBackends(tC.subroutines, tC.svc, tC.requestFQDNs)
+ if !reflect.DeepEqual(tC.expectedSubroutines, tC.subroutines) {
+ t.Errorf("expected %v got %v", tC.expectedSubroutines, tC.subroutines)
+ }
+ })
+ }
+}
diff --git a/lib/varnishcfg/vcl.go b/lib/varnishcfg/vcl.go
new file mode 100644
index 0000000000..5608b21ff8
--- /dev/null
+++ b/lib/varnishcfg/vcl.go
@@ -0,0 +1,101 @@
+package varnishcfg
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import "fmt"
+
+const defaultVCLVersion = "4.1"
+
+// vclFile contains all VCL components
+type vclFile struct {
+ version string
+ imports []string
+ acls map[string][]string
+ backends map[string]backend
+ subroutines map[string][]string
+}
+
+func newVCLFile(version string) vclFile {
+ return vclFile{
+ version: version,
+ imports: make([]string, 0),
+ acls: make(map[string][]string),
+ backends: make(map[string]backend),
+ subroutines: make(map[string][]string),
+ }
+}
+
+func (v vclFile) String() string {
+ txt := fmt.Sprintf("vcl %s;\n", v.version)
+ for _, i := range v.imports {
+ txt += fmt.Sprintf("import %s;\n", i)
+ }
+
+ for name, backend := range v.backends {
+ txt += fmt.Sprintf("backend %s {\n", name)
+ txt += fmt.Sprint(backend)
+ txt += fmt.Sprint("}\n")
+ }
+ // varnishd will fail if there are no backends defined
+ if len(v.backends) == 0 {
+ txt += fmt.Sprint("backend default none;\n")
+ }
+
+ for name, acl := range v.acls {
+ txt += fmt.Sprintf("acl %s {\n", name)
+ for _, entry := range acl {
+ txt += fmt.Sprintf("\t%s\n", entry)
+ }
+ txt += fmt.Sprint("}\n")
+ }
+
+ // has to be before other subroutines for variables initialization
+ if _, ok := v.subroutines["vcl_init"]; ok {
+ txt += fmt.Sprint("sub vcl_init {\n")
+ for _, entry := range v.subroutines["vcl_init"] {
+ txt += fmt.Sprintf("\t%s\n", entry)
+ }
+ txt += fmt.Sprint("}\n")
+ }
+
+ for name, subroutine := range v.subroutines {
+ if name == "vcl_init" {
+ continue
+ }
+ txt += fmt.Sprintf("sub %s {\n", name)
+ for _, entry := range subroutine {
+ txt += fmt.Sprintf("\t%s\n", entry)
+ }
+ txt += fmt.Sprint("}\n")
+ }
+
+ return txt
+}
+
+type backend struct {
+ host string
+ port int
+}
+
+func (b backend) String() string {
+ txt := fmt.Sprintf("\t.host = \"%s\";\n", b.host)
+ txt += fmt.Sprintf("\t.port = \"%d\";\n", b.port)
+ return txt
+}
diff --git a/lib/varnishcfg/vclbuilder.go b/lib/varnishcfg/vclbuilder.go
new file mode 100644
index 0000000000..a40221c87f
--- /dev/null
+++ b/lib/varnishcfg/vclbuilder.go
@@ -0,0 +1,75 @@
+// Package varnishcfg manages generating configuration files
+// for Varnish cache and Hitch proxy using data from Traffic Ops APIs.
+package varnishcfg
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import (
+ "fmt"
+ "strings"
+
+ "github.com/apache/trafficcontrol/cache-config/t3cutil"
+ "github.com/apache/trafficcontrol/lib/go-atscfg"
+)
+
+// VCLBuilder builds the default VCL file using TO data.
+type VCLBuilder struct {
+ toData *t3cutil.ConfigData
+ // opts
+}
+
+// NewVCLBuilder returns a new VCLBuilder object.
+func NewVCLBuilder(toData *t3cutil.ConfigData) VCLBuilder {
+ return VCLBuilder{
+ toData: toData,
+ }
+}
+
+// BuildVCLFile builds the default VCL file.
+func (vb *VCLBuilder) BuildVCLFile() (string, []string, error) {
+ warnings := make([]string, 0)
+ v := newVCLFile(defaultVCLVersion)
+
+ atsMajorVersion := uint(9)
+
+ parents, dataWarns, err := atscfg.MakeParentDotConfigData(
+ vb.toData.DeliveryServices,
+ vb.toData.Server,
+ vb.toData.Servers,
+ vb.toData.Topologies,
+ vb.toData.ServerParams,
+ vb.toData.ParentConfigParams,
+ vb.toData.ServerCapabilities,
+ vb.toData.DSRequiredCapabilities,
+ vb.toData.CacheGroups,
+ vb.toData.DeliveryServiceServers,
+ vb.toData.CDN,
+ &atscfg.ParentConfigOpts{},
+ atsMajorVersion,
+ )
+ warnings = append(warnings, dataWarns...)
+ if err != nil {
+ return "", nil, fmt.Errorf("(warnings: %s) %w", strings.Join(warnings, ", "), err)
+ }
+
+ dirWarnings, err := vb.configureDirectors(&v, parents)
+ warnings = append(warnings, dirWarnings...)
+ return fmt.Sprint(v), warnings, err
+}