You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@trafficcontrol.apache.org by ra...@apache.org on 2019/01/17 21:24:44 UTC
[trafficcontrol] branch master updated: Add Traffic Ops Golang
regex_revalidate.config (#3235)
This is an automated email from the ASF dual-hosted git repository.
rawlin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/trafficcontrol.git
The following commit(s) were added to refs/heads/master by this push:
new cb4ecb7 Add Traffic Ops Golang regex_revalidate.config (#3235)
cb4ecb7 is described below
commit cb4ecb7a9bd9c5b06b93e0f2549c495ef17c0cec
Author: Robert Butts <ro...@users.noreply.github.com>
AuthorDate: Thu Jan 17 14:24:38 2019 -0700
Add Traffic Ops Golang regex_revalidate.config (#3235)
* Add TO Go regex_revalidate.config
* Add Tests for TO API jobs, regex_revalidate.config
---
lib/go-tc/jobs.go | 89 ++++++++
traffic_ops/client/atsconfig.go | 39 ++++
traffic_ops/client/job.go | 74 ++++++
traffic_ops/testing/api/v14/jobs_test.go | 105 +++++++++
.../api/v14/regexrevalidatedotconfig_test.go | 40 ++++
traffic_ops/testing/api/v14/tc-fixtures.json | 18 ++
traffic_ops/testing/api/v14/todb.go | 12 +
traffic_ops/testing/api/v14/traffic_control.go | 6 +
.../traffic_ops_golang/ats/regexrevalidate.go | 249 +++++++++++++++++++++
.../traffic_ops_golang/cachegroup/queueupdate.go | 14 +-
.../cdnfederation/cdnfederations.go | 2 +-
.../traffic_ops_golang/dbhelpers/db_helpers.go | 14 +-
traffic_ops/traffic_ops_golang/routes.go | 3 +
13 files changed, 650 insertions(+), 15 deletions(-)
diff --git a/lib/go-tc/jobs.go b/lib/go-tc/jobs.go
new file mode 100644
index 0000000..4d12728
--- /dev/null
+++ b/lib/go-tc/jobs.go
@@ -0,0 +1,89 @@
+package tc
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import (
+ "encoding/json"
+ "errors"
+ "time"
+)
+
+type Job struct {
+ Parameters string `json:"parameters"`
+ Keyword string `json:"keyword"`
+ AssetURL string `json:"assetUrl"`
+ CreatedBy string `json:"createdBy"`
+ StartTime string `json:"startTime"`
+ ID int64 `json:"id"`
+ DeliveryService string `json:"deliveryService"`
+}
+
+// JobRequest contains the data to create a job.
+// Note this is a convenience struct for posting users; the actual JSON object is a JobRequestAPI
+type JobRequest struct {
+ TTL time.Duration
+ StartTime time.Time
+ DeliveryServiceID int64
+ Regex string
+ Urgent bool
+}
+
+// JobRequestTimeFormat is a Go reference time format, for use with time.Format, of the format required for Traffic Ops POST /user/current/jobs.
+const JobRequestTimeFormat = `2006-01-02 15:04:05`
+
+// JobTimeFormat is a Go reference time format, for use with time.Format, of the format sent by Traffic Ops GET /jobs
+const JobTimeFormat = `2006-01-02 15:04:05-07`
+
+func (jr JobRequest) MarshalJSON() ([]byte, error) {
+ return json.Marshal(JobRequestAPI{
+ TTLSeconds: int64(jr.TTL / time.Second),
+ StartTime: jr.StartTime.Format(JobRequestTimeFormat),
+ DSID: jr.DeliveryServiceID,
+ Regex: jr.Regex,
+ Urgent: jr.Urgent,
+ })
+}
+
+func (jr *JobRequest) UnmarshalJSON(b []byte) error {
+ jri := JobRequestAPI{}
+ if err := json.Unmarshal(b, &jri); err != nil {
+ return err
+ }
+ startTime, err := time.Parse(JobRequestTimeFormat, jri.StartTime)
+ if err != nil {
+ return errors.New("startTime '" + jri.StartTime + "' is not of the required format '" + JobRequestTimeFormat + "'")
+ }
+ *jr = JobRequest{
+ TTL: time.Duration(jri.TTLSeconds) * time.Second,
+ StartTime: startTime,
+ DeliveryServiceID: jri.DSID,
+ Regex: jri.Regex,
+ Urgent: jri.Urgent,
+ }
+ return nil
+}
+
+type JobRequestAPI struct {
+ TTLSeconds int64 `json:"ttl"`
+ StartTime string `json:"startTime"`
+ DSID int64 `json:"dsId"`
+ Regex string `json:"regex"`
+ Urgent bool `json:"urgent"`
+}
diff --git a/traffic_ops/client/atsconfig.go b/traffic_ops/client/atsconfig.go
new file mode 100644
index 0000000..c2ade03
--- /dev/null
+++ b/traffic_ops/client/atsconfig.go
@@ -0,0 +1,39 @@
+/*
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+*/
+
+package client
+
+import (
+ "io/ioutil"
+ "net/http"
+ "strconv"
+)
+
+func (to *Session) GetATSCDNConfig(cdnID int, fileName string) (string, ReqInf, error) {
+ return to.getConfigFile(apiBase + "/cdns/" + strconv.Itoa(cdnID) + "/configfiles/ats/" + fileName)
+}
+
+func (to *Session) GetATSCDNConfigByName(cdnName string, fileName string) (string, ReqInf, error) {
+ return to.getConfigFile(apiBase + "/cdns/" + cdnName + "/configfiles/ats/" + fileName)
+}
+
+func (to *Session) getConfigFile(uri string) (string, ReqInf, error) {
+ resp, remoteAddr, err := to.request(http.MethodGet, uri, nil)
+ reqInf := ReqInf{CacheHitStatus: CacheHitStatusMiss, RemoteAddr: remoteAddr}
+ if err != nil {
+ return "", reqInf, err
+ }
+ defer resp.Body.Close()
+
+ bts, err := ioutil.ReadAll(resp.Body)
+ return string(bts), reqInf, err
+}
diff --git a/traffic_ops/client/job.go b/traffic_ops/client/job.go
new file mode 100644
index 0000000..7df2434
--- /dev/null
+++ b/traffic_ops/client/job.go
@@ -0,0 +1,74 @@
+/*
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+*/
+
+package client
+
+import (
+ "encoding/json"
+ "net"
+ "net/http"
+ "strconv"
+
+ "github.com/apache/trafficcontrol/lib/go-tc"
+)
+
+// CreateJob creates a Job.
+func (to *Session) CreateJob(job tc.JobRequest) (tc.Alerts, ReqInf, error) {
+ remoteAddr := (net.Addr)(nil)
+ reqBody, err := json.Marshal(job)
+ reqInf := ReqInf{CacheHitStatus: CacheHitStatusMiss, RemoteAddr: remoteAddr}
+ if err != nil {
+ return tc.Alerts{}, reqInf, err
+ }
+ resp, remoteAddr, err := to.request(http.MethodPost, apiBase+`/user/current/jobs`, reqBody)
+ if err != nil {
+ return tc.Alerts{}, reqInf, err
+ }
+ defer resp.Body.Close()
+ alerts := tc.Alerts{}
+ err = json.NewDecoder(resp.Body).Decode(&alerts)
+ return alerts, reqInf, err
+}
+
+// GetJobs returns a list of Jobs.
+// If deliveryServiceID or userID are not nil, only jobs for that delivery service or belonging to that user are returned. Both deliveryServiceID and userID may be nil.
+func (to *Session) GetJobs(deliveryServiceID *int, userID *int) ([]tc.Job, ReqInf, error) {
+ path := apiBase + "/jobs"
+ if deliveryServiceID != nil || userID != nil {
+ path += "?"
+ if deliveryServiceID != nil {
+ path += "dsId=" + strconv.Itoa(*deliveryServiceID)
+ if userID != nil {
+ path += "&"
+ }
+ }
+ if userID != nil {
+ path += "userId=" + strconv.Itoa(*userID)
+ }
+ }
+
+ resp, remoteAddr, err := to.request(http.MethodGet, path, nil)
+ reqInf := ReqInf{CacheHitStatus: CacheHitStatusMiss, RemoteAddr: remoteAddr}
+ if err != nil {
+ return nil, reqInf, err
+ }
+ defer resp.Body.Close()
+
+ data := struct {
+ Response []tc.Job `json:"response"`
+ }{}
+ err = json.NewDecoder(resp.Body).Decode(&data)
+ return data.Response, reqInf, err
+}
diff --git a/traffic_ops/testing/api/v14/jobs_test.go b/traffic_ops/testing/api/v14/jobs_test.go
new file mode 100644
index 0000000..015c44b
--- /dev/null
+++ b/traffic_ops/testing/api/v14/jobs_test.go
@@ -0,0 +1,105 @@
+package v14
+
+/*
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+*/
+
+import (
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/apache/trafficcontrol/lib/go-tc"
+)
+
+func TestJobs(t *testing.T) {
+ WithObjs(t, []TCObj{CDNs, Types, Tenants, Parameters, Profiles, Statuses, Divisions, Regions, PhysLocations, CacheGroups, Servers, DeliveryServices}, func() {
+ CreateTestJobs(t)
+ GetTestJobs(t)
+ })
+}
+
+func CreateTestJobs(t *testing.T) {
+ toDSes, _, err := TOSession.GetDeliveryServices()
+ if err != nil {
+ t.Fatalf("cannot GET DeliveryServices: %v - %v\n", err, toDSes)
+ }
+ dsNameIDs := map[string]int64{}
+ for _, ds := range toDSes {
+ dsNameIDs[ds.XMLID] = int64(ds.ID)
+ }
+
+ for i, job := range testData.Jobs {
+ job.Request.StartTime = time.Now().UTC()
+ job.Request.DeliveryServiceID = dsNameIDs[job.DSName]
+ testData.Jobs[i] = job
+ }
+
+ for _, job := range testData.Jobs {
+ id, ok := dsNameIDs[job.DSName]
+ if !ok {
+ t.Fatalf("can't create test data job: delivery service '%v' not found in Traffic Ops", job.DSName)
+ }
+ job.Request.DeliveryServiceID = id
+ _, _, err := TOSession.CreateJob(job.Request)
+ if err != nil {
+ t.Errorf("could not CREATE job: %v\n", err)
+ }
+ }
+}
+
+func GetTestJobs(t *testing.T) {
+ toJobs, _, err := TOSession.GetJobs(nil, nil)
+ if err != nil {
+ t.Fatalf("error getting jobs: " + err.Error())
+ }
+
+ toDSes, _, err := TOSession.GetDeliveryServices()
+ if err != nil {
+ t.Fatalf("cannot GET DeliveryServices: %v - %v\n", err, toDSes)
+ }
+
+ dsIDNames := map[int64]string{}
+ for _, ds := range toDSes {
+ dsIDNames[int64(ds.ID)] = ds.XMLID
+ }
+
+ for _, testJob := range testData.Jobs {
+ found := false
+ for _, toJob := range toJobs {
+ if toJob.DeliveryService != dsIDNames[testJob.Request.DeliveryServiceID] {
+ continue
+ }
+ if !strings.HasSuffix(toJob.AssetURL, testJob.Request.Regex) {
+ continue
+ }
+ toJobTime, err := time.Parse(tc.JobTimeFormat, toJob.StartTime)
+ if err != nil {
+ t.Errorf("job ds %v regex %v start time expected format '%+v' actual '%+v' error '%+v'", testJob.Request.DeliveryServiceID, testJob.Request.Regex, tc.JobTimeFormat, toJob.StartTime, err)
+ continue
+ }
+ toJobTime = toJobTime.Round(time.Minute)
+ testJobTime := testJob.Request.StartTime.Round(time.Minute)
+ if !toJobTime.Equal(testJobTime) {
+ t.Errorf("test job ds %v regex %v start time expected '%+v' actual '%+v'", testJob.Request.DeliveryServiceID, testJob.Request.Regex, testJobTime, toJobTime)
+ continue
+ }
+ found = true
+ break
+ }
+ if !found {
+ t.Errorf("test job ds %v regex %v expected: exists, actual: not found", testJob.Request.DeliveryServiceID, testJob.Request.Regex)
+ }
+ }
+}
diff --git a/traffic_ops/testing/api/v14/regexrevalidatedotconfig_test.go b/traffic_ops/testing/api/v14/regexrevalidatedotconfig_test.go
new file mode 100644
index 0000000..af44c92
--- /dev/null
+++ b/traffic_ops/testing/api/v14/regexrevalidatedotconfig_test.go
@@ -0,0 +1,40 @@
+package v14
+
+/*
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+*/
+
+import (
+ "strings"
+ "testing"
+)
+
+func TestRegexRevalidateDotConfig(t *testing.T) {
+ WithObjs(t, []TCObj{CDNs, Types, Tenants, Parameters, Profiles, Statuses, Divisions, Regions, PhysLocations, CacheGroups, Servers, DeliveryServices}, func() {
+ CreateTestJobs(t)
+ GetTestRegexRevalidateDotConfig(t)
+ })
+}
+
+func GetTestRegexRevalidateDotConfig(t *testing.T) {
+ cdnName := "cdn1"
+
+ cfg, _, err := TOSession.GetATSCDNConfigByName(cdnName, "regex_revalidate.config")
+ if err != nil {
+ t.Fatalf("Getting cdn '" + cdnName + "' config regex_revalidate.config: " + err.Error() + "\n")
+ }
+
+ for _, testJob := range testData.Jobs {
+ if !strings.Contains(cfg, testJob.Request.Regex) {
+ t.Errorf("regex_revalidate.config '''%+v''' expected: contains '%+v' actual: missing", cfg, testJob.Request.Regex)
+ }
+ }
+}
diff --git a/traffic_ops/testing/api/v14/tc-fixtures.json b/traffic_ops/testing/api/v14/tc-fixtures.json
index bd88f2c..667ea45 100644
--- a/traffic_ops/testing/api/v14/tc-fixtures.json
+++ b/traffic_ops/testing/api/v14/tc-fixtures.json
@@ -1957,5 +1957,23 @@
"value": 42,
"type": "STEERING_WEIGHT"
}
+ ],
+ "jobs": [
+ {
+ "dsName": "ds1",
+ "request": {
+ "startTime": "2019-01-01 00:00:00",
+ "ttl": 2160,
+ "regex": "/foo"
+ }
+ },
+ {
+ "dsName": "ds1",
+ "request": {
+ "startTime": "2019-01-01 00:00:00",
+ "ttl": 2160,
+ "regex": "/foo"
+ }
+ }
]
}
diff --git a/traffic_ops/testing/api/v14/todb.go b/traffic_ops/testing/api/v14/todb.go
index d0c84a4..61bb12d 100644
--- a/traffic_ops/testing/api/v14/todb.go
+++ b/traffic_ops/testing/api/v14/todb.go
@@ -79,6 +79,18 @@ func SetupTestData(*sql.DB) error {
os.Exit(1)
}
+ err = SetupJobAgents(db)
+ if err != nil {
+ fmt.Printf("\nError setting up job agents %s - %s, %v\n", Config.TrafficOps.URL, Config.TrafficOps.Users.Admin, err)
+ os.Exit(1)
+ }
+
+ err = SetupJobStatuses(db)
+ if err != nil {
+ fmt.Printf("\nError setting up job agents %s - %s, %v\n", Config.TrafficOps.URL, Config.TrafficOps.Users.Admin, err)
+ os.Exit(1)
+ }
+
return err
}
diff --git a/traffic_ops/testing/api/v14/traffic_control.go b/traffic_ops/testing/api/v14/traffic_control.go
index 200bfdc..c27bb95 100644
--- a/traffic_ops/testing/api/v14/traffic_control.go
+++ b/traffic_ops/testing/api/v14/traffic_control.go
@@ -44,4 +44,10 @@ type TrafficControl struct {
Types []tc.Type `json:"types"`
SteeringTargets []tc.SteeringTargetNullable `json:"steeringTargets"`
Users []tc.User `json:"users"`
+ Jobs []JobRequest `json:"jobs"`
+}
+
+type JobRequest struct {
+ DSName string `json:"dsName"`
+ Request tc.JobRequest `json:"request"`
}
diff --git a/traffic_ops/traffic_ops_golang/ats/regexrevalidate.go b/traffic_ops/traffic_ops_golang/ats/regexrevalidate.go
new file mode 100644
index 0000000..5cfe985
--- /dev/null
+++ b/traffic_ops/traffic_ops_golang/ats/regexrevalidate.go
@@ -0,0 +1,249 @@
+package ats
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import (
+ "database/sql"
+ "errors"
+ "fmt"
+ "net/http"
+ "sort"
+ "strconv"
+ "strings"
+ "time"
+
+ "github.com/apache/trafficcontrol/lib/go-log"
+ "github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/api"
+ "github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/dbhelpers"
+)
+
+const DefaultMaxRevalDurationDays = 90
+const JobKeywordPurge = "PURGE"
+const HeaderCommentDateFormat = "Mon Jan 2 15:04:05 MST 2006"
+
+func GetRegexRevalidateDotConfig(w http.ResponseWriter, r *http.Request) {
+ inf, userErr, sysErr, errCode := api.NewInfo(r, []string{"cdn-name-or-id"}, nil)
+ if userErr != nil || sysErr != nil {
+ api.HandleErr(w, r, inf.Tx.Tx, errCode, userErr, sysErr)
+ return
+ }
+ defer inf.Close()
+
+ cdnName, userErr, sysErr, errCode := getCDNNameFromNameOrID(inf.Tx.Tx, inf.Params["cdn-name-or-id"])
+ if userErr != nil || sysErr != nil {
+ api.HandleErr(w, r, inf.Tx.Tx, errCode, userErr, sysErr)
+ return
+ }
+
+ regexRevalTxt, err := getRegexRevalidate(inf.Tx.Tx, cdnName)
+ if err != nil {
+ api.HandleErr(w, r, inf.Tx.Tx, http.StatusInternalServerError, nil, errors.New("getting regex_revalidate.config text: "+err.Error()))
+ return
+ }
+ w.Header().Set("Content-Type", "text/plain")
+ w.Write([]byte(regexRevalTxt))
+}
+
+// getCDNNameFromNameOrID returns the CDN name from a parameter which may be the name or ID.
+// This also checks and verifies the existence of the given CDN, and returns an appropriate user error if it doesn't exist.
+// Returns the name, any user error, any system error, and any error code.
+func getCDNNameFromNameOrID(tx *sql.Tx, cdnNameOrID string) (string, error, error, int) {
+ if cdnID, err := strconv.Atoi(cdnNameOrID); err == nil {
+ cdnName, ok, err := dbhelpers.GetCDNNameFromID(tx, int64(cdnID))
+ if err != nil {
+ return "", nil, fmt.Errorf("getting CDN name from id %v: %v", cdnID, err), http.StatusInternalServerError
+ } else if !ok {
+ return "", errors.New("cdn not found"), nil, http.StatusNotFound
+ }
+ return string(cdnName), nil, nil, http.StatusOK
+ }
+
+ cdnName := cdnNameOrID
+ if ok, err := dbhelpers.CDNExists(cdnName, tx); err != nil {
+ return "", nil, fmt.Errorf("checking CDN name '%v' existence: %v", cdnName, err), http.StatusInternalServerError
+ } else if !ok {
+ return "", errors.New("cdn not found"), nil, http.StatusNotFound
+ }
+ return cdnName, nil, nil, http.StatusOK
+}
+
+func getRegexRevalidate(tx *sql.Tx, cdnName string) (string, error) {
+ maxDays, ok, err := getMaxDays(tx)
+ if err != nil {
+ return "", errors.New("getting max reval duration days from Parameter: " + err.Error())
+ }
+ if !ok {
+ maxDays = DefaultMaxRevalDurationDays
+ log.Warnf("No maxRevalDurationDays regex_revalidate.config Parameter found, using default %v.\n", maxDays)
+ }
+ maxReval := time.Duration(maxDays) * time.Hour * 24
+ minTTL := time.Hour * 1
+
+ jobs, err := getJobs(tx, cdnName, maxReval, minTTL)
+ if err != nil {
+ return "", errors.New("getting jobs: " + err.Error())
+ }
+
+ text, err := headerComment(tx, "CDN "+cdnName)
+ if err != nil {
+ return "", errors.New("getting header comment: " + err.Error())
+ }
+ for _, job := range jobs {
+ text += job.AssetURL + " " + strconv.FormatInt(job.PurgeEnd.Unix(), 10) + "\n"
+ }
+
+ return text, nil
+}
+
+type Job struct {
+ AssetURL string
+ PurgeEnd time.Time
+}
+
+type Jobs []Job
+
+func (jb Jobs) Len() int { return len(jb) }
+func (jb Jobs) Swap(i, j int) { jb[i], jb[j] = jb[j], jb[i] }
+func (jb Jobs) Less(i, j int) bool {
+ if jb[i].AssetURL == jb[j].AssetURL {
+ return jb[i].PurgeEnd.Before(jb[j].PurgeEnd)
+ }
+ return strings.Compare(jb[i].AssetURL, jb[j].AssetURL) < 0
+}
+
+// getJobs returns jobs which
+// - have a non-null deliveryservice
+// - have parameters of the form TTL:%dh
+// - have a start time later than (now + maxReval days). That is, we don't query jobs older than maxReval in the past.
+// - are "purge" jobs
+// - have a start_time+ttl > now. That is, jobs that haven't expired yet.
+// The maxReval is used for both the max days, for which jobs older than that aren't selected, and for the maximum TTL.
+func getJobs(tx *sql.Tx, cdnName string, maxReval time.Duration, minTTL time.Duration) ([]Job, error) {
+ qry := `
+WITH
+ cdn_name AS (select $1::text as v),
+ max_days AS (select $2::integer as v)
+SELECT
+ j.asset_url,
+ CAST((SELECT REGEXP_MATCHES(j.parameters, 'TTL:(\d+)h') FETCH FIRST 1 ROWS ONLY)[1] AS INTEGER) as ttl,
+ j.start_time
+FROM
+ job j
+ JOIN deliveryservice ds ON j.job_deliveryservice = ds.id
+WHERE
+ j.parameters ~ 'TTL:(\d+)h'
+ AND j.start_time > (NOW() - ((select v from max_days) * INTERVAL '1 day'))
+ AND ds.cdn_id = (select id from cdn where name = (select v from cdn_name))
+ AND j.job_deliveryservice IS NOT NULL
+ AND j.keyword = '` + JobKeywordPurge + `'
+ AND (j.start_time + (CAST( (SELECT REGEXP_MATCHES(j.parameters, 'TTL:(\d+)h') FETCH FIRST 1 ROWS ONLY)[1] AS INTEGER) * INTERVAL '1 HOUR')) > NOW()
+`
+ maxRevalDays := maxReval / time.Hour / 24
+ rows, err := tx.Query(qry, cdnName, maxRevalDays)
+ if err != nil {
+ return nil, errors.New("querying: " + err.Error())
+ }
+ defer rows.Close()
+
+ jobMap := map[string]time.Time{}
+ for rows.Next() {
+ assetURL := ""
+ ttlHours := 0
+ startTime := time.Time{}
+ if err := rows.Scan(&assetURL, &ttlHours, &startTime); err != nil {
+ return nil, errors.New("scanning: " + err.Error())
+ }
+
+ ttl := time.Duration(ttlHours) * time.Hour
+ if ttl > maxReval {
+ ttl = maxReval
+ } else if ttl < minTTL {
+ ttl = minTTL
+ }
+
+ purgeEnd := startTime.Add(ttl)
+
+ if existingPurgeEnd, ok := jobMap[assetURL]; !ok || purgeEnd.After(existingPurgeEnd) {
+ jobMap[assetURL] = purgeEnd
+ }
+ }
+
+ jobs := []Job{}
+ for assetURL, purgeEnd := range jobMap {
+ jobs = append(jobs, Job{AssetURL: assetURL, PurgeEnd: purgeEnd})
+ }
+ sort.Sort(Jobs(jobs))
+ return jobs, nil
+}
+
+func getMaxDays(tx *sql.Tx) (int64, bool, error) {
+ daysStr := ""
+ if err := tx.QueryRow(`SELECT p.value FROM parameter p WHERE p.name = 'maxRevalDurationDays' AND p.config_file = 'regex_revalidate.config'`).Scan(&daysStr); err != nil {
+ if err == sql.ErrNoRows {
+ return 0, false, nil
+ }
+ return 0, false, errors.New("querying max reval duration days: " + err.Error())
+ }
+ days, err := strconv.ParseInt(daysStr, 10, 64)
+ if err != nil {
+ return 0, false, errors.New("querying max reval duration days: value '" + daysStr + "' is not an integer")
+ }
+ return days, true, nil
+}
+
+func headerComment(tx *sql.Tx, name string) (string, error) {
+ nameVersionStr, err := GetNameVersionString(tx)
+ if err != nil {
+ return "", errors.New("getting name version string: " + err.Error())
+ }
+ return "# DO NOT EDIT - Generated for " + name + " by " + nameVersionStr + " on " + time.Now().Format(HeaderCommentDateFormat) + "\n", nil
+}
+
+func GetNameVersionString(tx *sql.Tx) (string, error) {
+ qry := `
+SELECT
+ p.name,
+ p.value
+FROM
+ parameter p
+WHERE
+ (p.name = 'tm.toolname' OR p.name = 'tm.url') AND p.config_file = 'global'
+`
+ rows, err := tx.Query(qry)
+ if err != nil {
+ return "", errors.New("querying: " + err.Error())
+ }
+ defer rows.Close()
+ toolName := ""
+ url := ""
+ for rows.Next() {
+ name := ""
+ val := ""
+ if err := rows.Scan(&name, &val); err != nil {
+ return "", errors.New("scanning: " + err.Error())
+ }
+ if name == "tm.toolname" {
+ toolName = val
+ } else if name == "tm.url" {
+ url = val
+ }
+ }
+ return toolName + " (" + url + ")", nil
+}
diff --git a/traffic_ops/traffic_ops_golang/cachegroup/queueupdate.go b/traffic_ops/traffic_ops_golang/cachegroup/queueupdate.go
index b489e49..5b8e398 100644
--- a/traffic_ops/traffic_ops_golang/cachegroup/queueupdate.go
+++ b/traffic_ops/traffic_ops_golang/cachegroup/queueupdate.go
@@ -28,6 +28,7 @@ import (
"github.com/apache/trafficcontrol/lib/go-tc"
"github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/api"
+ "github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/dbhelpers"
)
func QueueUpdates(w http.ResponseWriter, r *http.Request) {
@@ -52,7 +53,7 @@ func QueueUpdates(w http.ResponseWriter, r *http.Request) {
return
}
if reqObj.CDN == nil || *reqObj.CDN == "" {
- cdn, ok, err := getCDNNameFromID(inf.Tx.Tx, int64(*reqObj.CDNID))
+ cdn, ok, err := dbhelpers.GetCDNNameFromID(inf.Tx.Tx, int64(*reqObj.CDNID))
if err != nil {
api.HandleErr(w, r, inf.Tx.Tx, http.StatusInternalServerError, nil, errors.New("getting CDN name from ID '"+strconv.Itoa(int(*reqObj.CDNID))+"': "+err.Error()))
return
@@ -98,17 +99,6 @@ type QueueUpdatesResp struct {
CacheGroupID int64 `json:"cachegroupID"`
}
-func getCDNNameFromID(tx *sql.Tx, id int64) (tc.CDNName, bool, error) {
- name := ""
- if err := tx.QueryRow(`SELECT name FROM cdn WHERE id = $1`, id).Scan(&name); err != nil {
- if err == sql.ErrNoRows {
- return "", false, nil
- }
- return "", false, errors.New("querying CDN ID: " + err.Error())
- }
- return tc.CDNName(name), true, nil
-}
-
func getCGNameFromID(tx *sql.Tx, id int64) (tc.CacheGroupName, bool, error) {
name := ""
if err := tx.QueryRow(`SELECT name FROM cachegroup WHERE id = $1`, id).Scan(&name); err != nil {
diff --git a/traffic_ops/traffic_ops_golang/cdnfederation/cdnfederations.go b/traffic_ops/traffic_ops_golang/cdnfederation/cdnfederations.go
index 19ff6a8..5ca1c82 100644
--- a/traffic_ops/traffic_ops_golang/cdnfederation/cdnfederations.go
+++ b/traffic_ops/traffic_ops_golang/cdnfederation/cdnfederations.go
@@ -183,7 +183,7 @@ func (fed *TOCDNFederation) Read() ([]interface{}, error, error, int) {
if fed.ID != nil {
return nil, errors.New("not found"), nil, http.StatusNotFound
}
- if ok, err := dbhelpers.CDNExists(fed.APIInfo().Params["name"], fed.APIInfo().Tx); err != nil {
+ if ok, err := dbhelpers.CDNExists(fed.APIInfo().Params["name"], fed.APIInfo().Tx.Tx); err != nil {
return nil, nil, errors.New("verifying CDN exists: " + err.Error()), http.StatusInternalServerError
} else if !ok {
return nil, errors.New("cdn not found"), nil, http.StatusNotFound
diff --git a/traffic_ops/traffic_ops_golang/dbhelpers/db_helpers.go b/traffic_ops/traffic_ops_golang/dbhelpers/db_helpers.go
index 7bc31ed..ae2f325 100644
--- a/traffic_ops/traffic_ops_golang/dbhelpers/db_helpers.go
+++ b/traffic_ops/traffic_ops_golang/dbhelpers/db_helpers.go
@@ -28,7 +28,6 @@ import (
"github.com/apache/trafficcontrol/lib/go-log"
"github.com/apache/trafficcontrol/lib/go-tc"
- "github.com/jmoiron/sqlx"
"github.com/lib/pq"
)
@@ -154,7 +153,7 @@ func GetProfileIDFromName(name string, tx *sql.Tx) (int, bool, error) {
}
// Returns true if the cdn exists
-func CDNExists(cdnName string, tx *sqlx.Tx) (bool, error) {
+func CDNExists(cdnName string, tx *sql.Tx) (bool, error) {
var id int
if err := tx.QueryRow(`SELECT id FROM cdn WHERE name = $1`, cdnName).Scan(&id); err != nil {
if err == sql.ErrNoRows {
@@ -164,3 +163,14 @@ func CDNExists(cdnName string, tx *sqlx.Tx) (bool, error) {
}
return true, nil
}
+
+func GetCDNNameFromID(tx *sql.Tx, id int64) (tc.CDNName, bool, error) {
+ name := ""
+ if err := tx.QueryRow(`SELECT name FROM cdn WHERE id = $1`, id).Scan(&name); err != nil {
+ if err == sql.ErrNoRows {
+ return "", false, nil
+ }
+ return "", false, errors.New("querying CDN ID: " + err.Error())
+ }
+ return tc.CDNName(name), true, nil
+}
diff --git a/traffic_ops/traffic_ops_golang/routes.go b/traffic_ops/traffic_ops_golang/routes.go
index a52fdba..1930889 100644
--- a/traffic_ops/traffic_ops_golang/routes.go
+++ b/traffic_ops/traffic_ops_golang/routes.go
@@ -38,6 +38,7 @@ import (
"github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/apiriak"
"github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/apitenant"
"github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/asn"
+ "github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/ats"
"github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/auth"
"github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/cachegroup"
"github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/cdn"
@@ -378,6 +379,8 @@ func Routes(d ServerData) ([]Route, []RawRoute, http.Handler, error) {
{1.1, http.MethodPut, `cdns/{id}/snapshot/?$`, crconfig.SnapshotHandler, auth.PrivLevelOperations, Authenticated, nil},
{1.1, http.MethodPut, `snapshot/{cdn}/?$`, crconfig.SnapshotHandler, auth.PrivLevelOperations, Authenticated, nil},
+ {1.1, http.MethodGet, `cdns/{cdn-name-or-id}/configfiles/ats/regex_revalidate.config/?(\.json)?$`, ats.GetRegexRevalidateDotConfig, auth.PrivLevelOperations, Authenticated, nil},
+
// Federations
{1.4, http.MethodGet, `federations/all/?(\.json)?$`, federations.GetAll, auth.PrivLevelAdmin, Authenticated, nil},
{1.1, http.MethodGet, `federations/?(\.json)?$`, federations.Get, auth.PrivLevelFederation, Authenticated, nil},