You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@trafficcontrol.apache.org by ra...@apache.org on 2021/11/18 19:47:10 UTC

[trafficcontrol] branch master updated: Make t3c jobs work with previous TO 4.0 (#6355)

This is an automated email from the ASF dual-hosted git repository.

rawlin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/trafficcontrol.git


The following commit(s) were added to refs/heads/master by this push:
     new 0ed2846  Make t3c jobs work with previous TO 4.0 (#6355)
0ed2846 is described below

commit 0ed284603bf7e5d6e5295ee74e7080e4f431d498
Author: Robert O Butts <ro...@users.noreply.github.com>
AuthorDate: Thu Nov 18 12:46:48 2021 -0700

    Make t3c jobs work with previous TO 4.0 (#6355)
    
    * Make t3c jobs work with previous TO 4.0
    
    * Add t3c comments per PR Review.
---
 cache-config/t3c-request/config/config.go       |   6 +-
 cache-config/t3cutil/toreq/clientfuncs.go       |   4 +-
 cache-config/t3cutil/toreq/conversions.go       | 105 ++++++++++++++++++++++++
 cache-config/testing/ort-tests/t3c-jobs_test.go |  96 ++++++++++++++++++++++
 lib/go-atscfg/atscfg.go                         |  16 +++-
 5 files changed, 223 insertions(+), 4 deletions(-)

diff --git a/cache-config/t3c-request/config/config.go b/cache-config/t3c-request/config/config.go
index 68da602..6a1d9e8 100644
--- a/cache-config/t3c-request/config/config.go
+++ b/cache-config/t3c-request/config/config.go
@@ -170,9 +170,11 @@ func InitConfig() (Cfg, error) {
 	// load old config after initializing the loggers, because we want to log how long it takes
 	oldCfg, err := LoadOldCfg(*oldCfgPtr)
 	if err != nil {
-		return Cfg{}, errors.New("loading old config: " + err.Error())
+		log.Warnf("loading old config failed, old config will not be used! Error: %v\n", err)
+	} else {
+		log.Infof("using old config for IMS requests")
+		cfg.OldCfg = oldCfg
 	}
-	cfg.OldCfg = oldCfg
 
 	return cfg, nil
 }
diff --git a/cache-config/t3cutil/toreq/clientfuncs.go b/cache-config/t3cutil/toreq/clientfuncs.go
index 7785e48..cec1cbe 100644
--- a/cache-config/t3cutil/toreq/clientfuncs.go
+++ b/cache-config/t3cutil/toreq/clientfuncs.go
@@ -531,7 +531,9 @@ func (cl *TOClient) GetJobs(reqHdr http.Header, cdnName string) ([]atscfg.Invali
 		opts := *ReqOpts(reqHdr)
 		opts.QueryParameters.Set("maxRevalDurationDays", "") // only get jobs with a start time within the window defined by the GLOBAL parameter 'maxRevalDurationDays'
 		opts.QueryParameters.Set("cdn", cdnName)             // only get jobs for delivery services in this server's CDN
-		toJobs, toReqInf, err := cl.c.GetInvalidationJobs(opts)
+		// GetJobsCompat can be changed back to 'cl.c.GetInvalidationJobs' when backwards compatibility
+		// with Traffic Ops from previous ATS 'master' changesets is no longer desired, presumably after the next major ATC release.
+		toJobs, toReqInf, err := cl.GetJobsCompat(opts)
 		if err != nil {
 			return errors.New("getting jobs from Traffic Ops '" + torequtil.MaybeIPStr(reqInf.RemoteAddr) + "': " + err.Error())
 		}
diff --git a/cache-config/t3cutil/toreq/conversions.go b/cache-config/t3cutil/toreq/conversions.go
index 3cd3687..94d870c 100644
--- a/cache-config/t3cutil/toreq/conversions.go
+++ b/cache-config/t3cutil/toreq/conversions.go
@@ -20,8 +20,16 @@ package toreq
  */
 
 import (
+	"errors"
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+
 	"github.com/apache/trafficcontrol/lib/go-atscfg"
 	"github.com/apache/trafficcontrol/lib/go-tc"
+	"github.com/apache/trafficcontrol/traffic_ops/toclientlib"
+	toclient "github.com/apache/trafficcontrol/traffic_ops/v4-client"
 )
 
 func serversToLatest(svs tc.ServersV4Response) ([]atscfg.Server, error) {
@@ -40,3 +48,100 @@ func dsesToLatest(dses []tc.DeliveryServiceV40) []atscfg.DeliveryService {
 func jobsToLatest(jobs []tc.InvalidationJobV4) []atscfg.InvalidationJob {
 	return atscfg.ToInvalidationJobs(jobs)
 }
+
+// GetJobsCompat gets jobs from any Traffic Ops built from the ATC `master` branch, and converts the different formats to the latest.
+// This makes t3c work with old or new Traffic Ops deployed from `master`,
+// though it doesn't make a version of t3c older than this work with a new TO,
+// which isn't logically possible from the client.
+func (cl *TOClient) GetJobsCompat(opts toclient.RequestOptions) (tc.InvalidationJobsResponseV4, toclientlib.ReqInf, error) {
+	path := "/jobs"
+
+	objs := struct {
+		Response []InvalidationJobV4PlusLegacy `json:"response"`
+		tc.Alerts
+	}{}
+
+	if len(opts.QueryParameters) > 0 {
+		path += "?" + opts.QueryParameters.Encode()
+	}
+	reqInf, err := cl.c.TOClient.Req(http.MethodGet, path, nil, opts.Header, &objs)
+	if err != nil {
+		return tc.InvalidationJobsResponseV4{}, reqInf, errors.New("request: " + err.Error())
+	}
+
+	resp := tc.InvalidationJobsResponseV4{Alerts: objs.Alerts}
+	for _, job := range objs.Response {
+		newJob, err := InvalidationJobV4FromLegacy(job) // (InvalidationJobV4, error) {
+		if err != nil {
+			return tc.InvalidationJobsResponseV4{}, reqInf, errors.New("converting job from possible legacy format: " + err.Error())
+		}
+		resp.Response = append(resp.Response, newJob)
+	}
+	return resp, reqInf, nil
+}
+
+// InvalidationJobV4ForLegacy is a type alias to prevent MarshalJSON recursion.
+type InvalidationJobV4ForLegacy tc.InvalidationJobV4
+
+// InvalidationJobV4PlusLegacy has the data to deserialize both the latest and older versions that Traffic Ops could return.
+type InvalidationJobV4PlusLegacy struct {
+	// StartTime overrides the StartTime in InvalidationJobV4 in order to unmarshal any string format.
+	//
+	// A json.Unmarshal will place a 'startTime' value in this field,
+	// rather than the anonymous embedded InvalidationJobV4ForLegacy (tc.InvalidationJobV4).
+	//
+	// InvalidationJobV4FromLegacy will then parse multiple time formats that different Traffic Ops servers may return,
+	// and put the parsed time in tc.InvalidationJobV4.StartTime.
+	StartTime *string `json:"startTime"`
+	InvalidationJobV4ForLegacy
+	InvalidationJobV4Legacy
+}
+
+type InvalidationJobV4Legacy struct {
+	Keyword    *string `json:"keyword"`
+	Parameters *string `json:"parameters"`
+}
+
+func InvalidationJobV4FromLegacy(job InvalidationJobV4PlusLegacy) (tc.InvalidationJobV4, error) {
+	if job.StartTime != nil {
+		err := error(nil)
+		job.InvalidationJobV4ForLegacy.StartTime, err = time.Parse(atscfg.JobV4TimeFormat, *job.StartTime)
+		if err != nil {
+			job.InvalidationJobV4ForLegacy.StartTime, err = time.Parse(atscfg.JobLegacyTimeFormat, *job.StartTime)
+			if err != nil {
+				return tc.InvalidationJobV4{}, errors.New("malformed startTime")
+			}
+		}
+	}
+
+	if job.TTLHours == 0 && job.Parameters != nil {
+		params := *job.Parameters
+		params = strings.TrimSpace(params)
+		params = strings.ToLower(params)
+		params = strings.Replace(params, " ", "", -1)
+
+		paramPrefix := strings.ToLower(atscfg.JobLegacyParamPrefix)
+		paramSuffix := strings.ToLower(atscfg.JobLegacyParamSuffix)
+		if !strings.HasPrefix(params, paramPrefix) || !strings.HasSuffix(params, paramSuffix) {
+			return tc.InvalidationJobV4{}, errors.New("legacy job.Parameters was not nil, but unexpected format '" + params + "'")
+		}
+
+		hoursStr := params[len(paramPrefix) : len(params)-len(paramSuffix)]
+		hours, err := strconv.Atoi(hoursStr)
+		if err != nil {
+			return tc.InvalidationJobV4{}, errors.New("legacy job.Parameters was not nil, but hours not an integer: '" + params + "'")
+		}
+		job.TTLHours = uint(hours)
+	}
+
+	if job.InvalidationType == "" && job.Parameters != nil {
+		job.InvalidationType = tc.REFRESH
+	}
+	if strings.HasSuffix(job.AssetURL, atscfg.JobLegacyRefetchSuffix) {
+		job.InvalidationType = tc.REFETCH
+	}
+	job.AssetURL = strings.TrimSuffix(job.AssetURL, atscfg.JobLegacyRefreshSuffix)
+	job.AssetURL = strings.TrimSuffix(job.AssetURL, atscfg.JobLegacyRefetchSuffix)
+
+	return tc.InvalidationJobV4(job.InvalidationJobV4ForLegacy), nil
+}
diff --git a/cache-config/testing/ort-tests/t3c-jobs_test.go b/cache-config/testing/ort-tests/t3c-jobs_test.go
new file mode 100644
index 0000000..9ace24a
--- /dev/null
+++ b/cache-config/testing/ort-tests/t3c-jobs_test.go
@@ -0,0 +1,96 @@
+package orttest
+
+/*
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+import (
+	"io/ioutil"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/apache/trafficcontrol/cache-config/testing/ort-tests/tcdata"
+	"github.com/apache/trafficcontrol/lib/go-tc"
+	"github.com/apache/trafficcontrol/lib/go-util"
+)
+
+func TestT3CJobs(t *testing.T) {
+	t.Logf("------------- Starting TestT3CJobs ---------------")
+	tcd.WithObjs(t, []tcdata.TCObj{
+		tcdata.CDNs, tcdata.Types, tcdata.Tenants, tcdata.Parameters,
+		tcdata.Profiles, tcdata.ProfileParameters, tcdata.Statuses,
+		tcdata.Divisions, tcdata.Regions, tcdata.PhysLocations,
+		tcdata.CacheGroups, tcdata.Servers, tcdata.Topologies,
+		tcdata.DeliveryServices}, func() {
+
+		now := tc.Time{Time: time.Now().Add(time.Minute)}
+		dsi := (interface{})("ds1")
+		ttli := (interface{})(72.0)
+		job := tc.InvalidationJobInput{
+			DeliveryService: &dsi,
+			Regex:           util.StrPtr(`/refetch-test\.png##REFETCH##`),
+			StartTime:       &now,
+			TTL:             &ttli,
+		}
+		if _, _, err := tcdata.TOSession.CreateInvalidationJob(job); err != nil {
+			t.Fatalf("ERROR: create refetch job failed: %v\n", err)
+		}
+		job.Regex = util.StrPtr(`/refresh-test\.png`)
+		if _, _, err := tcdata.TOSession.CreateInvalidationJob(job); err != nil {
+			t.Fatalf("ERROR: create refresh job failed: %v\n", err)
+		}
+
+		out, err := t3cUpdateWaitForParents("atlanta-edge-03", "badass", util.StrPtr("false"))
+		if err != nil {
+			t.Fatalf("ERROR: t3c badass failed: %v\n", err)
+		}
+		t.Log("TestT3CJobs t3c output: '''" + out + "'''")
+
+		fileName := filepath.Join(test_config_dir, "regex_revalidate.config")
+		revalFileBts, err := ioutil.ReadFile(fileName)
+		if err != nil {
+			t.Fatalf("reading %v: %v\n", fileName, err)
+		}
+		revalFile := string(revalFileBts)
+		lines := strings.Split(revalFile, "\n")
+		sawRefresh := false
+		sawRefetch := false
+		for _, line := range lines {
+			line = strings.TrimSpace(line)
+			if line == "" {
+				continue // blank lines are ok
+			}
+			if strings.Contains(line, "refresh-test") {
+				sawRefresh = true
+				if !strings.HasSuffix(line, "STALE") {
+					t.Errorf("expected regex_revalidate.config refresh-test line to contain 'STALE', actual '''%v'''", revalFile)
+				}
+			}
+			if strings.Contains(line, "refetch-test") {
+				sawRefetch = true
+				if !strings.HasSuffix(line, "MISS") {
+					t.Errorf("expected regex_revalidate.config refetch-test line to contain 'MISS', actual '''%v'''", revalFile)
+				}
+			}
+		}
+		if !sawRefresh {
+			t.Errorf("expected regex_revalidate.config to contain refresh-test, actual '''%v'''", revalFile)
+		}
+		if !sawRefetch {
+			t.Errorf("expected regex_revalidate.config to contain refetch-test, actual '''%v'''", revalFile)
+		}
+	})
+	t.Logf("------------- End of TestT3CJobs ---------------")
+}
diff --git a/lib/go-atscfg/atscfg.go b/lib/go-atscfg/atscfg.go
index ca3dc3b..c055d32 100644
--- a/lib/go-atscfg/atscfg.go
+++ b/lib/go-atscfg/atscfg.go
@@ -27,6 +27,7 @@ import (
 	"sort"
 	"strconv"
 	"strings"
+	"time"
 
 	"github.com/apache/trafficcontrol/lib/go-tc"
 )
@@ -695,6 +696,14 @@ func JobsToInvalidationJobs(oldJobs []tc.Job) ([]InvalidationJob, error) {
 	return jobs, nil
 }
 
+const JobV4TimeFormat = time.RFC3339Nano
+const JobLegacyTimeFormat = "2006-01-02 15:04:05-07"
+const JobLegacyRefetchSuffix = `##REFETCH##`
+const JobLegacyRefreshSuffix = `##REFRESH##`
+const JobLegacyParamPrefix = "TTL:"
+const JobLegacyParamSuffix = "h"
+const JobLegacyKeyword = "PURGE"
+
 func JobToInvalidationJob(jb tc.Job) (InvalidationJob, error) {
 	startTime := tc.Time{}
 	if err := json.Unmarshal([]byte(`"`+jb.StartTime+`"`), &startTime); err != nil {
@@ -704,13 +713,18 @@ func JobToInvalidationJob(jb tc.Job) (InvalidationJob, error) {
 	if err != nil {
 		return InvalidationJob{}, errors.New("unmarshalling ttl: " + err.Error())
 	}
+	invalType := tc.REFRESH
+	if strings.HasSuffix(jb.AssetURL, JobLegacyRefetchSuffix) {
+		invalType = tc.REFETCH
+	}
+
 	return InvalidationJob{
 		AssetURL:         jb.AssetURL,
 		CreatedBy:        jb.CreatedBy,
 		DeliveryService:  jb.DeliveryService,
 		ID:               uint64(jb.ID),
 		TTLHours:         uint(ttl),
-		InvalidationType: tc.REFRESH,
+		InvalidationType: invalType,
 		StartTime:        startTime.Time,
 	}, nil
 }