You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@trafficcontrol.apache.org by ro...@apache.org on 2018/05/23 16:34:06 UTC
[incubator-trafficcontrol] branch master updated (c17fc80 ->
cb54909)
This is an automated email from the ASF dual-hosted git repository.
rob pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-trafficcontrol.git.
from c17fc80 fix insecure field in example ldap.conf
new 784198d Add HitCount to cache object and cleanup of HTTP table in cacheinsepctor
new e829d14 add FreshFor
new 4449a94 Add rfc package, move stuff around to accomodate
new cb54909 Clean up
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
grove/cache/handler.go | 3 +-
grove/cache/retryinggetter.go | 6 +-
grove/cacheobj/cacheobj.go | 2 +
grove/diskcache/diskcache.go | 4 +-
grove/memcache/memcache.go | 1 +
grove/plugin/README_http_cacheinspector.md | 23 ++-
grove/plugin/http_cacheinspector.go | 11 +-
grove/remap/remap.go | 7 +-
grove/{remap => rfc}/rules.go | 228 ++++++++++++++++-------------
grove/{remap => rfc}/rules_test.go | 6 +-
10 files changed, 160 insertions(+), 131 deletions(-)
rename grove/{remap => rfc}/rules.go (95%)
rename grove/{remap => rfc}/rules_test.go (99%)
--
To stop receiving notification emails like this one, please contact
rob@apache.org.
[incubator-trafficcontrol] 03/04: Add rfc package,
move stuff around to accomodate
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rob pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-trafficcontrol.git
commit 4449a94fa84609a6d0949f3ffb6652686cd121c3
Author: Jan van Doorn <ja...@comcast.com>
AuthorDate: Wed May 16 08:58:06 2018 -0600
Add rfc package, move stuff around to accomodate
---
grove/cache/handler.go | 3 +-
grove/cache/retryinggetter.go | 5 +-
grove/integration_test/compare_gets.go | 140 +++++++++++++++++++++++++++
grove/plugin/http_cacheinspector.go | 3 +-
grove/remap/remap.go | 7 +-
grove/{remap => rfc}/rules.go | 166 +++++++++++++++++++++++++++++++--
grove/{remap => rfc}/rules_test.go | 6 +-
grove/web/util.go | 145 +---------------------------
8 files changed, 314 insertions(+), 161 deletions(-)
diff --git a/grove/cache/handler.go b/grove/cache/handler.go
index a7a7f4a..216378c 100644
--- a/grove/cache/handler.go
+++ b/grove/cache/handler.go
@@ -27,6 +27,7 @@ import (
"github.com/apache/incubator-trafficcontrol/grove/remap"
"github.com/apache/incubator-trafficcontrol/grove/remapdata"
+ "github.com/apache/incubator-trafficcontrol/grove/rfc"
"github.com/apache/incubator-trafficcontrol/grove/stat"
"github.com/apache/incubator-trafficcontrol/grove/thread"
"github.com/apache/incubator-trafficcontrol/grove/web"
@@ -261,7 +262,7 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
}
reqHeaders := r.Header
- canReuseStored := remap.CanReuseStored(reqHeaders, cacheObj.RespHeaders, reqCacheControl, cacheObj.RespCacheControl, cacheObj.ReqHeaders, cacheObj.ReqRespTime, cacheObj.RespRespTime, h.strictRFC)
+ canReuseStored := rfc.CanReuseStored(reqHeaders, cacheObj.RespHeaders, reqCacheControl, cacheObj.RespCacheControl, cacheObj.ReqHeaders, cacheObj.ReqRespTime, cacheObj.RespRespTime, h.strictRFC)
if canReuseStored != remapdata.ReuseCan { // run the BeforeParentRequest hook for revalidations / ReuseCannot
beforeParentRequestData := plugin.BeforeParentRequestData{Req: r, RemapRule: remappingProducer.Name()}
diff --git a/grove/cache/retryinggetter.go b/grove/cache/retryinggetter.go
index ee60017..b70a6d4 100644
--- a/grove/cache/retryinggetter.go
+++ b/grove/cache/retryinggetter.go
@@ -23,6 +23,7 @@ import (
"github.com/apache/incubator-trafficcontrol/grove/cacheobj"
"github.com/apache/incubator-trafficcontrol/grove/icache"
"github.com/apache/incubator-trafficcontrol/grove/remap"
+ "github.com/apache/incubator-trafficcontrol/grove/rfc"
"github.com/apache/incubator-trafficcontrol/grove/thread"
"github.com/apache/incubator-trafficcontrol/grove/web"
@@ -56,7 +57,7 @@ func (r *Retrier) Get(req *http.Request, obj *cacheobj.CacheObj) (*cacheobj.Cach
retryGetFunc := func(remapping remap.Remapping, retryFailures bool, obj *cacheobj.CacheObj) *cacheobj.CacheObj {
// return true for Revalidate, and issue revalidate requests separately.
canReuse := func(cacheObj *cacheobj.CacheObj) bool {
- return remap.CanReuse(r.ReqHdr, r.ReqCacheControl, cacheObj, r.H.strictRFC, true)
+ return rfc.CanReuse(r.ReqHdr, r.ReqCacheControl, cacheObj, r.H.strictRFC, true)
}
getAndCache := func() *cacheobj.CacheObj {
return GetAndCache(remapping.Request, remapping.ProxyURL, remapping.CacheKey, remapping.Name, remapping.Request.Header, r.ReqTime, r.H.strictRFC, remapping.Cache, r.H.ruleThrottlers[remapping.Name], obj, remapping.Timeout, retryFailures, remapping.RetryNum, remapping.RetryCodes, remapping.Transport, r.ReqID)
@@ -166,7 +167,7 @@ func GetAndCache(
if revalidateObj == nil || respCode != http.StatusNotModified {
log.Debugf("GetAndCache new %v (reqid %v)\n", cacheKey, reqID)
obj = cacheobj.New(reqHeader, respBody, respCode, respCode, proxyURLStr, respHeader, reqTime, reqRespTime, respRespTime, lastModified)
- if !remap.CanCache(req.Method, reqHeader, respCode, respHeader, strictRFC) {
+ if !rfc.CanCache(req.Method, reqHeader, respCode, respHeader, strictRFC) {
return obj // return without caching
}
} else {
diff --git a/grove/integration_test/compare_gets.go b/grove/integration_test/compare_gets.go
new file mode 100644
index 0000000..08d3aca
--- /dev/null
+++ b/grove/integration_test/compare_gets.go
@@ -0,0 +1,140 @@
+package main
+
+import (
+ "flag"
+ "fmt"
+ "github.com/apache/incubator-trafficcontrol/grove/web"
+ "io/ioutil"
+ "log"
+ "net/http"
+ "os"
+ "strings"
+)
+
+type responseType struct {
+ Headers http.Header
+ Body []byte
+}
+
+func httpGet(URL, headers string) responseType {
+ client := &http.Client{}
+ req, err := http.NewRequest("GET", URL, nil)
+ if err != nil {
+ fmt.Println("ERROR in httpGet")
+ }
+ //log.Printf(">>>%v<<< %v\n", headers, len(strings.Split(headers, ".")))
+ for _, hdrString := range strings.Split(headers, ",") {
+ //log.Println(">>> ", hdrString)
+ if hdrString == "" {
+ continue
+ }
+ parts := strings.Split(hdrString, ":")
+ if parts[0] == "Host" {
+ req.Host = parts[1]
+ } else {
+ //log.Println("> ", parts)
+ req.Header.Set(parts[0], parts[1])
+ }
+ }
+ //log.Printf(">>>> %v", req)
+ resp, err := client.Do(req)
+ if err != nil {
+ fmt.Println("ERROR in httpGet")
+ }
+ defer resp.Body.Close()
+ var response responseType
+ response.Headers = web.CopyHeader(resp.Header)
+ response.Body, err = ioutil.ReadAll(resp.Body)
+ if err != nil {
+ fmt.Println("ERROR in httpGet (readall)")
+ }
+ return response
+}
+
+func equal(a, b []byte) bool {
+ if a == nil || b == nil {
+ return false
+ }
+
+ if a == nil && b == nil {
+ return true
+ }
+ if len(a) != len(b) {
+ return false
+ }
+
+ for i := range a {
+ if a[i] != b[i] {
+ return false
+ }
+ }
+
+ return true
+}
+
+func equalStringSlices(a, b []string) bool {
+ if a == nil || b == nil {
+ return false
+ }
+
+ if a == nil && b == nil {
+ return true
+ }
+ if len(a) != len(b) {
+ return false
+ }
+
+ for i := range a {
+ if a[i] != b[i] {
+ return false
+ }
+ }
+
+ return true
+}
+
+func inStringSlice(str string, arr []string) bool {
+ for _, strEnt := range arr {
+ if strEnt == str {
+ return true
+ }
+ }
+ return false
+}
+
+func compareResponses(response1 responseType, response2 responseType, ignoreHdrs []string) bool {
+ if !equal(response1.Body, response2.Body) {
+ return false
+ }
+ for hdrKey, _ := range response1.Headers {
+ if inStringSlice(hdrKey, ignoreHdrs) {
+ continue
+ }
+ if !equalStringSlices(response1.Headers[hdrKey], response2.Headers[hdrKey]) {
+ log.Printf("ERROR hdr %v doesn't match: \"%v\" != \"%v\"\n", hdrKey, response1.Headers[hdrKey], response2.Headers[hdrKey])
+ return false
+ }
+ //fmt.Printf(">>>>> %v\n", hdrKey)
+ }
+
+ return true
+}
+func main() {
+ originURL := flag.String("org", "http://localhost", "The origin URL (default: \"http://localhost\")")
+ cacheURL := flag.String("cache", "http://localhost:8080", "The cache URL (default: \"http://localhost:8080\")")
+ path := flag.String("path", "", "The path to GET")
+ orgHdrs := flag.String("ohdrs", "", "Comma seperated list of headers to add to origin request")
+ cacheHdrs := flag.String("chdrs", "", "Comma separated list of headers to add to cache request")
+ ignoreHdrs := flag.String("ignorehdrs", "Server,Date", "Comma separated list of headers to ignore in the compare")
+ flag.Parse()
+
+ resp := httpGet(*originURL+"/"+*path, *orgHdrs)
+ cresp := httpGet(*cacheURL+"/"+*path, *cacheHdrs)
+ if !compareResponses(resp, cresp, strings.Split(*ignoreHdrs, ",")) {
+ fmt.Println("FAIL: Body bytes don't match")
+ os.Exit(1)
+
+ }
+ fmt.Println("PASS")
+ os.Exit(0)
+}
diff --git a/grove/plugin/http_cacheinspector.go b/grove/plugin/http_cacheinspector.go
index 2cdb158..9409aee 100644
--- a/grove/plugin/http_cacheinspector.go
+++ b/grove/plugin/http_cacheinspector.go
@@ -26,6 +26,7 @@ import (
"github.com/apache/incubator-trafficcontrol/grove/web"
"time"
+ "github.com/apache/incubator-trafficcontrol/grove/rfc"
"github.com/apache/incubator-trafficcontrol/lib/go-log"
)
@@ -178,7 +179,7 @@ func cacheinspect(icfg interface{}, d OnRequestData) bool {
cacheObject, _ := d.Stats.CachePeek(key, cName)
age := time.Now().Sub(cacheObject.ReqRespTime)
- freshFor := web.FreshFor(cacheObject.RespHeaders, cacheObject.RespCacheControl, cacheObject.ReqRespTime, cacheObject.RespRespTime)
+ freshFor := rfc.FreshFor(cacheObject.RespHeaders, cacheObject.RespCacheControl, cacheObject.ReqRespTime, cacheObject.RespRespTime)
w.Write([]byte(fmt.Sprintf(" %8d%8d%10s%22v%22v%12d <a href=\"http://%s%s?key=%s&cache=%s\">%s</a>\n",
i, cacheObject.Code, bytefmt.ByteSize(cacheObject.Size), age, freshFor, cacheObject.HitCount, req.Host, CacheStatsEndpoint, url.QueryEscape(key), cName, key)))
}
diff --git a/grove/remap/remap.go b/grove/remap/remap.go
index 2af529c..9793566 100644
--- a/grove/remap/remap.go
+++ b/grove/remap/remap.go
@@ -29,6 +29,7 @@ import (
"github.com/apache/incubator-trafficcontrol/grove/icache"
"github.com/apache/incubator-trafficcontrol/grove/plugin"
"github.com/apache/incubator-trafficcontrol/grove/remapdata"
+ "github.com/apache/incubator-trafficcontrol/grove/rfc"
"github.com/apache/incubator-trafficcontrol/grove/web"
"github.com/apache/incubator-trafficcontrol/lib/go-log"
@@ -336,7 +337,7 @@ func LoadRemapRules(path string, pluginConfigLoaders map[string]plugin.LoadFunc,
if remapRulesJSON.RetryCodes != nil {
remapRules.RetryCodes = make(map[int]struct{}, len(*remapRulesJSON.RetryCodes))
for _, code := range *remapRulesJSON.RetryCodes {
- if _, ok := ValidHTTPCodes[code]; !ok {
+ if _, ok := rfc.ValidHTTPCodes[code]; !ok {
return nil, nil, nil, fmt.Errorf("error parsing rules: retry code invalid: %v", code)
}
remapRules.RetryCodes[code] = struct{}{}
@@ -392,7 +393,7 @@ func LoadRemapRules(path string, pluginConfigLoaders map[string]plugin.LoadFunc,
if jsonRule.RetryCodes != nil {
rule.RetryCodes = make(map[int]struct{}, len(*jsonRule.RetryCodes))
for _, code := range *jsonRule.RetryCodes {
- if _, ok := ValidHTTPCodes[code]; !ok {
+ if _, ok := rfc.ValidHTTPCodes[code]; !ok {
return nil, nil, nil, fmt.Errorf("error parsing rule %v retry code invalid: %v", rule.Name, code)
}
rule.RetryCodes[code] = struct{}{}
@@ -511,7 +512,7 @@ func makeTo(tosJSON []RemapRuleToJSON, rule remapdata.RemapRule, baseTransport *
if toJSON.RetryCodes != nil {
to.RetryCodes = make(map[int]struct{}, len(*toJSON.RetryCodes))
for _, code := range *toJSON.RetryCodes {
- if _, ok := ValidHTTPCodes[code]; !ok {
+ if _, ok := rfc.ValidHTTPCodes[code]; !ok {
return nil, fmt.Errorf("error parsing to %v retry code invalid: %v", to.URL, code)
}
to.RetryCodes[code] = struct{}{}
diff --git a/grove/remap/rules.go b/grove/rfc/rules.go
similarity index 68%
rename from grove/remap/rules.go
rename to grove/rfc/rules.go
index 31ba715..1879bc7 100644
--- a/grove/remap/rules.go
+++ b/grove/rfc/rules.go
@@ -1,4 +1,4 @@
-package remap
+package rfc
/*
Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,6 +14,10 @@ package remap
limitations under the License.
*/
+// Package rfc contains functions implementing RFC 7234, 2616, and other RFCs.
+// When changing functions, be sure they still conform to the corresponding RFC.
+// When adding symbols, document the RFC and section they correspond to.
+
import (
"net/http"
"strings"
@@ -24,6 +28,8 @@ import (
"github.com/apache/incubator-trafficcontrol/grove/web"
"github.com/apache/incubator-trafficcontrol/lib/go-log"
+ "math"
+ "strconv"
)
// ValidHTTPCodes provides fast lookup whether a HTTP response code is valid per RFC7234§3
@@ -266,8 +272,8 @@ func fresh(
respReqTime time.Time,
respRespTime time.Time,
) bool {
- freshnessLifetime := web.GetFreshnessLifetime(respHeaders, respCacheControl)
- currentAge := web.GetCurrentAge(respHeaders, respReqTime, respRespTime)
+ freshnessLifetime := getFreshnessLifetime(respHeaders, respCacheControl)
+ currentAge := getCurrentAge(respHeaders, respReqTime, respRespTime)
log.Debugf("Fresh: freshnesslifetime %v currentAge %v\n", freshnessLifetime, currentAge)
fresh := freshnessLifetime > currentAge
return fresh
@@ -275,12 +281,12 @@ func fresh(
// inMinFresh returns whether the given response is within the `min-fresh` request directive. If no `min-fresh` directive exists in the request, `true` is returned.
func inMinFresh(respHeaders http.Header, reqCacheControl web.CacheControl, respCacheControl web.CacheControl, respReqTime time.Time, respRespTime time.Time) bool {
- minFresh, ok := web.GetHTTPDeltaSecondsCacheControl(reqCacheControl, "min-fresh")
+ minFresh, ok := getHTTPDeltaSecondsCacheControl(reqCacheControl, "min-fresh")
if !ok {
return true // no min-fresh => within min-fresh
}
- freshnessLifetime := web.GetFreshnessLifetime(respHeaders, respCacheControl)
- currentAge := web.GetCurrentAge(respHeaders, respReqTime, respRespTime)
+ freshnessLifetime := getFreshnessLifetime(respHeaders, respCacheControl)
+ currentAge := getCurrentAge(respHeaders, respReqTime, respRespTime)
inMinFresh := minFresh < (freshnessLifetime - currentAge)
log.Debugf("inMinFresh minFresh %v freshnessLifetime %v currentAge %v => %v < (%v - %v) = %v\n", minFresh, freshnessLifetime, currentAge, minFresh, freshnessLifetime, currentAge, inMinFresh)
return inMinFresh
@@ -322,13 +328,13 @@ func allowedStale(respHeaders http.Header, reqCacheControl web.CacheControl, res
// InMaxStale returns whether the given response is within the `max-stale` request directive. If no `max-stale` directive exists in the request, `true` is returned.
func inMaxStale(respHeaders http.Header, respCacheControl web.CacheControl, respReqTime time.Time, respRespTime time.Time) bool {
- maxStale, ok := web.GetHTTPDeltaSecondsCacheControl(respCacheControl, "max-stale")
+ maxStale, ok := getHTTPDeltaSecondsCacheControl(respCacheControl, "max-stale")
if !ok {
// maxStale = 5 // debug
return true // no max-stale => within max-stale
}
- freshnessLifetime := web.GetFreshnessLifetime(respHeaders, respCacheControl)
- currentAge := web.GetCurrentAge(respHeaders, respReqTime, respRespTime)
+ freshnessLifetime := getFreshnessLifetime(respHeaders, respCacheControl)
+ currentAge := getCurrentAge(respHeaders, respReqTime, respRespTime)
log.Errorf("DEBUGR InMaxStale maxStale %v freshnessLifetime %v currentAge %v => %v > (%v, %v)\n", maxStale, freshnessLifetime, currentAge, maxStale, currentAge, freshnessLifetime) // DEBUG
inMaxStale := maxStale > (currentAge - freshnessLifetime)
return inMaxStale
@@ -378,3 +384,145 @@ func hasPragmaNoCache(reqHeaders http.Header) bool {
}
return false
}
+
+// GetHTTPDeltaSeconds is a helper function which gets an HTTP Delta Seconds from the given map (which is typically a `http.Header` or `CacheControl`. Returns false if the given key doesn't exist in the map, or if the value isn't a valid Delta Seconds per RFC2616§3.3.2.
+func getHTTPDeltaSeconds(m map[string][]string, key string) (time.Duration, bool) {
+ maybeSeconds, ok := m[key]
+ if !ok {
+ return 0, false
+ }
+ if len(maybeSeconds) == 0 {
+ return 0, false
+ }
+ maybeSec := maybeSeconds[0]
+
+ seconds, err := strconv.ParseUint(maybeSec, 10, 64)
+ if err != nil {
+ return 0, false
+ }
+ return time.Duration(seconds) * time.Second, true
+}
+
+// getHTTPDeltaSeconds is a helper function which gets an HTTP Delta Seconds from the given map (which is typically a `http.Header` or `CacheControl`. Returns false if the given key doesn't exist in the map, or if the value isn't a valid Delta Seconds per RFC2616§3.3.2.
+func getHTTPDeltaSecondsCacheControl(m map[string]string, key string) (time.Duration, bool) {
+ maybeSec, ok := m[key]
+ if !ok {
+ return 0, false
+ }
+ seconds, err := strconv.ParseUint(maybeSec, 10, 64)
+ if err != nil {
+ return 0, false
+ }
+ return time.Duration(seconds) * time.Second, true
+}
+
+// getFreshnessLifetime calculates the freshness_lifetime per RFC7234§4.2.1
+func getFreshnessLifetime(respHeaders http.Header, respCacheControl web.CacheControl) time.Duration {
+ if s, ok := getHTTPDeltaSecondsCacheControl(respCacheControl, "s-maxage"); ok {
+ return s
+ }
+ if s, ok := getHTTPDeltaSecondsCacheControl(respCacheControl, "max-age"); ok {
+ return s
+ }
+
+ getExpires := func() (time.Duration, bool) {
+ expires, ok := web.GetHTTPDate(respHeaders, "Expires")
+ if !ok {
+ return 0, false
+ }
+ date, ok := web.GetHTTPDate(respHeaders, "Date")
+ if !ok {
+ return 0, false
+ }
+ return expires.Sub(date), true
+ }
+ if s, ok := getExpires(); ok {
+ return s
+ }
+ return heuristicFreshness(respHeaders)
+}
+
+const Day = time.Hour * time.Duration(24)
+
+// HeuristicFreshness follows the recommendation of RFC7234§4.2.2 and returns the min of 10% of the (Date - Last-Modified) headers and 24 hours, if they exist, and 24 hours if they don't.
+// TODO: smarter and configurable heuristics
+func heuristicFreshness(respHeaders http.Header) time.Duration {
+ sinceLastModified, ok := sinceLastModified(respHeaders)
+ if !ok {
+ return Day
+ }
+ freshness := time.Duration(math.Min(float64(Day), float64(sinceLastModified)))
+ return freshness
+}
+
+func sinceLastModified(headers http.Header) (time.Duration, bool) {
+ lastModified, ok := web.GetHTTPDate(headers, "last-modified")
+ if !ok {
+ return 0, false
+ }
+ date, ok := web.GetHTTPDate(headers, "date")
+ if !ok {
+ return 0, false
+ }
+ return date.Sub(lastModified), true
+}
+
+// ageValue is used to calculate current_age per RFC7234§4.2.3
+func ageValue(respHeaders http.Header) time.Duration {
+ s, ok := getHTTPDeltaSeconds(respHeaders, "age")
+ if !ok {
+ return 0
+ }
+ return s
+}
+
+// dateValue is used to calculate current_age per RFC7234§4.2.3. It returns time, or false if the response had no Date header (in violation of HTTP/1.1).
+func dateValue(respHeaders http.Header) (time.Time, bool) {
+ return web.GetHTTPDate(respHeaders, "date")
+}
+
+func apparentAge(respHeaders http.Header, respRespTime time.Time) time.Duration {
+ dateValue, ok := dateValue(respHeaders)
+ if !ok {
+ return 0 // TODO log warning?
+ }
+ rawAge := respRespTime.Sub(dateValue)
+ return time.Duration(math.Max(0.0, float64(rawAge)))
+}
+
+func responseDelay(respReqTime time.Time, respRespTime time.Time) time.Duration {
+ return respRespTime.Sub(respReqTime)
+}
+
+func correctedAgeValue(respHeaders http.Header, respReqTime time.Time, respRespTime time.Time) time.Duration {
+ return ageValue(respHeaders) + responseDelay(respReqTime, respRespTime)
+}
+
+func correctedInitialAge(respHeaders http.Header, respReqTime time.Time, respRespTime time.Time) time.Duration {
+ return time.Duration(math.Max(float64(apparentAge(respHeaders, respRespTime)), float64(correctedAgeValue(respHeaders, respReqTime, respRespTime))))
+}
+
+func residentTime(respRespTime time.Time) time.Duration {
+ return time.Now().Sub(respRespTime)
+}
+
+func getCurrentAge(respHeaders http.Header, respReqTime time.Time, respRespTime time.Time) time.Duration {
+ correctedInitial := correctedInitialAge(respHeaders, respReqTime, respRespTime)
+ resident := residentTime(respRespTime)
+ log.Debugf("getCurrentAge: correctedInitialAge %v residentTime %v\n", correctedInitial, resident)
+ return correctedInitial + resident
+}
+
+// FreshFor checks returns how long this object is still good for
+func FreshFor(
+ respHeaders http.Header,
+ respCacheControl web.CacheControl,
+ respReqTime time.Time,
+ respRespTime time.Time,
+) time.Duration {
+ freshnessLifetime := getFreshnessLifetime(respHeaders, respCacheControl)
+ currentAge := getCurrentAge(respHeaders, respReqTime, respRespTime)
+ log.Debugf("FreshFor: freshnesslifetime %v currentAge %v\n", freshnessLifetime, currentAge)
+ //fresh := freshnessLifetime > currentAge
+ return freshnessLifetime - currentAge
+}
diff --git a/grove/remap/rules_test.go b/grove/rfc/rules_test.go
similarity index 99%
rename from grove/remap/rules_test.go
rename to grove/rfc/rules_test.go
index 91bdde9..f364aad 100644
--- a/grove/remap/rules_test.go
+++ b/grove/rfc/rules_test.go
@@ -1,4 +1,4 @@
-package remap
+package rfc
/*
Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,6 +14,10 @@ package remap
limitations under the License.
*/
+// Package rfc contains functions implementing RFC 7234, 2616, and other RFCs.
+// When changing functions, be sure they still conform to the corresponding RFC.
+// When adding symbols, document the RFC and section they correspond to.
+
import (
"net/http"
"os"
diff --git a/grove/web/util.go b/grove/web/util.go
index 23c7922..ff13e44 100644
--- a/grove/web/util.go
+++ b/grove/web/util.go
@@ -22,9 +22,8 @@ import (
"strings"
"time"
+ //"github.com/apache/incubator-trafficcontrol/grove/rfc"
"github.com/apache/incubator-trafficcontrol/lib/go-log"
- "math"
- "strconv"
)
type Hdr struct {
@@ -200,145 +199,3 @@ func ParseHTTPDate(d string) (time.Time, bool) {
// RemapTextKey is the plugin shared data key inserted by grovetccfg for the Remap Line of the Delivery Service in Traffic Control, Traffic Ops.
const RemapTextKey = "remap_text"
-
-const Day = time.Hour * time.Duration(24)
-
-// GetHTTPDeltaSeconds is a helper function which gets an HTTP Delta Seconds from the given map (which is typically a `http.Header` or `CacheControl`. Returns false if the given key doesn't exist in the map, or if the value isn't a valid Delta Seconds per RFC2616§3.3.2.
-func GetHTTPDeltaSeconds(m map[string][]string, key string) (time.Duration, bool) {
- maybeSeconds, ok := m[key]
- if !ok {
- return 0, false
- }
- if len(maybeSeconds) == 0 {
- return 0, false
- }
- maybeSec := maybeSeconds[0]
-
- seconds, err := strconv.ParseUint(maybeSec, 10, 64)
- if err != nil {
- return 0, false
- }
- return time.Duration(seconds) * time.Second, true
-}
-
-// GetHTTPDeltaSeconds is a helper function which gets an HTTP Delta Seconds from the given map (which is typically a `http.Header` or `CacheControl`. Returns false if the given key doesn't exist in the map, or if the value isn't a valid Delta Seconds per RFC2616§3.3.2.
-func GetHTTPDeltaSecondsCacheControl(m map[string]string, key string) (time.Duration, bool) {
- maybeSec, ok := m[key]
- if !ok {
- return 0, false
- }
- seconds, err := strconv.ParseUint(maybeSec, 10, 64)
- if err != nil {
- return 0, false
- }
- return time.Duration(seconds) * time.Second, true
-}
-
-// HeuristicFreshness follows the recommendation of RFC7234§4.2.2 and returns the min of 10% of the (Date - Last-Modified) headers and 24 hours, if they exist, and 24 hours if they don't.
-// TODO: smarter and configurable heuristics
-func HeuristicFreshness(respHeaders http.Header) time.Duration {
- sinceLastModified, ok := sinceLastModified(respHeaders)
- if !ok {
- return Day
- }
- freshness := time.Duration(math.Min(float64(Day), float64(sinceLastModified)))
- return freshness
-}
-
-func sinceLastModified(headers http.Header) (time.Duration, bool) {
- lastModified, ok := GetHTTPDate(headers, "last-modified")
- if !ok {
- return 0, false
- }
- date, ok := GetHTTPDate(headers, "date")
- if !ok {
- return 0, false
- }
- return date.Sub(lastModified), true
-}
-
-// GetFreshnessLifetime calculates the freshness_lifetime per RFC7234§4.2.1
-func GetFreshnessLifetime(respHeaders http.Header, respCacheControl CacheControl) time.Duration {
- if s, ok := GetHTTPDeltaSecondsCacheControl(respCacheControl, "s-maxage"); ok {
- return s
- }
- if s, ok := GetHTTPDeltaSecondsCacheControl(respCacheControl, "max-age"); ok {
- return s
- }
-
- getExpires := func() (time.Duration, bool) {
- expires, ok := GetHTTPDate(respHeaders, "Expires")
- if !ok {
- return 0, false
- }
- date, ok := GetHTTPDate(respHeaders, "Date")
- if !ok {
- return 0, false
- }
- return expires.Sub(date), true
- }
- if s, ok := getExpires(); ok {
- return s
- }
- return HeuristicFreshness(respHeaders)
-}
-
-// t6AgeValue is used to calculate current_age per RFC7234§4.2.3
-func AgeValue(respHeaders http.Header) time.Duration {
- s, ok := GetHTTPDeltaSeconds(respHeaders, "age")
- if !ok {
- return 0
- }
- return s
-}
-
-func GetCurrentAge(respHeaders http.Header, respReqTime time.Time, respRespTime time.Time) time.Duration {
- correctedInitial := CorrectedInitialAge(respHeaders, respReqTime, respRespTime)
- resident := residentTime(respRespTime)
- log.Debugf("getCurrentAge: correctedInitialAge %v residentTime %v\n", correctedInitial, resident)
- return correctedInitial + resident
-}
-
-func CorrectedInitialAge(respHeaders http.Header, respReqTime time.Time, respRespTime time.Time) time.Duration {
- return time.Duration(math.Max(float64(ApparentAge(respHeaders, respRespTime)), float64(CorrectedAgeValue(respHeaders, respReqTime, respRespTime))))
-}
-
-func CorrectedAgeValue(respHeaders http.Header, respReqTime time.Time, respRespTime time.Time) time.Duration {
- return AgeValue(respHeaders) + responseDelay(respReqTime, respRespTime)
-}
-
-func responseDelay(respReqTime time.Time, respRespTime time.Time) time.Duration {
- return respRespTime.Sub(respReqTime)
-}
-
-func residentTime(respRespTime time.Time) time.Duration {
- return time.Now().Sub(respRespTime)
-}
-
-func ApparentAge(respHeaders http.Header, respRespTime time.Time) time.Duration {
- dateValue, ok := dateValue(respHeaders)
- if !ok {
- return 0 // TODO log warning?
- }
- rawAge := respRespTime.Sub(dateValue)
- return time.Duration(math.Max(0.0, float64(rawAge)))
-}
-
-// dateValue is used to calculate current_age per RFC7234§4.2.3. It returns time, or false if the response had no Date header (in violation of HTTP/1.1).
-func dateValue(respHeaders http.Header) (time.Time, bool) {
- return GetHTTPDate(respHeaders, "date")
-}
-
-// FreshFor checks returns how long this object is still good for
-func FreshFor(
- respHeaders http.Header,
- respCacheControl CacheControl,
- respReqTime time.Time,
- respRespTime time.Time,
-) time.Duration {
- freshnessLifetime := GetFreshnessLifetime(respHeaders, respCacheControl)
- currentAge := GetCurrentAge(respHeaders, respReqTime, respRespTime)
- log.Debugf("FreshFor: freshnesslifetime %v currentAge %v\n", freshnessLifetime, currentAge)
- //fresh := freshnessLifetime > currentAge
- return freshnessLifetime - currentAge
-}
--
To stop receiving notification emails like this one, please contact
rob@apache.org.
[incubator-trafficcontrol] 01/04: Add HitCount to cache object and
cleanup of HTTP table in cacheinsepctor
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rob pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-trafficcontrol.git
commit 784198d10e5bb41ff50d336850119b1034a6b6ae
Author: Jan van Doorn <ja...@comcast.com>
AuthorDate: Mon May 7 09:28:30 2018 -0600
Add HitCount to cache object and cleanup of HTTP table in cacheinsepctor
---
grove/cache/retryinggetter.go | 1 +
grove/cacheobj/cacheobj.go | 2 ++
grove/diskcache/diskcache.go | 4 ++--
grove/memcache/memcache.go | 1 +
grove/plugin/http_cacheinspector.go | 7 ++++---
5 files changed, 10 insertions(+), 5 deletions(-)
diff --git a/grove/cache/retryinggetter.go b/grove/cache/retryinggetter.go
index 12a0cf8..ee60017 100644
--- a/grove/cache/retryinggetter.go
+++ b/grove/cache/retryinggetter.go
@@ -187,6 +187,7 @@ func GetAndCache(
RespRespTime: respRespTime,
LastModified: revalidateObj.LastModified,
Size: revalidateObj.Size,
+ HitCount: revalidateObj.HitCount, // no need to +1 here, the cache Get did that
}
}
cache.Add(cacheKey, obj) // TODO store pointer?
diff --git a/grove/cacheobj/cacheobj.go b/grove/cacheobj/cacheobj.go
index 3e6b27f..c05888f 100644
--- a/grove/cacheobj/cacheobj.go
+++ b/grove/cacheobj/cacheobj.go
@@ -34,6 +34,7 @@ type CacheObj struct {
RespRespTime time.Time // the origin server's Date time when the object was sent
LastModified time.Time // the origin LastModified if it exists, or Date if it doesn't
Size uint64
+ HitCount uint64 // the number of times this object was hit
}
// ComputeSize computes the size of the given CacheObj. This computation is expensive, as the headers must be iterated over. Thus, the size should be computed once and stored, not computed on-the-fly for every new request for the cached object.
@@ -55,6 +56,7 @@ func New(reqHeader http.Header, bytes []byte, code int, originCode int, proxyURL
ReqRespTime: reqRespTime,
RespRespTime: respRespTime,
LastModified: lastModified,
+ HitCount: 1,
}
// copyHeader(reqHeader, &obj.reqHeaders)
// copyHeader(respHeader, &obj.respHeaders)
diff --git a/grove/diskcache/diskcache.go b/grove/diskcache/diskcache.go
index ae9ce35..a8e1b30 100644
--- a/grove/diskcache/diskcache.go
+++ b/grove/diskcache/diskcache.go
@@ -152,16 +152,16 @@ func (c *DiskCache) gc(cacheSizeBytes uint64) {
func (c *DiskCache) Get(key string) (*cacheobj.CacheObj, bool) {
val, found := c.Peek(key)
if found {
- // TODO JvD check to see if val.Size is good to use here.
c.lru.Add(key, val.Size) // TODO directly call c.ll.MoveToFront
log.Debugln("DiskCache.Get getting '" + key + "' from cache and updating LRU")
+ atomic.AddUint64(&val.HitCount, 1)
return val, true
}
return nil, false
}
-// Peek takes a key, and returns its value, and whether it was found, without changing the lru-ness or hit-count
+// Peek takes a key, and returns its value, and whether it was found, without changing the lru-ness or hitcount
func (c *DiskCache) Peek(key string) (*cacheobj.CacheObj, bool) {
log.Debugln("DiskCache.Get key '" + key + "'")
valBytes := []byte(nil)
diff --git a/grove/memcache/memcache.go b/grove/memcache/memcache.go
index 2cd29ce..29d0217 100644
--- a/grove/memcache/memcache.go
+++ b/grove/memcache/memcache.go
@@ -52,6 +52,7 @@ func (c *MemCache) Get(key string) (*cacheobj.CacheObj, bool) {
obj, ok := c.cache[key]
if ok {
c.lru.Add(key, obj.Size) // TODO directly call c.ll.MoveToFront
+ atomic.AddUint64(&obj.HitCount, 1)
}
c.cacheM.RUnlock()
return obj, ok
diff --git a/grove/plugin/http_cacheinspector.go b/grove/plugin/http_cacheinspector.go
index 23b40d1..ee792b4 100644
--- a/grove/plugin/http_cacheinspector.go
+++ b/grove/plugin/http_cacheinspector.go
@@ -119,6 +119,7 @@ func cacheinspect(icfg interface{}, d OnRequestData) bool {
w.Write([]byte(fmt.Sprintf(" ReqRespTime: %v\n", cacheObject.ReqRespTime)))
w.Write([]byte(fmt.Sprintf(" RespRespTime: %v\n", cacheObject.RespRespTime)))
w.Write([]byte(fmt.Sprintf(" LastModified: %v\n", cacheObject.LastModified)))
+ w.Write([]byte(fmt.Sprintf(" HitCount: %v\n", cacheObject.HitCount)))
} else {
w.Write([]byte("Not Found"))
}
@@ -169,7 +170,7 @@ func cacheinspect(icfg interface{}, d OnRequestData) bool {
w.Write([]byte(fmt.Sprintf("showing only first %d and last %d:\n\n", head, tail)))
}
- w.Write([]byte(fmt.Sprintf("<b> #\t\tCode\tSize\tAge\t\t\tKey</b>\n")))
+ w.Write([]byte(fmt.Sprintf("<b> # Code Size Age HitCount Key</b>\n")))
for i, key := range keys {
if (doSearch && !strings.Contains(key, searchArr[0])) || !doSearch && (i > tail && i < len(keys)-head) {
continue
@@ -177,8 +178,8 @@ func cacheinspect(icfg interface{}, d OnRequestData) bool {
cacheObject, _ := d.Stats.CachePeek(key, cName)
age := time.Now().Sub(cacheObject.ReqRespTime)
- w.Write([]byte(fmt.Sprintf(" %05d\t%d\t%s\t%-20v\t<a href=\"http://%s%s?key=%s&cache=%s\">%s</a>\n",
- i, cacheObject.Code, bytefmt.ByteSize(cacheObject.Size), age, req.Host, CacheStatsEndpoint, url.QueryEscape(key), cName, key)))
+ w.Write([]byte(fmt.Sprintf(" %8d%8d%10s%22v%12d <a href=\"http://%s%s?key=%s&cache=%s\">%s</a>\n",
+ i, cacheObject.Code, bytefmt.ByteSize(cacheObject.Size), age, cacheObject.HitCount, req.Host, CacheStatsEndpoint, url.QueryEscape(key), cName, key)))
}
}
--
To stop receiving notification emails like this one, please contact
rob@apache.org.
[incubator-trafficcontrol] 04/04: Clean up
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rob pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-trafficcontrol.git
commit cb549098ec9d44a830b71f9f1dbdae27092359c5
Author: Jan van Doorn <ja...@comcast.com>
AuthorDate: Wed May 16 09:01:26 2018 -0600
Clean up
---
grove/integration_test/compare_gets.go | 140 ---------------------------------
grove/web/util.go | 1 -
2 files changed, 141 deletions(-)
diff --git a/grove/integration_test/compare_gets.go b/grove/integration_test/compare_gets.go
deleted file mode 100644
index 08d3aca..0000000
--- a/grove/integration_test/compare_gets.go
+++ /dev/null
@@ -1,140 +0,0 @@
-package main
-
-import (
- "flag"
- "fmt"
- "github.com/apache/incubator-trafficcontrol/grove/web"
- "io/ioutil"
- "log"
- "net/http"
- "os"
- "strings"
-)
-
-type responseType struct {
- Headers http.Header
- Body []byte
-}
-
-func httpGet(URL, headers string) responseType {
- client := &http.Client{}
- req, err := http.NewRequest("GET", URL, nil)
- if err != nil {
- fmt.Println("ERROR in httpGet")
- }
- //log.Printf(">>>%v<<< %v\n", headers, len(strings.Split(headers, ".")))
- for _, hdrString := range strings.Split(headers, ",") {
- //log.Println(">>> ", hdrString)
- if hdrString == "" {
- continue
- }
- parts := strings.Split(hdrString, ":")
- if parts[0] == "Host" {
- req.Host = parts[1]
- } else {
- //log.Println("> ", parts)
- req.Header.Set(parts[0], parts[1])
- }
- }
- //log.Printf(">>>> %v", req)
- resp, err := client.Do(req)
- if err != nil {
- fmt.Println("ERROR in httpGet")
- }
- defer resp.Body.Close()
- var response responseType
- response.Headers = web.CopyHeader(resp.Header)
- response.Body, err = ioutil.ReadAll(resp.Body)
- if err != nil {
- fmt.Println("ERROR in httpGet (readall)")
- }
- return response
-}
-
-func equal(a, b []byte) bool {
- if a == nil || b == nil {
- return false
- }
-
- if a == nil && b == nil {
- return true
- }
- if len(a) != len(b) {
- return false
- }
-
- for i := range a {
- if a[i] != b[i] {
- return false
- }
- }
-
- return true
-}
-
-func equalStringSlices(a, b []string) bool {
- if a == nil || b == nil {
- return false
- }
-
- if a == nil && b == nil {
- return true
- }
- if len(a) != len(b) {
- return false
- }
-
- for i := range a {
- if a[i] != b[i] {
- return false
- }
- }
-
- return true
-}
-
-func inStringSlice(str string, arr []string) bool {
- for _, strEnt := range arr {
- if strEnt == str {
- return true
- }
- }
- return false
-}
-
-func compareResponses(response1 responseType, response2 responseType, ignoreHdrs []string) bool {
- if !equal(response1.Body, response2.Body) {
- return false
- }
- for hdrKey, _ := range response1.Headers {
- if inStringSlice(hdrKey, ignoreHdrs) {
- continue
- }
- if !equalStringSlices(response1.Headers[hdrKey], response2.Headers[hdrKey]) {
- log.Printf("ERROR hdr %v doesn't match: \"%v\" != \"%v\"\n", hdrKey, response1.Headers[hdrKey], response2.Headers[hdrKey])
- return false
- }
- //fmt.Printf(">>>>> %v\n", hdrKey)
- }
-
- return true
-}
-func main() {
- originURL := flag.String("org", "http://localhost", "The origin URL (default: \"http://localhost\")")
- cacheURL := flag.String("cache", "http://localhost:8080", "The cache URL (default: \"http://localhost:8080\")")
- path := flag.String("path", "", "The path to GET")
- orgHdrs := flag.String("ohdrs", "", "Comma seperated list of headers to add to origin request")
- cacheHdrs := flag.String("chdrs", "", "Comma separated list of headers to add to cache request")
- ignoreHdrs := flag.String("ignorehdrs", "Server,Date", "Comma separated list of headers to ignore in the compare")
- flag.Parse()
-
- resp := httpGet(*originURL+"/"+*path, *orgHdrs)
- cresp := httpGet(*cacheURL+"/"+*path, *cacheHdrs)
- if !compareResponses(resp, cresp, strings.Split(*ignoreHdrs, ",")) {
- fmt.Println("FAIL: Body bytes don't match")
- os.Exit(1)
-
- }
- fmt.Println("PASS")
- os.Exit(0)
-}
diff --git a/grove/web/util.go b/grove/web/util.go
index ff13e44..4d36e08 100644
--- a/grove/web/util.go
+++ b/grove/web/util.go
@@ -22,7 +22,6 @@ import (
"strings"
"time"
- //"github.com/apache/incubator-trafficcontrol/grove/rfc"
"github.com/apache/incubator-trafficcontrol/lib/go-log"
)
--
To stop receiving notification emails like this one, please contact
rob@apache.org.
[incubator-trafficcontrol] 02/04: add FreshFor
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rob pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-trafficcontrol.git
commit e829d1447c39ff8465c9f4ac75c6d19ed5205cc0
Author: Jan van Doorn <ja...@comcast.com>
AuthorDate: Tue May 8 15:57:40 2018 -0600
add FreshFor
---
grove/plugin/README_http_cacheinspector.md | 23 ++---
grove/plugin/http_cacheinspector.go | 9 +-
grove/remap/rules.go | 146 ++---------------------------
grove/web/util.go | 144 ++++++++++++++++++++++++++++
4 files changed, 167 insertions(+), 155 deletions(-)
diff --git a/grove/plugin/README_http_cacheinspector.md b/grove/plugin/README_http_cacheinspector.md
index e008e57..d9b1206 100644
--- a/grove/plugin/README_http_cacheinspector.md
+++ b/grove/plugin/README_http_cacheinspector.md
@@ -29,21 +29,18 @@ Jump to: disk my-disk-cache-two
*** Cache "" ***
- * Size of in use cache: 9.5M
- * Cache capacity: 9.5M
- * Number of elements in LRU: 169
+ * Size of in use cache: 1.5M
+ * Cache capacity: 9.5M
+ * Number of elements in LRU: 54
* Objects in cache sorted by Least Recently Used on top, showing only first 100 and last 100:
- # Code Size Age Key
- 00000 200 34K 16h19m9.135951506s GET:http://localhost/34k.bin?hah
- 00001 200 35K 16h19m9.103181146s GET:http://localhost/35k.bin?hah
- 00002 200 36K 16h19m9.079490592s GET:http://localhost/36k.bin?hah
- 00003 200 37K 16h19m9.050852429s GET:http://localhost/37k.bin?hah
- 00004 200 38K 16h19m9.020264008s GET:http://localhost/38k.bin?hah
- 00005 200 39K 16h19m8.989722029s GET:http://localhost/39k.bin?hah
- 00006 200 40K 16h19m8.967823063s GET:http://localhost/40k.bin?hah
- 00007 200 41K 16h19m8.939422783s GET:http://localhost/41k.bin?hah
- 00008 200 42K 16h19m8.913485123s GET:http://localhost/42k.bin?h
+ # Code Size Age FreshFor HitCount Key
+ 0 200 15K 600.922ms 50.541023s 1 GET:http://localhost/15k.bin?
+ 1 200 16K 627.929ms 50.540856s 1 GET:http://localhost/16k.bin?
+ 2 200 17K 650.813ms 50.540698s 1 GET:http://localhost/17k.bin?
+ 3 200 18K 671.732ms 50.540547s 1 GET:http://localhost/18k.bin?
+ 4 200 19K 691.981ms 50.540373s 1 GET:http://localhost/19k.bin?
+ 5 200 20K 715.566ms 50.540261s 1 GET:http://localhost/20k.bin?
```
diff --git a/grove/plugin/http_cacheinspector.go b/grove/plugin/http_cacheinspector.go
index ee792b4..2cdb158 100644
--- a/grove/plugin/http_cacheinspector.go
+++ b/grove/plugin/http_cacheinspector.go
@@ -170,16 +170,17 @@ func cacheinspect(icfg interface{}, d OnRequestData) bool {
w.Write([]byte(fmt.Sprintf("showing only first %d and last %d:\n\n", head, tail)))
}
- w.Write([]byte(fmt.Sprintf("<b> # Code Size Age HitCount Key</b>\n")))
+ w.Write([]byte(fmt.Sprintf("<b> # Code Size Age FreshFor HitCount Key</b>\n")))
for i, key := range keys {
- if (doSearch && !strings.Contains(key, searchArr[0])) || !doSearch && (i > tail && i < len(keys)-head) {
+ if (doSearch && !strings.Contains(key, searchArr[0])) || !doSearch && (i >= tail && i < len(keys)-head) {
continue
}
cacheObject, _ := d.Stats.CachePeek(key, cName)
age := time.Now().Sub(cacheObject.ReqRespTime)
- w.Write([]byte(fmt.Sprintf(" %8d%8d%10s%22v%12d <a href=\"http://%s%s?key=%s&cache=%s\">%s</a>\n",
- i, cacheObject.Code, bytefmt.ByteSize(cacheObject.Size), age, cacheObject.HitCount, req.Host, CacheStatsEndpoint, url.QueryEscape(key), cName, key)))
+ freshFor := web.FreshFor(cacheObject.RespHeaders, cacheObject.RespCacheControl, cacheObject.ReqRespTime, cacheObject.RespRespTime)
+ w.Write([]byte(fmt.Sprintf(" %8d%8d%10s%22v%22v%12d <a href=\"http://%s%s?key=%s&cache=%s\">%s</a>\n",
+ i, cacheObject.Code, bytefmt.ByteSize(cacheObject.Size), age, freshFor, cacheObject.HitCount, req.Host, CacheStatsEndpoint, url.QueryEscape(key), cName, key)))
}
}
diff --git a/grove/remap/rules.go b/grove/remap/rules.go
index d98a04a..31ba715 100644
--- a/grove/remap/rules.go
+++ b/grove/remap/rules.go
@@ -15,9 +15,7 @@ package remap
*/
import (
- "math"
"net/http"
- "strconv"
"strings"
"time"
@@ -268,149 +266,21 @@ func fresh(
respReqTime time.Time,
respRespTime time.Time,
) bool {
- freshnessLifetime := getFreshnessLifetime(respHeaders, respCacheControl)
- currentAge := getCurrentAge(respHeaders, respReqTime, respRespTime)
+ freshnessLifetime := web.GetFreshnessLifetime(respHeaders, respCacheControl)
+ currentAge := web.GetCurrentAge(respHeaders, respReqTime, respRespTime)
log.Debugf("Fresh: freshnesslifetime %v currentAge %v\n", freshnessLifetime, currentAge)
fresh := freshnessLifetime > currentAge
return fresh
}
-// GetHTTPDeltaSeconds is a helper function which gets an HTTP Delta Seconds from the given map (which is typically a `http.Header` or `CacheControl`. Returns false if the given key doesn't exist in the map, or if the value isn't a valid Delta Seconds per RFC2616§3.3.2.
-func getHTTPDeltaSeconds(m map[string][]string, key string) (time.Duration, bool) {
- maybeSeconds, ok := m[key]
- if !ok {
- return 0, false
- }
- if len(maybeSeconds) == 0 {
- return 0, false
- }
- maybeSec := maybeSeconds[0]
-
- seconds, err := strconv.ParseUint(maybeSec, 10, 64)
- if err != nil {
- return 0, false
- }
- return time.Duration(seconds) * time.Second, true
-}
-
-// getHTTPDeltaSeconds is a helper function which gets an HTTP Delta Seconds from the given map (which is typically a `http.Header` or `CacheControl`. Returns false if the given key doesn't exist in the map, or if the value isn't a valid Delta Seconds per RFC2616§3.3.2.
-func getHTTPDeltaSecondsCacheControl(m map[string]string, key string) (time.Duration, bool) {
- maybeSec, ok := m[key]
- if !ok {
- return 0, false
- }
- seconds, err := strconv.ParseUint(maybeSec, 10, 64)
- if err != nil {
- return 0, false
- }
- return time.Duration(seconds) * time.Second, true
-}
-
-// getFreshnessLifetime calculates the freshness_lifetime per RFC7234§4.2.1
-func getFreshnessLifetime(respHeaders http.Header, respCacheControl web.CacheControl) time.Duration {
- if s, ok := getHTTPDeltaSecondsCacheControl(respCacheControl, "s-maxage"); ok {
- return s
- }
- if s, ok := getHTTPDeltaSecondsCacheControl(respCacheControl, "max-age"); ok {
- return s
- }
-
- getExpires := func() (time.Duration, bool) {
- expires, ok := web.GetHTTPDate(respHeaders, "Expires")
- if !ok {
- return 0, false
- }
- date, ok := web.GetHTTPDate(respHeaders, "Date")
- if !ok {
- return 0, false
- }
- return expires.Sub(date), true
- }
- if s, ok := getExpires(); ok {
- return s
- }
- return heuristicFreshness(respHeaders)
-}
-
-const Day = time.Hour * time.Duration(24)
-
-// HeuristicFreshness follows the recommendation of RFC7234§4.2.2 and returns the min of 10% of the (Date - Last-Modified) headers and 24 hours, if they exist, and 24 hours if they don't.
-// TODO: smarter and configurable heuristics
-func heuristicFreshness(respHeaders http.Header) time.Duration {
- sinceLastModified, ok := sinceLastModified(respHeaders)
- if !ok {
- return Day
- }
- freshness := time.Duration(math.Min(float64(Day), float64(sinceLastModified)))
- return freshness
-}
-
-func sinceLastModified(headers http.Header) (time.Duration, bool) {
- lastModified, ok := web.GetHTTPDate(headers, "last-modified")
- if !ok {
- return 0, false
- }
- date, ok := web.GetHTTPDate(headers, "date")
- if !ok {
- return 0, false
- }
- return date.Sub(lastModified), true
-}
-
-// ageValue is used to calculate current_age per RFC7234§4.2.3
-func ageValue(respHeaders http.Header) time.Duration {
- s, ok := getHTTPDeltaSeconds(respHeaders, "age")
- if !ok {
- return 0
- }
- return s
-}
-
-// dateValue is used to calculate current_age per RFC7234§4.2.3. It returns time, or false if the response had no Date header (in violation of HTTP/1.1).
-func dateValue(respHeaders http.Header) (time.Time, bool) {
- return web.GetHTTPDate(respHeaders, "date")
-}
-
-func apparentAge(respHeaders http.Header, respRespTime time.Time) time.Duration {
- dateValue, ok := dateValue(respHeaders)
- if !ok {
- return 0 // TODO log warning?
- }
- rawAge := respRespTime.Sub(dateValue)
- return time.Duration(math.Max(0.0, float64(rawAge)))
-}
-
-func responseDelay(respReqTime time.Time, respRespTime time.Time) time.Duration {
- return respRespTime.Sub(respReqTime)
-}
-
-func correctedAgeValue(respHeaders http.Header, respReqTime time.Time, respRespTime time.Time) time.Duration {
- return ageValue(respHeaders) + responseDelay(respReqTime, respRespTime)
-}
-
-func correctedInitialAge(respHeaders http.Header, respReqTime time.Time, respRespTime time.Time) time.Duration {
- return time.Duration(math.Max(float64(apparentAge(respHeaders, respRespTime)), float64(correctedAgeValue(respHeaders, respReqTime, respRespTime))))
-}
-
-func residentTime(respRespTime time.Time) time.Duration {
- return time.Now().Sub(respRespTime)
-}
-
-func getCurrentAge(respHeaders http.Header, respReqTime time.Time, respRespTime time.Time) time.Duration {
- correctedInitial := correctedInitialAge(respHeaders, respReqTime, respRespTime)
- resident := residentTime(respRespTime)
- log.Debugf("getCurrentAge: correctedInitialAge %v residentTime %v\n", correctedInitial, resident)
- return correctedInitial + resident
-}
-
// inMinFresh returns whether the given response is within the `min-fresh` request directive. If no `min-fresh` directive exists in the request, `true` is returned.
func inMinFresh(respHeaders http.Header, reqCacheControl web.CacheControl, respCacheControl web.CacheControl, respReqTime time.Time, respRespTime time.Time) bool {
- minFresh, ok := getHTTPDeltaSecondsCacheControl(reqCacheControl, "min-fresh")
+ minFresh, ok := web.GetHTTPDeltaSecondsCacheControl(reqCacheControl, "min-fresh")
if !ok {
return true // no min-fresh => within min-fresh
}
- freshnessLifetime := getFreshnessLifetime(respHeaders, respCacheControl)
- currentAge := getCurrentAge(respHeaders, respReqTime, respRespTime)
+ freshnessLifetime := web.GetFreshnessLifetime(respHeaders, respCacheControl)
+ currentAge := web.GetCurrentAge(respHeaders, respReqTime, respRespTime)
inMinFresh := minFresh < (freshnessLifetime - currentAge)
log.Debugf("inMinFresh minFresh %v freshnessLifetime %v currentAge %v => %v < (%v - %v) = %v\n", minFresh, freshnessLifetime, currentAge, minFresh, freshnessLifetime, currentAge, inMinFresh)
return inMinFresh
@@ -452,13 +322,13 @@ func allowedStale(respHeaders http.Header, reqCacheControl web.CacheControl, res
// InMaxStale returns whether the given response is within the `max-stale` request directive. If no `max-stale` directive exists in the request, `true` is returned.
func inMaxStale(respHeaders http.Header, respCacheControl web.CacheControl, respReqTime time.Time, respRespTime time.Time) bool {
- maxStale, ok := getHTTPDeltaSecondsCacheControl(respCacheControl, "max-stale")
+ maxStale, ok := web.GetHTTPDeltaSecondsCacheControl(respCacheControl, "max-stale")
if !ok {
// maxStale = 5 // debug
return true // no max-stale => within max-stale
}
- freshnessLifetime := getFreshnessLifetime(respHeaders, respCacheControl)
- currentAge := getCurrentAge(respHeaders, respReqTime, respRespTime)
+ freshnessLifetime := web.GetFreshnessLifetime(respHeaders, respCacheControl)
+ currentAge := web.GetCurrentAge(respHeaders, respReqTime, respRespTime)
log.Errorf("DEBUGR InMaxStale maxStale %v freshnessLifetime %v currentAge %v => %v > (%v, %v)\n", maxStale, freshnessLifetime, currentAge, maxStale, currentAge, freshnessLifetime) // DEBUG
inMaxStale := maxStale > (currentAge - freshnessLifetime)
return inMaxStale
diff --git a/grove/web/util.go b/grove/web/util.go
index 4d36e08..23c7922 100644
--- a/grove/web/util.go
+++ b/grove/web/util.go
@@ -23,6 +23,8 @@ import (
"time"
"github.com/apache/incubator-trafficcontrol/lib/go-log"
+ "math"
+ "strconv"
)
type Hdr struct {
@@ -198,3 +200,145 @@ func ParseHTTPDate(d string) (time.Time, bool) {
// RemapTextKey is the plugin shared data key inserted by grovetccfg for the Remap Line of the Delivery Service in Traffic Control, Traffic Ops.
const RemapTextKey = "remap_text"
+
+const Day = time.Hour * time.Duration(24)
+
+// GetHTTPDeltaSeconds is a helper function which gets an HTTP Delta Seconds from the given map (which is typically a `http.Header` or `CacheControl`. Returns false if the given key doesn't exist in the map, or if the value isn't a valid Delta Seconds per RFC2616§3.3.2.
+func GetHTTPDeltaSeconds(m map[string][]string, key string) (time.Duration, bool) {
+ maybeSeconds, ok := m[key]
+ if !ok {
+ return 0, false
+ }
+ if len(maybeSeconds) == 0 {
+ return 0, false
+ }
+ maybeSec := maybeSeconds[0]
+
+ seconds, err := strconv.ParseUint(maybeSec, 10, 64)
+ if err != nil {
+ return 0, false
+ }
+ return time.Duration(seconds) * time.Second, true
+}
+
+// GetHTTPDeltaSeconds is a helper function which gets an HTTP Delta Seconds from the given map (which is typically a `http.Header` or `CacheControl`. Returns false if the given key doesn't exist in the map, or if the value isn't a valid Delta Seconds per RFC2616§3.3.2.
+func GetHTTPDeltaSecondsCacheControl(m map[string]string, key string) (time.Duration, bool) {
+ maybeSec, ok := m[key]
+ if !ok {
+ return 0, false
+ }
+ seconds, err := strconv.ParseUint(maybeSec, 10, 64)
+ if err != nil {
+ return 0, false
+ }
+ return time.Duration(seconds) * time.Second, true
+}
+
+// HeuristicFreshness follows the recommendation of RFC7234§4.2.2 and returns the min of 10% of the (Date - Last-Modified) headers and 24 hours, if they exist, and 24 hours if they don't.
+// TODO: smarter and configurable heuristics
+func HeuristicFreshness(respHeaders http.Header) time.Duration {
+ sinceLastModified, ok := sinceLastModified(respHeaders)
+ if !ok {
+ return Day
+ }
+ freshness := time.Duration(math.Min(float64(Day), float64(sinceLastModified)))
+ return freshness
+}
+
+func sinceLastModified(headers http.Header) (time.Duration, bool) {
+ lastModified, ok := GetHTTPDate(headers, "last-modified")
+ if !ok {
+ return 0, false
+ }
+ date, ok := GetHTTPDate(headers, "date")
+ if !ok {
+ return 0, false
+ }
+ return date.Sub(lastModified), true
+}
+
+// GetFreshnessLifetime calculates the freshness_lifetime per RFC7234§4.2.1
+func GetFreshnessLifetime(respHeaders http.Header, respCacheControl CacheControl) time.Duration {
+ if s, ok := GetHTTPDeltaSecondsCacheControl(respCacheControl, "s-maxage"); ok {
+ return s
+ }
+ if s, ok := GetHTTPDeltaSecondsCacheControl(respCacheControl, "max-age"); ok {
+ return s
+ }
+
+ getExpires := func() (time.Duration, bool) {
+ expires, ok := GetHTTPDate(respHeaders, "Expires")
+ if !ok {
+ return 0, false
+ }
+ date, ok := GetHTTPDate(respHeaders, "Date")
+ if !ok {
+ return 0, false
+ }
+ return expires.Sub(date), true
+ }
+ if s, ok := getExpires(); ok {
+ return s
+ }
+ return HeuristicFreshness(respHeaders)
+}
+
+// t6AgeValue is used to calculate current_age per RFC7234§4.2.3
+func AgeValue(respHeaders http.Header) time.Duration {
+ s, ok := GetHTTPDeltaSeconds(respHeaders, "age")
+ if !ok {
+ return 0
+ }
+ return s
+}
+
+func GetCurrentAge(respHeaders http.Header, respReqTime time.Time, respRespTime time.Time) time.Duration {
+ correctedInitial := CorrectedInitialAge(respHeaders, respReqTime, respRespTime)
+ resident := residentTime(respRespTime)
+ log.Debugf("getCurrentAge: correctedInitialAge %v residentTime %v\n", correctedInitial, resident)
+ return correctedInitial + resident
+}
+
+func CorrectedInitialAge(respHeaders http.Header, respReqTime time.Time, respRespTime time.Time) time.Duration {
+ return time.Duration(math.Max(float64(ApparentAge(respHeaders, respRespTime)), float64(CorrectedAgeValue(respHeaders, respReqTime, respRespTime))))
+}
+
+func CorrectedAgeValue(respHeaders http.Header, respReqTime time.Time, respRespTime time.Time) time.Duration {
+ return AgeValue(respHeaders) + responseDelay(respReqTime, respRespTime)
+}
+
+func responseDelay(respReqTime time.Time, respRespTime time.Time) time.Duration {
+ return respRespTime.Sub(respReqTime)
+}
+
+func residentTime(respRespTime time.Time) time.Duration {
+ return time.Now().Sub(respRespTime)
+}
+
+func ApparentAge(respHeaders http.Header, respRespTime time.Time) time.Duration {
+ dateValue, ok := dateValue(respHeaders)
+ if !ok {
+ return 0 // TODO log warning?
+ }
+ rawAge := respRespTime.Sub(dateValue)
+ return time.Duration(math.Max(0.0, float64(rawAge)))
+}
+
+// dateValue is used to calculate current_age per RFC7234§4.2.3. It returns time, or false if the response had no Date header (in violation of HTTP/1.1).
+func dateValue(respHeaders http.Header) (time.Time, bool) {
+ return GetHTTPDate(respHeaders, "date")
+}
+
+// FreshFor checks returns how long this object is still good for
+func FreshFor(
+ respHeaders http.Header,
+ respCacheControl CacheControl,
+ respReqTime time.Time,
+ respRespTime time.Time,
+) time.Duration {
+ freshnessLifetime := GetFreshnessLifetime(respHeaders, respCacheControl)
+ currentAge := GetCurrentAge(respHeaders, respReqTime, respRespTime)
+ log.Debugf("FreshFor: freshnesslifetime %v currentAge %v\n", freshnessLifetime, currentAge)
+ //fresh := freshnessLifetime > currentAge
+ return freshnessLifetime - currentAge
+}
--
To stop receiving notification emails like this one, please contact
rob@apache.org.