You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@trafficcontrol.apache.org by ro...@apache.org on 2018/05/23 16:41:57 UTC

[incubator-trafficcontrol] 04/04: Implement review comments

This is an automated email from the ASF dual-hosted git repository.

rob pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-trafficcontrol.git

commit 7061aea8b28eb9b854b6a0ec19d330780c12cf17
Author: Jan van Doorn <jv...@knutsel.com>
AuthorDate: Fri May 18 10:38:39 2018 -0600

    Implement review comments
---
 grove/cache/handler.go            |  4 +--
 grove/plugin/README.md            |  2 ++
 grove/plugin/plugin.go            | 15 +++++----
 grove/plugin/range_req_handler.go | 70 ++++++++++++++++++++++-----------------
 4 files changed, 51 insertions(+), 40 deletions(-)

diff --git a/grove/cache/handler.go b/grove/cache/handler.go
index 74b40a5..0b4e15e 100644
--- a/grove/cache/handler.go
+++ b/grove/cache/handler.go
@@ -224,8 +224,8 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 
 	connectionClose := h.connectionClose || remappingProducer.ConnectionClose()
 
-	beforeCacheLookUpData := plugin.BeforeCacheLookUpData{Req: r, DefaultCacheKey: remappingProducer.CacheKey()}
-	h.plugins.OnBeforeCacheLookup(remappingProducer.PluginCfg(), pluginContext, beforeCacheLookUpData, remappingProducer.OverrideCacheKey)
+	beforeCacheLookUpData := plugin.BeforeCacheLookUpData{Req: r, DefaultCacheKey: remappingProducer.CacheKey(), CacheKeyOverrideFunc: remappingProducer.OverrideCacheKey}
+	h.plugins.OnBeforeCacheLookup(remappingProducer.PluginCfg(), pluginContext, beforeCacheLookUpData)
 
 	cacheKey := remappingProducer.CacheKey()
 	retrier := NewRetrier(h, reqHeader, reqTime, reqCacheControl, remappingProducer, reqID)
diff --git a/grove/plugin/README.md b/grove/plugin/README.md
index 8dec9be..ec390d7 100644
--- a/grove/plugin/README.md
+++ b/grove/plugin/README.md
@@ -31,6 +31,8 @@ The `Funcs` object contains functions for each hook, as well as a load function
 
 * `onRequest` is called immediately when a request is received. It returns a boolean indicating whether to stop processing. Examples are IP blocking, or serving custom endpoints for statistics or to invalidate a cache entry.
 
+* `beforeCacheLookUp` is called immedidiately before looking the object up in the cache. It can be used to modify the cacheKey to be used to for this object using the passed `CacheKeyOverrideFunc` func. Once set using that function Grove will keep using that cacheKey throughout the life of the object in the cache.
+
 * `beforeParentRequest` is called immediately before making a request to a parent. It may manipulate the request being made to the parent. Examples are removing headers in the client request such as `Range`.
 
 * `beforeRespond` is called immediately before responding to a client. It may manipulate the code, headers, and body being returned. Examples are header modifications, or handling if-modified-since requests.
diff --git a/grove/plugin/plugin.go b/grove/plugin/plugin.go
index 32d1280..140c4e4 100644
--- a/grove/plugin/plugin.go
+++ b/grove/plugin/plugin.go
@@ -93,9 +93,10 @@ type BeforeRespondData struct {
 }
 
 type BeforeCacheLookUpData struct {
-	Req             *http.Request
-	DefaultCacheKey string
-	Context         *interface{}
+	Req                  *http.Request
+	CacheKeyOverrideFunc func(string)
+	DefaultCacheKey      string
+	Context              *interface{}
 }
 
 type AfterRespondData struct {
@@ -112,7 +113,7 @@ type AfterRespondData struct {
 type LoadFunc func(json.RawMessage) interface{}
 type StartupFunc func(icfg interface{}, d StartupData)
 type OnRequestFunc func(icfg interface{}, d OnRequestData) bool
-type BeforeCacheLookupFunc func(icfg interface{}, d BeforeCacheLookUpData, cacheKeyOverRideFunc func(string))
+type BeforeCacheLookupFunc func(icfg interface{}, d BeforeCacheLookUpData)
 type BeforeParentRequestFunc func(icfg interface{}, d BeforeParentRequestData)
 type BeforeRespondFunc func(icfg interface{}, d BeforeRespondData)
 type AfterRespondFunc func(icfg interface{}, d AfterRespondData)
@@ -140,7 +141,7 @@ type Plugins interface {
 	LoadFuncs() map[string]LoadFunc
 	OnStartup(cfgs map[string]interface{}, context map[string]*interface{}, d StartupData)
 	OnRequest(cfgs map[string]interface{}, context map[string]*interface{}, d OnRequestData) bool
-	OnBeforeCacheLookup(cfgs map[string]interface{}, context map[string]*interface{}, d BeforeCacheLookUpData, cacheKeyOverrideFunc func(string))
+	OnBeforeCacheLookup(cfgs map[string]interface{}, context map[string]*interface{}, d BeforeCacheLookUpData)
 	OnBeforeParentRequest(cfgs map[string]interface{}, context map[string]*interface{}, d BeforeParentRequestData)
 	OnBeforeRespond(cfgs map[string]interface{}, context map[string]*interface{}, d BeforeRespondData)
 	OnAfterRespond(cfgs map[string]interface{}, context map[string]*interface{}, d AfterRespondData)
@@ -184,13 +185,13 @@ func (ps pluginsSlice) OnRequest(cfgs map[string]interface{}, context map[string
 	return false
 }
 
-func (ps pluginsSlice) OnBeforeCacheLookup(cfgs map[string]interface{}, context map[string]*interface{}, d BeforeCacheLookUpData, cacheKeyOverrideFunc func(string)) {
+func (ps pluginsSlice) OnBeforeCacheLookup(cfgs map[string]interface{}, context map[string]*interface{}, d BeforeCacheLookUpData) {
 	for _, p := range ps {
 		if p.funcs.beforeCacheLookUp == nil {
 			continue
 		}
 		d.Context = context[p.name]
-		p.funcs.beforeCacheLookUp(cfgs[p.name], d, cacheKeyOverrideFunc)
+		p.funcs.beforeCacheLookUp(cfgs[p.name], d)
 	}
 }
 
diff --git a/grove/plugin/range_req_handler.go b/grove/plugin/range_req_handler.go
index 354a6cc..0bd8e0c 100644
--- a/grove/plugin/range_req_handler.go
+++ b/grove/plugin/range_req_handler.go
@@ -16,7 +16,6 @@ package plugin
 
 import (
 	"crypto/rand"
-	"encoding/hex"
 	"encoding/json"
 	"fmt"
 	"net/http"
@@ -33,7 +32,8 @@ type byteRange struct {
 }
 
 type rangeRequestConfig struct {
-	Mode string `json::"mode"`
+	Mode              string `json:"mode"`
+	MultiPartBoundary string // not in the json
 }
 
 func init() {
@@ -49,7 +49,7 @@ func init() {
 // rangeReqHandleLoad loads the configuration
 func rangeReqHandleLoad(b json.RawMessage) interface{} {
 	cfg := rangeRequestConfig{}
-	log.Errorf("rangeReqHandleLoad loading: %s", b)
+	log.Errorf("rangeReqHandleLoad loading: %s\n", b)
 
 	err := json.Unmarshal(b, &cfg)
 	if err != nil {
@@ -57,8 +57,16 @@ func rangeReqHandleLoad(b json.RawMessage) interface{} {
 		return nil
 	}
 	if !(cfg.Mode == "get_full_serve_range" || cfg.Mode == "patch") {
-		log.Errorf("Unknown mode for range_req_handler plugin: %s", cfg.Mode)
+		log.Errorf("Unknown mode for range_req_handler plugin: %s\n", cfg.Mode)
 	}
+
+	multipartBoundaryBytes := make([]byte, 16)
+	if _, err := rand.Read(multipartBoundaryBytes); err != nil {
+		log.Errorf("Error with rand.Read: %v\n", err)
+	}
+	// Use UUID format
+	cfg.MultiPartBoundary = fmt.Sprintf("%x-%x-%x-%x-%x", multipartBoundaryBytes[0:4], multipartBoundaryBytes[4:6], multipartBoundaryBytes[6:8], multipartBoundaryBytes[8:10], multipartBoundaryBytes[10:])
+
 	log.Debugf("range_rew_handler: load success: %+v\n", cfg)
 	return &cfg
 }
@@ -67,17 +75,18 @@ func rangeReqHandleLoad(b json.RawMessage) interface{} {
 func rangeReqHandlerOnRequest(icfg interface{}, d OnRequestData) bool {
 	rHeader := d.R.Header.Get("Range")
 	if rHeader == "" {
-		log.Debugf("No Range header found")
+		log.Debugf("No Range header found\n")
 		return false
 	}
-	log.Debugf("Range string is: %s", rHeader)
+	log.Debugf("Range string is: %s\n", rHeader)
 	// put the ranges [] in the context so we can use it later
 	byteRanges := parseRangeHeader(rHeader)
 	*d.Context = byteRanges
 	return false
 }
 
-func rangeReqHandleBeforeCacheLookup(icfg interface{}, d BeforeCacheLookUpData, overRideFunc func(string)) {
+// rangeReqHandleBeforeCacheLookup is used to override the cacheKey when in store_ranges mode.
+func rangeReqHandleBeforeCacheLookup(icfg interface{}, d BeforeCacheLookUpData) {
 	cfg, ok := icfg.(*rangeRequestConfig)
 	if !ok {
 		log.Errorf("range_req_handler config '%v' type '%T' expected *rangeRequestConfig\n", icfg, icfg)
@@ -89,8 +98,8 @@ func rangeReqHandleBeforeCacheLookup(icfg interface{}, d BeforeCacheLookUpData,
 			sep = "&"
 		}
 		newKey := d.DefaultCacheKey + sep + "grove_range_req_handler_plugin_data=" + d.Req.Header.Get("Range")
-		overRideFunc(newKey)
-		log.Debugf("range_req_handler: store_ranges default key:%s, new key:%s", d.DefaultCacheKey, newKey)
+		d.CacheKeyOverrideFunc(newKey)
+		log.Debugf("range_req_handler: store_ranges default key:%s, new key:%s\n", d.DefaultCacheKey, newKey)
 	}
 }
 
@@ -99,10 +108,10 @@ func rangeReqHandleBeforeParent(icfg interface{}, d BeforeParentRequestData) {
 	log.Debugf("rangeReqHandleBeforeParent calling.")
 	rHeader := d.Req.Header.Get("Range")
 	if rHeader == "" {
-		log.Debugf("No Range header found")
+		log.Debugln("No Range header found")
 		return
 	}
-	log.Debugf("Range string is: %s", rHeader)
+	log.Debugf("Range string is: %s\n", rHeader)
 	cfg, ok := icfg.(*rangeRequestConfig)
 	if !ok {
 		log.Errorf("range_req_handler config '%v' type '%T' expected *rangeRequestConfig\n", icfg, icfg)
@@ -117,12 +126,13 @@ func rangeReqHandleBeforeParent(icfg interface{}, d BeforeParentRequestData) {
 
 // rangeReqHandleBeforeRespond builds the 206 response
 // Assume all the needed ranges have been put in cache before, which is the truth for "get_full_serve_range" mode which gets the whole object into cache.
+// If mode == store_ranges, do nothing, we just return the object stored-as is
 func rangeReqHandleBeforeRespond(icfg interface{}, d BeforeRespondData) {
 	log.Debugf("rangeReqHandleBeforeRespond calling\n")
 	ictx := d.Context
 	ctx, ok := (*ictx).([]byteRange)
 	if !ok {
-		log.Errorf("Invalid context: %v", ictx)
+		log.Errorf("Invalid context: %v\n", ictx)
 	}
 	if len(ctx) == 0 {
 		return // there was no (valid) range header
@@ -137,35 +147,34 @@ func rangeReqHandleBeforeRespond(icfg interface{}, d BeforeRespondData) {
 		return // no need to do anything here.
 	}
 
-	multipartBoundaryString := ""
+	// mode != store_ranges
+	multipartBoundaryString := cfg.MultiPartBoundary
+	multipart := false
 	originalContentType := d.Hdr.Get("Content-type")
 	*d.Hdr = web.CopyHeader(*d.Hdr) // copy the headers, we don't want to mod the cacheObj
 	if len(ctx) > 1 {
-		//multipart = true
-		multipartBoundaryBytes := make([]byte, 8)
-		if _, err := rand.Read(multipartBoundaryBytes); err != nil {
-			log.Errorf("Error with rand.Read: %v", err)
-		}
-		multipartBoundaryString = hex.EncodeToString(multipartBoundaryBytes)
+		multipart = true
+		multipartBoundaryString = cfg.MultiPartBoundary
 		d.Hdr.Set("Content-Type", fmt.Sprintf("multipart/byteranges; boundary=%s", multipartBoundaryString))
 	}
 	totalContentLength, err := strconv.ParseInt(d.Hdr.Get("Content-Length"), 10, 64)
 	if err != nil {
-		log.Errorf("Invalid Content-Length header: %v", d.Hdr.Get("Content-Length"))
+		log.Errorf("Invalid Content-Length header: %v\n", d.Hdr.Get("Content-Length"))
 	}
 	body := make([]byte, 0)
 	for _, thisRange := range ctx {
 		if thisRange.End == -1 || thisRange.End >= totalContentLength { // if the end range is "", or too large serve until the end
 			thisRange.End = totalContentLength - 1
 		}
-		log.Debugf("range:%d-%d", thisRange.Start, thisRange.End)
-		if multipartBoundaryString != "" {
-			body = append(body, []byte(fmt.Sprintf("\r\n--%s\r\n", multipartBoundaryString))...)
-			body = append(body, []byte(fmt.Sprintf("Content-type: %s\r\n", originalContentType))...)
-			body = append(body, []byte(fmt.Sprintf("Content-range: bytes %d-%d/%d\r\n\r\n", thisRange.Start, thisRange.End, totalContentLength))...)
+
+		rangeString := "bytes " + strconv.FormatInt(thisRange.Start, 10) + "-" + strconv.FormatInt(thisRange.End, 10) + "\r\n\r\n"
+		log.Debugf("range:%d-%d\n", thisRange.Start, thisRange.End)
+		if multipart {
+			body = append(body, []byte("\r\n--"+multipartBoundaryString+"\r\n")...)
+			body = append(body, []byte("Content-type: "+originalContentType+"%s\r\n")...)
+			body = append(body, []byte("Content-range: "+rangeString)...)
 		} else {
-			byteRangeString := fmt.Sprintf("bytes %d-%d/%d", thisRange.Start, thisRange.End, totalContentLength)
-			d.Hdr.Add("Content-Range", byteRangeString)
+			d.Hdr.Add("Content-Range", rangeString+"/"+strconv.FormatInt(totalContentLength, 10))
 		}
 		bSlice := (*d.Body)[thisRange.Start : thisRange.End+1]
 		body = append(body, bSlice...)
@@ -188,7 +197,7 @@ func parseRange(rangeString string) (byteRange, error) {
 	} else {
 		start, err := strconv.ParseInt(parts[0], 10, 64)
 		if err != nil {
-			log.Errorf("Error converting rangeString start \"%\" to numbers", rangeString)
+			log.Errorf("Error converting rangeString start \"%\" to numbers\n", rangeString)
 			return byteRange{}, err
 		}
 		bRange.Start = start
@@ -198,7 +207,7 @@ func parseRange(rangeString string) (byteRange, error) {
 	} else {
 		end, err := strconv.ParseInt(parts[1], 10, 64)
 		if err != nil {
-			log.Errorf("Error converting rangeString end \"%\" to numbers", rangeString)
+			log.Errorf("Error converting rangeString end \"%\" to numbers\n", rangeString)
 			return byteRange{}, err
 		}
 		bRange.End = end
@@ -210,11 +219,10 @@ func parseRangeHeader(rHdrVal string) []byteRange {
 	byteRanges := make([]byteRange, 0)
 	rangeStringParts := strings.Split(rHdrVal, "=")
 	if rangeStringParts[0] != "bytes" {
-		log.Errorf("Not a valid Range type: \"%s\"", rangeStringParts[0])
+		log.Errorf("Not a valid Range type: \"%s\"\n", rangeStringParts[0])
 	}
 
 	for _, thisRangeString := range strings.Split(rangeStringParts[1], ",") {
-		log.Debugf("bRangeStr: %s", thisRangeString)
 		thisRange, err := parseRange(thisRangeString)
 		if err != nil {
 			return nil

-- 
To stop receiving notification emails like this one, please contact
rob@apache.org.