You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@trafficcontrol.apache.org by ne...@apache.org on 2017/04/19 15:54:53 UTC
[1/2] incubator-trafficcontrol git commit: Add TM2 HTTP gzip support
Repository: incubator-trafficcontrol
Updated Branches:
refs/heads/master 2853a5e9e -> d4a4eeecd
Add TM2 HTTP gzip support
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafficcontrol/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafficcontrol/commit/56fbacc9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafficcontrol/tree/56fbacc9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafficcontrol/diff/56fbacc9
Branch: refs/heads/master
Commit: 56fbacc98bcad264008117b934dd60d8fa0754d2
Parents: 2853a5e
Author: Robert Butts <ro...@gmail.com>
Authored: Mon Apr 3 22:13:33 2017 -0600
Committer: David Neuman <da...@gmail.com>
Committed: Wed Apr 19 09:54:30 2017 -0600
----------------------------------------------------------------------
.../traffic_monitor/datareq/datareq.go | 105 +++++++++++++++++--
1 file changed, 96 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafficcontrol/blob/56fbacc9/traffic_monitor_golang/traffic_monitor/datareq/datareq.go
----------------------------------------------------------------------
diff --git a/traffic_monitor_golang/traffic_monitor/datareq/datareq.go b/traffic_monitor_golang/traffic_monitor/datareq/datareq.go
index a5862b4..9e42f1a 100644
--- a/traffic_monitor_golang/traffic_monitor/datareq/datareq.go
+++ b/traffic_monitor_golang/traffic_monitor/datareq/datareq.go
@@ -20,11 +20,14 @@
package datareq
import (
+ "bytes"
+ "compress/gzip"
"fmt"
"net/http"
"net/url"
"strings"
"time"
+ "unicode"
"github.com/apache/incubator-trafficcontrol/traffic_monitor_golang/common/log"
"github.com/apache/incubator-trafficcontrol/traffic_monitor_golang/traffic_monitor/config"
@@ -157,8 +160,20 @@ func WrapErrCode(errorCount threadsafe.Uint, reqPath string, body []byte, err er
// WrapBytes takes a function which cannot error and returns only bytes, and wraps it as a http.HandlerFunc. The errContext is logged if the write fails, and should be enough information to trace the problem (function name, endpoint, request parameters, etc).
func WrapBytes(f func() []byte, contentType string) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
+ bytes := f()
+ bytes, err := gzipIfAccepts(r, w, bytes)
+ if err != nil {
+ log.Errorf("gzipping request '%v': %v\n", r.URL.EscapedPath(), err)
+ code := http.StatusInternalServerError
+ w.WriteHeader(code)
+ if _, err := w.Write([]byte(http.StatusText(code))); err != nil {
+ log.Warnf("received error writing data request %v: %v\n", r.URL.EscapedPath(), err)
+ }
+ return
+ }
+
w.Header().Set("Content-Type", contentType)
- log.Write(w, f(), r.URL.EscapedPath())
+ log.Write(w, bytes, r.URL.EscapedPath())
}
}
@@ -166,7 +181,13 @@ func WrapBytes(f func() []byte, contentType string) http.HandlerFunc {
func WrapErr(errorCount threadsafe.Uint, f func() ([]byte, error), contentType string) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
bytes, err := f()
- _, code := WrapErrCode(errorCount, r.URL.EscapedPath(), bytes, err)
+ code := http.StatusOK
+ if err != nil {
+ bytes, code = WrapErrCode(errorCount, r.URL.EscapedPath(), bytes, err)
+ } else {
+ bytes, err = gzipIfAccepts(r, w, bytes)
+ bytes, code = WrapErrCode(errorCount, r.URL.EscapedPath(), bytes, err)
+ }
w.Header().Set("Content-Type", contentType)
w.WriteHeader(code)
log.Write(w, bytes, r.URL.EscapedPath())
@@ -182,17 +203,30 @@ type SrvFunc func(params url.Values, path string) ([]byte, int)
func WrapParams(f SrvFunc, contentType string) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
bytes, code := f(r.URL.Query(), r.URL.EscapedPath())
- if len(bytes) > 0 {
- w.Header().Set("Content-Type", contentType)
+ if len(bytes) == 0 {
+ code := http.StatusInternalServerError
w.WriteHeader(code)
- if _, err := w.Write(bytes); err != nil {
+ if _, err := w.Write([]byte(http.StatusText(code))); err != nil {
log.Warnf("received error writing data request %v: %v\n", r.URL.EscapedPath(), err)
}
- } else {
- w.WriteHeader(http.StatusInternalServerError)
- if _, err := w.Write([]byte("Internal Server Error")); err != nil {
+ return
+ }
+
+ bytes, err := gzipIfAccepts(r, w, bytes)
+ if err != nil {
+ log.Errorf("gzipping '%v': %v\n", r.URL.EscapedPath(), err)
+ code := http.StatusInternalServerError
+ w.WriteHeader(code)
+ if _, err := w.Write([]byte(http.StatusText(code))); err != nil {
log.Warnf("received error writing data request %v: %v\n", r.URL.EscapedPath(), err)
}
+ return
+ }
+
+ w.Header().Set("Content-Type", contentType)
+ w.WriteHeader(code)
+ if _, err := w.Write(bytes); err != nil {
+ log.Warnf("received error writing data request %v: %v\n", r.URL.EscapedPath(), err)
}
}
}
@@ -200,10 +234,18 @@ func WrapParams(f SrvFunc, contentType string) http.HandlerFunc {
func WrapAgeErr(errorCount threadsafe.Uint, f func() ([]byte, time.Time, error), contentType string) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
bytes, contentTime, err := f()
- _, code := WrapErrCode(errorCount, r.URL.EscapedPath(), bytes, err)
+ code := http.StatusOK
+ if err != nil {
+ bytes, code = WrapErrCode(errorCount, r.URL.EscapedPath(), bytes, err)
+ } else {
+ bytes, err = gzipIfAccepts(r, w, bytes)
+ bytes, code = WrapErrCode(errorCount, r.URL.EscapedPath(), bytes, err)
+ }
+
w.Header().Set("Content-Type", contentType)
w.Header().Set("Age", fmt.Sprintf("%.0f", time.Since(contentTime).Seconds()))
w.WriteHeader(code)
+
log.Write(w, bytes, r.URL.EscapedPath())
}
}
@@ -225,6 +267,15 @@ func wrapUnpolledCheck(unpolledCaches threadsafe.UnpolledCaches, errorCount thre
const ContentTypeJSON = "application/json"
+func stripAllWhitespace(s string) string {
+ return strings.Map(func(r rune) rune {
+ if unicode.IsSpace(r) {
+ return -1
+ }
+ return r
+ }, s)
+}
+
// addTrailingEndpoints adds endpoints with trailing slashes to the given dispatch map. Without this, Go will match `route` and `route/` differently.
func addTrailingSlashEndpoints(dispatchMap map[string]http.HandlerFunc) map[string]http.HandlerFunc {
for route, handler := range dispatchMap {
@@ -235,3 +286,39 @@ func addTrailingSlashEndpoints(dispatchMap map[string]http.HandlerFunc) map[stri
}
return dispatchMap
}
+
+func acceptsGzip(r *http.Request) bool {
+ encodingHeaders := r.Header["Accept-Encoding"] // headers are case-insensitive, but Go promises to Canonical-Case requests
+ for _, encodingHeader := range encodingHeaders {
+ encodingHeader = stripAllWhitespace(encodingHeader)
+ encodings := strings.Split(encodingHeader, ",")
+ for _, encoding := range encodings {
+ if strings.ToLower(encoding) == "gzip" { // encoding is case-insensitive, per the RFC
+ return true
+ }
+ }
+ }
+ return false
+}
+
+// gzipIfAccepts gzips the given bytes, writes a `Content-Encoding: gzip` header to the given writer, and returns the gzipped bytes, if the Request supports GZip (has an Accept-Encoding header). Else, returns the bytes unmodified. Note the given bytes are NOT written to the given writer. It is assumed the bytes may need to pass thru other middleware before being written.
+func gzipIfAccepts(r *http.Request, w http.ResponseWriter, b []byte) ([]byte, error) {
+ // TODO this could be made more efficient by wrapping ResponseWriter with the GzipWriter, and letting callers writer directly to it - but then we'd have to deal with Closing the gzip.Writer.
+ if len(b) == 0 || !acceptsGzip(r) {
+ return b, nil
+ }
+ w.Header().Set("Content-Encoding", "gzip")
+
+ buf := bytes.Buffer{}
+ zw := gzip.NewWriter(&buf)
+
+ if _, err := zw.Write(b); err != nil {
+ return nil, fmt.Errorf("gzipping bytes: %v")
+ }
+
+ if err := zw.Close(); err != nil {
+ return nil, fmt.Errorf("closing gzip writer: %v")
+ }
+
+ return buf.Bytes(), nil
+}
[2/2] incubator-trafficcontrol git commit: This closes #425
Posted by ne...@apache.org.
This closes #425
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafficcontrol/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafficcontrol/commit/d4a4eeec
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafficcontrol/tree/d4a4eeec
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafficcontrol/diff/d4a4eeec
Branch: refs/heads/master
Commit: d4a4eeecd911f48775f4477f39107c5c9faee1e3
Parents: 56fbacc
Author: David Neuman <da...@gmail.com>
Authored: Wed Apr 19 09:54:41 2017 -0600
Committer: David Neuman <da...@gmail.com>
Committed: Wed Apr 19 09:54:41 2017 -0600
----------------------------------------------------------------------
----------------------------------------------------------------------