You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@trafficcontrol.apache.org by ro...@apache.org on 2018/06/04 20:49:45 UTC
[incubator-trafficcontrol] 02/15: RR tests
This is an automated email from the ASF dual-hosted git repository.
rob pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-trafficcontrol.git
commit 2418ed7c4beff63529544abf17da7987d90e2b5e
Author: Jan van Doorn <jv...@knutsel.com>
AuthorDate: Wed May 23 12:43:10 2018 -0600
RR tests
---
grove/integration_test/Dockerfile | 26 ++++
grove/integration_test/compare_gets | Bin 0 -> 5782484 bytes
grove/integration_test/compare_gets.go | 160 +++++++++++++++++++++
grove/integration_test/grove.cfg | 32 +++++
grove/integration_test/remap-base-test.json | 152 ++++++++++++++++++++
grove/integration_test/setup-and-run.sh | 45 ++++++
.../tests/plugins/range_req_handler/remap.json | 152 ++++++++++++++++++++
.../tests/plugins/range_req_handler/test.sh | 42 ++++++
8 files changed, 609 insertions(+)
diff --git a/grove/integration_test/Dockerfile b/grove/integration_test/Dockerfile
new file mode 100644
index 0000000..84d43cb
--- /dev/null
+++ b/grove/integration_test/Dockerfile
@@ -0,0 +1,26 @@
+FROM centos:7
+MAINTAINER The CentOS Project <cl...@centos.org>
+LABEL Vendor="CentOS" \
+ License=GPLv2 \
+ Version=2.4.6-40
+
+
+RUN yum -y --setopt=tsflags=nodocs update && \
+ yum -y --setopt=tsflags=nodocs install httpd && \
+ yum -y --setopt=tsflags=nodocs install perl && \
+ yum -y --setopt=tsflags=nodocs install git && \
+ yum -y --setopt=tsflags=nodocs install golang && \
+ yum -y --setopt=tsflags=nodocs install openssl && \
+ yum clean all
+
+#EXPOSE 80
+
+# Simple startup script to avoid some issues observed with container restart
+ADD setup-and-run.sh setup-and-run.sh /
+RUN chmod -v +x /setup-and-run.sh
+ADD remap-base-test.json /remap-base-test.json
+ADD grove.cfg /grove.cfg
+ADD tests /tests
+ADD compare_gets.go /compare_gets.go
+
+CMD ["/setup-and-run.sh"]
diff --git a/grove/integration_test/compare_gets b/grove/integration_test/compare_gets
new file mode 100755
index 0000000..c2a86a6
Binary files /dev/null and b/grove/integration_test/compare_gets differ
diff --git a/grove/integration_test/compare_gets.go b/grove/integration_test/compare_gets.go
new file mode 100644
index 0000000..385df26
--- /dev/null
+++ b/grove/integration_test/compare_gets.go
@@ -0,0 +1,160 @@
+package main
+
+import (
+ "flag"
+ "fmt"
+ "github.com/apache/incubator-trafficcontrol/grove/web"
+ "io/ioutil"
+ "log"
+ "net/http"
+ "os"
+ "strings"
+)
+
+type responseType struct {
+ Headers http.Header
+ Body []byte
+}
+
+func httpGet(URL, headers string) responseType {
+ client := &http.Client{}
+ req, err := http.NewRequest("GET", URL, nil)
+ if err != nil {
+ fmt.Println("ERROR in httpGet")
+ }
+ //log.Printf(">>>%v<<< %v\n", headers, len(strings.Split(headers, ".")))
+ for _, hdrString := range strings.Split(headers, " ") {
+ //log.Println(">>> ", hdrString)
+ if hdrString == "" {
+ continue
+ }
+ parts := strings.Split(hdrString, ":")
+ if parts[0] == "Host" {
+ req.Host = parts[1]
+ } else {
+ //log.Println("> ", parts)
+ req.Header.Set(parts[0], parts[1])
+ }
+ }
+ //log.Printf(">>>> %v", req)
+ resp, err := client.Do(req)
+ if err != nil {
+ fmt.Println("ERROR in httpGet")
+ }
+ defer resp.Body.Close()
+ var response responseType
+ response.Headers = web.CopyHeader(resp.Header)
+ response.Body, err = ioutil.ReadAll(resp.Body)
+ if err != nil {
+ fmt.Println("ERROR in httpGet (readall)")
+ }
+ return response
+}
+
+func equalBodies(a, b []byte) bool {
+ if a == nil || b == nil {
+ return false
+ }
+
+ if a == nil && b == nil {
+ return true
+ }
+ if len(a) != len(b) {
+ return false
+ }
+
+ for i := range a {
+ if a[i] != b[i] {
+ return false
+ }
+ }
+
+ return true
+}
+
+func equalStringSlices(a, b []string) bool {
+ if a == nil || b == nil {
+ return false
+ }
+
+ if a == nil && b == nil {
+ return true
+ }
+ if len(a) != len(b) {
+ return false
+ }
+
+ for i := range a {
+ if a[i] != b[i] {
+ return false
+ }
+ }
+
+ return true
+}
+
+func inStringSlice(str string, arr []string) bool {
+ for _, strEnt := range arr {
+ if strEnt == str {
+ return true
+ }
+ }
+ return false
+}
+
+func compareResponses(response1 responseType, response2 responseType, ignoreHdrs []string, ignoreMPB bool) bool {
+
+ if ignoreMPB {
+ contentTypeHdr := response1.Headers.Get("Content-type")
+ fmt.Println("ignoreing", contentTypeHdr, response1)
+ if strings.HasPrefix(contentTypeHdr, "multipart/byteranges") {
+ parts := strings.Split(contentTypeHdr, "=")
+ MPBoundary := parts[1]
+ //log.Println("+++")
+ //log.Printf("%s\n", string(response1.Body))
+ response1.Body = []byte(strings.Replace(string(response1.Body), MPBoundary, "", -1))
+ //log.Printf("%s\n", response1.Body)
+ }
+ contentTypeHdr = response2.Headers.Get("Content-type")
+ if strings.HasPrefix(contentTypeHdr, "multipart/byteranges") {
+ parts := strings.Split(contentTypeHdr, "=")
+ MPBoundary := parts[1]
+ response2.Body = []byte(strings.Replace(string(response2.Body), MPBoundary, "", -1))
+ }
+ }
+ if !equalBodies(response1.Body, response2.Body) {
+ return false
+ }
+ for hdrKey, _ := range response1.Headers {
+ if inStringSlice(hdrKey, ignoreHdrs) {
+ continue
+ }
+ if !equalStringSlices(response1.Headers[hdrKey], response2.Headers[hdrKey]) {
+ log.Printf("ERROR hdr %v doesn't match: \"%v\" != \"%v\"\n", hdrKey, response1.Headers[hdrKey], response2.Headers[hdrKey])
+ return false
+ }
+ //fmt.Printf(">>>>> %v\n", hdrKey)
+ }
+
+ return true
+}
+func main() {
+ originURL := flag.String("org", "http://localhost", "The origin URL (default: \"http://localhost\")")
+ cacheURL := flag.String("cache", "http://localhost:8080", "The cache URL (default: \"http://localhost:8080\")")
+ path := flag.String("path", "", "The path to GET")
+ orgHdrs := flag.String("ohdrs", "", "Comma seperated list of headers to add to origin request")
+ cacheHdrs := flag.String("chdrs", "", "Comma separated list of headers to add to cache request")
+ ignoreHdrs := flag.String("ignorehdrs", "Server,Date", "Comma separated list of headers to ignore in the compare")
+ ignoreMultiPartBoundary := flag.Bool("ignorembp", true, "Ignore multi part boundary in body comparison.")
+ flag.Parse()
+
+ resp := httpGet(*originURL+"/"+*path, *orgHdrs)
+ cresp := httpGet(*cacheURL+"/"+*path, *cacheHdrs)
+ if !compareResponses(resp, cresp, strings.Split(*ignoreHdrs, ","), *ignoreMultiPartBoundary) {
+ fmt.Println("FAIL: Body bytes don't match \n%s\n != \n%s\n", string(resp.Body), string(cresp.Body))
+ os.Exit(1)
+
+ }
+ fmt.Println("PASS")
+ os.Exit(0)
+}
diff --git a/grove/integration_test/grove.cfg b/grove/integration_test/grove.cfg
new file mode 100644
index 0000000..b5af0f0
--- /dev/null
+++ b/grove/integration_test/grove.cfg
@@ -0,0 +1,32 @@
+{
+ "rfc_compliant":false,
+ "port":8080,
+ "https_port":8443,
+ "cache_size_bytes":10000000,
+ "cache_files":{
+ "disk":[
+ {
+ "path":"/diskcachefile0.db",
+ "size_bytes":100000000
+ },
+ {
+ "path":"/diskcachefile1.db",
+ "size_bytes":100000000
+ }
+ ],
+ "my-disk-cache-two":[
+ {
+ "path":"/singlefilecache.db",
+ "size_bytes":10000000
+ }
+ ]
+ },
+ "remap_rules_file":"./remap.json",
+ "log_location_error":"./error.log",
+ "log_location_warning":"./error.log",
+ "log_location_info":"./error.log",
+ "log_location_debug":"./error.log",
+ "log_location_event":"./custom_ats_2.log",
+ "cert_file":"./cert.pem",
+ "key_file":"./key.pem"
+}
diff --git a/grove/integration_test/remap-base-test.json b/grove/integration_test/remap-base-test.json
new file mode 100644
index 0000000..4cf49e5
--- /dev/null
+++ b/grove/integration_test/remap-base-test.json
@@ -0,0 +1,152 @@
+{
+ "parent_selection": "consistent-hash",
+ "plugins": {
+ "modify_response_headers_global": {
+ "set": [
+ {
+ "name": "Server",
+ "value": "Grove/0.39999999"
+ }
+ ]
+ }
+ },
+ "retry_codes": null,
+ "retry_num": null,
+ "rules": [
+ {
+ "allow": null,
+ "certificate-file": "",
+ "certificate-key-file": "",
+ "concurrent_rule_requests": 0,
+ "connection-close": false,
+ "deny": null,
+ "from": "http://disk-test.cdn.kabletown.net",
+ "name": "jvd-test",
+ "parent_selection": "consistent-hash",
+ "cache_name": "disk",
+ "plugins": {
+ "modify_parent_request_headers": {
+ "set": [
+ {
+ "name": "X-From-CDN",
+ "value": "Traffic-Control"
+ }
+ ]
+ },
+ "modify_headers": {
+ "set": [
+ {
+ "name": "X-CDN-name",
+ "value": "JvD-Grove"
+ }
+ ]
+ },
+ "range_req_handler": {
+ "mode": "get_full_serve_range"
+ }
+ },
+ "query-string": {
+ "cache": true,
+ "remap": true
+ },
+ "retry_codes": [],
+ "retry_num": 5,
+ "timeout_ms": 5000000,
+ "to": [
+ {
+ "retry_codes": [],
+ "retry_num": 0,
+ "timeout_ms": 5000000,
+ "url": "http://localhost",
+ "weight": 1
+ }
+ ]
+ },
+ {
+ "allow": null,
+ "certificate-file": "",
+ "certificate-key-file": "",
+ "concurrent_rule_requests": 0,
+ "connection-close": false,
+ "deny": null,
+ "from": "http://disk1-test.cdn.kabletown.net",
+ "name": "jvd1-test",
+ "parent_selection": "consistent-hash",
+ "cache_name": "disk",
+ "plugins": {
+ "modify_parent_request_headers": {
+ "set": [
+ {
+ "name": "X-From-CDN",
+ "value": "Traffic-Control"
+ }
+ ]
+ },
+ "modify_headers": {
+ "set": [
+ {
+ "name": "X-CDN-name",
+ "value": "JvD-Grove"
+ }
+ ]
+ }
+ },
+ "query-string": {
+ "cache": true,
+ "remap": true
+ },
+ "retry_codes": [],
+ "retry_num": 5,
+ "timeout_ms": 5000000,
+ "to": [
+ {
+ "retry_codes": [],
+ "retry_num": 0,
+ "timeout_ms": 5000000,
+ "url": "http://localhost",
+ "weight": 1
+ }
+ ]
+ },
+ {
+ "allow": null,
+ "certificate-file": "",
+ "certificate-key-file": "",
+ "concurrent_rule_requests": 0,
+ "connection-close": false,
+ "deny": null,
+ "from": "http://mem-test.cdn.kabletown.net",
+ "name": "cim-linear-hds.http.http.cim-linear-hds",
+ "parent_selection": "consistent-hash",
+ "query-string": {
+ "cache": true,
+ "remap": true
+ },
+ "retry_codes": [],
+ "retry_num": 5,
+ "timeout_ms": 5000,
+ "plugins": {
+ "range_req_handler": {
+ "mode": "store_ranges"
+ }
+ },
+ "to": [
+ {
+ "retry_codes": [],
+ "retry_num": 0,
+ "timeout_ms": 5000000,
+ "url": "http://localhost",
+ "weight": 1
+ }
+ ]
+ }
+ ],
+ "stats": {
+ "allow": [
+ "127.0.0.1/32",
+ "::1/128"
+ ],
+ "deny": null
+ },
+ "timeout_ms": 5000
+}
diff --git a/grove/integration_test/setup-and-run.sh b/grove/integration_test/setup-and-run.sh
new file mode 100644
index 0000000..c9abf6b
--- /dev/null
+++ b/grove/integration_test/setup-and-run.sh
@@ -0,0 +1,45 @@
+#!/usr/bin/bash
+echo Configuring Grove Integration Test Environment.
+
+#perl -v
+#ls -l /var/www/html
+cd /var/www/html
+echo Generating origin test files...
+perl -e 'foreach $i ( 0 ... 1024*1024-1 ) { printf "%09d\n", $i*10 }' > 10Mb.txt
+for i in {1..1000} ; do dd if=/dev/urandom of=${i}k.bin bs=${i}k count=1 > /dev/null 2>&1 ; done
+httpd
+
+cd /
+echo Setting up go enviroment...
+export GOPATH=~/go
+go get golang.org/x/text
+go get golang.org/x/sys/unix
+go get golang.org/x/net/http2
+go get golang.org/x/net/ipv4
+go get golang.org/x/net/ipv6
+
+mkdir -p $GOPATH/src/github.com/apache/
+cd $GOPATH/src/github.com/apache/
+#git clone https://github.com/apache/incubator-trafficcontrol
+git clone $REPO
+cd $GOPATH/src/github.com/apache/incubator-trafficcontrol/grove
+git checkout $BRANCH
+go build
+
+cd /
+openssl req -newkey rsa:2048 -new -nodes -x509 -days 365 -keyout key.pem -out cert.pem -subj "/C=US/ST=CO/L=Denver/O=.../OU=.../CN=.../emailAddress=..."
+
+cp /remap-base-test.json /remap.json
+ls -l
+${GOPATH}/src/github.com/apache/incubator-trafficcontrol/grove/grove -cfg grove.cfg &
+
+
+sleep 3
+curl -H'Host: mem-test.cdn.kabletown.net' -Lsv -r 50000-50009 http://localhost:8080/10Mb.txt
+
+#cd $GOPATH/src/github.com/apache/incubator-trafficcontrol/grove/integration_test
+go build compare_gets.go
+
+
+
+
diff --git a/grove/integration_test/tests/plugins/range_req_handler/remap.json b/grove/integration_test/tests/plugins/range_req_handler/remap.json
new file mode 100644
index 0000000..4cf49e5
--- /dev/null
+++ b/grove/integration_test/tests/plugins/range_req_handler/remap.json
@@ -0,0 +1,152 @@
+{
+ "parent_selection": "consistent-hash",
+ "plugins": {
+ "modify_response_headers_global": {
+ "set": [
+ {
+ "name": "Server",
+ "value": "Grove/0.39999999"
+ }
+ ]
+ }
+ },
+ "retry_codes": null,
+ "retry_num": null,
+ "rules": [
+ {
+ "allow": null,
+ "certificate-file": "",
+ "certificate-key-file": "",
+ "concurrent_rule_requests": 0,
+ "connection-close": false,
+ "deny": null,
+ "from": "http://disk-test.cdn.kabletown.net",
+ "name": "jvd-test",
+ "parent_selection": "consistent-hash",
+ "cache_name": "disk",
+ "plugins": {
+ "modify_parent_request_headers": {
+ "set": [
+ {
+ "name": "X-From-CDN",
+ "value": "Traffic-Control"
+ }
+ ]
+ },
+ "modify_headers": {
+ "set": [
+ {
+ "name": "X-CDN-name",
+ "value": "JvD-Grove"
+ }
+ ]
+ },
+ "range_req_handler": {
+ "mode": "get_full_serve_range"
+ }
+ },
+ "query-string": {
+ "cache": true,
+ "remap": true
+ },
+ "retry_codes": [],
+ "retry_num": 5,
+ "timeout_ms": 5000000,
+ "to": [
+ {
+ "retry_codes": [],
+ "retry_num": 0,
+ "timeout_ms": 5000000,
+ "url": "http://localhost",
+ "weight": 1
+ }
+ ]
+ },
+ {
+ "allow": null,
+ "certificate-file": "",
+ "certificate-key-file": "",
+ "concurrent_rule_requests": 0,
+ "connection-close": false,
+ "deny": null,
+ "from": "http://disk1-test.cdn.kabletown.net",
+ "name": "jvd1-test",
+ "parent_selection": "consistent-hash",
+ "cache_name": "disk",
+ "plugins": {
+ "modify_parent_request_headers": {
+ "set": [
+ {
+ "name": "X-From-CDN",
+ "value": "Traffic-Control"
+ }
+ ]
+ },
+ "modify_headers": {
+ "set": [
+ {
+ "name": "X-CDN-name",
+ "value": "JvD-Grove"
+ }
+ ]
+ }
+ },
+ "query-string": {
+ "cache": true,
+ "remap": true
+ },
+ "retry_codes": [],
+ "retry_num": 5,
+ "timeout_ms": 5000000,
+ "to": [
+ {
+ "retry_codes": [],
+ "retry_num": 0,
+ "timeout_ms": 5000000,
+ "url": "http://localhost",
+ "weight": 1
+ }
+ ]
+ },
+ {
+ "allow": null,
+ "certificate-file": "",
+ "certificate-key-file": "",
+ "concurrent_rule_requests": 0,
+ "connection-close": false,
+ "deny": null,
+ "from": "http://mem-test.cdn.kabletown.net",
+ "name": "cim-linear-hds.http.http.cim-linear-hds",
+ "parent_selection": "consistent-hash",
+ "query-string": {
+ "cache": true,
+ "remap": true
+ },
+ "retry_codes": [],
+ "retry_num": 5,
+ "timeout_ms": 5000,
+ "plugins": {
+ "range_req_handler": {
+ "mode": "store_ranges"
+ }
+ },
+ "to": [
+ {
+ "retry_codes": [],
+ "retry_num": 0,
+ "timeout_ms": 5000000,
+ "url": "http://localhost",
+ "weight": 1
+ }
+ ]
+ }
+ ],
+ "stats": {
+ "allow": [
+ "127.0.0.1/32",
+ "::1/128"
+ ],
+ "deny": null
+ },
+ "timeout_ms": 5000
+}
diff --git a/grove/integration_test/tests/plugins/range_req_handler/test.sh b/grove/integration_test/tests/plugins/range_req_handler/test.sh
new file mode 100644
index 0000000..4654318
--- /dev/null
+++ b/grove/integration_test/tests/plugins/range_req_handler/test.sh
@@ -0,0 +1,42 @@
+#!/usr/bin/env bash -x
+
+#curl -H'Host: mem-test.cdn.kabletown.net' -Lsv -r 50000-50009 http://localhost:8080/10Mb.txt
+originurl="http://localhost/"
+host="mem-test.cdn.kabletown.net"
+cacheurl="http://localhost:8080/"
+file="10Mb.txt"
+
+result=0
+testno=0
+
+#curl -s -r 50000-50009 ${originurl}${file} > /tmp/out1 && echo FAIL test ${testno}
+#result=$(($result+$?))
+#testno=$(($testno+1))
+#
+#curl -s -r 50000-50009 -H"Host: ${host}" ${cacheurl}/${file} > /tmp/out2 && echo FAIL test ${testno}
+#
+#result=$(($result+$?))
+#testno=$(($testno+1))
+#
+#diff /tmp/out1 /tmp/out2 && echo FAIL test ${testno}
+#result=$(($result+$?))
+#testno=$(($testno+1))
+
+for host in "mem-test.cdn.kabletown.net", "disk1-test.cdn.kabletown.net"
+do
+ for r in "0-0", "0-100", "5000-", "-100", "0-0,10-15", "0-100,200-210", "33-99,66-88" "-"
+ do
+ test="/compare_gets --chdrs \"Host:$host,Range:bytes=\r${r}\" --ohdrs \"Range:bytes=${r}\" --path \"10Mb.txt\" --ignorehdrs \"Server,Date\""
+ testno=$(($testno+1))
+ echo -n "Test $testno ($test): "
+
+ /compare_gets --chdrs "Host:$host,Range:bytes=${r}" --ohdrs "Range:bytes=${r}" --path "10Mb.txt" --ignorehdrs "Server,Date"
+
+ result=$(($result+$?))
+ done
+done
+
+
+echo "$testno tests done, $result failed."
+
+exit $result
--
To stop receiving notification emails like this one, please contact
rob@apache.org.